Digital Drosophila Archive
272 lines
17 KiB
Python
272 lines
17 KiB
Python
"""Read-only metadata progress, NOT integrity, a scientific gate, or liveness.
|
|
|
|
One CLI invocation reads registration, small ledger-entry JSON files and file
|
|
names. Only the newest committed entry and pending child's small process/result
|
|
receipts are inspected. No model imports, array loads, neural execution, full
|
|
artifact SHA walk, PID inference, retries, writes or persistent cache.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
|
|
|
|
ENTRY_ID = re.compile(r'(?:input-generation|task-[0-9a-f]{64}--(?:brain|update|verify))\Z')
|
|
RESULT_NAME = re.compile(r'task-[0-9a-f]{64}\.json\Z')
|
|
SEQUENCE_NAME = re.compile(r'[0-9]{16}\.json\Z')
|
|
SHA = re.compile(r'[0-9a-f]{64}\Z')
|
|
MAX_METADATA_BYTES = 64 * 1024 * 1024
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def safe(path):
|
|
path = Path(path)
|
|
require(path.is_absolute() and '..' not in path.parts and not any(p.is_symlink() or
|
|
(hasattr(p, 'is_junction') and p.is_junction()) for p in (path, *path.parents)), 'Unsafe linked/nonabsolute path')
|
|
return path
|
|
|
|
|
|
def read_json(path):
|
|
path = safe(path)
|
|
require(path.is_file() and path.stat().st_size <= MAX_METADATA_BYTES, 'Missing/oversized metadata: ' + str(path))
|
|
raw = path.read_bytes()
|
|
def unique(items):
|
|
result = {}
|
|
for key, value in items:
|
|
require(key not in result, 'Duplicate JSON field'); result[key] = value
|
|
return result
|
|
value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique,
|
|
parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON metadata')))
|
|
require(type(value) is dict, 'Metadata object required')
|
|
return value, raw
|
|
|
|
|
|
def directory_files(path):
|
|
path = safe(path)
|
|
if not path.exists():
|
|
return []
|
|
require(path.is_dir(), 'Expected metadata directory')
|
|
rows = sorted(path.iterdir())
|
|
for item in rows:
|
|
require(safe(item).is_file(), 'Unexpected directory/nonregular metadata child: ' + str(item))
|
|
return rows
|
|
|
|
|
|
def named_record(path, regsha):
|
|
if not path.exists():
|
|
return None
|
|
value, _ = read_json(path)
|
|
require(value.get('registration_sha256') == regsha, 'Record registration differs: ' + str(path))
|
|
return value
|
|
|
|
|
|
def process_record(exp, entry_id, regsha, *, expected_sha=None, job_sha=None):
|
|
path = exp / 'receipts' / entry_id / 'process.json'
|
|
if not path.exists():
|
|
return dict(path=str(path), present=False, actual_exit='UNKNOWN')
|
|
value, raw = read_json(path)
|
|
require(value.get('entry_id') == entry_id and value.get('registration_sha256') == regsha and
|
|
type(value.get('complete')) is bool and type(value.get('timed_out')) is bool and
|
|
type(value.get('exit_observed_by_parent')) is bool and
|
|
(value.get('exit_code') is None or type(value.get('exit_code')) is int), 'Incomplete parent process receipt')
|
|
require(SHA.fullmatch(value.get('job_sha256', '')) and (job_sha is None or value['job_sha256'] == job_sha), 'Parent job binding differs')
|
|
require(expected_sha is None or hashlib.sha256(raw).hexdigest() == expected_sha, 'Latest committed process receipt SHA differs')
|
|
code = value['exit_code']
|
|
require(code is None or value['exit_observed_by_parent'] is True, 'Exit code lacks parent observation')
|
|
require(not value['complete'] or (type(code) is int and code == 0 and not value['timed_out']), 'Completion contradicts recorded exit')
|
|
return dict(path=str(path), present=True, exit_code=code, timed_out=value['timed_out'],
|
|
exit_observed_by_parent=value['exit_observed_by_parent'], complete=value['complete'],
|
|
actual_exit='RECORDED-BY-PARENT' if code is not None else 'UNKNOWN',
|
|
OS_liveness_checked=False, sha_checked_against_latest_entry=expected_sha is not None)
|
|
|
|
|
|
def _progress(exp, control, expected_registration_sha):
|
|
regpath = exp / 'protocol/preregistered.json'
|
|
if not regpath.exists():
|
|
return dict(status='PREPARING-OR-NOT-REGISTERED', registered=False,
|
|
reason='No complete registration file is available; preparation activity is not inferred')
|
|
reg, raw = read_json(regpath)
|
|
regsha = hashlib.sha256(raw).hexdigest()
|
|
require(expected_registration_sha is None or regsha == expected_registration_sha, 'External registration SHA differs')
|
|
require(reg.get('schema') == 'normal-conditioning-registration-v1' and
|
|
reg.get('scientific_scope') == 'normal-acquisition-only' and reg.get('subsequent_assays_authorized') is False,
|
|
'Unexpected registration schema/scope')
|
|
inventory = reg['normal_inventory']
|
|
for field in ('train_calls', 'probe_calls', 'brain_calls', 'original_E_calls', 'training_cpu_validation_calls'):
|
|
require(type(inventory[field]) is int and inventory[field] >= 0, 'Missing registered inventory: ' + field)
|
|
require(inventory['brain_calls'] == inventory['train_calls'] + inventory['probe_calls'] and
|
|
inventory['original_E_calls'] == inventory['training_cpu_validation_calls'] == inventory['train_calls'],
|
|
'Registered normal inventory is inconsistent')
|
|
expected_processes = 1 + inventory['brain_calls'] + inventory['original_E_calls'] + inventory['training_cpu_validation_calls']
|
|
ledger = exp / '.append-ledger'
|
|
files = directory_files(ledger / 'entries')
|
|
starts = directory_files(ledger / 'started')
|
|
results = directory_files(exp / 'results')
|
|
require(all(RESULT_NAME.fullmatch(p.name) for p in results), 'Unrecognized/partial result file')
|
|
if ledger.exists():
|
|
header, _ = read_json(ledger / 'header.json')
|
|
require(header.get('schema') == 'normal-append-ledger-v1' and header.get('root') == str(exp) and
|
|
header.get('registration_sha256') == regsha and
|
|
header.get('initial_inventory', {}).get('protocol/preregistered.json', {}).get('sha256') == regsha,
|
|
'Ledger header registration binding differs')
|
|
seen, committed_results, counts = set(), set(), Counter()
|
|
latest = None
|
|
for number, path in enumerate(files, 1):
|
|
require(SEQUENCE_NAME.fullmatch(path.name) and path.name == f'{number:016d}.json', 'Noncontiguous/partial ledger sequence')
|
|
row, _ = read_json(path)
|
|
entry_id = row.get('entry_id')
|
|
require(row.get('schema') == 'normal-append-ledger-v1-entry' and type(row.get('sequence')) is int and
|
|
row['sequence'] == number and row.get('registration_sha256') == regsha and
|
|
type(entry_id) is str and ENTRY_ID.fullmatch(entry_id) and entry_id not in seen and
|
|
row.get('scientific_gate_claimed') is False, 'Ledger entry identity/schema differs')
|
|
metadata = row['metadata']
|
|
require(type(metadata.get('observed_returncode')) is int and metadata['observed_returncode'] == 0 and
|
|
metadata.get('job_file') == str(exp / 'jobs' / (entry_id + '.json')) and
|
|
metadata.get('process_receipt_file') == str(exp / 'receipts' / entry_id / 'process.json'), 'Ledger process metadata differs')
|
|
outgoing = row['new_outputs']
|
|
require(type(outgoing) is list and len({x['path'] for x in outgoing}) == len(outgoing), 'Invalid ledger output rows')
|
|
process_rows = [x for x in outgoing if x['path'] == metadata['process_receipt_file']]
|
|
require(len(process_rows) == 1 and SHA.fullmatch(process_rows[0]['sha256']), 'Committed process receipt is not ledger owned')
|
|
result_rows = [x for x in outgoing if Path(x['path']).parent == exp / 'results']
|
|
require(len(result_rows) <= 1, 'Multiple task results in one process entry')
|
|
if entry_id == 'input-generation':
|
|
require(number == 1 and not result_rows, 'Sampler must be the first committed process')
|
|
counts['sampler'] += 1
|
|
else:
|
|
require('input-generation' in seen, 'Task commit precedes sampler commit')
|
|
key, phase = entry_id.rsplit('--', 1)
|
|
counts[phase] += 1
|
|
if phase in ('update', 'verify'):
|
|
require(key + '--brain' in seen and (phase != 'verify' or key + '--update' in seen), 'Training commit lacks earlier brain/update')
|
|
if result_rows:
|
|
expected = str(exp / 'results' / (key + '.json'))
|
|
require(result_rows[0]['path'] == expected and expected not in committed_results and phase in ('brain', 'verify'),
|
|
'Unexpected/repeated committed task result')
|
|
committed_results.add(expected)
|
|
counts['probe_complete' if phase == 'brain' else 'train_complete'] += 1
|
|
else:
|
|
require(phase != 'verify', 'Committed training verification lacks its task result')
|
|
seen.add(entry_id)
|
|
latest = dict(path=path, row=row, process=process_rows[0], result=result_rows[0] if result_rows else None)
|
|
require(len(files) <= expected_processes and counts['train_complete'] <= inventory['train_calls'] and
|
|
counts['probe_complete'] <= inventory['probe_calls'], 'Recorded progress exceeds registered inventory')
|
|
start_ids = {p.stem for p in starts}
|
|
require(len(start_ids) == len(starts) and all(p.suffix == '.json' and ENTRY_ID.fullmatch(p.stem) for p in starts) and
|
|
seen <= start_ids, 'Missing/invalid ledger start inventory')
|
|
pending = sorted(start_ids - seen)
|
|
require(len(pending) <= 1, 'More than one uncommitted child in serial ledger')
|
|
result_paths = {str(p) for p in results}
|
|
require(committed_results <= result_paths, 'Ledger-owned result file is missing')
|
|
latest_report = None
|
|
if latest is not None:
|
|
row = latest['row']; entry_id = row['entry_id']
|
|
start, _ = read_json(ledger / 'started' / (entry_id + '.json'))
|
|
require(start['entry_id'] == entry_id and start['registration_sha256'] == regsha and start['next_sequence'] == len(files), 'Latest start binding differs')
|
|
receipt = process_record(exp, entry_id, regsha, expected_sha=latest['process']['sha256'], job_sha=start['job']['sha256'])
|
|
require(receipt['present'] and receipt['complete'] and receipt['exit_code'] == 0 and not receipt['timed_out'], 'Latest committed process is not recorded successful')
|
|
latest_report = dict(sequence=len(files), entry_id=entry_id, process=receipt)
|
|
if latest['result'] is not None:
|
|
record, raw_result = read_json(latest['result']['path'])
|
|
task = record['task']; kind = 'train' if entry_id.endswith('--verify') else 'probe'
|
|
key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest()
|
|
require(hashlib.sha256(raw_result).hexdigest() == latest['result']['sha256'] and
|
|
record.get('schema') == 'normal-task-result-v1' and record.get('registration_sha256') == regsha and
|
|
record.get('complete') is True and task['kind'] == kind and entry_id.startswith(key + '--'), 'Latest task result binding differs')
|
|
latest_report['task'] = {k: task[k] for k in ('task_id', 'kind', 'block_id', 'arm')}
|
|
pending_report = None
|
|
if pending:
|
|
entry_id = pending[0]
|
|
start, _ = read_json(ledger / 'started' / (entry_id + '.json'))
|
|
require(start.get('schema') == 'normal-append-ledger-v1-start' and start.get('entry_id') == entry_id and
|
|
start.get('registration_sha256') == regsha and start.get('next_sequence') == len(files) + 1 and
|
|
start.get('terminal_state') == 'unknown' and start.get('retry_authorized') is False, 'Pending start identity differs')
|
|
pending_report = dict(entry_id=entry_id, process=process_record(exp, entry_id, regsha, job_sha=start['job']['sha256']))
|
|
control_records = {name: named_record(control / filename, regsha) for name, filename in
|
|
(('launch_attempt', 'launch-attempt.json'), ('launch', 'launch.json'), ('supervisor_start', 'supervisor-start.json'), ('exit', 'exit.json'))}
|
|
launch = control_records['launch']
|
|
if launch is not None:
|
|
require(launch.get('launch_observed') is True and launch.get('final_exit_observed') is False and
|
|
launch.get('not_evidence_of_completion') is True and type(launch.get('supervisor_pid')) is int, 'Incomplete launch observation')
|
|
attempt = control_records['launch_attempt']
|
|
if attempt is not None:
|
|
require(attempt.get('complete') is False and attempt.get('terminal_state') == 'UNKNOWN' and
|
|
attempt.get('automatic_retry') is False, 'Incomplete launch attempt record')
|
|
supervisor = control_records['supervisor_start']
|
|
if supervisor is not None:
|
|
require(supervisor.get('complete') is False and type(supervisor.get('pid')) is int, 'Incomplete supervisor start record')
|
|
stopped = named_record(exp / 'audit/stopped.json', regsha)
|
|
completion = named_record(exp / 'audit/execution-complete.json', regsha)
|
|
exit_record = control_records['exit']
|
|
status = 'REGISTERED-NOT-STARTED'
|
|
if attempt is not None:
|
|
status = 'LAUNCH-ATTEMPT-RECORDED-EXIT-UNKNOWN'
|
|
if control_records['launch'] is not None or control_records['supervisor_start'] is not None:
|
|
status = 'LAUNCH-RECORDED-EXIT-UNKNOWN'
|
|
if files:
|
|
status = 'BETWEEN-COMMITS-OR-PARENT-STATE-UNKNOWN'
|
|
if pending_report:
|
|
receipt = pending_report['process']
|
|
status = 'SAMPLER-STARTED-EXIT-UNKNOWN' if pending[0] == 'input-generation' else 'CHILD-STARTED-UNCOMMITTED-EXIT-UNKNOWN'
|
|
if receipt['present']:
|
|
if receipt['timed_out'] or (receipt['exit_code'] is not None and receipt['exit_code'] != 0): status = 'CHILD-FAILED-UNCOMMITTED'
|
|
elif receipt['complete']: status = 'CHILD-EXITED-ZERO-NOT-COMMITTED'
|
|
elif receipt['exit_code'] == 0: status = 'CHILD-EXITED-ZERO-NOT-VALIDATED'
|
|
if completion is not None:
|
|
require(completion.get('complete') is True and completion.get('inventory') == inventory and
|
|
len(files) == expected_processes and not pending, 'Execution completion contradicts ledger inventory')
|
|
status = 'EXECUTION-COMPLETION-RECORDED-PARENT-EXIT-UNKNOWN'
|
|
if exit_record is not None:
|
|
require(type(exit_record.get('final_exit_observed')) is bool and type(exit_record.get('complete')) is bool and
|
|
(exit_record.get('exit_code') is None or type(exit_record.get('exit_code')) is int), 'Incomplete supervisor exit record')
|
|
code = exit_record['exit_code']
|
|
require((code is None) == (exit_record['final_exit_observed'] is False) and exit_record['complete'] is (code == 0), 'Contradictory supervisor exit record')
|
|
status = 'SUPERVISOR-ERROR-EXIT-UNKNOWN' if code is None else ('PARENT-EXITED-ZERO' if code == 0 else 'PARENT-EXITED-NONZERO')
|
|
if stopped is not None:
|
|
require(stopped.get('complete') is False, 'Malformed stopped record')
|
|
status = 'STOPPED-RECORD-PRESENT'
|
|
return dict(status=status, registered=True, registration_sha256=regsha, registered_inventory=inventory,
|
|
expected_process_entries=expected_processes, committed_process_entries=len(files),
|
|
sampler_committed=counts['sampler'] == 1, brain_entries=counts['brain'], update_entries=counts['update'],
|
|
training_epochs_completed=counts['train_complete'], probes_completed=counts['probe_complete'],
|
|
result_files_present=len(results), task_results_ledger_owned=len(committed_results),
|
|
result_files_without_commit=sorted(result_paths - committed_results), latest_entry=latest_report,
|
|
uncommitted_child=pending_report, control_records=control_records, stopped_record=stopped,
|
|
execution_completion_record_present=completion is not None,
|
|
registered_workflow_inventory_recorded_complete=(len(files) == expected_processes and
|
|
counts['train_complete'] == inventory['train_calls'] and counts['probe_complete'] == inventory['probe_calls'] and not pending),
|
|
ledger_writer_lock_present=(ledger / 'writer.lock').exists(),
|
|
entries_appeared_during_read=len(directory_files(ledger / 'entries')) != len(files))
|
|
|
|
|
|
def read_progress(exp, control, registration_sha256=None):
|
|
"""Unknown/inconsistent metadata never become completed work or learning."""
|
|
base = dict(schema='normal-fast-progress-v1', scope='READ-ONLY-METADATA-NOT-INTEGRITY',
|
|
integrity_verified=False, historical_receipts_reverified=False, OS_liveness_checked=False,
|
|
learning_pass_claimed=False, scientific_metrics_evaluated=False, candidate_generated=False,
|
|
writes_performed=False, model_executed=False, automatic_retry_authorized=False,
|
|
snapshot_scope='Non-atomic observation; concurrent publication can require reading again')
|
|
try:
|
|
exp, control = safe(exp), safe(control)
|
|
require(registration_sha256 is None or SHA.fullmatch(registration_sha256), 'Invalid external registration SHA')
|
|
return dict(base, **_progress(exp, control, registration_sha256))
|
|
except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError) as error:
|
|
return dict(base, status='UNKNOWN-METADATA-INCONSISTENT', error=type(error).__name__ + ': ' + str(error),
|
|
completion_counts_available=False)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--exp', required=True)
|
|
parser.add_argument('--control', required=True)
|
|
parser.add_argument('--registration-sha256')
|
|
args = parser.parse_args()
|
|
print(json.dumps(read_progress(args.exp, args.control, args.registration_sha256), ensure_ascii=False, sort_keys=True, indent=2))
|