"""Read-only metadata progress, NOT integrity, a scientific gate, or liveness. One CLI invocation reads registration, small ledger-entry JSON files and file names. Only the newest committed entry and pending child's small process/result receipts are inspected. No model imports, array loads, neural execution, full artifact SHA walk, PID inference, retries, writes or persistent cache. """ from __future__ import annotations import argparse from collections import Counter import hashlib import json from pathlib import Path import re ENTRY_ID = re.compile(r'(?:input-generation|task-[0-9a-f]{64}--(?:brain|update|verify))\Z') RESULT_NAME = re.compile(r'task-[0-9a-f]{64}\.json\Z') SEQUENCE_NAME = re.compile(r'[0-9]{16}\.json\Z') SHA = re.compile(r'[0-9a-f]{64}\Z') MAX_METADATA_BYTES = 64 * 1024 * 1024 def require(ok, message): if not ok: raise ValueError(message) def safe(path): path = Path(path) require(path.is_absolute() and '..' not in path.parts and not any(p.is_symlink() or (hasattr(p, 'is_junction') and p.is_junction()) for p in (path, *path.parents)), 'Unsafe linked/nonabsolute path') return path def read_json(path): path = safe(path) require(path.is_file() and path.stat().st_size <= MAX_METADATA_BYTES, 'Missing/oversized metadata: ' + str(path)) raw = path.read_bytes() def unique(items): result = {} for key, value in items: require(key not in result, 'Duplicate JSON field'); result[key] = value return result value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON metadata'))) require(type(value) is dict, 'Metadata object required') return value, raw def directory_files(path): path = safe(path) if not path.exists(): return [] require(path.is_dir(), 'Expected metadata directory') rows = sorted(path.iterdir()) for item in rows: require(safe(item).is_file(), 'Unexpected directory/nonregular metadata child: ' + str(item)) return rows def named_record(path, regsha): if not path.exists(): return None value, _ = read_json(path) require(value.get('registration_sha256') == regsha, 'Record registration differs: ' + str(path)) return value def process_record(exp, entry_id, regsha, *, expected_sha=None, job_sha=None): path = exp / 'receipts' / entry_id / 'process.json' if not path.exists(): return dict(path=str(path), present=False, actual_exit='UNKNOWN') value, raw = read_json(path) require(value.get('entry_id') == entry_id and value.get('registration_sha256') == regsha and type(value.get('complete')) is bool and type(value.get('timed_out')) is bool and type(value.get('exit_observed_by_parent')) is bool and (value.get('exit_code') is None or type(value.get('exit_code')) is int), 'Incomplete parent process receipt') require(SHA.fullmatch(value.get('job_sha256', '')) and (job_sha is None or value['job_sha256'] == job_sha), 'Parent job binding differs') require(expected_sha is None or hashlib.sha256(raw).hexdigest() == expected_sha, 'Latest committed process receipt SHA differs') code = value['exit_code'] require(code is None or value['exit_observed_by_parent'] is True, 'Exit code lacks parent observation') require(not value['complete'] or (type(code) is int and code == 0 and not value['timed_out']), 'Completion contradicts recorded exit') return dict(path=str(path), present=True, exit_code=code, timed_out=value['timed_out'], exit_observed_by_parent=value['exit_observed_by_parent'], complete=value['complete'], actual_exit='RECORDED-BY-PARENT' if code is not None else 'UNKNOWN', OS_liveness_checked=False, sha_checked_against_latest_entry=expected_sha is not None) def _progress(exp, control, expected_registration_sha): regpath = exp / 'protocol/preregistered.json' if not regpath.exists(): return dict(status='PREPARING-OR-NOT-REGISTERED', registered=False, reason='No complete registration file is available; preparation activity is not inferred') reg, raw = read_json(regpath) regsha = hashlib.sha256(raw).hexdigest() require(expected_registration_sha is None or regsha == expected_registration_sha, 'External registration SHA differs') require(reg.get('schema') == 'normal-conditioning-registration-v1' and reg.get('scientific_scope') == 'normal-acquisition-only' and reg.get('subsequent_assays_authorized') is False, 'Unexpected registration schema/scope') inventory = reg['normal_inventory'] for field in ('train_calls', 'probe_calls', 'brain_calls', 'original_E_calls', 'training_cpu_validation_calls'): require(type(inventory[field]) is int and inventory[field] >= 0, 'Missing registered inventory: ' + field) require(inventory['brain_calls'] == inventory['train_calls'] + inventory['probe_calls'] and inventory['original_E_calls'] == inventory['training_cpu_validation_calls'] == inventory['train_calls'], 'Registered normal inventory is inconsistent') expected_processes = 1 + inventory['brain_calls'] + inventory['original_E_calls'] + inventory['training_cpu_validation_calls'] ledger = exp / '.append-ledger' files = directory_files(ledger / 'entries') starts = directory_files(ledger / 'started') results = directory_files(exp / 'results') require(all(RESULT_NAME.fullmatch(p.name) for p in results), 'Unrecognized/partial result file') if ledger.exists(): header, _ = read_json(ledger / 'header.json') require(header.get('schema') == 'normal-append-ledger-v1' and header.get('root') == str(exp) and header.get('registration_sha256') == regsha and header.get('initial_inventory', {}).get('protocol/preregistered.json', {}).get('sha256') == regsha, 'Ledger header registration binding differs') seen, committed_results, counts = set(), set(), Counter() latest = None for number, path in enumerate(files, 1): require(SEQUENCE_NAME.fullmatch(path.name) and path.name == f'{number:016d}.json', 'Noncontiguous/partial ledger sequence') row, _ = read_json(path) entry_id = row.get('entry_id') require(row.get('schema') == 'normal-append-ledger-v1-entry' and type(row.get('sequence')) is int and row['sequence'] == number and row.get('registration_sha256') == regsha and type(entry_id) is str and ENTRY_ID.fullmatch(entry_id) and entry_id not in seen and row.get('scientific_gate_claimed') is False, 'Ledger entry identity/schema differs') metadata = row['metadata'] require(type(metadata.get('observed_returncode')) is int and metadata['observed_returncode'] == 0 and metadata.get('job_file') == str(exp / 'jobs' / (entry_id + '.json')) and metadata.get('process_receipt_file') == str(exp / 'receipts' / entry_id / 'process.json'), 'Ledger process metadata differs') outgoing = row['new_outputs'] require(type(outgoing) is list and len({x['path'] for x in outgoing}) == len(outgoing), 'Invalid ledger output rows') process_rows = [x for x in outgoing if x['path'] == metadata['process_receipt_file']] require(len(process_rows) == 1 and SHA.fullmatch(process_rows[0]['sha256']), 'Committed process receipt is not ledger owned') result_rows = [x for x in outgoing if Path(x['path']).parent == exp / 'results'] require(len(result_rows) <= 1, 'Multiple task results in one process entry') if entry_id == 'input-generation': require(number == 1 and not result_rows, 'Sampler must be the first committed process') counts['sampler'] += 1 else: require('input-generation' in seen, 'Task commit precedes sampler commit') key, phase = entry_id.rsplit('--', 1) counts[phase] += 1 if phase in ('update', 'verify'): require(key + '--brain' in seen and (phase != 'verify' or key + '--update' in seen), 'Training commit lacks earlier brain/update') if result_rows: expected = str(exp / 'results' / (key + '.json')) require(result_rows[0]['path'] == expected and expected not in committed_results and phase in ('brain', 'verify'), 'Unexpected/repeated committed task result') committed_results.add(expected) counts['probe_complete' if phase == 'brain' else 'train_complete'] += 1 else: require(phase != 'verify', 'Committed training verification lacks its task result') seen.add(entry_id) latest = dict(path=path, row=row, process=process_rows[0], result=result_rows[0] if result_rows else None) require(len(files) <= expected_processes and counts['train_complete'] <= inventory['train_calls'] and counts['probe_complete'] <= inventory['probe_calls'], 'Recorded progress exceeds registered inventory') start_ids = {p.stem for p in starts} require(len(start_ids) == len(starts) and all(p.suffix == '.json' and ENTRY_ID.fullmatch(p.stem) for p in starts) and seen <= start_ids, 'Missing/invalid ledger start inventory') pending = sorted(start_ids - seen) require(len(pending) <= 1, 'More than one uncommitted child in serial ledger') result_paths = {str(p) for p in results} require(committed_results <= result_paths, 'Ledger-owned result file is missing') latest_report = None if latest is not None: row = latest['row']; entry_id = row['entry_id'] start, _ = read_json(ledger / 'started' / (entry_id + '.json')) require(start['entry_id'] == entry_id and start['registration_sha256'] == regsha and start['next_sequence'] == len(files), 'Latest start binding differs') receipt = process_record(exp, entry_id, regsha, expected_sha=latest['process']['sha256'], job_sha=start['job']['sha256']) require(receipt['present'] and receipt['complete'] and receipt['exit_code'] == 0 and not receipt['timed_out'], 'Latest committed process is not recorded successful') latest_report = dict(sequence=len(files), entry_id=entry_id, process=receipt) if latest['result'] is not None: record, raw_result = read_json(latest['result']['path']) task = record['task']; kind = 'train' if entry_id.endswith('--verify') else 'probe' key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest() require(hashlib.sha256(raw_result).hexdigest() == latest['result']['sha256'] and record.get('schema') == 'normal-task-result-v1' and record.get('registration_sha256') == regsha and record.get('complete') is True and task['kind'] == kind and entry_id.startswith(key + '--'), 'Latest task result binding differs') latest_report['task'] = {k: task[k] for k in ('task_id', 'kind', 'block_id', 'arm')} pending_report = None if pending: entry_id = pending[0] start, _ = read_json(ledger / 'started' / (entry_id + '.json')) require(start.get('schema') == 'normal-append-ledger-v1-start' and start.get('entry_id') == entry_id and start.get('registration_sha256') == regsha and start.get('next_sequence') == len(files) + 1 and start.get('terminal_state') == 'unknown' and start.get('retry_authorized') is False, 'Pending start identity differs') pending_report = dict(entry_id=entry_id, process=process_record(exp, entry_id, regsha, job_sha=start['job']['sha256'])) control_records = {name: named_record(control / filename, regsha) for name, filename in (('launch_attempt', 'launch-attempt.json'), ('launch', 'launch.json'), ('supervisor_start', 'supervisor-start.json'), ('exit', 'exit.json'))} launch = control_records['launch'] if launch is not None: require(launch.get('launch_observed') is True and launch.get('final_exit_observed') is False and launch.get('not_evidence_of_completion') is True and type(launch.get('supervisor_pid')) is int, 'Incomplete launch observation') attempt = control_records['launch_attempt'] if attempt is not None: require(attempt.get('complete') is False and attempt.get('terminal_state') == 'UNKNOWN' and attempt.get('automatic_retry') is False, 'Incomplete launch attempt record') supervisor = control_records['supervisor_start'] if supervisor is not None: require(supervisor.get('complete') is False and type(supervisor.get('pid')) is int, 'Incomplete supervisor start record') stopped = named_record(exp / 'audit/stopped.json', regsha) completion = named_record(exp / 'audit/execution-complete.json', regsha) exit_record = control_records['exit'] status = 'REGISTERED-NOT-STARTED' if attempt is not None: status = 'LAUNCH-ATTEMPT-RECORDED-EXIT-UNKNOWN' if control_records['launch'] is not None or control_records['supervisor_start'] is not None: status = 'LAUNCH-RECORDED-EXIT-UNKNOWN' if files: status = 'BETWEEN-COMMITS-OR-PARENT-STATE-UNKNOWN' if pending_report: receipt = pending_report['process'] status = 'SAMPLER-STARTED-EXIT-UNKNOWN' if pending[0] == 'input-generation' else 'CHILD-STARTED-UNCOMMITTED-EXIT-UNKNOWN' if receipt['present']: if receipt['timed_out'] or (receipt['exit_code'] is not None and receipt['exit_code'] != 0): status = 'CHILD-FAILED-UNCOMMITTED' elif receipt['complete']: status = 'CHILD-EXITED-ZERO-NOT-COMMITTED' elif receipt['exit_code'] == 0: status = 'CHILD-EXITED-ZERO-NOT-VALIDATED' if completion is not None: require(completion.get('complete') is True and completion.get('inventory') == inventory and len(files) == expected_processes and not pending, 'Execution completion contradicts ledger inventory') status = 'EXECUTION-COMPLETION-RECORDED-PARENT-EXIT-UNKNOWN' if exit_record is not None: require(type(exit_record.get('final_exit_observed')) is bool and type(exit_record.get('complete')) is bool and (exit_record.get('exit_code') is None or type(exit_record.get('exit_code')) is int), 'Incomplete supervisor exit record') code = exit_record['exit_code'] require((code is None) == (exit_record['final_exit_observed'] is False) and exit_record['complete'] is (code == 0), 'Contradictory supervisor exit record') status = 'SUPERVISOR-ERROR-EXIT-UNKNOWN' if code is None else ('PARENT-EXITED-ZERO' if code == 0 else 'PARENT-EXITED-NONZERO') if stopped is not None: require(stopped.get('complete') is False, 'Malformed stopped record') status = 'STOPPED-RECORD-PRESENT' return dict(status=status, registered=True, registration_sha256=regsha, registered_inventory=inventory, expected_process_entries=expected_processes, committed_process_entries=len(files), sampler_committed=counts['sampler'] == 1, brain_entries=counts['brain'], update_entries=counts['update'], training_epochs_completed=counts['train_complete'], probes_completed=counts['probe_complete'], result_files_present=len(results), task_results_ledger_owned=len(committed_results), result_files_without_commit=sorted(result_paths - committed_results), latest_entry=latest_report, uncommitted_child=pending_report, control_records=control_records, stopped_record=stopped, execution_completion_record_present=completion is not None, registered_workflow_inventory_recorded_complete=(len(files) == expected_processes and counts['train_complete'] == inventory['train_calls'] and counts['probe_complete'] == inventory['probe_calls'] and not pending), ledger_writer_lock_present=(ledger / 'writer.lock').exists(), entries_appeared_during_read=len(directory_files(ledger / 'entries')) != len(files)) def read_progress(exp, control, registration_sha256=None): """Unknown/inconsistent metadata never become completed work or learning.""" base = dict(schema='normal-fast-progress-v1', scope='READ-ONLY-METADATA-NOT-INTEGRITY', integrity_verified=False, historical_receipts_reverified=False, OS_liveness_checked=False, learning_pass_claimed=False, scientific_metrics_evaluated=False, candidate_generated=False, writes_performed=False, model_executed=False, automatic_retry_authorized=False, snapshot_scope='Non-atomic observation; concurrent publication can require reading again') try: exp, control = safe(exp), safe(control) require(registration_sha256 is None or SHA.fullmatch(registration_sha256), 'Invalid external registration SHA') return dict(base, **_progress(exp, control, registration_sha256)) except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError) as error: return dict(base, status='UNKNOWN-METADATA-INCONSISTENT', error=type(error).__name__ + ': ' + str(error), completion_counts_available=False) if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--exp', required=True) parser.add_argument('--control', required=True) parser.add_argument('--registration-sha256') args = parser.parse_args() print(json.dumps(read_progress(args.exp, args.control, args.registration_sha256), ensure_ascii=False, sort_keys=True, indent=2))