qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add read-only normal-run progress audit and verified startup observation

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-09-30 21:17:50 +08:00
parent 703edfcb31
commit 1e429fd50d
4 changed files with 484 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/normal-conditioning-audit/progress_normal.py
+271
View File
@@ -0,0 +1,271 @@
"""Read-only metadata progress, NOT integrity, a scientific gate, or liveness.
One CLI invocation reads registration, small ledger-entry JSON files and file
names. Only the newest committed entry and pending child's small process/result
receipts are inspected. No model imports, array loads, neural execution, full
artifact SHA walk, PID inference, retries, writes or persistent cache.
"""
from __future__ import annotations
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
import re
ENTRY_ID = re.compile(r'(?:input-generation|task-[0-9a-f]{64}--(?:brain|update|verify))\Z')
RESULT_NAME = re.compile(r'task-[0-9a-f]{64}\.json\Z')
SEQUENCE_NAME = re.compile(r'[0-9]{16}\.json\Z')
SHA = re.compile(r'[0-9a-f]{64}\Z')
MAX_METADATA_BYTES = 64 * 1024 * 1024
def require(ok, message):
if not ok:
raise ValueError(message)
def safe(path):
path = Path(path)
require(path.is_absolute() and '..' not in path.parts and not any(p.is_symlink() or
(hasattr(p, 'is_junction') and p.is_junction()) for p in (path, *path.parents)), 'Unsafe linked/nonabsolute path')
return path
def read_json(path):
path = safe(path)
require(path.is_file() and path.stat().st_size <= MAX_METADATA_BYTES, 'Missing/oversized metadata: ' + str(path))
raw = path.read_bytes()
def unique(items):
result = {}
for key, value in items:
require(key not in result, 'Duplicate JSON field'); result[key] = value
return result
value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique,
parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON metadata')))
require(type(value) is dict, 'Metadata object required')
return value, raw
def directory_files(path):
path = safe(path)
if not path.exists():
return []
require(path.is_dir(), 'Expected metadata directory')
rows = sorted(path.iterdir())
for item in rows:
require(safe(item).is_file(), 'Unexpected directory/nonregular metadata child: ' + str(item))
return rows
def named_record(path, regsha):
if not path.exists():
return None
value, _ = read_json(path)
require(value.get('registration_sha256') == regsha, 'Record registration differs: ' + str(path))
return value
def process_record(exp, entry_id, regsha, *, expected_sha=None, job_sha=None):
path = exp / 'receipts' / entry_id / 'process.json'
if not path.exists():
return dict(path=str(path), present=False, actual_exit='UNKNOWN')
value, raw = read_json(path)
require(value.get('entry_id') == entry_id and value.get('registration_sha256') == regsha and
type(value.get('complete')) is bool and type(value.get('timed_out')) is bool and
type(value.get('exit_observed_by_parent')) is bool and
(value.get('exit_code') is None or type(value.get('exit_code')) is int), 'Incomplete parent process receipt')
require(SHA.fullmatch(value.get('job_sha256', '')) and (job_sha is None or value['job_sha256'] == job_sha), 'Parent job binding differs')
require(expected_sha is None or hashlib.sha256(raw).hexdigest() == expected_sha, 'Latest committed process receipt SHA differs')
code = value['exit_code']
require(code is None or value['exit_observed_by_parent'] is True, 'Exit code lacks parent observation')
require(not value['complete'] or (type(code) is int and code == 0 and not value['timed_out']), 'Completion contradicts recorded exit')
return dict(path=str(path), present=True, exit_code=code, timed_out=value['timed_out'],
exit_observed_by_parent=value['exit_observed_by_parent'], complete=value['complete'],
actual_exit='RECORDED-BY-PARENT' if code is not None else 'UNKNOWN',
OS_liveness_checked=False, sha_checked_against_latest_entry=expected_sha is not None)
def _progress(exp, control, expected_registration_sha):
regpath = exp / 'protocol/preregistered.json'
if not regpath.exists():
return dict(status='PREPARING-OR-NOT-REGISTERED', registered=False,
reason='No complete registration file is available; preparation activity is not inferred')
reg, raw = read_json(regpath)
regsha = hashlib.sha256(raw).hexdigest()
require(expected_registration_sha is None or regsha == expected_registration_sha, 'External registration SHA differs')
require(reg.get('schema') == 'normal-conditioning-registration-v1' and
reg.get('scientific_scope') == 'normal-acquisition-only' and reg.get('subsequent_assays_authorized') is False,
'Unexpected registration schema/scope')
inventory = reg['normal_inventory']
for field in ('train_calls', 'probe_calls', 'brain_calls', 'original_E_calls', 'training_cpu_validation_calls'):
require(type(inventory[field]) is int and inventory[field] >= 0, 'Missing registered inventory: ' + field)
require(inventory['brain_calls'] == inventory['train_calls'] + inventory['probe_calls'] and
inventory['original_E_calls'] == inventory['training_cpu_validation_calls'] == inventory['train_calls'],
'Registered normal inventory is inconsistent')
expected_processes = 1 + inventory['brain_calls'] + inventory['original_E_calls'] + inventory['training_cpu_validation_calls']
ledger = exp / '.append-ledger'
files = directory_files(ledger / 'entries')
starts = directory_files(ledger / 'started')
results = directory_files(exp / 'results')
require(all(RESULT_NAME.fullmatch(p.name) for p in results), 'Unrecognized/partial result file')
if ledger.exists():
header, _ = read_json(ledger / 'header.json')
require(header.get('schema') == 'normal-append-ledger-v1' and header.get('root') == str(exp) and
header.get('registration_sha256') == regsha and
header.get('initial_inventory', {}).get('protocol/preregistered.json', {}).get('sha256') == regsha,
'Ledger header registration binding differs')
seen, committed_results, counts = set(), set(), Counter()
latest = None
for number, path in enumerate(files, 1):
require(SEQUENCE_NAME.fullmatch(path.name) and path.name == f'{number:016d}.json', 'Noncontiguous/partial ledger sequence')
row, _ = read_json(path)
entry_id = row.get('entry_id')
require(row.get('schema') == 'normal-append-ledger-v1-entry' and type(row.get('sequence')) is int and
row['sequence'] == number and row.get('registration_sha256') == regsha and
type(entry_id) is str and ENTRY_ID.fullmatch(entry_id) and entry_id not in seen and
row.get('scientific_gate_claimed') is False, 'Ledger entry identity/schema differs')
metadata = row['metadata']
require(type(metadata.get('observed_returncode')) is int and metadata['observed_returncode'] == 0 and
metadata.get('job_file') == str(exp / 'jobs' / (entry_id + '.json')) and
metadata.get('process_receipt_file') == str(exp / 'receipts' / entry_id / 'process.json'), 'Ledger process metadata differs')
outgoing = row['new_outputs']
require(type(outgoing) is list and len({x['path'] for x in outgoing}) == len(outgoing), 'Invalid ledger output rows')
process_rows = [x for x in outgoing if x['path'] == metadata['process_receipt_file']]
require(len(process_rows) == 1 and SHA.fullmatch(process_rows[0]['sha256']), 'Committed process receipt is not ledger owned')
result_rows = [x for x in outgoing if Path(x['path']).parent == exp / 'results']
require(len(result_rows) <= 1, 'Multiple task results in one process entry')
if entry_id == 'input-generation':
require(number == 1 and not result_rows, 'Sampler must be the first committed process')
counts['sampler'] += 1
else:
require('input-generation' in seen, 'Task commit precedes sampler commit')
key, phase = entry_id.rsplit('--', 1)
counts[phase] += 1
if phase in ('update', 'verify'):
require(key + '--brain' in seen and (phase != 'verify' or key + '--update' in seen), 'Training commit lacks earlier brain/update')
if result_rows:
expected = str(exp / 'results' / (key + '.json'))
require(result_rows[0]['path'] == expected and expected not in committed_results and phase in ('brain', 'verify'),
'Unexpected/repeated committed task result')
committed_results.add(expected)
counts['probe_complete' if phase == 'brain' else 'train_complete'] += 1
else:
require(phase != 'verify', 'Committed training verification lacks its task result')
seen.add(entry_id)
latest = dict(path=path, row=row, process=process_rows[0], result=result_rows[0] if result_rows else None)
require(len(files) <= expected_processes and counts['train_complete'] <= inventory['train_calls'] and
counts['probe_complete'] <= inventory['probe_calls'], 'Recorded progress exceeds registered inventory')
start_ids = {p.stem for p in starts}
require(len(start_ids) == len(starts) and all(p.suffix == '.json' and ENTRY_ID.fullmatch(p.stem) for p in starts) and
seen <= start_ids, 'Missing/invalid ledger start inventory')
pending = sorted(start_ids - seen)
require(len(pending) <= 1, 'More than one uncommitted child in serial ledger')
result_paths = {str(p) for p in results}
require(committed_results <= result_paths, 'Ledger-owned result file is missing')
latest_report = None
if latest is not None:
row = latest['row']; entry_id = row['entry_id']
start, _ = read_json(ledger / 'started' / (entry_id + '.json'))
require(start['entry_id'] == entry_id and start['registration_sha256'] == regsha and start['next_sequence'] == len(files), 'Latest start binding differs')
receipt = process_record(exp, entry_id, regsha, expected_sha=latest['process']['sha256'], job_sha=start['job']['sha256'])
require(receipt['present'] and receipt['complete'] and receipt['exit_code'] == 0 and not receipt['timed_out'], 'Latest committed process is not recorded successful')
latest_report = dict(sequence=len(files), entry_id=entry_id, process=receipt)
if latest['result'] is not None:
record, raw_result = read_json(latest['result']['path'])
task = record['task']; kind = 'train' if entry_id.endswith('--verify') else 'probe'
key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest()
require(hashlib.sha256(raw_result).hexdigest() == latest['result']['sha256'] and
record.get('schema') == 'normal-task-result-v1' and record.get('registration_sha256') == regsha and
record.get('complete') is True and task['kind'] == kind and entry_id.startswith(key + '--'), 'Latest task result binding differs')
latest_report['task'] = {k: task[k] for k in ('task_id', 'kind', 'block_id', 'arm')}
pending_report = None
if pending:
entry_id = pending[0]
start, _ = read_json(ledger / 'started' / (entry_id + '.json'))
require(start.get('schema') == 'normal-append-ledger-v1-start' and start.get('entry_id') == entry_id and
start.get('registration_sha256') == regsha and start.get('next_sequence') == len(files) + 1 and
start.get('terminal_state') == 'unknown' and start.get('retry_authorized') is False, 'Pending start identity differs')
pending_report = dict(entry_id=entry_id, process=process_record(exp, entry_id, regsha, job_sha=start['job']['sha256']))
control_records = {name: named_record(control / filename, regsha) for name, filename in
(('launch_attempt', 'launch-attempt.json'), ('launch', 'launch.json'), ('supervisor_start', 'supervisor-start.json'), ('exit', 'exit.json'))}
launch = control_records['launch']
if launch is not None:
require(launch.get('launch_observed') is True and launch.get('final_exit_observed') is False and
launch.get('not_evidence_of_completion') is True and type(launch.get('supervisor_pid')) is int, 'Incomplete launch observation')
attempt = control_records['launch_attempt']
if attempt is not None:
require(attempt.get('complete') is False and attempt.get('terminal_state') == 'UNKNOWN' and
attempt.get('automatic_retry') is False, 'Incomplete launch attempt record')
supervisor = control_records['supervisor_start']
if supervisor is not None:
require(supervisor.get('complete') is False and type(supervisor.get('pid')) is int, 'Incomplete supervisor start record')
stopped = named_record(exp / 'audit/stopped.json', regsha)
completion = named_record(exp / 'audit/execution-complete.json', regsha)
exit_record = control_records['exit']
status = 'REGISTERED-NOT-STARTED'
if attempt is not None:
status = 'LAUNCH-ATTEMPT-RECORDED-EXIT-UNKNOWN'
if control_records['launch'] is not None or control_records['supervisor_start'] is not None:
status = 'LAUNCH-RECORDED-EXIT-UNKNOWN'
if files:
status = 'BETWEEN-COMMITS-OR-PARENT-STATE-UNKNOWN'
if pending_report:
receipt = pending_report['process']
status = 'SAMPLER-STARTED-EXIT-UNKNOWN' if pending[0] == 'input-generation' else 'CHILD-STARTED-UNCOMMITTED-EXIT-UNKNOWN'
if receipt['present']:
if receipt['timed_out'] or (receipt['exit_code'] is not None and receipt['exit_code'] != 0): status = 'CHILD-FAILED-UNCOMMITTED'
elif receipt['complete']: status = 'CHILD-EXITED-ZERO-NOT-COMMITTED'
elif receipt['exit_code'] == 0: status = 'CHILD-EXITED-ZERO-NOT-VALIDATED'
if completion is not None:
require(completion.get('complete') is True and completion.get('inventory') == inventory and
len(files) == expected_processes and not pending, 'Execution completion contradicts ledger inventory')
status = 'EXECUTION-COMPLETION-RECORDED-PARENT-EXIT-UNKNOWN'
if exit_record is not None:
require(type(exit_record.get('final_exit_observed')) is bool and type(exit_record.get('complete')) is bool and
(exit_record.get('exit_code') is None or type(exit_record.get('exit_code')) is int), 'Incomplete supervisor exit record')
code = exit_record['exit_code']
require((code is None) == (exit_record['final_exit_observed'] is False) and exit_record['complete'] is (code == 0), 'Contradictory supervisor exit record')
status = 'SUPERVISOR-ERROR-EXIT-UNKNOWN' if code is None else ('PARENT-EXITED-ZERO' if code == 0 else 'PARENT-EXITED-NONZERO')
if stopped is not None:
require(stopped.get('complete') is False, 'Malformed stopped record')
status = 'STOPPED-RECORD-PRESENT'
return dict(status=status, registered=True, registration_sha256=regsha, registered_inventory=inventory,
expected_process_entries=expected_processes, committed_process_entries=len(files),
sampler_committed=counts['sampler'] == 1, brain_entries=counts['brain'], update_entries=counts['update'],
training_epochs_completed=counts['train_complete'], probes_completed=counts['probe_complete'],
result_files_present=len(results), task_results_ledger_owned=len(committed_results),
result_files_without_commit=sorted(result_paths - committed_results), latest_entry=latest_report,
uncommitted_child=pending_report, control_records=control_records, stopped_record=stopped,
execution_completion_record_present=completion is not None,
registered_workflow_inventory_recorded_complete=(len(files) == expected_processes and
counts['train_complete'] == inventory['train_calls'] and counts['probe_complete'] == inventory['probe_calls'] and not pending),
ledger_writer_lock_present=(ledger / 'writer.lock').exists(),
entries_appeared_during_read=len(directory_files(ledger / 'entries')) != len(files))
def read_progress(exp, control, registration_sha256=None):
"""Unknown/inconsistent metadata never become completed work or learning."""
base = dict(schema='normal-fast-progress-v1', scope='READ-ONLY-METADATA-NOT-INTEGRITY',
integrity_verified=False, historical_receipts_reverified=False, OS_liveness_checked=False,
learning_pass_claimed=False, scientific_metrics_evaluated=False, candidate_generated=False,
writes_performed=False, model_executed=False, automatic_retry_authorized=False,
snapshot_scope='Non-atomic observation; concurrent publication can require reading again')
try:
exp, control = safe(exp), safe(control)
require(registration_sha256 is None or SHA.fullmatch(registration_sha256), 'Invalid external registration SHA')
return dict(base, **_progress(exp, control, registration_sha256))
except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError) as error:
return dict(base, status='UNKNOWN-METADATA-INCONSISTENT', error=type(error).__name__ + ': ' + str(error),
completion_counts_available=False)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--exp', required=True)
parser.add_argument('--control', required=True)
parser.add_argument('--registration-sha256')
args = parser.parse_args()
print(json.dumps(read_progress(args.exp, args.control, args.registration_sha256), ensure_ascii=False, sort_keys=True, indent=2))