Seal AO interface failure and verified isolated postprocessing diagnosis

tools/phase4ao-publication/finalize_failed_phase4ao.py

@@ -0,0 +1,128 @@
+"""Additive failure sealing, never an AO execution retry or scientific PASS."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sys
+
+ROOT = Path('/data/digital-drosophila')
+EXP = ROOT / 'experiments/phase4ao-conditioning-interface'
+DIAG = ROOT / 'experiments/phase4ao-postprocess-isolation'
+MODEL = ROOT / 'models/male-cns-lif/olfactory-conditioning-lab-v1'
+
+
+def digest(p):
+    h = hashlib.sha256()
+    with Path(p).open('rb') as f:
+        for b in iter(lambda: f.read(8 << 20), b''): h.update(b)
+    return h.hexdigest()
+
+
+def read(p): return json.loads(Path(p).read_text())
+
+
+def save(p, value):
+    with Path(p).open('x', encoding='utf-8') as f:
+        json.dump(value, f, sort_keys=True, indent=2, allow_nan=False); f.write('\n')
+
+
+def seal(directory, helper):
+    assert not (directory / 'manifest.json').exists() and not (directory / 'checksums.sha256').exists()
+    paths = sorted(directory.rglob('*'))
+    assert not any(p.is_symlink() or '__pycache__' in p.parts for p in paths)
+    rows = [dict(path=p.relative_to(directory).as_posix(), bytes=p.stat().st_size, sha256=digest(p))
+            for p in paths if p.is_file()]
+    save(directory / 'manifest.json', dict(root=str(directory), files=rows))
+    with (directory / 'checksums.sha256').open('x', encoding='utf-8', newline='\n') as f:
+        f.writelines(r['sha256'] + '  ' + r['path'] + '\n' for r in rows)
+        f.write(digest(directory / 'manifest.json') + '  manifest.json\n')
+    return helper.verify_seal(directory)
+
+
+def main(reg_sha, checkpoint_sha, stopped_sha, diagnostic_sha):
+    assert sys.dont_write_bytecode
+    assert not (EXP / 'manifest.json').exists() and not (EXP / 'audit/final-status.json').exists()
+    assert digest(EXP / 'audit/stopped.json') == stopped_sha
+    assert digest(DIAG / 'diagnostic-receipt.json') == diagnostic_sha
+    sys.path.insert(0, str(MODEL))
+    import conditioning_preflight_prepare as P
+    import runtime_continuation as C
+    reg = P.read_registration(reg_sha)
+    assert reg['plan']['total_calls'] == 12
+    row_id = reg['plan']['calls'][0]['row_id']
+    assert row_id == 'D1--reference--epoch-000'
+    assert sorted(p.name for p in (EXP / 'runs').iterdir()) == [row_id]
+    assert not any((EXP / 'updates').iterdir())
+    assert not (EXP / 'audit/execution-complete.json').exists()
+    assert sorted(p.name for p in (EXP / 'receipts').glob('*.process.json')) == ['input-generation.process.json']
+    stopped = read(EXP / 'audit/stopped.json')
+    assert stopped['completed_rows'] == [] and stopped['registration_sha256'] == reg_sha
+    stderr = (EXP / 'logs' / (row_id + '.stderr.log')).read_text()
+    assert 'E.postprocess_epoch' in stderr and 'Existing unverified module binding: phase4an_sealed_am_overlay' in stderr
+    checkpoint = C.load_checkpoint(EXP / 'runs' / row_id, checkpoint_sha)
+    assert checkpoint['fresh'] and checkpoint['steps'] == 1200 and checkpoint['summary']['all_finite']
+    diagnostic = read(DIAG / 'diagnostic-receipt.json')
+    assert diagnostic['status'] == 'PASS-ISOLATED-POSTPROCESS-DIAGNOSTIC-ONLY'
+    assert diagnostic['registration_sha256'] == reg_sha and diagnostic['checkpoint_manifest_sha256'] == checkpoint_sha
+    assert diagnostic['stopped_sha256'] == stopped_sha and diagnostic['new_brain_runs'] == 0
+    assert diagnostic['weights_installed'] is False and diagnostic['original_protocol_remains_stopped'] is True
+    P.verify_pins(diagnostic['result_files_sha256'])
+    original = P.P.read_registration(P.AN_REG)
+    helper = P.P.load_module('phase4ao_failure_original_seal', ROOT / P.P.AI_HELPER_REL, P.P.AI_HELPER_SHA)
+    # Recheck previous complete seals without rerunning their neural trials.
+    save(EXP / 'regression/AN-seal-after.json', helper.verify_seal(P.AN))
+    save(EXP / 'regression/AM-seals-after.json', P.P.verify_authority(original['anchors']))
+    old = P.P.verify_old_integrity(original)
+    save(EXP / 'regression/old-integrity-after.json', old)
+    P.read_registration(reg_sha)
+    C.load_checkpoint(EXP / 'runs' / row_id, checkpoint_sha)
+    source = Path(__file__).resolve().parent
+    for name in ('diagnose_postprocess_isolation.py', 'test_postprocess_isolation.py', 'finalize_failed_phase4ao.py'):
+        target = DIAG / name
+        assert not target.exists()
+        shutil.copyfile(source / name, target)
+    diag_seal = seal(DIAG, helper)
+    summary = checkpoint['summary']
+    analysis = dict(status='FAIL-POSTPROCESS-MODULE-IDENTITY', registration_sha256=reg_sha,
+        planned_brain_calls=12, actual_complete_brain_trajectories=1, complete_workflow_rows=0,
+        executed_biological_ms=120, executed_ticks=1200, stopped_sha256=stopped_sha,
+        first_checkpoint_manifest_sha256=checkpoint_sha,
+        failure_stage='after neural runtime and immediate seal, before epoch delta output creation',
+        cause='Two separately loaded frozen runtime_light module objects use private _LOADED caches but share a sys.modules helper name; the second correctly rejects unowned existing binding.',
+        previous_synthetic_test_gap='Tests covered isolated loader and synthetic overlay fixture, not the mixed actual loader integration.',
+        unchanged_protection='No loader cache or frozen source was patched or bypassed.',
+        runtime=dict(neurons=summary['nodes'], edges=summary['edges'], spikes=summary['total_spikes'],
+            all_finite=summary['all_finite'], simulation_seconds=summary['diagnostic_simulation_seconds'],
+            wall_seconds=summary['wall_seconds'], build_or_cache_seconds=summary['build_or_cache_seconds'],
+            peak_allocated_vram_bytes=summary['peak_allocated_vram_bytes']),
+        diagnostic=dict(receipt_sha256=diagnostic_sha, experiment=str(DIAG), manifest_sha256=diag_seal['manifest_sha256'],
+            status=diagnostic['status'], actual_changed_edges=diagnostic['counts']['actual_bitwise_changed_edges'],
+            off_compartment_changed_edges=diagnostic['counts']['actual_off_compartment_bitwise_changed_edges'],
+            weights_installed=False, old_trajectory_rerun=False, new_brain_runs=0),
+        old_integrity_roots=len(old), old_integrity_pass=True, prior_AN_AM_seals_preserved=True,
+        original_sources_unchanged=True, scientific_learning_assessed=False, learning_success_claimed=False,
+        full_conditioning_executed=False, learned_candidate=False,
+        next_step='Separately register a new process-isolated coordinator. Do not overwrite or resume this stopped AO inventory; no result-selected seed or brain parameter change.')
+    save(EXP / 'audit/failure-analysis.json', analysis)
+    final = dict(status=analysis['status'], analysis_path='audit/failure-analysis.json',
+        analysis_sha256=digest(EXP / 'audit/failure-analysis.json'), registration_sha256=reg_sha,
+        planned_calls=12, actual_complete_brain_trajectories=1, completed_calls=0,
+        old_integrity_roots=len(old), old_integrity_pass=True, prior_AN_AM_seals_preserved=True,
+        frozen_21_and_previous_draft_20_unchanged=True, registered_51_sources_unchanged=True,
+        learned_candidate=False, learning_success_claimed=False, full_conditioning_executed=False,
+        old_trajectory_rerun=False, original_failure_preserved=True, model_directory_sealed=False,
+        five_reference_trials='Sealed AM evidence retained; not rerun by AO or the CPU diagnosis.',
+        finalizer_source_sha256=digest(__file__), diagnostic_manifest_sha256=diag_seal['manifest_sha256'])
+    save(EXP / 'audit/final-status.json', final)
+    verified = seal(EXP, helper)
+    print(json.dumps(dict(status=final['status'], manifest_sha256=verified['manifest_sha256'],
+        final_sha256=digest(EXP / 'audit/final-status.json'), diagnostic_manifest_sha256=diag_seal['manifest_sha256'])), flush=True)
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser()
+    for name in ('registration-sha256', 'checkpoint-sha256', 'stopped-sha256', 'diagnostic-sha256'):
+        p.add_argument('--' + name, required=True)
+    a = p.parse_args()
+    main(a.registration_sha256, a.checkpoint_sha256, a.stopped_sha256, a.diagnostic_sha256)