Seal Phase 4AG author-APL audit and fixed CAL failure

models/male-cns-lif/author-apl-lab-v2/run_suite.py

@@ -0,0 +1,68 @@
+"""Ordered numerical gates before any full CAL; never tune from outcomes."""
+import argparse
+import json
+from pathlib import Path
+import subprocess
+import time
+import traceback
+from context import ROOT,MODEL,EXP,CFG,check,save,budget
+
+SCIENCE=ROOT/'models/male-cns-lif/.venv/bin/python'
+ORACLE=ROOT/'experiments/phase4ad-author-semantics/environment/.venv/bin/python'
+
+
+def child(backend,name,repeat=0):
+    log=EXP/'logs'/f'{backend}-{name}-r{repeat}.log'
+    cmd=[str(ORACLE if backend=='oracle' else SCIENCE),'-B',str(MODEL/'run_case.py'),backend,name,'--repeat',str(repeat)]
+    print('START',backend,name,repeat,flush=True);t0=time.perf_counter()
+    with log.open('x',encoding='utf-8') as f:
+        result=subprocess.run(cmd,stdout=f,stderr=subprocess.STDOUT,timeout=1800,check=False)
+    print('FINISH',backend,name,repeat,'exit',result.returncode,'wall',round(time.perf_counter()-t0,2),flush=True)
+    if result.returncode:raise RuntimeError(f'{backend} {name} failed; inspect {log}')
+
+
+def gate_case(name,repeats):
+    from compare import compare,compare_gpu_repeats
+    child('gpu',name)
+    for r in range(1,repeats):child('gpu',name,r)
+    child('oracle',name)
+    result=compare(EXP/'cases'/name,EXP/'gpu'/(name+'-r0'),EXP/'oracle'/name,EXP/'audit'/('compare-'+name+'.json'))
+    assert result['pass_all'],f'Closed-loop exact comparison failed: {name}; no later CAL permitted'
+    for r in range(1,repeats):
+        report=compare_gpu_repeats(EXP/'gpu'/(name+'-r0'),EXP/'gpu'/f'{name}-r{r}',EXP/'audit'/f'repeat-{name}-r{r}.json')
+        assert report['pass_all'],f'Exact GPU repeat failed: {name}'
+    child('component',name)
+    return dict(case=name,pass_all=True,gpu_repeats=repeats)
+
+
+def main():
+    parser=argparse.ArgumentParser();parser.add_argument('stage',choices=['fixtures','probes','calibration']);args=parser.parse_args()
+    check();budget();receipt=EXP/'audit'/(args.stage+'-gate.json');assert not receipt.exists()
+    try:
+        if args.stage=='fixtures':
+            rows=[gate_case(n,CFG['fixture_repeats']) for n in CFG['fixtures']]
+            save(receipt,dict(pass_all=True,cases=rows,scope='Fixed real-edge induced fixtures only'))
+        elif args.stage=='probes':
+            assert json.loads((EXP/'audit/fixtures-gate.json').read_text())['pass_all']
+            rows=[gate_case(n,1) for n in CFG['calibration']['full_oracle_probes']]
+            save(receipt,dict(pass_all=True,cases=rows,scope='Only two original first-CAL fixed fullgraph trajectories'))
+        else:
+            assert json.loads((EXP/'audit/fixtures-gate.json').read_text())['pass_all']
+            assert json.loads((EXP/'audit/probes-gate.json').read_text())['pass_all']
+            records=json.loads((EXP/'protocol/calibration-cases.json').read_text());assert len(records)==60
+            probes=CFG['calibration']['full_oracle_probes']
+            for rec in records:
+                if rec['name'] not in probes:child('gpu',rec['name'])
+                if rec['seed']==CFG['calibration']['seeds'][0]:child('gpu',rec['name'],1)
+            from calibration import summarize
+            result=summarize()
+            save(receipt,dict(pass_all=result['pass_all'],scope='Complete original six-CAL representation gate, not heldout or learning',summary='calibration/summary.json'))
+            if not result['pass_all']:raise SystemExit(2)
+    except Exception as e:
+        save(EXP/'audit'/('stopped-'+args.stage+'.json'),dict(stage=args.stage,type=type(e).__name__,message=str(e),traceback=traceback.format_exc(),
+             later_stages_not_permitted=True,partial_outputs_preserved=True,parameters_changed=False))
+        raise
+    budget()
+
+
+if __name__=='__main__':main()