qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add bounded same-state update validation and functional gates

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-09-30 19:49:02 +08:00
parent b33e7ea047
commit d54f0b3ef1
31 changed files with 7951 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/male-cns-lif/conditioning-execution-v1/test_execution_workflow.py
+333
View File
@@ -0,0 +1,333 @@
"""CPU-only workflow mocks: real local receipts/files, never GPU/remote calls."""
from copy import deepcopy
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import numpy as np
import boundary_plan as P
import execution_workflow as F
class WorkflowTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
path=Path(__file__).resolve().parent.parent/'conditioning-interface-v2/isolated_workers.py'
spec=importlib.util.spec_from_file_location('workflow_test_frozen_AP_worker',path)
cls.ap=importlib.util.module_from_spec(spec);spec.loader.exec_module(cls.ap)
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.base = Path(self.tmp.name)
self.exp,self.model,self.lab = (self.base/n for n in ('phase4-fixture','model','lab'))
for name in ('protocol/case-templates','inputs/static','cases','runs','updates','jobs','authorities',
'receipts','logs','audit','extension-build'):
(self.exp/name).mkdir(parents=True,exist_ok=True)
self.model.mkdir(); self.lab.mkdir()
self.ap_worker = self.base/'ap_worker.py'
self.put(self.ap_worker,'# fixture AP worker')
sources = {}
for name in set().union(*F.W.SOURCES.values()) | {'isolated_execution.py'}:
path = self.model/name; self.put(path,'# fixture source '+name); sources[str(path)]=F.digest(path)
for name in ('runtime_light','runtime_continuation','conditioning_update','eligibility_bridge',
'phase4j','am_overlay','conditioning_epoch','conditioning_preflight_compare','engineering_compare'):
self.put(self.lab/(name+'.py'),'# fixture '+name)
self.patches = [patch.object(F,'EXP',self.exp),patch.object(F,'MODEL',self.model),
patch.object(F,'LAB',self.lab),patch.object(F,'AP_WORKER',self.ap_worker)]
for item in self.patches:
item.start(); self.addCleanup(item.stop)
self.plan=P.build_plan(); self.calls={c['call_id']:c for c in self.plan['calls']}
self.requests=dict(tick=np.array([3,900],np.int32),neuron_idx=np.array([12,15],np.int32))
F.save_npz(self.exp/'inputs/static/mechanical-requests.npz',self.requests)
self.initial=self.overlay('zero')
parents={}
for arm,descriptor in self.plan['imported_parents'].items():
directory=self.base/('AP-'+arm); directory.mkdir()
self.checkpoint(directory,1200,'reference')
update=self.base/('AP-'+arm+'-update'); self.update_files(update)
before=self.overlay('before-'+arm)
case=self.base/('AP-'+arm+'-case');case.mkdir()
F.save(case/'spec.json',dict(steps=1200));F.save_npz(case/'requests.npz',self.requests)
epoch=dict(schema='phase4an-epoch-postprocess-authority-v1',checkpoint_dir=str(directory),
checkpoint_manifest_sha256=F.digest(directory/'checkpoint-manifest.json'),
files={'requests':F.A.pin(case/'requests.npz')},frozen_inputs={})
for role in ('runtime_continuation','runtime_light','conditioning_update','eligibility_bridge','phase4j','am_overlay'):
epoch['files'][role+'_source']=F.A.pin(self.lab/(role+'.py'))
ep=self.base/('AP-'+arm+'.epoch.json');F.save(ep,epoch)
parents[descriptor['role']]=dict(directory=str(directory),checkpoint_sha256=epoch['checkpoint_manifest_sha256'],
epoch_authority=F.A.pin(ep),update_dir=str(update),before_overlay=before,
after_overlay=dict(overlay_file=str(update/'delta-after.npz'),overlay_manifest=str(update/'overlay-manifest.json')))
templates={}
for call in self.plan['calls']:
group=call['case_group']
if group not in templates:
path=self.exp/'protocol/case-templates'/f'{group}.json'
F.save(path,dict(name=group,steps=call['steps'],max_steps=call['steps'],
within_trial_weight_updates=call.get('within_trial_weight_updates',False)))
templates[group]=str(path)
self.reg=dict(plan=self.plan,parent_records=parents,initial_overlay=self.initial,case_templates=templates,
source_sha256=sources,input_sha256={},prepared_file_sha256={},budget=dict(root=str(self.exp),
output_cap_bytes=2**30,minimum_free_reserve_bytes=0,child_timeout_seconds=20,workflow_timeout_seconds=7200))
F.save(self.exp/'protocol/preregistered.json',self.reg)
self.w=F.Workflow.__new__(F.Workflow)
self.w.sha=F.digest(self.exp/'protocol/preregistered.json');self.w.reg=self.reg
self.w.parents=deepcopy(parents);self.w.jobs={};self.w.boundaries={};self.w.completed=[];self.w.cycles=[];self.w.sampler=None
self.w.started=0.;self.w.deadline=10**12
self.w.h=SimpleNamespace(GPU='fixture-gpu',COMFY_URLS=('fixture-queue',),
SOURCE_PINS={n:F.digest(self.lab/(n+'.py')) for n in ('runtime_light','runtime_continuation','conditioning_epoch')})
self.launched=[];self.boundary_args=[]
def put(self,path,text):
path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
path.write_text(text,encoding='utf-8')
def overlay(self,name):
npz=self.base/(name+'.npz');meta=self.base/(name+'.json')
F.save_npz(npz,dict(delta=np.array([0.],np.float32)));F.save(meta,dict(fixture=name))
return dict(overlay_file=str(npz),overlay_manifest=str(meta))
def checkpoint(self,directory,steps,mode,offset=0):
F.save(directory/'summary.json',dict(steps=steps,diagnostic_mode=mode,global_next_tick=offset+steps,
effective_weights_raw_sha256='f'*64))
F.save_npz(directory/'state.npz',dict(step=np.array([0],np.int32)))
F.save(directory/'checkpoint-manifest.json',dict(fixture=True,steps=steps))
def update_files(self,directory):
directory.mkdir()
for name in ('chain-receipt.json','update-receipt.json','overlay-manifest.json'):
F.save(directory/name,dict(fixture=True))
F.save_npz(directory/'delta-after.npz',dict(delta=np.array([-.01],np.float32)))
def fake_boundary(self,**kw):
self.boundary_args.append(deepcopy(kw))
case=Path(kw['case']);parent=Path(kw['parent'])
return dict(schema='conditioning-updated-boundary-v1',transition=kw['transition'],
diagnostic_mode=kw['mode'],boundary_id=kw['boundary_id'],
case=dict(spec=F.A.pin(case/'spec.json'),requests=F.A.pin(case/'requests.npz')),
parent=dict(directory=str(parent),checkpoint_manifest_sha256=F.digest(parent/'checkpoint-manifest.json')),
overlay_before={k:F.A.pin(v) for k,v in kw['before_overlay'].items()},
overlay_after={k:F.A.pin(v) for k,v in kw['after_overlay'].items()},
source_files=kw['source_files'],frozen_inputs={})
def fake_launch(self,name,job,*,ap=False):
self.launched.append((name,deepcopy(job),ap));output=Path(job['output_dir'])
kind=job['kind'];result={}
if kind=='update':self.update_files(output)
else:
output.mkdir()
if kind=='sample':
for cue,node in (('A',20),('B',30)):
F.save_npz(output/f'{cue}-requests.npz',dict(tick=np.array([5],np.int32),neuron_idx=np.array([node],np.int32)))
elif kind=='validate_cpu':F.save(output/'validation.json',dict(fixture=True))
else:
if kind=='brain':case=Path(job['case_dir'])
elif kind=='oracle':case=Path(job['oracle_case'])
else:case=Path(F.read(job['authority_file'])['case']['spec']['path']).parent
spec=F.read(case/'spec.json')
self.checkpoint(output,spec['steps'],job.get('diagnostic_mode','reference'),spec.get('global_tick_offset',0))
result['checkpoint_manifest_sha256']=F.digest(output/'checkpoint-manifest.json')
receipt=dict(output_dir=str(output),result=result)
loc=self.exp/'receipts'/f'{name}.fixture-process.json';F.save(loc,dict(name=name,fixture=True))
self.w.jobs[name]=dict(loc=dict(process=str(loc)),job=job,receipt=receipt)
self.put(self.exp/'extension-build/.ninja_log',str(len(self.launched)))
self.put(self.exp/'extension-build/new-extension.bin','compiled fixture artifact')
return receipt
def test_new_job_exact_schema_and_direct_authority_parent_closure(self):
cp=self.base/'AP-D1'
auth=self.exp/'authorities/test.json'
file=self.base/'binding.txt';self.put(file,'bound')
F.save(auth,dict(parent=dict(directory=str(cp),checkpoint_manifest_sha256=F.digest(cp/'checkpoint-manifest.json')),
files=dict(x=F.A.pin(file)),frozen_inputs={str(self.ap_worker):F.digest(self.ap_worker)}))
job=self.w.new_job('test','updated',auth,build_root=str(self.exp/'extension-build'),diagnostic_mode='light')
self.assertEqual(set(job),F.W.COMMON | F.W.FIELDS['updated'])
self.assertEqual(set(job['source_sha256']),F.W.SOURCES['updated'])
for path in (file,auth,cp/'checkpoint-manifest.json',self.exp/'protocol/preregistered.json'):
self.assertEqual(job['input_sha256'][str(path)],F.digest(path))
sample=self.w.new_job('evaluation-inputs','sample',self.exp/'protocol/preregistered.json')
self.assertEqual(set(sample),F.W.COMMON | F.W.FIELDS['sample'])
self.assertEqual(set(sample['source_sha256']),F.W.SOURCES['sample'])
def test_full_fixed_workflow_23_brains_31_processes_and_shared_case_lineage(self):
with patch.object(self.w,'launch',side_effect=self.fake_launch),patch.object(F.A,'boundary_authority',side_effect=self.fake_boundary):
self.w.run()
self.assertEqual(len(self.w.completed),23);self.assertEqual(len(self.w.jobs),31)
self.assertEqual(len(list((self.exp/'cases').iterdir())),17)
byname={n:j for n,j,_ in self.launched}
for _,job,ap in self.launched:
expected=self.ap.COMMON | (self.ap.BRAIN_FIELDS if job['kind']=='brain' else self.ap.UPDATE_FIELDS) if ap else F.W.COMMON | F.W.FIELDS[job['kind']]
self.assertEqual(set(job),expected)
self.assertEqual(sum(j['kind'] in ('brain','updated','oracle','unchanged') for j in byname.values()),23)
self.assertEqual(sum(j['kind']=='sample' for j in byname.values()),1)
for arm in ('D1','D0','frozen','mechanical'):
r=F.read(byname[f'{arm}--updated--reference']['authority_file'])
l=F.read(byname[f'{arm}--updated--light']['authority_file'])
self.assertEqual(r['case'],l['case']);self.assertEqual(r['parent'],l['parent'])
for arm in ('D0','frozen'):
job=byname[f'{arm}--old-C--reference'];p=self.reg['parent_records'][f'AP.{arm}.reference.epoch001']
self.assertEqual(job['overlay_file'],p['before_overlay']['overlay_file'])
self.assertNotEqual(job['overlay_file'],p['after_overlay']['overlay_file'])
for arm in ('D1','mechanical'):
self.assertEqual(byname[f'{arm}--live-oracle']['authority_file'],byname[f'{arm}--updated--reference']['authority_file'])
after=self.reg['parent_records']['AP.D1.reference.epoch001']['after_overlay']
idle=next(v for v in self.boundary_args if v['boundary_id']=='D1--idle--9s')
self.assertEqual(idle['before_overlay'],after);self.assertEqual(idle['after_overlay'],after)
self.assertIsNone(idle['epoch']);self.assertIsNone(idle['update_dir']);self.assertIsNone(idle['cpu_receipt'])
self.assertEqual(self.w.parents['D1--idle--9s']['after_overlay'],after)
for t in ('t0','t1','t10'):
for cue,node in (('A',20),('B',30)):
request=F.npz(self.exp/'cases'/f'D1--probe--{t}--{cue}'/'requests.npz')
np.testing.assert_array_equal(request['neuron_idx'],np.array([node],np.int32))
performance=byname['performance--natural-light-fresh']
prior=self.reg['parent_records']['AP.D1.reference.epoch001']
self.assertEqual(performance['overlay_file'],prior['before_overlay']['overlay_file'])
update=F.read(byname['performance--natural-light-fresh--original-E']['authority_file'])
self.assertEqual((update['command']['us_tick'],update['command']['trace_backend'],update['diagnostic_mode']),(900,'torch','light'))
self.assertIn('performance--natural-light-fresh--cpu-verify',byname)
cpu=byname['performance--natural-light-fresh--cpu-verify']
self.assertEqual(cpu['authority_file'],byname['performance--natural-light-fresh--original-E']['authority_file'])
self.assertEqual(cpu['update_dir'],byname['performance--natural-light-fresh--original-E']['output_dir'])
for output in Path(cpu['update_dir']).iterdir():
self.assertEqual(cpu['input_sha256'][str(output)],F.digest(output))
probe=byname['performance--evaluation-A-light-fresh']
self.assertEqual(probe['overlay_file'],self.initial['overlay_file'])
self.assertNotIn('parent_checkpoint_manifest_sha256',F.read(Path(probe['case_dir'])/'spec.json'))
self.assertFalse(any(n.startswith('performance--evaluation-A-light-fresh--') for n in byname))
complete=F.read(self.exp/'audit/execution-complete.json')
self.assertFalse(complete['scientific_learning_pass_claimed'])
def test_mutable_build_inventory_visible_new_artifacts_not_silently_ignored(self):
self.w.reg=deepcopy(self.reg)
self.w.reg['plan']=dict(imported_parents={},calls=self.plan['calls'][:2],
counts=dict(new_brain_calls=2,total_children=0))
def execute(call):
self.put(self.exp/'extension-build/.ninja_log',call['call_id'])
self.put(self.exp/'extension-build/new-extension.bin','compiled fixture artifact')
F.save(self.exp/'runs'/(call['call_id']+'.json'),dict(complete=True))
with patch.object(self.w,'execute_call',side_effect=execute):
self.w.run()
self.assertNotIn('extension-build/.ninja_log',F.tree(self.exp,exclude_build=True))
self.assertIn('extension-build/.ninja_log',F.tree(self.exp))
for cycle in self.w.cycles:
self.assertIn('.ninja_log',cycle['mutable_build_inventory_after'])
self.assertIn('new-extension.bin',cycle['mutable_build_inventory_after'])
self.assertGreater(cycle['mutable_build_bytes_after'],0)
self.assertTrue(cycle['output_inventory'])
old=self.exp/'audit/pinned-result.json';F.save(old,dict(value=1))
before=F.tree(self.exp,exclude_build=True);self.put(old,'{"value":2}')
after=F.tree(self.exp,exclude_build=True)
self.assertNotEqual(before['audit/pinned-result.json'],after['audit/pinned-result.json'])
def test_probe_scope_mutation_rejected_and_shared_case_mismatch_rejected(self):
call=deepcopy(self.calls['performance--evaluation-A-light-fresh']);call['US']=True
with self.assertRaises(ValueError):self.w.execute_call(call)
call=self.calls['D1--updated--reference'];parent=self.w.parents[call['parent']]
case=self.w.materialize(call,parent)
altered=deepcopy(parent);altered['checkpoint_sha256']='0'*64
with self.assertRaises(ValueError):self.w.materialize(call,altered)
self.assertEqual(F.read(case/'spec.json')['parent_checkpoint_manifest_sha256'],parent['checkpoint_sha256'])
def child_receipt(self,command,*,ap=False,mutate=None,**kwargs):
jobfile=Path(command[command.index('--job')+1]);job=F.read(jobfile)
out=Path(job['output_dir']);out.mkdir();F.save(out/'result.json',dict(fixture=True))
attempt=Path(job['receipt_file']+'.attempt.json');F.save(attempt,dict(fixture=True))
script=self.ap_worker if ap else self.model/'isolated_execution.py'
pins={**job['input_sha256'],str(jobfile):F.digest(jobfile),str(script):job['worker_sha256']}
if ap:pins.update({v['path']:v['sha256'] for v in job['sources'].values()})
else:
pins[str(self.ap_worker)]=F.W.AP_SHA
pins.update({str(self.model/k):v for k,v in job['source_sha256'].items()})
receipt=dict(schema='phase4ap-isolated-worker-receipt-v1' if ap else 'conditioning-isolated-execution-receipt-v1',
complete=True,job_sha256=F.digest(jobfile),registration_sha256=self.w.sha,kind=job['kind'],
job_id=job['job_id'],worker_sha256=job['worker_sha256'],output_dir=job['output_dir'],
output_files_sha256={'result.json':F.digest(out/'result.json')},attempt_sha256=F.digest(attempt))
receipt['verified_input_sha256' if ap else 'input_sha256']=pins
if mutate:mutate(receipt)
F.save(job['receipt_file'],receipt)
return SimpleNamespace(returncode=0)
def sample_job(self,name):
return self.w.new_job(name,'sample',self.exp/'protocol/preregistered.json')
def test_real_parent_process_receipt_binds_exit_inputs_outputs_and_wall_layers(self):
job=self.sample_job('success')
with patch.object(self.w,'admission'),patch.object(F,'verify'), \
patch.object(F.subprocess,'run',side_effect=self.child_receipt) as run, \
patch.object(F.time,'perf_counter',side_effect=[100.,104.,107.]):
self.w.deadline=110.;self.w.launch('success',job)
process=F.read(self.exp/'receipts/success.process.json')
self.assertEqual((process['exit_code'],process['child_wall_seconds'],process['validation_wall_seconds'],process['end_to_end_seconds']),(0,4.,3.,7.))
self.assertEqual(run.call_args.kwargs['timeout'],10.)
self.assertEqual(process['worker_receipt_sha256'],F.digest(self.exp/'receipts/success.worker.json'))
self.assertGreater(process['output_bytes'],0)
self.assertTrue(process['complete'])
def test_frozen_AP_worker_parent_receipt_uses_verified_source_closure(self):
prior=self.reg['parent_records']['AP.D1.reference.epoch001']
case=Path(F.read(prior['epoch_authority']['path'])['files']['requests']['path']).parent
job=self.w.ap_job('performance-ap','brain',case=case,overlay=prior['before_overlay'],output=self.exp/'runs/performance-ap')
self.assertEqual(set(job),self.ap.COMMON | self.ap.BRAIN_FIELDS)
with patch.object(self.w,'admission'),patch.object(F,'verify'), \
patch.object(F.subprocess,'run',side_effect=lambda *a,**kw:self.child_receipt(*a,ap=True,**kw)) as run:
self.w.launch('performance-ap',job,ap=True)
self.assertEqual(run.call_args.args[0][3],'brain')
receipt=F.read(self.exp/'receipts/performance-ap.worker.json')
for role in ('runtime_light','runtime_continuation'):
binding=job['sources'][role]
self.assertEqual(receipt['verified_input_sha256'][binding['path']],binding['sha256'])
def test_exit_failure_timeout_and_receipt_mutation_preserved_and_not_retried(self):
variants=[('exit',lambda *a,**kw:SimpleNamespace(returncode=9)),
('timeout',lambda *a,**kw:(_ for _ in ()).throw(subprocess.TimeoutExpired('fixture',1))),
('identity',lambda *a,**kw:self.child_receipt(*a,mutate=lambda r:r.update(job_id='wrong'),**kw)),
('closure',lambda *a,**kw:self.child_receipt(*a,mutate=lambda r:r['input_sha256'].clear(),**kw)),
('output',lambda *a,**kw:self.child_receipt(*a,mutate=lambda r:r['output_files_sha256'].clear(),**kw))]
for name,action in variants:
job=self.sample_job(name)
with patch.object(self.w,'admission'),patch.object(F,'verify'),patch.object(F.subprocess,'run',side_effect=action) as run:
with self.assertRaises(ValueError):self.w.launch(name,job)
self.assertFalse(F.read(self.exp/f'receipts/{name}.process.json')['complete'])
with self.assertRaises(ValueError):self.w.launch(name,job)
self.assertEqual(run.call_count,1)
self.assertNotIn(name,self.w.jobs)
def test_workflow_attempt_prevents_restart(self):
F.save(self.exp/'audit/execution-attempt.json',dict(partial=True))
with patch.object(self.w,'cpu_verify') as verify:
with self.assertRaises(ValueError):self.w.run()
verify.assert_not_called()
def test_whole_deadline_starts_before_registration_verification(self):
clock=[10.]
def registration(sha):clock[0]=50.;return self.reg
with patch.object(F,'registration',side_effect=registration),patch.object(F.W,'ap_helpers',return_value=self.w.h), \
patch.object(F.time,'perf_counter',side_effect=lambda:clock[0]):
workflow=F.Workflow('a'*64)
self.assertEqual(workflow.started,10.)
self.assertEqual(workflow.deadline,10.+self.reg['budget']['workflow_timeout_seconds'])
def test_no_complete_after_final_cycle_inventory_crosses_deadline(self):
call=self.calls['D1--updated--reference'];self.w.reg=deepcopy(self.reg)
self.w.reg['plan']=dict(imported_parents={},calls=[call],counts=dict(new_brain_calls=1,total_children=0))
clock=[1.];self.w.started=0.;self.w.deadline=5.
def execute(_):clock[0]=6.
with patch.object(self.w,'execute_call',side_effect=execute),patch.object(F.time,'perf_counter',side_effect=lambda:clock[0]):
with self.assertRaisesRegex(ValueError,'wall limit'):
self.w.run()
self.assertFalse((self.exp/'audit/execution-complete.json').exists())
def test_import_does_not_load_old_coordinator_neural_runtime_or_torch(self):
code="import sys;sys.path.insert(0,sys.argv[1]);import execution_workflow;bad=[k for k in sys.modules if k in ('torch','runtime_light','runtime_continuation','conditioning_epoch','_conditioning_execution_prior_integrity')];assert not bad,bad"
completed=subprocess.run([sys.executable,'-B','-c',code,str(Path(__file__).resolve().parent)],
capture_output=True,text=True,timeout=20)
self.assertEqual(completed.returncode,0,completed.stderr)
if __name__=='__main__':unittest.main()