Digital Drosophila Archive
214 lines
14 KiB
Python
214 lines
14 KiB
Python
"""Small synthetic filesystem receipts; these are NOT real cost measurements."""
|
|
from copy import deepcopy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
import boundary_plan as P
|
|
import budget_report as R
|
|
|
|
|
|
class BudgetReportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
|
self.root=Path(self.tmp.name);self.exp=self.root/'phase4-fixture';self.model=self.root/'model'
|
|
for name in ('audit','receipts','jobs','logs','runs','updates','authorities','cases','protocol','extension-build'):
|
|
(self.exp/name).mkdir(parents=True,exist_ok=True)
|
|
self.model.mkdir();self.worker=self.model/'isolated_execution.py';self.ap=self.model/'ap_worker.py'
|
|
for path in (self.worker,self.ap,self.model/'runtime_light.py',self.model/'runtime_continuation.py',self.model/'conditioning_epoch.py'):
|
|
path.write_text('# CPU fixture '+path.name,encoding='utf-8')
|
|
self.patch('EXP',self.exp);self.patch('MODEL',self.model);self.patch('AP_WORKER',self.ap)
|
|
p=patch.object(R.F.W,'AP_SHA',R.A.digest(self.ap));p.start();self.addCleanup(p.stop)
|
|
self.plan=P.build_plan();self.calls={c['call_id']:c for c in self.plan['calls']}
|
|
self.zero=self.exp/'protocol/zero.npz';R.F.save_npz(self.zero,dict(delta=np.array([0.],np.float32)))
|
|
self.reg=dict(plan=self.plan,source_sha256={str(self.worker):R.A.digest(self.worker)},
|
|
initial_overlay=dict(overlay_file=str(self.zero),overlay_manifest=str(self.zero)),
|
|
input_sha256={},prepared_file_sha256={})
|
|
R.F.save(self.exp/'protocol/preregistered.json',self.reg);self.regsha=R.A.digest(self.exp/'protocol/preregistered.json')
|
|
p=patch.object(R.F,'registration',side_effect=self.registration);p.start();self.addCleanup(p.stop)
|
|
self.specs={};self.processes={};self.cycle_times={}
|
|
self.put(self.exp/'extension-build/kernel.bin','actual-fixture-cache')
|
|
for index,call in enumerate(self.plan['calls']):
|
|
name=call['call_id'];start=(index+1)*100.;self.cycle_times[name]=(start,start+40.)
|
|
if call['runner'] in ('runtime_light','fresh_probe'):
|
|
case=self.exp/'cases'/name;case.mkdir()
|
|
spec=dict(steps=call['steps'],dt_ms=.1,model=dict(frozen_fixture=True),apl=dict(gain=.01),
|
|
precision='float64',apl_precision='float32',apl_delay_steps=18,
|
|
graph_dir='pinned-fullgraph-fixture',apl_bundle_dir='pinned-APL-fixture')
|
|
R.F.save(case/'spec.json',spec)
|
|
R.F.save_npz(case/'requests.npz',dict(tick=np.array([5],np.int32),neuron_idx=np.array([20],np.int32)))
|
|
self.specs[name]=case
|
|
self._epoch(R.TRAIN);self._epoch('mechanical--fresh-parent')
|
|
for arm in ('D1','D0','frozen'):self._process('AP-'+arm+'--cpu-verify','validate_cpu',10.)
|
|
for call in self.plan['calls']:
|
|
name=call['call_id'];start=self.cycle_times[name][0]
|
|
kind={'runtime_light':'brain','fresh_probe':'brain','runtime_continuation':'unchanged',
|
|
'updated_continuation':'updated','boundary_oracle':'oracle'}[call['runner']]
|
|
self._process(name,kind,start+1)
|
|
if name in (R.TRAIN,'mechanical--fresh-parent'):
|
|
self._process(name+'--original-E','update',start+11,base=name)
|
|
self._process(name+'--cpu-verify','validate_cpu',start+21,base=name)
|
|
self._process('evaluation-inputs','sample',self.cycle_times[R.TRAIN][1]+10)
|
|
build=R.F.tree(self.exp/'extension-build')
|
|
for call in self.plan['calls']:
|
|
name=call['call_id'];start,end=self.cycle_times[name]
|
|
rows=[self.processes[event] for _,event in R.SELECTED[name]] if name in R.SELECTED else []
|
|
inventory=R._cycle_files(name,rows,R.Evidence()) if rows else {}
|
|
value=dict(call=call,complete=True,start_monotonic=start,end_monotonic=end,wall_seconds=end-start,
|
|
output_inventory=inventory,output_bytes=sum(v['bytes'] for v in inventory.values()),
|
|
mutable_build_inventory_before=build,mutable_build_inventory_after=build,
|
|
mutable_build_bytes_before=sum(v['bytes'] for v in build.values()),
|
|
mutable_build_bytes_after=sum(v['bytes'] for v in build.values()),
|
|
source_registration_sha256=self.regsha,learning_pass_claimed=False)
|
|
R.F.save(self.exp/'receipts'/f'{name}.cycle.json',value)
|
|
R.F.save(self.exp/'audit/execution-attempt.json',dict(registration_sha256=self.regsha,monotonic_origin=1.))
|
|
R.F.save(self.exp/'audit/execution-complete.json',dict(complete=True,registration_sha256=self.regsha,
|
|
scientific_learning_pass_claimed=False,completed_calls=[c['call_id'] for c in self.plan['calls']],
|
|
processes={name:str(self.exp/'receipts'/f'{name}.process.json') for name in self.processes}))
|
|
|
|
def patch(self,key,value):
|
|
p=patch.object(R.F,key,value);p.start();self.addCleanup(p.stop)
|
|
|
|
def registration(self,sha):
|
|
R.require(sha==R.A.digest(self.exp/'protocol/preregistered.json'),'Registration differs')
|
|
return self.reg
|
|
|
|
def put(self,path,text):
|
|
Path(path).write_text(text,encoding='utf-8')
|
|
|
|
def rewrite(self,path,mutate):
|
|
value=R.A.read(path);mutate(value)
|
|
self.put(path,json.dumps(value,sort_keys=True,allow_nan=False))
|
|
|
|
def _epoch(self,name):
|
|
R.F.save(self.exp/'authorities'/f'{name}.epoch.json',dict(checkpoint_dir=str(self.exp/'runs'/name),
|
|
command=dict(steps=1200,compartment="y5B'2a",us_tick=900 if name==R.TRAIN else 1199,
|
|
d_command=1.,frozen=False,trace_backend='torch')))
|
|
|
|
def _process(self,name,kind,start,base=None):
|
|
ap=kind in ('brain','update');script=self.ap if ap else self.worker
|
|
output=self.exp/('updates' if kind=='update' else 'runs')/(base if kind=='update' else name);output.mkdir()
|
|
sources={role:dict(path=str(self.model/(role+'.py')),sha256=R.A.digest(self.model/(role+'.py')))
|
|
for role in (('runtime_light','runtime_continuation') if kind=='brain' else ('conditioning_epoch',))} if ap else {}
|
|
job=dict(job_id=name,kind=kind,registration_sha256=self.regsha,worker_sha256=R.A.digest(script),
|
|
receipt_file=str(self.exp/'receipts'/f'{name}.worker.json'),output_dir=str(output),
|
|
input_sha256={str(self.exp/'protocol/preregistered.json'):self.regsha},sources=sources,source_sha256={})
|
|
result={};environment=dict(gpu_uuid='FIXTURE',python_version='fixture',torch_version='fixture')
|
|
if kind=='brain':
|
|
job.update(case_dir=str(self.specs[name]),diagnostic_mode=self.calls[name]['diagnostic_mode'],overlay_file=str(self.zero))
|
|
summary=dict(steps=1200,completed_ticks=1200,explicit_steps_executed=1200,max_steps=1200,nodes=R.B.N,edges=R.B.E,
|
|
diagnostic_mode=job['diagnostic_mode'],entire_neural_trial_executed=True,within_trial_weight_updates=False,
|
|
precision='float64',apl_precision='float32')
|
|
R.F.save(output/'summary.json',summary);result['complete_fresh_trial']=True
|
|
elif kind=='sample':
|
|
for cue in ('A','B'):
|
|
R.F.save_npz(output/f'{cue}-requests.npz',dict(tick=np.array([5],np.int32),neuron_idx=np.array([20],np.int32)))
|
|
else:R.F.save(output/'fixture.json',dict(complete=True))
|
|
if base:
|
|
job.update(authority_file=str(self.exp/'authorities'/f'{base}.epoch.json'))
|
|
if kind=='validate_cpu':job['update_dir']=str(self.exp/'updates'/base)
|
|
jobpath=self.exp/'jobs'/f'{name}.json';R.F.save(jobpath,job);jobsha=R.A.digest(jobpath)
|
|
expected={**job['input_sha256'],str(jobpath):jobsha,str(script):job['worker_sha256']}
|
|
if ap:expected.update({v['path']:v['sha256'] for v in sources.values()})
|
|
else:expected[str(self.ap)]=R.F.W.AP_SHA
|
|
logs={}
|
|
for side in ('stdout','stderr'):
|
|
path=self.exp/'logs'/f'{name}.{side}.log';self.put(path,'');logs[side+'_sha256']=R.A.digest(path)
|
|
attempt=self.exp/'receipts'/f'{name}.worker.json.attempt.json';R.F.save(attempt,dict(fixture=True))
|
|
inventory=R.F.tree(output);hashes={k:v['sha256'] for k,v in inventory.items()}
|
|
worker=dict(schema='phase4ap-isolated-worker-receipt-v1' if ap else 'conditioning-isolated-execution-receipt-v1',
|
|
complete=True,job_id=name,kind=kind,registration_sha256=self.regsha,worker_sha256=job['worker_sha256'],
|
|
job_sha256=jobsha,attempt_sha256=R.A.digest(attempt),output_dir=str(output),output_files_sha256=hashes,
|
|
environment=environment,result=result,wall_seconds=2.)
|
|
worker['verified_input_sha256' if ap else 'input_sha256']=expected
|
|
workerpath=Path(job['receipt_file']);R.F.save(workerpath,worker)
|
|
command=['fixture-python','-B',str(script)]+([kind] if ap else [])+['--job',str(jobpath),'--job-sha256',jobsha]
|
|
process=dict(schema='conditioning-execution-process-v1',complete=True,name=name,kind=kind,
|
|
registration_sha256=self.regsha,exit_code=0,timed_out=False,job_sha256=jobsha,argv=command,
|
|
worker_receipt_sha256=R.A.digest(workerpath),spawn_monotonic=start,exit_monotonic=start+4.,
|
|
validation_end_monotonic=start+5.,child_wall_seconds=4.,end_to_end_seconds=5.,validation_wall_seconds=1.,
|
|
output_bytes=sum(v['bytes'] for v in inventory.values()),
|
|
output_inventory_sha256=hashlib.sha256(json.dumps(hashes,sort_keys=True).encode()).hexdigest(),**logs)
|
|
processpath=self.exp/'receipts'/f'{name}.process.json';R.F.save(processpath,process)
|
|
self.processes[name]=dict(process=process,job=job,worker=worker,output_inventory=inventory,
|
|
binding=dict(receipt_id=str(processpath),sha256=R.A.digest(processpath)))
|
|
|
|
def report(self,save=False):return R.report(self.regsha,save_report=save)
|
|
|
|
def test_actual_cycles_tail_complete_bytes_and_no_forecast_without_future_policy(self):
|
|
value=self.report()
|
|
self.assertEqual(value['status'],'INSUFFICIENT-MEASUREMENTS')
|
|
self.assertEqual(value['policy_status'],'FUTURE-POLICY-NOT-REGISTERED')
|
|
self.assertEqual(value['registration_sha256'],self.regsha)
|
|
self.assertEqual(value['inventory']['brain_calls'],33920)
|
|
self.assertEqual({r['cycle_id'] for r in value['measurements']},{R.TRAIN,R.PROBE})
|
|
for name in (R.TRAIN,R.PROBE):
|
|
account=value['accounting'][name]
|
|
self.assertEqual((account['recorded_cycle_seconds'],account['tail_upper_bound_seconds'],
|
|
account['conservative_complete_cycle_seconds']),(40.,60.,100.))
|
|
self.assertEqual(account['total_conservative_bytes'],account['created_bytes']+
|
|
account['cycle_receipt_bytes']+account['repeated_full_cache_bytes'])
|
|
self.assertGreater(account['cycle_receipt_bytes'],0)
|
|
self.assertIsNone(value['estimated_wall_seconds']);self.assertIsNone(value['estimated_output_bytes'])
|
|
self.assertIsNone(value['strict_projection']['estimated_wall_seconds'])
|
|
self.assertFalse(value['learning_pass_claimed']);self.assertFalse(value['execution_authorized'])
|
|
|
|
def test_saved_report_reproduces_exactly_and_is_exclusive(self):
|
|
first=self.report(True);second=self.report(False)
|
|
self.assertEqual(R._canonical(first),R._canonical(second))
|
|
self.assertEqual(R._canonical(first),R._canonical(R.A.read(self.exp/'audit/budget-report.json')))
|
|
with self.assertRaisesRegex(ValueError,'never overwrite'):self.report(True)
|
|
self.assertNotIn('generated_at',first);self.assertNotIn('free_bytes',first)
|
|
|
|
def test_worker_byte_tampering_and_wrong_registration_rejected(self):
|
|
path=self.exp/'receipts'/f'{R.TRAIN}.worker.json'
|
|
self.rewrite(path,lambda r:r.update(wall_seconds=.01))
|
|
with self.assertRaisesRegex(ValueError,'Evidence bytes changed'):self.report()
|
|
with self.assertRaises(ValueError):R.report('f'*64,save_report=False)
|
|
|
|
def test_unsuccessful_parent_process_cannot_be_reclassified_by_worker(self):
|
|
path=self.exp/'receipts'/f'{R.TRAIN}.process.json'
|
|
self.rewrite(path,lambda r:r.update(exit_code=1))
|
|
with self.assertRaisesRegex(ValueError,'Unsuccessful'):self.report()
|
|
|
|
def test_cycle_inventory_omission_not_mistaken_for_full_output_bytes(self):
|
|
path=self.exp/'receipts'/f'{R.TRAIN}.cycle.json'
|
|
def omit(row):
|
|
key=next(iter(row['output_inventory']));del row['output_inventory'][key]
|
|
row['output_bytes']=sum(v['bytes'] for v in row['output_inventory'].values())
|
|
self.rewrite(path,omit)
|
|
with self.assertRaisesRegex(ValueError,'complete fixed-branch scope'):self.report()
|
|
|
|
def test_missing_cycle_evidence_returns_insufficient_without_inventing_measurement(self):
|
|
(self.exp/'receipts'/f'{R.TRAIN}.cycle.json').unlink()
|
|
value=self.report()
|
|
self.assertEqual(value['status'],'INSUFFICIENT-MEASUREMENTS')
|
|
self.assertEqual(value['measurements'],[])
|
|
self.assertTrue(any('evidence missing' in item for item in value['missing']))
|
|
self.assertIsNone(value['estimated_output_bytes'])
|
|
|
|
def test_inconsistent_adjacent_clock_cannot_supply_tail_upper_bound(self):
|
|
path=self.exp/'receipts'/f'{R.PROBE}.cycle.json'
|
|
self.rewrite(path,lambda r:r.update(start_monotonic=self.cycle_times[R.TRAIN][1]-1))
|
|
with self.assertRaisesRegex(ValueError,'adjacent-cycle tail'):self.report()
|
|
|
|
def test_mutable_cache_final_bytes_must_match_observed_final_inventory(self):
|
|
self.put(self.exp/'extension-build/kernel.bin','changed-after-observation')
|
|
with self.assertRaisesRegex(ValueError,'mutable cache inventory changed'):self.report()
|
|
|
|
def test_reference_or_continuation_not_admitted_as_fresh_performance(self):
|
|
real=R._parent_process
|
|
def different(name,*args):
|
|
row=real(name,*args)
|
|
if name==R.PROBE:row=deepcopy(row);row['job']['diagnostic_mode']='reference'
|
|
return row
|
|
with patch.object(R,'_parent_process',side_effect=different):
|
|
with self.assertRaisesRegex(ValueError,'fresh light'):self.report()
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|