Digital Drosophila Archive
342 lines
24 KiB
Python
342 lines
24 KiB
Python
"""Serial normal conditioning. Original R/E/U executables stay unchanged.
|
|
|
|
Every fresh brain trial is followed by original E and independent original-U
|
|
recomputation in separate exec processes. Probes never run E or feed back.
|
|
Only fully committed steps may be skipped on explicit resume. An uncommitted
|
|
start, failed child or partial task always stops; no inferred OS exit or retry.
|
|
"""
|
|
from __future__ import annotations
|
|
import argparse
|
|
import copy
|
|
from datetime import datetime, timezone
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import numpy as np
|
|
from normal_io import *
|
|
from append_ledger import Ledger
|
|
|
|
def registration(expected):
|
|
require(digest(EXP/'protocol/preregistered.json')==expected,'Registration hash differs')
|
|
reg=read(EXP/'protocol/preregistered.json')
|
|
require(reg['schema']=='normal-conditioning-registration-v1' and reg['scientific_scope']=='normal-acquisition-only'
|
|
and reg['subsequent_assays_authorized'] is False,'Wrong authority')
|
|
verify(reg['source_sha256']);verify(reg['input_sha256']);verify(reg['prepared_sha256'])
|
|
return reg
|
|
|
|
def derive_spec(original,task,*,budget_root,output_cap):
|
|
require(original['kind']=='fullgraph' and original['dt_ms']==.1 and original['apl_delay_steps']==18 and
|
|
original['precision']=='float64' and original['apl_precision']=='float32' and original['record_history'] is False,
|
|
'Original fixed D18 spec required')
|
|
require(task['steps']==1200 and task['kind'] in ('train','probe'),'Wrong task')
|
|
allowed=('validation_cohort','odor_index','expected_external_events','training','cs_input_seed','cs_input_seed_bits',
|
|
'seed_scope','odor_id','odor_name','source_panel_row','source_panel_split','rate_sum_hz','target_count',
|
|
'expected_CS_requests','CS_window','outer_epoch_id','outer_arm','outer_protocol')
|
|
result={k:copy.deepcopy(v) for k,v in original.items() if k not in allowed}
|
|
result.update(name=task_key(task),steps=1200,duration_ms=120.,max_steps=1200,budget_root=str(budget_root),
|
|
output_budget_bytes=output_cap,mode=task['mode'],within_trial_weight_updates=False,
|
|
outer_protocol='normal-acquisition-v1',outer_task_id=task['task_id'],outer_arm=task['arm'])
|
|
# The frozen independent reset-control seed is retained, not derived from CS.
|
|
require(result['seed']==original['seed'],'Reset seed changed')
|
|
return result
|
|
|
|
def compose_requests(cs,dan,task,inputs):
|
|
require(task['steps']==1200,'Only normal120ms tasks')
|
|
if task['kind']=='probe':
|
|
recipe=task['recipe'];require(recipe['US_requested'] is False and recipe['D_command']==0 and
|
|
recipe['weight_updates_enabled'] is False and task['feedback_to_training_or_idle'] is False,'Probe command differs')
|
|
nodes=np.empty(0,np.int32)
|
|
else:
|
|
recipe=task['recipe'];us=recipe['us']
|
|
require(us['nominal_slot_tick']==900 and us['rng_seed'] is None and us['rng_draws']==0 and
|
|
((us['requested'] is True and us['tick']==900) or (us['requested'] is False and us['tick'] is None)),
|
|
'Scheduled artificial US differs')
|
|
require(recipe['D_command']==(1. if us['requested'] and task['arm']!='d_zero' else 0.) and
|
|
recipe['update_enabled'] is (task['arm']!='frozen'),'D/update command differs')
|
|
require(dan.dtype==np.int32 and dan.ndim==1 and len(dan) in (2,4) and np.all(dan[1:]>dan[:-1]),'Bound DAN IDs required')
|
|
nodes=dan.copy() if us['requested'] else np.empty(0,np.int32)
|
|
us=dict(tick=np.full(len(nodes),900,np.int32),neuron_idx=nodes)
|
|
composed=inputs.merge_requests(cs,us,steps=1200,conflict_policy='error')
|
|
return composed,us
|
|
|
|
class Workflow:
|
|
def __init__(self,sha):
|
|
self.sha=sha;self.reg=registration(sha);self.start=time.perf_counter()
|
|
require(not (EXP/'audit/stopped.json').exists() and not (EXP/'manifest.json').exists(), 'Stopped or sealed phase cannot run')
|
|
self.plan=read(EXP/'protocol/normal-plan.json');self.input_plan=read(EXP/'protocol/input-inventory.json')
|
|
self.tasks={x['task_id']:x for x in self.plan['tasks']}
|
|
self.h=load_source('_normal_AP_helpers',AP/'isolated_workers.py',AP_SHA)
|
|
self.inputs=load_source('_normal_request_composer',LAB/'conditioning_inputs.py',self.reg['static_input_sha256'][str(LAB/'conditioning_inputs.py')])
|
|
self.metric=load_source('_normal_CPU_state_validation',LAB/'metrics.py',self.reg['input_sha256'][str(LAB/'metrics.py')])
|
|
self.nodes=np.load(Path(read(EXP/'protocol/case-template.json')['graph_dir'])/'nodes.npy',allow_pickle=False)
|
|
self.populations=npz(self.reg['population_file'])
|
|
initial={Path(p).relative_to(EXP).as_posix():identity(p) for p in self.reg['prepared_sha256']}
|
|
initial['protocol/preregistered.json']=identity(EXP/'protocol/preregistered.json')
|
|
self.ledger=Ledger(EXP,sha,initial,self.reg['budget']['output_cap_bytes'],self.reg['budget']['minimum_free_reserve_bytes'])
|
|
status=self.ledger.inspect();require(not status['uncommitted_entry_ids'],'Uncommitted OS state unknown; no automatic retry')
|
|
self.completed=set(status['entry_ids']);self.expected=[]
|
|
|
|
def basepins(self):
|
|
return {**self.reg['runtime_input_sha256'],str(EXP/'protocol/preregistered.json'):self.sha}
|
|
|
|
def immutable_pin(self,path):
|
|
path=unlinked(path);rel=path.relative_to(EXP).as_posix()
|
|
require(rel in self.ledger._tracked and identity(path)==self.ledger._tracked[rel],
|
|
'Consumed immutable file differs from committed bytes: '+rel)
|
|
return dict(path=str(path),sha256=self.ledger._tracked[rel]['sha256'])
|
|
|
|
def budget(self):
|
|
require(time.perf_counter()-self.start < self.reg['budget']['workflow_timeout_seconds'],'Invocation wall cap exceeded')
|
|
require(datetime.now(timezone.utc)<datetime.fromisoformat(self.reg['budget']['not_after_utc']),
|
|
'Registered absolute deadline exceeded; resume cannot reset wall allowance')
|
|
require(shutil.disk_usage(EXP).free>=self.reg['budget']['minimum_free_reserve_bytes'],'Free space reserve exceeded')
|
|
# Running totals account only committed payload, ledger and shared build.
|
|
# Each child also gets a fresh per-child output allowance; full tree at final.
|
|
accounting=self.ledger._budget()
|
|
require(accounting['total_accounted_bytes']+self.reg['budget']['next_child_headroom_bytes'] <=
|
|
self.reg['budget']['output_cap_bytes'],'No registered headroom for another complete child')
|
|
|
|
def launch(self,name,job,argv,outputs,*,pre_outputs=(),post=None,ap=False):
|
|
self.expected.append(name)
|
|
require(name not in self.completed,'Internal attempt to rerun committed job')
|
|
self.budget()
|
|
unlinked(EXP/'receipts'/name).mkdir()
|
|
jobfile=EXP/f'jobs/{name}.json';save(jobfile,job);jobsha=digest(jobfile)
|
|
argv=[*argv]
|
|
if ap:argv+=['--job',str(jobfile),'--job-sha256',jobsha]
|
|
start=self.ledger.begin(name,jobfile,dict(kind=job['kind']))
|
|
stdout=EXP/f'logs/{name}.stdout.log';stderr=EXP/f'logs/{name}.stderr.log'
|
|
process=EXP/f'receipts/{name}/process.json'
|
|
started=time.perf_counter();rc=None;timed=False;launch_error=None
|
|
env=dict(os.environ);env['PYTHONDONTWRITEBYTECODE']='1'
|
|
if job['kind'] not in ('brain','sample'):env['CUDA_VISIBLE_DEVICES']=''
|
|
with unlinked(stdout).open('xb') as out,unlinked(stderr).open('xb') as err:
|
|
try:rc=subprocess.run(argv,cwd=MODEL,env=env,stdout=out,stderr=err,
|
|
timeout=self.reg['budget']['child_timeout_seconds']).returncode
|
|
except subprocess.TimeoutExpired:timed=True
|
|
except OSError as error:launch_error=type(error).__name__+': '+str(error)
|
|
base=dict(entry_id=name,registration_sha256=self.sha,job_sha256=jobsha,argv=argv,
|
|
exit_code=rc,timed_out=timed,exit_observed_by_parent=rc is not None or timed,launch_error=launch_error,
|
|
child_wall_seconds=time.perf_counter()-started,observed_utc=datetime.now(timezone.utc).isoformat(),
|
|
stdout_sha256=digest(stdout),stderr_sha256=digest(stderr))
|
|
try:
|
|
require(type(rc)is int and rc==0 and not timed,'Child failed; retain outputs and stop')
|
|
verify(job['input_sha256'])
|
|
if ap:
|
|
receipt=read(job['receipt_file'])
|
|
require(receipt['schema']=='phase4ap-isolated-worker-receipt-v1' and receipt['complete'] is True and
|
|
receipt['job_sha256']==jobsha and receipt['registration_sha256']==self.sha and
|
|
receipt['kind']==job['kind'] and receipt['job_id']==name,'Actual child receipt differs')
|
|
expected={**job['input_sha256'],str(jobfile):jobsha,str(AP/'isolated_workers.py'):AP_SHA,
|
|
**{v['path']:v['sha256'] for v in job['sources'].values()}}
|
|
require(receipt['verified_input_sha256']==expected,'Worker input closure differs')
|
|
actual={str(Path(p).relative_to(job['output_dir'])):v['sha256'] for p,v in inventory(Path(job['output_dir'])).items()}
|
|
require(actual==receipt['output_files_sha256'],'Worker output inventory differs')
|
|
require(digest(job['receipt_file']+'.attempt.json')==receipt['attempt_sha256'],'Attempt changed')
|
|
if post is not None:post()
|
|
base.update(complete=True,end_to_end_seconds=time.perf_counter()-started)
|
|
save(process,base)
|
|
files=[*pre_outputs,jobfile,stdout,stderr,process]
|
|
for p in outputs:
|
|
p=Path(p);files.extend(Path(x) for x in inventory(p)) if p.is_dir() else files.append(p)
|
|
outgoing={str(p):identity(p) for p in files}
|
|
incoming={p:identity(p) for p in job['input_sha256']}
|
|
incoming[str(jobfile)]=identity(jobfile)
|
|
self.ledger.append(name,incoming,outgoing,dict(job_file=str(jobfile),process_receipt_file=str(process),
|
|
observed_returncode=rc,scope='actual serial child exit and immutable artifact verification'))
|
|
self.completed.add(name)
|
|
print(json.dumps(dict(complete=True,entry_id=name,wall_seconds=base['end_to_end_seconds']),sort_keys=True),flush=True)
|
|
except BaseException as error:
|
|
if not process.exists():save(process,dict(base,complete=False,error=str(error)))
|
|
raise
|
|
|
|
def input_generation(self):
|
|
name='input-generation'
|
|
if name in self.completed:self.expected.append(name);return
|
|
pins={**self.reg['source_sha256'],**self.reg['static_input_sha256'],
|
|
**{p:s for p,s in self.reg['prepared_sha256'].items() if '/inputs/static/' in p or p.endswith('/input-inventory.json')},
|
|
str(EXP/'protocol/preregistered.json'):self.sha}
|
|
job=dict(kind='sample',registration_sha256=self.sha,input_sha256=pins)
|
|
def validate_sample():
|
|
report=read(EXP/'receipts/input-generation.json')
|
|
require(report['complete'] is True and report['registration_sha256']==self.sha and
|
|
set(report['inputs'])==set(self.input_plan['descriptors']) and report['descriptors']==4080,
|
|
'Incomplete original sampling inventory')
|
|
for key,d in self.input_plan['descriptors'].items():
|
|
f=EXP/f'inputs/generated/{key}.npz';a=EXP/f'inputs/generated/{key}.json'
|
|
audit=read(a)
|
|
require(audit['descriptor']==d and audit['requests_sha256']==digest(f) and
|
|
report['inputs'][key]==dict(requests=pin(f),audit=pin(a)), 'Saved sampler binding differs')
|
|
self.inputs.validate_requests(npz(f),1200)
|
|
self.launch(name,job,[sys.executable,'-B',str(MODEL/'normal_inputs.py'),'--registration-sha256',self.sha],
|
|
[EXP/'inputs/generated',EXP/'receipts/input-generation.json'],post=validate_sample)
|
|
|
|
def overlay(self,task):
|
|
self.overlay_dependency_pins={}
|
|
dependency=task['delta_dependency']
|
|
if dependency==dict(kind='zero',source=None):return copy.deepcopy(self.reg['initial_overlay'])
|
|
require(dependency['kind']=='epoch_delta_after' and dependency['source'].endswith('/delta-after'),'Unknown delta dependency')
|
|
previous=self.tasks[dependency['source'][:-len('/delta-after')]]
|
|
key=task_key(previous);require(key+'--verify' in self.completed,'Previous real E+CPU validation missing')
|
|
result_path=EXP/f'results/{key}.json'
|
|
def committed(path):
|
|
checked=self.immutable_pin(path)
|
|
self.overlay_dependency_pins[checked['path']]=checked['sha256']
|
|
committed(result_path);record=read(result_path)
|
|
require(record['complete'] is True and record['task']==previous and previous['kind']=='train' and
|
|
all(previous[k]==task[k] for k in ('block_id','arm')),'Delta cannot cross unrelated branch')
|
|
validation=Path(record['cpu_validation']['path']);committed(validation)
|
|
require(digest(validation)==record['cpu_validation']['sha256'],'CPU receipt changed')
|
|
valid=read(validation)
|
|
committed(Path(record['cpu_process_receipt_path']))
|
|
process=read(record['cpu_process_receipt_path'])
|
|
require(process['complete'] is True and type(process['exit_code']) is int and process['exit_code']==0 and
|
|
process['entry_id']==key+'--verify' and process['registration_sha256']==self.sha,
|
|
'Previous CPU exit missing')
|
|
for role,filename,field in (('overlay_file','delta-after.npz','after_overlay_sha256'),
|
|
('overlay_manifest','overlay-manifest.json','after_manifest_sha256')):
|
|
path=Path(record['after_overlay'][role]);committed(path)
|
|
require(path==Path(record['update_dir'])/filename and digest(path)==valid[field],
|
|
'Next installed delta is not the actual independently verified output')
|
|
return copy.deepcopy(record['after_overlay'])
|
|
|
|
def prepare_case(self,task):
|
|
key=task_key(task);case=unlinked(EXP/'cases'/key);case.mkdir()
|
|
template_file=EXP/'protocol/case-template.json'
|
|
checked=self.immutable_pin(template_file)
|
|
self.case_dependency_pins={checked['path']:checked['sha256'],
|
|
self.reg['population_file']:self.reg['input_sha256'][self.reg['population_file']]}
|
|
template=read(template_file)
|
|
spec=derive_spec(template,task,budget_root=EXP,output_cap=self.reg['budget']['output_cap_bytes'])
|
|
csid=self.input_plan['task_inputs'][task['task_id']]
|
|
if csid is None:cs=dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32));cs_source=None
|
|
else:
|
|
cs_source=EXP/f'inputs/generated/{csid}.npz'
|
|
audit_file=EXP/'inputs/generated'/(csid+'.json')
|
|
for path in (cs_source,audit_file):
|
|
checked=self.immutable_pin(path);self.case_dependency_pins[checked['path']]=checked['sha256']
|
|
generated=read(audit_file)
|
|
require(generated['descriptor']==self.input_plan['descriptors'][csid] and
|
|
generated['requests_sha256']==digest(cs_source),'Saved shared CS changed')
|
|
cs=npz(cs_source)
|
|
combined,us=compose_requests(cs,self.populations[task['dan_type']],task,self.inputs)
|
|
save(case/'spec.json',spec);save_npz(case/'requests.npz',combined['requests'])
|
|
save_npz(case/'cs_requests.npz',cs);save_npz(case/'us_requests.npz',us)
|
|
save_npz(case/'source-mask.npz',dict(source_mask=combined['source_mask']))
|
|
save(case/'input-binding.json',dict(task_id=task['task_id'],CS_source=None if cs_source is None else pin(cs_source),
|
|
CS_descriptor_id=csid,composition=combined['audit'],DAN_type=task['dan_type'],
|
|
DAN_membership_sha256=hashlib.sha256(self.populations[task['dan_type']].tobytes()).hexdigest(),
|
|
population_file=pin(self.reg['population_file']),artificial_US_not_dopamine_concentration=True))
|
|
return case
|
|
|
|
def ap_job(self,name,kind,output,*,case=None,overlay=None,authority=None):
|
|
pins=self.basepins()
|
|
job=dict(schema='phase4ap-isolated-worker-job-v1',job_id=name,kind=kind,registration_sha256=self.sha,
|
|
worker_sha256=AP_SHA,output_dir=str(output),receipt_file=str(EXP/f'receipts/{name}/worker.json'))
|
|
if kind=='brain':
|
|
pins.update(self.overlay_dependency_pins)
|
|
pins.update(self.case_dependency_pins)
|
|
for p in (case/'spec.json',case/'requests.npz',overlay['overlay_file'],overlay['overlay_manifest']):pins[str(p)]=digest(p)
|
|
job.update(case_dir=str(case),build_root=str(EXP/'extension-build'),**overlay,diagnostic_mode='light',
|
|
admission=dict(gpu_uuid=GPU,comfy_queue_urls=list(self.h.COMFY_URLS)),
|
|
sources={n:dict(path=str(LAB/(n+'.py')),sha256=self.h.SOURCE_PINS[n]) for n in ('runtime_light','runtime_continuation')})
|
|
else:
|
|
auth=read(authority);pins[str(authority)]=digest(authority)
|
|
pins.update({v['path']:v['sha256'] for v in auth['files'].values()})
|
|
pins.update(auth['frozen_inputs']);pins[str(Path(auth['checkpoint_dir'])/'checkpoint-manifest.json')]=auth['checkpoint_manifest_sha256']
|
|
job.update(authority_file=str(authority),authority_sha256=digest(authority),
|
|
sources={'conditioning_epoch':dict(path=str(LAB/'conditioning_epoch.py'),sha256=self.h.SOURCE_PINS['conditioning_epoch'])})
|
|
job['input_sha256']=pins;return job
|
|
|
|
def brain_verify(self,task,case,output):
|
|
summary=read(output/'summary.json');state=npz(output/'state.npz')
|
|
checked=self.metric.validate_state(state,nodes=self.nodes,steps=1200)
|
|
require(summary['all_finite'] is True and summary['steps']==1200 and summary['diagnostic_mode']=='light','Nonfinite/incomplete brain')
|
|
require(int(state['counts'].sum())==summary['total_spikes'],'Spike event/count mismatch')
|
|
manifest=read(output/'checkpoint-manifest.json')
|
|
for rel,sha in manifest['files_sha256'].items():require(digest(output/rel)==sha,'Checkpoint payload changed')
|
|
return dict(complete=True,state_raw_sha256=checked['hashes'],finite=True,
|
|
checkpoint_manifest_sha256=digest(output/'checkpoint-manifest.json'),task_id=task['task_id'])
|
|
|
|
def task(self,task):
|
|
key=task_key(task);brain=key+'--brain';up=key+'--update';cpu=key+'--verify'
|
|
planned=[brain,up,cpu] if task['kind']=='train' else [brain]
|
|
if all(x in self.completed for x in planned):self.expected.extend(planned);return
|
|
require(not any(x in self.completed for x in planned),'Partially completed task: preserve and inspect; no implicit replay')
|
|
overlay=self.overlay(task);case=self.prepare_case(task);output=EXP/'runs'/key
|
|
record=dict(schema='normal-task-result-v1',complete=True,registration_sha256=self.sha,task=task,
|
|
checkpoint_dir=str(output),overlay_before=overlay,brain_process_receipt_path=str(EXP/f'receipts/{brain}/process.json'))
|
|
def after_brain():
|
|
record['brain_audit']=self.brain_verify(task,case,output)
|
|
record['checkpoint_manifest_sha256']=digest(output/'checkpoint-manifest.json')
|
|
# Parent process receipt is produced later and is bound by the ledger,
|
|
# avoiding a self-reference in a result written before process exit audit.
|
|
if task['kind']=='probe':save(EXP/f'results/{key}.json',record)
|
|
job=self.ap_job(brain,'brain',output,case=case,overlay=overlay)
|
|
outs=[output,Path(job['receipt_file']),Path(job['receipt_file']+'.attempt.json')]
|
|
if task['kind']=='probe':outs.append(EXP/f'results/{key}.json')
|
|
self.launch(brain,job,[sys.executable,'-B',str(AP/'isolated_workers.py'),'brain'],outs,
|
|
pre_outputs=[Path(p) for p in inventory(case)],post=after_brain,ap=True)
|
|
if task['kind']=='probe':return
|
|
template_pin=self.immutable_pin(EXP/'protocol/epoch-template.json')
|
|
template=read(template_pin['path']);auth=copy.deepcopy(template)
|
|
r=task['recipe'];auth.update(epoch_id=task['task_id'],diagnostic_mode='light',checkpoint_dir=str(output),
|
|
checkpoint_manifest_sha256=digest(output/'checkpoint-manifest.json'),
|
|
command=dict(steps=1200,compartment=task['compartment_verbatim'],us_tick=r['us']['tick'],d_command=r['D_command'],
|
|
frozen=not r['update_enabled'],trace_backend='torch'))
|
|
for role,path in dict(spec=case/'spec.json',requests=case/'requests.npz',state=output/'state.npz',
|
|
overlay_before=overlay['overlay_file'],overlay_manifest_before=overlay['overlay_manifest']).items():auth['files'][role]=pin(path)
|
|
authority=EXP/f'jobs/{key}.authority.json';save(authority,auth);update=EXP/'updates'/key
|
|
job=self.ap_job(up,'update',update,authority=authority)
|
|
job['input_sha256'][template_pin['path']]=template_pin['sha256']
|
|
for path in (EXP/f'receipts/{brain}/process.json',EXP/f'receipts/{brain}/worker.json'):
|
|
checked=self.immutable_pin(path);job['input_sha256'][checked['path']]=checked['sha256']
|
|
self.launch(up,job,[sys.executable,'-B',str(AP/'isolated_workers.py'),'update'],
|
|
[update,Path(job['receipt_file']),Path(job['receipt_file']+'.attempt.json')],pre_outputs=[authority],ap=True)
|
|
out=unlinked(EXP/'validation'/key);out.mkdir();validation=out/'validation.json'
|
|
script=AQ/'validate_update_cpu.py';verify({str(script):self.reg['input_sha256'][str(script)]})
|
|
pins={**job['input_sha256'],str(script):self.reg['input_sha256'][str(script)]}
|
|
for name in ('conditioning_epoch.py','conditioning_preflight_compare.py','engineering_compare.py'):
|
|
path=str(LAB/name);pins[path]=self.reg['input_sha256'][path]
|
|
pins.update({p:v['sha256'] for p,v in inventory(update).items()})
|
|
job=dict(kind='cpu_verify',registration_sha256=self.sha,input_sha256=pins,authority_file=str(authority),
|
|
authority_sha256=digest(authority),update_dir=str(update),chain_sha256=digest(update/'chain-receipt.json'),
|
|
receipt_file=str(validation),validator_sha256=self.reg['input_sha256'][str(script)])
|
|
def after_cpu():
|
|
valid=read(validation)
|
|
require(valid['complete'] is True and valid['frozen_U_verify_receipt_recomputed'] is True and
|
|
valid['actual_sealed_events_used'] is True and valid['after_overlay_sha256']==digest(update/'delta-after.npz') and
|
|
valid['epoch_authority_sha256']==digest(authority),'Independent update verification differs')
|
|
if task['arm'] in ('d_zero','frozen','cs_only'):
|
|
require(not npz(update/'delta-after.npz')['plastic_delta_mv'].view(np.uint32).any(),'Zero-update control became nonzero')
|
|
record.update(epoch_authority=pin(authority),update_dir=str(update),cpu_validation=pin(validation),
|
|
after_overlay=dict(overlay_file=str(update/'delta-after.npz'),overlay_manifest=str(update/'overlay-manifest.json')),
|
|
update_process_receipt_path=str(EXP/f'receipts/{up}/process.json'),
|
|
cpu_process_receipt_path=str(EXP/f'receipts/{cpu}/process.json'))
|
|
save(EXP/f'results/{key}.json',record)
|
|
self.launch(cpu,job,[sys.executable,'-B',str(script),'--authority',str(authority),'--authority-sha256',digest(authority),
|
|
'--update-dir',str(update),'--chain-sha256',digest(update/'chain-receipt.json'),'--receipt',str(validation)],
|
|
[out,EXP/f'results/{key}.json'],post=after_cpu)
|
|
|
|
def run(self):
|
|
try:
|
|
self.input_generation()
|
|
for task in self.plan['tasks']:self.task(task)
|
|
report=self.ledger.final_verify(self.expected)
|
|
require(report['pass_all'],'Final ledger coverage/integrity failed')
|
|
save(EXP/'audit/execution-complete.json',dict(complete=True,registration_sha256=self.sha,
|
|
inventory=self.plan['inventory'],ledger=report,learning_success_claimed=False,
|
|
next='independent scientific analysis and frozen integrity, no subsequent assay'))
|
|
except BaseException as error:
|
|
if not (EXP/'audit/stopped.json').exists():save(EXP/'audit/stopped.json',dict(complete=False,
|
|
registration_sha256=self.sha,error_type=type(error).__name__,error=str(error),
|
|
traceback=traceback.format_exc(),partial_files_preserved=True,automatic_retry=False))
|
|
raise
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('--registration-sha256',required=True)
|
|
Workflow(p.parse_args().registration_sha256).run()
|