"""Serial normal conditioning. Original R/E/U executables stay unchanged. Every fresh brain trial is followed by original E and independent original-U recomputation in separate exec processes. Probes never run E or feed back. Only fully committed steps may be skipped on explicit resume. An uncommitted start, failed child or partial task always stops; no inferred OS exit or retry. """ from __future__ import annotations import argparse import copy from datetime import datetime, timezone import os import shutil import subprocess import sys import time import traceback import numpy as np from normal_io import * from append_ledger import Ledger def registration(expected): require(digest(EXP/'protocol/preregistered.json')==expected,'Registration hash differs') reg=read(EXP/'protocol/preregistered.json') require(reg['schema']=='normal-conditioning-registration-v1' and reg['scientific_scope']=='normal-acquisition-only' and reg['subsequent_assays_authorized'] is False,'Wrong authority') verify(reg['source_sha256']);verify(reg['input_sha256']);verify(reg['prepared_sha256']) return reg def derive_spec(original,task,*,budget_root,output_cap): require(original['kind']=='fullgraph' and original['dt_ms']==.1 and original['apl_delay_steps']==18 and original['precision']=='float64' and original['apl_precision']=='float32' and original['record_history'] is False, 'Original fixed D18 spec required') require(task['steps']==1200 and task['kind'] in ('train','probe'),'Wrong task') allowed=('validation_cohort','odor_index','expected_external_events','training','cs_input_seed','cs_input_seed_bits', 'seed_scope','odor_id','odor_name','source_panel_row','source_panel_split','rate_sum_hz','target_count', 'expected_CS_requests','CS_window','outer_epoch_id','outer_arm','outer_protocol') result={k:copy.deepcopy(v) for k,v in original.items() if k not in allowed} result.update(name=task_key(task),steps=1200,duration_ms=120.,max_steps=1200,budget_root=str(budget_root), output_budget_bytes=output_cap,mode=task['mode'],within_trial_weight_updates=False, outer_protocol='normal-acquisition-v1',outer_task_id=task['task_id'],outer_arm=task['arm']) # The frozen independent reset-control seed is retained, not derived from CS. require(result['seed']==original['seed'],'Reset seed changed') return result def compose_requests(cs,dan,task,inputs): require(task['steps']==1200,'Only normal120ms tasks') if task['kind']=='probe': recipe=task['recipe'];require(recipe['US_requested'] is False and recipe['D_command']==0 and recipe['weight_updates_enabled'] is False and task['feedback_to_training_or_idle'] is False,'Probe command differs') nodes=np.empty(0,np.int32) else: recipe=task['recipe'];us=recipe['us'] require(us['nominal_slot_tick']==900 and us['rng_seed'] is None and us['rng_draws']==0 and ((us['requested'] is True and us['tick']==900) or (us['requested'] is False and us['tick'] is None)), 'Scheduled artificial US differs') require(recipe['D_command']==(1. if us['requested'] and task['arm']!='d_zero' else 0.) and recipe['update_enabled'] is (task['arm']!='frozen'),'D/update command differs') require(dan.dtype==np.int32 and dan.ndim==1 and len(dan) in (2,4) and np.all(dan[1:]>dan[:-1]),'Bound DAN IDs required') nodes=dan.copy() if us['requested'] else np.empty(0,np.int32) us=dict(tick=np.full(len(nodes),900,np.int32),neuron_idx=nodes) composed=inputs.merge_requests(cs,us,steps=1200,conflict_policy='error') return composed,us class Workflow: def __init__(self,sha): self.sha=sha;self.reg=registration(sha);self.start=time.perf_counter() require(not (EXP/'audit/stopped.json').exists() and not (EXP/'manifest.json').exists(), 'Stopped or sealed phase cannot run') self.plan=read(EXP/'protocol/normal-plan.json');self.input_plan=read(EXP/'protocol/input-inventory.json') self.tasks={x['task_id']:x for x in self.plan['tasks']} self.h=load_source('_normal_AP_helpers',AP/'isolated_workers.py',AP_SHA) self.inputs=load_source('_normal_request_composer',LAB/'conditioning_inputs.py',self.reg['static_input_sha256'][str(LAB/'conditioning_inputs.py')]) self.metric=load_source('_normal_CPU_state_validation',LAB/'metrics.py',self.reg['input_sha256'][str(LAB/'metrics.py')]) self.nodes=np.load(Path(read(EXP/'protocol/case-template.json')['graph_dir'])/'nodes.npy',allow_pickle=False) self.populations=npz(self.reg['population_file']) initial={Path(p).relative_to(EXP).as_posix():identity(p) for p in self.reg['prepared_sha256']} initial['protocol/preregistered.json']=identity(EXP/'protocol/preregistered.json') self.ledger=Ledger(EXP,sha,initial,self.reg['budget']['output_cap_bytes'],self.reg['budget']['minimum_free_reserve_bytes']) status=self.ledger.inspect();require(not status['uncommitted_entry_ids'],'Uncommitted OS state unknown; no automatic retry') self.completed=set(status['entry_ids']);self.expected=[] def basepins(self): return {**self.reg['runtime_input_sha256'],str(EXP/'protocol/preregistered.json'):self.sha} def immutable_pin(self,path): path=unlinked(path);rel=path.relative_to(EXP).as_posix() require(rel in self.ledger._tracked and identity(path)==self.ledger._tracked[rel], 'Consumed immutable file differs from committed bytes: '+rel) return dict(path=str(path),sha256=self.ledger._tracked[rel]['sha256']) def budget(self): require(time.perf_counter()-self.start < self.reg['budget']['workflow_timeout_seconds'],'Invocation wall cap exceeded') require(datetime.now(timezone.utc)=self.reg['budget']['minimum_free_reserve_bytes'],'Free space reserve exceeded') # Running totals account only committed payload, ledger and shared build. # Each child also gets a fresh per-child output allowance; full tree at final. accounting=self.ledger._budget() require(accounting['total_accounted_bytes']+self.reg['budget']['next_child_headroom_bytes'] <= self.reg['budget']['output_cap_bytes'],'No registered headroom for another complete child') def launch(self,name,job,argv,outputs,*,pre_outputs=(),post=None,ap=False): self.expected.append(name) require(name not in self.completed,'Internal attempt to rerun committed job') self.budget() unlinked(EXP/'receipts'/name).mkdir() jobfile=EXP/f'jobs/{name}.json';save(jobfile,job);jobsha=digest(jobfile) argv=[*argv] if ap:argv+=['--job',str(jobfile),'--job-sha256',jobsha] start=self.ledger.begin(name,jobfile,dict(kind=job['kind'])) stdout=EXP/f'logs/{name}.stdout.log';stderr=EXP/f'logs/{name}.stderr.log' process=EXP/f'receipts/{name}/process.json' started=time.perf_counter();rc=None;timed=False;launch_error=None env=dict(os.environ);env['PYTHONDONTWRITEBYTECODE']='1' if job['kind'] not in ('brain','sample'):env['CUDA_VISIBLE_DEVICES']='' with unlinked(stdout).open('xb') as out,unlinked(stderr).open('xb') as err: try:rc=subprocess.run(argv,cwd=MODEL,env=env,stdout=out,stderr=err, timeout=self.reg['budget']['child_timeout_seconds']).returncode except subprocess.TimeoutExpired:timed=True except OSError as error:launch_error=type(error).__name__+': '+str(error) base=dict(entry_id=name,registration_sha256=self.sha,job_sha256=jobsha,argv=argv, exit_code=rc,timed_out=timed,exit_observed_by_parent=rc is not None or timed,launch_error=launch_error, child_wall_seconds=time.perf_counter()-started,observed_utc=datetime.now(timezone.utc).isoformat(), stdout_sha256=digest(stdout),stderr_sha256=digest(stderr)) try: require(type(rc)is int and rc==0 and not timed,'Child failed; retain outputs and stop') verify(job['input_sha256']) if ap: receipt=read(job['receipt_file']) require(receipt['schema']=='phase4ap-isolated-worker-receipt-v1' and receipt['complete'] is True and receipt['job_sha256']==jobsha and receipt['registration_sha256']==self.sha and receipt['kind']==job['kind'] and receipt['job_id']==name,'Actual child receipt differs') expected={**job['input_sha256'],str(jobfile):jobsha,str(AP/'isolated_workers.py'):AP_SHA, **{v['path']:v['sha256'] for v in job['sources'].values()}} require(receipt['verified_input_sha256']==expected,'Worker input closure differs') actual={str(Path(p).relative_to(job['output_dir'])):v['sha256'] for p,v in inventory(Path(job['output_dir'])).items()} require(actual==receipt['output_files_sha256'],'Worker output inventory differs') require(digest(job['receipt_file']+'.attempt.json')==receipt['attempt_sha256'],'Attempt changed') if post is not None:post() base.update(complete=True,end_to_end_seconds=time.perf_counter()-started) save(process,base) files=[*pre_outputs,jobfile,stdout,stderr,process] for p in outputs: p=Path(p);files.extend(Path(x) for x in inventory(p)) if p.is_dir() else files.append(p) outgoing={str(p):identity(p) for p in files} incoming={p:identity(p) for p in job['input_sha256']} incoming[str(jobfile)]=identity(jobfile) self.ledger.append(name,incoming,outgoing,dict(job_file=str(jobfile),process_receipt_file=str(process), observed_returncode=rc,scope='actual serial child exit and immutable artifact verification')) self.completed.add(name) print(json.dumps(dict(complete=True,entry_id=name,wall_seconds=base['end_to_end_seconds']),sort_keys=True),flush=True) except BaseException as error: if not process.exists():save(process,dict(base,complete=False,error=str(error))) raise def input_generation(self): name='input-generation' if name in self.completed:self.expected.append(name);return pins={**self.reg['source_sha256'],**self.reg['static_input_sha256'], **{p:s for p,s in self.reg['prepared_sha256'].items() if '/inputs/static/' in p or p.endswith('/input-inventory.json')}, str(EXP/'protocol/preregistered.json'):self.sha} job=dict(kind='sample',registration_sha256=self.sha,input_sha256=pins) def validate_sample(): report=read(EXP/'receipts/input-generation.json') require(report['complete'] is True and report['registration_sha256']==self.sha and set(report['inputs'])==set(self.input_plan['descriptors']) and report['descriptors']==4080, 'Incomplete original sampling inventory') for key,d in self.input_plan['descriptors'].items(): f=EXP/f'inputs/generated/{key}.npz';a=EXP/f'inputs/generated/{key}.json' audit=read(a) require(audit['descriptor']==d and audit['requests_sha256']==digest(f) and report['inputs'][key]==dict(requests=pin(f),audit=pin(a)), 'Saved sampler binding differs') self.inputs.validate_requests(npz(f),1200) self.launch(name,job,[sys.executable,'-B',str(MODEL/'normal_inputs.py'),'--registration-sha256',self.sha], [EXP/'inputs/generated',EXP/'receipts/input-generation.json'],post=validate_sample) def overlay(self,task): self.overlay_dependency_pins={} dependency=task['delta_dependency'] if dependency==dict(kind='zero',source=None):return copy.deepcopy(self.reg['initial_overlay']) require(dependency['kind']=='epoch_delta_after' and dependency['source'].endswith('/delta-after'),'Unknown delta dependency') previous=self.tasks[dependency['source'][:-len('/delta-after')]] key=task_key(previous);require(key+'--verify' in self.completed,'Previous real E+CPU validation missing') result_path=EXP/f'results/{key}.json' def committed(path): checked=self.immutable_pin(path) self.overlay_dependency_pins[checked['path']]=checked['sha256'] committed(result_path);record=read(result_path) require(record['complete'] is True and record['task']==previous and previous['kind']=='train' and all(previous[k]==task[k] for k in ('block_id','arm')),'Delta cannot cross unrelated branch') validation=Path(record['cpu_validation']['path']);committed(validation) require(digest(validation)==record['cpu_validation']['sha256'],'CPU receipt changed') valid=read(validation) committed(Path(record['cpu_process_receipt_path'])) process=read(record['cpu_process_receipt_path']) require(process['complete'] is True and type(process['exit_code']) is int and process['exit_code']==0 and process['entry_id']==key+'--verify' and process['registration_sha256']==self.sha, 'Previous CPU exit missing') for role,filename,field in (('overlay_file','delta-after.npz','after_overlay_sha256'), ('overlay_manifest','overlay-manifest.json','after_manifest_sha256')): path=Path(record['after_overlay'][role]);committed(path) require(path==Path(record['update_dir'])/filename and digest(path)==valid[field], 'Next installed delta is not the actual independently verified output') return copy.deepcopy(record['after_overlay']) def prepare_case(self,task): key=task_key(task);case=unlinked(EXP/'cases'/key);case.mkdir() template_file=EXP/'protocol/case-template.json' checked=self.immutable_pin(template_file) self.case_dependency_pins={checked['path']:checked['sha256'], self.reg['population_file']:self.reg['input_sha256'][self.reg['population_file']]} template=read(template_file) spec=derive_spec(template,task,budget_root=EXP,output_cap=self.reg['budget']['output_cap_bytes']) csid=self.input_plan['task_inputs'][task['task_id']] if csid is None:cs=dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32));cs_source=None else: cs_source=EXP/f'inputs/generated/{csid}.npz' audit_file=EXP/'inputs/generated'/(csid+'.json') for path in (cs_source,audit_file): checked=self.immutable_pin(path);self.case_dependency_pins[checked['path']]=checked['sha256'] generated=read(audit_file) require(generated['descriptor']==self.input_plan['descriptors'][csid] and generated['requests_sha256']==digest(cs_source),'Saved shared CS changed') cs=npz(cs_source) combined,us=compose_requests(cs,self.populations[task['dan_type']],task,self.inputs) save(case/'spec.json',spec);save_npz(case/'requests.npz',combined['requests']) save_npz(case/'cs_requests.npz',cs);save_npz(case/'us_requests.npz',us) save_npz(case/'source-mask.npz',dict(source_mask=combined['source_mask'])) save(case/'input-binding.json',dict(task_id=task['task_id'],CS_source=None if cs_source is None else pin(cs_source), CS_descriptor_id=csid,composition=combined['audit'],DAN_type=task['dan_type'], DAN_membership_sha256=hashlib.sha256(self.populations[task['dan_type']].tobytes()).hexdigest(), population_file=pin(self.reg['population_file']),artificial_US_not_dopamine_concentration=True)) return case def ap_job(self,name,kind,output,*,case=None,overlay=None,authority=None): pins=self.basepins() job=dict(schema='phase4ap-isolated-worker-job-v1',job_id=name,kind=kind,registration_sha256=self.sha, worker_sha256=AP_SHA,output_dir=str(output),receipt_file=str(EXP/f'receipts/{name}/worker.json')) if kind=='brain': pins.update(self.overlay_dependency_pins) pins.update(self.case_dependency_pins) for p in (case/'spec.json',case/'requests.npz',overlay['overlay_file'],overlay['overlay_manifest']):pins[str(p)]=digest(p) job.update(case_dir=str(case),build_root=str(EXP/'extension-build'),**overlay,diagnostic_mode='light', admission=dict(gpu_uuid=GPU,comfy_queue_urls=list(self.h.COMFY_URLS)), sources={n:dict(path=str(LAB/(n+'.py')),sha256=self.h.SOURCE_PINS[n]) for n in ('runtime_light','runtime_continuation')}) else: auth=read(authority);pins[str(authority)]=digest(authority) pins.update({v['path']:v['sha256'] for v in auth['files'].values()}) pins.update(auth['frozen_inputs']);pins[str(Path(auth['checkpoint_dir'])/'checkpoint-manifest.json')]=auth['checkpoint_manifest_sha256'] job.update(authority_file=str(authority),authority_sha256=digest(authority), sources={'conditioning_epoch':dict(path=str(LAB/'conditioning_epoch.py'),sha256=self.h.SOURCE_PINS['conditioning_epoch'])}) job['input_sha256']=pins;return job def brain_verify(self,task,case,output): summary=read(output/'summary.json');state=npz(output/'state.npz') checked=self.metric.validate_state(state,nodes=self.nodes,steps=1200) require(summary['all_finite'] is True and summary['steps']==1200 and summary['diagnostic_mode']=='light','Nonfinite/incomplete brain') require(int(state['counts'].sum())==summary['total_spikes'],'Spike event/count mismatch') manifest=read(output/'checkpoint-manifest.json') for rel,sha in manifest['files_sha256'].items():require(digest(output/rel)==sha,'Checkpoint payload changed') return dict(complete=True,state_raw_sha256=checked['hashes'],finite=True, checkpoint_manifest_sha256=digest(output/'checkpoint-manifest.json'),task_id=task['task_id']) def task(self,task): key=task_key(task);brain=key+'--brain';up=key+'--update';cpu=key+'--verify' planned=[brain,up,cpu] if task['kind']=='train' else [brain] if all(x in self.completed for x in planned):self.expected.extend(planned);return require(not any(x in self.completed for x in planned),'Partially completed task: preserve and inspect; no implicit replay') overlay=self.overlay(task);case=self.prepare_case(task);output=EXP/'runs'/key record=dict(schema='normal-task-result-v1',complete=True,registration_sha256=self.sha,task=task, checkpoint_dir=str(output),overlay_before=overlay,brain_process_receipt_path=str(EXP/f'receipts/{brain}/process.json')) def after_brain(): record['brain_audit']=self.brain_verify(task,case,output) record['checkpoint_manifest_sha256']=digest(output/'checkpoint-manifest.json') # Parent process receipt is produced later and is bound by the ledger, # avoiding a self-reference in a result written before process exit audit. if task['kind']=='probe':save(EXP/f'results/{key}.json',record) job=self.ap_job(brain,'brain',output,case=case,overlay=overlay) outs=[output,Path(job['receipt_file']),Path(job['receipt_file']+'.attempt.json')] if task['kind']=='probe':outs.append(EXP/f'results/{key}.json') self.launch(brain,job,[sys.executable,'-B',str(AP/'isolated_workers.py'),'brain'],outs, pre_outputs=[Path(p) for p in inventory(case)],post=after_brain,ap=True) if task['kind']=='probe':return template_pin=self.immutable_pin(EXP/'protocol/epoch-template.json') template=read(template_pin['path']);auth=copy.deepcopy(template) r=task['recipe'];auth.update(epoch_id=task['task_id'],diagnostic_mode='light',checkpoint_dir=str(output), checkpoint_manifest_sha256=digest(output/'checkpoint-manifest.json'), command=dict(steps=1200,compartment=task['compartment_verbatim'],us_tick=r['us']['tick'],d_command=r['D_command'], frozen=not r['update_enabled'],trace_backend='torch')) for role,path in dict(spec=case/'spec.json',requests=case/'requests.npz',state=output/'state.npz', overlay_before=overlay['overlay_file'],overlay_manifest_before=overlay['overlay_manifest']).items():auth['files'][role]=pin(path) authority=EXP/f'jobs/{key}.authority.json';save(authority,auth);update=EXP/'updates'/key job=self.ap_job(up,'update',update,authority=authority) job['input_sha256'][template_pin['path']]=template_pin['sha256'] for path in (EXP/f'receipts/{brain}/process.json',EXP/f'receipts/{brain}/worker.json'): checked=self.immutable_pin(path);job['input_sha256'][checked['path']]=checked['sha256'] self.launch(up,job,[sys.executable,'-B',str(AP/'isolated_workers.py'),'update'], [update,Path(job['receipt_file']),Path(job['receipt_file']+'.attempt.json')],pre_outputs=[authority],ap=True) out=unlinked(EXP/'validation'/key);out.mkdir();validation=out/'validation.json' script=AQ/'validate_update_cpu.py';verify({str(script):self.reg['input_sha256'][str(script)]}) pins={**job['input_sha256'],str(script):self.reg['input_sha256'][str(script)]} for name in ('conditioning_epoch.py','conditioning_preflight_compare.py','engineering_compare.py'): path=str(LAB/name);pins[path]=self.reg['input_sha256'][path] pins.update({p:v['sha256'] for p,v in inventory(update).items()}) job=dict(kind='cpu_verify',registration_sha256=self.sha,input_sha256=pins,authority_file=str(authority), authority_sha256=digest(authority),update_dir=str(update),chain_sha256=digest(update/'chain-receipt.json'), receipt_file=str(validation),validator_sha256=self.reg['input_sha256'][str(script)]) def after_cpu(): valid=read(validation) require(valid['complete'] is True and valid['frozen_U_verify_receipt_recomputed'] is True and valid['actual_sealed_events_used'] is True and valid['after_overlay_sha256']==digest(update/'delta-after.npz') and valid['epoch_authority_sha256']==digest(authority),'Independent update verification differs') if task['arm'] in ('d_zero','frozen','cs_only'): require(not npz(update/'delta-after.npz')['plastic_delta_mv'].view(np.uint32).any(),'Zero-update control became nonzero') record.update(epoch_authority=pin(authority),update_dir=str(update),cpu_validation=pin(validation), after_overlay=dict(overlay_file=str(update/'delta-after.npz'),overlay_manifest=str(update/'overlay-manifest.json')), update_process_receipt_path=str(EXP/f'receipts/{up}/process.json'), cpu_process_receipt_path=str(EXP/f'receipts/{cpu}/process.json')) save(EXP/f'results/{key}.json',record) self.launch(cpu,job,[sys.executable,'-B',str(script),'--authority',str(authority),'--authority-sha256',digest(authority), '--update-dir',str(update),'--chain-sha256',digest(update/'chain-receipt.json'),'--receipt',str(validation)], [out,EXP/f'results/{key}.json'],post=after_cpu) def run(self): try: self.input_generation() for task in self.plan['tasks']:self.task(task) report=self.ledger.final_verify(self.expected) require(report['pass_all'],'Final ledger coverage/integrity failed') save(EXP/'audit/execution-complete.json',dict(complete=True,registration_sha256=self.sha, inventory=self.plan['inventory'],ledger=report,learning_success_claimed=False, next='independent scientific analysis and frozen integrity, no subsequent assay')) except BaseException as error: if not (EXP/'audit/stopped.json').exists():save(EXP/'audit/stopped.json',dict(complete=False, registration_sha256=self.sha,error_type=type(error).__name__,error=str(error), traceback=traceback.format_exc(),partial_files_preserved=True,automatic_retry=False)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('--registration-sha256',required=True) Workflow(p.parse_args().registration_sha256).run()