Digital Drosophila Archive
228 lines
16 KiB
Python
228 lines
16 KiB
Python
"""Read-only CS/US semantics diagnostics; no learning or parameter selection."""
|
|
from pathlib import Path
|
|
import argparse
|
|
import json
|
|
import math
|
|
import time
|
|
import numpy as np
|
|
import pandas as pd
|
|
import torch
|
|
import yaml
|
|
import phase4p
|
|
import phase4p_finalize
|
|
import phase4s
|
|
import phase4u
|
|
from phase4p_revalidate import DeterministicAPLSim, door_inputs, make_drive
|
|
from phase4r import save, digest, arrhash
|
|
from phase4h_audit import verify_checksum
|
|
|
|
|
|
def checks(cfg):
|
|
old = yaml.safe_load((Path(cfg['prior_model'])/'phase4u.yaml').read_text())
|
|
out = phase4u.checks(old)
|
|
for key in ('prior_root', 'prior_model'):
|
|
root = Path(cfg[key]); out['U_'+key] = verify_checksum(root, root/'checksums.sha256')
|
|
assert all(x['passed'] for x in out.values())
|
|
return out
|
|
|
|
|
|
def forcing(sim, targets, rates, seed, cfg, mode):
|
|
block = round(cfg['forcing_block_ms']/sim.dt)
|
|
steps = round((cfg['observation_duration_ms'] if mode in ('sustained','offset') else cfg['short_duration_ms'])/sim.dt)
|
|
probs = torch.as_tensor(np.asarray(rates, np.float32)*sim.dt/1000., device=sim.device)
|
|
parts = []
|
|
for b in range(math.ceil(steps/block)):
|
|
gen = torch.Generator(device=sim.device); gen.manual_seed(int(seed+b*cfg['forcing_block_seed_stride']))
|
|
parts.append((torch.rand((block,len(targets)),generator=gen,device=sim.device)<probs[None,:]).to(torch.uint8))
|
|
out = torch.cat(parts)[:steps]
|
|
if mode == 'offset': out[block:] = 0
|
|
if mode == 'warmup':
|
|
out = torch.cat((torch.zeros((round(cfg['warmup_ms']/sim.dt),len(targets)),dtype=torch.uint8,device=sim.device),out))
|
|
return out
|
|
|
|
|
|
def observed(sim, targets, forced, target_kc, cfg, dan=None, pulse=None):
|
|
"""Statement ordering and all state mutations mirror frozen DeterministicAPLSim.run."""
|
|
rt=sim.rt; sim.install(None); rt.reset(); steps=len(forced)
|
|
tg=torch.as_tensor(targets,dtype=torch.int64,device=sim.device)
|
|
dan=np.empty(0,np.int32) if dan is None else dan
|
|
dg=torch.as_tensor(dan,dtype=torch.int64,device=sim.device)
|
|
df=torch.zeros((steps,len(dan)),dtype=torch.uint8,device=sim.device)
|
|
ds=set() if pulse is None else {round(pulse/sim.dt)}
|
|
for s in ds: df[s]=1
|
|
apl=torch.zeros(len(sim.apl),dtype=torch.float32,device=sim.device)
|
|
trace=torch.zeros(len(sim.kc),dtype=torch.float32,device=sim.device)
|
|
first=torch.full((sim.n,),-1,dtype=torch.int32,device=sim.device)
|
|
ad=np.float32(math.exp(-sim.dt/20.)); td=np.float32(math.exp(-sim.dt/100.))
|
|
trace_at=[]; evs=[]; evn=[]; samples=[]; us=[]; prefix=None
|
|
stride=round(cfg['sampling_ms']/sim.dt); short=round(cfg['short_duration_ms']/sim.dt)
|
|
requests=0; accepted=0; existing=0; blocked=0
|
|
target_gpu=torch.as_tensor(target_kc,dtype=torch.int64,device=sim.device)
|
|
torch.cuda.synchronize(); start=time.perf_counter()
|
|
for step in range(steps):
|
|
rt.ext.state_step(rt.v,rt.g,rt.ref,rt.queue[step%rt.qlen],rt.spike_u8,rt.counts,
|
|
float(rt.gd),float(rt.alpha),float(rt.rest),float(rt.resetv),float(rt.threshold),rt.ref_steps)
|
|
pre=rt.spike_u8[tg].clone(); pre_ref=rt.ref[tg].clone()
|
|
rt.ext.force_step(rt.v,rt.ref,rt.spike_u8,rt.counts,tg,forced[step],float(rt.resetv),rt.ref_steps)
|
|
req=forced[step].bool(); requests+=int(req.sum())
|
|
accepted+=int((req & ~pre.bool() & (pre_ref==0)).sum())
|
|
existing+=int((req & pre.bool()).sum()); blocked+=int((req & ~pre.bool() & (pre_ref!=0)).sum())
|
|
if step in ds: before=rt.spike_u8[dg].clone(); refs=rt.ref[dg].clone()
|
|
if len(dan): rt.ext.force_step(rt.v,rt.ref,rt.spike_u8,rt.counts,dg,df[step],float(rt.resetv),rt.ref_steps)
|
|
if step in ds:
|
|
after=rt.spike_u8[dg]
|
|
for j,node in enumerate(dan):
|
|
us.append({'step':step,'time_ms':step*sim.dt,'neuron_idx':int(node),'preexisting_spike':bool(before[j]),
|
|
'pre_force_refractory':int(refs[j]),'accepted_new_spike':bool(after[j] and not before[j]),
|
|
'actual_output_spike':bool(after[j]),'teaching_gate_sampled':True,'first_edge_arrival_ms':(step+rt.delay_steps)*sim.dt})
|
|
sources=torch.nonzero(rt.spike_u8,as_tuple=False).flatten()
|
|
if sources.numel():
|
|
missing=first[sources]<0; first[sources[missing]]=step
|
|
rt.canonical_fanout(sources,rt.queue[(step+rt.delay_steps)%rt.qlen])
|
|
kc_spike=rt.spike_u8[sim.kc_gpu]
|
|
trace.mul_(float(td)).add_(kc_spike.to(torch.float32)).clamp_max_(1.)
|
|
if step in ds: trace_at.append(trace.clone())
|
|
active=torch.nonzero(kc_spike,as_tuple=False).flatten()
|
|
if active.numel():
|
|
node=sim.kc_gpu[active].cpu().numpy().astype(np.int32)
|
|
evn.append(node); evs.append(np.full(len(node),step,np.int32))
|
|
apl.mul_(float(ad)).add_(torch.mv(sim.input_gpu,rt.spike_u8.to(torch.float32)))
|
|
inhibition=torch.matmul(apl,sim.output_gpu)*np.float32(.01)
|
|
rt.g[sim.kc_gpu]=rt.g[sim.kc_gpu]-inhibition
|
|
rt.step_index+=1
|
|
if step==short-1:
|
|
prefix={k:x.cpu().numpy().copy() for k,x in [('counts',rt.counts),('final_v',rt.v),('final_g',rt.g)]}
|
|
if (step+1)%stride==0:
|
|
samples.append({'step':step,'time_ms':(step+1)*sim.dt,'PN_requested_cumulative':requests,
|
|
'PN_accepted_cumulative':accepted,'PN_preexisting_cumulative':existing,'PN_blocked_cumulative':blocked,
|
|
'KC_spikes_cumulative':int(rt.counts[sim.kc_gpu].sum()),'target_KC_spikes_cumulative':int(rt.counts[target_gpu].sum()),
|
|
'KC_active_cumulative':int((rt.counts[sim.kc_gpu]>0).sum()),'KC_g_mean':float(rt.g[sim.kc_gpu].mean()),
|
|
'KC_V_mean':float(rt.v[sim.kc_gpu].mean()),'KC_threshold_margin_min':float((rt.threshold-rt.v[sim.kc_gpu]).min()),
|
|
'APL_state_0':float(apl[0]),'APL_state_1':float(apl[1]),'APL_inhibition_mean':float(inhibition.mean()),
|
|
'finite':bool(torch.isfinite(rt.v).all() and torch.isfinite(rt.g).all())})
|
|
torch.cuda.synchronize()
|
|
counts=rt.counts.cpu().numpy().copy()
|
|
return {'counts':counts,'kc_counts':counts[sim.kc], 'first':first.cpu().numpy().copy(),
|
|
'final_v':rt.v.cpu().numpy().copy(),'final_g':rt.g.cpu().numpy().copy(),
|
|
'trace':np.zeros(len(sim.kc),np.float32) if not trace_at else trace_at[-1].cpu().numpy().copy(),
|
|
'kc_event_step':np.concatenate(evs) if evs else np.empty(0,np.int32),
|
|
'kc_event_node':np.concatenate(evn) if evn else np.empty(0,np.int32),
|
|
'prefix':prefix,'samples':samples,'US':us,'wall_seconds':time.perf_counter()-start,
|
|
'finite':all(x['finite'] for x in samples)}
|
|
|
|
|
|
def exact(a,b,keys):
|
|
return {k:bool(np.array_equal(a[k],b[k])) for k in keys}
|
|
|
|
|
|
def run(cfg,exp,model):
|
|
for name in ('protocol','audit','events','states','observations','regression','performance'): (exp/name).mkdir(parents=True,exist_ok=True)
|
|
assert not (exp/'protocol/preregistered.json').exists(),'Refusing overwrite'
|
|
save(exp/'audit/preflight.json',checks(cfg))
|
|
oc,panel,vectors,selection,mapping=door_inputs(); panel=panel[panel.split=='calibration'].reset_index(drop=True)
|
|
assert len(panel)==6
|
|
save(exp/'protocol/preregistered.json',{'config':cfg,'panel':panel.to_dict('records'),
|
|
'sources':{p.name:digest(p) for p in model.iterdir() if p.is_file()},'before_simulation':True,'no_parameter_selection':True})
|
|
pc=yaml.safe_load((Path(cfg['runtime_root'])/'phase4p.yaml').read_text())
|
|
rt=phase4p.CanonicalRuntime(pc,Path(cfg['runtime_root']),exp); sim=DeterministicAPLSim(rt)
|
|
assert len(sim.identity)==14232
|
|
target=np.unique(sim.identity.loc[sim.identity.compartment.eq(cfg['target_compartment']),'source_neuron_idx'].to_numpy(np.int32))
|
|
dan=sim.dan_targets(cfg['matched_DAN'],cfg['target_compartment'])
|
|
save(exp/'audit/semantics.json',{'CS':'frozen physiological DoOR-to-PN rates; Bernoulli force request per 0.1ms step; refractory may refuse',
|
|
'US':'one scheduled force request per confirmed PAM15 neuron; engineering teaching trace sampled at scheduled emission step regardless acceptance',
|
|
'dopamine_concentration_measured':False,'initial_state':'all V at rest; g/ref/queue/APL/trace zero',
|
|
'delay_steps':rt.delay_steps,'delay_ms':rt.delay_steps*sim.dt,'DAN_ids':dan.tolist(),'target_KC_ids':target.tolist(),
|
|
'long_input':'120ms RNG blocks with explicit seed stride; first block identical, later sequence not claimed to equal legacy long run',
|
|
'time_scaling_to_biological_seconds':'unknown; 1000ms is observation, not conditioning replication'})
|
|
keys=['counts','first','final_v','final_g','trace','kc_event_step','kc_event_node']; regress=[]; summaries=[]; usrows=[]
|
|
start=time.perf_counter(); torch.cuda.reset_peak_memory_stats()
|
|
def persist(name,r,oi,seed,mode):
|
|
assert r['finite'],'Nonfinite state'
|
|
phase4s.persist_run(exp,name,r)
|
|
np.savez_compressed(exp/'states'/f'{name}.npz',**{k:r[k] for k in ('counts','final_v','final_g','trace')})
|
|
pd.DataFrame(r['samples']).to_parquet(exp/'observations'/f'{name}.parquet',index=False)
|
|
steps=r['kc_event_step']; nodes=r['kc_event_node']; t=steps[np.isin(nodes,target)]*sim.dt
|
|
row={'odor_index':oi,'odor_name':panel.iloc[oi].odor_name,'seed':seed,'condition':mode,
|
|
'KC_active':int(np.count_nonzero(r['kc_counts'])),'KC_events':len(steps),'target_events':len(t),
|
|
'first_target_ms':float(t.min()) if len(t) else None,'median_target_ms':float(np.median(t)) if len(t) else None,
|
|
'target_events_after120ms':int(np.count_nonzero(t>=120)), 'KC_events_after120ms':int(np.count_nonzero(steps*sim.dt>=120)),
|
|
'last_KC_event_ms':float(steps.max()*sim.dt) if len(steps) else None,'wall_seconds':r['wall_seconds'],'finite':r['finite']}
|
|
summaries.append(row)
|
|
for oi,odor in panel.iterrows():
|
|
targets,rates=make_drive(oc,panel,vectors,selection,mapping,odor.odor_id,'physiological')
|
|
for si,seed in enumerate(cfg['seeds']):
|
|
actual=cfg['seed_namespace']+oi*cfg['odor_seed_offset']+seed
|
|
legacy=sim.run(targets,rates,actual,cfg['short_duration_ms'],collect_events=True)
|
|
f=forcing(sim,targets,rates,actual,cfg,'short')
|
|
short=observed(sim,targets,f,target,cfg)
|
|
eq=exact(legacy,short,keys); regress.append({'odor':oi,'seed':seed,'test':'short-exact','fields':eq})
|
|
save(exp/'regression/instrumentation.json',regress); assert all(eq.values()),'Instrumentation diverges'
|
|
persist(f'{oi}-{seed}-short',short,oi,seed,'short')
|
|
for mode in ('sustained','offset'):
|
|
ff=forcing(sim,targets,rates,actual,cfg,mode); assert torch.equal(ff[:len(f)],f)
|
|
r=observed(sim,targets,ff,target,cfg)
|
|
eq=exact(legacy,r['prefix'],['counts','final_v','final_g'])
|
|
mask=r['kc_event_step']<len(f)
|
|
eq['events']=arrhash(r['kc_event_step'][mask],r['kc_event_node'][mask])==arrhash(legacy['kc_event_step'],legacy['kc_event_node'])
|
|
regress.append({'odor':oi,'seed':seed,'test':mode+'-prefix','fields':eq}); assert all(eq.values())
|
|
persist(f'{oi}-{seed}-{mode}',r,oi,seed,mode)
|
|
if si==0:
|
|
warm=observed(sim,targets,forcing(sim,targets,rates,actual,cfg,'warmup'),target,cfg)
|
|
eq=exact(legacy,warm,['counts','final_v','final_g','trace','kc_event_node'])
|
|
eq['shifted_events']=np.array_equal(warm['kc_event_step']-round(cfg['warmup_ms']/sim.dt),legacy['kc_event_step'])
|
|
regress.append({'odor':oi,'seed':seed,'test':'quiet-warmup','fields':eq})
|
|
persist(f'{oi}-{seed}-warmup',warm,oi,seed,'warmup')
|
|
for pulse in cfg['US_audit_ms']:
|
|
lr=sim.run(targets,rates,actual,cfg['short_duration_ms'],dan_targets=dan,dan_pulses=[pulse],collect_events=True)
|
|
r=observed(sim,targets,f,target,cfg,dan,pulse); eq=exact(lr,r,keys)
|
|
regress.append({'odor':oi,'seed':seed,'test':f'US{pulse}-exact','fields':eq}); assert all(eq.values())
|
|
usrows.extend([dict(x,odor_index=oi,seed=seed,US_ms=pulse) for x in r['US']])
|
|
persist(f'{oi}-{seed}-US{pulse}',r,oi,seed,f'US{pulse}')
|
|
save(exp/'regression/instrumentation.json',regress)
|
|
print(json.dumps({'completed_odor':oi,'seed':seed,'trials':len(summaries)}),flush=True)
|
|
pd.DataFrame(summaries).to_parquet(exp/'audit/trials.parquet',index=False)
|
|
pd.DataFrame(usrows).to_parquet(exp/'audit/US-acceptance.parquet',index=False)
|
|
reference=phase4s.evidence_regression(cfg,exp,rt,sim)
|
|
result={'phase':'4V','status':'COMPLETE-DIAGNOSTIC-ONLY','reference_pass':reference,
|
|
'exact_instrumentation_pass':all(all(x['fields'].values()) for x in regress),'diagnostic_trials':len(summaries),
|
|
'weight_updates':0,'candidate_created':False,'reserved_test_odors_consumed':0,
|
|
'DAN_requests':len(usrows),'DAN_accepted_new':sum(x['accepted_new_spike'] for x in usrows),
|
|
'DAN_preexisting':sum(x['preexisting_spike'] for x in usrows),
|
|
'DAN_refused':sum(not x['actual_output_spike'] for x in usrows)}
|
|
assert result['exact_instrumentation_pass'] and reference
|
|
save(exp/'audit/results.json',result)
|
|
save(exp/'performance/summary.json',{'wall_seconds_excluding_build':time.perf_counter()-start,
|
|
'peak_VRAM_bytes':torch.cuda.max_memory_allocated(),'GPU':torch.cuda.get_device_name(0),
|
|
'measurement':'instrumented diagnostics include synchronous observation overhead; not runtime speed benchmark'})
|
|
print(json.dumps(result),flush=True)
|
|
|
|
|
|
def seal(cfg,exp,model):
|
|
now=checks(cfg); history=phase4p_finalize.integrity(); pre=json.loads((exp/'protocol/preregistered.json').read_text())
|
|
passed=history['all_static_checks_passed'] and now==json.loads((exp/'audit/preflight.json').read_text()) and all(digest(model/n)==h for n,h in pre['sources'].items())
|
|
save(exp/'audit/frozen-integrity.json',{'pass':passed,'history':history,'current':now}); assert passed
|
|
r=json.loads((exp/'audit/results.json').read_text()); r['integrity_pass']=True; save(exp/'audit/results.json',r)
|
|
df=pd.read_parquet(exp/'audit/trials.parquet'); lines=['# Phase 4V — CS/US semantics and timescale audit','',
|
|
'Status: COMPLETE-DIAGNOSTIC-ONLY. No training, selection, new timing policy or learned candidate.',
|
|
'Frozen instrumentation and five reference trials: exact PASS. Frozen checksums: PASS.',
|
|
'Only six old calibration odors were used; six reserved test odors remain unconsumed.','',
|
|
'| Odor | Condition | mean KC events | mean target events | mean target events >=120ms |',
|
|
'|---|---|---:|---:|---:|']
|
|
for (odor,mode),g in df.groupby(['odor_name','condition'],sort=False):
|
|
lines.append(f"| {odor} | {mode} | {g.KC_events.mean():.2f} | {g.target_events.mean():.2f} | {g.target_events_after120ms.mean():.2f} |")
|
|
lines += ['',f"DAN requests: {r['DAN_requests']}; accepted new spikes: {r['DAN_accepted_new']}; preexisting: {r['DAN_preexisting']}; refused: {r['DAN_refused']}.",
|
|
'US is a force request, not measured dopamine. Teaching trace is sampled at the scheduled emission step; synaptic arrivals are 1.8ms later.',
|
|
'Observation does not establish biological time scaling or associative memory. Literature boundaries are in phase4v-literature.md.',
|
|
'No game, decoder, RL, visual input, gain/threshold changes, eligibility changes, or plastic-mask changes.']
|
|
(exp/'REPORT.md').write_text('\n'.join(lines)+'\n',encoding='utf-8')
|
|
for root in (model,exp):
|
|
rows=[{'path':p.relative_to(root).as_posix(),'sha256':digest(p),'bytes':p.stat().st_size} for p in sorted(root.rglob('*')) if p.is_file() and p.name not in ('manifest.json','checksums.sha256')]
|
|
save(root/'manifest.json',{'files':rows}); rows.append({'path':'manifest.json','sha256':digest(root/'manifest.json')})
|
|
(root/'checksums.sha256').write_text(''.join(x['sha256']+' '+x['path']+'\n' for x in rows))
|
|
check=verify_checksum(root,root/'checksums.sha256'); assert check['passed']; print(json.dumps(check),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser(); p.add_argument('task',choices=['run','seal']); p.add_argument('--config',required=True); a=p.parse_args()
|
|
cfg=yaml.safe_load(Path(a.config).read_text()); (run if a.task=='run' else seal)(cfg,Path(cfg['experiment_root']),Path(cfg['model_root']))
|