Digital Drosophila Archive
370 lines
28 KiB
Python
370 lines
28 KiB
Python
"""New-path-only Phase 4AB scope diagnostics; never edits frozen scientific files."""
|
||
import argparse
|
||
from datetime import datetime, timezone
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
import numpy as np
|
||
import pandas as pd
|
||
import yaml
|
||
from scope import contact_scope, edge_ids_for_pairs, blocked_weights
|
||
|
||
|
||
def load_context():
|
||
cfg = yaml.safe_load(Path(__file__).with_name('config.yaml').read_text())
|
||
root = Path(cfg['root']); model = root/cfg['model']; exp = root/cfg['experiment']
|
||
sys.path.insert(0, str(root/cfg['prior_model']))
|
||
from audit import save, digest, immutable_check, seal
|
||
return cfg, root, model, exp, save, digest, immutable_check, seal
|
||
|
||
|
||
def manifest_check(folder, digest):
|
||
manifest = folder/'manifest.json'
|
||
rows = json.loads(manifest.read_text())['files']
|
||
bad = []
|
||
for row in rows:
|
||
path = folder/row['path']
|
||
if not path.is_file() or path.stat().st_size != row['bytes'] or digest(path) != row['sha256']:
|
||
bad.append(row['path'])
|
||
return {'pass': not bad, 'checked': len(rows), 'mismatches': bad,
|
||
'manifest_sha256': digest(manifest), 'checksums_sha256': digest(folder/'checksums.sha256')}
|
||
|
||
|
||
def integrity(cfg, root, digest, immutable_check):
|
||
result = {'original': immutable_check(root, root/cfg['snapshot'], cfg['workers'])}
|
||
for key in ('prior_model', 'prior_experiment'):
|
||
result[key] = manifest_check(root/cfg[key], digest)
|
||
assert all(x['pass'] for x in result.values()), result
|
||
return result
|
||
|
||
|
||
def queues():
|
||
out = []
|
||
for port in (8188, 8189, 8190):
|
||
with urllib.request.urlopen(f'http://192.168.10.51:{port}/queue', timeout=5) as r:
|
||
q = json.load(r)
|
||
out.append({'port': port, 'running': len(q['queue_running']), 'pending': len(q['queue_pending'])})
|
||
assert all(x['running'] == x['pending'] == 0 for x in out), 'ComfyUI busy: defer, never evict'
|
||
return out
|
||
|
||
|
||
def prepare():
|
||
cfg, root, model, exp, save, digest, immutable_check, _ = load_context()
|
||
assert exp.parent == root/'experiments' and model.parent == root/'models/male-cns-lif'
|
||
if exp.exists() and any(exp.iterdir()): raise RuntimeError('Refusing nonempty experiment')
|
||
exp.mkdir(exist_ok=True)
|
||
for d in ('protocol', 'audit', 'masks', 'events', 'states', 'observations', 'requests', 'replay', 'regression', 'performance'):
|
||
(exp/d).mkdir()
|
||
save(exp/'protocol/preregistered.json', {'config': cfg, 'before_analysis': True,
|
||
'timestamp_utc': datetime.now(timezone.utc).isoformat(),
|
||
'parent_commit': subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip(),
|
||
'source_hashes': {p.name: digest(p) for p in sorted(model.iterdir()) if p.is_file()}})
|
||
save(exp/'regression/preflight.json', integrity(cfg, root, digest, immutable_check))
|
||
can = root/cfg['canonical']; prior = root/cfg['prior_experiment']
|
||
m = pd.read_parquet(can/'neuron-index/unified-metadata.parquet').sort_values('neuron_idx').reset_index(drop=True)
|
||
ptr = np.load(can/'gpu/csr-indptr.int64.npy', mmap_mode='r')
|
||
dst = np.load(can/'gpu/csr-indices.int32.npy', mmap_mode='r')
|
||
raw = np.load(can/'gpu/csr-weights.int64.npy', mmap_mode='r')
|
||
assert len(m) == 166700 and len(dst) == 25582938 and int(raw.sum()) == 124177617
|
||
assert m.body_id.dtype == np.uint64 and np.array_equal(m.neuron_idx, np.arange(len(m)))
|
||
src = np.repeat(np.arange(len(m), dtype=np.int32), np.diff(ptr))
|
||
apl = np.flatnonzero(m.annotation_type.eq('APL')); pn = np.flatnonzero(m.annotation_class.eq('ALPN'))
|
||
kc = np.flatnonzero(m.annotation_class.eq('Kenyon_Cell'))
|
||
contacts = pd.read_parquet(prior/'localization/APL-incident.parquet')
|
||
ap = contacts[contacts.source_neuron_idx.isin(apl) & contacts.target_neuron_idx.isin(pn)].copy()
|
||
pk = pd.read_parquet(prior/'localization/APL-target-PN-to-KC.parquet')
|
||
assert len(ap) == 4600 and ap.target_neuron_idx.nunique() == 259 and len(pk) == 385229
|
||
assert pk.source_neuron_idx.isin(pn).all() and pk.target_neuron_idx.isin(kc).all()
|
||
pairs, allowed, keep = contact_scope(ap, pk, cfg['roi_labels'])
|
||
ids = edge_ids_for_pairs(pairs, ptr, dst)
|
||
assert np.array_equal(raw[ids], pairs.raw_partner_count)
|
||
full_expected = np.flatnonzero(np.isin(src, ap.target_neuron_idx.unique()) & np.isin(dst, kc))
|
||
assert np.array_equal(np.sort(ids), full_expected), 'Missing whole PN->KC pairs'
|
||
pairs['edge_id'] = ids
|
||
pairs['source_body_id'] = m.body_id.to_numpy()[pairs.source_neuron_idx]
|
||
pairs['target_body_id'] = m.body_id.to_numpy()[pairs.target_neuron_idx]
|
||
ca = pairs[pairs.blocked_contact_count > 0].sort_values('edge_id').copy()
|
||
eligible = np.sort(ca.source_neuron_idx.unique()).astype(np.int32)
|
||
all_pk = np.flatnonzero(np.isin(src, eligible) & np.isin(dst, kc))
|
||
whole = np.flatnonzero(np.isin(src, eligible))
|
||
assert set(ca.edge_id).issubset(all_pk) and set(all_pk).issubset(whole)
|
||
assert not np.any(np.isin(src[ca.edge_id], apl)) and not np.any(ca.retained_contact_count < 0)
|
||
for name, eids, count in [
|
||
('calyx_PN_KC_contact_block', ca.edge_id.to_numpy(np.int64), ca.blocked_contact_count.to_numpy(np.int64)),
|
||
('all_PN_KC_block', all_pk, raw[all_pk]), ('whole_PN_output_block', whole, raw[whole])]:
|
||
frame = pd.DataFrame({'edge_id': eids, 'source_neuron_idx': src[eids], 'target_neuron_idx': dst[eids],
|
||
'raw_synapse_count': raw[eids], 'blocked_contact_count': count})
|
||
frame['source_body_id'] = m.body_id.to_numpy()[src[eids]]
|
||
frame['target_body_id'] = m.body_id.to_numpy()[dst[eids]]
|
||
frame.to_parquet(exp/'masks'/f'{name}.parquet', index=False)
|
||
pairs.to_parquet(exp/'audit/all-PN-KC-contact-partitions.parquet', index=False)
|
||
allowed.to_parquet(exp/'audit/observed-APL-PN-ROI-pairs.parquet', index=False)
|
||
pns = m.iloc[np.sort(ap.target_neuron_idx.unique().astype(np.int32))].copy()
|
||
pns['eligible_for_diagnostic'] = pns.neuron_idx.isin(eligible)
|
||
pns.to_parquet(exp/'audit/all-259-APL-target-PNs.parquet', index=False)
|
||
roi = pk.assign(in_scope=keep).groupby(['primary_post', 'in_scope'], observed=True, dropna=False).size().reset_index(name='contacts')
|
||
roi.to_csv(exp/'audit/ROI-partition.csv', index=False)
|
||
summary = {'neurons': len(m), 'edges': len(dst), 'raw_contacts': int(raw.sum()), 'APL_target_PNs': len(pns),
|
||
'eligible_PNs': len(eligible), 'ineligible_PNs': int((~pns.eligible_for_diagnostic).sum()),
|
||
'calyx_mask_edges': len(ca), 'calyx_blocked_contacts': int(ca.blocked_contact_count.sum()),
|
||
'calyx_affected_KCs': int(ca.target_neuron_idx.nunique()), 'mixed_CA_nonCA_edges': int((ca.retained_contact_count > 0).sum()),
|
||
'calyx_other_contacts_retained_on_affected_edges': int(ca.retained_contact_count.sum()),
|
||
'all_PN_KC_edges': len(all_pk), 'all_PN_KC_contacts': int(raw[all_pk].sum()),
|
||
'whole_PN_edges': len(whole), 'whole_PN_contacts': int(raw[whole].sum()),
|
||
'unknown_ROI_contacts_in_original_PN_KC': int(pk.primary_post.eq('CentralBrain-unspecified').sum()),
|
||
'unknown_contacts_relabelled': 0, 'same_bouton_confirmed': False, 'biophysical_coefficient_known': False}
|
||
save(exp/'audit/scope-summary.json', summary)
|
||
inputs = [prior/'localization/APL-incident.parquet', prior/'localization/APL-target-PN-to-KC.parquet',
|
||
prior/'external-resources/synaptic-partners/download-verification.json',
|
||
can/'neuron-index/unified-metadata.parquet'] + list((can/'gpu').glob('csr-*.npy'))
|
||
save(exp/'protocol/input-hashes.json', [{'path': str(p.relative_to(root)), 'sha256': digest(p), 'bytes': p.stat().st_size} for p in inputs])
|
||
save(exp/'protocol/mask-hashes.json', {p.name: digest(p) for p in sorted((exp/'masks').iterdir())})
|
||
print(json.dumps(summary), flush=True)
|
||
|
||
|
||
def run():
|
||
cfg, root, model, exp, save, digest, _, _ = load_context()
|
||
pre = json.loads((exp/'protocol/preregistered.json').read_text())
|
||
assert all(digest(model/n) == h for n, h in pre['source_hashes'].items()), 'Source changed since preregistration'
|
||
assert not (exp/'regression/run-start.json').exists(), 'Refusing rerun into existing outputs'
|
||
assert all(digest(exp/'masks'/n) == h for n, h in json.loads((exp/'protocol/mask-hashes.json').read_text()).items())
|
||
q = queues(); runtime = root/cfg['runtime']
|
||
pc = yaml.safe_load((runtime/'phase4p.yaml').read_text())
|
||
os.environ['CUDA_VISIBLE_DEVICES'] = pc['device']['visible_gpu_uuid']
|
||
os.environ['CUDA_HOME'] = '/usr/local/cuda'; os.environ['TORCH_CUDA_ARCH_LIST'] = '8.9'; os.environ['MAX_JOBS'] = '2'
|
||
os.environ['PATH'] = str(root/'models/male-cns-lif/.venv/bin') + ':/usr/local/cuda/bin:' + os.environ['PATH']
|
||
sys.path.insert(0, str(root/'models/male-cns-lif')); sys.path.insert(0, str(runtime))
|
||
import torch
|
||
import phase4p, phase4v, phase4z
|
||
from phase4p_revalidate import DeterministicAPLSim, door_inputs, make_drive
|
||
from phase4r import arrhash
|
||
free, total = torch.cuda.mem_get_info(); assert free > 2*1024**3
|
||
save(exp/'regression/run-start.json', {'ComfyUI_queues': q, 'free_VRAM': free, 'total_VRAM': total,
|
||
'device': torch.cuda.get_device_name(), 'timestamp_utc': datetime.now(timezone.utc).isoformat()})
|
||
start = time.perf_counter(); torch.cuda.reset_peak_memory_stats()
|
||
rt = phase4p.CanonicalRuntime(pc, runtime, exp)
|
||
sim = DeterministicAPLSim(rt); sim.install(None)
|
||
assert len(sim.identity) == 14232 and sim.n == 166700 and len(rt.raw_np) == 25582938 and rt.delay_steps == 18
|
||
frozen = rt.weights.cpu().numpy().copy(); anatomical_hash = arrhash(rt.raw_np)
|
||
original_install = sim.install
|
||
apl_input_hash = arrhash(sim.input_gpu.cpu().numpy(), sim.output_gpu.cpu().numpy())
|
||
base_hash = arrhash(sim.base.cpu().numpy())
|
||
masks = {n: pd.read_parquet(exp/'masks'/f'{n}.parquet') for n in cfg['conditions'][1:]}
|
||
updates = {}
|
||
for name, mask in masks.items():
|
||
ids = mask.edge_id.to_numpy(np.int64)
|
||
values = blocked_weights(frozen[ids], rt.raw_np[ids], mask.blocked_contact_count.to_numpy(np.int64))
|
||
assert np.all(np.abs(values) <= np.abs(frozen[ids]))
|
||
updates[name] = (torch.as_tensor(ids, device=rt.device), torch.as_tensor(values, device=rt.device))
|
||
mask.assign(frozen_effective_weight=frozen[ids], diagnostic_effective_weight=values,
|
||
diagnostic_delta=(values.astype(float)-frozen[ids])).to_parquet(exp/'audit'/f'{name}-effective-overlay.parquet', index=False)
|
||
condition = 'frozen_reference'
|
||
def install(E):
|
||
assert E is None, 'No learning allowed'
|
||
original_install(None)
|
||
if condition != 'frozen_reference':
|
||
ix, values = updates[condition]; rt.weights[ix] = values
|
||
sim.install = install
|
||
vc = yaml.safe_load((root/cfg['input_model']/'phase4v.yaml').read_text())
|
||
assert vc['short_duration_ms'] == 120 and vc['observation_duration_ms'] == 1000
|
||
oldwatch = json.loads((root/cfg['event_prior']/'protocol/watched.json').read_text())
|
||
watch = np.asarray(oldwatch['all_watched'], np.int32)
|
||
targetkc = np.unique(sim.identity.loc[sim.identity.compartment.eq(vc['target_compartment']), 'source_neuron_idx'].to_numpy(np.int32))
|
||
oc, panel, vectors, selection, mapping = door_inputs(); panel = panel[panel.split == 'calibration'].reset_index(drop=True)
|
||
assert len(panel) == 6
|
||
save(exp/'protocol/calibration-panel.json', panel.to_dict('records'))
|
||
codes = phase4z.role_codes(sim.meta)
|
||
trials, regress, request_summary, populations, watched, hashes = [], [], [], [], [], []
|
||
baseline0 = None
|
||
try:
|
||
for oi in cfg['odor_indices']:
|
||
odor = panel.iloc[oi]
|
||
targets, rates = make_drive(oc, panel, vectors, selection, mapping, odor.odor_id, 'physiological')
|
||
forcing_seed = vc['seed_namespace'] + oi*vc['odor_seed_offset'] + cfg['seed']
|
||
force = phase4v.forcing(sim, targets, rates, forcing_seed, vc, 'offset'); ff = force.cpu().numpy()
|
||
hashes.append({'odor_index': oi, 'seed': forcing_seed, 'forcing': arrhash(ff), 'targets_rates': arrhash(targets, rates)})
|
||
np.savez_compressed(exp/'requests'/f'{oi}-scheduled.npz', forcing=ff, targets=targets, rates=rates)
|
||
requests0 = None
|
||
for condition in cfg['conditions']:
|
||
queues(); name = f'{oi}-{cfg["seed"]}-{condition}'
|
||
obs = phase4z.Observer(rt, watch, ff, targets); obs.attach()
|
||
try: result = phase4v.observed(sim, targets, force, targetkc, vc)
|
||
finally: obs.detach()
|
||
assert result['finite'] and np.isfinite(result['final_v']).all() and np.isfinite(result['final_g']).all()
|
||
ev = obs.events(); states = np.asarray(obs.states); req = pd.DataFrame(obs.request_rows)
|
||
assert len(req) == int(ff.sum())
|
||
assert np.array_equal(np.bincount(ev.neuron_idx, minlength=sim.n), result['counts'])
|
||
expected_weights = frozen.copy()
|
||
if condition != 'frozen_reference':
|
||
ids = masks[condition].edge_id.to_numpy(np.int64)
|
||
expected_weights[ids] = updates[condition][1].cpu().numpy()
|
||
assert np.array_equal(rt.weights.cpu().numpy(), expected_weights), 'Undeclared weight change'
|
||
row = {'trial': name, 'declared_weight_scope_exact': True}
|
||
if condition == 'frozen_reference':
|
||
old_ev = pd.read_parquet(root/cfg['event_prior']/'events'/f'{name}-whole-brain.parquet')
|
||
old_state = np.load(root/cfg['event_prior']/'states'/f'{name}-watched.npz')
|
||
old_final = np.load(root/cfg['state_prior']/'states'/f'{name}.npz')
|
||
row['prior_exact'] = {'events': ev.equals(old_ev), 'watched_states': np.array_equal(states, old_state['state']),
|
||
**{key: bool(np.array_equal(result[key], old_final[key])) for key in ('counts', 'final_v', 'final_g', 'trace')}}
|
||
assert all(row['prior_exact'].values()), row
|
||
requests0 = req
|
||
if oi == cfg['repeat_odor_index']: baseline0 = ev, states, req, result
|
||
paired = requests0.merge(req, on=['step', 'neuron_idx'], suffixes=('_ref', '_diagnostic'), validate='one_to_one', how='outer', indicator=True)
|
||
assert paired['_merge'].eq('both').all()
|
||
request_summary.append({'trial': name, 'scheduled': len(req), 'identical_request_keys': True,
|
||
'accepted': int(req.status.eq('accepted').sum()), 'blocked': int(req.status.eq('blocked').sum()), 'preexisting': int(req.status.eq('preexisting').sum()),
|
||
'status_differences_vs_reference': int((paired.status_ref != paired.status_diagnostic).sum()),
|
||
'actual_output_differences_vs_reference': int((paired.actual_output_spike_ref != paired.actual_output_spike_diagnostic).sum())})
|
||
late = ev[ev.step >= round(cfg['late_start_ms']/rt.dt)]
|
||
period_count = np.bincount(late.neuron_idx, minlength=sim.n)
|
||
for code in np.unique(codes):
|
||
nodes = np.flatnonzero(codes == code)
|
||
populations.append({'trial': name, 'population': str(code), 'neurons': len(nodes), 'active': int((result['counts'][nodes] > 0).sum()),
|
||
'spikes': int(result['counts'][nodes].sum()), 'late_spikes': int(period_count[nodes].sum())})
|
||
for node in watch:
|
||
watched.append({'trial': name, 'neuron_idx': int(node), 'body_id': int(sim.meta.iloc[node].body_id),
|
||
'type': str(sim.meta.iloc[node].annotation_type), 'population': str(codes[node]),
|
||
'spikes': int(result['counts'][node]), 'late_spikes': int(period_count[node]),
|
||
'late_rate_hz': float(period_count[node]/((1000-cfg['late_start_ms'])/1000))})
|
||
trials.append({'trial': name, 'odor_index': oi, 'odor_name': str(odor.odor_name), 'seed': cfg['seed'], 'condition': condition,
|
||
'whole_spikes': len(ev), 'KC_active': int(np.count_nonzero(result['counts'][sim.kc])),
|
||
'KC_active_fraction': float(np.count_nonzero(result['counts'][sim.kc])/len(sim.kc)),
|
||
'KC_spikes': int(result['counts'][sim.kc].sum()), 'KC_late_spikes': int(period_count[sim.kc].sum()),
|
||
'KC82312_late_spikes': int(period_count[cfg['target_KC']]), 'VA6_late_spikes': int(period_count[cfg['watched_VA6']]),
|
||
'finite': True, 'wall_seconds': result['wall_seconds'], 'biological_ms': 1000})
|
||
ev.to_parquet(exp/'events'/f'{name}.parquet', index=False)
|
||
req.to_parquet(exp/'requests'/f'{name}.parquet', index=False)
|
||
pd.DataFrame(result['samples']).to_parquet(exp/'observations'/f'{name}.parquet', index=False)
|
||
np.savez_compressed(exp/'states'/f'{name}.npz', counts=result['counts'], final_v=result['final_v'], final_g=result['final_g'], trace=result['trace'])
|
||
np.savez_compressed(exp/'states'/f'{name}-watched.npz', state=states, watch=watch)
|
||
if oi == cfg['repeat_odor_index']:
|
||
queues(); again = phase4z.Observer(rt, watch, ff, targets); again.attach()
|
||
try: repeat = phase4v.observed(sim, targets, force, targetkc, vc)
|
||
finally: again.detach()
|
||
row['repeat_exact'] = {'events': ev.equals(again.events()), 'states': np.array_equal(states, again.states),
|
||
'requests': req.equals(pd.DataFrame(again.request_rows)),
|
||
**{key: bool(np.array_equal(result[key], repeat[key])) for key in ('counts', 'final_v', 'final_g', 'trace')}}
|
||
assert all(row['repeat_exact'].values()), row
|
||
regress.append(row); save(exp/'regression'/f'{name}.json', row)
|
||
print(json.dumps(trials[-1]), flush=True)
|
||
# Fixed-event replay is an independent direct-operator check, not the dynamic whole-brain result.
|
||
replay_check(cfg, exp, sim, masks, frozen, baseline0, watch, phase4z, save)
|
||
finally:
|
||
sim.install = original_install; original_install(None)
|
||
assert np.array_equal(rt.weights.cpu().numpy(), frozen)
|
||
assert arrhash(rt.raw_np) == anatomical_hash and arrhash(sim.base.cpu().numpy()) == base_hash
|
||
assert arrhash(sim.input_gpu.cpu().numpy(), sim.output_gpu.cpu().numpy()) == apl_input_hash
|
||
save(exp/'regression/restored.json', {'base': True, 'raw': True, 'APL_matrices': True, 'frozen_effective_weights': True})
|
||
for name, rows in [('trials', trials), ('request-comparison', request_summary), ('population-counts', populations), ('watched-counts', watched)]:
|
||
pd.DataFrame(rows).to_csv(exp/'audit'/f'{name}.csv', index=False)
|
||
save(exp/'protocol/forcing-hashes.json', hashes)
|
||
# Separate reference trials do not include the graded APL overlay, exactly as archived.
|
||
rt.weights.copy_(sim.base)
|
||
reference = []
|
||
for spec in pc['reference_trials']:
|
||
queues(); ref = root/'experiments/phase4p-deterministic-runtime/phase4c-v2/reference'/spec['name']
|
||
ev = pd.read_parquet(ref/'spike-events.parquet')
|
||
expected = {'step': ev.step.to_numpy(), 'neuron_idx': ev.neuron_idx.to_numpy(),
|
||
'counts': np.load(ref/'per-neuron-spike-counts.int32.npy'), 'v': np.load(ref/'final-voltage.float32.npy'), 'g': np.load(ref/'final-g.float32.npy')}
|
||
r = rt.run(np.flatnonzero(rt.pops[spec['population']]), spec['rate_hz'], spec['seed'], spec['duration_ms'])
|
||
eq = {key: bool(np.array_equal(r[key], value)) for key, value in expected.items()}
|
||
assert all(eq.values()), eq
|
||
reference.append({'trial': spec['name'], 'exact': eq, 'wall_seconds': r['metrics']['wall_seconds']})
|
||
original_install(None)
|
||
save(exp/'regression/five-frozen-trials.json', {'pass': True, 'trials': reference, 'original_v1_preserved': True, 'oracle': 'frozen deterministic-v2'})
|
||
save(exp/'performance/summary.json', {'wall_seconds_including_build': time.perf_counter()-start,
|
||
'peak_VRAM_bytes': int(torch.cuda.max_memory_allocated()), 'device': torch.cuda.get_device_name(),
|
||
'diagnostic_overhead_included': True, 'not_runtime_benchmark': True, 'ComfyUI_models_unloaded': False})
|
||
save(exp/'audit/run-complete.json', {'trials': len(trials), 'exact_repeat_trials': 4, 'old_trial_reproductions': 6,
|
||
'frozen_reference_trials': 5, 'all_finite': True, 'candidate': False, 'training': False})
|
||
|
||
|
||
def replay_check(cfg, exp, sim, masks, frozen, baseline, watch, phase4z, save):
|
||
ev, state, req, result = baseline; rt = sim.rt; T = len(state)
|
||
ids = np.flatnonzero(np.isin(rt.indices_np, watch))
|
||
edges = pd.DataFrame({'edge_id': ids, 'source_neuron_idx': sim.sources[ids], 'target_neuron_idx': rt.indices_np[ids], 'weight': frozen[ids]})
|
||
arrivals = ev.merge(edges, left_on='neuron_idx', right_on='source_neuron_idx')
|
||
arrivals['arrival_step'] = arrivals.step + rt.delay_steps
|
||
arrivals = arrivals[arrivals.arrival_step < T].sort_values(['target_neuron_idx', 'arrival_step', 'edge_id'], kind='stable')
|
||
rows = []
|
||
for wi, node in enumerate(watch):
|
||
subset = arrivals[arrivals.target_neuron_idx == node].copy()
|
||
after = np.r_[state[1:, 1, wi], result['final_g'][node]]
|
||
overlay = after.astype(float)-state[:, 5, wi].astype(float)
|
||
request = np.zeros(T, bool); rq = req[req.neuron_idx == node]; request[rq.step.to_numpy(np.int32)] = True
|
||
base_arrival = state[:, 3, wi]
|
||
for name in cfg['conditions']:
|
||
weights = subset.weight.to_numpy(np.float32).copy()
|
||
if name != 'frozen_reference':
|
||
frame = masks[name]; eids = frame.edge_id.to_numpy(np.int64)
|
||
values = blocked_weights(frozen[eids], rt.raw_np[eids], frame.blocked_contact_count.to_numpy(np.int64))
|
||
lookup = pd.Series(values, index=eids)
|
||
mapped = subset.edge_id.map(lookup); keep = mapped.notna().to_numpy()
|
||
weights[keep] = mapped[keep].to_numpy(np.float32)
|
||
subset['diagnostic_weight'] = weights
|
||
a = np.zeros(T, np.float32)
|
||
for step, group in subset.groupby('arrival_step', sort=False):
|
||
a[int(step)] = np.cumsum(group.diagnostic_weight.to_numpy(np.float32), dtype=np.float32)[-1]
|
||
rr = phase4z.replay(a, overlay, request, rt)
|
||
if name == 'frozen_reference':
|
||
exact = np.array_equal(a, base_arrival) and np.array_equal(rr[0], ev.loc[ev.neuron_idx == node, 'step']) and np.array_equal(rr[1], state[:, 4:7, wi])
|
||
assert exact and rr[2] == result['final_v'][node] and rr[3] == result['final_g'][node]
|
||
unchanged = np.array_equal(a, base_arrival)
|
||
if name == 'calyx_PN_KC_contact_block' and node not in sim.kc: assert unchanged
|
||
rows.append({'condition': name, 'neuron_idx': int(node), 'arrival_unchanged': bool(unchanged),
|
||
'arrival_L1_difference': float(np.abs(a.astype(float)-base_arrival).sum()),
|
||
'conditional_late_spikes': int((rr[0] >= round(cfg['late_start_ms']/rt.dt)).sum()),
|
||
'all_other_recorded_events_and_APL_fixed': True, 'whole_brain_causality_claim': False})
|
||
pd.DataFrame(rows).to_csv(exp/'replay/fixed-event-scope.csv', index=False)
|
||
save(exp/'regression/fixed-event-replay.json', {'baseline_exact_neurons': len(watch), 'terminal_nonKC_arrivals_exact': True})
|
||
|
||
|
||
def finalize():
|
||
cfg, root, model, exp, save, digest, immutable_check, seal = load_context()
|
||
assert (exp/'audit/run-complete.json').exists()
|
||
pre = json.loads((exp/'protocol/preregistered.json').read_text())
|
||
assert all(digest(model/n) == h for n, h in pre['source_hashes'].items())
|
||
post = integrity(cfg, root, digest, immutable_check)
|
||
assert post == json.loads((exp/'regression/preflight.json').read_text())
|
||
save(exp/'regression/final-integrity.json', post)
|
||
q = queues(); save(exp/'audit/final-service-health.json', {'ComfyUI_queues': q, 'services_stopped': False})
|
||
trials = pd.read_csv(exp/'audit/trials.csv'); scope = json.loads((exp/'audit/scope-summary.json').read_text())
|
||
requests = pd.read_csv(exp/'audit/request-comparison.csv')
|
||
lines = ['# Phase 4AB — calyx contact-scope diagnostic', '',
|
||
'Status: COMPLETE-SCOPE-COUNTERFACTUAL / NO-DYNAMICS-CANDIDATE.',
|
||
'Diagnostic interventions only; no calibrated biological inhibition, no training, no parameter winner.', '',
|
||
f"Eligible PNs: {scope['eligible_PNs']}/259. CA mask: {scope['calyx_mask_edges']} edges / {scope['calyx_blocked_contacts']} contacts.",
|
||
f"Affected KCs: {scope['calyx_affected_KCs']}. Mixed-ROI affected edges: {scope['mixed_CA_nonCA_edges']}; retained other contacts: {scope['calyx_other_contacts_retained_on_affected_edges']}.",
|
||
f"Broader PN→KC control: {scope['all_PN_KC_edges']} edges / {scope['all_PN_KC_contacts']} contacts. All-fast-PN-output control: {scope['whole_PN_edges']} edges / {scope['whole_PN_contacts']} contacts.", '',
|
||
'| Odor | Condition | Active KC | KC late spikes | KC82312 late | VA6 late |',
|
||
'|---|---|---:|---:|---:|---:|']
|
||
for r in trials.itertuples():
|
||
lines.append(f'| {r.odor_name} | {r.condition} | {r.KC_active} | {r.KC_late_spikes} | {r.KC82312_late_spikes} | {r.VA6_late_spikes} |')
|
||
lines += ['', 'Late means 500–1,000 ms; 120 ms old calibration odor input; one seed, six old odors. No held-out generalization claim.',
|
||
f"Identical scheduled requests across conditions. Delivered request-associated output differences versus reference: {int(requests.actual_output_differences_vs_reference.sum())} (summed over all comparisons). See per-trial table; this is state/refractory feedback, not a changed RNG.", '',
|
||
'## Interpretation limits', '',
|
||
'CA selection uses exact primary_post labels plus matched APL→PN/ROI metadata, not bouton identities or inferred receptors. Unknown ROIs are not reclassified.',
|
||
'Whole-PN control removes fast output, not somatic spikes or the frozen graded APL input. It is not a complete PN knockout.',
|
||
'Reduction/silencing is not restored information or associative memory. No condition becomes a formal model. Fixed-event replay freezes other recorded events/APL; dynamic trials allow natural recurrent feedback.',
|
||
'Local-inhibition motivation: [Prisco et al.](https://doi.org/10.7554/eLife.74172), [Amin et al.](https://doi.org/10.7554/eLife.56954). These sources do not establish the complete-block operator as physiological.', '',
|
||
'## Verification', '',
|
||
'Six original trials reproduced exactly (whole spike events, watched state and final counts/V/g). Four first-odor repeats exact. Five deterministic-v2 frozen reference trials exact. Finite state throughout.',
|
||
'Original 230,239 files and both Phase 4AA manifests rechecked before/after. Base CSR/count/sign, frozen LIF/APL, eligibility-v2 and 14,232 plastic mask preserved.',
|
||
'Timing includes diagnostic CPU/GPU observation overhead and is not a runtime speed benchmark. ComfyUI was not stopped or unloaded.',
|
||
'Top-level acceptance gates remain 5/10 (50%); anatomical diagnosis is not evidence of learning success.']
|
||
(exp/'REPORT.md').write_text('\n'.join(lines)+'\n', encoding='utf-8')
|
||
save(exp/'audit/final-summary.json', {'status': 'COMPLETE-SCOPE-COUNTERFACTUAL / NO-DYNAMICS-CANDIDATE',
|
||
'integrity_pass': True, 'reference_pass': True, 'scope': scope, 'candidate': False, 'training': False,
|
||
'preserved_acceptance_gates': '5/10', 'frozen_baseline_modified': False})
|
||
for folder in (model, exp):
|
||
count = seal(folder); assert manifest_check(folder, digest)['pass']; print('SEALED', folder, count, flush=True)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
parser = argparse.ArgumentParser(); parser.add_argument('task', choices=['prepare', 'run', 'finalize'])
|
||
globals()[parser.parse_args().task]()
|