Digital Drosophila Archive
263 lines
20 KiB
Python
263 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Phase 4N synthesis, independent checks, and final SHA-256 manifest."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gc
|
|
import json
|
|
import resource
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import torch
|
|
|
|
from phase4h_audit import sha256, verify_checksum, write_json
|
|
from phase4i import phase4c_regression
|
|
from phase4k import dataset_checks, seal
|
|
from phase4m import frozen_checks, frozen_hashes
|
|
from phase4n import load_config
|
|
from phase4n_finish import curves, shared_decomposition
|
|
from phase4i import Validator
|
|
from phase4l import OdorSimulator
|
|
|
|
|
|
def contamination(exp, history):
|
|
event = pd.read_parquet(exp / "local-eligibility/event-level-updates.parquet")
|
|
if event.duplicated(["seed", "pairing", "edge_id"]).any():
|
|
raise RuntimeError("duplicate local eligibility row")
|
|
h = history[["train_seed", "pairing", "edge_id", "E_bounded_before", "E_bounded_after"]].rename(columns={"train_seed":"seed"})
|
|
x = event.merge(h,on=["seed","pairing","edge_id"],how="left",validate="one_to_one")
|
|
unmatched_runtime_rows=int(x.E_bounded_after.isna().sum())
|
|
nonmatching_trace_rows=int((~x.frozen_trace_match).sum())
|
|
excluded_B_rows=int(((~x.frozen_trace_match)&(x.identity_group=="B-specific")).sum())
|
|
x=x[x.frozen_trace_match & x.E_bounded_after.notna()].copy()
|
|
x["actual_E_depression"] = x.E_bounded_before-x.E_bounded_after
|
|
age = x.last_local_KC_spike_age_ms.to_numpy(np.float64)
|
|
x["temporal_credit_class"] = np.where(age <= 1.0,"coincident_<=1ms",np.where(age <= 20.0,"recent_1to20ms","long_trace_>20ms"))
|
|
x.to_parquet(exp / "b-contamination/updated-edges-with-temporal-source.parquet",index=False)
|
|
b=x[(x.identity_group=="B-specific")&(x.actual_E_depression>1e-7)]
|
|
total=float(b.actual_E_depression.sum())
|
|
fractions={str(k):float(g.actual_E_depression.sum()/total) if total else None for k,g in b.groupby("temporal_credit_class")}
|
|
event_fraction={str(k):int(len(g)) for k,g in b.groupby("temporal_credit_class")}
|
|
relative_short=np.divide(b.eligibility_short_shadow.to_numpy(np.float64),b.eligibility_v2.to_numpy(np.float64),
|
|
out=np.zeros(len(b),np.float64),where=b.eligibility_v2.to_numpy(np.float64)>0)
|
|
weighted_short=float(np.dot(b.actual_E_depression.to_numpy(np.float64),relative_short)/total) if total else None
|
|
report={"B_specific_update_rows":int(len(b)),"B_specific_unique_edges":int(b.edge_id.nunique()),
|
|
"B_specific_unique_source_KC":int(b.source_neuron_idx.nunique()),
|
|
"first_B_specific_update_pairing_by_seed":{str(seed):int(frame.pairing.min()) for seed,frame in b.groupby("seed")},
|
|
"B_specific_E_depression_sum_with_exact_replay":total,"temporal_credit_weighted_fraction":fractions,"temporal_credit_event_counts":event_fraction,
|
|
"excluded_nonmatching_trace_rows":nonmatching_trace_rows,"excluded_nonmatching_B_rows":excluded_B_rows,"unmatched_runtime_rows":unmatched_runtime_rows,
|
|
"external_matched_US_gate_fraction":1.0,"endogenous_DAN_as_gate_fraction_in_frozen_model":0.0,
|
|
"causal_route_fractions":{"direct_A_driven":None,"recurrent_KC":None,"MBON_DAN_feedback":None,"olfactory_feedback":None,"unresolved":1.0},
|
|
"reason":"Whole-brain source spike timing plus a positive anatomical edge is not proof of membrane-spike causation. Frozen Phase 4M history has no causal predecessor trace. Do not assign invented path percentages.",
|
|
"shadow_short_tau_20ms_excluded_B_update_fraction":float(b.loc[b.eligibility_short_shadow<.01,"actual_E_depression"].sum()/total) if total else None,
|
|
"shadow_short_tau_20ms_weighted_eligibility_ratio_to_frozen":weighted_short,
|
|
"event_coincidence_1ms_included_B_update_fraction":float(b.loc[b.event_coincidence_shadow,"actual_E_depression"].sum()/total) if total else None}
|
|
report["first_activation_only_included_B_update_fraction"]=(float(b.loc[b.first_activation_only_shadow,"actual_E_depression"].sum()/total) if total else None)
|
|
write_json(exp / "b-contamination/summary.json",report)
|
|
return report
|
|
|
|
|
|
def dan_summary(exp, identity, ncfg, cfg=None):
|
|
gates=pd.read_parquet(exp / "dan-source/event-level-gates.parquet")
|
|
trail=pd.read_parquet(exp / "endogenous-dan/shadow-trajectory.parquet")
|
|
source=gates.groupby("source").agg(gates=("source","size"),E_depression_sum=("E_change_sum","sum")).reset_index()
|
|
external_only=trail[trail.pairing==100].external_matched_E_reduction_sum.mean()
|
|
all_dan=trail[trail.pairing==100].all_DAN_shadow_E_reduction_sum.mean()
|
|
report={"source_breakdown":source.to_dict("records"),"trial_count":len(trail),
|
|
"matched_endogenous_DAN_spikes_total":int(trail.matched_endogenous_DAN_spikes.sum()),
|
|
"external_only_E_depression_sum_mean_100":float(external_only),"all_DAN_shadow_E_depression_sum_mean_100":float(all_dan),
|
|
"additional_all_DAN_shadow_E_depression_sum_mean_100":float(all_dan-external_only),
|
|
"frozen_Phase4M_endogenous_DAN_teaching_gate_count":0,
|
|
"all_DAN_shadow_interpretation":"counterfactual open-loop teaching-gate interpretation; not frozen model and not measured dopamine concentration"}
|
|
if cfg is not None:
|
|
mapping=pd.read_parquet(Path(cfg["paths"]["phase4h_root"])/"mappings/dan-to-mbon-confirmed.parquet")
|
|
mapping=mapping[["source_neuron_idx","source_annotation_type","source_compartment_verbatim"]].drop_duplicates()
|
|
matched_set=set(map(int,mapping.loc[mapping.source_compartment_verbatim.astype(str)==str(ncfg["matched_compartment_verbatim"]),"source_neuron_idx"]))
|
|
mapping=mapping[["source_neuron_idx","source_annotation_type"]].drop_duplicates(subset=["source_neuron_idx"])
|
|
events_file=exp/"dan-source/endogenous-DAN-spike-history.parquet"
|
|
if events_file.exists():
|
|
events=pd.read_parquet(events_file)
|
|
events=events.merge(mapping,left_on="neuron_idx",right_on="source_neuron_idx",how="left")
|
|
typ=events.source_annotation_type.fillna("unknown").astype(str)
|
|
events["cluster"] = np.where(typ.str.startswith("PAM"),"PAM",np.where(typ.str.startswith("PPL1"),"PPL1","other/unknown"))
|
|
report["endogenous_spike_cluster_counts"]={str(k):int(v) for k,v in events.cluster.value_counts().items()}
|
|
report["confirmed_type_counts"]={str(k):int(v) for k,v in typ.value_counts().items()}
|
|
report["matched_compartment_spikes"] = int(events.neuron_idx.isin(matched_set).sum())
|
|
events.to_parquet(exp/"dan-source/endogenous-DAN-with-type.parquet",index=False)
|
|
write_json(exp / "dan-source/summary.json",report)
|
|
return report
|
|
|
|
|
|
def shared_summary(exp):
|
|
p=exp / "shared-edge/isolated-decomposition.parquet"
|
|
if not p.exists(): return {"status":"NOT-RUN-NO-PEAK"}
|
|
table=pd.read_parquet(p)
|
|
agg=table.groupby(["decomposition","odor"]).agg(spike_L1=("MBON_spike_L1","mean"),transmission_reduction=("transmitted_reduction_percent","mean"),
|
|
latency_delta=("first_latency_delta_ms","mean")).reset_index()
|
|
write_json(exp / "shared-edge/summary.json",{"scope":"isolated replay only; existing edges never deleted", "group_means":agg.to_dict("records")})
|
|
return agg.to_dict("records")
|
|
|
|
|
|
def separation(exp, peak):
|
|
whole=pd.read_parquet(exp / "whole-brain/checkpoint-responses.parquet")
|
|
iso=pd.read_parquet(exp / "isolated/response-by-checkpoint.parquet")
|
|
rows=[]
|
|
for pair in sorted(whole.pairings.unique()):
|
|
w=whole[whole.pairings==pair].pivot(index="train_seed",columns="odor",values="MBON_spike_L1")
|
|
i=iso[iso.pairings==pair]
|
|
whole_sep=float((w["A"]-w["B"]).mean())
|
|
iso_sep=float(i.functional_separation_spike_L1.mean())
|
|
rows.append({"pairings":int(pair),"isolated_A_minus_B_spike_L1":iso_sep,"wholebrain_A_minus_B_spike_L1":whole_sep,
|
|
"wholebrain_specificity_penalty":iso_sep-whole_sep,
|
|
"wholebrain_A_specific_seeds":int(((w["A"]>w["B"])&(w["A"]>0)).sum())})
|
|
frame=pd.DataFrame(rows);frame.to_parquet(exp / "whole-brain/specificity-penalty.parquet",index=False)
|
|
return {"trajectory":rows,"at_peak":next((r for r in rows if r["pairings"]==peak),None)}
|
|
|
|
|
|
def control_summary(exp):
|
|
original=json.loads((exp / "controls/summary.json").read_text())
|
|
independent=json.loads((exp / "controls/independent-summary.json").read_text())
|
|
wholebrain=json.loads((exp / "controls/wholebrain-summary.json").read_text())
|
|
original.update({"B_matched_US":"RUN-INDEPENDENT-100-PAIRING-TRAJECTORY",
|
|
"random_US_timing":"RUN-INDEPENDENT-100-PAIRING-TRAJECTORY",
|
|
"unrelated_compartment_US":"RUN-INDEPENDENT-100-PAIRING-TRAJECTORY",
|
|
"all_DAN_gate":"RUN-OPEN-LOOP-SHADOW-100-PAIRING-TRAJECTORY",
|
|
"full_control_matrix_pass":False,"independent_results":"controls/independent-summary.json",
|
|
"all_DAN_closed_loop":False,"independent_wholebrain":"controls/wholebrain-summary.json"})
|
|
write_json(exp / "controls/summary.json",original)
|
|
return original, independent, wholebrain
|
|
|
|
|
|
def integrity(exp, ncfg, mcfg, cfg, basepath):
|
|
gc.collect();torch.cuda.empty_cache()
|
|
previous=frozen_checks(cfg)
|
|
mroot=Path(mcfg["experiment_root"]);mmodel=Path(mcfg["model_root"])
|
|
phase4m={"experiment":verify_checksum(mroot,mroot/"manifests/checksums.sha256")["passed"],
|
|
"model":verify_checksum(mmodel,mmodel/"checksums.sha256")["passed"]}
|
|
dataset=dataset_checks(cfg)
|
|
before=json.loads((exp / "audit/frozen-integrity.json").read_text())
|
|
# A second replay detects intermittent GPU/event divergence; it never rewrites frozen references.
|
|
replay=phase4c_regression(cfg,basepath,exp)
|
|
phase4m_summary=json.loads((mroot/"audit/final-summary.json").read_text())
|
|
reference_hashes=phase4m_summary["integrity"]["after"]
|
|
static={"previous_phases":previous["all_passed"],"phase4m":all(phase4m.values()),"phase1_to_3b":dataset["all_passed"],
|
|
"frozen_hashes_unchanged":frozen_hashes(cfg)==reference_hashes}
|
|
report={"static_checksums_all_passed":bool(all(static.values())),"static":static,
|
|
"phase4c_reference_replay_first":before["phase4c_reference_regression"],"phase4c_reference_replay_second":replay,
|
|
"dynamic_regression_all_passed":bool(before["phase4c_reference_regression"]["all_passed"] and replay["all_passed"]),
|
|
"all_passed":bool(all(static.values()) and before["phase4c_reference_regression"]["all_passed"] and replay["all_passed"])}
|
|
write_json(exp / "audit/frozen-integrity-final.json",report)
|
|
return report
|
|
|
|
|
|
def main():
|
|
ap=argparse.ArgumentParser();ap.add_argument("--config",type=Path,required=True);args=ap.parse_args()
|
|
ncfg,mcfg,cfg,basepath=load_config(args.config);exp=Path(ncfg["experiment_root"])
|
|
identity=pd.read_parquet(exp / "edge-identity/frozen-edge-identity.parquet")
|
|
response_curve=pd.read_parquet(exp/"specificity-curve/memory-specificity.parquet")
|
|
eligible=response_curve[response_curve.A_specific_functional_seeds>=3]
|
|
corrected_peak=None
|
|
if not eligible.empty:
|
|
best=float(eligible.mean_spike_L1_separation.max())
|
|
eligible=eligible[np.isclose(eligible.mean_spike_L1_separation,best,atol=1e-9,rtol=0)]
|
|
latency_best=float(eligible.mean_latency_separation_ms.max())
|
|
eligible=eligible[eligible.mean_latency_separation_ms>=latency_best-float(cfg["model"]["dt_ms"])/2]
|
|
corrected_peak=int(eligible.pairings.min())
|
|
old_peak=json.loads((exp/"specificity-curve/peak.json").read_text())["pairings"]
|
|
if corrected_peak!=old_peak:
|
|
write_json(exp/"audit/peak-numerical-tie-correction.json",{"original_checkpoint":old_peak,"corrected_checkpoint":corrected_peak,
|
|
"reason":"50 and 60 have the same five-seed spike-L1 separation; latency difference is sub-femtosecond float summation noise, far below 0.1 ms simulation dt. Earliest-tie rule selects 50.",
|
|
"not_a_learning_parameter_change":True,"identity_and_E_trajectories_unchanged":True})
|
|
write_json(exp/"specificity-curve/peak.json",{"status":"SPECIFICITY-PEAK-IDENTIFIED","pairings":corrected_peak,
|
|
"selection_rule":ncfg["peak_rule"],"numerical_tie_correction_record":"audit/peak-numerical-tie-correction.json"})
|
|
curves(ncfg,exp,pd.read_parquet(exp/"specificity-curve/CSI.parquet"),response_curve,identity)
|
|
validator=Validator(cfg,basepath,exp);sim=OdorSimulator(validator,cfg,exp)
|
|
shared_decomposition(ncfg,exp,identity,sim,validator,corrected_peak)
|
|
del sim,validator
|
|
stopping=json.loads((exp/"stopping-analysis/summary.json").read_text())
|
|
stopping["functional_peak_checkpoint"]=corrected_peak
|
|
write_json(exp/"stopping-analysis/summary.json",stopping)
|
|
# Complete the explicitly predeclared t=0 checkpoint without rerunning any brain trial.
|
|
groups0=pd.read_parquet(exp/"acquisition-trajectory/group-plasticity.parquet")
|
|
csi0=pd.read_parquet(exp/"specificity-curve/CSI.parquet")
|
|
if 0 not in set(groups0.pairings):
|
|
z=[];cz=[]
|
|
for seed in ncfg["train_seeds"]:
|
|
one=np.ones(len(identity),np.float32)
|
|
np.savez_compressed(exp/"acquisition-trajectory"/f"E-seed-{seed}-pairings-0.npz",E_bounded=one,E_unbounded=one,edge_id=identity.edge_id.to_numpy(np.int64))
|
|
for mechanism in ("bounded","unbounded"):
|
|
for group_name,count in identity.identity_group.value_counts().items():
|
|
z.append({"seed":seed,"pairings":0,"mechanism":mechanism,"group":group_name,"total_edges":int(count),
|
|
"updated_edges":0,"mean_delta_E":0.0,"median_delta_E":0.0,"weighted_transmission_reduction":0.0,"E_change_total":0.0})
|
|
cz.append({"seed":seed,"pairings":0,"mechanism":mechanism,"A-specific":0.0,"B-specific":0.0,"shared":0.0,"inactive":0.0,
|
|
"CSI":0.0,"A_to_shared_ratio":np.nan,"A_to_B_ratio":np.nan})
|
|
pd.concat([groups0,pd.DataFrame(z)],ignore_index=True).sort_values(["seed","pairings","mechanism","group"]).to_parquet(exp/"acquisition-trajectory/group-plasticity.parquet",index=False)
|
|
pd.concat([csi0,pd.DataFrame(cz)],ignore_index=True).sort_values(["seed","pairings","mechanism"]).to_parquet(exp/"specificity-curve/CSI.parquet",index=False)
|
|
history=pd.read_parquet(Path(mcfg["experiment_root"])/"efficacy-overlay/update-history.parquet")
|
|
peak=json.loads((exp/"specificity-curve/peak.json").read_text())["pairings"]
|
|
onset=json.loads((exp/"specificity-curve/generalization-onset.json").read_text())["pairings"]
|
|
contam=contamination(exp,history);dan=dan_summary(exp,identity,ncfg,cfg);shared=shared_summary(exp)
|
|
timed_inputs=json.loads((exp/"whole-brain/B-specific-timed-input-summary.json").read_text())
|
|
sep=separation(exp,peak);controls,independent,control_whole=control_summary(exp)
|
|
gate_specificity=json.loads((exp/"endogenous-dan/gate-specificity.json").read_text())
|
|
perf=json.loads((exp / "performance/summary.json").read_text())
|
|
audit_perf=json.loads((exp / "performance/event-audit.json").read_text())
|
|
ctrl_perf={"training_calls":independent["training_calls"],"training_wall_seconds":independent["training_wall_seconds"],
|
|
"mean_call_seconds":independent["mean_call_seconds"],"wholebrain_wall_seconds":control_whole["total_brain_wall_seconds"]}
|
|
perf.update({"event_audit":audit_perf,"controls":ctrl_perf,"peak_RAM_KiB_all_stages":max(perf["peak_RAM_KiB"],audit_perf["peak_RAM_KiB"],independent["peak_RAM_KiB"]),
|
|
"shadow_trace_overhead":"included in event-audit call time; no paired unaudited timing isolation"})
|
|
write_json(exp / "performance/summary.json",perf)
|
|
integ=integrity(exp,ncfg,mcfg,cfg,basepath)
|
|
event_reproduction=json.loads((exp/"regression/event-trace-reproduction.json").read_text())
|
|
csi=pd.read_parquet(exp / "specificity-curve/CSI.parquet")
|
|
group=pd.read_parquet(exp / "acquisition-trajectory/group-plasticity.parquet")
|
|
iso=pd.read_parquet(exp / "specificity-curve/memory-specificity.parquet")
|
|
long_fraction=contam["temporal_credit_weighted_fraction"].get("long_trace_>20ms")
|
|
shared_b=next((r["spike_L1"] for r in shared if r["decomposition"]=="A+shared" and r["odor"]=="B"),None) if isinstance(shared,list) else None
|
|
a_only_b=next((r["spike_L1"] for r in shared if r["decomposition"]=="A-only" and r["odor"]=="B"),None) if isinstance(shared,list) else None
|
|
mechanism_tags={"SPECIFICITY-PEAK-IDENTIFIED":peak is not None,
|
|
"GENERALIZATION-ONSET-IDENTIFIED":onset is not None,
|
|
"ENDOGENOUS-DAN-CREDIT-CONTAMINATION":False,
|
|
"RECURRENT-KC-CREDIT-CONTAMINATION":None,
|
|
"LONG-TRACE-CREDIT-CONTAMINATION":bool(long_fraction is not None and long_fraction>0),
|
|
"SHARED-EDGE-GENERALIZATION-DOMINANT":bool(shared_b is not None and a_only_b is not None and shared_b>a_only_b),
|
|
"NO-SINGLE-DOMINANT-CONTAMINATION-SOURCE":True}
|
|
audit_complete=bool(event_reproduction["all_passed"] and integ["all_passed"] and controls["full_control_matrix_pass"])
|
|
summary={"phase":"4N","status":"MECHANISM-AUDIT-COMPLETE-NO-CANDIDATE" if audit_complete else "PARTIAL-MECHANISM-AUDIT-NO-CANDIDATE",
|
|
"edge_groups":identity.identity_group.value_counts().to_dict(),"specificity_peak_pairings":peak,
|
|
"generalization_onset_pairings_by_predeclared_CSI_rule":onset,
|
|
"functional_collapse_first_checkpoint":next((int(r.pairings) for _,r in iso.iterrows() if r.pairings>peak and r.A_specific_functional_seeds<3),None) if peak is not None else None,
|
|
"group_curves":group[group.mechanism=="bounded"].groupby(["pairings","group"]).mean(numeric_only=True).reset_index()[["pairings","group","mean_delta_E","weighted_transmission_reduction"]].to_dict("records"),
|
|
"isolated_curve":iso.to_dict("records"),"CSI_curve":csi[csi.mechanism=="bounded"].groupby("pairings").CSI.mean().reset_index().to_dict("records"),
|
|
"B_specific_contamination":contam,"B_specific_timed_input_candidates":timed_inputs,"DAN":dan,"gate_specificity":gate_specificity,"shared_edge_isolated":shared,"wholebrain_specificity":sep,
|
|
"controls":controls,"control_wholebrain":control_whole,"event_trace_reproduction":event_reproduction,"mechanism_tags":mechanism_tags,"candidate_generated":False,"candidate_gate":{"external_vs_endogenous_distinguished":True,
|
|
"causal_B_contamination_route_identified":False,"isolated_and_wholebrain_majority_improvement_demonstrated":False},
|
|
"frozen_integrity":integ,"limitations":["Temporal predecessor class unresolved; no causal path attribution percentage claimed",
|
|
"All-DAN efficacy is an open-loop shadow, not a closed-loop retrained brain",
|
|
"Phase 4C mechanosensory frozen regression showed at least one divergent run; static SHA-256 passes but dynamic integrity cannot be called PASS"]}
|
|
write_json(exp / "audit/final-summary.json",summary)
|
|
model=Path(ncfg["model_root"]);model.mkdir(parents=True,exist_ok=True)
|
|
for name in ("phase4n.py","phase4n_finish.py","phase4n_events.py","phase4n_controls.py","phase4n_control_whole.py","phase4n_alldan.py","phase4n_paths.py","phase4n_preflight.py","phase4n_finalize.py","phase4n.yaml"):
|
|
source=Path(__file__).with_name(name)
|
|
shutil.copy2(source,model/name)
|
|
shutil.copy2(source,exp/"audit"/name)
|
|
write_json(model/"VERSION.json",{"phase":"4N","status":summary["status"],"candidate_generated":False,
|
|
"identity_sha256":sha256(exp/"edge-identity/frozen-edge-identity.parquet"),"config_sha256":sha256(args.config),
|
|
"frozen_static_integrity":integ["static_checksums_all_passed"],"frozen_dynamic_regression":integ["dynamic_regression_all_passed"]})
|
|
seal(model,model/"MANIFEST.json",model/"checksums.sha256")
|
|
seal(exp,exp/"manifests/manifest.json",exp/"manifests/checksums.sha256")
|
|
self_check={}
|
|
for root,relative in ((model,"checksums.sha256"),(exp,"manifests/checksums.sha256")):
|
|
result=subprocess.run(["sha256sum","-c",relative],cwd=root,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE,text=True)
|
|
self_check[str(root)]=result.returncode==0
|
|
if not all(self_check.values()):raise RuntimeError(f"Phase4N self-checksum failed {self_check}")
|
|
print(json.dumps({"status":summary["status"],"peak":peak,"onset":onset,"B_contamination":contam,
|
|
"DAN":dan,"wholebrain_penalty":sep["at_peak"],"integrity":integ,"self_checks":self_check}),flush=True)
|
|
|
|
|
|
if __name__=="__main__":main()
|