Digital Drosophila Archive
255 lines
17 KiB
Python
255 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Phase 4N checkpoint-specific whole-brain/credit audit and conservative sealing."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gc
|
|
import json
|
|
import resource
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import torch
|
|
|
|
from phase4h_audit import sha256, verify_checksum, write_json
|
|
from phase4i import Validator, phase4c_regression
|
|
from phase4j import effect
|
|
from phase4k import dataset_checks, seal
|
|
from phase4l import OdorSimulator, pn_drive
|
|
from phase4m import frozen_checks, frozen_hashes, load_odor_inputs
|
|
from phase4n import GROUPS, load_config
|
|
|
|
|
|
def state(exp, seed, pair):
|
|
if pair == 0:
|
|
return np.ones(14232, np.float32)
|
|
with np.load(exp / "acquisition-trajectory" / f"E-seed-{seed}-pairings-{pair}.npz") as f:
|
|
return f["E_bounded"].copy()
|
|
|
|
|
|
def curves(ncfg, exp, csi, replay, identity):
|
|
csi = csi[csi.mechanism == "bounded"]
|
|
peak = json.loads((exp / "specificity-curve/peak.json").read_text())["pairings"]
|
|
onset = None
|
|
if peak is not None:
|
|
p = csi[csi.pairings == peak]
|
|
baseline_csi = p.CSI.mean()
|
|
baseline_other = max(p.shared.mean(), p["B-specific"].mean())
|
|
for pair in ncfg["checkpoints"]:
|
|
if pair <= peak:
|
|
continue
|
|
q = csi[csi.pairings == pair]
|
|
other = max(q.shared.mean(), q["B-specific"].mean())
|
|
if q.CSI.mean() <= .8 * baseline_csi and other >= 1.1 * baseline_other and baseline_csi > 0 and baseline_other > 0:
|
|
onset = pair
|
|
break
|
|
write_json(exp / "specificity-curve/generalization-onset.json", {"status": "GENERALIZATION-ONSET-IDENTIFIED" if onset is not None else "UNIDENTIFIED",
|
|
"pairings": onset, "rule": ncfg["generalization_onset_rule"]})
|
|
groups = pd.read_parquet(exp / "acquisition-trajectory/group-plasticity.parquet")
|
|
groups = groups[groups.mechanism == "bounded"]
|
|
# Dependency-free vector plot; values remain in the accompanying Parquet files.
|
|
parts = ['<svg xmlns="http://www.w3.org/2000/svg" width="900" height="640" viewBox="0 0 900 640">',
|
|
'<rect width="900" height="640" fill="white"/>', '<text x="55" y="30" font-size="17">Phase 4N acquisition and specificity</text>']
|
|
def panel(series, y0, height, title):
|
|
allv = np.concatenate([np.asarray(v, np.float64) for _, v, _ in series]); lo, hi = float(np.min(allv)), float(np.max(allv))
|
|
if hi <= lo: hi = lo + 1.0
|
|
parts.append(f'<text x="55" y="{y0-10}" font-size="13">{title}</text>')
|
|
parts.append(f'<path d="M60 {y0} V{y0+height} H850" fill="none" stroke="black"/>')
|
|
for label, vals, color in series:
|
|
xs = np.asarray(ncfg["checkpoints"], np.float64)
|
|
ys = np.asarray(vals, np.float64)
|
|
coords = ' '.join(f'{60+790*x/100:.1f},{y0+height-20-(height-35)*(y-lo)/(hi-lo):.1f}' for x,y in zip(xs,ys))
|
|
parts.append(f'<polyline points="{coords}" fill="none" stroke="{color}" stroke-width="2"/>')
|
|
parts.append(f'<text x="{625}" y="{y0+18+17*len([s for s in series if s[0] < label])}" font-size="11" fill="{color}">{label}</text>')
|
|
for p,color in ((peak,"green"),(onset,"red")):
|
|
if p is not None:
|
|
x = 60 + 790*p/100
|
|
parts.append(f'<path d="M{x:.1f} {y0} V{y0+height}" stroke="{color}" stroke-dasharray="4,4"/>')
|
|
first = []
|
|
for name,color in zip(GROUPS,("#1261a0","#d4871e","#b72230","#777777")):
|
|
g = groups[groups.group == name].groupby("pairings").weighted_transmission_reduction.mean().reindex(ncfg["checkpoints"]).fillna(0.0)
|
|
first.append((name,g.to_numpy(),color))
|
|
panel(first, 80, 210, "Mean anatomical-weighted E reduction")
|
|
panel([("A-B spike L1",replay.mean_spike_L1_separation.to_numpy(),"#2059a6"),
|
|
("A-B latency ms",replay.mean_latency_separation_ms.to_numpy(),"#b72230")], 370, 210, "Functional MBON effect difference")
|
|
parts.append('</svg>')
|
|
(exp / "specificity-curve/trajectory.svg").write_text('\n'.join(parts), encoding="utf-8")
|
|
return peak, onset
|
|
|
|
|
|
def shared_decomposition(ncfg, exp, identity, sim, v, peak):
|
|
if peak is None:
|
|
write_json(exp / "shared-edge/summary.json", {"status": "NOT-RUN-NO-SPECIFICITY-PEAK"})
|
|
return None
|
|
from phase4m import ReplayEngine, replay_effect
|
|
comp = identity.compartment.astype(str).to_numpy() == str(ncfg["matched_compartment_verbatim"])
|
|
targets = np.unique(identity.loc[comp, "target_neuron_idx"].to_numpy(np.int32))
|
|
engine = ReplayEngine(sim, targets)
|
|
w = identity.anatomical_weight_mv.to_numpy(np.float32)
|
|
groups = identity.identity_group.astype(str).to_numpy()
|
|
rows = []
|
|
from phase4n import load_config
|
|
_, mcfg, _, _ = load_config(Path(__file__).with_name("phase4n.yaml"))
|
|
for seed, es in zip(ncfg["train_seeds"], ncfg["evaluation_seeds"]):
|
|
E = state(exp, seed, peak)
|
|
for odor in ("A", "B"):
|
|
spikes = pd.read_parquet(Path(mcfg["experiment_root"]) / "isolated-replay" / f"KC-spikes-ODOR_{odor}-seed-{es}.parquet")
|
|
baseline = engine.run(spikes, np.zeros(len(E), np.float32), float(mcfg["protocol"]["duration_ms"]))
|
|
for label, included in (("A-only", ("A-specific",)), ("A+shared", ("A-specific", "shared")),
|
|
("A+shared+B", ("A-specific", "shared", "B-specific")), ("all-updated", GROUPS)):
|
|
delta = np.where(np.isin(groups, included), w * (E - 1), 0).astype(np.float32)
|
|
post = engine.run(spikes, delta, float(mcfg["protocol"]["duration_ms"]))
|
|
metrics = replay_effect(baseline, post, sim.dt)
|
|
rows.append({"train_seed": seed, "eval_seed": es, "pairings": peak, "odor": odor, "decomposition": label,
|
|
**{k: val for k, val in metrics.items() if k != "isolated_input_scope"}})
|
|
pd.DataFrame(rows).to_parquet(exp / "shared-edge/isolated-decomposition.parquet", index=False)
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def wholebrain(ncfg, mcfg, cfg, exp, identity, sim, v, peak):
|
|
wanted = sorted(set([10, 20, 50, 75, 100] + ([peak] if peak is not None else [])))
|
|
panel, vectors, selection, mapping = load_odor_inputs(cfg)
|
|
gloms = selection["glomeruli_order"]
|
|
matched = float(np.median(panel.mapped_response_sum))
|
|
pn = {}
|
|
for label in ("A", "B"):
|
|
oid = selection[f"ODOR_{label}"]["odor_id"]
|
|
pn[label] = pn_drive(oid, vectors[oid], mapping, gloms, str(mcfg["protocol"]["input_mode"]), matched, cfg)[:2]
|
|
dan = np.unique(v.confirmed.source_neuron_idx.to_numpy(np.int32))
|
|
group_sources = {name: np.unique(identity.loc[identity.identity_group == name, "source_neuron_idx"].to_numpy(np.int32)) for name in GROUPS}
|
|
w = identity.anatomical_weight_mv.to_numpy(np.float32)
|
|
rows, calls = [], []
|
|
for seed, es in zip(ncfg["train_seeds"], ncfg["evaluation_seeds"]):
|
|
baseline = {label: sim.run_rates(*pn[label], es, float(mcfg["protocol"]["duration_ms"]), collect_kc_spikes=True) for label in pn}
|
|
for pair in wanted:
|
|
delta = w * (state(exp, seed, pair) - 1)
|
|
for label in ("A", "B"):
|
|
post = sim.run_rates(*pn[label], es, float(mcfg["protocol"]["duration_ms"]), delta=delta, collect_kc_spikes=True)
|
|
calls.append(post["wall_seconds"])
|
|
eff = effect(baseline[label], post)
|
|
row = {"train_seed": seed, "eval_seed": es, "pairings": pair, "odor": label,
|
|
"MBON_spike_L1": eff["MBON_spike_l1"], "MBON_g_max_abs_mv": eff["MBON_g_max_abs"],
|
|
"MBON_v_max_abs_mv": eff["MBON_v_max_abs"], "MBON_latency_delta_ms": eff["MBON_first_latency_delta_ms"],
|
|
"downstream_spike_L1": eff["downstream_spike_l1"], "KC_spike_L1": int(np.abs(post["kc_counts"]-baseline[label]["kc_counts"]).sum()),
|
|
"endogenous_DAN_spikes": int(post["counts"][dan].sum()), "baseline_endogenous_DAN_spikes": int(baseline[label]["counts"][dan].sum()),
|
|
"finite": bool(post["finite"]), "peak_vram_bytes": int(post["peak_vram_bytes"]), "wall_seconds": post["wall_seconds"]}
|
|
for group, nodes in group_sources.items():
|
|
row[f"{group}_KC_spikes"] = int(post["counts"][nodes].sum())
|
|
row[f"{group}_KC_newly_active_vs_E1"] = int(np.count_nonzero((post["counts"][nodes] > 0) & (baseline[label]["counts"][nodes] == 0)))
|
|
row["MBON_spikes"] = int(post["mbon_counts"].sum())
|
|
row["other_feedback_spikes"] = int(post["downstream_counts"].sum())
|
|
rows.append(row)
|
|
frame = pd.DataFrame(rows)
|
|
frame.to_parquet(exp / "whole-brain/checkpoint-responses.parquet", index=False)
|
|
write_json(exp / "performance/whole-brain.json", {"calls": len(calls), "mean_wall_seconds": float(np.mean(calls)),
|
|
"sum_wall_seconds": float(np.sum(calls)), "max_peak_vram_bytes": int(frame.peak_vram_bytes.max())})
|
|
return frame
|
|
|
|
|
|
def audit_updates(ncfg, mcfg, exp, identity, history, csi, peak):
|
|
group = identity.set_index("edge_id").identity_group
|
|
hist = history.copy()
|
|
hist["identity_group"] = group.loc[hist.edge_id.to_numpy(np.int64)].to_numpy()
|
|
hist["delta_E_bounded"] = hist.E_bounded_after - hist.E_bounded_before
|
|
hist["DAN_source"] = "external-US-associated"
|
|
hist["matched_compartment"] = True
|
|
hist["endogenous_DAN_used_as_gate"] = False
|
|
hist["local_KC_trace_nonzero"] = hist.eligibility > 0
|
|
hist.to_parquet(exp / "local-eligibility/phase4m-updates-classified.parquet", index=False)
|
|
first = hist[(hist.identity_group == "B-specific") & (hist.delta_E_bounded < -1e-7)].groupby("train_seed").pairing.min().reset_index(name="first_B_specific_update_pairing")
|
|
first.to_parquet(exp / "b-contamination/first-B-updates.parquet", index=False)
|
|
causes = {"direct_A_driven_KC_activity": None, "recurrent_KC_reactivation": None, "MBON_DAN_feedback": None,
|
|
"olfactory_feedback": None, "long_eligibility_trace": None, "endogenous_DAN_teaching_interpretation": 0.0,
|
|
"unresolved_runtime_KC_recruitment_or_trace": 1.0}
|
|
write_json(exp / "b-contamination/provenance.json", {"classification": causes,
|
|
"reason": "Frozen Phase 4M update history records trace at forced matched-US but no event-level causal predecessor or KC spike times. Assigning a recurrent/feedback percentage would be invented.",
|
|
"earliest_B_update_by_seed": first.to_dict("records"), "all_updates_gated_by_external_matched_US": True})
|
|
write_json(exp / "dan-source/phase4m-gate-audit.json", {"external_US_gates": int(history[["train_seed", "pairing"]].drop_duplicates().shape[0]),
|
|
"endogenous_DAN_as_teaching_gate": 0, "policy": "Phase 4M implementation opens plasticity once at imposed 90 ms US, not at endogenous DAN spike times",
|
|
"observed_endogenous_DAN_activity_is_not_equivalent_to_dopamine_concentration": True})
|
|
score = csi[csi.mechanism == "bounded"].groupby("pairings").CSI.mean()
|
|
write_json(exp / "stopping-analysis/summary.json", {"status": "POST-HOC-ONLY-NO-ADAPTIVE-POLICY",
|
|
"peak_CSI_checkpoint": int(score.idxmax()), "peak_CSI": float(score.max()), "functional_peak_checkpoint": peak,
|
|
"online_validated_criterion": False, "heldout_training_validation_required": True})
|
|
|
|
|
|
def controls(exp, identity, ncfg, mcfg):
|
|
rows = []
|
|
for seed in ncfg["train_seeds"]:
|
|
for pair in ncfg["checkpoints"]:
|
|
E = state(exp, seed, pair)
|
|
rows += [{"seed": seed, "pairings": pair, "condition": "A+matched-US", "E_changed": int(np.count_nonzero(E < 1 - 1e-7)), "status": "RUN-RECONSTRUCTED-FROZEN-PHASE4M"},
|
|
{"seed": seed, "pairings": pair, "condition": "A+D0", "E_changed": 0, "status": "EXACT-ZERO-GATE-IDENTITY"},
|
|
{"seed": seed, "pairings": pair, "condition": "frozen-E1", "E_changed": 0, "status": "EXACT-FROZEN-IDENTITY"},
|
|
{"seed": seed, "pairings": pair, "condition": "external-US-only-gate", "E_changed": int(np.count_nonzero(E < 1 - 1e-7)), "status": "IDENTICAL-TO-FROZEN-PHASE4M-RULE"}]
|
|
pd.DataFrame(rows).to_parquet(exp / "controls/trajectory-status.parquet", index=False)
|
|
report = {"A_matched_US": "RUN-PHASE4M-TRAJECTORY-RECONSTRUCTED", "A_D0": "ANALYTIC-EXACT-IDENTITY", "frozen_E1": "ANALYTIC-EXACT-IDENTITY",
|
|
"external_US_only": "RUN-IDENTICAL-TO-PHASE4M", "B_matched_US": "NOT-RUN", "random_US_timing": "NOT-RUN",
|
|
"unrelated_compartment_US": "NOT-RUN", "all_DAN_gate": "NOT-RUN", "full_control_matrix_pass": False}
|
|
write_json(exp / "controls/summary.json", report)
|
|
return report
|
|
|
|
|
|
def integrity(ncfg, mcfg, cfg, base_path, exp, frozen_before):
|
|
gc.collect(); torch.cuda.empty_cache()
|
|
checks = frozen_checks(cfg)
|
|
mroot, mm = Path(mcfg["experiment_root"]), Path(mcfg["model_root"])
|
|
mchecks = {"phase4m_exp": verify_checksum(mroot, mroot / "manifests/checksums.sha256")["passed"],
|
|
"phase4m_model": verify_checksum(mm, mm / "checksums.sha256")["passed"]}
|
|
dataset = dataset_checks(cfg)
|
|
regression = phase4c_regression(cfg, base_path, exp)
|
|
after = frozen_hashes(cfg)
|
|
report = {"all_passed": bool(checks["all_passed"] and all(mchecks.values()) and dataset["all_passed"] and regression["all_passed"] and frozen_before == after),
|
|
"phase4c_to_4l": checks, "phase4m": mchecks, "phase1_to_3b": dataset, "phase4c_reference_regression": regression,
|
|
"before_after_identical": frozen_before == after, "base_csr_unmodified": True, "APL_v2_encoder_mask_unmodified": True}
|
|
write_json(exp / "audit/frozen-integrity.json", report)
|
|
return report
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(); ap.add_argument("--config", type=Path, required=True); args = ap.parse_args()
|
|
ncfg, mcfg, cfg, base_path = load_config(args.config)
|
|
exp = Path(ncfg["experiment_root"])
|
|
identity = pd.read_parquet(exp / "edge-identity/frozen-edge-identity.parquet")
|
|
csi = pd.read_parquet(exp / "specificity-curve/CSI.parquet")
|
|
replay = pd.read_parquet(exp / "specificity-curve/memory-specificity.parquet")
|
|
history = pd.read_parquet(Path(mcfg["experiment_root"]) / "efficacy-overlay/update-history.parquet")
|
|
before = frozen_hashes(cfg)
|
|
peak, onset = curves(ncfg, exp, csi, replay, identity)
|
|
print(json.dumps({"stage": "specificity", "peak": peak, "generalization_onset": onset}), flush=True)
|
|
v = Validator(cfg, base_path, exp); sim = OdorSimulator(v, cfg, exp)
|
|
shared = shared_decomposition(ncfg, exp, identity, sim, v, peak)
|
|
print(json.dumps({"stage": "shared_decomposition", "rows": 0 if shared is None else len(shared)}), flush=True)
|
|
whole = wholebrain(ncfg, mcfg, cfg, exp, identity, sim, v, peak)
|
|
print(json.dumps({"stage": "wholebrain", "rows": len(whole)}), flush=True)
|
|
audit_updates(ncfg, mcfg, exp, identity, history, csi, peak)
|
|
control = controls(exp, identity, ncfg, mcfg)
|
|
del sim, v
|
|
integ = integrity(ncfg, mcfg, cfg, base_path, exp, before)
|
|
ru = resource.getrusage(resource.RUSAGE_SELF)
|
|
perf = {"peak_RAM_KiB": int(ru.ru_maxrss), "edge_audit_rows": int(len(history)),
|
|
"edge_audit_memory_bytes": int(history.memory_usage(deep=True).sum()), "shadow_trace_overhead": "NOT-MEASURED-NOT-RUN",
|
|
"wholebrain": json.loads((exp / "performance/whole-brain.json").read_text()),
|
|
"isolated": json.loads((exp / "performance/isolated.json").read_text())}
|
|
write_json(exp / "performance/summary.json", perf)
|
|
summary = {"phase": "4N", "status": "PARTIAL-MECHANISM-AUDIT-NO-CANDIDATE", "peak": peak, "generalization_onset": onset,
|
|
"edge_groups": identity.identity_group.value_counts().to_dict(), "controls": control, "frozen_integrity": integ["all_passed"],
|
|
"candidate_generated": False, "limitations": ["Endogenous DAN event-level gate overlay not run", "Full matched control trajectories not run", "Runtime causal predecessor and local shadow traces not captured; contamination source percentages unresolved"]}
|
|
write_json(exp / "audit/final-summary.json", summary)
|
|
model = Path(ncfg["model_root"]); model.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(args.config, model / "phase4n.yaml"); shutil.copy2(Path(__file__).resolve(), model / "phase4n_finish.py")
|
|
shutil.copy2(Path(__file__).with_name("phase4n.py"), model / "phase4n.py")
|
|
shutil.copy2(Path(__file__).resolve(), exp / "audit/phase4n_finish.py")
|
|
write_json(model / "VERSION.json", {"phase": "4N", "status": summary["status"], "candidate_generated": False,
|
|
"identity_sha256": sha256(exp / "edge-identity/frozen-edge-identity.parquet"), "config_sha256": sha256(args.config)})
|
|
seal(model, model / "MANIFEST.json", model / "checksums.sha256")
|
|
seal(exp, exp / "manifests/manifest.json", exp / "manifests/checksums.sha256")
|
|
print(json.dumps(summary), flush=True)
|
|
|
|
|
|
if __name__ == "__main__": main()
|