Archive MaleCNS research sources, provenance, bibliography and immutable data inventory
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"file_count": 11,
|
||||
"files": [
|
||||
{
|
||||
"bytes": 353,
|
||||
"path": "VERSION.json",
|
||||
"sha256": "6d5cf119bde4963fe78b1407ff1452e42d0cf70168661d60cc52bdf5a61b8667"
|
||||
},
|
||||
{
|
||||
"bytes": 13925,
|
||||
"path": "phase4n.py",
|
||||
"sha256": "dbbb49447aecbc862d3f54bba9de41f6b8ab90a6afb1ef43a686149fce832d13"
|
||||
},
|
||||
{
|
||||
"bytes": 1712,
|
||||
"path": "phase4n.yaml",
|
||||
"sha256": "6e1b85fc923c7c969df0a7002557256cb719106f8a028c2c506c5ccfe0edd2b2"
|
||||
},
|
||||
{
|
||||
"bytes": 6158,
|
||||
"path": "phase4n_alldan.py",
|
||||
"sha256": "df12477905d7da70e0e337a8d4408cc247135e7876ac7b197e1997f2c4611a54"
|
||||
},
|
||||
{
|
||||
"bytes": 3237,
|
||||
"path": "phase4n_control_whole.py",
|
||||
"sha256": "560ebffba3e8c3457827a0a682476d125a2bbef81958f5b621d32c1203aada1e"
|
||||
},
|
||||
{
|
||||
"bytes": 7198,
|
||||
"path": "phase4n_controls.py",
|
||||
"sha256": "229e48aa63a725e65202dfbabec58d9a092822482c6d0b461ab5a72ee323a901"
|
||||
},
|
||||
{
|
||||
"bytes": 17592,
|
||||
"path": "phase4n_events.py",
|
||||
"sha256": "1fd10326d9e9e4dd6e78b71a6a7c784058c51119a3a5523d4fe50550cd23bbda"
|
||||
},
|
||||
{
|
||||
"bytes": 20496,
|
||||
"path": "phase4n_finalize.py",
|
||||
"sha256": "2ed9b2a7282f0319238646d2c9be0533650a0146292c2de8201af27bb9cb2e5c"
|
||||
},
|
||||
{
|
||||
"bytes": 16993,
|
||||
"path": "phase4n_finish.py",
|
||||
"sha256": "690d33c18d6438d11050ba1fdee9a8015c6bc25b7c85f78cb56f4d06820e35a7"
|
||||
},
|
||||
{
|
||||
"bytes": 6839,
|
||||
"path": "phase4n_paths.py",
|
||||
"sha256": "41554ae9b712b1710a3e8559e86e01e2aa5d95ca0f88fcaf93a4a5baa8dc3877"
|
||||
},
|
||||
{
|
||||
"bytes": 757,
|
||||
"path": "phase4n_preflight.py",
|
||||
"sha256": "aa339208407f4dd00282201ce362ea06fa042f3115e57483670196c6b3c3d642"
|
||||
}
|
||||
],
|
||||
"generated_utc": "2026-09-21T10:57:20.327961+00:00",
|
||||
"root": "/data/digital-drosophila/models/male-cns-lif/credit-assignment-v1",
|
||||
"total_bytes": 95260
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"candidate_generated": false,
|
||||
"config_sha256": "6e1b85fc923c7c969df0a7002557256cb719106f8a028c2c506c5ccfe0edd2b2",
|
||||
"frozen_dynamic_regression": false,
|
||||
"frozen_static_integrity": true,
|
||||
"identity_sha256": "aa63eacdfe4f63751a3bdf1efe3071a69d07babcd64b479aa91c9f3b512cd45d",
|
||||
"phase": "4N",
|
||||
"status": "PARTIAL-MECHANISM-AUDIT-NO-CANDIDATE"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
8782baf95df0bc5d4a17e1ecca99fd7f3af55a25a7619b8ae5028d22c70b0417 MANIFEST.json
|
||||
6d5cf119bde4963fe78b1407ff1452e42d0cf70168661d60cc52bdf5a61b8667 VERSION.json
|
||||
dbbb49447aecbc862d3f54bba9de41f6b8ab90a6afb1ef43a686149fce832d13 phase4n.py
|
||||
6e1b85fc923c7c969df0a7002557256cb719106f8a028c2c506c5ccfe0edd2b2 phase4n.yaml
|
||||
df12477905d7da70e0e337a8d4408cc247135e7876ac7b197e1997f2c4611a54 phase4n_alldan.py
|
||||
560ebffba3e8c3457827a0a682476d125a2bbef81958f5b621d32c1203aada1e phase4n_control_whole.py
|
||||
229e48aa63a725e65202dfbabec58d9a092822482c6d0b461ab5a72ee323a901 phase4n_controls.py
|
||||
1fd10326d9e9e4dd6e78b71a6a7c784058c51119a3a5523d4fe50550cd23bbda phase4n_events.py
|
||||
2ed9b2a7282f0319238646d2c9be0533650a0146292c2de8201af27bb9cb2e5c phase4n_finalize.py
|
||||
690d33c18d6438d11050ba1fdee9a8015c6bc25b7c85f78cb56f4d06820e35a7 phase4n_finish.py
|
||||
41554ae9b712b1710a3e8559e86e01e2aa5d95ca0f88fcaf93a4a5baa8dc3877 phase4n_paths.py
|
||||
aa339208407f4dd00282201ce362ea06fa042f3115e57483670196c6b3c3d642 phase4n_preflight.py
|
||||
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4N read-only reconstruction of frozen Phase 4M acquisition and replay."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import yaml
|
||||
|
||||
from phase4h_audit import sha256, verify_checksum, write_json
|
||||
from phase4i import Validator
|
||||
from phase4k import seal
|
||||
from phase4m import ReplayEngine, compare_A_B, frozen_checks, frozen_hashes, load_odor_inputs, replay_effect
|
||||
from phase4l import OdorSimulator, pn_drive
|
||||
from phase4j import effect
|
||||
|
||||
|
||||
GROUPS = ("A-specific", "shared", "B-specific", "inactive")
|
||||
|
||||
|
||||
def load_config(path: Path):
|
||||
ncfg = yaml.safe_load(path.read_text())
|
||||
mpath = Path(ncfg["frozen_phase4m_config"])
|
||||
mcfg = yaml.safe_load(mpath.read_text())
|
||||
lpath = Path(mcfg["frozen_phase4l_config"])
|
||||
cfg = yaml.safe_load(lpath.read_text())
|
||||
cfg["paths"]["phase4l_root"] = cfg["paths"]["experiment_root"]
|
||||
cfg["paths"]["phase4l_model_root"] = str(lpath.parent)
|
||||
return ncfg, mcfg, cfg, lpath
|
||||
|
||||
|
||||
def prepare(ncfg, mcfg, cfg, exp: Path):
|
||||
if exp.exists() and (exp / "edge-identity/definition.json").exists():
|
||||
raise RuntimeError("Phase 4N identity/definition already frozen; refusing to overwrite")
|
||||
previous = frozen_checks(cfg)
|
||||
mroot = Path(mcfg["experiment_root"])
|
||||
mmodel = Path(mcfg["model_root"])
|
||||
for root, rel in ((mroot, "manifests/checksums.sha256"), (mmodel, "checksums.sha256")):
|
||||
check = verify_checksum(root, root / rel)
|
||||
if not check["passed"]:
|
||||
raise RuntimeError(f"Phase 4M frozen checksum failed: {root}")
|
||||
for sub in ("edge-identity", "acquisition-trajectory", "specificity-curve", "dan-source", "endogenous-dan", "local-eligibility", "shared-edge", "b-contamination", "isolated", "whole-brain", "controls", "stopping-analysis", "performance", "regression", "audit", "manifests"):
|
||||
(exp / sub).mkdir(parents=True, exist_ok=True)
|
||||
src = Path(cfg["paths"]["phase4l_root"]) / "eligible-edges/edge-identity.parquet"
|
||||
identity = pd.read_parquet(src)
|
||||
required = {"edge_id", "source_neuron_idx", "source_body_id", "target_neuron_idx", "target_body_id", "raw_synapse_count", "target_compartment_verbatim", "odor_eligibility_category"}
|
||||
if not required.issubset(identity.columns) or len(identity) != 14232:
|
||||
raise RuntimeError("Phase 4L identity schema/size mismatch")
|
||||
if set(identity.odor_eligibility_category.astype(str)) != set(GROUPS):
|
||||
raise RuntimeError("identity groups differ from four frozen groups")
|
||||
identity["anatomical_weight_mv"] = identity.raw_synapse_count.astype(np.float32) * np.float32(cfg["model"]["w_syn_mv_per_anatomical_synapse"])
|
||||
identity["initial_E"] = np.float32(1.0)
|
||||
identity.rename(columns={"odor_eligibility_category": "identity_group", "target_compartment_verbatim": "compartment"}, inplace=True)
|
||||
out = exp / "edge-identity/frozen-edge-identity.parquet"
|
||||
identity.to_parquet(out, index=False)
|
||||
definition = {"source": str(src), "source_sha256": sha256(src), "source_semantics": ncfg["edge_identity_source"],
|
||||
"group_counts": identity.identity_group.value_counts().to_dict(), "edge_count": len(identity),
|
||||
"checkpoint_rule": ncfg["checkpoints"], "CSI": ncfg["credit_specificity_index"],
|
||||
"functional_separation": ncfg["functional_separation"], "peak_rule": ncfg["peak_rule"],
|
||||
"generalization_onset_rule": ncfg["generalization_onset_rule"], "frozen_before_learning": True,
|
||||
"phase4m_experiment_checksum_sha256": sha256(mroot / "manifests/checksums.sha256"),
|
||||
"phase4m_model_checksum_sha256": sha256(mmodel / "checksums.sha256"), "prior_checksum_passed": previous["all_passed"]}
|
||||
write_json(exp / "edge-identity/definition.json", definition)
|
||||
shutil.copy2(Path(__file__).resolve(), exp / "audit/phase4n.py")
|
||||
return identity, definition
|
||||
|
||||
|
||||
def reconstruct(ncfg, mcfg, cfg, exp: Path, identity: pd.DataFrame):
|
||||
hist = pd.read_parquet(Path(mcfg["experiment_root"]) / "efficacy-overlay/update-history.parquet")
|
||||
required = {"train_seed", "pairing", "edge_id", "E_bounded_before", "E_bounded_after", "E_multiplicative_before", "E_multiplicative_after", "eligibility"}
|
||||
if not required.issubset(hist.columns):
|
||||
raise RuntimeError("Phase 4M update history schema mismatch")
|
||||
edge_ids = identity.edge_id.to_numpy(np.int64)
|
||||
lookup = pd.Series(np.arange(len(edge_ids), dtype=np.int32), index=edge_ids)
|
||||
checkpoints = {}
|
||||
verify = []
|
||||
points = list(map(int, ncfg["checkpoints"]))
|
||||
all_rows = []
|
||||
for seed in ncfg["train_seeds"]:
|
||||
by = hist[hist.train_seed == seed]
|
||||
E = np.ones(len(edge_ids), np.float32)
|
||||
U = np.ones(len(edge_ids), np.float32)
|
||||
checkpoints[seed] = {0: {"bounded": E.copy(), "unbounded": U.copy()}}
|
||||
np.savez_compressed(exp / "acquisition-trajectory" / f"E-seed-{seed}-pairings-0.npz", E_bounded=E, E_unbounded=U, edge_id=edge_ids)
|
||||
for mechanism in ("bounded", "unbounded"):
|
||||
for group in GROUPS:
|
||||
mask = identity.identity_group.to_numpy() == group
|
||||
all_rows.append({"seed": seed, "pairings": 0, "mechanism": mechanism, "group": group,
|
||||
"total_edges": int(mask.sum()), "updated_edges": 0, "mean_delta_E": 0.0, "median_delta_E": 0.0,
|
||||
"weighted_transmission_reduction": 0.0, "E_change_total": 0.0})
|
||||
for pairing in range(1, max(points) + 1):
|
||||
item = by[by.pairing == pairing]
|
||||
local = lookup.loc[item.edge_id.to_numpy(np.int64)].to_numpy(np.int32)
|
||||
before_error = max(float(np.max(np.abs(E[local] - item.E_bounded_before.to_numpy(np.float32)))) if len(local) else 0.0,
|
||||
float(np.max(np.abs(U[local] - item.E_multiplicative_before.to_numpy(np.float32)))) if len(local) else 0.0)
|
||||
if before_error > 1e-6:
|
||||
raise RuntimeError(f"Phase 4M event history not reconstructible seed={seed} pairing={pairing}: {before_error}")
|
||||
E[local] = item.E_bounded_after.to_numpy(np.float32)
|
||||
U[local] = item.E_multiplicative_after.to_numpy(np.float32)
|
||||
if pairing not in points:
|
||||
continue
|
||||
checkpoints[seed][pairing] = {"bounded": E.copy(), "unbounded": U.copy()}
|
||||
np.savez_compressed(exp / "acquisition-trajectory" / f"E-seed-{seed}-pairings-{pairing}.npz", E_bounded=E, E_unbounded=U, edge_id=edge_ids)
|
||||
old = Path(mcfg["experiment_root"]) / "efficacy-overlay" / f"E-seed-{seed}-pairings-{pairing}.npz"
|
||||
if old.exists():
|
||||
with np.load(old) as f:
|
||||
err = max(float(np.max(np.abs(E - f["E_bounded_multiplicative"]))), float(np.max(np.abs(U - f["E_multiplicative"]))))
|
||||
verify.append({"seed": seed, "pairings": pairing, "max_abs_error": err})
|
||||
if err > 1e-6:
|
||||
raise RuntimeError("reconstructed E differs from sealed Phase 4M checkpoint")
|
||||
for mechanism, value in (("bounded", E), ("unbounded", U)):
|
||||
for group in GROUPS:
|
||||
mask = identity.identity_group.to_numpy() == group
|
||||
w = identity.anatomical_weight_mv.to_numpy(np.float32)[mask]
|
||||
reduction = 1 - value[mask]
|
||||
all_rows.append({"seed": seed, "pairings": pairing, "mechanism": mechanism, "group": group,
|
||||
"total_edges": int(mask.sum()), "updated_edges": int(np.count_nonzero(reduction > 1e-7)),
|
||||
"mean_delta_E": float((-reduction).mean()), "median_delta_E": float(np.median(-reduction)),
|
||||
"weighted_transmission_reduction": float(np.dot(w.astype(np.float64), reduction.astype(np.float64)) / w.sum()),
|
||||
"E_change_total": float(reduction.sum())})
|
||||
table = pd.DataFrame(all_rows)
|
||||
table.to_parquet(exp / "acquisition-trajectory/group-plasticity.parquet", index=False)
|
||||
wide = table.pivot_table(index=["seed", "pairings", "mechanism"], columns="group", values="weighted_transmission_reduction").reset_index()
|
||||
a, b, shared = (wide[x].to_numpy(np.float64) for x in ("A-specific", "B-specific", "shared"))
|
||||
wide["CSI"] = a - np.maximum(shared, b)
|
||||
wide["A_to_shared_ratio"] = np.divide(a, shared, out=np.full_like(a, np.nan), where=shared != 0)
|
||||
wide["A_to_B_ratio"] = np.divide(a, b, out=np.full_like(a, np.nan), where=b != 0)
|
||||
wide.to_parquet(exp / "specificity-curve/CSI.parquet", index=False)
|
||||
write_json(exp / "regression/phase4m-history-reconstruction.json", {"passed": True, "old_checkpoint_comparisons": verify, "max_error": max((v["max_abs_error"] for v in verify), default=0.0)})
|
||||
return checkpoints, table, wide, hist
|
||||
|
||||
|
||||
def isolated(ncfg, mcfg, cfg, exp: Path, identity: pd.DataFrame, checkpoints):
|
||||
v = Validator(cfg, Path(mcfg["frozen_phase4l_config"]), exp)
|
||||
if not np.array_equal(v.edge_ids, identity.edge_id.to_numpy(np.int64)):
|
||||
raise RuntimeError("plastic candidate ordering mismatch")
|
||||
sim = OdorSimulator(v, cfg, exp)
|
||||
comp = identity.compartment.astype(str).to_numpy() == str(ncfg["matched_compartment_verbatim"])
|
||||
targets = np.unique(identity.loc[comp, "target_neuron_idx"].to_numpy(np.int32))
|
||||
replay = ReplayEngine(sim, targets)
|
||||
anatomical = identity.anatomical_weight_mv.to_numpy(np.float32)
|
||||
old = Path(mcfg["experiment_root"])
|
||||
evaldata = {}
|
||||
zero = np.zeros(len(v.edge_ids), np.float32)
|
||||
for seed in ncfg["evaluation_seeds"]:
|
||||
for odor in ("A", "B"):
|
||||
spikes = pd.read_parquet(old / "isolated-replay" / f"KC-spikes-ODOR_{odor}-seed-{seed}.parquet")
|
||||
evaldata[(seed, odor)] = (spikes, replay.run(spikes, zero, float(mcfg["protocol"]["duration_ms"])))
|
||||
rows = []
|
||||
replay_wall = []
|
||||
for train_seed, eval_seed in zip(ncfg["train_seeds"], ncfg["evaluation_seeds"]):
|
||||
for pair in ncfg["checkpoints"]:
|
||||
E = checkpoints[train_seed][pair]["bounded"]
|
||||
delta = anatomical * (E - 1)
|
||||
rec = {"train_seed": train_seed, "eval_seed": eval_seed, "pairings": pair}
|
||||
effects = {}
|
||||
for odor in ("A", "B"):
|
||||
spikes, baseline = evaldata[(eval_seed, odor)]
|
||||
post = replay.run(spikes, delta, float(mcfg["protocol"]["duration_ms"]))
|
||||
replay_wall.append(post["wall_seconds"])
|
||||
effects[odor] = replay_effect(baseline, post, sim.dt)
|
||||
rec.update({f"{odor}_{k}": value for k, value in effects[odor].items() if k != "isolated_input_scope"})
|
||||
rec["A_specific_functional"] = compare_A_B(effects["A"], effects["B"])
|
||||
rec["functional_separation_spike_L1"] = float(effects["A"]["MBON_spike_L1"] - effects["B"]["MBON_spike_L1"])
|
||||
al = effects["A"]["first_latency_delta_ms"] or 0.0
|
||||
bl = effects["B"]["first_latency_delta_ms"] or 0.0
|
||||
rec["functional_separation_latency_ms"] = float(al - bl)
|
||||
rows.append(rec)
|
||||
frame = pd.DataFrame(rows)
|
||||
frame.to_parquet(exp / "isolated/response-by-checkpoint.parquet", index=False)
|
||||
curve = frame.groupby("pairings").agg(A_specific_functional_seeds=("A_specific_functional", "sum"),
|
||||
mean_spike_L1_separation=("functional_separation_spike_L1", "mean"),
|
||||
mean_latency_separation_ms=("functional_separation_latency_ms", "mean"),
|
||||
A_MBON_spike_L1=("A_MBON_spike_L1", "mean"), B_MBON_spike_L1=("B_MBON_spike_L1", "mean"),
|
||||
A_transmission_reduction_percent=("A_transmitted_reduction_percent", "mean"),
|
||||
B_transmission_reduction_percent=("B_transmitted_reduction_percent", "mean")).reset_index()
|
||||
curve.to_parquet(exp / "specificity-curve/memory-specificity.parquet", index=False)
|
||||
valid = curve[curve.A_specific_functional_seeds >= 3].copy()
|
||||
if valid.empty:
|
||||
peak = None
|
||||
else:
|
||||
best_spikes = float(valid.mean_spike_L1_separation.max())
|
||||
valid = valid[np.isclose(valid.mean_spike_L1_separation, best_spikes, atol=1e-9, rtol=0)]
|
||||
best_latency = float(valid.mean_latency_separation_ms.max())
|
||||
valid = valid[valid.mean_latency_separation_ms >= best_latency - sim.dt / 2]
|
||||
peak = int(valid.pairings.min())
|
||||
if peak is not None:
|
||||
# Equal score/latency checkpoints are resolved to earliest by the predeclared rule.
|
||||
peak_csi = float(np.mean(pd.read_parquet(exp / "specificity-curve/CSI.parquet").query("mechanism == 'bounded' and pairings == @peak").CSI))
|
||||
else:
|
||||
peak_csi = None
|
||||
write_json(exp / "specificity-curve/peak.json", {"status": "SPECIFICITY-PEAK-IDENTIFIED" if peak is not None else "UNIDENTIFIED", "pairings": peak,
|
||||
"peak_CSI": peak_csi, "selection_rule": ncfg["peak_rule"]})
|
||||
write_json(exp / "performance/isolated.json", {"mean_replay_wall_seconds": float(np.mean(replay_wall)), "total_replay_wall_seconds": float(np.sum(replay_wall)), "count": len(replay_wall)})
|
||||
return v, sim, frame, curve, peak
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--config", required=True, type=Path)
|
||||
args = ap.parse_args()
|
||||
ncfg, mcfg, cfg, base_path = load_config(args.config)
|
||||
exp = Path(ncfg["experiment_root"])
|
||||
identity, definition = prepare(ncfg, mcfg, cfg, exp)
|
||||
print(json.dumps({"stage": "identity_frozen", "group_counts": definition["group_counts"], "identity_sha256": sha256(exp / "edge-identity/frozen-edge-identity.parquet")}), flush=True)
|
||||
checkpoints, group, csi, history = reconstruct(ncfg, mcfg, cfg, exp, identity)
|
||||
print(json.dumps({"stage": "trajectory_reconstructed", "checkpoint_count": len(ncfg["checkpoints"]), "updates": len(history)}), flush=True)
|
||||
v, sim, frame, curve, peak = isolated(ncfg, mcfg, cfg, exp, identity, checkpoints)
|
||||
print(json.dumps({"stage": "isolated_complete", "peak": peak, "curve": curve.to_dict("records")}), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
phase: 4N
|
||||
name: MaleCNS-CreditAssignment-audit
|
||||
frozen_phase4m_config: /data/digital-drosophila/models/male-cns-lif/kcmbon-efficacy-v1/phase4m.yaml
|
||||
experiment_root: /data/digital-drosophila/experiments/phase4n-credit-assignment
|
||||
model_root: /data/digital-drosophila/models/male-cns-lif/credit-assignment-v1
|
||||
mechanism: bounded_multiplicative
|
||||
checkpoints: [0, 1, 2, 5, 10, 20, 30, 40, 50, 60, 75, 100]
|
||||
train_seeds: [51201, 51202, 51203, 51204, 51205]
|
||||
evaluation_seeds: [51301, 51302, 51303, 51304, 51305]
|
||||
edge_identity_source: phase4l matched_energy held-out ODOR_A/B majority-of-five KC representation
|
||||
credit_specificity_index: group mean anatomical-weighted E depression for A-specific minus maximum of shared and B-specific groups
|
||||
functional_separation: A MBON spike-count L1 change minus B MBON spike-count L1 change; first-spike latency used only as tie-breaker
|
||||
peak_rule: earliest checkpoint with at least 3 of 5 A-specific functional seeds and maximum five-seed mean spike-L1 separation; spike-L1 ties within 1e-9 use mean first-latency separation only if difference exceeds half a dt step (0.05 ms), otherwise choose earliest; no peak if no such checkpoint
|
||||
generalization_onset_rule: earliest checkpoint after peak with CSI at least 20 percent below peak and shared-or-B group depression at least 10 percent above its peak value; otherwise unidentified
|
||||
random_us_seed_offset: 700000
|
||||
random_us_window_ms: [30.0, 110.0]
|
||||
shadow_trace_tau_ms: 20.0
|
||||
event_coincidence_window_ms: 1.0
|
||||
first_activation_only: true
|
||||
matched_compartment_verbatim: "y5B'2a"
|
||||
unrelated_compartment_verbatim: "y2a'1"
|
||||
matched_dan_type: PAM15
|
||||
unrelated_dan_type: PPL103
|
||||
no_frozen_artifact_writes: true
|
||||
no_formal_associative_memory_claim: true
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare external-only E with open-loop all-confirmed-DAN gate shadow."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from phase4h_audit import write_json
|
||||
from phase4i import Validator
|
||||
from phase4j import effect
|
||||
from phase4l import OdorSimulator, pn_drive
|
||||
from phase4m import ReplayEngine, compare_A_B, load_odor_inputs, replay_effect
|
||||
from phase4n import load_config
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument("--config",type=Path,required=True);args=ap.parse_args()
|
||||
ncfg,mcfg,cfg,basepath=load_config(args.config);exp=Path(ncfg["experiment_root"])
|
||||
identity=pd.read_parquet(exp/"edge-identity/frozen-edge-identity.parquet")
|
||||
reproduction=json.loads((exp/"regression/event-trace-reproduction.json").read_text())
|
||||
bad={int(seed):min(int(x["pairing"]) for x in reproduction["mismatched_trials"] if int(x["seed"])==int(seed))
|
||||
for seed in ncfg["train_seeds"] if any(int(x["seed"])==int(seed) for x in reproduction["mismatched_trials"])}
|
||||
v=Validator(cfg,basepath,exp);sim=OdorSimulator(v,cfg,exp)
|
||||
comp=identity.compartment.astype(str).to_numpy()==str(ncfg["matched_compartment_verbatim"])
|
||||
engine=ReplayEngine(sim,np.unique(identity.loc[comp,"target_neuron_idx"].to_numpy(np.int32)))
|
||||
w=identity.anatomical_weight_mv.to_numpy(np.float32);old=Path(mcfg["experiment_root"])
|
||||
rows=[];calls=[]
|
||||
for seed,es in zip(ncfg["train_seeds"],ncfg["evaluation_seeds"]):
|
||||
for pair in [20,50,60,75,100]:
|
||||
with np.load(exp/"endogenous-dan"/f"E-all-DAN-shadow-seed-{seed}-pairings-{pair}.npz") as f:
|
||||
states={"external-only":f["E_external_only"].copy(),"all-confirmed-DAN-shadow":f["E_all_DAN"].copy()}
|
||||
for mechanism,E in states.items():
|
||||
delta=w*(E-1);results={}
|
||||
for odor in ("A","B"):
|
||||
spikes=pd.read_parquet(old/"isolated-replay"/f"KC-spikes-ODOR_{odor}-seed-{es}.parquet")
|
||||
zero=engine.run(spikes,np.zeros(len(E),np.float32),float(mcfg["protocol"]["duration_ms"]))
|
||||
result=engine.run(spikes,delta,float(mcfg["protocol"]["duration_ms"]))
|
||||
calls.append(result["wall_seconds"]);results[odor]=replay_effect(zero,result,sim.dt)
|
||||
rows.append({"seed":seed,"eval_seed":es,"pairings":pair,"mechanism":mechanism,
|
||||
"trace_exact_prefix":bool(pair < bad.get(int(seed),10**9)),
|
||||
"A_specific_functional":compare_A_B(results["A"],results["B"]),
|
||||
"A_MBON_spike_L1":results["A"]["MBON_spike_L1"],"B_MBON_spike_L1":results["B"]["MBON_spike_L1"],
|
||||
"A_transmission_reduction_percent":results["A"]["transmitted_reduction_percent"],
|
||||
"B_transmission_reduction_percent":results["B"]["transmitted_reduction_percent"]})
|
||||
table=pd.DataFrame(rows);table.to_parquet(exp/"endogenous-dan/isolated-gate-comparison.parquet",index=False)
|
||||
panel,vectors,selection,mapping=load_odor_inputs(cfg);gloms=selection["glomeruli_order"];ms=float(np.median(panel.mapped_response_sum));pn={}
|
||||
for odor in ("A","B"):
|
||||
oid=selection[f"ODOR_{odor}"]["odor_id"]
|
||||
pn[odor]=pn_drive(oid,vectors[oid],mapping,gloms,str(mcfg["protocol"]["input_mode"]),ms,cfg)[:2]
|
||||
whole=[];brain_calls=[]
|
||||
for seed,es in zip(ncfg["train_seeds"],ncfg["evaluation_seeds"]):
|
||||
baseline={odor:sim.run_rates(*pn[odor],es,float(mcfg["protocol"]["duration_ms"])) for odor in pn}
|
||||
for pair in (60,100):
|
||||
with np.load(exp/"endogenous-dan"/f"E-all-DAN-shadow-seed-{seed}-pairings-{pair}.npz") as f:
|
||||
states={"external-only":f["E_external_only"].copy(),"all-confirmed-DAN-shadow":f["E_all_DAN"].copy()}
|
||||
for mechanism,E in states.items():
|
||||
delta=w*(E-1)
|
||||
for odor in ("A","B"):
|
||||
post=sim.run_rates(*pn[odor],es,float(mcfg["protocol"]["duration_ms"]),delta=delta)
|
||||
brain_calls.append(post["wall_seconds"]);e=effect(baseline[odor],post)
|
||||
whole.append({"seed":seed,"eval_seed":es,"pairings":pair,"mechanism":mechanism,"odor":odor,
|
||||
"trace_exact_prefix":bool(pair < bad.get(int(seed),10**9)),
|
||||
"MBON_spike_L1":e["MBON_spike_l1"],"downstream_spike_L1":e["downstream_spike_l1"],
|
||||
"KC_spike_L1":int(np.abs(post["kc_counts"]-baseline[odor]["kc_counts"]).sum()),
|
||||
"finite":bool(post["finite"]),"wall_seconds":post["wall_seconds"]})
|
||||
wt=pd.DataFrame(whole);wt.to_parquet(exp/"endogenous-dan/wholebrain-gate-comparison.parquet",index=False)
|
||||
iso=table.groupby(["mechanism","pairings"]).agg(A_specific_seeds=("A_specific_functional","sum"),A_spike_L1=("A_MBON_spike_L1","mean"),B_spike_L1=("B_MBON_spike_L1","mean")).reset_index()
|
||||
pivot=wt.pivot(index=["mechanism","pairings","seed"],columns="odor",values="MBON_spike_L1").reset_index()
|
||||
pivot["A_minus_B"]=pivot["A"]-pivot["B"]
|
||||
wh=pivot.groupby(["mechanism","pairings"]).agg(A_minus_B_mean=("A_minus_B","mean"),A_specific_seeds=("A_minus_B",lambda x:int((x>0).sum()))).reset_index()
|
||||
clean=table[table.trace_exact_prefix].groupby(["mechanism","pairings"]).agg(clean_seed_count=("seed","nunique"),A_specific_seeds=("A_specific_functional","sum"),
|
||||
A_spike_L1=("A_MBON_spike_L1","mean"),B_spike_L1=("B_MBON_spike_L1","mean")).reset_index()
|
||||
comp_summary={"isolated":iso.to_dict("records"),"wholebrain":wh.to_dict("records"),
|
||||
"isolated_exact_prefix_only":clean.to_dict("records"),"nonexact_trials":reproduction["mismatched_trials"],
|
||||
"all_DAN_branch_status":"counterfactual open-loop shadow gate, not frozen Phase 4M dynamics or physiological dopamine",
|
||||
"isolated_replay_wall_seconds":float(sum(calls)),"wholebrain_wall_seconds":float(sum(brain_calls)),
|
||||
"external_US_only_is_frozen_Phase4M_rule":True}
|
||||
write_json(exp/"endogenous-dan/gate-specificity.json",comp_summary)
|
||||
print(json.dumps(comp_summary),flush=True)
|
||||
|
||||
|
||||
if __name__=="__main__":main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Held-out whole-brain readout for independent Phase 4N control E overlays."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from phase4h_audit import write_json
|
||||
from phase4i import Validator
|
||||
from phase4j import effect
|
||||
from phase4l import OdorSimulator, pn_drive
|
||||
from phase4m import load_odor_inputs
|
||||
from phase4n import load_config
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument("--config",type=Path,required=True);args=ap.parse_args()
|
||||
ncfg,mcfg,cfg,basepath=load_config(args.config);exp=Path(ncfg["experiment_root"])
|
||||
identity=pd.read_parquet(exp/"edge-identity/frozen-edge-identity.parquet")
|
||||
v=Validator(cfg,basepath,exp);sim=OdorSimulator(v,cfg,exp)
|
||||
w=identity.anatomical_weight_mv.to_numpy(np.float32)
|
||||
panel,vectors,selection,mapping=load_odor_inputs(cfg);gloms=selection["glomeruli_order"];ms=float(np.median(panel.mapped_response_sum))
|
||||
pn={}
|
||||
for odor in ("A","B"):
|
||||
oid=selection[f"ODOR_{odor}"]["odor_id"]
|
||||
pn[odor]=pn_drive(oid,vectors[oid],mapping,gloms,str(mcfg["protocol"]["input_mode"]),ms,cfg)[:2]
|
||||
rows=[];walls=[];conditions=("B+matched-US","A+random-US-timing","A+unrelated-compartment-US")
|
||||
for seed,es in zip(ncfg["train_seeds"],ncfg["evaluation_seeds"]):
|
||||
baseline={odor:sim.run_rates(*pn[odor],es,float(mcfg["protocol"]["duration_ms"])) for odor in pn}
|
||||
for pair in (50,100):
|
||||
for condition in conditions:
|
||||
with np.load(exp/"controls"/f"E-{condition}-seed-{seed}-pairings-{pair}.npz") as f:E=f["E"].copy()
|
||||
delta=w*(E-1)
|
||||
for odor in ("A","B"):
|
||||
post=sim.run_rates(*pn[odor],es,float(mcfg["protocol"]["duration_ms"]),delta=delta)
|
||||
e=effect(baseline[odor],post);walls.append(post["wall_seconds"])
|
||||
rows.append({"condition":condition,"train_seed":seed,"eval_seed":es,"pairings":pair,"odor":odor,
|
||||
"MBON_spike_L1":e["MBON_spike_l1"],"downstream_spike_L1":e["downstream_spike_l1"],
|
||||
"KC_spike_L1":int(np.abs(post["kc_counts"]-baseline[odor]["kc_counts"]).sum()),
|
||||
"finite":bool(post["finite"]),"wall_seconds":post["wall_seconds"]})
|
||||
print(json.dumps({"seed":seed,"completed_control_wholebrain":True}),flush=True)
|
||||
table=pd.DataFrame(rows);table.to_parquet(exp/"controls/wholebrain-heldout-control-replay.parquet",index=False)
|
||||
pivot=table.pivot(index=["condition","pairings","train_seed"],columns="odor",values="MBON_spike_L1").reset_index()
|
||||
pivot["A_minus_B"]=pivot["A"]-pivot["B"]
|
||||
summary=pivot.groupby(["condition","pairings"]).agg(A_minus_B_mean=("A_minus_B","mean"),A_specific_seeds=("A_minus_B",lambda x:int((x>0).sum()))).reset_index()
|
||||
write_json(exp/"controls/wholebrain-summary.json",{"rows":len(table),"conditions":summary.to_dict("records"),
|
||||
"total_brain_wall_seconds":float(sum(walls)),"frozen_E1_reference_reused_per_evaluation_seed":True,
|
||||
"A_D0_and_frozen_E1_effect_are_exactly_zero_by_E_identity":True})
|
||||
print(json.dumps(summary.to_dict("records")),flush=True)
|
||||
|
||||
|
||||
if __name__=="__main__":main()
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4N independent control trajectories; no frozen artifact writes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import resource
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from phase4h_audit import write_json
|
||||
from phase4i import Validator
|
||||
from phase4j import apply_v2, effect
|
||||
from phase4l import OdorSimulator, pn_drive
|
||||
from phase4m import ReplayEngine, compare_A_B, load_odor_inputs, replay_effect
|
||||
from phase4n import load_config
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument("--config",type=Path,required=True);ap.add_argument("--max-pairings",type=int,default=100)
|
||||
args=ap.parse_args();ncfg,mcfg,cfg,basepath=load_config(args.config);exp=Path(ncfg["experiment_root"])
|
||||
identity=pd.read_parquet(exp/"edge-identity/frozen-edge-identity.parquet")
|
||||
v=Validator(cfg,basepath,exp);sim=OdorSimulator(v,cfg,exp)
|
||||
panel,vectors,selection,mapping=load_odor_inputs(cfg);gloms=selection["glomeruli_order"];ms=float(np.median(panel.mapped_response_sum))
|
||||
pn={}
|
||||
for label in ("A","B"):
|
||||
oid=selection[f"ODOR_{label}"]["odor_id"]
|
||||
pn[label]=pn_drive(oid,vectors[oid],mapping,gloms,str(mcfg["protocol"]["input_mode"]),ms,cfg)[:2]
|
||||
matched=v.dan_targets(str(ncfg["matched_dan_type"]),str(ncfg["matched_compartment_verbatim"]))
|
||||
unrelated=v.dan_targets(str(ncfg["unrelated_dan_type"]),str(ncfg["unrelated_compartment_verbatim"]))
|
||||
src=identity.source_neuron_idx.to_numpy(np.int32);loc=np.full(v.n,-1,np.int32);loc[v.kc]=np.arange(len(v.kc),dtype=np.int32);loc=loc[src]
|
||||
k=np.float32(cfg["eligibility_v2"]["eta_effective_mv"]/cfg["model"]["w_syn_mv_per_anatomical_synapse"])
|
||||
bound=np.float32(1-cfg["eligibility_v2"]["max_fractional_depression"])
|
||||
points=set(map(int,ncfg["checkpoints"]));rows=[];walls=[]
|
||||
conditions={"B+matched-US":("B",matched,str(ncfg["matched_compartment_verbatim"])),
|
||||
"A+random-US-timing":("A",matched,str(ncfg["matched_compartment_verbatim"])),
|
||||
"A+unrelated-compartment-US":("A",unrelated,str(ncfg["unrelated_compartment_verbatim"]))}
|
||||
for condition,(odor,dan,compartment) in conditions.items():
|
||||
group_mask=identity.compartment.astype(str).to_numpy()==compartment
|
||||
for seed in ncfg["train_seeds"]:
|
||||
add=np.zeros(len(v.edge_ids),np.float32);E=np.ones(len(v.edge_ids),np.float32)
|
||||
rng=np.random.default_rng(seed+int(ncfg["random_us_seed_offset"]))
|
||||
np.savez_compressed(exp/"controls"/f"E-{condition}-seed-{seed}-pairings-0.npz",E=E,edge_id=v.edge_ids)
|
||||
rows.append({"condition":condition,"seed":seed,"pairings":0,"odor":odor,"DAN_compartment":compartment,
|
||||
"US_time_ms":None,"updated_edges":0,"matched_compartment_E_reduction_sum":0.0,
|
||||
"all_mask_E_reduction_sum":0.0,"brain_wall_seconds":0.0})
|
||||
for pairing in range(1,args.max_pairings+1):
|
||||
pulse=float(rng.uniform(*ncfg["random_us_window_ms"])) if condition=="A+random-US-timing" else float(mcfg["protocol"]["dan_time_ms"])
|
||||
run=sim.run_rates(*pn[odor],seed+pairing,float(mcfg["protocol"]["duration_ms"]),delta=add,dan_targets=dan,dan_pulses=[pulse])
|
||||
walls.append(run["wall_seconds"]);trace=run["kc_trace_at_dan"];elig=trace[loc]
|
||||
active=group_mask&(elig>0)
|
||||
E[active]=np.maximum(bound,E[active]*(np.float32(1.0)-k*elig[active]))
|
||||
add,_=apply_v2(v,trace,compartment,1.0,add,cfg)
|
||||
if pairing in points:
|
||||
out=exp/"controls"/f"E-{condition}-seed-{seed}-pairings-{pairing}.npz"
|
||||
np.savez_compressed(out,E=E,edge_id=v.edge_ids)
|
||||
rows.append({"condition":condition,"seed":seed,"pairings":pairing,"odor":odor,"DAN_compartment":compartment,
|
||||
"US_time_ms":pulse,"updated_edges":int(np.count_nonzero(E<1-1e-7)),"matched_compartment_E_reduction_sum":float(np.sum((1-E)[identity.compartment.astype(str).to_numpy()==str(ncfg["matched_compartment_verbatim"])])),
|
||||
"all_mask_E_reduction_sum":float(np.sum(1-E)),"brain_wall_seconds":run["wall_seconds"]})
|
||||
if pairing % 20 == 0:
|
||||
print(json.dumps({"condition":condition,"seed":seed,"pairing":pairing,"E_reduction_sum":float(np.sum(1-E))}),flush=True)
|
||||
print(json.dumps({"condition":condition,"seed":seed,"pairings":args.max_pairings,"E_reduction_sum":float(np.sum(1-E))}),flush=True)
|
||||
frame=pd.DataFrame(rows);frame.to_parquet(exp/"controls/independent-training-trajectories.parquet",index=False)
|
||||
# Common frozen held-out A/B KC spike trains; changes below are diagnostic, not memory claims.
|
||||
target_comp=identity.compartment.astype(str).to_numpy()==str(ncfg["matched_compartment_verbatim"])
|
||||
engine=ReplayEngine(sim,np.unique(identity.loc[target_comp,"target_neuron_idx"].to_numpy(np.int32)))
|
||||
w=identity.anatomical_weight_mv.to_numpy(np.float32);replay_rows=[]
|
||||
for condition in conditions:
|
||||
for seed,es in zip(ncfg["train_seeds"],ncfg["evaluation_seeds"]):
|
||||
for pair in [20,50,60,75,100]:
|
||||
with np.load(exp/"controls"/f"E-{condition}-seed-{seed}-pairings-{pair}.npz") as f:E=f["E"].copy()
|
||||
delta=w*(E-1);effects={}
|
||||
for odor in ("A","B"):
|
||||
spikes=pd.read_parquet(Path(mcfg["experiment_root"])/"isolated-replay"/f"KC-spikes-ODOR_{odor}-seed-{es}.parquet")
|
||||
base=engine.run(spikes,np.zeros(len(E),np.float32),float(mcfg["protocol"]["duration_ms"]))
|
||||
post=engine.run(spikes,delta,float(mcfg["protocol"]["duration_ms"]))
|
||||
effects[odor]=replay_effect(base,post,sim.dt)
|
||||
replay_rows.append({"condition":condition,"train_seed":seed,"eval_seed":es,"pairings":pair,
|
||||
"A_specific_functional":compare_A_B(effects["A"],effects["B"]),
|
||||
"A_MBON_spike_L1":effects["A"]["MBON_spike_L1"],"B_MBON_spike_L1":effects["B"]["MBON_spike_L1"],
|
||||
"A_transmission_reduction_percent":effects["A"]["transmitted_reduction_percent"],
|
||||
"B_transmission_reduction_percent":effects["B"]["transmitted_reduction_percent"]})
|
||||
rframe=pd.DataFrame(replay_rows);rframe.to_parquet(exp/"controls/isolated-heldout-control-replay.parquet",index=False)
|
||||
write_json(exp/"controls/independent-summary.json",{"conditions":list(conditions),"seeds":list(ncfg["train_seeds"]),
|
||||
"pairings_per_seed":args.max_pairings,"training_calls":len(walls),"training_wall_seconds":float(sum(walls)),
|
||||
"mean_call_seconds":float(np.mean(walls)),"peak_RAM_KiB":int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
|
||||
"matched_compartment_target_not_modified_by_unrelated_US":bool(frame[frame.condition=="A+unrelated-compartment-US"].matched_compartment_E_reduction_sum.eq(0).all()),
|
||||
"isolated_replay":rframe.groupby(["condition","pairings"]).agg(A_specific_seeds=("A_specific_functional","sum"),A_spike_L1=("A_MBON_spike_L1","mean"),B_spike_L1=("B_MBON_spike_L1","mean")).reset_index().to_dict("records")})
|
||||
|
||||
|
||||
if __name__=="__main__":main()
|
||||
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4N event-level KC/DAN side-channel on unmodified frozen LIF dynamics."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import resource
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
|
||||
from phase4h_audit import write_json
|
||||
from phase4i import Validator
|
||||
from phase4l import OdorSimulator, pn_drive
|
||||
from phase4m import load_odor_inputs
|
||||
from phase4n import load_config
|
||||
|
||||
|
||||
class EventSimulator(OdorSimulator):
|
||||
def run_events(self, targets, rates_hz, seed, duration_ms, delta, dan_targets, dan_pulse_ms, all_dan,
|
||||
save_all_events=False):
|
||||
targets = np.asarray(targets, np.int32); rates_hz = np.asarray(rates_hz, np.float32)
|
||||
self.v.install(delta); self.rt.weights[self.apl_edge_gpu] = 0.0; self.rt.reset(int(seed))
|
||||
steps = int(round(duration_ms / self.dt)); tg = torch.as_tensor(targets, dtype=torch.int64, device=self.device)
|
||||
rg = torch.as_tensor(rates_hz * self.dt / 1000.0, dtype=torch.float32, device=self.device)
|
||||
gen = torch.Generator(device=self.device); gen.manual_seed(int(seed))
|
||||
forced = torch.rand((steps, len(targets)), generator=gen, device=self.device) < rg[None, :]
|
||||
dan_targets = np.asarray(dan_targets, np.int32); dg = torch.as_tensor(dan_targets, dtype=torch.int64, device=self.device)
|
||||
dan_forced = self._forced(steps, dan_targets, seed + 991, None, [dan_pulse_ms])
|
||||
dan_all_gpu = torch.as_tensor(all_dan, dtype=torch.int64, device=self.device)
|
||||
apl = self.cfg["apl_overlay"]; apl_decay = math.exp(-self.dt / float(apl["tau_ms"]))
|
||||
trace_decay = math.exp(-self.dt / float(self.cfg["eligibility_v2"]["kc_trace_tau_ms"]))
|
||||
apl_state = torch.zeros(len(self.apl), dtype=torch.float32, device=self.device)
|
||||
trace = torch.zeros(len(self.kc), dtype=torch.float32, device=self.device)
|
||||
kc_history = torch.zeros((steps, len(self.kc)), dtype=torch.bool, device=self.device)
|
||||
dan_history = torch.zeros((steps, len(all_dan)), dtype=torch.bool, device=self.device)
|
||||
natural_dan_history = torch.zeros_like(dan_history)
|
||||
trace_at_gate = None; all_nodes = []; all_steps = []
|
||||
alpha, gd, rest, reset, threshold, refsteps = self.rt.alpha, self.rt.g_decay, self.rt.rest, self.rt.reset_value, self.rt.threshold, self.rt.refractory_steps
|
||||
t0 = time.perf_counter()
|
||||
for step in range(steps):
|
||||
slot = step % self.rt.queue_len; self.rt.g.mul_(gd).add_(self.rt.delay_queue[slot]); self.rt.delay_queue[slot].zero_()
|
||||
self.rt.refractory.clamp_min_(1).sub_(1); available = self.rt.refractory == 0
|
||||
self.rt.v[available] += alpha * (self.rt.g[available] - (self.rt.v[available] - rest)); self.rt.v[~available] = reset
|
||||
spikes = available & (self.rt.v >= threshold)
|
||||
accepted = forced[step] & available[tg]
|
||||
if accepted.any(): spikes[tg[accepted]] = True
|
||||
natural_dan_history[step] = spikes[dan_all_gpu]
|
||||
if len(dan_targets):
|
||||
da = dan_forced[step] & available[dg]
|
||||
if da.any(): spikes[dg[da]] = True
|
||||
sources = torch.nonzero(spikes, as_tuple=False).flatten()
|
||||
kc_history[step] = spikes[self.kc_gpu]
|
||||
dan_history[step] = spikes[dan_all_gpu]
|
||||
if save_all_events and sources.numel():
|
||||
all_nodes.append(sources.to(torch.int32)); all_steps.append(torch.full((sources.numel(),), step, dtype=torch.int32, device=self.device))
|
||||
if sources.numel():
|
||||
self.rt.v[sources] = reset; self.rt.refractory[sources] = refsteps; eids = self.rt._hybrid_edges(sources)
|
||||
if eids.numel(): self.rt.delay_queue[(step + self.rt.delay_steps) % self.rt.queue_len].scatter_add_(0, self.rt.indices[eids].long(), self.rt.weights[eids])
|
||||
trace.mul_(trace_decay).add_(spikes[self.kc_gpu].to(torch.float32)).clamp_max_(1.0)
|
||||
if step == int(round(dan_pulse_ms / self.dt)): trace_at_gate = trace.clone()
|
||||
apl_state.mul_(apl_decay).add_(torch.mv(self.input_gpu, spikes.to(torch.float32)) * float(apl["normalization"]))
|
||||
inhibition = torch.matmul(apl_state, self.output_gpu) * float(apl["gain"])
|
||||
self.rt.g.index_add_(0, self.kc_gpu, -inhibition)
|
||||
torch.cuda.synchronize()
|
||||
event_table = None
|
||||
if save_all_events:
|
||||
event_table = pd.DataFrame({"step": torch.cat(all_steps).cpu().numpy() if all_steps else np.empty(0,np.int32),
|
||||
"neuron_idx": torch.cat(all_nodes).cpu().numpy() if all_nodes else np.empty(0,np.int32)})
|
||||
return {"kc": kc_history.cpu().numpy(), "dan": dan_history.cpu().numpy(), "natural_dan": natural_dan_history.cpu().numpy(), "all_events": event_table,
|
||||
"trace_at_external": trace_at_gate.cpu().numpy(), "wall_seconds": time.perf_counter()-t0,
|
||||
"peak_vram_bytes": int(torch.cuda.max_memory_allocated())}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("--config", type=Path, required=True); ap.add_argument("--max-pairings", type=int, default=100); ap.add_argument("--train-seed", type=int)
|
||||
args = ap.parse_args(); ncfg, mcfg, cfg, basepath = load_config(args.config)
|
||||
exp = Path(ncfg["experiment_root"])
|
||||
identity = pd.read_parquet(exp / "edge-identity/frozen-edge-identity.parquet")
|
||||
history = pd.read_parquet(Path(mcfg["experiment_root"]) / "efficacy-overlay/update-history.parquet")
|
||||
v = Validator(cfg, basepath, exp); sim = EventSimulator(v, cfg, exp)
|
||||
if not np.array_equal(v.edge_ids, identity.edge_id.to_numpy(np.int64)):
|
||||
raise RuntimeError("edge identity order mismatch")
|
||||
all_dan = np.unique(v.confirmed.source_neuron_idx.to_numpy(np.int32))
|
||||
matched = v.dan_targets(str(ncfg["matched_dan_type"]), str(ncfg["matched_compartment_verbatim"]))
|
||||
matched_set = set(map(int, matched)); all_dan_to_local = {int(n): i for i,n in enumerate(all_dan)}
|
||||
matched_loc = np.array([all_dan_to_local[int(n)] for n in matched], np.int32)
|
||||
panel, vectors, selection, mapping = load_odor_inputs(cfg)
|
||||
gloms = selection["glomeruli_order"]; matched_sum = float(np.median(panel.mapped_response_sum))
|
||||
oid = selection["ODOR_A"]["odor_id"]
|
||||
pn = pn_drive(oid, vectors[oid], mapping, gloms, str(mcfg["protocol"]["input_mode"]), matched_sum, cfg)[:2]
|
||||
src = identity.source_neuron_idx.to_numpy(np.int32)
|
||||
kclook = np.full(v.n, -1, np.int32); kclook[v.kc] = np.arange(len(v.kc), dtype=np.int32)
|
||||
loc = kclook[src]
|
||||
comp = identity.compartment.astype(str).to_numpy() == str(ncfg["matched_compartment_verbatim"])
|
||||
bound = np.float32(1.0 - cfg["eligibility_v2"]["max_fractional_depression"])
|
||||
k = np.float32(cfg["eligibility_v2"]["eta_effective_mv"] / cfg["model"]["w_syn_mv_per_anatomical_synapse"])
|
||||
tau = float(cfg["eligibility_v2"]["kc_trace_tau_ms"])
|
||||
short_tau = float(ncfg["shadow_trace_tau_ms"])
|
||||
pulse_step = int(round(float(mcfg["protocol"]["dan_time_ms"]) / sim.dt))
|
||||
hist_by = {(int(s),int(p)): frame for (s,p),frame in history.groupby(["train_seed","pairing"])}
|
||||
eid_to_local = pd.Series(np.arange(len(identity), dtype=np.int32), index=identity.edge_id.to_numpy(np.int64))
|
||||
audit_rows = []; gate_rows = []; shadow_rows = []; perf = []; kc_event_frames = []; dan_event_frames = []; mismatches = []
|
||||
key = {10,20,50,60,75,100}
|
||||
for seed in ([args.train_seed] if args.train_seed is not None else ncfg["train_seeds"]):
|
||||
additive = np.zeros(len(identity), np.float32)
|
||||
E_ext = np.ones(len(identity), np.float32)
|
||||
E_all = np.ones(len(identity), np.float32)
|
||||
rng = np.random.default_rng(seed + int(ncfg["random_us_seed_offset"]))
|
||||
for pairing in range(1, args.max_pairings+1):
|
||||
audit_start = len(audit_rows); gate_start = len(gate_rows)
|
||||
capture = pairing in key
|
||||
run = sim.run_events(*pn, seed+pairing, float(mcfg["protocol"]["duration_ms"]), additive, matched,
|
||||
float(mcfg["protocol"]["dan_time_ms"]), all_dan, save_all_events=capture)
|
||||
perf.append({"seed":seed,"pairing":pairing,"wall_seconds":run["wall_seconds"],"peak_vram_bytes":run["peak_vram_bytes"]})
|
||||
kc = run["kc"]; dan = run["natural_dan"]
|
||||
ks, kl = np.nonzero(kc)
|
||||
if len(ks):
|
||||
kc_event_frames.append(pd.DataFrame({"seed":seed,"pairing":pairing,"step":ks.astype(np.int32),
|
||||
"neuron_idx":v.kc[kl].astype(np.int32)}))
|
||||
ds, dl = np.nonzero(dan)
|
||||
if len(ds):
|
||||
dan_event_frames.append(pd.DataFrame({"seed":seed,"pairing":pairing,"step":ds.astype(np.int32),
|
||||
"neuron_idx":all_dan[dl].astype(np.int32),"source":"endogenous-spike"}))
|
||||
trace = np.zeros(len(v.kc), np.float32); short = np.zeros_like(trace)
|
||||
last = np.full(len(v.kc), -1, np.int32); first = np.full(len(v.kc), -1, np.int32)
|
||||
external_at_gate = None
|
||||
endogenous_steps = 0; endogenous_events = 0
|
||||
random_step = int(round(float(rng.uniform(*ncfg["random_us_window_ms"])) / sim.dt))
|
||||
random_trace = None
|
||||
for step in range(len(kc)):
|
||||
spike = kc[step]
|
||||
trace *= np.float32(math.exp(-sim.dt / tau)); trace += spike.astype(np.float32); np.minimum(trace, 1.0, out=trace)
|
||||
short *= np.float32(math.exp(-sim.dt / short_tau)); short += spike.astype(np.float32); np.minimum(short, 1.0, out=short)
|
||||
last[spike] = step; first[spike & (first < 0)] = step
|
||||
if step == random_step: random_trace = trace[loc].copy()
|
||||
endogenous = bool(np.any(dan[step, matched_loc]))
|
||||
external = step == pulse_step
|
||||
if endogenous:
|
||||
endogenous_steps += 1; endogenous_events += int(np.count_nonzero(dan[step, matched_loc]))
|
||||
if external:
|
||||
external_at_gate = trace[loc].copy()
|
||||
max_error = float(np.max(np.abs(trace - run["trace_at_external"])))
|
||||
if max_error > 2e-5: raise RuntimeError(f"KC trace reconstruction mismatch {max_error}")
|
||||
if external or endogenous:
|
||||
elig = trace[loc]; active = comp & (elig > 0)
|
||||
before = E_all[active].copy()
|
||||
E_all[active] = np.maximum(bound, E_all[active] * (np.float32(1.0)-k*elig[active]))
|
||||
gate_rows.append({"seed":seed,"pairing":pairing,"step":step,"time_ms":step*sim.dt,
|
||||
"source": "both" if external and endogenous else ("external-US-associated" if external else "endogenous-feedback-associated"),
|
||||
"matched_endogenous_DAN_spikes":int(np.count_nonzero(dan[step,matched_loc])),
|
||||
"eligible_edges":int(active.sum()),"E_change_sum":float(np.sum(before-E_all[active]))})
|
||||
if external:
|
||||
last_age_ms = np.where(last[loc] >= 0, (step-last[loc])*sim.dt, np.inf)
|
||||
a = comp & (trace[loc] > 0)
|
||||
for local in np.flatnonzero(a):
|
||||
source = int(src[local]); age = float(last_age_ms[local])
|
||||
audit_rows.append({"seed":seed,"pairing":pairing,"edge_id":int(identity.edge_id.iat[local]),
|
||||
"source_neuron_idx":source,"source_body_id":int(identity.source_body_id.iat[local]),
|
||||
"target_neuron_idx":int(identity.target_neuron_idx.iat[local]),"compartment":str(identity.compartment.iat[local]),
|
||||
"identity_group":str(identity.identity_group.iat[local]),"update_time_ms":step*sim.dt,
|
||||
"eligibility_v2":float(trace[loc[local]]),"eligibility_short_shadow":float(short[loc[local]]),
|
||||
"event_coincidence_shadow":bool(age <= float(ncfg["event_coincidence_window_ms"])),
|
||||
"first_activation_only_shadow":bool(first[loc[local]] == last[loc[local]]),
|
||||
"last_local_KC_spike_age_ms":age,"first_local_KC_spike_ms":float(first[loc[local]]*sim.dt),
|
||||
"matched_external_DAN_arrival_ms":step*sim.dt,
|
||||
"endogenous_matched_DAN_simultaneous":bool(endogenous),
|
||||
"source_class": "external-US-associated", "delta_E_bounded":float(-k*trace[loc[local]]*E_ext[local])})
|
||||
if external_at_gate is None: raise RuntimeError("external gate missing")
|
||||
rec = hist_by[(int(seed),int(pairing))]
|
||||
active_loc = eid_to_local.loc[rec.edge_id.to_numpy(np.int64)].to_numpy(np.int32)
|
||||
err = float(np.max(np.abs(external_at_gate[active_loc] - rec.eligibility.to_numpy(np.float32)))) if len(active_loc) else 0.0
|
||||
exact = err <= 2e-5
|
||||
if not exact:
|
||||
mismatches.append({"seed":seed,"pairing":pairing,"max_abs_eligibility_error":err})
|
||||
for row in audit_rows[audit_start:]: row["frozen_trace_match"] = exact
|
||||
for row in gate_rows[gate_start:]: row["frozen_trace_match"] = exact
|
||||
# Advance the next trial from the sealed Phase 4M overlay, never from a divergent audit replay.
|
||||
old_E = rec.E_bounded_before.to_numpy(np.float32); new_E = rec.E_bounded_after.to_numpy(np.float32)
|
||||
if len(active_loc) and float(np.max(np.abs(E_ext[active_loc]-old_E))) > 1e-6:
|
||||
raise RuntimeError(f"sealed E history before-state mismatch {seed}/{pairing}")
|
||||
E_ext[active_loc] = new_E
|
||||
true_elig = rec.eligibility.to_numpy(np.float32)
|
||||
additive[active_loc] = np.maximum(-np.float32(cfg["eligibility_v2"]["max_fractional_depression"])*v.base_candidate[active_loc],
|
||||
additive[active_loc] - np.float32(cfg["eligibility_v2"]["eta_effective_mv"])*true_elig)
|
||||
if pairing in ncfg["checkpoints"]:
|
||||
with np.load(exp / "acquisition-trajectory" / f"E-seed-{seed}-pairings-{pairing}.npz") as f:
|
||||
chk_err = float(np.max(np.abs(E_ext-f["E_bounded"])))
|
||||
if chk_err > 1e-6: raise RuntimeError(f"external E reconstruction mismatch {seed}/{pairing}: {chk_err}")
|
||||
np.savez_compressed(exp / "endogenous-dan" / f"E-all-DAN-shadow-seed-{seed}-pairings-{pairing}.npz", E_all_DAN=E_all,
|
||||
E_external_only=E_ext, edge_id=identity.edge_id.to_numpy(np.int64))
|
||||
shadow_rows.append({"seed":seed,"pairing":pairing,"random_US_step":random_step,
|
||||
"external_matched_E_reduction_sum":float(np.sum(1-E_ext)),"all_DAN_shadow_E_reduction_sum":float(np.sum(1-E_all)),
|
||||
"matched_endogenous_DAN_steps":endogenous_steps,"matched_endogenous_DAN_spikes":endogenous_events,
|
||||
"random_US_trace_positive_edges":int(np.count_nonzero(comp & (random_trace>0))) if random_trace is not None else 0,
|
||||
"frozen_trace_match":exact,"max_abs_eligibility_error":err})
|
||||
if capture and run["all_events"] is not None:
|
||||
run["all_events"].to_parquet(exp / "whole-brain" / f"spike-events-seed-{seed}-pairings-{pairing}.parquet", index=False)
|
||||
if pairing % 20 == 0:
|
||||
print(json.dumps({"seed":seed,"pairing":pairing,"endogenous_steps":endogenous_steps,"E_external_sum":float(np.sum(1-E_ext)),"E_all_sum":float(np.sum(1-E_all))}),flush=True)
|
||||
pd.DataFrame(audit_rows).to_parquet(exp / "local-eligibility/event-level-updates.parquet", index=False)
|
||||
pd.DataFrame(gate_rows).to_parquet(exp / "dan-source/event-level-gates.parquet", index=False)
|
||||
pd.concat(kc_event_frames,ignore_index=True).to_parquet(exp / "local-eligibility/KC-spike-history.parquet",index=False)
|
||||
if dan_event_frames:
|
||||
pd.concat(dan_event_frames,ignore_index=True).to_parquet(exp / "dan-source/endogenous-DAN-spike-history.parquet",index=False)
|
||||
pd.DataFrame(shadow_rows).to_parquet(exp / "endogenous-dan/shadow-trajectory.parquet", index=False)
|
||||
write_json(exp / "regression/event-trace-reproduction.json", {"all_passed":len(mismatches)==0,"mismatched_trials":mismatches,
|
||||
"mismatched_count":len(mismatches),"audited_trial_count":len(shadow_rows),
|
||||
"next_trial_overlay_restored_from_sealed_Phase4M_history":True})
|
||||
pd.DataFrame(perf).to_parquet(exp / "performance/event-audit-calls.parquet", index=False)
|
||||
write_json(exp / "endogenous-dan/summary.json", {"mode_A": "observational frozen Phase 4M LIF with endogenous DAN spikes recorded",
|
||||
"mode_B": "external US-only plasticity gate, exact Phase 4M E reproduction",
|
||||
"all_DAN_mode": "counterfactual open-loop shadow E; same frozen Phase 4M additive brain trajectory; not a new brain model",
|
||||
"all_DAN_scope": "only metadata-confirmed DAN mapped to matched compartment; other DAN unresolved",
|
||||
"frozen_brain_dynamics_changed": False, "training_seed_count":len(ncfg["train_seeds"]),"pairings_per_seed":args.max_pairings})
|
||||
write_json(exp / "performance/event-audit.json", {"calls":len(perf),"total_wall_seconds":float(sum(x["wall_seconds"] for x in perf)),
|
||||
"mean_wall_seconds":float(np.mean([x["wall_seconds"] for x in perf])),"peak_VRAM_bytes":max(x["peak_vram_bytes"] for x in perf),
|
||||
"peak_RAM_KiB":int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
|
||||
"KC_boolean_history_bytes_per_trial":int(len(run["kc"])*len(v.kc)),"DAN_boolean_history_bytes_per_trial":int(len(run["dan"])*len(all_dan)),
|
||||
"no_dense_time_by_edge_tensor":True})
|
||||
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4N synthesis, independent checks, and final SHA-256 manifest."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import json
|
||||
import resource
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
|
||||
from phase4h_audit import sha256, verify_checksum, write_json
|
||||
from phase4i import phase4c_regression
|
||||
from phase4k import dataset_checks, seal
|
||||
from phase4m import frozen_checks, frozen_hashes
|
||||
from phase4n import load_config
|
||||
from phase4n_finish import curves, shared_decomposition
|
||||
from phase4i import Validator
|
||||
from phase4l import OdorSimulator
|
||||
|
||||
|
||||
def contamination(exp, history):
|
||||
event = pd.read_parquet(exp / "local-eligibility/event-level-updates.parquet")
|
||||
if event.duplicated(["seed", "pairing", "edge_id"]).any():
|
||||
raise RuntimeError("duplicate local eligibility row")
|
||||
h = history[["train_seed", "pairing", "edge_id", "E_bounded_before", "E_bounded_after"]].rename(columns={"train_seed":"seed"})
|
||||
x = event.merge(h,on=["seed","pairing","edge_id"],how="left",validate="one_to_one")
|
||||
unmatched_runtime_rows=int(x.E_bounded_after.isna().sum())
|
||||
nonmatching_trace_rows=int((~x.frozen_trace_match).sum())
|
||||
excluded_B_rows=int(((~x.frozen_trace_match)&(x.identity_group=="B-specific")).sum())
|
||||
x=x[x.frozen_trace_match & x.E_bounded_after.notna()].copy()
|
||||
x["actual_E_depression"] = x.E_bounded_before-x.E_bounded_after
|
||||
age = x.last_local_KC_spike_age_ms.to_numpy(np.float64)
|
||||
x["temporal_credit_class"] = np.where(age <= 1.0,"coincident_<=1ms",np.where(age <= 20.0,"recent_1to20ms","long_trace_>20ms"))
|
||||
x.to_parquet(exp / "b-contamination/updated-edges-with-temporal-source.parquet",index=False)
|
||||
b=x[(x.identity_group=="B-specific")&(x.actual_E_depression>1e-7)]
|
||||
total=float(b.actual_E_depression.sum())
|
||||
fractions={str(k):float(g.actual_E_depression.sum()/total) if total else None for k,g in b.groupby("temporal_credit_class")}
|
||||
event_fraction={str(k):int(len(g)) for k,g in b.groupby("temporal_credit_class")}
|
||||
relative_short=np.divide(b.eligibility_short_shadow.to_numpy(np.float64),b.eligibility_v2.to_numpy(np.float64),
|
||||
out=np.zeros(len(b),np.float64),where=b.eligibility_v2.to_numpy(np.float64)>0)
|
||||
weighted_short=float(np.dot(b.actual_E_depression.to_numpy(np.float64),relative_short)/total) if total else None
|
||||
report={"B_specific_update_rows":int(len(b)),"B_specific_unique_edges":int(b.edge_id.nunique()),
|
||||
"B_specific_unique_source_KC":int(b.source_neuron_idx.nunique()),
|
||||
"first_B_specific_update_pairing_by_seed":{str(seed):int(frame.pairing.min()) for seed,frame in b.groupby("seed")},
|
||||
"B_specific_E_depression_sum_with_exact_replay":total,"temporal_credit_weighted_fraction":fractions,"temporal_credit_event_counts":event_fraction,
|
||||
"excluded_nonmatching_trace_rows":nonmatching_trace_rows,"excluded_nonmatching_B_rows":excluded_B_rows,"unmatched_runtime_rows":unmatched_runtime_rows,
|
||||
"external_matched_US_gate_fraction":1.0,"endogenous_DAN_as_gate_fraction_in_frozen_model":0.0,
|
||||
"causal_route_fractions":{"direct_A_driven":None,"recurrent_KC":None,"MBON_DAN_feedback":None,"olfactory_feedback":None,"unresolved":1.0},
|
||||
"reason":"Whole-brain source spike timing plus a positive anatomical edge is not proof of membrane-spike causation. Frozen Phase 4M history has no causal predecessor trace. Do not assign invented path percentages.",
|
||||
"shadow_short_tau_20ms_excluded_B_update_fraction":float(b.loc[b.eligibility_short_shadow<.01,"actual_E_depression"].sum()/total) if total else None,
|
||||
"shadow_short_tau_20ms_weighted_eligibility_ratio_to_frozen":weighted_short,
|
||||
"event_coincidence_1ms_included_B_update_fraction":float(b.loc[b.event_coincidence_shadow,"actual_E_depression"].sum()/total) if total else None}
|
||||
report["first_activation_only_included_B_update_fraction"]=(float(b.loc[b.first_activation_only_shadow,"actual_E_depression"].sum()/total) if total else None)
|
||||
write_json(exp / "b-contamination/summary.json",report)
|
||||
return report
|
||||
|
||||
|
||||
def dan_summary(exp, identity, ncfg, cfg=None):
|
||||
gates=pd.read_parquet(exp / "dan-source/event-level-gates.parquet")
|
||||
trail=pd.read_parquet(exp / "endogenous-dan/shadow-trajectory.parquet")
|
||||
source=gates.groupby("source").agg(gates=("source","size"),E_depression_sum=("E_change_sum","sum")).reset_index()
|
||||
external_only=trail[trail.pairing==100].external_matched_E_reduction_sum.mean()
|
||||
all_dan=trail[trail.pairing==100].all_DAN_shadow_E_reduction_sum.mean()
|
||||
report={"source_breakdown":source.to_dict("records"),"trial_count":len(trail),
|
||||
"matched_endogenous_DAN_spikes_total":int(trail.matched_endogenous_DAN_spikes.sum()),
|
||||
"external_only_E_depression_sum_mean_100":float(external_only),"all_DAN_shadow_E_depression_sum_mean_100":float(all_dan),
|
||||
"additional_all_DAN_shadow_E_depression_sum_mean_100":float(all_dan-external_only),
|
||||
"frozen_Phase4M_endogenous_DAN_teaching_gate_count":0,
|
||||
"all_DAN_shadow_interpretation":"counterfactual open-loop teaching-gate interpretation; not frozen model and not measured dopamine concentration"}
|
||||
if cfg is not None:
|
||||
mapping=pd.read_parquet(Path(cfg["paths"]["phase4h_root"])/"mappings/dan-to-mbon-confirmed.parquet")
|
||||
mapping=mapping[["source_neuron_idx","source_annotation_type","source_compartment_verbatim"]].drop_duplicates()
|
||||
matched_set=set(map(int,mapping.loc[mapping.source_compartment_verbatim.astype(str)==str(ncfg["matched_compartment_verbatim"]),"source_neuron_idx"]))
|
||||
mapping=mapping[["source_neuron_idx","source_annotation_type"]].drop_duplicates(subset=["source_neuron_idx"])
|
||||
events_file=exp/"dan-source/endogenous-DAN-spike-history.parquet"
|
||||
if events_file.exists():
|
||||
events=pd.read_parquet(events_file)
|
||||
events=events.merge(mapping,left_on="neuron_idx",right_on="source_neuron_idx",how="left")
|
||||
typ=events.source_annotation_type.fillna("unknown").astype(str)
|
||||
events["cluster"] = np.where(typ.str.startswith("PAM"),"PAM",np.where(typ.str.startswith("PPL1"),"PPL1","other/unknown"))
|
||||
report["endogenous_spike_cluster_counts"]={str(k):int(v) for k,v in events.cluster.value_counts().items()}
|
||||
report["confirmed_type_counts"]={str(k):int(v) for k,v in typ.value_counts().items()}
|
||||
report["matched_compartment_spikes"] = int(events.neuron_idx.isin(matched_set).sum())
|
||||
events.to_parquet(exp/"dan-source/endogenous-DAN-with-type.parquet",index=False)
|
||||
write_json(exp / "dan-source/summary.json",report)
|
||||
return report
|
||||
|
||||
|
||||
def shared_summary(exp):
|
||||
p=exp / "shared-edge/isolated-decomposition.parquet"
|
||||
if not p.exists(): return {"status":"NOT-RUN-NO-PEAK"}
|
||||
table=pd.read_parquet(p)
|
||||
agg=table.groupby(["decomposition","odor"]).agg(spike_L1=("MBON_spike_L1","mean"),transmission_reduction=("transmitted_reduction_percent","mean"),
|
||||
latency_delta=("first_latency_delta_ms","mean")).reset_index()
|
||||
write_json(exp / "shared-edge/summary.json",{"scope":"isolated replay only; existing edges never deleted", "group_means":agg.to_dict("records")})
|
||||
return agg.to_dict("records")
|
||||
|
||||
|
||||
def separation(exp, peak):
|
||||
whole=pd.read_parquet(exp / "whole-brain/checkpoint-responses.parquet")
|
||||
iso=pd.read_parquet(exp / "isolated/response-by-checkpoint.parquet")
|
||||
rows=[]
|
||||
for pair in sorted(whole.pairings.unique()):
|
||||
w=whole[whole.pairings==pair].pivot(index="train_seed",columns="odor",values="MBON_spike_L1")
|
||||
i=iso[iso.pairings==pair]
|
||||
whole_sep=float((w["A"]-w["B"]).mean())
|
||||
iso_sep=float(i.functional_separation_spike_L1.mean())
|
||||
rows.append({"pairings":int(pair),"isolated_A_minus_B_spike_L1":iso_sep,"wholebrain_A_minus_B_spike_L1":whole_sep,
|
||||
"wholebrain_specificity_penalty":iso_sep-whole_sep,
|
||||
"wholebrain_A_specific_seeds":int(((w["A"]>w["B"])&(w["A"]>0)).sum())})
|
||||
frame=pd.DataFrame(rows);frame.to_parquet(exp / "whole-brain/specificity-penalty.parquet",index=False)
|
||||
return {"trajectory":rows,"at_peak":next((r for r in rows if r["pairings"]==peak),None)}
|
||||
|
||||
|
||||
def control_summary(exp):
|
||||
original=json.loads((exp / "controls/summary.json").read_text())
|
||||
independent=json.loads((exp / "controls/independent-summary.json").read_text())
|
||||
wholebrain=json.loads((exp / "controls/wholebrain-summary.json").read_text())
|
||||
original.update({"B_matched_US":"RUN-INDEPENDENT-100-PAIRING-TRAJECTORY",
|
||||
"random_US_timing":"RUN-INDEPENDENT-100-PAIRING-TRAJECTORY",
|
||||
"unrelated_compartment_US":"RUN-INDEPENDENT-100-PAIRING-TRAJECTORY",
|
||||
"all_DAN_gate":"RUN-OPEN-LOOP-SHADOW-100-PAIRING-TRAJECTORY",
|
||||
"full_control_matrix_pass":False,"independent_results":"controls/independent-summary.json",
|
||||
"all_DAN_closed_loop":False,"independent_wholebrain":"controls/wholebrain-summary.json"})
|
||||
write_json(exp / "controls/summary.json",original)
|
||||
return original, independent, wholebrain
|
||||
|
||||
|
||||
def integrity(exp, ncfg, mcfg, cfg, basepath):
|
||||
gc.collect();torch.cuda.empty_cache()
|
||||
previous=frozen_checks(cfg)
|
||||
mroot=Path(mcfg["experiment_root"]);mmodel=Path(mcfg["model_root"])
|
||||
phase4m={"experiment":verify_checksum(mroot,mroot/"manifests/checksums.sha256")["passed"],
|
||||
"model":verify_checksum(mmodel,mmodel/"checksums.sha256")["passed"]}
|
||||
dataset=dataset_checks(cfg)
|
||||
before=json.loads((exp / "audit/frozen-integrity.json").read_text())
|
||||
# A second replay detects intermittent GPU/event divergence; it never rewrites frozen references.
|
||||
replay=phase4c_regression(cfg,basepath,exp)
|
||||
phase4m_summary=json.loads((mroot/"audit/final-summary.json").read_text())
|
||||
reference_hashes=phase4m_summary["integrity"]["after"]
|
||||
static={"previous_phases":previous["all_passed"],"phase4m":all(phase4m.values()),"phase1_to_3b":dataset["all_passed"],
|
||||
"frozen_hashes_unchanged":frozen_hashes(cfg)==reference_hashes}
|
||||
report={"static_checksums_all_passed":bool(all(static.values())),"static":static,
|
||||
"phase4c_reference_replay_first":before["phase4c_reference_regression"],"phase4c_reference_replay_second":replay,
|
||||
"dynamic_regression_all_passed":bool(before["phase4c_reference_regression"]["all_passed"] and replay["all_passed"]),
|
||||
"all_passed":bool(all(static.values()) and before["phase4c_reference_regression"]["all_passed"] and replay["all_passed"])}
|
||||
write_json(exp / "audit/frozen-integrity-final.json",report)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument("--config",type=Path,required=True);args=ap.parse_args()
|
||||
ncfg,mcfg,cfg,basepath=load_config(args.config);exp=Path(ncfg["experiment_root"])
|
||||
identity=pd.read_parquet(exp / "edge-identity/frozen-edge-identity.parquet")
|
||||
response_curve=pd.read_parquet(exp/"specificity-curve/memory-specificity.parquet")
|
||||
eligible=response_curve[response_curve.A_specific_functional_seeds>=3]
|
||||
corrected_peak=None
|
||||
if not eligible.empty:
|
||||
best=float(eligible.mean_spike_L1_separation.max())
|
||||
eligible=eligible[np.isclose(eligible.mean_spike_L1_separation,best,atol=1e-9,rtol=0)]
|
||||
latency_best=float(eligible.mean_latency_separation_ms.max())
|
||||
eligible=eligible[eligible.mean_latency_separation_ms>=latency_best-float(cfg["model"]["dt_ms"])/2]
|
||||
corrected_peak=int(eligible.pairings.min())
|
||||
old_peak=json.loads((exp/"specificity-curve/peak.json").read_text())["pairings"]
|
||||
if corrected_peak!=old_peak:
|
||||
write_json(exp/"audit/peak-numerical-tie-correction.json",{"original_checkpoint":old_peak,"corrected_checkpoint":corrected_peak,
|
||||
"reason":"50 and 60 have the same five-seed spike-L1 separation; latency difference is sub-femtosecond float summation noise, far below 0.1 ms simulation dt. Earliest-tie rule selects 50.",
|
||||
"not_a_learning_parameter_change":True,"identity_and_E_trajectories_unchanged":True})
|
||||
write_json(exp/"specificity-curve/peak.json",{"status":"SPECIFICITY-PEAK-IDENTIFIED","pairings":corrected_peak,
|
||||
"selection_rule":ncfg["peak_rule"],"numerical_tie_correction_record":"audit/peak-numerical-tie-correction.json"})
|
||||
curves(ncfg,exp,pd.read_parquet(exp/"specificity-curve/CSI.parquet"),response_curve,identity)
|
||||
validator=Validator(cfg,basepath,exp);sim=OdorSimulator(validator,cfg,exp)
|
||||
shared_decomposition(ncfg,exp,identity,sim,validator,corrected_peak)
|
||||
del sim,validator
|
||||
stopping=json.loads((exp/"stopping-analysis/summary.json").read_text())
|
||||
stopping["functional_peak_checkpoint"]=corrected_peak
|
||||
write_json(exp/"stopping-analysis/summary.json",stopping)
|
||||
# Complete the explicitly predeclared t=0 checkpoint without rerunning any brain trial.
|
||||
groups0=pd.read_parquet(exp/"acquisition-trajectory/group-plasticity.parquet")
|
||||
csi0=pd.read_parquet(exp/"specificity-curve/CSI.parquet")
|
||||
if 0 not in set(groups0.pairings):
|
||||
z=[];cz=[]
|
||||
for seed in ncfg["train_seeds"]:
|
||||
one=np.ones(len(identity),np.float32)
|
||||
np.savez_compressed(exp/"acquisition-trajectory"/f"E-seed-{seed}-pairings-0.npz",E_bounded=one,E_unbounded=one,edge_id=identity.edge_id.to_numpy(np.int64))
|
||||
for mechanism in ("bounded","unbounded"):
|
||||
for group_name,count in identity.identity_group.value_counts().items():
|
||||
z.append({"seed":seed,"pairings":0,"mechanism":mechanism,"group":group_name,"total_edges":int(count),
|
||||
"updated_edges":0,"mean_delta_E":0.0,"median_delta_E":0.0,"weighted_transmission_reduction":0.0,"E_change_total":0.0})
|
||||
cz.append({"seed":seed,"pairings":0,"mechanism":mechanism,"A-specific":0.0,"B-specific":0.0,"shared":0.0,"inactive":0.0,
|
||||
"CSI":0.0,"A_to_shared_ratio":np.nan,"A_to_B_ratio":np.nan})
|
||||
pd.concat([groups0,pd.DataFrame(z)],ignore_index=True).sort_values(["seed","pairings","mechanism","group"]).to_parquet(exp/"acquisition-trajectory/group-plasticity.parquet",index=False)
|
||||
pd.concat([csi0,pd.DataFrame(cz)],ignore_index=True).sort_values(["seed","pairings","mechanism"]).to_parquet(exp/"specificity-curve/CSI.parquet",index=False)
|
||||
history=pd.read_parquet(Path(mcfg["experiment_root"])/"efficacy-overlay/update-history.parquet")
|
||||
peak=json.loads((exp/"specificity-curve/peak.json").read_text())["pairings"]
|
||||
onset=json.loads((exp/"specificity-curve/generalization-onset.json").read_text())["pairings"]
|
||||
contam=contamination(exp,history);dan=dan_summary(exp,identity,ncfg,cfg);shared=shared_summary(exp)
|
||||
timed_inputs=json.loads((exp/"whole-brain/B-specific-timed-input-summary.json").read_text())
|
||||
sep=separation(exp,peak);controls,independent,control_whole=control_summary(exp)
|
||||
gate_specificity=json.loads((exp/"endogenous-dan/gate-specificity.json").read_text())
|
||||
perf=json.loads((exp / "performance/summary.json").read_text())
|
||||
audit_perf=json.loads((exp / "performance/event-audit.json").read_text())
|
||||
ctrl_perf={"training_calls":independent["training_calls"],"training_wall_seconds":independent["training_wall_seconds"],
|
||||
"mean_call_seconds":independent["mean_call_seconds"],"wholebrain_wall_seconds":control_whole["total_brain_wall_seconds"]}
|
||||
perf.update({"event_audit":audit_perf,"controls":ctrl_perf,"peak_RAM_KiB_all_stages":max(perf["peak_RAM_KiB"],audit_perf["peak_RAM_KiB"],independent["peak_RAM_KiB"]),
|
||||
"shadow_trace_overhead":"included in event-audit call time; no paired unaudited timing isolation"})
|
||||
write_json(exp / "performance/summary.json",perf)
|
||||
integ=integrity(exp,ncfg,mcfg,cfg,basepath)
|
||||
event_reproduction=json.loads((exp/"regression/event-trace-reproduction.json").read_text())
|
||||
csi=pd.read_parquet(exp / "specificity-curve/CSI.parquet")
|
||||
group=pd.read_parquet(exp / "acquisition-trajectory/group-plasticity.parquet")
|
||||
iso=pd.read_parquet(exp / "specificity-curve/memory-specificity.parquet")
|
||||
long_fraction=contam["temporal_credit_weighted_fraction"].get("long_trace_>20ms")
|
||||
shared_b=next((r["spike_L1"] for r in shared if r["decomposition"]=="A+shared" and r["odor"]=="B"),None) if isinstance(shared,list) else None
|
||||
a_only_b=next((r["spike_L1"] for r in shared if r["decomposition"]=="A-only" and r["odor"]=="B"),None) if isinstance(shared,list) else None
|
||||
mechanism_tags={"SPECIFICITY-PEAK-IDENTIFIED":peak is not None,
|
||||
"GENERALIZATION-ONSET-IDENTIFIED":onset is not None,
|
||||
"ENDOGENOUS-DAN-CREDIT-CONTAMINATION":False,
|
||||
"RECURRENT-KC-CREDIT-CONTAMINATION":None,
|
||||
"LONG-TRACE-CREDIT-CONTAMINATION":bool(long_fraction is not None and long_fraction>0),
|
||||
"SHARED-EDGE-GENERALIZATION-DOMINANT":bool(shared_b is not None and a_only_b is not None and shared_b>a_only_b),
|
||||
"NO-SINGLE-DOMINANT-CONTAMINATION-SOURCE":True}
|
||||
audit_complete=bool(event_reproduction["all_passed"] and integ["all_passed"] and controls["full_control_matrix_pass"])
|
||||
summary={"phase":"4N","status":"MECHANISM-AUDIT-COMPLETE-NO-CANDIDATE" if audit_complete else "PARTIAL-MECHANISM-AUDIT-NO-CANDIDATE",
|
||||
"edge_groups":identity.identity_group.value_counts().to_dict(),"specificity_peak_pairings":peak,
|
||||
"generalization_onset_pairings_by_predeclared_CSI_rule":onset,
|
||||
"functional_collapse_first_checkpoint":next((int(r.pairings) for _,r in iso.iterrows() if r.pairings>peak and r.A_specific_functional_seeds<3),None) if peak is not None else None,
|
||||
"group_curves":group[group.mechanism=="bounded"].groupby(["pairings","group"]).mean(numeric_only=True).reset_index()[["pairings","group","mean_delta_E","weighted_transmission_reduction"]].to_dict("records"),
|
||||
"isolated_curve":iso.to_dict("records"),"CSI_curve":csi[csi.mechanism=="bounded"].groupby("pairings").CSI.mean().reset_index().to_dict("records"),
|
||||
"B_specific_contamination":contam,"B_specific_timed_input_candidates":timed_inputs,"DAN":dan,"gate_specificity":gate_specificity,"shared_edge_isolated":shared,"wholebrain_specificity":sep,
|
||||
"controls":controls,"control_wholebrain":control_whole,"event_trace_reproduction":event_reproduction,"mechanism_tags":mechanism_tags,"candidate_generated":False,"candidate_gate":{"external_vs_endogenous_distinguished":True,
|
||||
"causal_B_contamination_route_identified":False,"isolated_and_wholebrain_majority_improvement_demonstrated":False},
|
||||
"frozen_integrity":integ,"limitations":["Temporal predecessor class unresolved; no causal path attribution percentage claimed",
|
||||
"All-DAN efficacy is an open-loop shadow, not a closed-loop retrained brain",
|
||||
"Phase 4C mechanosensory frozen regression showed at least one divergent run; static SHA-256 passes but dynamic integrity cannot be called PASS"]}
|
||||
write_json(exp / "audit/final-summary.json",summary)
|
||||
model=Path(ncfg["model_root"]);model.mkdir(parents=True,exist_ok=True)
|
||||
for name in ("phase4n.py","phase4n_finish.py","phase4n_events.py","phase4n_controls.py","phase4n_control_whole.py","phase4n_alldan.py","phase4n_paths.py","phase4n_preflight.py","phase4n_finalize.py","phase4n.yaml"):
|
||||
source=Path(__file__).with_name(name)
|
||||
shutil.copy2(source,model/name)
|
||||
shutil.copy2(source,exp/"audit"/name)
|
||||
write_json(model/"VERSION.json",{"phase":"4N","status":summary["status"],"candidate_generated":False,
|
||||
"identity_sha256":sha256(exp/"edge-identity/frozen-edge-identity.parquet"),"config_sha256":sha256(args.config),
|
||||
"frozen_static_integrity":integ["static_checksums_all_passed"],"frozen_dynamic_regression":integ["dynamic_regression_all_passed"]})
|
||||
seal(model,model/"MANIFEST.json",model/"checksums.sha256")
|
||||
seal(exp,exp/"manifests/manifest.json",exp/"manifests/checksums.sha256")
|
||||
self_check={}
|
||||
for root,relative in ((model,"checksums.sha256"),(exp,"manifests/checksums.sha256")):
|
||||
result=subprocess.run(["sha256sum","-c",relative],cwd=root,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE,text=True)
|
||||
self_check[str(root)]=result.returncode==0
|
||||
if not all(self_check.values()):raise RuntimeError(f"Phase4N self-checksum failed {self_check}")
|
||||
print(json.dumps({"status":summary["status"],"peak":peak,"onset":onset,"B_contamination":contam,
|
||||
"DAN":dan,"wholebrain_penalty":sep["at_peak"],"integrity":integ,"self_checks":self_check}),flush=True)
|
||||
|
||||
|
||||
if __name__=="__main__":main()
|
||||
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4N checkpoint-specific whole-brain/credit audit and conservative sealing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import json
|
||||
import resource
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
|
||||
from phase4h_audit import sha256, verify_checksum, write_json
|
||||
from phase4i import Validator, phase4c_regression
|
||||
from phase4j import effect
|
||||
from phase4k import dataset_checks, seal
|
||||
from phase4l import OdorSimulator, pn_drive
|
||||
from phase4m import frozen_checks, frozen_hashes, load_odor_inputs
|
||||
from phase4n import GROUPS, load_config
|
||||
|
||||
|
||||
def state(exp, seed, pair):
|
||||
if pair == 0:
|
||||
return np.ones(14232, np.float32)
|
||||
with np.load(exp / "acquisition-trajectory" / f"E-seed-{seed}-pairings-{pair}.npz") as f:
|
||||
return f["E_bounded"].copy()
|
||||
|
||||
|
||||
def curves(ncfg, exp, csi, replay, identity):
|
||||
csi = csi[csi.mechanism == "bounded"]
|
||||
peak = json.loads((exp / "specificity-curve/peak.json").read_text())["pairings"]
|
||||
onset = None
|
||||
if peak is not None:
|
||||
p = csi[csi.pairings == peak]
|
||||
baseline_csi = p.CSI.mean()
|
||||
baseline_other = max(p.shared.mean(), p["B-specific"].mean())
|
||||
for pair in ncfg["checkpoints"]:
|
||||
if pair <= peak:
|
||||
continue
|
||||
q = csi[csi.pairings == pair]
|
||||
other = max(q.shared.mean(), q["B-specific"].mean())
|
||||
if q.CSI.mean() <= .8 * baseline_csi and other >= 1.1 * baseline_other and baseline_csi > 0 and baseline_other > 0:
|
||||
onset = pair
|
||||
break
|
||||
write_json(exp / "specificity-curve/generalization-onset.json", {"status": "GENERALIZATION-ONSET-IDENTIFIED" if onset is not None else "UNIDENTIFIED",
|
||||
"pairings": onset, "rule": ncfg["generalization_onset_rule"]})
|
||||
groups = pd.read_parquet(exp / "acquisition-trajectory/group-plasticity.parquet")
|
||||
groups = groups[groups.mechanism == "bounded"]
|
||||
# Dependency-free vector plot; values remain in the accompanying Parquet files.
|
||||
parts = ['<svg xmlns="http://www.w3.org/2000/svg" width="900" height="640" viewBox="0 0 900 640">',
|
||||
'<rect width="900" height="640" fill="white"/>', '<text x="55" y="30" font-size="17">Phase 4N acquisition and specificity</text>']
|
||||
def panel(series, y0, height, title):
|
||||
allv = np.concatenate([np.asarray(v, np.float64) for _, v, _ in series]); lo, hi = float(np.min(allv)), float(np.max(allv))
|
||||
if hi <= lo: hi = lo + 1.0
|
||||
parts.append(f'<text x="55" y="{y0-10}" font-size="13">{title}</text>')
|
||||
parts.append(f'<path d="M60 {y0} V{y0+height} H850" fill="none" stroke="black"/>')
|
||||
for label, vals, color in series:
|
||||
xs = np.asarray(ncfg["checkpoints"], np.float64)
|
||||
ys = np.asarray(vals, np.float64)
|
||||
coords = ' '.join(f'{60+790*x/100:.1f},{y0+height-20-(height-35)*(y-lo)/(hi-lo):.1f}' for x,y in zip(xs,ys))
|
||||
parts.append(f'<polyline points="{coords}" fill="none" stroke="{color}" stroke-width="2"/>')
|
||||
parts.append(f'<text x="{625}" y="{y0+18+17*len([s for s in series if s[0] < label])}" font-size="11" fill="{color}">{label}</text>')
|
||||
for p,color in ((peak,"green"),(onset,"red")):
|
||||
if p is not None:
|
||||
x = 60 + 790*p/100
|
||||
parts.append(f'<path d="M{x:.1f} {y0} V{y0+height}" stroke="{color}" stroke-dasharray="4,4"/>')
|
||||
first = []
|
||||
for name,color in zip(GROUPS,("#1261a0","#d4871e","#b72230","#777777")):
|
||||
g = groups[groups.group == name].groupby("pairings").weighted_transmission_reduction.mean().reindex(ncfg["checkpoints"]).fillna(0.0)
|
||||
first.append((name,g.to_numpy(),color))
|
||||
panel(first, 80, 210, "Mean anatomical-weighted E reduction")
|
||||
panel([("A-B spike L1",replay.mean_spike_L1_separation.to_numpy(),"#2059a6"),
|
||||
("A-B latency ms",replay.mean_latency_separation_ms.to_numpy(),"#b72230")], 370, 210, "Functional MBON effect difference")
|
||||
parts.append('</svg>')
|
||||
(exp / "specificity-curve/trajectory.svg").write_text('\n'.join(parts), encoding="utf-8")
|
||||
return peak, onset
|
||||
|
||||
|
||||
def shared_decomposition(ncfg, exp, identity, sim, v, peak):
|
||||
if peak is None:
|
||||
write_json(exp / "shared-edge/summary.json", {"status": "NOT-RUN-NO-SPECIFICITY-PEAK"})
|
||||
return None
|
||||
from phase4m import ReplayEngine, replay_effect
|
||||
comp = identity.compartment.astype(str).to_numpy() == str(ncfg["matched_compartment_verbatim"])
|
||||
targets = np.unique(identity.loc[comp, "target_neuron_idx"].to_numpy(np.int32))
|
||||
engine = ReplayEngine(sim, targets)
|
||||
w = identity.anatomical_weight_mv.to_numpy(np.float32)
|
||||
groups = identity.identity_group.astype(str).to_numpy()
|
||||
rows = []
|
||||
from phase4n import load_config
|
||||
_, mcfg, _, _ = load_config(Path(__file__).with_name("phase4n.yaml"))
|
||||
for seed, es in zip(ncfg["train_seeds"], ncfg["evaluation_seeds"]):
|
||||
E = state(exp, seed, peak)
|
||||
for odor in ("A", "B"):
|
||||
spikes = pd.read_parquet(Path(mcfg["experiment_root"]) / "isolated-replay" / f"KC-spikes-ODOR_{odor}-seed-{es}.parquet")
|
||||
baseline = engine.run(spikes, np.zeros(len(E), np.float32), float(mcfg["protocol"]["duration_ms"]))
|
||||
for label, included in (("A-only", ("A-specific",)), ("A+shared", ("A-specific", "shared")),
|
||||
("A+shared+B", ("A-specific", "shared", "B-specific")), ("all-updated", GROUPS)):
|
||||
delta = np.where(np.isin(groups, included), w * (E - 1), 0).astype(np.float32)
|
||||
post = engine.run(spikes, delta, float(mcfg["protocol"]["duration_ms"]))
|
||||
metrics = replay_effect(baseline, post, sim.dt)
|
||||
rows.append({"train_seed": seed, "eval_seed": es, "pairings": peak, "odor": odor, "decomposition": label,
|
||||
**{k: val for k, val in metrics.items() if k != "isolated_input_scope"}})
|
||||
pd.DataFrame(rows).to_parquet(exp / "shared-edge/isolated-decomposition.parquet", index=False)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def wholebrain(ncfg, mcfg, cfg, exp, identity, sim, v, peak):
|
||||
wanted = sorted(set([10, 20, 50, 75, 100] + ([peak] if peak is not None else [])))
|
||||
panel, vectors, selection, mapping = load_odor_inputs(cfg)
|
||||
gloms = selection["glomeruli_order"]
|
||||
matched = float(np.median(panel.mapped_response_sum))
|
||||
pn = {}
|
||||
for label in ("A", "B"):
|
||||
oid = selection[f"ODOR_{label}"]["odor_id"]
|
||||
pn[label] = pn_drive(oid, vectors[oid], mapping, gloms, str(mcfg["protocol"]["input_mode"]), matched, cfg)[:2]
|
||||
dan = np.unique(v.confirmed.source_neuron_idx.to_numpy(np.int32))
|
||||
group_sources = {name: np.unique(identity.loc[identity.identity_group == name, "source_neuron_idx"].to_numpy(np.int32)) for name in GROUPS}
|
||||
w = identity.anatomical_weight_mv.to_numpy(np.float32)
|
||||
rows, calls = [], []
|
||||
for seed, es in zip(ncfg["train_seeds"], ncfg["evaluation_seeds"]):
|
||||
baseline = {label: sim.run_rates(*pn[label], es, float(mcfg["protocol"]["duration_ms"]), collect_kc_spikes=True) for label in pn}
|
||||
for pair in wanted:
|
||||
delta = w * (state(exp, seed, pair) - 1)
|
||||
for label in ("A", "B"):
|
||||
post = sim.run_rates(*pn[label], es, float(mcfg["protocol"]["duration_ms"]), delta=delta, collect_kc_spikes=True)
|
||||
calls.append(post["wall_seconds"])
|
||||
eff = effect(baseline[label], post)
|
||||
row = {"train_seed": seed, "eval_seed": es, "pairings": pair, "odor": label,
|
||||
"MBON_spike_L1": eff["MBON_spike_l1"], "MBON_g_max_abs_mv": eff["MBON_g_max_abs"],
|
||||
"MBON_v_max_abs_mv": eff["MBON_v_max_abs"], "MBON_latency_delta_ms": eff["MBON_first_latency_delta_ms"],
|
||||
"downstream_spike_L1": eff["downstream_spike_l1"], "KC_spike_L1": int(np.abs(post["kc_counts"]-baseline[label]["kc_counts"]).sum()),
|
||||
"endogenous_DAN_spikes": int(post["counts"][dan].sum()), "baseline_endogenous_DAN_spikes": int(baseline[label]["counts"][dan].sum()),
|
||||
"finite": bool(post["finite"]), "peak_vram_bytes": int(post["peak_vram_bytes"]), "wall_seconds": post["wall_seconds"]}
|
||||
for group, nodes in group_sources.items():
|
||||
row[f"{group}_KC_spikes"] = int(post["counts"][nodes].sum())
|
||||
row[f"{group}_KC_newly_active_vs_E1"] = int(np.count_nonzero((post["counts"][nodes] > 0) & (baseline[label]["counts"][nodes] == 0)))
|
||||
row["MBON_spikes"] = int(post["mbon_counts"].sum())
|
||||
row["other_feedback_spikes"] = int(post["downstream_counts"].sum())
|
||||
rows.append(row)
|
||||
frame = pd.DataFrame(rows)
|
||||
frame.to_parquet(exp / "whole-brain/checkpoint-responses.parquet", index=False)
|
||||
write_json(exp / "performance/whole-brain.json", {"calls": len(calls), "mean_wall_seconds": float(np.mean(calls)),
|
||||
"sum_wall_seconds": float(np.sum(calls)), "max_peak_vram_bytes": int(frame.peak_vram_bytes.max())})
|
||||
return frame
|
||||
|
||||
|
||||
def audit_updates(ncfg, mcfg, exp, identity, history, csi, peak):
|
||||
group = identity.set_index("edge_id").identity_group
|
||||
hist = history.copy()
|
||||
hist["identity_group"] = group.loc[hist.edge_id.to_numpy(np.int64)].to_numpy()
|
||||
hist["delta_E_bounded"] = hist.E_bounded_after - hist.E_bounded_before
|
||||
hist["DAN_source"] = "external-US-associated"
|
||||
hist["matched_compartment"] = True
|
||||
hist["endogenous_DAN_used_as_gate"] = False
|
||||
hist["local_KC_trace_nonzero"] = hist.eligibility > 0
|
||||
hist.to_parquet(exp / "local-eligibility/phase4m-updates-classified.parquet", index=False)
|
||||
first = hist[(hist.identity_group == "B-specific") & (hist.delta_E_bounded < -1e-7)].groupby("train_seed").pairing.min().reset_index(name="first_B_specific_update_pairing")
|
||||
first.to_parquet(exp / "b-contamination/first-B-updates.parquet", index=False)
|
||||
causes = {"direct_A_driven_KC_activity": None, "recurrent_KC_reactivation": None, "MBON_DAN_feedback": None,
|
||||
"olfactory_feedback": None, "long_eligibility_trace": None, "endogenous_DAN_teaching_interpretation": 0.0,
|
||||
"unresolved_runtime_KC_recruitment_or_trace": 1.0}
|
||||
write_json(exp / "b-contamination/provenance.json", {"classification": causes,
|
||||
"reason": "Frozen Phase 4M update history records trace at forced matched-US but no event-level causal predecessor or KC spike times. Assigning a recurrent/feedback percentage would be invented.",
|
||||
"earliest_B_update_by_seed": first.to_dict("records"), "all_updates_gated_by_external_matched_US": True})
|
||||
write_json(exp / "dan-source/phase4m-gate-audit.json", {"external_US_gates": int(history[["train_seed", "pairing"]].drop_duplicates().shape[0]),
|
||||
"endogenous_DAN_as_teaching_gate": 0, "policy": "Phase 4M implementation opens plasticity once at imposed 90 ms US, not at endogenous DAN spike times",
|
||||
"observed_endogenous_DAN_activity_is_not_equivalent_to_dopamine_concentration": True})
|
||||
score = csi[csi.mechanism == "bounded"].groupby("pairings").CSI.mean()
|
||||
write_json(exp / "stopping-analysis/summary.json", {"status": "POST-HOC-ONLY-NO-ADAPTIVE-POLICY",
|
||||
"peak_CSI_checkpoint": int(score.idxmax()), "peak_CSI": float(score.max()), "functional_peak_checkpoint": peak,
|
||||
"online_validated_criterion": False, "heldout_training_validation_required": True})
|
||||
|
||||
|
||||
def controls(exp, identity, ncfg, mcfg):
|
||||
rows = []
|
||||
for seed in ncfg["train_seeds"]:
|
||||
for pair in ncfg["checkpoints"]:
|
||||
E = state(exp, seed, pair)
|
||||
rows += [{"seed": seed, "pairings": pair, "condition": "A+matched-US", "E_changed": int(np.count_nonzero(E < 1 - 1e-7)), "status": "RUN-RECONSTRUCTED-FROZEN-PHASE4M"},
|
||||
{"seed": seed, "pairings": pair, "condition": "A+D0", "E_changed": 0, "status": "EXACT-ZERO-GATE-IDENTITY"},
|
||||
{"seed": seed, "pairings": pair, "condition": "frozen-E1", "E_changed": 0, "status": "EXACT-FROZEN-IDENTITY"},
|
||||
{"seed": seed, "pairings": pair, "condition": "external-US-only-gate", "E_changed": int(np.count_nonzero(E < 1 - 1e-7)), "status": "IDENTICAL-TO-FROZEN-PHASE4M-RULE"}]
|
||||
pd.DataFrame(rows).to_parquet(exp / "controls/trajectory-status.parquet", index=False)
|
||||
report = {"A_matched_US": "RUN-PHASE4M-TRAJECTORY-RECONSTRUCTED", "A_D0": "ANALYTIC-EXACT-IDENTITY", "frozen_E1": "ANALYTIC-EXACT-IDENTITY",
|
||||
"external_US_only": "RUN-IDENTICAL-TO-PHASE4M", "B_matched_US": "NOT-RUN", "random_US_timing": "NOT-RUN",
|
||||
"unrelated_compartment_US": "NOT-RUN", "all_DAN_gate": "NOT-RUN", "full_control_matrix_pass": False}
|
||||
write_json(exp / "controls/summary.json", report)
|
||||
return report
|
||||
|
||||
|
||||
def integrity(ncfg, mcfg, cfg, base_path, exp, frozen_before):
|
||||
gc.collect(); torch.cuda.empty_cache()
|
||||
checks = frozen_checks(cfg)
|
||||
mroot, mm = Path(mcfg["experiment_root"]), Path(mcfg["model_root"])
|
||||
mchecks = {"phase4m_exp": verify_checksum(mroot, mroot / "manifests/checksums.sha256")["passed"],
|
||||
"phase4m_model": verify_checksum(mm, mm / "checksums.sha256")["passed"]}
|
||||
dataset = dataset_checks(cfg)
|
||||
regression = phase4c_regression(cfg, base_path, exp)
|
||||
after = frozen_hashes(cfg)
|
||||
report = {"all_passed": bool(checks["all_passed"] and all(mchecks.values()) and dataset["all_passed"] and regression["all_passed"] and frozen_before == after),
|
||||
"phase4c_to_4l": checks, "phase4m": mchecks, "phase1_to_3b": dataset, "phase4c_reference_regression": regression,
|
||||
"before_after_identical": frozen_before == after, "base_csr_unmodified": True, "APL_v2_encoder_mask_unmodified": True}
|
||||
write_json(exp / "audit/frozen-integrity.json", report)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("--config", type=Path, required=True); args = ap.parse_args()
|
||||
ncfg, mcfg, cfg, base_path = load_config(args.config)
|
||||
exp = Path(ncfg["experiment_root"])
|
||||
identity = pd.read_parquet(exp / "edge-identity/frozen-edge-identity.parquet")
|
||||
csi = pd.read_parquet(exp / "specificity-curve/CSI.parquet")
|
||||
replay = pd.read_parquet(exp / "specificity-curve/memory-specificity.parquet")
|
||||
history = pd.read_parquet(Path(mcfg["experiment_root"]) / "efficacy-overlay/update-history.parquet")
|
||||
before = frozen_hashes(cfg)
|
||||
peak, onset = curves(ncfg, exp, csi, replay, identity)
|
||||
print(json.dumps({"stage": "specificity", "peak": peak, "generalization_onset": onset}), flush=True)
|
||||
v = Validator(cfg, base_path, exp); sim = OdorSimulator(v, cfg, exp)
|
||||
shared = shared_decomposition(ncfg, exp, identity, sim, v, peak)
|
||||
print(json.dumps({"stage": "shared_decomposition", "rows": 0 if shared is None else len(shared)}), flush=True)
|
||||
whole = wholebrain(ncfg, mcfg, cfg, exp, identity, sim, v, peak)
|
||||
print(json.dumps({"stage": "wholebrain", "rows": len(whole)}), flush=True)
|
||||
audit_updates(ncfg, mcfg, exp, identity, history, csi, peak)
|
||||
control = controls(exp, identity, ncfg, mcfg)
|
||||
del sim, v
|
||||
integ = integrity(ncfg, mcfg, cfg, base_path, exp, before)
|
||||
ru = resource.getrusage(resource.RUSAGE_SELF)
|
||||
perf = {"peak_RAM_KiB": int(ru.ru_maxrss), "edge_audit_rows": int(len(history)),
|
||||
"edge_audit_memory_bytes": int(history.memory_usage(deep=True).sum()), "shadow_trace_overhead": "NOT-MEASURED-NOT-RUN",
|
||||
"wholebrain": json.loads((exp / "performance/whole-brain.json").read_text()),
|
||||
"isolated": json.loads((exp / "performance/isolated.json").read_text())}
|
||||
write_json(exp / "performance/summary.json", perf)
|
||||
summary = {"phase": "4N", "status": "PARTIAL-MECHANISM-AUDIT-NO-CANDIDATE", "peak": peak, "generalization_onset": onset,
|
||||
"edge_groups": identity.identity_group.value_counts().to_dict(), "controls": control, "frozen_integrity": integ["all_passed"],
|
||||
"candidate_generated": False, "limitations": ["Endogenous DAN event-level gate overlay not run", "Full matched control trajectories not run", "Runtime causal predecessor and local shadow traces not captured; contamination source percentages unresolved"]}
|
||||
write_json(exp / "audit/final-summary.json", summary)
|
||||
model = Path(ncfg["model_root"]); model.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(args.config, model / "phase4n.yaml"); shutil.copy2(Path(__file__).resolve(), model / "phase4n_finish.py")
|
||||
shutil.copy2(Path(__file__).with_name("phase4n.py"), model / "phase4n.py")
|
||||
shutil.copy2(Path(__file__).resolve(), exp / "audit/phase4n_finish.py")
|
||||
write_json(model / "VERSION.json", {"phase": "4N", "status": summary["status"], "candidate_generated": False,
|
||||
"identity_sha256": sha256(exp / "edge-identity/frozen-edge-identity.parquet"), "config_sha256": sha256(args.config)})
|
||||
seal(model, model / "MANIFEST.json", model / "checksums.sha256")
|
||||
seal(exp, exp / "manifests/manifest.json", exp / "manifests/checksums.sha256")
|
||||
print(json.dumps(summary), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Timing-compatible incoming KC drive paths; not unique causal-spike proofs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from phase4h_audit import write_json
|
||||
from phase4n import load_config
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument("--config",type=Path,required=True);args=ap.parse_args()
|
||||
ncfg,mcfg,cfg,_=load_config(args.config);exp=Path(ncfg["experiment_root"])
|
||||
identity=pd.read_parquet(exp/"edge-identity/frozen-edge-identity.parquet")
|
||||
bnodes=np.unique(identity.loc[identity.identity_group=="B-specific","source_neuron_idx"].to_numpy(np.int32))
|
||||
root=Path(cfg["paths"]["connectome_root"])
|
||||
indptr=np.load(root/"gpu/csr-indptr.int64.npy",mmap_mode="r")
|
||||
indices=np.load(root/"gpu/csr-indices.int32.npy",mmap_mode="r")
|
||||
raw=np.load(root/"gpu/csr-weights.int64.npy",mmap_mode="r")
|
||||
signs=np.load("/data/digital-drosophila/experiments/phase4a-baseline/outputs/derived/synaptic-sign.int8.npy",mmap_mode="r")
|
||||
meta=pd.read_parquet(root/"neuron-index/unified-metadata.parquet",columns=["neuron_idx","body_id","annotation_class","annotation_type"])
|
||||
meta=meta.sort_values("neuron_idx").reset_index(drop=True)
|
||||
if len(signs)!=len(meta) or len(indptr)!=len(meta)+1:raise RuntimeError("frozen graph/schema mismatch")
|
||||
dan=pd.read_parquet(Path(cfg["paths"]["phase4h_root"])/"mappings/dan-to-mbon-confirmed.parquet",columns=["source_neuron_idx"])
|
||||
danset=set(map(int,dan.source_neuron_idx.unique()))
|
||||
edge_ids=np.flatnonzero(np.isin(np.asarray(indices),bnodes)).astype(np.int64)
|
||||
targets=np.asarray(indices[edge_ids],np.int32)
|
||||
sources=(np.searchsorted(indptr,edge_ids,side="right")-1).astype(np.int32)
|
||||
weights=np.asarray(raw[edge_ids],np.float32)*np.asarray(signs[sources],np.float32)*np.float32(cfg["model"]["w_syn_mv_per_anatomical_synapse"])
|
||||
by_target={int(n):np.flatnonzero(targets==n) for n in bnodes}
|
||||
classes=meta.annotation_class.fillna("").astype(str).to_numpy()
|
||||
def source_class(n):
|
||||
cls=classes[n]
|
||||
if cls in ("ALPN","olfactory"):return "PN/olfactory"
|
||||
if cls=="Kenyon_Cell":return "KC recurrent"
|
||||
if cls=="MBON":return "MBON feedback"
|
||||
if n in danset:return "DAN"
|
||||
if "APL" in str(meta.annotation_type.iat[n]):return "APL anatomical"
|
||||
return "other CNS"
|
||||
rows=[];pairings=[10,20,50,60,75,100]
|
||||
dt=float(cfg["model"]["dt_ms"]);delay=int(round(float(cfg["model"]["synaptic_delay_ms"])/dt))
|
||||
tau=float(cfg["model"]["synaptic_tau_ms"]);window=int(round(5*tau/dt))
|
||||
for seed in ncfg["train_seeds"]:
|
||||
for pair in pairings:
|
||||
file=exp/"whole-brain"/f"spike-events-seed-{seed}-pairings-{pair}.parquet"
|
||||
events=pd.read_parquet(file)
|
||||
steps_by_node={int(n):g.step.to_numpy(np.int32) for n,g in events.groupby("neuron_idx")}
|
||||
for target in bnodes:
|
||||
times=steps_by_node.get(int(target))
|
||||
if times is None or not len(times):continue
|
||||
t=int(times[0])
|
||||
if t>int(round(float(mcfg["protocol"]["dan_time_ms"])/dt)):continue
|
||||
cat_totals=defaultdict(float);rank=[]
|
||||
for position in by_target[int(target)]:
|
||||
src=int(sources[position]);w=float(weights[position]);
|
||||
if w<=0:continue
|
||||
source_steps=steps_by_node.get(src)
|
||||
if source_steps is None:continue
|
||||
arrived=source_steps+delay
|
||||
active=arrived[(arrived<=t)&(arrived>=t-window)]
|
||||
if not len(active):continue
|
||||
contribution=float(np.sum(w*np.exp(-(t-active)*dt/tau)))
|
||||
category=source_class(src);cat_totals[category]+=contribution
|
||||
latest=int(active[-1])
|
||||
rank.append((contribution,int(edge_ids[position]),src,latest,category,w))
|
||||
rank.sort(reverse=True)
|
||||
top=rank[0] if rank else None
|
||||
rows.append({"seed":seed,"pairing":pair,"B_specific_KC_neuron_idx":int(target),"B_specific_KC_body_id":int(meta.body_id.iat[target]),
|
||||
"first_activation_step":t,"first_activation_ms":t*dt,"candidate_predecessor_neuron_idx":None if top is None else top[2],
|
||||
"candidate_predecessor_body_id":None if top is None else int(meta.body_id.iat[top[2]]),
|
||||
"actual_edge_id":None if top is None else top[1],"arrival_ms":None if top is None else top[3]*dt,
|
||||
"candidate_predecessor_population":None if top is None else top[4],
|
||||
"candidate_predecessor_type":None if top is None else str(meta.annotation_type.iat[top[2]]),
|
||||
"anatomical_effective_weight_mv":None if top is None else top[5],
|
||||
"decayed_positive_input_at_spike_mv":None if top is None else top[0],
|
||||
"positive_input_by_source_class_json":json.dumps(dict(cat_totals),sort_keys=True),
|
||||
"APL_graded_inhibition_accounted":False,"unique_causal_predecessor_proven":False,
|
||||
"reason":"Multiple delayed inputs and APL graded inhibition converge; strongest positive timed edge is not a counterfactual causal proof."})
|
||||
table=pd.DataFrame(rows);table.to_parquet(exp/"whole-brain/B-specific-timed-input-candidates.parquet",index=False)
|
||||
updates=pd.read_parquet(exp/"local-eligibility/event-level-updates.parquet")
|
||||
updates=updates[(updates.identity_group=="B-specific")&updates.pairing.isin(pairings)].copy()
|
||||
linked=updates.merge(table,left_on=["seed","pairing","source_neuron_idx"],
|
||||
right_on=["seed","pairing","B_specific_KC_neuron_idx"],how="left",validate="many_to_one")
|
||||
linked.to_parquet(exp/"whole-brain/B-specific-credit-timed-input.parquet",index=False)
|
||||
distribution=table.candidate_predecessor_population.fillna("none").value_counts().to_dict() if len(table) else {}
|
||||
write_json(exp/"whole-brain/B-specific-timed-input-summary.json",{"audited_B_specific_KC":int(len(bnodes)),
|
||||
"rows":int(len(table)),"source_class_of_strongest_timing_compatible_positive_input":distribution,
|
||||
"linked_B_specific_plastic_update_rows":int(len(linked)),
|
||||
"linked_updates_with_timed_input":int(linked.actual_edge_id.notna().sum()),
|
||||
"causal_predecessor_unambiguously_identified":False,
|
||||
"APL_graded_inhibition_in_source_decomposition":False,
|
||||
"five_synaptic_tau_lookback_ms":5*tau,"exact_frozen_edge_and_delay_used":True,
|
||||
"not_interpretable_as_causal_source_percentages":True})
|
||||
print(json.dumps({"B_nodes":len(bnodes),"rows":len(table),"timed_positive_source_distribution":distribution}),flush=True)
|
||||
|
||||
|
||||
if __name__=="__main__":main()
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4N early consistency check for frozen event and DAN audit tables."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import pandas as pd
|
||||
from phase4n import load_config
|
||||
from phase4n_finalize import contamination, dan_summary, shared_summary, separation
|
||||
|
||||
ncfg, mcfg, cfg, _ = load_config(Path(__file__).with_name("phase4n.yaml"))
|
||||
exp = Path(ncfg["experiment_root"])
|
||||
identity = pd.read_parquet(exp / "edge-identity/frozen-edge-identity.parquet")
|
||||
history = pd.read_parquet(Path(mcfg["experiment_root"]) / "efficacy-overlay/update-history.parquet")
|
||||
print(json.dumps({"contamination": contamination(exp, history), "DAN": dan_summary(exp, identity, ncfg),
|
||||
"shared": shared_summary(exp), "wholebrain": separation(exp, 50)}))
|
||||
Reference in New Issue
Block a user