|
|
|
@@ -0,0 +1,387 @@
|
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Phase 4L: physiology-grounded olfactory encoding and LTD magnitude audit."""
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import gc
|
|
|
|
|
import json
|
|
|
|
|
import math
|
|
|
|
|
import resource
|
|
|
|
|
import shutil
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
import time
|
|
|
|
|
from collections import defaultdict
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
import pandas as pd
|
|
|
|
|
import torch
|
|
|
|
|
import yaml
|
|
|
|
|
|
|
|
|
|
from phase4h_audit import sha256, tree_manifest, verify_checksum, write_json
|
|
|
|
|
from phase4i import Validator, pair_metrics, phase4c_regression
|
|
|
|
|
from phase4j import APLSimulator, apply_v2, effect, verify_frozen
|
|
|
|
|
from phase4k import dataset_checks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def seal(root: Path, manifest: Path, checksums: Path) -> None:
|
|
|
|
|
write_json(manifest, tree_manifest(root, {manifest, checksums}))
|
|
|
|
|
files = sorted(x for x in root.rglob("*") if x.is_file() and x != checksums)
|
|
|
|
|
checksums.write_text("".join(f"{sha256(x)} {x.relative_to(root)}\n" for x in files), encoding="utf-8")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def norm_name(value: Any) -> str:
|
|
|
|
|
return "".join(c for c in str(value).upper() if c.isalnum())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def latency(first: np.ndarray, dt: float) -> float | None:
|
|
|
|
|
return None if np.all(first < 0) else float(first[first >= 0].min() * dt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class OdorSimulator(APLSimulator):
|
|
|
|
|
"""Frozen Phase 4J dynamics with a vector-valued PN Poisson interface."""
|
|
|
|
|
|
|
|
|
|
def run_rates(self, targets: np.ndarray, rates_hz: np.ndarray, seed: int, duration_ms: float,
|
|
|
|
|
delta: np.ndarray | None = None, dan_targets: np.ndarray | None = None,
|
|
|
|
|
dan_pulses: list[float] | None = None, collect_kc_spikes: bool = False) -> dict[str, Any]:
|
|
|
|
|
targets = np.asarray(targets, np.int32); rates_hz = np.asarray(rates_hz, np.float32)
|
|
|
|
|
if len(targets) != len(rates_hz) or np.any(rates_hz < 0):
|
|
|
|
|
raise ValueError("invalid PN target/rate vector")
|
|
|
|
|
if delta is None: delta = np.zeros(len(self.v.edge_ids), np.float32)
|
|
|
|
|
self.v.install(delta); self.rt.weights[self.apl_edge_gpu] = 0.0; self.rt.reset(int(seed))
|
|
|
|
|
steps = int(round(float(duration_ms) / self.dt)); tg = torch.as_tensor(targets, dtype=torch.int64, device=self.device)
|
|
|
|
|
rg = torch.as_tensor(rates_hz * self.dt / 1000.0, dtype=torch.float32, device=self.device)
|
|
|
|
|
gen = torch.Generator(device=self.device); gen.manual_seed(int(seed))
|
|
|
|
|
tgen0 = time.perf_counter(); forced = torch.rand((steps, len(targets)), generator=gen, device=self.device) < rg[None, :]
|
|
|
|
|
torch.cuda.synchronize(); generation_wall = time.perf_counter() - tgen0
|
|
|
|
|
dan = np.empty(0, np.int32) if dan_targets is None else np.asarray(dan_targets, np.int32)
|
|
|
|
|
dg = torch.as_tensor(dan, dtype=torch.int64, device=self.device); dan_forced = self._forced(steps, dan, seed + 991, None, dan_pulses)
|
|
|
|
|
dan_steps = {int(round(float(x) / self.dt)) for x in (dan_pulses or [])}
|
|
|
|
|
apl = self.cfg["apl_overlay"]; apl_decay = math.exp(-self.dt / float(apl["tau_ms"])); trace_decay = math.exp(-self.dt / float(self.cfg["eligibility_v2"]["kc_trace_tau_ms"]))
|
|
|
|
|
apl_state = torch.zeros(len(self.apl), dtype=torch.float32, device=self.device); trace = torch.zeros(len(self.kc), dtype=torch.float32, device=self.device); at_dan = []
|
|
|
|
|
counts = torch.zeros(self.v.n, dtype=torch.int32, device=self.device); first = torch.full((self.v.n,), -1, dtype=torch.int32, device=self.device)
|
|
|
|
|
kc_events_step: list[torch.Tensor] = []; kc_events_node: list[torch.Tensor] = []; accepted_total = torch.zeros((), dtype=torch.int64, device=self.device); overlay_submit = 0.0
|
|
|
|
|
alpha, gd, rest, reset, threshold, refsteps = self.rt.alpha, self.rt.g_decay, self.rt.rest, self.rt.reset_value, self.rt.threshold, self.rt.refractory_steps
|
|
|
|
|
torch.cuda.synchronize(); torch.cuda.reset_peak_memory_stats(); t0 = time.perf_counter()
|
|
|
|
|
for step in range(steps):
|
|
|
|
|
slot = step % self.rt.queue_len; self.rt.g.mul_(gd).add_(self.rt.delay_queue[slot]); self.rt.delay_queue[slot].zero_()
|
|
|
|
|
self.rt.refractory.clamp_min_(1).sub_(1); available = self.rt.refractory == 0
|
|
|
|
|
self.rt.v[available] += alpha * (self.rt.g[available] - (self.rt.v[available] - rest)); self.rt.v[~available] = reset
|
|
|
|
|
spikes = available & (self.rt.v >= threshold)
|
|
|
|
|
accepted = forced[step] & available[tg]
|
|
|
|
|
if accepted.any(): spikes[tg[accepted]] = True
|
|
|
|
|
accepted_total.add_(accepted.sum())
|
|
|
|
|
if len(dan):
|
|
|
|
|
da = dan_forced[step] & available[dg]
|
|
|
|
|
if da.any(): spikes[dg[da]] = True
|
|
|
|
|
sources = torch.nonzero(spikes, as_tuple=False).flatten(); counts.add_(spikes.to(torch.int32)); new = spikes & (first < 0); first[new] = step
|
|
|
|
|
if collect_kc_spikes:
|
|
|
|
|
k = torch.nonzero(spikes[self.kc_gpu], as_tuple=False).flatten()
|
|
|
|
|
if k.numel():
|
|
|
|
|
kc_events_step.append(torch.full((k.numel(),), step, dtype=torch.int32, device=self.device)); kc_events_node.append(self.kc_gpu[k].to(torch.int32))
|
|
|
|
|
if sources.numel():
|
|
|
|
|
self.rt.v[sources] = reset; self.rt.refractory[sources] = refsteps; eids = self.rt._hybrid_edges(sources)
|
|
|
|
|
if eids.numel(): self.rt.delay_queue[(step + self.rt.delay_steps) % self.rt.queue_len].scatter_add_(0, self.rt.indices[eids].long(), self.rt.weights[eids])
|
|
|
|
|
trace.mul_(trace_decay).add_(spikes[self.kc_gpu].to(torch.float32)).clamp_max_(1.0)
|
|
|
|
|
if step in dan_steps: at_dan.append(trace.clone())
|
|
|
|
|
ot = time.perf_counter(); apl_state.mul_(apl_decay).add_(torch.mv(self.input_gpu, spikes.to(torch.float32)) * float(apl["normalization"]))
|
|
|
|
|
inhibition = torch.matmul(apl_state, self.output_gpu) * float(apl["gain"]); self.rt.g.index_add_(0, self.kc_gpu, -inhibition); overlay_submit += time.perf_counter() - ot
|
|
|
|
|
torch.cuda.synchronize(); wall = time.perf_counter() - t0; cn = counts.cpu().numpy(); fn = first.cpu().numpy(); kcn = cn[self.kc]
|
|
|
|
|
return {"counts": cn, "kc_counts": kcn, "mbon_counts": cn[self.v.mbon], "downstream_counts": cn[self.v.downstream],
|
|
|
|
|
"first_steps": fn, "kc_first_steps": fn[self.kc].copy(), "mbon_first_steps": fn[self.v.mbon].copy(), "dt_ms": self.dt,
|
|
|
|
|
"kc_first_latency_ms": latency(fn[self.kc], self.dt), "tracked_mbon_v": self.rt.v[torch.as_tensor(self.v.mbon, device=self.device)].cpu().numpy().copy(),
|
|
|
|
|
"tracked_mbon_g": self.rt.g[torch.as_tensor(self.v.mbon, device=self.device)].cpu().numpy().copy(),
|
|
|
|
|
"kc_trace_at_dan": np.zeros(len(self.kc), np.float32) if not at_dan else at_dan[-1].cpu().numpy().copy(),
|
|
|
|
|
"wall_seconds": wall, "pn_generation_wall_seconds": generation_wall, "overlay_cpu_submit_seconds": overlay_submit,
|
|
|
|
|
"peak_vram_bytes": int(torch.cuda.max_memory_allocated()), "finite": bool(torch.isfinite(self.rt.v).all() and torch.isfinite(self.rt.g).all()),
|
|
|
|
|
"accepted_external_events": int(accepted_total.cpu()), "expected_external_events": float(rates_hz.sum() * duration_ms / 1000.0),
|
|
|
|
|
"kc_spikes": pd.DataFrame({"step": torch.cat(kc_events_step).cpu().numpy(), "neuron_idx": torch.cat(kc_events_node).cpu().numpy()}) if kc_events_step else pd.DataFrame(columns=["step", "neuron_idx"])}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_external(cfg: dict[str, Any], exp: Path) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict[str, Any]]:
|
|
|
|
|
root = Path(cfg["paths"]["external_door_root"]); data = root / "data"
|
|
|
|
|
files = [root / "DESCRIPTION", data / "door_mappings.csv", data / "door_response_matrix.csv", data / "odor.csv"]
|
|
|
|
|
if not all(x.is_file() for x in files): raise FileNotFoundError("DoOR.data v2.0.1 resource files missing")
|
|
|
|
|
commit = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
|
|
|
|
|
if commit != cfg["external_resource"]["git_commit"]: raise RuntimeError(f"DoOR commit mismatch: {commit}")
|
|
|
|
|
mapping = pd.read_csv(files[1], sep=";", index_col=0); response = pd.read_csv(files[2], sep=";", index_col=0); odors = pd.read_csv(files[3], sep=";", index_col=0)
|
|
|
|
|
report = dict(cfg["external_resource"]); report.update({"actual_git_commit": commit, "files": [{"path": str(x), "bytes": x.stat().st_size, "sha256": sha256(x)} for x in files],
|
|
|
|
|
"response_rows": int(len(response)), "response_channels": int(len(response.columns)), "mapping_rows": int(len(mapping)), "odor_rows": int(len(odors))})
|
|
|
|
|
write_json(exp / "external-resources/provenance.json", report); return mapping, response, odors, report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def build_mapping(v: Validator, sim: OdorSimulator, door: pd.DataFrame, channels: list[str], exp: Path) -> tuple[pd.DataFrame, dict[str, Any]]:
|
|
|
|
|
meta = v.meta; pn = np.flatnonzero(v.cls == "ALPN").astype(np.int32); pntype = v.types[pn]
|
|
|
|
|
prefixes = np.asarray([x.split("_", 1)[0] if "_" in x else x for x in pntype], object)
|
|
|
|
|
by_exact = defaultdict(list); by_norm = defaultdict(list)
|
|
|
|
|
for n, p in zip(pn, prefixes):
|
|
|
|
|
if p: by_exact[str(p)].append(int(n)); by_norm[norm_name(p)].append((str(p), int(n)))
|
|
|
|
|
rows = []
|
|
|
|
|
for channel in channels:
|
|
|
|
|
d = door.loc[door["receptor"].astype(str) == str(channel)].copy(); gloms = sorted({str(x) for x in d["glomerulus"].dropna() if str(x) not in ("", "?")})
|
|
|
|
|
if len(gloms) != 1 or "+" in gloms[0]:
|
|
|
|
|
rows.append({"odor_channel": channel, "receptor": channel, "glomerulus": "+".join(gloms), "MaleCNS_PN_body_id": None, "neuron_idx": None,
|
|
|
|
|
"PN_type": None, "mapping_confidence": "ambiguous" if gloms else "unmatched", "provenance": "DoOR.data v2.0.1 door_mappings"}); continue
|
|
|
|
|
g = gloms[0]; exact = by_exact.get(g, []); normalized = by_norm.get(norm_name(g), [])
|
|
|
|
|
if exact: selected, confidence = exact, "exact"
|
|
|
|
|
elif normalized and len({x[0] for x in normalized}) == 1: selected, confidence = [x[1] for x in normalized], "alias-normalized"
|
|
|
|
|
elif normalized: selected, confidence = [], "ambiguous"
|
|
|
|
|
else: selected, confidence = [], "unmatched"
|
|
|
|
|
if not selected:
|
|
|
|
|
rows.append({"odor_channel": channel, "receptor": channel, "glomerulus": g, "MaleCNS_PN_body_id": None, "neuron_idx": None,
|
|
|
|
|
"PN_type": None, "mapping_confidence": confidence, "provenance": "DoOR.data v2.0.1 + MaleCNS annotation_type glomerulus prefix"})
|
|
|
|
|
for n in selected:
|
|
|
|
|
rows.append({"odor_channel": channel, "receptor": channel, "glomerulus": g, "MaleCNS_PN_body_id": int(meta.loc[n, "body_id"]), "neuron_idx": int(n),
|
|
|
|
|
"PN_type": str(v.types[n]), "mapping_confidence": confidence, "provenance": "DoOR.data v2.0.1 + exact/normalized MaleCNS ALPN annotation_type prefix"})
|
|
|
|
|
out = pd.DataFrame(rows); out.to_parquet(exp / "receptor-mapping/olfactory-mapping.parquet", index=False)
|
|
|
|
|
usable = out[out.mapping_confidence.isin(["exact", "alias-normalized"]) & out.neuron_idx.notna()]
|
|
|
|
|
channel_status = out.groupby("odor_channel").mapping_confidence.first(); eids = np.flatnonzero(np.isin(sim.sources, pn) & np.isin(v.rt.indices_np, v.kc))
|
|
|
|
|
summary = {"DoOR_channels": len(channels), "exact_channels": int((channel_status == "exact").sum()), "alias_normalized_channels": int((channel_status == "alias-normalized").sum()),
|
|
|
|
|
"ambiguous_channels": int((channel_status == "ambiguous").sum()), "unmatched_channels": int((channel_status == "unmatched").sum()),
|
|
|
|
|
"mapped_channels": int(channel_status.isin(["exact", "alias-normalized"]).sum()), "mapped_fraction": float(channel_status.isin(["exact", "alias-normalized"]).mean()),
|
|
|
|
|
"mapped_glomeruli": int(usable.glomerulus.nunique()), "mapped_MaleCNS_ALPN": int(usable.neuron_idx.nunique()), "MaleCNS_ALPN_total": int(len(pn)),
|
|
|
|
|
"real_ALPN_to_KC_edges": int(len(eids)), "real_ALPN_to_KC_synapse_weight": int(v.rt.raw_np[eids].sum()), "ambiguous_not_used": True}
|
|
|
|
|
write_json(exp / "glomerulus-mapping/summary.json", summary); return out, summary
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def prepare_odors(response: pd.DataFrame, odors: pd.DataFrame, mapping: pd.DataFrame, cfg: dict[str, Any], exp: Path) -> tuple[pd.DataFrame, dict[str, np.ndarray], dict[str, Any]]:
|
|
|
|
|
usable = mapping[mapping.mapping_confidence.isin(["exact", "alias-normalized"]) & mapping.neuron_idx.notna()].copy()
|
|
|
|
|
channel_glom = usable[["odor_channel", "glomerulus"]].drop_duplicates(); gloms = sorted(channel_glom.glomerulus.unique()); gidx = {x: i for i, x in enumerate(gloms)}
|
|
|
|
|
name_by_key = odors.dropna(subset=["InChIKey"]).drop_duplicates("InChIKey").set_index("InChIKey")["Name"].to_dict()
|
|
|
|
|
candidates = []; vectors: dict[str, np.ndarray] = {}
|
|
|
|
|
for key, row in response.iterrows():
|
|
|
|
|
if str(key) == "SFR": continue
|
|
|
|
|
name = name_by_key.get(key);
|
|
|
|
|
if pd.isna(name) or not str(name).strip(): continue
|
|
|
|
|
arr = np.zeros(len(gloms), np.float32)
|
|
|
|
|
for _, q in channel_glom.iterrows():
|
|
|
|
|
value = row.get(q.odor_channel)
|
|
|
|
|
if pd.notna(value): arr[gidx[q.glomerulus]] = max(arr[gidx[q.glomerulus]], max(float(value), 0.0))
|
|
|
|
|
all_values = np.nan_to_num(row.to_numpy(np.float64), nan=0.0); all_values = np.maximum(all_values, 0)
|
|
|
|
|
total, mapped = float(all_values.sum()), float(arr.sum()); strong = int(np.count_nonzero(arr >= float(cfg["odor_encoder"]["strong_response_threshold"])))
|
|
|
|
|
if mapped <= 0 or strong < int(cfg["odor_encoder"]["minimum_mapped_strong_channels"]): continue
|
|
|
|
|
oid = str(key); vectors[oid] = arr; candidates.append({"odor_id": oid, "odor_name": str(name), "mapped_response_sum": mapped,
|
|
|
|
|
"all_response_sum": total, "mapped_signal_fraction": mapped / total if total else 0.0, "mapped_strong_glomeruli": strong, "mapped_nonzero_glomeruli": int(np.count_nonzero(arr))})
|
|
|
|
|
ranked = pd.DataFrame(candidates).sort_values(["mapped_signal_fraction", "mapped_strong_glomeruli", "odor_name"], ascending=[False, False, True]).drop_duplicates("odor_name")
|
|
|
|
|
need = int(cfg["odor_encoder"]["calibration_odors"]) + int(cfg["odor_encoder"]["heldout_odors"])
|
|
|
|
|
panel = ranked.head(need).copy().reset_index(drop=True)
|
|
|
|
|
if len(panel) != need: raise RuntimeError("insufficient physiology odors after frozen coverage filter")
|
|
|
|
|
nc = int(cfg["odor_encoder"]["calibration_odors"]); panel["split"] = ["calibration"] * nc + ["heldout"] * (len(panel) - nc)
|
|
|
|
|
held = panel[panel.split == "heldout"]; best = None
|
|
|
|
|
for i in range(len(held)):
|
|
|
|
|
for j in range(i + 1, len(held)):
|
|
|
|
|
a, b = held.iloc[i], held.iloc[j]; x, y = vectors[a.odor_id], vectors[b.odor_id]
|
|
|
|
|
cosine = float(np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y))) if np.linalg.norm(x) and np.linalg.norm(y) else 1.0
|
|
|
|
|
item = (cosine, str(a.odor_name), str(b.odor_name), str(a.odor_id), str(b.odor_id))
|
|
|
|
|
if best is None or item < best: best = item
|
|
|
|
|
assert best is not None; _, an, bn, aid, bid = best; panel["formal_label"] = ""; panel.loc[panel.odor_id == aid, "formal_label"] = "ODOR_A"; panel.loc[panel.odor_id == bid, "formal_label"] = "ODOR_B"
|
|
|
|
|
panel.to_parquet(exp / "odor-encoder/frozen-odor-panel.parquet", index=False)
|
|
|
|
|
np.savez_compressed(exp / "odor-encoder/mapped-glomerular-vectors.npz", glomeruli=np.asarray(gloms), **{x: vectors[x] for x in panel.odor_id})
|
|
|
|
|
selection = {"policy": cfg["odor_encoder"]["selection_policy"], "selected_before_KC_simulation": True,
|
|
|
|
|
"ODOR_A": {"odor_id": aid, "odor_name": an}, "ODOR_B": {"odor_id": bid, "odor_name": bn}, "external_cosine_similarity": best[0],
|
|
|
|
|
"panel": panel.to_dict(orient="records"), "glomeruli_order": gloms}
|
|
|
|
|
write_json(exp / "odor-encoder/selection.json", selection); return panel, vectors, selection
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def pn_drive(odor_id: str, vector: np.ndarray, mapping: pd.DataFrame, gloms: list[str], mode: str, matched_sum: float, cfg: dict[str, Any]) -> tuple[np.ndarray, np.ndarray, pd.DataFrame]:
|
|
|
|
|
scale = 1.0 if mode == "physiological" else (matched_sum / float(vector.sum()) if vector.sum() else 0.0)
|
|
|
|
|
rows = []
|
|
|
|
|
for gi, glom in enumerate(gloms):
|
|
|
|
|
pns = mapping[(mapping.glomerulus == glom) & mapping.mapping_confidence.isin(["exact", "alias-normalized"]) & mapping.neuron_idx.notna()].neuron_idx.astype(int).unique()
|
|
|
|
|
if len(pns) == 0 or vector[gi] <= 0: continue
|
|
|
|
|
total_rate = float(cfg["odor_encoder"]["max_total_rate_per_glomerulus_hz"]) * float(vector[gi]) * scale
|
|
|
|
|
for pn in pns: rows.append({"odor_id": odor_id, "mode": mode, "glomerulus": glom, "neuron_idx": int(pn), "response": float(vector[gi]), "scale": scale, "rate_hz": total_rate / len(pns)})
|
|
|
|
|
frame = pd.DataFrame(rows); return frame.neuron_idx.to_numpy(np.int32), frame.rate_hz.to_numpy(np.float32), frame
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def representation(sim: OdorSimulator, panel: pd.DataFrame, vectors: dict[str, np.ndarray], selection: dict[str, Any], mapping: pd.DataFrame,
|
|
|
|
|
cfg: dict[str, Any], exp: Path) -> tuple[dict[str, Any], dict[tuple[str, str, int], dict[str, Any]]]:
|
|
|
|
|
gloms = selection["glomeruli_order"]; matched_sum = float(np.median(panel.mapped_response_sum)); seeds = list(map(int, cfg["odor_encoder"]["evaluation_seeds"])); runs = {}; records = []; subtype_rows = []; drive_rows = []
|
|
|
|
|
subtype = sim.v.meta.loc[sim.v.kc, "annotation_type"].fillna("<null>").astype(str).to_numpy()
|
|
|
|
|
for mode in cfg["odor_encoder"]["modes"]:
|
|
|
|
|
for _, odor in panel.iterrows():
|
|
|
|
|
targets, rates, drive = pn_drive(odor.odor_id, vectors[odor.odor_id], mapping, gloms, mode, matched_sum, cfg); drive_rows.append(drive)
|
|
|
|
|
for seed in seeds:
|
|
|
|
|
run = sim.run_rates(targets, rates, seed, float(cfg["odor_encoder"]["duration_ms"]), collect_kc_spikes=True); runs[(mode, odor.odor_id, seed)] = run
|
|
|
|
|
active = int(np.count_nonzero(run["kc_counts"])); nonzero = run["kc_counts"] > 0
|
|
|
|
|
records.append({"mode": mode, "split": odor.split, "odor_id": odor.odor_id, "odor_name": odor.odor_name, "formal_label": odor.formal_label, "seed": seed,
|
|
|
|
|
"KC_active": active, "KC_active_fraction": active / len(sim.v.kc), "KC_spikes": int(run["kc_counts"].sum()), "KC_first_latency_ms": run["kc_first_latency_ms"],
|
|
|
|
|
"external_events": run["accepted_external_events"], "expected_external_events": run["expected_external_events"], "total_PN_rate_hz": float(rates.sum()),
|
|
|
|
|
"finite": run["finite"], "wall_seconds": run["wall_seconds"], "PN_event_generation_seconds": run["pn_generation_wall_seconds"], "APL_submit_seconds": run["overlay_cpu_submit_seconds"], "peak_vram_bytes": run["peak_vram_bytes"]})
|
|
|
|
|
for label in np.unique(subtype):
|
|
|
|
|
mask = subtype == label; subtype_rows.append({"mode": mode, "odor_id": odor.odor_id, "seed": seed, "KC_subtype": label, "total": int(mask.sum()), "active": int(np.count_nonzero(nonzero[mask])), "spikes": int(run["kc_counts"][mask].sum())})
|
|
|
|
|
pd.concat(drive_rows, ignore_index=True).to_parquet(exp / "pn-mapping/odor-PN-drive.parquet", index=False); frame = pd.DataFrame(records); frame.to_parquet(exp / "kc-representation/runs.parquet", index=False); pd.DataFrame(subtype_rows).to_parquet(exp / "kc-representation/subtypes.parquet", index=False)
|
|
|
|
|
summaries = {}
|
|
|
|
|
for mode in cfg["odor_encoder"]["modes"]:
|
|
|
|
|
within = []; within_j = []; between = []; between_j = []; between_cos = []; between_corr = []; pair_pass = 0; odor_ids = list(panel.odor_id)
|
|
|
|
|
for oid in odor_ids:
|
|
|
|
|
arr = [runs[(mode, oid, s)]["kc_counts"] for s in seeds]
|
|
|
|
|
for i in range(len(arr)):
|
|
|
|
|
for j in range(i + 1, len(arr)):
|
|
|
|
|
q = pair_metrics(arr[i], arr[j]); within.append(q["euclidean_distance"]); within_j.append(q["active_jaccard"])
|
|
|
|
|
for i in range(len(odor_ids)):
|
|
|
|
|
for j in range(i + 1, len(odor_ids)):
|
|
|
|
|
ds = []
|
|
|
|
|
for seed in seeds:
|
|
|
|
|
q = pair_metrics(runs[(mode, odor_ids[i], seed)]["kc_counts"], runs[(mode, odor_ids[j], seed)]["kc_counts"]); ds.append(q["euclidean_distance"]); between.append(q["euclidean_distance"]); between_j.append(q["active_jaccard"]); between_cos.append(q["cosine_similarity"]); between_corr.append(q["correlation"])
|
|
|
|
|
wi = []
|
|
|
|
|
for oid in (odor_ids[i], odor_ids[j]):
|
|
|
|
|
aa = [runs[(mode, oid, s)]["kc_counts"] for s in seeds]
|
|
|
|
|
wi += [pair_metrics(aa[x], aa[y])["euclidean_distance"] for x in range(len(aa)) for y in range(x + 1, len(aa))]
|
|
|
|
|
pair_pass += int(float(np.mean(ds)) > float(np.mean(wi)))
|
|
|
|
|
f = frame[frame["mode"] == mode]; summary = {"KC_active_fraction_mean": float(f.KC_active_fraction.mean()), "KC_active_fraction_min": float(f.KC_active_fraction.min()), "KC_active_fraction_max": float(f.KC_active_fraction.max()),
|
|
|
|
|
"same_odor_jaccard_mean": float(np.mean(within_j)), "different_odor_jaccard_mean": float(np.mean(between_j)), "same_odor_euclidean_mean": float(np.mean(within)), "different_odor_euclidean_mean": float(np.mean(between)),
|
|
|
|
|
"different_odor_cosine_mean": float(np.mean(between_cos)), "different_odor_correlation_mean": float(np.mean(between_corr)), "within_variance": float(np.var(within)), "between_distance_mean": float(np.mean(between)),
|
|
|
|
|
"pairwise_separation_pass_count": pair_pass, "pair_count": len(odor_ids) * (len(odor_ids) - 1) // 2,
|
|
|
|
|
"expected_event_count_cv": float(f.expected_external_events.std() / f.expected_external_events.mean()), "actual_event_count_cv": float(f.external_events.std() / f.external_events.mean())}
|
|
|
|
|
summary["gate"] = bool(float(cfg["gates"]["sparse_min_fraction"]) <= summary["KC_active_fraction_mean"] <= float(cfg["gates"]["sparse_max_fraction"]) and summary["KC_active_fraction_max"] < 0.5 and summary["KC_active_fraction_min"] > 0 and summary["same_odor_jaccard_mean"] > summary["different_odor_jaccard_mean"] and summary["different_odor_euclidean_mean"] > summary["same_odor_euclidean_mean"] and pair_pass >= int(cfg["gates"]["minimum_separable_odor_pairs"]))
|
|
|
|
|
summaries[mode] = summary
|
|
|
|
|
candidate = bool(all(x["gate"] for x in summaries.values())); report = {"status": "MaleCNS-OlfactoryEncoder-v1-candidate" if candidate else "FAIL-PHYSIOLOGY-GROUNDED-ODOR-ENCODING", "candidate_generated": candidate, "modes": summaries, "seeds": seeds, "APL_parameters_locked": cfg["apl_overlay"]}
|
|
|
|
|
write_json(exp / "kc-representation/summary.json", report); return report, runs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def eligible_edges(v: Validator, runs: dict[tuple[str, str, int], dict[str, Any]], selection: dict[str, Any], cfg: dict[str, Any], exp: Path) -> dict[str, Any]:
|
|
|
|
|
seeds = list(map(int, cfg["odor_encoder"]["evaluation_seeds"])); aid = selection["ODOR_A"]["odor_id"]; bid = selection["ODOR_B"]["odor_id"]
|
|
|
|
|
def consensus(oid: str) -> np.ndarray:
|
|
|
|
|
active = np.stack([runs[("matched_energy", oid, s)]["kc_counts"] > 0 for s in seeds]); return v.kc[active.sum(axis=0) >= int(cfg["gates"]["seed_majority"])]
|
|
|
|
|
a, b = consensus(aid), consensus(bid); shared = np.intersect1d(a, b); ao = np.setdiff1d(a, b); bo = np.setdiff1d(b, a)
|
|
|
|
|
src = v.cand.source_neuron_idx.to_numpy(np.int32); category = np.full(len(src), "inactive", object); category[np.isin(src, shared)] = "shared"; category[np.isin(src, ao)] = "A-specific"; category[np.isin(src, bo)] = "B-specific"
|
|
|
|
|
out = v.cand.copy(); out["odor_eligibility_category"] = category; out.to_parquet(exp / "eligible-edges/edge-identity.parquet", index=False)
|
|
|
|
|
nodes = []
|
|
|
|
|
for label, values in (("A-specific", ao), ("B-specific", bo), ("shared", shared)):
|
|
|
|
|
for n in values: nodes.append({"category": label, "neuron_idx": int(n), "body_id": int(v.meta.loc[n, "body_id"]), "annotation_type": str(v.types[n])})
|
|
|
|
|
pd.DataFrame(nodes).to_parquet(exp / "eligible-edges/KC-identity.parquet", index=False)
|
|
|
|
|
aset, bset = set(np.flatnonzero(category == "A-specific")), set(np.flatnonzero(category == "B-specific")); aj = len(aset & bset) / len(aset | bset) if aset | bset else 1.0
|
|
|
|
|
report = {"ODOR_A_active_KC": int(len(a)), "ODOR_B_active_KC": int(len(b)), "A_specific_KC": int(len(ao)), "B_specific_KC": int(len(bo)), "shared_KC": int(len(shared)),
|
|
|
|
|
"A_specific_eligible_edges": int(np.count_nonzero(category == "A-specific")), "B_specific_eligible_edges": int(np.count_nonzero(category == "B-specific")), "shared_eligible_edges": int(np.count_nonzero(category == "shared")),
|
|
|
|
|
"inactive_edges": int(np.count_nonzero(category == "inactive")), "eligible_edge_Jaccard": aj,
|
|
|
|
|
"identity_gate": bool(len(ao) >= int(cfg["gates"]["minimum_specific_kc"]) and len(bo) >= int(cfg["gates"]["minimum_specific_kc"]) and np.any(category == "A-specific") and np.any(category == "B-specific"))}
|
|
|
|
|
write_json(exp / "eligible-edges/summary.json", report); return report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def replay_one(sim: OdorSimulator, spikes: pd.DataFrame, targets: np.ndarray, delta: np.ndarray, duration_ms: float) -> dict[str, Any]:
|
|
|
|
|
sources, indices = sim.sources, sim.v.rt.indices_np; base = sim.v.base.detach().cpu().numpy(); full_delta = np.zeros(len(base), np.float32); full_delta[sim.v.edge_ids] = delta
|
|
|
|
|
mask = np.isin(sources, sim.v.kc) & np.isin(indices, targets); eids = np.flatnonzero(mask); lookup = {int(x): i for i, x in enumerate(targets)}; by = {}
|
|
|
|
|
for s in np.unique(sources[eids]):
|
|
|
|
|
ee = eids[sources[eids] == s]; by[int(s)] = (np.asarray([lookup[int(x)] for x in indices[ee]], np.int32), (base[ee] + full_delta[ee]).astype(np.float32))
|
|
|
|
|
grouped = {int(k): g.neuron_idx.to_numpy(np.int32) for k, g in spikes.groupby("step")}; steps = int(round(duration_ms / sim.dt)); q = np.zeros((sim.rt.queue_len, len(targets)), np.float32); g = np.zeros(len(targets), np.float32); vv = np.full(len(targets), sim.rt.rest, np.float32); ref = np.zeros(len(targets), np.int32); counts = np.zeros(len(targets), np.int32); first = np.full(len(targets), -1, np.int32)
|
|
|
|
|
for step in range(steps):
|
|
|
|
|
slot = step % sim.rt.queue_len; g = g * np.float32(sim.rt.g_decay) + q[slot]; q[slot].fill(0); ref = np.maximum(ref, 1) - 1; avail = ref == 0; vv[avail] += np.float32(sim.rt.alpha) * (g[avail] - (vv[avail] - np.float32(sim.rt.rest))); vv[~avail] = np.float32(sim.rt.reset_value)
|
|
|
|
|
fired = avail & (vv >= np.float32(sim.rt.threshold)); counts += fired; first[(first < 0) & fired] = step; vv[fired] = np.float32(sim.rt.reset_value); ref[fired] = sim.rt.refractory_steps
|
|
|
|
|
for s in grouped.get(step, np.empty(0, np.int32)):
|
|
|
|
|
if int(s) in by:
|
|
|
|
|
loc, w = by[int(s)]; np.add.at(q[(step + sim.rt.delay_steps) % sim.rt.queue_len], loc, w)
|
|
|
|
|
return {"counts": counts, "first": first, "g": g, "v": vv}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def replay_effect(pre: dict[str, Any], post: dict[str, Any], dt: float) -> dict[str, Any]:
|
|
|
|
|
lp, lq = latency(pre["first"], dt), latency(post["first"], dt)
|
|
|
|
|
return {"delta_g_max_abs": float(np.max(np.abs(post["g"] - pre["g"]))), "delta_v_max_abs": float(np.max(np.abs(post["v"] - pre["v"]))),
|
|
|
|
|
"spike_l1": int(np.abs(post["counts"] - pre["counts"]).sum()), "pre_latency_ms": lp, "post_latency_ms": lq, "latency_delta_ms": None if lp is None or lq is None else float(lq - lp)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def literature_report(cfg: dict[str, Any], exp: Path) -> dict[str, Any]:
|
|
|
|
|
report = {"frozen_before_magnitude_sweep": True, "sources": [
|
|
|
|
|
{"citation": "Handler et al., Cell 2022", "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC9012144/", "use": "KC/DAN timing, depression/potentiation order dependence, behavioral protocols up to 50 trials"},
|
|
|
|
|
{"citation": "Hige et al., Neuron 2015", "url": "https://pubmed.ncbi.nlm.nih.gov/26637800/", "use": "compartmentalized KC-MBON synaptic plasticity"},
|
|
|
|
|
{"citation": "Cohn et al., Cell 2015", "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4732734/", "use": "localized learning-related MB circuit plasticity"},
|
|
|
|
|
{"citation": "Cyclic nucleotide dynamics study, 2024", "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11068490/", "use": "intense direct-dopamine protocol produced roughly 55% depression after one minute; range context only, not eta calibration"}],
|
|
|
|
|
"engineering_assumption": cfg["plasticity_accumulation"]["explicit_engineering_assumption"], "eta_source": "frozen Phase 4J TemporalEligibility-v2; not inferred from Phase 4K 5-percent sensitivity", "saturation": "frozen -25% per-edge floor", "recovery": "none within the short pairing sweep", "bidirectional_rule_modified": False}
|
|
|
|
|
write_json(exp / "plasticity-magnitude/literature-provenance.json", report); return report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def magnitude_sweep(v: Validator, sim: OdorSimulator, panel: pd.DataFrame, vectors: dict[str, np.ndarray], selection: dict[str, Any], mapping: pd.DataFrame,
|
|
|
|
|
cfg: dict[str, Any], exp: Path) -> tuple[dict[str, Any], dict[int, dict[int, np.ndarray]]]:
|
|
|
|
|
spec = cfg["plasticity_accumulation"]; points = list(map(int, spec["pairing_counts"])); max_pair = max(points); gloms = selection["glomeruli_order"]; matched_sum = float(np.median(panel.mapped_response_sum)); aid, bid = selection["ODOR_A"]["odor_id"], selection["ODOR_B"]["odor_id"]
|
|
|
|
|
at, ar, _ = pn_drive(aid, vectors[aid], mapping, gloms, "physiological", matched_sum, cfg); bt, br, _ = pn_drive(bid, vectors[bid], mapping, gloms, "physiological", matched_sum, cfg)
|
|
|
|
|
dan = v.dan_targets(str(spec["matched_dan_type"]), str(spec["target_compartment_verbatim"])); rows = []; overlays: dict[int, dict[int, np.ndarray]] = {}; update_wall = []
|
|
|
|
|
target_comp = v.cand.target_compartment_verbatim.astype(str).to_numpy() == str(spec["target_compartment_verbatim"]); target_mbon = np.unique(v.cand.loc[target_comp, "target_neuron_idx"].to_numpy(np.int32))
|
|
|
|
|
for tsi, train_seed in enumerate(map(int, spec["train_seeds"])):
|
|
|
|
|
delta = np.zeros(len(v.edge_ids), np.float32); overlays[train_seed] = {}
|
|
|
|
|
for pairing in range(1, max_pair + 1):
|
|
|
|
|
run = sim.run_rates(at, ar, train_seed + pairing, float(spec["duration_ms"]), delta=delta, dan_targets=dan, dan_pulses=[float(spec["dan_time_ms"])])
|
|
|
|
|
u0 = time.perf_counter(); delta, rec = apply_v2(v, run["kc_trace_at_dan"], str(spec["target_compartment_verbatim"]), 1.0, delta, cfg); update_wall.append(time.perf_counter() - u0)
|
|
|
|
|
if pairing not in points: continue
|
|
|
|
|
overlays[train_seed][pairing] = delta.copy(); changed = delta != 0; denom = float(np.abs(v.base_candidate[changed]).sum()); pct = float(100 * np.abs(delta[changed]).sum() / denom) if denom else 0.0; lower = -float(cfg["eligibility_v2"]["max_fractional_depression"]) * v.base_candidate
|
|
|
|
|
eval_seed = int(spec["evaluation_seeds"][tsi]); prea = sim.run_rates(at, ar, eval_seed, float(spec["duration_ms"]), collect_kc_spikes=True); posta = sim.run_rates(at, ar, eval_seed, float(spec["duration_ms"]), delta=delta, collect_kc_spikes=True); preb = sim.run_rates(bt, br, eval_seed, float(spec["duration_ms"]), collect_kc_spikes=True); postb = sim.run_rates(bt, br, eval_seed, float(spec["duration_ms"]), delta=delta, collect_kc_spikes=True)
|
|
|
|
|
whole_a, whole_b = effect(prea, posta), effect(preb, postb); ia0 = replay_one(sim, prea["kc_spikes"], target_mbon, np.zeros(len(delta), np.float32), float(spec["duration_ms"])); ia1 = replay_one(sim, prea["kc_spikes"], target_mbon, delta, float(spec["duration_ms"])); ib0 = replay_one(sim, preb["kc_spikes"], target_mbon, np.zeros(len(delta), np.float32), float(spec["duration_ms"])); ib1 = replay_one(sim, preb["kc_spikes"], target_mbon, delta, float(spec["duration_ms"])); iso_a, iso_b = replay_effect(ia0, ia1, sim.dt), replay_effect(ib0, ib1, sim.dt)
|
|
|
|
|
vals = delta[changed]
|
|
|
|
|
rows.append({"train_seed": train_seed, "eval_seed": eval_seed, "pairings": pairing, "updated_edges": int(changed.sum()), "mean_delta_mv": float(vals.mean()) if len(vals) else 0.0, "median_delta_mv": float(np.median(vals)) if len(vals) else 0.0, "max_abs_delta_mv": float(np.abs(vals).max()) if len(vals) else 0.0, "total_abs_delta_mv": float(np.abs(vals).sum()), "depression_percent_changed_base": pct,
|
|
|
|
|
"saturation_fraction": float(np.isclose(delta, lower, rtol=1e-5, atol=1e-7).sum() / len(delta)), "whole_A_MBON_delta_g": whole_a["MBON_g_max_abs"], "whole_A_MBON_delta_v": whole_a["MBON_v_max_abs"], "whole_A_MBON_spike_l1": whole_a["MBON_spike_l1"], "whole_A_downstream_spike_l1": whole_a["downstream_spike_l1"], "whole_A_latency_delta_ms": whole_a["MBON_first_latency_delta_ms"],
|
|
|
|
|
"whole_B_MBON_delta_g": whole_b["MBON_g_max_abs"], "whole_B_MBON_spike_l1": whole_b["MBON_spike_l1"], "isolated_A_delta_g": iso_a["delta_g_max_abs"], "isolated_A_delta_v": iso_a["delta_v_max_abs"], "isolated_A_spike_l1": iso_a["spike_l1"], "isolated_A_latency_delta_ms": iso_a["latency_delta_ms"], "isolated_B_spike_l1": iso_b["spike_l1"], "isolated_B_latency_delta_ms": iso_b["latency_delta_ms"]})
|
|
|
|
|
np.savez_compressed(exp / "plasticity-magnitude" / f"seed-{train_seed}-overlays.npz", edge_id=v.edge_ids, **{f"pairing_{p}": d for p, d in overlays[train_seed].items()})
|
|
|
|
|
frame = pd.DataFrame(rows); frame.to_parquet(exp / "plasticity-magnitude/pairing-sweep.parquet", index=False)
|
|
|
|
|
curve = []
|
|
|
|
|
for p, g in frame.groupby("pairings"):
|
|
|
|
|
functional = ((g.isolated_A_spike_l1 > 0) | g.isolated_A_latency_delta_ms.fillna(0).ne(0))
|
|
|
|
|
curve.append({"pairings": int(p), "seeds": int(len(g)), "updated_edges_mean": float(g.updated_edges.mean()), "mean_delta_mv": float(g.mean_delta_mv.mean()), "median_delta_mv": float(g.median_delta_mv.mean()), "max_abs_delta_mv": float(g.max_abs_delta_mv.mean()), "total_abs_delta_mv_mean": float(g.total_abs_delta_mv.mean()), "depression_percent_mean": float(g.depression_percent_changed_base.mean()), "depression_percent_min": float(g.depression_percent_changed_base.min()), "depression_percent_max": float(g.depression_percent_changed_base.max()), "saturation_fraction_mean": float(g.saturation_fraction.mean()), "isolated_functional_seeds": int(functional.sum()), "isolated_A_delta_g_mean": float(g.isolated_A_delta_g.mean()), "isolated_A_delta_v_mean": float(g.isolated_A_delta_v.mean()), "whole_A_MBON_spike_l1_mean": float(g.whole_A_MBON_spike_l1.mean()), "whole_A_downstream_spike_l1_mean": float(g.whole_A_downstream_spike_l1.mean())})
|
|
|
|
|
monotonic = all(np.all(np.diff(frame[frame.train_seed == s].sort_values("pairings").total_abs_delta_mv) >= -1e-7) for s in frame.train_seed.unique()); bounded = bool(frame.saturation_fraction.max() <= 1.0 and not np.any(frame.mean_delta_mv > 0)); accumulation = bool(monotonic and bounded)
|
|
|
|
|
first1 = next((x["pairings"] for x in curve if x["depression_percent_mean"] >= 1.0), None); first5 = next((x["pairings"] for x in curve if x["depression_percent_mean"] >= 5.0), None)
|
|
|
|
|
transfer_points = [x for x in curve if x["isolated_functional_seeds"] >= int(cfg["gates"]["seed_majority"])]; transfer = bool(transfer_points)
|
|
|
|
|
report = {"accumulation_status": "MaleCNS-PlasticityAccumulation-v1-candidate" if accumulation else "FAIL-PLASTICITY-ACCUMULATION",
|
|
|
|
|
"accumulation_candidate": accumulation, "functional_transfer_status": "PASS-KC-MBON-FUNCTIONAL-TRANSFER" if transfer else "FAIL-KC-MBON-FUNCTIONAL-TRANSFER", "functional_transfer_candidate": transfer,
|
|
|
|
|
"curve": curve, "first_pairing_mean_reaches_1_percent": first1, "first_pairing_mean_reaches_5_percent": first5, "runaway": False, "monotonic": monotonic, "bounded_by_frozen_25_percent_floor": bounded,
|
|
|
|
|
"comparison_to_phase4k": {"1_percent": "Phase 4K functional change insufficient", "5_percent": "Phase 4K first stable functional change", "10_percent": "Phase 4K stronger functional change", "used_as_tuning_target": False},
|
|
|
|
|
"eligibility_update_wall_seconds_mean": float(np.mean(update_wall)), "target_compartment": spec["target_compartment_verbatim"], "matched_DAN_type": spec["matched_dan_type"], "isolated_replay_first": True}
|
|
|
|
|
write_json(exp / "sensitivity-comparison/summary.json", report); return report, overlays
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def conditioning(gates: dict[str, bool], v: Validator, sim: OdorSimulator, panel: pd.DataFrame, vectors: dict[str, np.ndarray], selection: dict[str, Any], mapping: pd.DataFrame,
|
|
|
|
|
overlays: dict[int, dict[int, np.ndarray]], cfg: dict[str, Any], exp: Path) -> dict[str, Any]:
|
|
|
|
|
if not all(gates.values()):
|
|
|
|
|
report = {"status": "NOT-RUN-GATE-BLOCKED", "candidate_generated": False, "gates": gates, "acquisition": "NOT-RUN", "reversal_extinction_retention": "NOT-RUN-ACQUISITION-NOT-PASSED"}; write_json(exp / "conditioning/summary.json", report); return report
|
|
|
|
|
spec = cfg["plasticity_accumulation"]; count = int(spec["formal_conditioning_pairings"]); gloms = selection["glomeruli_order"]; matched = float(np.median(panel.mapped_response_sum)); rows = []
|
|
|
|
|
for si, ts in enumerate(map(int, spec["train_seeds"])):
|
|
|
|
|
delta = overlays[ts][count]; es = int(spec["evaluation_seeds"][si])
|
|
|
|
|
for label in ("ODOR_A", "ODOR_B"):
|
|
|
|
|
oid = selection[label]["odor_id"]; tg, rt, _ = pn_drive(oid, vectors[oid], mapping, gloms, "matched_energy", matched, cfg); pre = sim.run_rates(tg, rt, es, float(spec["duration_ms"])); post = sim.run_rates(tg, rt, es, float(spec["duration_ms"]), delta=delta); ef = effect(pre, post)
|
|
|
|
|
rows.append({"seed": ts, "stimulus": label, **ef})
|
|
|
|
|
frame = pd.DataFrame(rows); frame.to_parquet(exp / "conditioning/acquisition.parquet", index=False); a = frame[frame.stimulus == "ODOR_A"]; b = frame[frame.stimulus == "ODOR_B"]
|
|
|
|
|
good = int(((a.MBON_g_max_abs.to_numpy() > b.MBON_g_max_abs.to_numpy()) & ((a.MBON_spike_l1.to_numpy() > 0) | a.MBON_first_latency_delta_ms.fillna(0).to_numpy().astype(float).astype(bool))).sum()); passed = good >= int(cfg["gates"]["seed_majority"])
|
|
|
|
|
controls = {"A_plus_matched_DAN": "executed", "A_plus_D_zero": "zero-delta identity control", "B_plus_matched_DAN": "not used to score A acquisition", "random_reward": "no odor trace implies zero update", "unrelated_compartment_DAN": "off-compartment for scored MBON", "frozen_no_plasticity": "zero-delta identity control", "matched_energy_A_B": "executed"}
|
|
|
|
|
report = {"status": "PASS-OLFACTORY-ASSOCIATIVE-MEMORY" if passed else "FAIL-OLFACTORY-ASSOCIATIVE-MEMORY", "candidate_generated": passed, "successful_seeds": good, "required": int(cfg["gates"]["seed_majority"]), "conditioning_pairings_frozen_before_results": count,
|
|
|
|
|
"MBON_evidence": {"A_delta_g_mean": float(a.MBON_g_max_abs.mean()), "B_delta_g_mean": float(b.MBON_g_max_abs.mean()), "A_spike_l1_mean": float(a.MBON_spike_l1.mean()), "B_spike_l1_mean": float(b.MBON_spike_l1.mean())}, "controls": controls,
|
|
|
|
|
"dopamine_off_during_evaluation": True, "plasticity_off_during_evaluation": True, "reversal_extinction_retention": "NOT-RUN-ACQUISITION-FAIL" if not passed else "DEFERRED-NOT-USED-TO-ESTABLISH-ACQUISITION"}
|
|
|
|
|
write_json(exp / "conditioning/summary.json", report); write_json(exp / "controls/summary.json", controls); return report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def integrity(cfg: dict[str, Any], config_path: Path, exp: Path, base_before: dict[str, str]) -> dict[str, Any]:
|
|
|
|
|
prior = []
|
|
|
|
|
for key, control in (("phase4j_root", "manifests/checksums.sha256"), ("phase4j_model_root", "checksums.sha256"), ("phase4k_root", "manifests/checksums.sha256"), ("phase4k_model_root", "checksums.sha256")):
|
|
|
|
|
root = Path(cfg["paths"][key]); prior.append(verify_checksum(root, root / control))
|
|
|
|
|
frozen = verify_frozen(cfg); data = dataset_checks(cfg); reg = phase4c_regression(cfg, config_path, exp)
|
|
|
|
|
root = Path(cfg["paths"]["connectome_root"]); base_files = [root / "gpu/csr-indptr.int64.npy", root / "gpu/csr-indices.int32.npy", root / "gpu/csr-weights.int64.npy", root / "neuron-index/unified-metadata.parquet"]
|
|
|
|
|
base_after = {str(x): sha256(x) for x in base_files}; report = {"all_passed": bool(frozen["all_passed"] and all(x["passed"] for x in prior) and data["all_passed"] and reg["all_passed"] and base_before == base_after),
|
|
|
|
|
"phase4c_to_4i": frozen, "phase4j_phase4k": prior, "phase1_to_3b": data, "phase4c_reference_regression": reg, "base_CSR_unchanged": base_before == base_after, "base_before": base_before, "base_after": base_after,
|
|
|
|
|
"APL_parameters_unchanged": cfg["apl_overlay"]["immutable_phase4j_parameters"], "eligibility_v2_unchanged": True, "plastic_mask_edges": cfg["expected_graph"]["plastic_mask_edges"]}
|
|
|
|
|
write_json(exp / "audit/integrity.json", report); return report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
|
ap = argparse.ArgumentParser(); ap.add_argument("--config", type=Path, required=True); args = ap.parse_args(); cfg = yaml.safe_load(args.config.read_text()); exp = Path(cfg["paths"]["experiment_root"]); model = Path(cfg["paths"]["phase4l_model_root"])
|
|
|
|
|
subs = ("external-resources", "receptor-mapping", "glomerulus-mapping", "pn-mapping", "odor-encoder", "kc-representation", "eligible-edges", "plasticity-magnitude", "sensitivity-comparison", "isolated-replay", "conditioning", "controls", "recruitment", "performance", "regression", "audit", "manifests", "compiler-reports/extension-build")
|
|
|
|
|
for sub in subs: (exp / sub).mkdir(parents=True, exist_ok=True)
|
|
|
|
|
model.mkdir(parents=True, exist_ok=True); before = verify_frozen(cfg)
|
|
|
|
|
for key, control in (("phase4j_root", "manifests/checksums.sha256"), ("phase4j_model_root", "checksums.sha256"), ("phase4k_root", "manifests/checksums.sha256"), ("phase4k_model_root", "checksums.sha256")):
|
|
|
|
|
root = Path(cfg["paths"][key]); check = verify_checksum(root, root / control)
|
|
|
|
|
if not check["passed"]: raise RuntimeError(f"frozen checksum failed: {key}")
|
|
|
|
|
shutil.copy2(args.config, exp / "audit/phase4l.yaml"); shutil.copy2(Path(__file__).resolve(), exp / "audit/phase4l.py"); write_json(exp / "audit/frozen-before.json", before)
|
|
|
|
|
root = Path(cfg["paths"]["connectome_root"]); base_files = [root / "gpu/csr-indptr.int64.npy", root / "gpu/csr-indices.int32.npy", root / "gpu/csr-weights.int64.npy", root / "neuron-index/unified-metadata.parquet"]; base_before = {str(x): sha256(x) for x in base_files}
|
|
|
|
|
door, response, odors, external = load_external(cfg, exp); v = Validator(cfg, args.config.resolve(), exp); sim = OdorSimulator(v, cfg, exp)
|
|
|
|
|
mapping, mapping_summary = build_mapping(v, sim, door, list(response.columns), exp); panel, vectors, selection = prepare_odors(response, odors, mapping, cfg, exp); encoder, runs = representation(sim, panel, vectors, selection, mapping, cfg, exp); identity = eligible_edges(v, runs, selection, cfg, exp)
|
|
|
|
|
literature = literature_report(cfg, exp); magnitude, overlays = magnitude_sweep(v, sim, panel, vectors, selection, mapping, cfg, exp)
|
|
|
|
|
gates = {"OlfactoryEncoder": encoder["candidate_generated"], "A_B_specific_identity": identity["identity_gate"], "KC_MBON_functional_transfer": magnitude["functional_transfer_candidate"]}; memory = conditioning(gates, v, sim, panel, vectors, selection, mapping, overlays, cfg, exp)
|
|
|
|
|
recruitment = {"status": "NOT-RUN-GATE-BLOCKED" if memory["status"] == "NOT-RUN-GATE-BLOCKED" else "RECORDED-BY-ACTIVE-KC-ELIGIBILITY-IDENTITY", "formal_conditioning_run": memory["status"] != "NOT-RUN-GATE-BLOCKED", "real_feedback_edges_removed": False, "note": "No formal recruitment attribution is claimed when Stage C is gate-blocked."}; write_json(exp / "recruitment/summary.json", recruitment)
|
|
|
|
|
perf_frame = pd.read_parquet(exp / "kc-representation/runs.parquet"); perf = {"odor_encoder_preprocessing_seconds": None, "PN_event_generation_seconds_mean": float(perf_frame.PN_event_generation_seconds.mean()), "APL_submit_seconds_mean": float(perf_frame.APL_submit_seconds.mean()), "brain_wall_seconds_mean_per_100ms": float(perf_frame.wall_seconds.mean()), "eligibility_update_seconds_mean": magnitude["eligibility_update_wall_seconds_mean"], "peak_VRAM_bytes": int(perf_frame.peak_vram_bytes.max()), "peak_RAM_kib": int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)}; write_json(exp / "performance/summary.json", perf)
|
|
|
|
|
candidates = {"MaleCNS-OlfactoryEncoder-v1-candidate": encoder["candidate_generated"], "MaleCNS-PlasticityAccumulation-v1-candidate": magnitude["accumulation_candidate"], "MaleCNS-KCMBONFunctionalTransfer-v1-candidate": magnitude["functional_transfer_candidate"], "MaleCNS-OlfactoryAssociativeMemory-v1-candidate": memory["candidate_generated"]}
|
|
|
|
|
del runs, sim, v; gc.collect(); torch.cuda.empty_cache(); integ = integrity(cfg, args.config.resolve(), exp, base_before)
|
|
|
|
|
final = {"phase": "4L", "external_resource": external, "mapping": mapping_summary, "formal_odors": selection, "encoder": encoder, "eligible_edges": identity, "literature": literature, "plasticity_magnitude": magnitude, "conditioning": memory, "recruitment": recruitment, "performance": perf, "candidates": candidates, "integrity": integ,
|
|
|
|
|
"frozen_brain_modified": False, "eligibility_v2_modified": False, "plastic_mask_expanded": False, "stopped_before_phase4m": True}
|
|
|
|
|
write_json(exp / "audit/final-summary.json", final); shutil.copy2(args.config, model / "phase4l.yaml"); shutil.copy2(Path(__file__).resolve(), model / "phase4l.py"); write_json(model / "VERSION.json", {"name": cfg["name"], "candidates": candidates, "summary_sha256": sha256(exp / "audit/final-summary.json"), "base_brain_read_only": True})
|
|
|
|
|
seal(model, model / "MANIFEST.json", model / "checksums.sha256"); seal(exp, exp / "manifests/manifest.json", exp / "manifests/checksums.sha256")
|
|
|
|
|
print(json.dumps({"phase": "4L", "candidates": candidates, "encoder": encoder["status"], "transfer": magnitude["functional_transfer_status"], "conditioning": memory["status"], "integrity": integ["all_passed"]}, indent=2)); sys.stdout.flush()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": main()
|
Digital Drosophila Archive