Digital Drosophila Archive
610 lines
48 KiB
Python
610 lines
48 KiB
Python
#!/usr/bin/env python3
|
|
"""Phase 4K: read-only KC recruitment and MBON causal-input audit."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gc
|
|
import json
|
|
import math
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import torch
|
|
import yaml
|
|
|
|
from phase4h_audit import sha256, tree_manifest, verify_checksum, write_json
|
|
from phase4i import Validator, phase4c_regression
|
|
from phase4j import APLSimulator, verify_frozen
|
|
|
|
|
|
KC_CATEGORIES = ("PN_olfactory", "KC", "APL", "MBON", "DAN", "other")
|
|
MBON_CATEGORIES = ("plastic_KC", "nonplastic_KC", "DAN", "MBON", "feedback_neuron", "other")
|
|
|
|
|
|
def seal(root: Path, manifest: Path, checksums: Path) -> None:
|
|
write_json(manifest, tree_manifest(root, {manifest, checksums}))
|
|
targets = sorted(x for x in root.rglob("*") if x.is_file() and x != checksums)
|
|
checksums.write_text("".join(f"{sha256(x)} {x.relative_to(root)}\n" for x in targets), encoding="utf-8")
|
|
|
|
|
|
def first_latency(first_steps: np.ndarray, members: np.ndarray, dt: float) -> float | None:
|
|
values = first_steps[members]
|
|
return None if np.all(values < 0) else float(values[values >= 0].min() * dt)
|
|
|
|
|
|
def response(run: dict[str, Any], v: Validator) -> dict[str, Any]:
|
|
return {
|
|
"MBON_spikes": int(run["counts"][v.mbon].sum()),
|
|
"MBON_first_latency_ms": first_latency(run["first_steps"], v.mbon, run["dt_ms"]),
|
|
"downstream_spikes": int(run["counts"][v.downstream].sum()),
|
|
"downstream_first_latency_ms": first_latency(run["first_steps"], v.downstream, run["dt_ms"]),
|
|
"MBON_final_g": run["final_g_mbon"],
|
|
"MBON_final_v": run["final_v_mbon"],
|
|
}
|
|
|
|
|
|
def compare(pre: dict[str, Any], post: dict[str, Any]) -> dict[str, Any]:
|
|
def delta_latency(key: str) -> float | None:
|
|
a, b = pre[key], post[key]
|
|
return None if a is None or b is None else float(b - a)
|
|
return {
|
|
"MBON_delta_g_max_abs": float(np.max(np.abs(post["MBON_final_g"] - pre["MBON_final_g"]))),
|
|
"MBON_delta_v_max_abs": float(np.max(np.abs(post["MBON_final_v"] - pre["MBON_final_v"]))),
|
|
"MBON_spike_count_delta": int(post["MBON_spikes"] - pre["MBON_spikes"]),
|
|
"MBON_spike_l1": int(abs(post["MBON_spikes"] - pre["MBON_spikes"])),
|
|
"MBON_first_latency_delta_ms": delta_latency("MBON_first_latency_ms"),
|
|
"downstream_spike_count_delta": int(post["downstream_spikes"] - pre["downstream_spikes"]),
|
|
"downstream_first_latency_delta_ms": delta_latency("downstream_first_latency_ms"),
|
|
}
|
|
|
|
|
|
class AuditSimulator(APLSimulator):
|
|
"""Frozen LIF runtime with side-channel drive and event-causality accounting."""
|
|
|
|
def __init__(self, v: Validator, cfg: dict[str, Any], exp: Path):
|
|
super().__init__(v, cfg, exp)
|
|
self.base_np = v.base.detach().cpu().numpy().copy()
|
|
self.indices_np = v.rt.indices_np
|
|
self.raw_np = v.rt.raw_np
|
|
self.edge_sources = self.sources
|
|
self.dan = np.unique(v.confirmed["source_neuron_idx"].to_numpy(np.int32))
|
|
|
|
pn_olf = (v.cls == "ALPN") | (v.cls == "olfactory")
|
|
node_code = np.full(v.n, KC_CATEGORIES.index("other"), np.int8)
|
|
node_code[pn_olf] = KC_CATEGORIES.index("PN_olfactory")
|
|
node_code[v.kc] = KC_CATEGORIES.index("KC")
|
|
node_code[self.apl] = KC_CATEGORIES.index("APL")
|
|
node_code[v.mbon] = KC_CATEGORIES.index("MBON")
|
|
node_code[self.dan] = KC_CATEGORIES.index("DAN")
|
|
self.node_kc_code = node_code
|
|
self.edge_kc_code_gpu = torch.as_tensor(node_code[self.edge_sources], dtype=torch.int8, device=self.device)
|
|
|
|
target_is_kc = np.zeros(v.n, bool); target_is_kc[v.kc] = True
|
|
feedback = np.unique(self.edge_sources[target_is_kc[self.indices_np]])
|
|
is_candidate = np.zeros(len(self.indices_np), bool); is_candidate[v.edge_ids] = True
|
|
edge_mbon_code = np.full(len(self.indices_np), MBON_CATEGORIES.index("other"), np.int8)
|
|
src_kc = v.cls[self.edge_sources] == "Kenyon_Cell"
|
|
edge_mbon_code[src_kc & is_candidate] = MBON_CATEGORIES.index("plastic_KC")
|
|
edge_mbon_code[src_kc & ~is_candidate] = MBON_CATEGORIES.index("nonplastic_KC")
|
|
edge_mbon_code[np.isin(self.edge_sources, self.dan)] = MBON_CATEGORIES.index("DAN")
|
|
edge_mbon_code[np.isin(self.edge_sources, v.mbon)] = MBON_CATEGORIES.index("MBON")
|
|
structurally_feedback = np.isin(self.edge_sources, feedback) & ~src_kc & ~np.isin(self.edge_sources, np.concatenate([self.dan, v.mbon]))
|
|
edge_mbon_code[structurally_feedback] = MBON_CATEGORIES.index("feedback_neuron")
|
|
self.edge_mbon_code_gpu = torch.as_tensor(edge_mbon_code, dtype=torch.int8, device=self.device)
|
|
|
|
kc_lookup = np.full(v.n, -1, np.int32); kc_lookup[v.kc] = np.arange(len(v.kc), dtype=np.int32)
|
|
mbon_lookup = np.full(v.n, -1, np.int32); mbon_lookup[v.mbon] = np.arange(len(v.mbon), dtype=np.int32)
|
|
self.kc_lookup_gpu = torch.as_tensor(kc_lookup, dtype=torch.int32, device=self.device)
|
|
self.mbon_lookup_gpu = torch.as_tensor(mbon_lookup, dtype=torch.int32, device=self.device)
|
|
self.mbon_gpu = torch.as_tensor(v.mbon, dtype=torch.int64, device=self.device)
|
|
|
|
def population_label(self, neuron: int) -> str:
|
|
return KC_CATEGORIES[int(self.node_kc_code[int(neuron)])]
|
|
|
|
@staticmethod
|
|
def _dominant_arrivals(
|
|
item: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray] | None,
|
|
score: np.ndarray, parent: np.ndarray, edge: np.ndarray, arrival: np.ndarray,
|
|
) -> None:
|
|
if item is None or not len(item[0]):
|
|
return
|
|
targets, values, parents, edges, arrivals = item
|
|
order = np.lexsort((-values, targets))
|
|
st = targets[order]
|
|
take = np.r_[True, st[1:] != st[:-1]]
|
|
chosen = order[take]; nodes = targets[chosen]; better = values[chosen] > score[nodes]
|
|
nodes = nodes[better]; chosen = chosen[better]
|
|
score[nodes] = values[chosen]; parent[nodes] = parents[chosen]
|
|
edge[nodes] = edges[chosen]; arrival[nodes] = arrivals[chosen]
|
|
|
|
def run_audit(
|
|
self, targets: np.ndarray, seed: int, mode: str, duration_ms: float,
|
|
delta: np.ndarray | None = None, rate_hz: float | None = None, pulses: list[float] | None = None,
|
|
dan_targets: np.ndarray | None = None, dan_pulses: list[float] | None = None,
|
|
causal: bool = False, collect_kc_spikes: bool = False,
|
|
) -> dict[str, Any]:
|
|
if delta is None:
|
|
delta = np.zeros(len(self.v.edge_ids), np.float32)
|
|
self.v.install(delta)
|
|
if mode in {"graded", "disabled"}:
|
|
self.rt.weights[self.apl_edge_gpu] = 0.0
|
|
self.rt.reset(int(seed)); steps = int(round(float(duration_ms) / self.dt))
|
|
target_gpu = torch.as_tensor(targets, dtype=torch.int64, device=self.device)
|
|
forced = self._forced(steps, targets, seed, rate_hz, pulses)
|
|
dan = np.empty(0, np.int32) if dan_targets is None else np.asarray(dan_targets, np.int32)
|
|
dan_gpu = torch.as_tensor(dan, dtype=torch.int64, device=self.device)
|
|
dan_forced = self._forced(steps, dan, seed + 991, None, dan_pulses)
|
|
|
|
qlen = self.rt.queue_len; nk, nm = len(self.v.kc), len(self.v.mbon)
|
|
kq = torch.zeros((qlen, len(KC_CATEGORIES), nk), dtype=torch.float32, device=self.device)
|
|
mq = torch.zeros((qlen, len(MBON_CATEGORIES), nm), dtype=torch.float32, device=self.device)
|
|
mpq = torch.zeros_like(mq); mdq = torch.zeros((qlen, nm), dtype=torch.float32, device=self.device)
|
|
kg = torch.zeros((len(KC_CATEGORIES), nk), dtype=torch.float32, device=self.device)
|
|
mg = torch.zeros((len(MBON_CATEGORIES), nm), dtype=torch.float32, device=self.device)
|
|
mdg = torch.zeros(nm, dtype=torch.float32, device=self.device)
|
|
kr = torch.zeros_like(kg); mr = torch.zeros_like(mg); mrp = torch.zeros_like(mg); mdr = torch.zeros(nm, dtype=torch.float32, device=self.device)
|
|
kt = torch.zeros((steps, len(KC_CATEGORIES)), dtype=torch.float32, device=self.device)
|
|
mt = torch.zeros((steps, len(MBON_CATEGORIES)), dtype=torch.float32, device=self.device)
|
|
counts = torch.zeros(self.v.n, dtype=torch.int32, device=self.device)
|
|
first = torch.full((self.v.n,), -1, dtype=torch.int32, device=self.device)
|
|
kc_apl_first = torch.zeros(nk, dtype=torch.float32, device=self.device)
|
|
|
|
params = self.cfg["apl_overlay"]
|
|
apl_state = torch.zeros(len(self.apl), dtype=torch.float32, device=self.device)
|
|
apl_decay = math.exp(-self.dt / float(params["tau_ms"]))
|
|
alpha, gd = self.rt.alpha, self.rt.g_decay
|
|
rest, reset, threshold, refsteps = self.rt.rest, self.rt.reset_value, self.rt.threshold, self.rt.refractory_steps
|
|
delta_by_edge = torch.zeros(len(self.indices_np), dtype=torch.float32, device=self.device)
|
|
delta_by_edge[torch.as_tensor(self.v.edge_ids, dtype=torch.int64, device=self.device)] = torch.as_tensor(delta, dtype=torch.float32, device=self.device)
|
|
|
|
event_rows: list[dict[str, Any]] = []; event_parent: list[int] = []; event_depth: list[int] = []
|
|
causal_score = np.zeros(self.v.n, np.float32); causal_parent = np.full(self.v.n, -1, np.int64)
|
|
causal_edge = np.full(self.v.n, -1, np.int64); causal_arrival = np.full(self.v.n, -1, np.int32)
|
|
causal_ring: list[tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray] | None] = [None] * qlen
|
|
current_event = np.full(self.v.n, -1, np.int64)
|
|
kc_spike_steps: list[np.ndarray] = []; kc_spike_nodes: list[np.ndarray] = []
|
|
kc_local_lookup = np.full(self.v.n, -1, np.int32); kc_local_lookup[self.v.kc] = np.arange(nk, dtype=np.int32)
|
|
forced_set = set(map(int, targets)); dan_set = set(map(int, dan))
|
|
|
|
torch.cuda.synchronize(); t0 = time.perf_counter()
|
|
for step in range(steps):
|
|
slot = step % qlen
|
|
self.rt.g.mul_(gd).add_(self.rt.delay_queue[slot]); self.rt.delay_queue[slot].zero_()
|
|
ka = kq[slot]; ma = mq[slot]; mpa = mpq[slot]; mda = mdq[slot]
|
|
kg.mul_(gd).add_(ka); mg.mul_(gd).add_(ma); mdg.mul_(gd).add_(mda)
|
|
kr.add_(ka); mr.add_(ma); mrp.add_(mpa); mdr.add_(mda)
|
|
kt[step] = ka.sum(dim=1); mt[step] = ma.sum(dim=1)
|
|
ka.zero_(); ma.zero_(); mpa.zero_(); mda.zero_()
|
|
if causal:
|
|
causal_score *= float(gd)
|
|
self._dominant_arrivals(causal_ring[slot], causal_score, causal_parent, causal_edge, causal_arrival)
|
|
causal_ring[slot] = None
|
|
|
|
self.rt.refractory.clamp_min_(1).sub_(1); available = self.rt.refractory == 0
|
|
self.rt.v[available] += alpha * (self.rt.g[available] - (self.rt.v[available] - rest)); self.rt.v[~available] = reset
|
|
spikes = available & (self.rt.v >= threshold)
|
|
accepted = forced[step] & available[target_gpu]
|
|
if accepted.any():
|
|
spikes[target_gpu[accepted]] = True
|
|
dacc = torch.zeros(len(dan), dtype=torch.bool, device=self.device)
|
|
if len(dan):
|
|
dacc = dan_forced[step] & available[dan_gpu]
|
|
if dacc.any():
|
|
spikes[dan_gpu[dacc]] = True
|
|
kc_new = spikes[self.kc_gpu] & (first[self.kc_gpu] < 0)
|
|
if kc_new.any():
|
|
kc_apl_first[kc_new] = kr[KC_CATEGORIES.index("APL"), kc_new]
|
|
sources = torch.nonzero(spikes, as_tuple=False).flatten()
|
|
counts.add_(spikes.to(torch.int32)); new = spikes & (first < 0); first[new] = step
|
|
if collect_kc_spikes:
|
|
kn = torch.nonzero(spikes[self.kc_gpu], as_tuple=False).flatten().cpu().numpy()
|
|
if len(kn):
|
|
kc_spike_steps.append(np.full(len(kn), step, np.int32)); kc_spike_nodes.append(self.v.kc[kn])
|
|
|
|
accepted_nodes = set(target_gpu[accepted].cpu().numpy().astype(int).tolist()) if bool(accepted.any()) else set()
|
|
dan_nodes = set(dan_gpu[dacc].cpu().numpy().astype(int).tolist()) if len(dan) and bool(dacc.any()) else set()
|
|
source_np = sources.cpu().numpy().astype(np.int32) if causal and sources.numel() else np.empty(0, np.int32)
|
|
if causal and len(source_np):
|
|
ids = np.arange(len(event_rows), len(event_rows) + len(source_np), dtype=np.int64)
|
|
current_event[source_np] = ids
|
|
for eid, node in zip(ids, source_np):
|
|
root = int(node) in accepted_nodes and int(node) in forced_set
|
|
par = -1 if root else int(causal_parent[node]); is_causal = root or par >= 0
|
|
depth = 0 if root else (event_depth[par] + 1 if is_causal else -1)
|
|
event_rows.append({"event_id": int(eid), "step": int(step), "timestamp_ms": float(step * self.dt), "neuron_idx": int(node),
|
|
"forced_kind": "injected_A" if root else ("matched_DAN" if int(node) in dan_nodes and int(node) in dan_set else "none"),
|
|
"causally_reachable_from_A": bool(is_causal), "parent_event_id": par,
|
|
"parent_edge_id": -1 if root else int(causal_edge[node]), "parent_arrival_step": -1 if root else int(causal_arrival[node]),
|
|
"dominant_decayed_contribution": None if root else float(causal_score[node]), "path_depth": int(depth)})
|
|
event_parent.append(par); event_depth.append(depth)
|
|
|
|
if sources.numel():
|
|
self.rt.v[sources] = reset; self.rt.refractory[sources] = refsteps
|
|
edge_ids = self.rt._hybrid_edges(sources)
|
|
if edge_ids.numel():
|
|
next_slot = (step + self.rt.delay_steps) % qlen
|
|
targets_all = self.rt.indices[edge_ids].long(); weights = self.rt.weights[edge_ids]
|
|
self.rt.delay_queue[next_slot].scatter_add_(0, targets_all, weights)
|
|
kloc = self.kc_lookup_gpu[targets_all]; kcode = self.edge_kc_code_gpu[edge_ids]
|
|
for ci in range(len(KC_CATEGORIES)):
|
|
mask = (kloc >= 0) & (kcode == ci)
|
|
if mask.any():
|
|
kq[next_slot, ci].scatter_add_(0, kloc[mask].long(), weights[mask])
|
|
mloc = self.mbon_lookup_gpu[targets_all]; mcode = self.edge_mbon_code_gpu[edge_ids]
|
|
for ci in range(len(MBON_CATEGORIES)):
|
|
mask = (mloc >= 0) & (mcode == ci)
|
|
if mask.any():
|
|
mq[next_slot, ci].scatter_add_(0, mloc[mask].long(), weights[mask])
|
|
mpq[next_slot, ci].scatter_add_(0, mloc[mask].long(), weights[mask].clamp_min(0))
|
|
mm = mloc >= 0
|
|
if mm.any():
|
|
mdq[next_slot].scatter_add_(0, mloc[mm].long(), delta_by_edge[edge_ids[mm]])
|
|
|
|
if causal:
|
|
eids = edge_ids.cpu().numpy().astype(np.int64); pre = self.edge_sources[eids]
|
|
pe = current_event[pre]; w = self.base_np[eids]
|
|
cm = (pe >= 0) & (w > 0)
|
|
if np.any(cm):
|
|
causal_ring[next_slot] = (self.indices_np[eids[cm]].astype(np.int32), w[cm].astype(np.float32), pe[cm].astype(np.int64),
|
|
eids[cm], np.full(np.count_nonzero(cm), step + self.rt.delay_steps, np.int32))
|
|
if causal and len(source_np):
|
|
current_event[source_np] = -1
|
|
|
|
if mode == "graded":
|
|
apl_state.mul_(apl_decay).add_(torch.mv(self.input_gpu, spikes.to(torch.float32)) * float(params["normalization"]))
|
|
inhibition = torch.matmul(apl_state, self.output_gpu) * float(params["gain"])
|
|
self.rt.g.index_add_(0, self.kc_gpu, -inhibition)
|
|
kg[KC_CATEGORIES.index("APL")].sub_(inhibition); kr[KC_CATEGORIES.index("APL")].sub_(inhibition)
|
|
kt[step, KC_CATEGORIES.index("APL")].sub_(inhibition.sum())
|
|
|
|
torch.cuda.synchronize(); wall = time.perf_counter() - t0
|
|
counts_np = counts.cpu().numpy(); first_np = first.cpu().numpy()
|
|
kshadow = kg.cpu().numpy(); mshadow = mg.cpu().numpy(); mdshadow = mdg.cpu().numpy()
|
|
final_k = self.rt.g[self.kc_gpu].cpu().numpy(); final_m = self.rt.g[self.mbon_gpu].cpu().numpy()
|
|
kc_spikes = pd.DataFrame({"step": np.concatenate(kc_spike_steps) if kc_spike_steps else np.empty(0, np.int32),
|
|
"neuron_idx": np.concatenate(kc_spike_nodes) if kc_spike_nodes else np.empty(0, np.int32)})
|
|
return {
|
|
"counts": counts_np, "first_steps": first_np, "dt_ms": self.dt, "wall_seconds": wall,
|
|
"final_g_kc": final_k, "final_g_mbon": final_m, "final_v_mbon": self.rt.v[self.mbon_gpu].cpu().numpy().copy(),
|
|
"kc_drive_received": kr.cpu().numpy(), "kc_g_shadow": kshadow, "kc_apl_at_first": kc_apl_first.cpu().numpy(),
|
|
"mbon_drive_received": mr.cpu().numpy(), "mbon_positive_received": mrp.cpu().numpy(),
|
|
"mbon_delta_received": mdr.cpu().numpy(), "mbon_g_shadow": mshadow, "mbon_delta_g_shadow": mdshadow,
|
|
"kc_timeseries": kt.cpu().numpy(), "mbon_timeseries": mt.cpu().numpy(), "events": pd.DataFrame(event_rows), "kc_spikes": kc_spikes,
|
|
"shadow_validation": {"KC_final_g_max_abs_error": float(np.max(np.abs(kshadow.sum(axis=0) - final_k))),
|
|
"MBON_final_g_max_abs_error": float(np.max(np.abs(mshadow.sum(axis=0) - final_m)))},
|
|
}
|
|
|
|
|
|
def save_drive(condition: str, run: dict[str, Any], v: Validator, kc_rows: list[dict[str, Any]], mbon_rows: list[dict[str, Any]], time_rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
for ci, category in enumerate(KC_CATEGORIES):
|
|
for local, neuron in enumerate(v.kc):
|
|
kc_rows.append({"condition": condition, "category": category, "neuron_idx": int(neuron), "body_id": int(v.meta.loc[neuron, "body_id"]),
|
|
"cumulative_signed_drive": float(run["kc_drive_received"][ci, local]), "final_g_component": float(run["kc_g_shadow"][ci, local])})
|
|
for ci, category in enumerate(MBON_CATEGORIES):
|
|
for local, neuron in enumerate(v.mbon):
|
|
mbon_rows.append({"condition": condition, "category": category, "neuron_idx": int(neuron), "body_id": int(v.meta.loc[neuron, "body_id"]),
|
|
"cumulative_signed_drive": float(run["mbon_drive_received"][ci, local]), "cumulative_positive_drive": float(run["mbon_positive_received"][ci, local]),
|
|
"final_g_component": float(run["mbon_g_shadow"][ci, local]), "actual_delta_cumulative_drive": float(run["mbon_delta_received"][local]),
|
|
"actual_delta_final_g": float(run["mbon_delta_g_shadow"][local]), "final_total_g": float(run["final_g_mbon"][local]),
|
|
"final_v": float(run["final_v_mbon"][local]), "threshold_margin_mv": float(v.rt.threshold - run["final_v_mbon"][local])})
|
|
for step in range(len(run["kc_timeseries"])):
|
|
row = {"condition": condition, "step": step, "timestamp_ms": step * run["dt_ms"]}
|
|
row.update({f"KC_{x}": float(run["kc_timeseries"][step, i]) for i, x in enumerate(KC_CATEGORIES)})
|
|
row.update({f"MBON_{x}": float(run["mbon_timeseries"][step, i]) for i, x in enumerate(MBON_CATEGORIES)})
|
|
time_rows.append(row)
|
|
ksum = run["kc_drive_received"].sum(axis=1); msum = run["mbon_drive_received"].sum(axis=1); mpos = run["mbon_positive_received"].sum(axis=1)
|
|
return {"KC_cumulative_signed_drive": {x: float(ksum[i]) for i, x in enumerate(KC_CATEGORIES)},
|
|
"MBON_cumulative_signed_drive": {x: float(msum[i]) for i, x in enumerate(MBON_CATEGORIES)},
|
|
"MBON_cumulative_positive_drive": {x: float(mpos[i]) for i, x in enumerate(MBON_CATEGORIES)},
|
|
"wall_seconds": run["wall_seconds"], "shadow_validation": run["shadow_validation"]}
|
|
|
|
|
|
def classify_path(labels: list[str], apl_modulation: bool, path_length: int) -> str:
|
|
if path_length == 1:
|
|
return "direct recurrent"
|
|
middle = labels[1:-1]
|
|
if any(x == "PN_olfactory" for x in middle):
|
|
return "PN/olfactory feedback"
|
|
if any(x in {"MBON", "DAN"} for x in middle):
|
|
return "MBON/DAN feedback"
|
|
if any(x == "APL" for x in middle) or apl_modulation:
|
|
return "APL-related"
|
|
return "other CNS loop" if path_length > 0 else "unexplained"
|
|
|
|
|
|
def recruitment_paths(sim: AuditSimulator, run: dict[str, Any], set_a: np.ndarray, set_b: np.ndarray, actual_delta: np.ndarray, exp: Path) -> dict[str, Any]:
|
|
events = run["events"].copy(); events.to_parquet(exp / "recruitment/direct-A-all-spike-events.parquet", index=False)
|
|
first_event = events.sort_values(["step", "event_id"]).drop_duplicates("neuron_idx").set_index("neuron_idx")
|
|
changed = actual_delta != 0; cand_src = sim.v.cand["source_neuron_idx"].to_numpy(np.int32)
|
|
b_changed_edges = changed & np.isin(cand_src, set_b); b_changed_sources = np.unique(cand_src[b_changed_edges])
|
|
event_map = {int(row["event_id"]): row for row in events.to_dict(orient="records")}
|
|
recruited = np.setdiff1d(sim.v.kc[run["counts"][sim.v.kc] > 0], set_a)
|
|
node_rows: list[dict[str, Any]] = []; edge_rows: list[dict[str, Any]] = []; summary_rows: list[dict[str, Any]] = []
|
|
kc_local = {int(x): i for i, x in enumerate(sim.v.kc)}
|
|
for target in recruited:
|
|
if int(target) not in first_event.index:
|
|
continue
|
|
terminal = first_event.loc[int(target)]; eid = int(terminal["event_id"]); chain: list[dict[str, Any]] = []
|
|
seen = set()
|
|
while eid >= 0 and eid not in seen:
|
|
seen.add(eid); ev = event_map[eid]; chain.append(ev); eid = int(ev["parent_event_id"])
|
|
chain.reverse(); rooted = bool(chain and chain[0]["forced_kind"] == "injected_A")
|
|
labels = [sim.population_label(int(x["neuron_idx"])) for x in chain]
|
|
apl_mod = abs(float(run["kc_apl_at_first"][kc_local[int(target)]])) > 0
|
|
path_length = len(chain) - 1 if rooted else 0
|
|
category = classify_path(labels, apl_mod, path_length) if rooted else "unexplained"
|
|
for order, ev in enumerate(chain):
|
|
neuron = int(ev["neuron_idx"])
|
|
node_rows.append({"target_recruited_kc": int(target), "path_order": order, "event_id": int(ev["event_id"]), "neuron_idx": neuron,
|
|
"body_id": int(sim.v.meta.loc[neuron, "body_id"]), "population": sim.population_label(neuron),
|
|
"annotation_type": sim.v.types[neuron], "annotation_class": sim.v.cls[neuron], "timestamp_ms": float(ev["timestamp_ms"]),
|
|
"forced_kind": ev["forced_kind"]})
|
|
for order, ev in enumerate(chain[1:], start=1):
|
|
edge_id = int(ev["parent_edge_id"]); parent = chain[order - 1]
|
|
edge_rows.append({"target_recruited_kc": int(target), "path_order": order, "edge_id": edge_id,
|
|
"source_neuron_idx": int(parent["neuron_idx"]), "target_neuron_idx": int(ev["neuron_idx"]),
|
|
"source_spike_time_ms": float(parent["timestamp_ms"]), "arrival_time_ms": float(ev["parent_arrival_step"] * run["dt_ms"]),
|
|
"target_spike_time_ms": float(ev["timestamp_ms"]), "effective_weight": float(sim.base_np[edge_id]),
|
|
"raw_synapse_count": int(sim.raw_np[edge_id]), "synaptic_sign": int(np.sign(sim.base_np[edge_id])),
|
|
"source_population": sim.population_label(int(parent["neuron_idx"])), "target_population": sim.population_label(int(ev["neuron_idx"]))})
|
|
summary_rows.append({"neuron_idx": int(target), "body_id": int(sim.v.meta.loc[target, "body_id"]), "annotation_type": sim.v.types[target],
|
|
"first_activation_ms": float(terminal["timestamp_ms"]), "causal_path_found": rooted, "path_length": path_length,
|
|
"recruitment_class": category, "APL_involved_or_modulatory": apl_mod or ("APL" in labels),
|
|
"is_injected_B": bool(target in set_b), "is_B_source_with_changed_edge": bool(target in b_changed_sources),
|
|
"changed_B_edge_count": int(np.count_nonzero(b_changed_edges & (cand_src == target)))})
|
|
summary_df = pd.DataFrame(summary_rows); summary_df.to_parquet(exp / "recruitment/recruited-kc-summary.parquet", index=False)
|
|
pd.DataFrame(node_rows).to_parquet(exp / "recruitment/causal-path-nodes.parquet", index=False)
|
|
pd.DataFrame(edge_rows).to_parquet(exp / "recruitment/causal-path-edges.parquet", index=False)
|
|
counts = summary_df["recruitment_class"].value_counts().to_dict() if len(summary_df) else {}
|
|
b_rows = summary_df[summary_df["is_B_source_with_changed_edge"]] if len(summary_df) else summary_df
|
|
report = {"label": "KC-RECRUITMENT-SOURCE-IDENTIFIED" if len(summary_df) and bool(summary_df["causal_path_found"].all()) else "FAIL-KC-RECRUITMENT-SOURCE-IDENTIFIED",
|
|
"injected_A_KC": int(len(set_a)), "active_non_A_KC": int(len(recruited)), "traced_recruited_KC": int(len(summary_df)),
|
|
"causal_paths_found": int(summary_df["causal_path_found"].sum()) if len(summary_df) else 0,
|
|
"classification": {str(k): int(v) for k, v in counts.items()},
|
|
"B_set_KC_active": int(np.count_nonzero(run["counts"][set_b])), "B_source_changed_edges": int(np.count_nonzero(b_changed_edges)),
|
|
"unique_B_source_KC_with_changed_edges": int(len(b_changed_sources)),
|
|
"B_source_path_classes": {str(k): int(v) for k, v in b_rows["recruitment_class"].value_counts().to_dict().items()} if len(b_rows) else {},
|
|
"causal_rule": sim.cfg["causal_attribution"]["rule"], "static_shortest_path_used": False,
|
|
"interpretation_limit": "dominant positive arrival under exact runtime timing is an attribution path, not an intervention-based proof"}
|
|
write_json(exp / "recruitment/summary.json", report); return report
|
|
|
|
|
|
def drive_audit(sim: AuditSimulator, set_a: np.ndarray, set_b: np.ndarray, olf_a: np.ndarray, cfg: dict[str, Any], exp: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
|
|
dc, oc = cfg["direct_kc"], cfg["olfactory"]; params = cfg["apl_overlay"]
|
|
specs = {
|
|
"original_olfactory_CS": dict(targets=olf_a, seed=oc["seed"], mode="original", duration_ms=oc["duration_ms"], rate_hz=oc["rate_hz"]),
|
|
"APL_overlay_olfactory_CS": dict(targets=olf_a, seed=oc["seed"], mode="graded", duration_ms=oc["duration_ms"], rate_hz=oc["rate_hz"]),
|
|
"direct_KC_SET_A": dict(targets=set_a, seed=dc["evaluation_seed"], mode="original", duration_ms=dc["duration_ms"], pulses=dc["pulse_times_ms"], collect_kc_spikes=True),
|
|
"direct_KC_SET_B": dict(targets=set_b, seed=dc["evaluation_seed"], mode="original", duration_ms=dc["duration_ms"], pulses=dc["pulse_times_ms"], collect_kc_spikes=True),
|
|
}
|
|
kr: list[dict[str, Any]] = []; mr: list[dict[str, Any]] = []; tr: list[dict[str, Any]] = []; runs = {}; summary = {}
|
|
for name, spec in specs.items():
|
|
run = sim.run_audit(delta=np.zeros(len(sim.v.edge_ids), np.float32), **spec); runs[name] = run
|
|
summary[name] = save_drive(name, run, sim.v, kr, mr, tr)
|
|
pd.DataFrame(kr).to_parquet(exp / "drive-decomposition/kc-incoming-drive.parquet", index=False)
|
|
pd.DataFrame(mr).to_parquet(exp / "drive-decomposition/mbon-incoming-drive-baseline.parquet", index=False)
|
|
pd.DataFrame(tr).to_parquet(exp / "drive-decomposition/drive-timeseries.parquet", index=False)
|
|
write_json(exp / "drive-decomposition/summary.json", summary); return summary, runs
|
|
|
|
|
|
def mbon_audit(sim: AuditSimulator, base_runs: dict[str, dict[str, Any]], set_a: np.ndarray, set_b: np.ndarray, actual_delta: np.ndarray, cfg: dict[str, Any], exp: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
|
|
dc = cfg["direct_kc"]; posts = {}
|
|
for label, targets in (("A", set_a), ("B", set_b)):
|
|
posts[label] = sim.run_audit(targets, dc["evaluation_seed"], "original", dc["duration_ms"], delta=actual_delta, pulses=dc["pulse_times_ms"], collect_kc_spikes=True)
|
|
rows = []; report = {}
|
|
for label, pre in (("A", base_runs["direct_KC_SET_A"]), ("B", base_runs["direct_KC_SET_B"])):
|
|
post = posts[label]; p = response(pre, sim.v); q = response(post, sim.v); eff = compare(p, q)
|
|
plastic_i = MBON_CATEGORIES.index("plastic_KC")
|
|
plastic_positive = float(post["mbon_positive_received"][plastic_i].sum()); total_positive = float(post["mbon_positive_received"].sum())
|
|
total_signed = float(post["mbon_drive_received"].sum()); delta_signed = float(post["mbon_delta_received"].sum())
|
|
report[label] = {"effect": eff, "total_cumulative_signed_drive": total_signed, "total_excitatory_drive": total_positive,
|
|
"plastic_KC_excitatory_drive": plastic_positive, "plastic_KC_fraction_of_total_excitatory_drive": plastic_positive / total_positive if total_positive else None,
|
|
"actual_delta_cumulative_drive": delta_signed, "actual_delta_over_total_excitatory_drive": delta_signed / total_positive if total_positive else None,
|
|
"actual_delta_over_total_signed_drive": delta_signed / total_signed if total_signed else None,
|
|
"final_total_g_sum": float(post["final_g_mbon"].sum()), "actual_delta_final_g_sum": float(post["mbon_delta_g_shadow"].sum())}
|
|
for state, run in (("pre", pre), ("post", post)):
|
|
for ci, category in enumerate(MBON_CATEGORIES):
|
|
for local, neuron in enumerate(sim.v.mbon):
|
|
rows.append({"stimulus": label, "state": state, "category": category, "neuron_idx": int(neuron), "body_id": int(sim.v.meta.loc[neuron, "body_id"]),
|
|
"cumulative_signed_drive": float(run["mbon_drive_received"][ci, local]), "cumulative_positive_drive": float(run["mbon_positive_received"][ci, local]),
|
|
"final_g_component": float(run["mbon_g_shadow"][ci, local]), "actual_delta_final_g": float(run["mbon_delta_g_shadow"][local]),
|
|
"final_total_g": float(run["final_g_mbon"][local]), "final_v": float(run["final_v_mbon"][local]),
|
|
"threshold_margin_mv": float(sim.v.rt.threshold - run["final_v_mbon"][local])})
|
|
pd.DataFrame(rows).to_parquet(exp / "mbon-decomposition/pre-post-mbon-drive.parquet", index=False)
|
|
write_json(exp / "mbon-decomposition/summary.json", report); return report, posts
|
|
|
|
|
|
def sensitivity(sim: AuditSimulator, set_a: np.ndarray, actual_delta: np.ndarray, cfg: dict[str, Any], exp: Path) -> dict[str, Any]:
|
|
dc, sc = cfg["direct_kc"], cfg["sensitivity"]; changed = actual_delta != 0; rows = []
|
|
for seed in sc["evaluation_seeds"]:
|
|
base_delta = np.zeros(len(sim.v.edge_ids), np.float32)
|
|
base = sim.run_audit(set_a, int(seed), "original", dc["duration_ms"], delta=base_delta, pulses=dc["pulse_times_ms"])
|
|
bp = response(base, sim.v)
|
|
for fraction in sc["depression_fractions"]:
|
|
delta = np.zeros(len(sim.v.edge_ids), np.float32); delta[changed] = -float(fraction) * sim.v.base_candidate[changed]
|
|
run = base if float(fraction) == 0 else sim.run_audit(set_a, int(seed), "original", dc["duration_ms"], delta=delta, pulses=dc["pulse_times_ms"])
|
|
q = response(run, sim.v); eff = compare(bp, q)
|
|
functional = bool(eff["MBON_spike_l1"] > 0 or (eff["MBON_first_latency_delta_ms"] not in (None, 0.0)))
|
|
rows.append({"seed": int(seed), "depression_fraction": float(fraction), "changed_edges": int(np.count_nonzero(changed)),
|
|
**eff, "functional_MBON_output_change": functional})
|
|
frame = pd.DataFrame(rows); frame.to_parquet(exp / "sensitivity/controlled-depression-curve.parquet", index=False)
|
|
aggregate = []
|
|
for frac, group in frame.groupby("depression_fraction"):
|
|
aggregate.append({"depression_fraction": float(frac), "seeds": int(len(group)), "functional_seed_count": int(group.functional_MBON_output_change.sum()),
|
|
"MBON_delta_g_max_abs_mean": float(group.MBON_delta_g_max_abs.mean()), "MBON_delta_v_max_abs_mean": float(group.MBON_delta_v_max_abs.mean()),
|
|
"MBON_spike_l1_mean": float(group.MBON_spike_l1.mean()), "downstream_spike_abs_delta_mean": float(group.downstream_spike_count_delta.abs().mean())})
|
|
functional_levels = [x["depression_fraction"] for x in aggregate if x["functional_seed_count"] > 0]
|
|
at50 = next(x for x in aggregate if x["depression_fraction"] == 0.5)
|
|
insensitive = at50["functional_seed_count"] == 0
|
|
report = {"changed_real_edges": int(np.count_nonzero(changed)), "curve": aggregate,
|
|
"first_fraction_with_any_functional_MBON_change": min(functional_levels) if functional_levels else None,
|
|
"MBON_READOUT_INSENSITIVE": bool(insensitive), "label": "MBON-READOUT-INSENSITIVE" if insensitive else "MBON-READOUT-SENSITIVE-WITHIN-TESTED-RANGE",
|
|
"diagnostic_only_not_saved_as_learned_model": True, "criterion": sc["functional_change_definition"]}
|
|
write_json(exp / "sensitivity/summary.json", report); return report
|
|
|
|
|
|
def isolated_replay_one(sim: AuditSimulator, spikes: pd.DataFrame, targets: np.ndarray, delta: np.ndarray, duration_ms: float) -> dict[str, Any]:
|
|
target_lookup = {int(x): i for i, x in enumerate(targets)}; nt = len(targets); steps = int(round(duration_ms / sim.dt)); qlen = sim.rt.queue_len
|
|
relevant = np.isin(sim.edge_sources, sim.v.kc) & np.isin(sim.indices_np, targets)
|
|
edge_ids = np.flatnonzero(relevant); by_source: dict[int, tuple[np.ndarray, np.ndarray]] = {}
|
|
full_delta = np.zeros(len(sim.indices_np), np.float32); full_delta[sim.v.edge_ids] = delta
|
|
weights = sim.base_np[edge_ids] + full_delta[edge_ids]
|
|
for source in np.unique(sim.edge_sources[edge_ids]):
|
|
mask = sim.edge_sources[edge_ids] == source; ids = edge_ids[mask]
|
|
by_source[int(source)] = (np.asarray([target_lookup[int(x)] for x in sim.indices_np[ids]], np.int32), weights[mask].astype(np.float32))
|
|
grouped = {int(k): g.neuron_idx.to_numpy(np.int32) for k, g in spikes.groupby("step")}
|
|
q = np.zeros((qlen, nt), np.float32); g = np.zeros(nt, np.float32); v = np.full(nt, sim.rt.rest, np.float32); ref = np.zeros(nt, np.int32)
|
|
counts = np.zeros(nt, np.int32); first = np.full(nt, -1, np.int32)
|
|
for step in range(steps):
|
|
slot = step % qlen; g = g * np.float32(sim.rt.g_decay) + q[slot]; q[slot].fill(0)
|
|
ref = np.maximum(ref, 1) - 1; available = ref == 0
|
|
v[available] += np.float32(sim.rt.alpha) * (g[available] - (v[available] - np.float32(sim.rt.rest))); v[~available] = np.float32(sim.rt.reset_value)
|
|
fired = available & (v >= np.float32(sim.rt.threshold)); counts += fired; first[(first < 0) & fired] = step; v[fired] = np.float32(sim.rt.reset_value); ref[fired] = sim.rt.refractory_steps
|
|
for source in grouped.get(step, np.empty(0, np.int32)):
|
|
if int(source) in by_source:
|
|
loc, w = by_source[int(source)]; np.add.at(q[(step + sim.rt.delay_steps) % qlen], loc, w)
|
|
return {"counts": counts, "first": first, "g": g, "v": v, "targets": targets}
|
|
|
|
|
|
def isolated_replay(sim: AuditSimulator, base_runs: dict[str, dict[str, Any]], actual_delta: np.ndarray, cfg: dict[str, Any], whole_direct_status: str, exp: Path) -> dict[str, Any]:
|
|
changed = actual_delta != 0; targets = np.unique(sim.v.cand.loc[changed, "target_neuron_idx"].to_numpy(np.int32)); rows = []; effects = {}
|
|
for label, key in (("A", "direct_KC_SET_A"), ("B", "direct_KC_SET_B")):
|
|
spikes = base_runs[key]["kc_spikes"]
|
|
pre = isolated_replay_one(sim, spikes, targets, np.zeros(len(actual_delta), np.float32), cfg["direct_kc"]["duration_ms"])
|
|
post = isolated_replay_one(sim, spikes, targets, actual_delta, cfg["direct_kc"]["duration_ms"])
|
|
def lat(x: dict[str, Any]) -> float | None:
|
|
return None if np.all(x["first"] < 0) else float(x["first"][x["first"] >= 0].min() * sim.dt)
|
|
effect = {"delta_g_max_abs": float(np.max(np.abs(post["g"] - pre["g"]))), "delta_v_max_abs": float(np.max(np.abs(post["v"] - pre["v"]))),
|
|
"spike_l1": int(np.abs(post["counts"] - pre["counts"]).sum()), "pre_first_latency_ms": lat(pre), "post_first_latency_ms": lat(post),
|
|
"latency_delta_ms": None if lat(pre) is None or lat(post) is None else float(lat(post) - lat(pre))}
|
|
effects[label] = effect
|
|
for state, x in (("pre", pre), ("post", post)):
|
|
for local, neuron in enumerate(targets):
|
|
rows.append({"stimulus": label, "state": state, "neuron_idx": int(neuron), "body_id": int(sim.v.meta.loc[neuron, "body_id"]),
|
|
"spikes": int(x["counts"][local]), "first_latency_ms": None if x["first"][local] < 0 else float(x["first"][local] * sim.dt),
|
|
"final_g": float(x["g"][local]), "final_v": float(x["v"][local])})
|
|
pd.DataFrame(rows).to_parquet(exp / "isolated-replay/real-edge-replay.parquet", index=False)
|
|
af = effects["A"]["spike_l1"] > 0 or effects["A"]["latency_delta_ms"] not in (None, 0.0)
|
|
bf = effects["B"]["spike_l1"] > 0 or effects["B"]["latency_delta_ms"] not in (None, 0.0)
|
|
transfer = bool(af and not bf); contamination = bool(transfer and whole_direct_status != "PASS-DIRECT-KC-MEMORY-v2")
|
|
report = {"target_MBON": int(len(targets)), "actual_changed_edges": int(np.count_nonzero(changed)), "effects": effects,
|
|
"PLASTICITY_TRANSFER_PASS": transfer, "plasticity_label": "PLASTICITY-TRANSFER-PASS" if transfer else "FAIL-PLASTICITY-TRANSFER",
|
|
"WHOLE_BRAIN_RECRUITMENT_CONTAMINATION": contamination,
|
|
"weak_effect_label": None if af else "KC-MBON-PLASTICITY-EFFECT-TOO-WEAK",
|
|
"spike_train_source": "deterministically regenerated from frozen Phase 4J protocol because exact Phase 4J event trains were not persisted",
|
|
"new_edges_added": False}
|
|
write_json(exp / "isolated-replay/summary.json", report); return report
|
|
|
|
|
|
def olfactory_provenance(sim: AuditSimulator, cfg: dict[str, Any], exp: Path) -> dict[str, Any]:
|
|
meta = sim.v.meta; olf = np.flatnonzero(sim.v.rt.population_masks["olfactory"]).astype(np.int32); olf.sort(); take = int(cfg["olfactory"]["targets_per_cs"])
|
|
current = np.concatenate([olf[:take], olf[take:2 * take]])
|
|
explicit = [x for x in meta.columns if any(term in x.lower() for term in ("glomer", "receptor", "sensillum", "antenn"))]
|
|
fields = [x for x in ("neuron_idx", "body_id", "annotation_superclass", "annotation_class", "annotation_subclass", "annotation_type", "annotation_instance", "side", "neurotransmitter", *explicit) if x in meta.columns]
|
|
fields = list(dict.fromkeys(fields))
|
|
table = meta.loc[current, fields].copy(); table.insert(0, "CS", np.repeat(["A", "B"], take)); table.to_parquet(exp / "olfactory-provenance/current-CS-neurons.parquet", index=False)
|
|
pn = np.flatnonzero(sim.v.cls == "ALPN").astype(np.int32); is_pn_kc = np.isin(sim.edge_sources, pn) & np.isin(sim.indices_np, sim.v.kc); eids = np.flatnonzero(is_pn_kc)
|
|
edges = pd.DataFrame({"edge_id": eids, "source_neuron_idx": sim.edge_sources[eids], "target_neuron_idx": sim.indices_np[eids], "raw_synapse_count": sim.raw_np[eids]})
|
|
edges["source_body_id"] = meta.loc[edges.source_neuron_idx, "body_id"].to_numpy(); edges["target_body_id"] = meta.loc[edges.target_neuron_idx, "body_id"].to_numpy()
|
|
edges.to_parquet(exp / "olfactory-provenance/ALPN-to-KC-real-edges.parquet", index=False)
|
|
identity_coverage = {x: int((table[x].notna() & ~table[x].fillna("").astype(str).eq("")).sum()) for x in explicit}
|
|
# Phase 4J selected two contiguous neuron_idx slices from the broad olfactory
|
|
# mask; it did not select or match by any receptor/glomerulus/PN identity.
|
|
grounded = False
|
|
report = {"olfactory_population_definition": "annotation_class == olfactory", "olfactory_population_count": int(len(olf)),
|
|
"current_CS_definition": "first 64 and next 64 neuron_idx after sorting the broad olfactory population", "current_CS_neurons": int(len(current)),
|
|
"explicit_glomerulus_or_receptor_columns": explicit, "explicit_identity_nonempty_counts_in_current_CS": identity_coverage,
|
|
"current_CS_selection_used_explicit_identity": False, "current_CS_glomerulus_grounded": grounded,
|
|
"label": "GLOMERULUS-GROUNDED" if grounded else "ODOR-INPUT-NOT-GLOMERULUS-GROUNDED",
|
|
"ALPN_count": int(len(pn)), "ALPN_to_KC_edges": int(len(eids)), "ALPN_to_KC_synapse_weight": int(sim.raw_np[eids].sum()),
|
|
"DoOR_imported": False, "reason_DoOR_not_imported": "Phase 4K mapping audit found no explicit MaleCNS glomerulus/receptor key for a non-guessing join",
|
|
"prepared_next_input_scheme": "odor response vector -> explicit receptor/glomerulus mapping -> matched MaleCNS PN identities -> real ALPN-to-KC edges",
|
|
"unmatched_channels": "all until an explicit official receptor/glomerulus-to-MaleCNS mapping is supplied"}
|
|
write_json(exp / "olfactory-provenance/summary.json", report); return report
|
|
|
|
|
|
def dataset_checks(cfg: dict[str, Any]) -> dict[str, Any]:
|
|
root = Path(cfg["paths"]["dataset_root"])
|
|
commands = {
|
|
"raw_core_sha256": f"cd {root}/raw && sha256sum -c _manifests/checksums.sha256 >/dev/null",
|
|
"raw_skeleton_md5": f"cd {root}/raw/skeletons && md5sum -c ../_manifests/skeletons/checksums.md5 >/dev/null",
|
|
"phase3_sha256": f"cd {root}/processed && sha256sum -c audit/checksums.sha256 >/dev/null",
|
|
"phase3b_audit_sha256": f"cd {root}/processed/neuron-only/audit && sha256sum -c checksums.sha256 >/dev/null",
|
|
"phase3b_166700_sha256": f"cd {root}/processed/neuron-only && sha256sum -c audit/phase3b-166700/checksums.sha256 >/dev/null",
|
|
"phase1_phase2_backup_sha256": f"sha256sum -c {root}/backups/male-cns-v1.0-phase1-phase2-20260919-1518.tar.zst.sha256 >/dev/null",
|
|
}
|
|
results = {}
|
|
for name, command in commands.items():
|
|
proc = subprocess.run(["bash", "-lc", command], capture_output=True, text=True)
|
|
results[name] = {"passed": proc.returncode == 0, "returncode": proc.returncode, "stderr": proc.stderr[-1000:]}
|
|
results["all_passed"] = all(x["passed"] for x in results.values())
|
|
return results
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(); ap.add_argument("--config", type=Path, required=True); args = ap.parse_args()
|
|
cfg = yaml.safe_load(args.config.read_text()); exp = Path(cfg["paths"]["experiment_root"]); model = Path(cfg["paths"]["phase4k_model_root"])
|
|
for sub in ("recruitment", "drive-decomposition", "mbon-decomposition", "sensitivity", "isolated-replay", "olfactory-provenance", "regression", "audit", "manifests", "compiler-reports/extension-build"):
|
|
(exp / sub).mkdir(parents=True, exist_ok=True)
|
|
model.mkdir(parents=True, exist_ok=True)
|
|
before = verify_frozen(cfg); p4j_checks = [verify_checksum(Path(cfg["paths"]["phase4j_root"]), Path(cfg["paths"]["phase4j_root"]) / "manifests/checksums.sha256"),
|
|
verify_checksum(Path(cfg["paths"]["phase4j_model_root"]), Path(cfg["paths"]["phase4j_model_root"]) / "checksums.sha256")]
|
|
if not all(x["passed"] for x in p4j_checks): raise RuntimeError("Phase 4J frozen checksum failure")
|
|
shutil.copy2(args.config, exp / "audit/phase4k.yaml"); shutil.copy2(Path(__file__).resolve(), exp / "audit/phase4k.py")
|
|
root = Path(cfg["paths"]["connectome_root"]); base_files = [root / "gpu/csr-indptr.int64.npy", root / "gpu/csr-indices.int32.npy", root / "gpu/csr-weights.int64.npy", root / "neuron-index/unified-metadata.parquet"]
|
|
base_before = {str(x): sha256(x) for x in base_files}
|
|
v = Validator(cfg, args.config.resolve(), exp); sim = AuditSimulator(v, cfg, exp)
|
|
sets = pd.read_parquet(Path(cfg["paths"]["phase4j_root"]) / "direct-kc-memory/direct-kc-ensembles.parquet")
|
|
set_a = np.sort(sets.loc[sets["set"] == "KC_SET_A", "neuron_idx"].to_numpy(np.int32)); set_b = np.sort(sets.loc[sets["set"] == "KC_SET_B", "neuron_idx"].to_numpy(np.int32))
|
|
overlay = np.load(Path(cfg["paths"]["phase4j_root"]) / f"direct-kc-memory/seed-{cfg['direct_kc']['trace_seed']}/overlay.npz")
|
|
if not np.array_equal(overlay["edge_id"], v.edge_ids): raise RuntimeError("Phase 4J overlay edge identity mismatch")
|
|
actual_delta = overlay["plastic_delta_mv"].astype(np.float32)
|
|
olf = np.flatnonzero(v.rt.population_masks["olfactory"]).astype(np.int32); olf.sort(); olf_a = olf[:cfg["olfactory"]["targets_per_cs"]]
|
|
|
|
drive_summary, base_runs = drive_audit(sim, set_a, set_b, olf_a, cfg, exp)
|
|
dan = v.dan_targets(cfg["direct_kc"]["matched_dan_type"], cfg["direct_kc"]["target_compartment_verbatim"])
|
|
causal_run = sim.run_audit(set_a, cfg["direct_kc"]["trace_seed"], "original", cfg["direct_kc"]["duration_ms"],
|
|
delta=np.zeros(len(v.edge_ids), np.float32), pulses=cfg["direct_kc"]["pulse_times_ms"], dan_targets=dan,
|
|
dan_pulses=[cfg["direct_kc"]["dan_time_ms"]], causal=True, collect_kc_spikes=True)
|
|
recruitment = recruitment_paths(sim, causal_run, set_a, set_b, actual_delta, exp)
|
|
mbon, post_runs = mbon_audit(sim, base_runs, set_a, set_b, actual_delta, cfg, exp)
|
|
curve = sensitivity(sim, set_a, actual_delta, cfg, exp)
|
|
p4j_summary = json.loads((Path(cfg["paths"]["phase4j_root"]) / "direct-kc-memory/summary.json").read_text())
|
|
replay = isolated_replay(sim, base_runs, actual_delta, cfg, p4j_summary["status"], exp)
|
|
odor = olfactory_provenance(sim, cfg, exp)
|
|
|
|
labels = {"KC_recruitment": recruitment["label"], "plasticity_transfer": replay["plasticity_label"],
|
|
"whole_brain_recruitment_contamination": replay["WHOLE_BRAIN_RECRUITMENT_CONTAMINATION"],
|
|
"MBON_readout_insensitive": curve["MBON_READOUT_INSENSITIVE"], "odor_input": odor["label"]}
|
|
del sim, v; gc.collect(); torch.cuda.empty_cache()
|
|
reg = phase4c_regression(cfg, args.config.resolve(), exp)
|
|
after = verify_frozen(cfg); p4j_after = [verify_checksum(Path(cfg["paths"]["phase4j_root"]), Path(cfg["paths"]["phase4j_root"]) / "manifests/checksums.sha256"),
|
|
verify_checksum(Path(cfg["paths"]["phase4j_model_root"]), Path(cfg["paths"]["phase4j_model_root"]) / "checksums.sha256")]
|
|
dataset = dataset_checks(cfg); base_after = {str(x): sha256(x) for x in base_files}
|
|
integrity = before["all_passed"] and after["all_passed"] and all(x["passed"] for x in p4j_checks + p4j_after) and dataset["all_passed"] and base_before == base_after and reg["all_passed"]
|
|
integrity_report = {"all_passed": integrity, "phase4c_to_4i_before": before["all_passed"], "phase4c_to_4i_after": after["all_passed"],
|
|
"phase4j_before": all(x["passed"] for x in p4j_checks), "phase4j_after": all(x["passed"] for x in p4j_after),
|
|
"dataset_phase1_to_3b": dataset, "base_CSR_unchanged": base_before == base_after, "phase4c_reference_regression": reg["all_passed"]}
|
|
write_json(exp / "audit/integrity.json", integrity_report); write_json(exp / "audit/base-sha256.json", {"before": base_before, "after": base_after})
|
|
summary = {"phase": "4K", "labels": labels, "recruitment": recruitment, "drive_decomposition": drive_summary, "MBON_decomposition": mbon,
|
|
"sensitivity": curve, "isolated_replay": replay, "olfactory_provenance": odor, "integrity": integrity_report,
|
|
"recommendation": "repair upstream odor/glomerulus-grounded encoding and KC recruitment before changing MBON plasticity/readout" if odor["label"] == "ODOR-INPUT-NOT-GLOMERULUS-GROUNDED" else "use causal audit to choose upstream recruitment versus readout work",
|
|
"eligibility_v2_modified": False, "base_modified": False, "plastic_mask_expanded": False, "stopped_before_phase4l": True}
|
|
write_json(exp / "audit/final-summary.json", summary)
|
|
shutil.copy2(args.config, model / "phase4k.yaml"); shutil.copy2(Path(__file__).resolve(), model / "phase4k.py")
|
|
write_json(model / "VERSION.json", {"name": cfg["name"], "diagnostic_only": True, "labels": labels, "eligibility_v2_modified": False,
|
|
"summary_sha256": sha256(exp / "audit/final-summary.json")})
|
|
print(json.dumps({"labels": labels, "integrity": integrity}, indent=2)); sys.stdout.flush()
|
|
seal(model, model / "MANIFEST.json", model / "checksums.sha256"); seal(exp, exp / "manifests/manifest.json", exp / "manifests/checksums.sha256")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|