qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-09-29 03:38:32 +08:00
commit 3981628ff8
682 changed files with 161475 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/male-cns-lif/visual-interface-v2/MANIFEST.json
+25
View File
@@ -0,0 +1,25 @@
{
"artifact": "MaleCNS-VisualAdapter-v2-candidate",
"files": [
{
"bytes": 2063,
"path": "VERSION.json",
"sha256": "2cd29361a570970e02ca1bb9943e5b3c6fc42688cacfd5082688960aa8808038"
},
{
"bytes": 23822,
"path": "brain_runtime.py",
"sha256": "1c4e3e6f06708bfec30b24e92297f320b2b8b87ebbbe6da501f5f2db4f74762a"
},
{
"bytes": 43339,
"path": "phase4e.py",
"sha256": "a56fead8424e4e79be370dade0a86fec10438348968819481960b0b693411f73"
},
{
"bytes": 4590,
"path": "phase4e.yaml",
"sha256": "708b6bb75b82a967f8c1e65cce646f080a79f7b6048ca72e61de1dc184d9923a"
}
]
}
models/male-cns-lif/visual-interface-v2/VERSION.json
+51
View File
@@ -0,0 +1,51 @@
{
"connectome_changed": false,
"formal_v2_frozen": false,
"gateway": {
"adapter_v2_eligible": false,
"adapter_v2_gate_note": "gateway selection uses anatomy/path/propagation; formal adapter eligibility additionally requires deterministic separability",
"condition": {
"aborted_runaway": false,
"active_fraction": 0.008014397120575886,
"active_neurons": 1336,
"amplitude_multiple_w_syn": 32.0,
"amplitude_mv": 8.8,
"descending_active": 142,
"descending_first_spike_latency_ms": 87.9,
"descending_spikes": 248,
"external_drive_events": 2334,
"layer": "engineering-visual-projection",
"mean_firing_rate_hz": 0.11271745650869826,
"motor_active": 93,
"motor_first_spike_latency_ms": 93.30000000000001,
"motor_spikes": 121,
"multi_hop_output_propagation": true,
"non_target_active": 1324,
"numerically_finite": true,
"output_first_spike_latency_ms": 87.9,
"peak_step_fraction": 0.0003299340131973605,
"propagation_depth_from_full_layer": 2,
"rate_hz": 50.0,
"requested_biological_ms": 100.0,
"seed": 60758,
"simulated_biological_ms": 100.0,
"sparsity_percent": 5.0,
"sweep": "amplitude-extended",
"target_active": 12,
"target_count": 460,
"total_spikes": 1879,
"wall_seconds": 0.8299968829960562
},
"handpicked": false,
"layer": "engineering-visual-projection",
"minimum_output_hops": 1,
"name": "visual_gateway_candidate",
"rule": "gateway iff multi-hop propagation reaches descending or motor output and is non-runaway; rank by lowest amplitude multiple, then lowest min output hops, then hierarchy order. Formal adapter-v2 eligibility additionally requires deterministic nonzero output separability and matched-input-count validation",
"selected": true
},
"lif_semantics_changed": false,
"name": "MaleCNS-VisualAdapter-v2-candidate",
"phase4f_started": false,
"status": "CANDIDATE-REJECTED",
"transmitter_sign_changed": false
}
models/male-cns-lif/visual-interface-v2/brain_runtime.py
+415
View File
@@ -0,0 +1,415 @@
#!/usr/bin/env python3
"""Stable game-agnostic Brain I/O API for MaleCNS-LIF-runtime-v1."""
from __future__ import annotations
import copy
import json
import math
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
import numpy as np
import pyarrow.parquet as pq
import torch
import yaml
API_VERSION = "1.0.0"
INPUT_BATCH_VERSION = "1.0.0"
ACTIVITY_STREAM_VERSION = "1.0.0"
@dataclass(frozen=True)
class InputEvent:
target_neuron_idx: np.ndarray
stimulation_mode: str
value_kind: str
value: float | np.ndarray
start_ms: float
duration_ms: float
source_id: str
channel_id: str
amplitude_mv: float = 0.0
metadata: dict[str, Any] = field(default_factory=dict)
def validate(self, neuron_count: int, dt_ms: float) -> None:
target = np.asarray(self.target_neuron_idx)
if target.ndim != 1 or target.dtype.kind not in "iu":
raise ValueError("target_neuron_idx must be a one-dimensional integer array")
if len(target) and (int(target.min()) < 0 or int(target.max()) >= neuron_count):
raise ValueError("target neuron_idx outside canonical range")
if len(np.unique(target)) != len(target):
raise ValueError("target neuron_idx values must be unique")
if self.stimulation_mode not in {"spike_injection", "external_synaptic_drive"}:
raise ValueError(f"unsupported stimulation_mode {self.stimulation_mode!r}")
if self.value_kind not in {"binary", "scalar_intensity", "poisson_rate_hz", "signed_modulation"}:
raise ValueError(f"unsupported value_kind {self.value_kind!r}")
if self.start_ms < 0 or self.duration_ms <= 0:
raise ValueError("timestamps/duration must be non-negative/positive")
for x in (self.start_ms, self.duration_ms):
if not math.isclose(x / dt_ms, round(x / dt_ms), abs_tol=1e-8):
raise ValueError(f"{x} ms is not aligned to dt={dt_ms} ms")
value = np.asarray(self.value)
if value.ndim > 1 or (value.ndim == 1 and len(value) not in (1, len(target))):
raise ValueError("value must be scalar or match target count")
if self.value_kind == "poisson_rate_hz" and np.any(value < 0):
raise ValueError("Poisson rate cannot be negative")
@dataclass(frozen=True)
class InputBatch:
events: tuple[InputEvent, ...]
batch_id: str
version: str = INPUT_BATCH_VERSION
metadata: dict[str, Any] = field(default_factory=dict)
def validate(self, neuron_count: int, dt_ms: float) -> None:
if self.version != INPUT_BATCH_VERSION:
raise ValueError(f"InputBatch version must be {INPUT_BATCH_VERSION}")
for event in self.events:
event.validate(neuron_count, dt_ms)
class VisualAdapterV1:
"""Maps abstract channels to audited visual hierarchy populations."""
def __init__(self, config: dict[str, Any], phase4c_root: Path):
self.config = config
self.phase4c_root = phase4c_root
self.modes: dict[str, np.ndarray] = {}
for mode, spec in config["visual_adapter"]["modes"].items():
population = spec["hierarchy_population"]
path = phase4c_root / "visual-audit" / f"hierarchy-{population}.parquet"
if not path.is_file():
raise FileNotFoundError(path)
self.modes[mode] = pq.read_table(path, columns=["neuron_idx"])["neuron_idx"].to_numpy().astype(np.int64)
def population(self, mode: str) -> np.ndarray:
if mode not in self.modes:
raise KeyError(mode)
return self.modes[mode].copy()
def encode(
self, mode: str, channel_values: np.ndarray, start_ms: float, duration_ms: float,
source_id: str, channel_id: str, value_kind: str = "poisson_rate_hz",
) -> InputBatch:
if mode not in self.modes:
raise KeyError(mode)
spec = self.config["visual_adapter"]["modes"][mode]
targets = self.modes[mode]
values = np.asarray(channel_values, dtype=np.float32)
if values.shape != (len(targets),):
raise ValueError(f"visual channel must have shape {(len(targets),)}, got {values.shape}")
selected = np.flatnonzero(values != 0).astype(np.int64)
target = targets[selected]
selected_values = values[selected]
event = InputEvent(
target_neuron_idx=target,
stimulation_mode=spec["input_mode"],
value_kind=value_kind,
value=selected_values,
amplitude_mv=float(self.config["visual_adapter"]["external_drive_mv"]),
start_ms=start_ms,
duration_ms=duration_ms,
source_id=source_id,
channel_id=channel_id,
metadata={
"adapter": self.config["names"]["visual_adapter"],
"adapter_mode": mode,
"biological_fidelity": spec["biological_fidelity"],
"connectome_weights_modified": False,
},
)
return InputBatch(events=(event,), batch_id=f"{source_id}:{channel_id}:{start_ms}", metadata={"adapter_mode": mode})
class BrainRuntime:
"""Persistent exact-state BrainRuntime API. It has no game-specific dependency."""
def __init__(self, config_path: Path, extension_build_dir: Path):
self.config_path = config_path.resolve()
self.cfg = yaml.safe_load(self.config_path.read_text(encoding="utf-8"))
self.dt = float(self.cfg["model"]["dt_ms"])
self.n = 166700
self.model_root = Path(self.cfg["paths"]["model_root"])
self.runtime_root = Path(self.cfg["paths"]["runtime_root"])
self.input_root = Path(self.cfg["paths"]["connectome_root"])
runtime_module_dir = str(self.runtime_root)
if runtime_module_dir not in sys.path:
sys.path.insert(0, runtime_module_dir)
import phase4c as runtime_module
self.runtime_module = runtime_module
p4cfg = yaml.safe_load((self.runtime_root / "phase4c.yaml").read_text(encoding="utf-8"))
self.paths = runtime_module.exact_paths(p4cfg)
self.indptr_np, self.indices_np, self.raw_np = runtime_module.p4b.validate_graph(p4cfg, self.paths)
self.table = runtime_module.full_metadata(self.paths["metadata"])
self.population_masks = runtime_module.populations(self.table)
if not torch.cuda.is_available() or torch.cuda.device_count() != 1:
raise RuntimeError("BrainRuntime requires exactly one visible CUDA GPU")
self.device = torch.device("cuda:0")
props = torch.cuda.get_device_properties(self.device)
if self.cfg["device"]["required_name_substring"] not in props.name:
raise RuntimeError(f"required RTX 4090, got {props.name}")
extension_build_dir.mkdir(parents=True, exist_ok=True)
build_experiment_root = extension_build_dir.parent.parent
(build_experiment_root / "compiler-reports/extension-build").mkdir(parents=True, exist_ok=True)
self.extension, report = runtime_module.build_extension(self.runtime_root, build_experiment_root)
if self.extension is None:
raise RuntimeError(f"runtime-v1 extension unavailable: {report}")
base = runtime_module.Runtime(p4cfg, self.table, self.indptr_np, self.indices_np, self.raw_np, self.device, self.extension)
self.indptr, self.indices, self.weights = base.indptr, base.indices, base.weights
self.signs, self.superclass = base.signs, base.superclass
self.neuron_type = np.asarray(self.table["annotation_type"].to_pylist(), dtype=object)
self.neuron_class = np.asarray(self.table["annotation_class"].to_pylist(), dtype=object)
self.body_id = self.table["body_id"].to_numpy().astype(np.uint64)
m = self.cfg["model"]
self.delay_steps = int(round(float(m["synaptic_delay_ms"]) / self.dt))
self.queue_len = self.delay_steps + 1
self.refractory_steps = int(round(float(m["refractory_ms"]) / self.dt))
self.alpha = self.dt / float(m["membrane_tau_ms"])
self.g_decay = 1.0 - self.dt / float(m["synaptic_tau_ms"])
self.rest = float(m["resting_potential_mv"])
self.reset_value = float(m["reset_potential_mv"])
self.threshold = float(m["threshold_mv"])
self.output_populations = self._output_populations()
self.reset(0)
def _output_populations(self) -> dict[str, np.ndarray]:
s = self.superclass
return {
"descending": np.flatnonzero(np.isin(s, ["descending_neuron", "descending_neuron_tbc"])).astype(np.int32),
"motor_related": np.flatnonzero(np.isin(s, ["vnc_motor", "cb_motor"])).astype(np.int32),
"ascending": np.flatnonzero(np.isin(s, ["ascending_neuron", "sensory_ascending", "sensory_ascending_tbc", "efferent_ascending"])).astype(np.int32),
}
def reset(self, seed: int, initial_state: dict[str, Any] | None = None) -> dict[str, Any]:
self.seed = int(seed)
self.generator = torch.Generator(device=self.device); self.generator.manual_seed(self.seed)
self.v = torch.full((self.n,), self.rest, dtype=torch.float32, device=self.device)
self.g = torch.zeros(self.n, dtype=torch.float32, device=self.device)
self.refractory = torch.zeros(self.n, dtype=torch.int32, device=self.device)
self.delay_queue = torch.zeros((self.queue_len, self.n), dtype=torch.float32, device=self.device)
self.spike_counts = torch.zeros(self.n, dtype=torch.int32, device=self.device)
self.simulation_step = 0; self.sequence_id = 0
self.spike_steps: list[np.ndarray] = []; self.spike_nodes: list[np.ndarray] = []
self.spike_origins: list[np.ndarray] = []
self.drive_steps: list[np.ndarray] = []; self.drive_nodes: list[np.ndarray] = []
self.drive_values: list[np.ndarray] = []; self.input_audit: list[dict[str, Any]] = []
if initial_state is not None:
self.restore_state(initial_state)
return self.get_state_summary()
@property
def simulation_time_ms(self) -> float:
return self.simulation_step * self.dt
def _event_values(self, event: InputEvent, targets: torch.Tensor) -> torch.Tensor:
value = np.asarray(event.value, dtype=np.float32)
if value.ndim == 0:
return torch.full((len(targets),), float(value), dtype=torch.float32, device=self.device)
return torch.as_tensor(value, dtype=torch.float32, device=self.device)
def _active(self, event: InputEvent, time_ms: float) -> bool:
return event.start_ms - 1e-9 <= time_ms < event.start_ms + event.duration_ms - 1e-9
def _hybrid_edges(self, sources: torch.Tensor) -> torch.Tensor:
starts = self.indptr[sources]
degrees = self.indptr[sources + 1] - starts
cumulative = torch.cumsum(degrees, 0)
total = int(cumulative[-1].item()) if cumulative.numel() else 0
output = torch.empty(total, dtype=torch.int64, device=self.device)
if total:
self.extension.fill_edges(self.indptr, sources, cumulative - degrees, output)
return output
def step(self, input_batch: InputBatch | None, biological_ms: float) -> dict[str, Any]:
steps = int(round(float(biological_ms) / self.dt))
if not math.isclose(steps * self.dt, float(biological_ms), abs_tol=1e-8) or steps <= 0:
raise ValueError("biological_ms must be a positive multiple of dt")
batch = input_batch or InputBatch(events=(), batch_id="zero-input")
batch.validate(self.n, self.dt)
batch_start = self.simulation_time_ms
prepared = [(e, torch.as_tensor(e.target_neuron_idx, dtype=torch.int64, device=self.device)) for e in batch.events]
for event in batch.events:
self.input_audit.append({
"batch_id": batch.batch_id, "source_id": event.source_id, "channel_id": event.channel_id,
"mode": event.stimulation_mode, "value_kind": event.value_kind,
"start_ms": event.start_ms, "duration_ms": event.duration_ms,
"target_count": int(len(event.target_neuron_idx)), "metadata": event.metadata,
"external_drive_is_connectome_transmission": False,
})
start_event_count = sum(len(x) for x in self.spike_nodes)
start_drive_count = sum(len(x) for x in self.drive_nodes)
for _ in range(steps):
global_step = self.simulation_step; time_ms = self.simulation_time_ms
slot = global_step % self.queue_len
self.g.mul_(self.g_decay).add_(self.delay_queue[slot]); self.delay_queue[slot].zero_()
spike_requests: list[tuple[InputEvent, torch.Tensor, torch.Tensor]] = []
for event, targets in prepared:
if not self._active(event, time_ms) or targets.numel() == 0:
continue
values = self._event_values(event, targets)
if event.value_kind == "poisson_rate_hz":
selected = torch.rand(len(targets), generator=self.generator, device=self.device) < (values * self.dt / 1000.0)
elif event.value_kind == "binary":
selected = values != 0
elif event.value_kind == "scalar_intensity":
selected = values != 0
else: # signed_modulation
selected = torch.rand(len(targets), generator=self.generator, device=self.device) < (torch.abs(values) * self.dt / 1000.0)
if event.stimulation_mode == "external_synaptic_drive":
chosen = targets[selected]
if chosen.numel():
signed_value = values[selected] * float(event.amplitude_mv)
if event.value_kind == "poisson_rate_hz":
signed_value = torch.full_like(signed_value, float(event.amplitude_mv))
self.g.index_add_(0, chosen, signed_value)
node_np = chosen.cpu().numpy().astype(np.int32, copy=True)
val_np = signed_value.cpu().numpy().astype(np.float32, copy=True)
self.drive_steps.append(np.full(len(node_np), global_step, dtype=np.int32))
self.drive_nodes.append(node_np); self.drive_values.append(val_np)
else:
spike_requests.append((event, targets, selected))
self.refractory.clamp_min_(1).sub_(1)
available = self.refractory == 0
self.v[available] += self.alpha * (self.g[available] - (self.v[available] - self.rest))
self.v[~available] = self.reset_value
natural = available & (self.v >= self.threshold)
spikes = natural.clone()
injected = torch.zeros(self.n, dtype=torch.bool, device=self.device)
for _event, targets, selected in spike_requests:
accepted = selected & available[targets]
chosen = targets[accepted]
if chosen.numel():
spikes[chosen] = True; injected[chosen] = ~natural[chosen]
sources = torch.nonzero(spikes, as_tuple=False).flatten()
self.spike_counts.add_(spikes.to(torch.int32))
if sources.numel():
self.v[sources] = self.reset_value; self.refractory[sources] = self.refractory_steps
edge_ids = self._hybrid_edges(sources)
if edge_ids.numel():
self.delay_queue[(global_step + self.delay_steps) % self.queue_len].scatter_add_(0, self.indices[edge_ids].long(), self.weights[edge_ids])
src_np = sources.cpu().numpy().astype(np.int32, copy=True)
origin_np = np.where(injected[sources].cpu().numpy(), "external_spike_injection", "network_spike")
self.spike_steps.append(np.full(len(src_np), global_step, dtype=np.int32))
self.spike_nodes.append(src_np); self.spike_origins.append(origin_np)
self.simulation_step += 1
torch.cuda.synchronize()
self.sequence_id += 1
return {
"api_version": API_VERSION, "sequence_id": self.sequence_id,
"batch_id": batch.batch_id, "start_time_ms": batch_start,
"end_time_ms": self.simulation_time_ms, "biological_ms": biological_ms,
"new_spikes": sum(len(x) for x in self.spike_nodes) - start_event_count,
"new_external_drive_events": sum(len(x) for x in self.drive_nodes) - start_drive_count,
}
def _events(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
steps = np.concatenate(self.spike_steps) if self.spike_steps else np.empty(0, np.int32)
nodes = np.concatenate(self.spike_nodes) if self.spike_nodes else np.empty(0, np.int32)
origins = np.concatenate(self.spike_origins) if self.spike_origins else np.empty(0, object)
return steps, nodes, origins
def read_spikes(self, window_ms: float | None = None) -> dict[str, Any]:
steps, nodes, origins = self._events()
if window_ms is not None:
threshold = self.simulation_step - int(round(window_ms / self.dt))
mask = steps >= threshold; steps, nodes, origins = steps[mask], nodes[mask], origins[mask]
return {
"simulation_time_ms": self.simulation_time_ms,
"step": steps, "timestamp_ms": steps.astype(np.float64) * self.dt,
"neuron_idx": nodes, "origin": origins,
}
def read_population(self, population_id: str, window_ms: float) -> dict[str, Any]:
if population_id not in self.output_populations:
raise KeyError(population_id)
members = self.output_populations[population_id]
events = self.read_spikes(window_ms)
mask = np.isin(events["neuron_idx"], members)
nodes = events["neuron_idx"][mask]
counts = np.zeros(len(members), dtype=np.int32)
if len(nodes):
lookup = np.full(self.n, -1, dtype=np.int32); lookup[members] = np.arange(len(members), dtype=np.int32)
np.add.at(counts, lookup[nodes], 1)
seconds = window_ms / 1000.0
return {
"simulation_time_ms": self.simulation_time_ms, "population_id": population_id,
"neuron_idx": members.copy(), "spike_counts": counts,
"firing_rate_hz": counts.astype(np.float32) / seconds,
"active_mask": counts > 0, "aggregate_rate_hz": float(counts.sum() / (len(members) * seconds)),
"total_spikes": int(counts.sum()), "active_neurons": int(np.count_nonzero(counts)),
}
def read_output(self, output_spec: dict[str, Any], window_ms: float) -> dict[str, Any]:
population = output_spec.get("population", "descending")
aggregation = output_spec.get("aggregation", "per-neuron")
base = self.read_population(population, window_ms)
members = base["neuron_idx"]; counts = base["spike_counts"]
if aggregation == "per-neuron":
return base
if aggregation not in {"per-type", "per-class"}:
raise ValueError("aggregation must be per-neuron, per-type, or per-class")
labels = self.neuron_type if aggregation == "per-type" else self.neuron_class
result: dict[str, dict[str, float | int]] = {}
for label in sorted({"<null>" if labels[i] is None else str(labels[i]) for i in members}):
mask = np.asarray([("<null>" if labels[i] is None else str(labels[i])) == label for i in members])
result[label] = {"neuron_count": int(mask.sum()), "spike_count": int(counts[mask].sum()), "mean_rate_hz": float(base["firing_rate_hz"][mask].mean())}
return {"simulation_time_ms": self.simulation_time_ms, "population_id": population, "aggregation": aggregation, "features": result}
def fixed_temporal_bins(self, population_id: str, bin_ms: float, window_ms: float) -> dict[str, Any]:
if bin_ms not in self.cfg["readout"]["bins_ms"]:
raise ValueError("unsupported fixed bin")
ev = self.read_spikes(window_ms); members = self.output_populations[population_id]
mask = np.isin(ev["neuron_idx"], members); times = ev["timestamp_ms"][mask]
start = self.simulation_time_ms - window_ms
edges = np.arange(start, self.simulation_time_ms + bin_ms + 1e-9, bin_ms)
counts, _ = np.histogram(times, bins=edges)
return {"simulation_time_ms": self.simulation_time_ms, "population_id": population_id, "bin_ms": bin_ms, "bin_start_ms": edges[:-1], "spike_counts": counts.astype(np.int32)}
def get_state_summary(self) -> dict[str, Any]:
return {
"api_version": API_VERSION, "runtime": self.cfg["names"]["runtime"],
"simulation_step": self.simulation_step, "simulation_time_ms": self.simulation_time_ms,
"sequence_id": self.sequence_id, "seed": self.seed,
"neurons": self.n, "edges": int(self.indices.numel()),
"active_neurons_total": int(torch.count_nonzero(self.spike_counts).item()),
"spikes_total": int(self.spike_counts.sum().item()),
"external_drive_events_total": sum(len(x) for x in self.drive_nodes),
}
def checkpoint_state(self) -> dict[str, Any]:
return {
"api_version": API_VERSION, "seed": self.seed, "simulation_step": self.simulation_step,
"sequence_id": self.sequence_id, "generator_state": self.generator.get_state().cpu(),
"v": self.v.clone(), "g": self.g.clone(), "refractory": self.refractory.clone(),
"delay_queue": self.delay_queue.clone(), "spike_counts": self.spike_counts.clone(),
"spike_steps": copy.deepcopy(self.spike_steps), "spike_nodes": copy.deepcopy(self.spike_nodes),
"spike_origins": copy.deepcopy(self.spike_origins), "drive_steps": copy.deepcopy(self.drive_steps),
"drive_nodes": copy.deepcopy(self.drive_nodes), "drive_values": copy.deepcopy(self.drive_values),
"input_audit": copy.deepcopy(self.input_audit),
}
def restore_state(self, state: dict[str, Any]) -> dict[str, Any]:
if state.get("api_version") != API_VERSION:
raise ValueError("checkpoint API version mismatch")
self.seed = int(state["seed"]); self.simulation_step = int(state["simulation_step"]); self.sequence_id = int(state["sequence_id"])
self.generator.set_state(state["generator_state"].cpu())
for name in ("v", "g", "refractory", "delay_queue", "spike_counts"):
setattr(self, name, state[name].clone().to(self.device))
for name in ("spike_steps", "spike_nodes", "spike_origins", "drive_steps", "drive_nodes", "drive_values", "input_audit"):
setattr(self, name, copy.deepcopy(state[name]))
return self.get_state_summary()
def activity_packet(self, window_ms: float, input_metadata: dict[str, Any] | None = None) -> dict[str, Any]:
ev = self.read_spikes(window_ms)
aggregates = {name: {k: v for k, v in self.read_population(name, window_ms).items() if k in ("active_neurons", "total_spikes", "aggregate_rate_hz")} for name in self.output_populations}
return {
"contract_version": ACTIVITY_STREAM_VERSION, "simulation_time_ms": self.simulation_time_ms,
"sequence_id": self.sequence_id, "window_ms": window_ms,
"spike_neuron_idx": ev["neuron_idx"].tolist(), "spike_timestamp_ms": ev["timestamp_ms"].tolist(),
"population_aggregates": aggregates, "active_neuron_count": int(len(np.unique(ev["neuron_idx"]))),
"input_channel_metadata": input_metadata or {}, "representation": "event-only",
}
models/male-cns-lif/visual-interface-v2/checksums.sha256
+5
View File
@@ -0,0 +1,5 @@
2cd29361a570970e02ca1bb9943e5b3c6fc42688cacfd5082688960aa8808038 VERSION.json
1c4e3e6f06708bfec30b24e92297f320b2b8b87ebbbe6da501f5f2db4f74762a brain_runtime.py
a56fead8424e4e79be370dade0a86fec10438348968819481960b0b693411f73 phase4e.py
708b6bb75b82a967f8c1e65cce646f080a79f7b6048ca72e61de1dc184d9923a phase4e.yaml
f41b9f4667a335af45b01e811b58ac1d4fac9256d5227016596e87cec94132bf MANIFEST.json
models/male-cns-lif/visual-interface-v2/phase4e.py
+668
View File
@@ -0,0 +1,668 @@
#!/usr/bin/env python3
"""Phase 4E: auditable engineering visual-input calibration for MaleCNS-LIF."""
from __future__ import annotations
import argparse
import gc
import hashlib
import json
import math
import os
import platform
import shutil
import sys
import time
from collections import Counter, deque
from pathlib import Path
from typing import Any
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import torch
import yaml
from scipy import sparse
from scipy.sparse import csgraph
from brain_runtime import BrainRuntime, InputBatch, InputEvent
def sha256(path: Path, chunk: int = 8 << 20) -> str:
h = hashlib.sha256()
with path.open("rb") as f:
while data := f.read(chunk):
h.update(data)
return h.hexdigest()
def write_json(path: Path, value: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, indent=2, sort_keys=True, default=np_json) + "\n", encoding="utf-8")
def np_json(value: Any) -> Any:
if isinstance(value, np.ndarray):
return value.tolist()
if isinstance(value, np.generic):
return value.item()
if isinstance(value, Path):
return str(value)
raise TypeError(type(value).__name__)
def stable_seed(base: int, text: str) -> int:
return base + int.from_bytes(hashlib.sha256(text.encode()).digest()[:4], "little") % 1_000_000
def verify_checksum_file(root: Path, checksum_file: Path) -> dict[str, Any]:
rows, failures = 0, []
for line in checksum_file.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
expected, rel = line.split(None, 1)
path = root / rel.strip().lstrip("*")
rows += 1
if not path.is_file() or sha256(path) != expected:
failures.append(str(path))
return {"checksum_file": str(checksum_file), "checked": rows, "failures": failures, "passed": not failures}
def immutable_audit(cfg: dict[str, Any]) -> tuple[list[Path], dict[str, str], dict[str, Any]]:
p = cfg["paths"]
controls = [
Path(p["reference_root"]) / "checksums.sha256",
Path(p["runtime_root"]) / "checksums.sha256",
Path(p["io_root"]) / "checksums.sha256",
Path(p["phase4c_root"]) / "manifests/checksums.sha256",
Path(p["phase4d_root"]) / "manifests/checksums.sha256",
]
missing = [str(x) for x in controls if not x.is_file()]
if missing:
raise FileNotFoundError(f"immutable checksum controls missing: {missing}")
snapshots = {str(x): sha256(x) for x in controls}
checks = []
for x in controls:
# Model seals are relative to their model directory; experiment seals
# live in a manifests/ subdirectory but list paths relative to the
# experiment root.
root = x.parent.parent if x.parent.name == "manifests" else x.parent
checks.append(verify_checksum_file(root, x))
if not all(x["passed"] for x in checks):
raise RuntimeError(f"immutable baseline checksum failure: {checks}")
return controls, snapshots, {"controls": checks, "all_passed": True}
def load_populations(cfg: dict[str, Any], runtime: BrainRuntime, exp: Path) -> tuple[dict[str, np.ndarray], dict[str, Any]]:
root = Path(cfg["paths"]["phase4c_root"]) / "visual-audit"
populations: dict[str, np.ndarray] = {}
summary: dict[str, Any] = {}
for name, spec in cfg["visual_populations"].items():
if spec["source"] == "hierarchy":
path = root / f"hierarchy-{spec['value']}.parquet"
nodes = pq.read_table(path, columns=["neuron_idx"])["neuron_idx"].to_numpy().astype(np.int32)
expression = f"Phase4C audited hierarchy={spec['value']}"
else:
field = spec["field"]
if field not in runtime.table.column_names:
raise RuntimeError(f"metadata field absent: {field}")
values = np.asarray(runtime.table[field].to_pylist(), dtype=object)
nodes = np.flatnonzero(np.isin(values, spec["values"])).astype(np.int32)
expression = f"{field} in {spec['values']}"
if not len(nodes) or len(np.unique(nodes)) != len(nodes):
raise RuntimeError(f"invalid population {name}")
populations[name] = nodes
path = exp / "populations" / f"{name}.parquet"
pq.write_table(pa.table({
"neuron_idx": pa.array(nodes, type=pa.int32()),
"body_id": pa.array(runtime.body_id[nodes], type=pa.uint64()),
"superclass": pa.array([runtime.superclass[i] for i in nodes], type=pa.string()),
"class": pa.array([runtime.neuron_class[i] for i in nodes], type=pa.string()),
"type": pa.array([runtime.neuron_type[i] for i in nodes], type=pa.string()),
}), path, compression="zstd")
summary[name] = {"count": int(len(nodes)), "selection": expression, **spec, "membership": str(path)}
write_json(exp / "populations/summary.json", summary)
return populations, summary
def reverse_bfs(indptr: np.ndarray, indices: np.ndarray, sources: np.ndarray, n: int) -> np.ndarray:
dist = np.full(n, -1, dtype=np.int16)
dist[sources] = 0
frontier = sources.astype(np.int64, copy=False)
depth = 0
while len(frontier):
depth += 1
mask = np.zeros(n, dtype=bool)
for node in frontier:
mask[indices[int(indptr[node]):int(indptr[node + 1])]] = True
mask[dist >= 0] = False
frontier = np.flatnonzero(mask).astype(np.int64)
if len(frontier):
dist[frontier] = depth
return dist
def graph_reachability(cfg: dict[str, Any], runtime: BrainRuntime, populations: dict[str, np.ndarray], exp: Path) -> dict[str, Any]:
cache = exp / "reachability/visual-output-reachability.parquet"
summary_path = exp / "reachability/summary.json"
if cache.is_file() and summary_path.is_file():
return json.loads(summary_path.read_text(encoding="utf-8"))
n = runtime.n
data = np.ones(len(runtime.indices_np), dtype=np.uint8)
graph = sparse.csr_matrix((data, runtime.indices_np.astype(np.int32, copy=False), runtime.indptr_np), shape=(n, n))
reverse = graph.transpose().tocsr()
desc = runtime.output_populations["descending"].astype(np.int64)
motor = runtime.output_populations["motor_related"].astype(np.int64)
desc_dist = reverse_bfs(reverse.indptr, reverse.indices, desc, n)
motor_dist = reverse_bfs(reverse.indptr, reverse.indices, motor, n)
component_count, labels = csgraph.connected_components(graph, directed=True, connection="strong", return_labels=True)
src = np.repeat(np.arange(n, dtype=np.int32), np.diff(runtime.indptr_np))
csrc, cdst = labels[src], labels[runtime.indices_np]
keep = csrc != cdst
condensation = sparse.csr_matrix((np.ones(np.count_nonzero(keep), np.uint8), (csrc[keep], cdst[keep])), shape=(component_count, component_count))
condensation.sum_duplicates(); condensation.data[:] = 1
indegree = np.bincount(condensation.indices, minlength=component_count).astype(np.int64)
queue = deque(np.flatnonzero(indegree == 0).tolist()); topo: list[int] = []
while queue:
c = queue.popleft(); topo.append(c)
for nxt in condensation.indices[condensation.indptr[c]:condensation.indptr[c + 1]]:
indegree[nxt] -= 1
if indegree[nxt] == 0:
queue.append(int(nxt))
if len(topo) != component_count:
raise RuntimeError("SCC condensation is unexpectedly cyclic")
output = np.concatenate([desc, motor])
bits = [0] * component_count
for bit, node in enumerate(output):
bits[int(labels[node])] |= 1 << bit
for c in reversed(topo):
value = bits[c]
for nxt in condensation.indices[condensation.indptr[c]:condensation.indptr[c + 1]]:
value |= bits[int(nxt)]
bits[c] = value
desc_mask = (1 << len(desc)) - 1
reach_desc_comp = np.fromiter(((x & desc_mask).bit_count() for x in bits), dtype=np.int32, count=component_count)
reach_motor_comp = np.fromiter(((x >> len(desc)).bit_count() for x in bits), dtype=np.int32, count=component_count)
rows: list[pa.Table] = []
by_layer: dict[str, Any] = {}
raw = runtime.raw_np
for layer, nodes in populations.items():
starts, stops = runtime.indptr_np[nodes], runtime.indptr_np[nodes + 1]
edge_count = int(np.sum(stops - starts, dtype=np.int64))
weight = int(sum(int(raw[int(a):int(b)].sum(dtype=np.int64)) for a, b in zip(starts, stops)))
rd, rm = reach_desc_comp[labels[nodes]], reach_motor_comp[labels[nodes]]
dd, dm = desc_dist[nodes], motor_dist[nodes]
rows.append(pa.table({
"neuron_idx": pa.array(nodes, type=pa.int32()),
"body_id": pa.array(runtime.body_id[nodes], type=pa.uint64()),
"visual_layer": pa.array([layer] * len(nodes), type=pa.string()),
"min_hops_descending": pa.array(dd, type=pa.int16()),
"min_hops_motor": pa.array(dm, type=pa.int16()),
"reachable_descending_count": pa.array(rd, type=pa.int32()),
"reachable_motor_count": pa.array(rm, type=pa.int32()),
}))
by_layer[layer] = {
"count": int(len(nodes)), "outgoing_edges": edge_count, "outgoing_synapse_weight": weight,
"can_reach_descending": int(np.count_nonzero(rd)), "can_reach_motor": int(np.count_nonzero(rm)),
"min_hops_descending_distribution": dict(sorted(Counter(map(int, dd)).items())),
"min_hops_motor_distribution": dict(sorted(Counter(map(int, dm)).items())),
"reachable_descending_count_min_max": [int(rd.min(initial=0)), int(rd.max(initial=0))],
"reachable_motor_count_min_max": [int(rm.min(initial=0)), int(rm.max(initial=0))],
}
pq.write_table(pa.concat_tables(rows), cache, compression="zstd")
summary = {
"method": "exact directed reachability via SCC-condensation DAG bitsets; min hops via reverse multi-source BFS",
"thresholding": False, "strongly_connected_components": int(component_count),
"descending_count": int(len(desc)), "motor_count": int(len(motor)), "layers": by_layer,
"table": str(cache),
}
write_json(summary_path, summary)
return summary
def forward_distances(runtime: BrainRuntime, populations: dict[str, np.ndarray], exp: Path) -> dict[str, np.ndarray]:
result = {}
for name, nodes in populations.items():
path = exp / "reachability" / f"forward-distance-{name}.int16.npy"
if path.is_file():
result[name] = np.load(path)
else:
result[name] = runtime.runtime_module.p4b.directed_distances(runtime.indptr_np, runtime.indices_np, nodes.astype(np.int64), runtime.n)
np.save(path, result[name])
return result
def subset(nodes: np.ndarray, percent: float, seed: int, offset: int = 0) -> np.ndarray:
count = max(1, int(round(len(nodes) * percent / 100.0)))
order = np.random.default_rng(seed).permutation(len(nodes))
if offset + count <= len(nodes):
chosen = order[offset:offset + count]
else:
chosen = np.roll(order, -offset)[:count]
return np.sort(nodes[chosen].astype(np.int64))
def make_batch(layer: str, nodes: np.ndarray, rate: float, amplitude: float, duration: float, source: str,
stimulation_mode: str = "external_synaptic_drive", events: tuple[InputEvent, ...] | None = None) -> InputBatch:
if events is None:
event = InputEvent(
target_neuron_idx=nodes.astype(np.int64), stimulation_mode=stimulation_mode,
value_kind="poisson_rate_hz", value=float(rate), start_ms=0.0, duration_ms=float(duration),
source_id=source, channel_id=layer, amplitude_mv=float(amplitude),
metadata={"visual_layer": layer, "engineering_interface": stimulation_mode == "external_synaptic_drive",
"connectome_modified": False, "external_drive_is_connectome_transmission": False},
)
events = (event,)
return InputBatch(events=events, batch_id=f"{source}:{layer}", metadata={"visual_layer": layer})
def peak_step_fraction(runtime: BrainRuntime) -> float:
events = runtime.read_spikes()
if not len(events["step"]):
return 0.0
return float(np.bincount(events["step"].astype(np.int64)).max(initial=0) / runtime.n)
def run_trial(runtime: BrainRuntime, batch: InputBatch, targets: np.ndarray, duration: float, seed: int,
distance: np.ndarray, cfg: dict[str, Any], capture: bool = False) -> tuple[dict[str, Any], dict[str, np.ndarray] | None]:
runtime.reset(seed)
chunk = min(float(cfg["calibration"]["chunk_ms"]), duration)
elapsed, wall, aborted = 0.0, 0.0, False
while elapsed < duration - 1e-9:
step_ms = min(chunk, duration - elapsed)
t0 = time.perf_counter(); runtime.step(batch, step_ms); wall += time.perf_counter() - t0
active = int(torch.count_nonzero(runtime.spike_counts).item())
total = int(runtime.spike_counts.sum().item())
actual_s = runtime.simulation_time_ms / 1000.0
runaway = cfg["calibration"]["runaway"]
if (active / runtime.n >= float(runaway["active_fraction_gte"]) or
total / (runtime.n * actual_s) >= float(runaway["mean_firing_rate_hz_gte"]) or
peak_step_fraction(runtime) >= float(runaway["peak_step_fraction_gte"])):
aborted = True
break
elapsed += step_ms
events = runtime.read_spikes(); nodes = events["neuron_idx"]
unique = np.unique(nodes)
target_mask = np.zeros(runtime.n, bool); target_mask[targets] = True
downstream = unique[~target_mask[unique]] if len(unique) else unique
d = runtime.read_population("descending", runtime.simulation_time_ms)
m = runtime.read_population("motor_related", runtime.simulation_time_ms)
valid_depth = distance[unique] if len(unique) else np.empty(0, np.int16)
valid_depth = valid_depth[valid_depth >= 0]
descending_mask = np.isin(nodes, runtime.output_populations["descending"])
motor_mask = np.isin(nodes, runtime.output_populations["motor_related"])
out_mask = descending_mask | motor_mask
out_latency = float(events["timestamp_ms"][out_mask].min()) if np.any(out_mask) else None
total = int(len(nodes)); actual_s = runtime.simulation_time_ms / 1000.0
record = {
"seed": seed, "requested_biological_ms": duration, "simulated_biological_ms": runtime.simulation_time_ms,
"wall_seconds": wall, "aborted_runaway": aborted, "total_spikes": total,
"active_neurons": int(len(unique)), "active_fraction": float(len(unique) / runtime.n),
"mean_firing_rate_hz": float(total / (runtime.n * actual_s)), "peak_step_fraction": peak_step_fraction(runtime),
"target_count": int(len(targets)), "target_active": int(np.count_nonzero(target_mask[unique])) if len(unique) else 0,
"non_target_active": int(len(downstream)), "propagation_depth_from_full_layer": int(valid_depth.max(initial=0)),
"descending_active": d["active_neurons"], "descending_spikes": d["total_spikes"],
"motor_active": m["active_neurons"], "motor_spikes": m["total_spikes"],
"output_first_spike_latency_ms": out_latency,
"descending_first_spike_latency_ms": float(events["timestamp_ms"][descending_mask].min()) if np.any(descending_mask) else None,
"motor_first_spike_latency_ms": float(events["timestamp_ms"][motor_mask].min()) if np.any(motor_mask) else None,
"external_drive_events": int(sum(len(x) for x in runtime.drive_nodes)),
"numerically_finite": bool(torch.isfinite(runtime.v).all().item() and torch.isfinite(runtime.g).all().item()),
}
success = (record["descending_active"] + record["motor_active"] > 0 and record["non_target_active"] > 0 and
record["propagation_depth_from_full_layer"] >= int(cfg["calibration"]["propagation_success"]["minimum_graph_hops"]))
record["multi_hop_output_propagation"] = bool(success)
captured = None
if capture:
captured = {
"step": events["step"].copy(), "neuron_idx": nodes.copy(),
"counts": runtime.spike_counts.cpu().numpy().copy(),
"v": runtime.v.cpu().numpy().copy(), "g": runtime.g.cpu().numpy().copy(),
"feature": np.concatenate([d["spike_counts"], m["spike_counts"]]).astype(np.float32),
}
return record, captured
def save_trial(exp: Path, category: str, key: str, record: dict[str, Any]) -> None:
write_json(exp / "calibration" / category / f"{key}.json", record)
def calibration_sweeps(cfg: dict[str, Any], runtime: BrainRuntime, populations: dict[str, np.ndarray],
distances: dict[str, np.ndarray], exp: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
cal = cfg["calibration"]; base = float(cfg["model"]["w_syn_mv_per_anatomical_synapse"])
layers = cfg["candidate_selection"]["hierarchy_order"]
records: list[dict[str, Any]] = []
amplitude_choice: dict[str, float | None] = {}
comparison = []
for i, layer in enumerate(populations):
nodes = subset(populations[layer], float(cal["fixed_sparsity_percent"]), stable_seed(int(cal["seed"]), layer))
mode = cfg["visual_populations"][layer]["stimulation_mode"]
amplitude = 0.0 if mode == "spike_injection" else 8.0
batch = make_batch(layer, nodes, float(cal["fixed_rate_hz"]), amplitude, float(cal["trial_duration_ms"]), "layer-comparison", mode)
rec, _ = run_trial(runtime, batch, nodes, float(cal["trial_duration_ms"]), int(cal["seed"]) + i, distances[layer], cfg)
rec.update({"sweep": "five-layer-comparison", "layer": layer, "rate_hz": cal["fixed_rate_hz"],
"sparsity_percent": cal["fixed_sparsity_percent"], "amplitude_mv": amplitude,
"amplitude_multiple_w_syn": amplitude / base if amplitude else None})
comparison.append(rec); records.append(rec); save_trial(exp, "five-layer-comparison", layer, rec)
for li, layer in enumerate(layers):
seed0 = stable_seed(int(cal["seed"]), layer)
nodes = subset(populations[layer], float(cal["fixed_sparsity_percent"]), seed0)
layer_records = []
multipliers = list(map(float, cal["amplitude_multiples"]))
found = False
for multiplier in multipliers:
amp = base * multiplier
batch = make_batch(layer, nodes, float(cal["fixed_rate_hz"]), amp, float(cal["trial_duration_ms"]), "amplitude-sweep")
rec, _ = run_trial(runtime, batch, nodes, float(cal["trial_duration_ms"]), seed0 + int(multiplier * 10), distances[layer], cfg)
rec.update({"sweep": "amplitude", "layer": layer, "rate_hz": cal["fixed_rate_hz"],
"sparsity_percent": cal["fixed_sparsity_percent"], "amplitude_mv": amp,
"amplitude_multiple_w_syn": multiplier})
layer_records.append(rec); records.append(rec); save_trial(exp, "amplitude", f"{layer}-{multiplier:g}x", rec)
found |= rec["multi_hop_output_propagation"] and not rec["aborted_runaway"]
if not found:
for multiplier in map(float, cal["extended_amplitude_multiples"]):
amp = base * multiplier
batch = make_batch(layer, nodes, float(cal["fixed_rate_hz"]), amp, float(cal["trial_duration_ms"]), "amplitude-sweep-extended")
rec, _ = run_trial(runtime, batch, nodes, float(cal["trial_duration_ms"]), seed0 + int(multiplier * 10), distances[layer], cfg)
rec.update({"sweep": "amplitude-extended", "layer": layer, "rate_hz": cal["fixed_rate_hz"],
"sparsity_percent": cal["fixed_sparsity_percent"], "amplitude_mv": amp,
"amplitude_multiple_w_syn": multiplier})
layer_records.append(rec); records.append(rec); save_trial(exp, "amplitude", f"{layer}-{multiplier:g}x", rec)
if rec["multi_hop_output_propagation"] and not rec["aborted_runaway"]:
break
viable = [x for x in layer_records if x["multi_hop_output_propagation"] and not x["aborted_runaway"] and x["numerically_finite"]]
amplitude_choice[layer] = min((x["amplitude_multiple_w_syn"] for x in viable), default=None)
for layer in layers:
multiplier = amplitude_choice[layer]
if multiplier is None:
continue
seed0 = stable_seed(int(cal["seed"]), layer)
fixed_nodes = subset(populations[layer], float(cal["fixed_sparsity_percent"]), seed0)
for rate in map(float, cal["rate_hz"]):
batch = make_batch(layer, fixed_nodes, rate, base * multiplier, float(cal["trial_duration_ms"]), "rate-sweep")
rec, _ = run_trial(runtime, batch, fixed_nodes, float(cal["trial_duration_ms"]), seed0 + int(rate), distances[layer], cfg)
rec.update({"sweep": "rate", "layer": layer, "rate_hz": rate, "sparsity_percent": cal["fixed_sparsity_percent"],
"amplitude_mv": base * multiplier, "amplitude_multiple_w_syn": multiplier})
records.append(rec); save_trial(exp, "rate", f"{layer}-{rate:g}hz", rec)
for sp in map(float, cal["sparsity_percent"]):
nodes = subset(populations[layer], sp, seed0)
batch = make_batch(layer, nodes, float(cal["fixed_rate_hz"]), base * multiplier, float(cal["trial_duration_ms"]), "sparsity-sweep")
rec, _ = run_trial(runtime, batch, nodes, float(cal["trial_duration_ms"]), seed0 + int(sp * 100), distances[layer], cfg)
rec.update({"sweep": "sparsity", "layer": layer, "rate_hz": cal["fixed_rate_hz"], "sparsity_percent": sp,
"amplitude_mv": base * multiplier, "amplitude_multiple_w_syn": multiplier})
records.append(rec); save_trial(exp, "sparsity", f"{layer}-{sp:g}pct", rec)
viable_by_layer: dict[str, dict[str, Any]] = {}
for layer in layers:
candidates = [x for x in records if x.get("layer") == layer and x["multi_hop_output_propagation"] and not x["aborted_runaway"]]
if candidates:
candidates.sort(key=lambda x: (float(x.get("amplitude_multiple_w_syn") or 1e9), float(x.get("rate_hz", 1e9)), float(x.get("sparsity_percent", 1e9))))
viable_by_layer[layer] = candidates[0]
for layer, chosen in viable_by_layer.items():
seed0 = stable_seed(int(cal["seed"]), layer)
for window in map(float, cal["latency_windows_ms"]):
nodes = subset(populations[layer], float(chosen["sparsity_percent"]), seed0)
batch = make_batch(layer, nodes, float(chosen["rate_hz"]), float(chosen["amplitude_mv"]), window, "latency-sweep")
rec, _ = run_trial(runtime, batch, nodes, window, seed0 + int(window * 10), distances[layer], cfg)
rec.update({"sweep": "latency", "layer": layer, "rate_hz": chosen["rate_hz"],
"sparsity_percent": chosen["sparsity_percent"], "amplitude_mv": chosen["amplitude_mv"],
"amplitude_multiple_w_syn": chosen["amplitude_multiple_w_syn"], "window_ms": window})
records.append(rec); save_trial(exp, "latency", f"{layer}-{window:g}ms", rec)
table = pa.Table.from_pylist(records)
pq.write_table(table, exp / "calibration/all-trials.parquet", compression="zstd")
summary = {"base_w_syn_mv": base, "phase4d_external_drive_mv": 8.0, "phase4d_external_drive_multiple_w_syn": 8.0 / base,
"amplitude_choice_by_layer": amplitude_choice, "viable_condition_by_layer": viable_by_layer,
"trial_count": len(records), "five_layer_comparison": comparison}
write_json(exp / "calibration/summary.json", summary)
return summary, records
def pair_metric(a: np.ndarray, b: np.ndarray) -> dict[str, float]:
an, bn = float(np.linalg.norm(a)), float(np.linalg.norm(b))
cosine = float(np.dot(a, b) / (an * bn)) if an and bn else (1.0 if an == bn else 0.0)
active_a, active_b = a > 0, b > 0; union = int(np.count_nonzero(active_a | active_b))
return {"euclidean": float(np.linalg.norm(a - b)), "cosine_distance": 1.0 - cosine,
"active_jaccard": float(np.count_nonzero(active_a & active_b) / union) if union else 1.0}
def pattern_batch(layer: str, populations: dict[str, np.ndarray], kind: str, rate: float, amp: float,
duration: float, seed: int, cfg: dict[str, Any], sparse_percent: float | None = None) -> tuple[InputBatch, np.ndarray]:
all_nodes = populations[layer]
sparse_pct = float(cfg["patterns"]["sparse_percent"] if sparse_percent is None else sparse_percent)
count = max(1, int(round(len(all_nodes) * sparse_pct / 100)))
order = np.random.default_rng(seed).permutation(len(all_nodes)); sparse_a = np.sort(all_nodes[order[:count]].astype(np.int64))
sparse_b = np.sort(all_nodes[order[count:2 * count]].astype(np.int64))
if kind == "sparse-A": nodes = sparse_a
elif kind == "sparse-B": nodes = sparse_b
elif kind == "dense": nodes = subset(all_nodes, float(cfg["patterns"]["dense_percent"]), seed)
elif kind == "zero": nodes = np.empty(0, np.int64)
elif kind == "moving":
segs = int(cfg["patterns"]["moving_segments"]); segment_ms = duration / segs; events = []
union = []
for i in range(segs):
nodes_i = np.sort(all_nodes[order[(2 + i) * count:(3 + i) * count]].astype(np.int64)); union.append(nodes_i)
events.append(InputEvent(target_neuron_idx=nodes_i, stimulation_mode="external_synaptic_drive", value_kind="poisson_rate_hz",
value=rate, start_ms=i * segment_ms, duration_ms=segment_ms, source_id="pattern", channel_id=f"moving-{i}",
amplitude_mv=amp, metadata={"pattern": "moving", "segment": i}))
nodes = np.unique(np.concatenate(union)); return make_batch(layer, nodes, rate, amp, duration, "pattern", events=tuple(events)), nodes
else: raise KeyError(kind)
return make_batch(layer, nodes, rate, amp, duration, "pattern"), nodes
def separability_and_determinism(cfg: dict[str, Any], runtime: BrainRuntime, populations: dict[str, np.ndarray],
distances: dict[str, np.ndarray], viable: dict[str, Any], exp: Path) -> dict[str, Any]:
result: dict[str, Any] = {}
duration = float(cfg["patterns"]["trial_duration_ms"])
for layer, condition in viable.items():
features: dict[str, list[np.ndarray]] = {k: [] for k in ("sparse-A", "sparse-B", "dense", "moving")}
trials, drive_counts = [], {}
for kind in features:
for seed in map(int, cfg["patterns"]["seeds"]):
batch, nodes = pattern_batch(layer, populations, kind, float(condition["rate_hz"]), float(condition["amplitude_mv"]), duration, seed, cfg,
float(condition.get("sparsity_percent", cfg["patterns"]["sparse_percent"])))
rec, cap = run_trial(runtime, batch, nodes, duration, seed, distances[layer], cfg, capture=True)
assert cap is not None; features[kind].append(cap["feature"]); trials.append({"pattern": kind, **rec})
drive_counts[f"{kind}-{seed}"] = rec["external_drive_events"]
means = {k: np.mean(v, axis=0) for k, v in features.items()}
pairs = {}; names = sorted(means)
for i, a in enumerate(names):
for b in names[i + 1:]: pairs[f"{a}__{b}"] = pair_metric(means[a], means[b])
det_seed = int(cfg["patterns"]["seeds"][0])
batch, nodes = pattern_batch(layer, populations, "sparse-A", float(condition["rate_hz"]), float(condition["amplitude_mv"]), duration, det_seed, cfg,
float(condition.get("sparsity_percent", cfg["patterns"]["sparse_percent"])))
_, c1 = run_trial(runtime, batch, nodes, duration, det_seed, distances[layer], cfg, capture=True)
_, c2 = run_trial(runtime, batch, nodes, duration, det_seed, distances[layer], cfg, capture=True)
assert c1 is not None and c2 is not None
deterministic = bool(np.array_equal(c1["step"], c2["step"]) and np.array_equal(c1["neuron_idx"], c2["neuron_idx"]) and np.array_equal(c1["v"], c2["v"]) and np.array_equal(c1["g"], c2["g"]))
matched = all(drive_counts[f"sparse-A-{s}"] == drive_counts[f"sparse-B-{s}"] for s in map(int, cfg["patterns"]["seeds"]))
separable = all(x["euclidean"] > 0 for x in pairs.values())
result[layer] = {"condition": condition, "pairwise_output_features": pairs, "trials": trials,
"matched_input_count_sparse_A_B": matched, "deterministic_exact": deterministic,
"all_mean_output_features_distinct": separable, "candidate_eligible": deterministic and separable and matched}
write_json(exp / "separability" / f"{layer}.json", result[layer])
write_json(exp / "separability/summary.json", result)
return result
def select_gateway(cfg: dict[str, Any], reach: dict[str, Any], sweep: dict[str, Any], sep: dict[str, Any]) -> dict[str, Any]:
eligible = []
for rank, layer in enumerate(cfg["candidate_selection"]["hierarchy_order"]):
condition = sweep["viable_condition_by_layer"].get(layer)
if condition:
hop_hist_d = reach["layers"][layer]["min_hops_descending_distribution"]
hop_hist_m = reach["layers"][layer]["min_hops_motor_distribution"]
positive = [int(k) for k in list(hop_hist_d) + list(hop_hist_m) if int(k) >= 0]
min_hops = min(positive) if positive else 32767
eligible.append((float(condition["amplitude_multiple_w_syn"]), min_hops, rank, layer, condition))
eligible.sort(key=lambda x: x[:3])
if not eligible:
return {"selected": False, "reason": "no condition produced multi-hop, non-runaway output propagation", "rule": cfg["candidate_selection"]["rule"]}
_, hops, _, layer, condition = eligible[0]
return {"selected": True, "name": "visual_gateway_candidate", "layer": layer, "condition": condition,
"minimum_output_hops": hops, "rule": cfg["candidate_selection"]["rule"], "handpicked": False,
"adapter_v2_eligible": bool(sep.get(layer, {}).get("candidate_eligible")),
"adapter_v2_gate_note": "gateway selection uses anatomy/path/propagation; formal adapter eligibility additionally requires deterministic separability"}
def continuous_test(cfg: dict[str, Any], runtime: BrainRuntime, populations: dict[str, np.ndarray],
distances: dict[str, np.ndarray], gateway: dict[str, Any], exp: Path) -> dict[str, Any]:
if not gateway["selected"]:
result = {"executed": False, "reason": gateway["reason"]}; write_json(exp / "continuous/summary.json", result); return result
layer, cond = gateway["layer"], gateway["condition"]; result: dict[str, Any] = {}
duration = float(cfg["continuous"]["biological_duration_ms"])
for period in map(float, cfg["continuous"]["periods_ms"]):
runtime.reset(int(cfg["continuous"]["seed"])); intervals = []; previous = None
sequence = cfg["continuous"]["sequence"]
for i, start in enumerate(np.arange(0.0, duration, period)):
kind = sequence[i % len(sequence)]
batch0, nodes = pattern_batch(layer, populations, kind, float(cond["rate_hz"]), float(cond["amplitude_mv"]), period,
int(cfg["continuous"]["seed"]), cfg,
float(cond.get("sparsity_percent", cfg["patterns"]["sparse_percent"])))
shifted = tuple(InputEvent(target_neuron_idx=e.target_neuron_idx, stimulation_mode=e.stimulation_mode, value_kind=e.value_kind,
value=e.value, start_ms=e.start_ms + float(start), duration_ms=e.duration_ms, source_id="continuous",
channel_id=e.channel_id, amplitude_mv=e.amplitude_mv, metadata=e.metadata) for e in batch0.events)
batch = InputBatch(events=shifted, batch_id=f"continuous:{i}")
t0 = time.perf_counter(); step = runtime.step(batch, period); wall = time.perf_counter() - t0
d = runtime.read_population("descending", period); m = runtime.read_population("motor_related", period)
feature = np.concatenate([d["spike_counts"], m["spike_counts"]]).astype(np.float32)
recent = runtime.read_spikes(period); out = np.isin(recent["neuron_idx"], np.concatenate([d["neuron_idx"], m["neuron_idx"]]))
latency = float(recent["timestamp_ms"][out].min() - start) if np.any(out) else None
intervals.append({"index": i, "start_ms": start, "pattern": kind, "target_count": len(nodes), "wall_seconds": wall,
"drive_events": step["new_external_drive_events"], "descending_spikes": d["total_spikes"],
"motor_spikes": m["total_spikes"], "output_latency_ms": latency,
"output_feature_change_l2": None if previous is None else float(np.linalg.norm(feature - previous))})
previous = feature
zero = [x for x in intervals if x["pattern"] == "zero"]
result[str(int(period))] = {"period_ms": period, "interval_count": len(intervals), "continuous_state_without_reset": True,
"wall_seconds": float(sum(x["wall_seconds"] for x in intervals)),
"zero_period_output_spikes": int(sum(x["descending_spikes"] + x["motor_spikes"] for x in zero)),
"intervals": intervals}
write_json(exp / "continuous" / f"period-{int(period)}ms.json", result[str(int(period))])
write_json(exp / "continuous/summary.json", result)
return result
def regression_suite(cfg: dict[str, Any], runtime: BrainRuntime, exp: Path) -> dict[str, Any]:
io_root = Path(cfg["paths"]["io_root"]); sys.path.insert(0, str(io_root))
import phase4d as p4d
reg = p4d.regression(exp, cfg, runtime)
api = p4d.api_tests(exp, runtime)
stream = p4d.stream_contract(exp, runtime)
nodes = np.flatnonzero(runtime.population_masks["auditory"]).astype(np.int64)
batch = p4d.direct_batch(nodes, "auditory", 10, 0.0, 20.0, "phase4e-continuous-contract")
runtime.reset(45999); runtime.step(batch, 10.0); checkpoint = runtime.checkpoint_state(); runtime.step(batch, 10.0)
first = runtime.read_spikes(); first_v = runtime.v.cpu().numpy().copy(); runtime.restore_state(checkpoint); runtime.step(batch, 10.0)
replay = runtime.read_spikes(); replay_v = runtime.v.cpu().numpy().copy()
continuous = {"checkpoint_replay_events_exact": bool(np.array_equal(first["step"], replay["step"]) and np.array_equal(first["neuron_idx"], replay["neuron_idx"])),
"checkpoint_replay_voltage_exact": bool(np.array_equal(first_v, replay_v)), "clock_ms": runtime.simulation_time_ms}
continuous["passed"] = continuous["checkpoint_replay_events_exact"] and continuous["checkpoint_replay_voltage_exact"]
write_json(exp / "audit/phase4d-continuous-contract.json", continuous)
result = {"phase4c_five_trials_passed": all(x["passed"] for x in reg.values()), "phase4d_api_passed": api["all_passed"],
"phase4d_continuous_checkpoint_passed": continuous["passed"], "activity_replay_passed": stream["replay_exact"],
"trials": reg, "api": api, "continuous": continuous, "activity_stream": stream}
result["all_passed"] = all(result[k] for k in ("phase4c_five_trials_passed", "phase4d_api_passed", "phase4d_continuous_checkpoint_passed", "activity_replay_passed"))
write_json(exp / "audit/regression-summary.json", result)
return result
def freeze_candidate(cfg: dict[str, Any], gateway: dict[str, Any], sep: dict[str, Any], continuous: dict[str, Any],
regression: dict[str, Any], code: Path, config: Path, exp: Path) -> dict[str, Any]:
root = Path(cfg["paths"]["candidate_root"]); root.mkdir(parents=True, exist_ok=True)
for src in (code, config):
dst = root / src.name
if src.resolve() != dst.resolve():
shutil.copy2(src, dst)
passed = bool(gateway["selected"] and gateway.get("adapter_v2_eligible") and regression["all_passed"] and continuous.get("executed", True))
version = {"name": cfg["names"]["candidate"], "status": "CANDIDATE-PASS" if passed else "CANDIDATE-REJECTED",
"formal_v2_frozen": False, "gateway": gateway, "connectome_changed": False, "lif_semantics_changed": False,
"transmitter_sign_changed": False, "phase4f_started": False}
write_json(root / "VERSION.json", version)
files = sorted(p for p in root.iterdir() if p.is_file() and p.name not in {"MANIFEST.json", "checksums.sha256"})
write_json(root / "MANIFEST.json", {"artifact": cfg["names"]["candidate"], "files": [{"path": p.name, "bytes": p.stat().st_size, "sha256": sha256(p)} for p in files]})
targets = files + [root / "MANIFEST.json"]
(root / "checksums.sha256").write_text("".join(f"{sha256(p)} {p.name}\n" for p in targets), encoding="utf-8")
return version
def materialize_requested_layout(exp: Path) -> None:
"""Expose the requested top-level sweep/gateway layout as real files."""
aliases = {
"calibration/amplitude": "amplitude-sweep",
"calibration/rate": "rate-sweep",
"calibration/sparsity": "sparsity-sweep",
"calibration/latency": "latency",
}
for source, destination in aliases.items():
dst = exp / destination; dst.mkdir(parents=True, exist_ok=True)
for path in (exp / source).glob("*.json"):
shutil.copy2(path, dst / path.name)
gateway_dir = exp / "gateway"; gateway_dir.mkdir(parents=True, exist_ok=True)
shutil.copy2(exp / "audit/visual-gateway-candidate.json", gateway_dir / "visual-gateway-candidate.json")
def seal(cfg: dict[str, Any], exp: Path, code: Path, config: Path, controls: list[Path], summary: dict[str, Any]) -> None:
excluded = {exp / "manifests/manifest.json", exp / "manifests/checksums.sha256"}
outputs = sorted(p for p in exp.rglob("*") if p.is_file() and p not in excluded)
manifest = {"phase": "4E", "status": summary["status"], "created_unix_time": time.time(),
"software": {"python": platform.python_version(), "torch": torch.__version__, "numpy": np.__version__, "scipy": __import__("scipy").__version__, "pyarrow": pa.__version__},
"code": [{"path": str(x), "sha256": sha256(x)} for x in (code, config)],
"immutable_inputs": [{"path": str(x), "sha256": sha256(x)} for x in controls],
"outputs": [{"path": str(x), "bytes": x.stat().st_size, "sha256": sha256(x)} for x in outputs],
"prohibitions": cfg["prohibitions"]}
write_json(exp / "manifests/manifest.json", manifest)
targets = outputs + [exp / "manifests/manifest.json"]
(exp / "manifests/checksums.sha256").write_text("".join(f"{sha256(x)} {x.relative_to(exp)}\n" for x in targets), encoding="utf-8")
def main() -> None:
ap = argparse.ArgumentParser(); ap.add_argument("--config", type=Path, required=True); args = ap.parse_args()
cfg = yaml.safe_load(args.config.read_text(encoding="utf-8")); exp = Path(cfg["paths"]["experiment_root"])
for sub in ("configs", "populations", "reachability", "calibration", "amplitude-sweep", "rate-sweep", "sparsity-sweep", "latency", "gateway", "separability", "continuous", "regression", "activity-stream", "audit", "manifests", "compiler-reports/extension-build"):
(exp / sub).mkdir(parents=True, exist_ok=True)
shutil.copy2(args.config, exp / "configs/phase4e.yaml")
controls, before, immutable = immutable_audit(cfg); write_json(exp / "audit/immutable-inputs-before.json", immutable)
# Frozen runtime-v1 imports the shared, unchanged Phase 4B graph helpers
# from the model root.
sys.path.insert(0, cfg["paths"]["model_root"])
runtime = BrainRuntime(args.config, exp / "compiler-reports/extension-build")
populations, population_summary = load_populations(cfg, runtime, exp)
semantics = {"phase4d_external_drive_application": "g[target] += amplitude_mv per selected external Poisson event, after g decay/connectome arrival and before membrane integration",
"units": "mV-equivalent conductance state increment", "phase4d_amplitude_mv": 8.0,
"w_syn_mv_per_anatomical_synapse": cfg["model"]["w_syn_mv_per_anatomical_synapse"],
"ratio_phase4d_drive_to_one_anatomical_synapse": 8.0 / float(cfg["model"]["w_syn_mv_per_anatomical_synapse"]),
"connectome_transmission": False, "raw_csr_weight_modified": False}
write_json(exp / "audit/external-drive-semantics.json", semantics)
reach = graph_reachability(cfg, runtime, populations, exp)
distances = forward_distances(runtime, populations, exp)
sweep, records = calibration_sweeps(cfg, runtime, populations, distances, exp)
viable = sweep["viable_condition_by_layer"]
sep = separability_and_determinism(cfg, runtime, populations, distances, viable, exp)
gateway = select_gateway(cfg, reach, sweep, sep); write_json(exp / "audit/visual-gateway-candidate.json", gateway)
continuous = continuous_test(cfg, runtime, populations, distances, gateway, exp)
# Run frozen regressions in a fresh runtime, isolated from the long visual
# stress sweeps and their allocator history.
del runtime
gc.collect(); torch.cuda.empty_cache()
regression_runtime = BrainRuntime(args.config, exp / "compiler-reports/extension-build")
regression = regression_suite(cfg, regression_runtime, exp)
del regression_runtime
gc.collect(); torch.cuda.empty_cache()
materialize_requested_layout(exp)
after = {str(x): sha256(x) for x in controls}; unchanged = before == after
candidate = freeze_candidate(cfg, gateway, sep, continuous, regression, Path(__file__).resolve(), args.config.resolve(), exp)
stable = all(not x["aborted_runaway"] and x["numerically_finite"] for x in records if x.get("multi_hop_output_propagation"))
invariant_pass = regression["all_passed"] and unchanged and stable
status = ("PASS" if invariant_pass and gateway.get("adapter_v2_eligible") else
"FAIL-NO-V2-CANDIDATE" if invariant_pass else "FAILED")
summary = {"status": status, "reference_phase4d_unchanged": unchanged, "immutable_checks": immutable,
"visual_population_summary": population_summary, "external_drive_semantics": semantics,
"reachability": reach, "calibration": sweep, "gateway": gateway, "separability": sep,
"continuous": continuous, "regression": regression, "candidate_artifact": candidate,
"propagating_trials_non_runaway_and_finite": stable, "phase4f_started": False,
"game_rl_web_3d_started": False}
write_json(exp / "audit/summary.json", summary)
seal(cfg, exp, Path(__file__).resolve(), args.config.resolve(), controls, summary)
if status == "FAILED":
raise RuntimeError("Phase 4E invariant/regression failure")
if __name__ == "__main__":
main()
models/male-cns-lif/visual-interface-v2/phase4e.yaml
+115
View File
@@ -0,0 +1,115 @@
phase: 4E
names:
reference: MaleCNS-LIF-reference-v1
runtime: MaleCNS-LIF-runtime-v1
io_api: MaleCNS-BrainIO-v1
candidate: MaleCNS-VisualAdapter-v2-candidate
paths:
connectome_root: /data/digital-drosophila/datasets/male-cns-v1.0/processed/neuron-only
model_root: /data/digital-drosophila/models/male-cns-lif
reference_root: /data/digital-drosophila/models/male-cns-lif/reference-v1
runtime_root: /data/digital-drosophila/models/male-cns-lif/runtime-v1
io_root: /data/digital-drosophila/models/male-cns-lif/io-v1
phase4c_root: /data/digital-drosophila/experiments/phase4c-runtime
phase4d_root: /data/digital-drosophila/experiments/phase4d-io
experiment_root: /data/digital-drosophila/experiments/phase4e-visual-calibration
candidate_root: /data/digital-drosophila/models/male-cns-lif/visual-interface-v2
device:
required_name_substring: RTX 4090
launcher_visible_gpu_uuid: GPU-79f3a85d-adce-7833-7003-6488140ca596
model:
dt_ms: 0.1
membrane_tau_ms: 20.0
synaptic_tau_ms: 5.0
refractory_ms: 2.2
resting_potential_mv: -52.0
reset_potential_mv: -52.0
threshold_mv: -45.0
synaptic_delay_ms: 1.8
w_syn_mv_per_anatomical_synapse: 0.275
sign_policy:
neurotransmitter_sign:
acetylcholine: 1
gaba: -1
glutamate: -1
histamine: -1
dopamine: 1
octopamine: 1
serotonin: 1
unknown_sign: 0
unclear_sign: 0
visual_populations:
reference-photoreceptor: {source: hierarchy, value: visual_sensory, interface_kind: biological-reference, stimulation_mode: spike_injection}
engineering-first-order: {source: hierarchy, value: first_order, interface_kind: engineering, stimulation_mode: external_synaptic_drive}
engineering-second-order: {source: hierarchy, value: second_order, interface_kind: engineering, stimulation_mode: external_synaptic_drive}
engineering-higher-visual: {source: hierarchy, value: higher_visual, interface_kind: engineering, stimulation_mode: external_synaptic_drive}
engineering-visual-projection: {source: metadata-expression, field: annotation_superclass, values: [visual_projection, visual_projection_tbc], interface_kind: engineering, stimulation_mode: external_synaptic_drive}
calibration:
seed: 45000
trial_duration_ms: 100.0
chunk_ms: 10.0
fixed_rate_hz: 50.0
fixed_sparsity_percent: 5.0
amplitude_multiples: [0.25, 0.5, 1, 2, 4, 8, 16]
extended_amplitude_multiples: [32, 64]
rate_hz: [10, 25, 50, 100, 150, 200]
sparsity_percent: [0.1, 0.5, 1, 5, 10, 25, 50, 100]
latency_windows_ms: [5, 10, 20, 30, 50, 100, 200]
deterministic_repeats: 2
runaway:
active_fraction_gte: 0.80
mean_firing_rate_hz_gte: 100.0
peak_step_fraction_gte: 0.10
propagation_success:
require_output_active: true
require_non_target_active: true
minimum_graph_hops: 2
patterns:
# Separability uses the selected propagating condition's sparsity. This is
# retained only as a fallback if that condition has no sparsity field.
sparse_percent: 5.0
dense_percent: 25.0
moving_segments: 5
trial_duration_ms: 100.0
seeds: [45101, 45102, 45103]
continuous:
biological_duration_ms: 1000.0
periods_ms: [10, 20, 50, 100]
sequence: [sparse-A, sparse-B, sparse-A, moving, zero, dense]
seed: 45200
readout:
bins_ms: [1, 5, 10, 20, 50]
regression:
float_absolute_tolerance: 0.00390625
reference_trials:
- {name: auditory-10hz, population: auditory, rate_hz: 10, seed: 42001, duration_ms: 1000.0}
- {name: auditory-100hz, population: auditory, rate_hz: 100, seed: 42001, duration_ms: 1000.0}
- {name: olfactory-50hz, population: olfactory, rate_hz: 50, seed: 42003, duration_ms: 1000.0}
- {name: mechanosensory-50hz, population: mechanosensory, rate_hz: 50, seed: 42003, duration_ms: 1000.0}
- {name: gustatory-50hz, population: gustatory, rate_hz: 50, seed: 42003, duration_ms: 1000.0}
candidate_selection:
rule: "gateway iff multi-hop propagation reaches descending or motor output and is non-runaway; rank by lowest amplitude multiple, then lowest min output hops, then hierarchy order. Formal adapter-v2 eligibility additionally requires deterministic nonzero output separability and matched-input-count validation"
hierarchy_order: [engineering-first-order, engineering-second-order, engineering-higher-visual, engineering-visual-projection]
prohibitions:
connectome_changes: false
raw_weight_changes: false
lif_semantics_changes: false
transmitter_sign_changes: false
training: false
reinforcement_learning: false
game_integration: false
web_server: false
three_d_frontend: false
phase4f: false