"""Phase 4AF: a streaming float64 oracle for three fixed short full-graph trials. Importing this module does not import Brian. queue_summary is shared with the GPU comparison code. All scientific inputs and the sealed 4AD/4AE code remain read-only. Every run_case call belongs in a fresh process. """ from __future__ import annotations import csv import hashlib import importlib.metadata import json import os import platform import struct import sys import types from contextlib import ExitStack from pathlib import Path import numpy as np N_NEURONS = 166700 N_EDGES = 25582938 BLOCK_BYTES = 8 << 20 QUEUE_HEADER = b"phase4af-queue-v1\0" AUTHOR_PATH = Path("/data/digital-drosophila/experiments/phase4ad-author-semantics/external-resources/model.py") AUTHOR_SHA256 = "fc45837d7122c6ce2a7f3f2f23c515992e4b232aadb919efabb72337fac88e4e" AE_ORACLE_PATH = Path("/data/digital-drosophila/models/male-cns-lif/author-runtime-lab-v1/oracle_stress.py") AE_ORACLE_SHA256 = "32eb1c42383068ce52701d2139a376ce155ad7072cde019949f8ca2d55492409" EVENT_DTYPE = np.dtype([("step", " np.iinfo(np.int64).max): raise ValueError("Queue edge ID cannot be encoded as signed int64") chunk = np.ascontiguousarray(original, dtype=" limit: raise RuntimeError(f"New-output budget would be exceeded: {actual} + {reserve} > {limit} bytes; inputs were not adjusted") return actual def _new_memmap(path, dtype, shape): if Path(path).exists(): raise FileExistsError(path) return np.lib.format.open_memmap(path, mode="w+", dtype=np.dtype(dtype), shape=shape) def _load_checked_module(path, expected_hash, name): data = Path(path).read_bytes() actual = hashlib.sha256(data).hexdigest() _check(actual == expected_hash, f"Pinned source hash mismatch: {path}") module = types.ModuleType(name) module.__file__ = str(path) # Execute checked bytes without producing __pycache__ beside a sealed file. exec(compile(data, str(path), "exec"), module.__dict__) return module def _graph_inputs(graph_dir): definitions = { "nodes": (np.int32, (N_NEURONS,)), "edge_id": (np.int64, (N_EDGES,)), "src": (np.int32, (N_EDGES,)), "dst": (np.int32, (N_EDGES,)), "signed_raw": (np.int64, (N_EDGES,)), "ptr": (np.int64, (N_NEURONS + 1,)), } arrays = {} for name, (dtype, shape) in definitions.items(): value = np.load(graph_dir / (name + ".npy"), mmap_mode="r", allow_pickle=False) _check(value.dtype == np.dtype(dtype) and value.shape == shape, f"Incorrect canonical {name} shape/dtype") arrays[name] = value _check(np.array_equal(arrays["nodes"], np.arange(N_NEURONS, dtype=np.int32)), "This full-graph protocol requires canonical global node indices 0..N-1") ptr = arrays["ptr"] _check(ptr[0] == 0 and ptr[-1] == N_EDGES and np.all(np.diff(ptr) >= 0), "Invalid canonical CSR pointer") source_counts = np.zeros(N_NEURONS, dtype=np.int64) previous_source = -1 width = BLOCK_BYTES // 8 for begin in range(0, N_EDGES, width): stop = min(N_EDGES, begin + width) _check(np.array_equal(arrays["edge_id"][begin:stop], np.arange(begin, stop, dtype=np.int64)), "Full-graph edge identity/order is not the original canonical CSR order") source, target = arrays["src"][begin:stop], arrays["dst"][begin:stop] _check(source[0] >= previous_source and np.all(np.diff(source) >= 0), "Sources are not canonically sorted") _check(source[0] >= 0 and source[-1] < N_NEURONS and np.all((target >= 0) & (target < N_NEURONS)), "Canonical endpoint outside the full graph") previous_source = int(source[-1]) source_counts += np.bincount(source, minlength=N_NEURONS) _check(np.array_equal(source_counts, np.diff(ptr)), "src and CSR pointer disagree") for name in ("completeness.csv", "connectivity.parquet"): _check((graph_dir / name).is_file(), f"Missing author input file {name}") return arrays class _StreamingQueueProxy: """The 4AE forwarding contract with bounded hashing instead of retained lists.""" def __init__(self, original, get_tick, get_available, arrays, due_counts, spike_ring, delay_steps, queue_file, records): self._original_queue = original self._get_tick = get_tick self._get_available = get_available self._arrays = arrays self._due_counts = due_counts self._spike_ring = spike_ring self._delay_steps = delay_steps self._queue_file = queue_file self._records = records def peek(self, *args, **kwargs): original = self._original_queue.peek(*args, **kwargs) edge_view = np.asarray(original) tick = self._get_tick() _check(tick == len(self._records["rows"]), "Queue peek ticks must be unique and consecutive") _check(len(edge_view) == int(self._due_counts[tick % len(self._due_counts)]), "Queue arrival count differs from delayed outgoing degree of real spikes") available = np.asarray(self._get_available(), dtype=bool) fired_then = np.zeros(N_NEURONS, dtype=bool) if tick >= self._delay_steps: fired_then[self._spike_ring[(tick - self._delay_steps) % len(self._spike_ring)]] = True def accepted_chunk(begin, stop): local_edges = edge_view[begin:stop] _check(np.all((local_edges >= 0) & (local_edges < N_EDGES)), "Queue contains an invalid local edge index") _check(np.all(fired_then[self._arrays["src"][local_edges]]), "Queue arrival lacks a presynaptic spike at the fixed 18-step delay") return available[self._arrays["dst"][local_edges]] # edge_id[k] == k was independently checked for every canonical edge. # Thus the untouched local queue array is also the original edge-ID stream. summary = queue_summary(tick, edge_view, accepted_chunk) self._queue_file.write(json.dumps(summary, sort_keys=True) + "\n") self._records["rows"].append(summary) self._records["peek_calls"] += 1 return original def advance(self, *args, **kwargs): self._records["advance_calls"] += 1 return self._original_queue.advance(*args, **kwargs) def __getattr__(self, name): return getattr(self._original_queue, name) def run_case(case_dir: Path, out_dir: Path) -> dict: """Run exactly one preregistered float64 case in this process. Outputs: v/g/ref history .npy memmaps; actual weights.npy; sparse state.npz; queue.jsonl; events.bin with two little-endian int32 fields; request records; small per-tick statistics and audit JSON. A failed trial retains partial outputs and error.json and never writes summary.json with complete=True. """ global _PROCESS_ALREADY_USED _check(not _PROCESS_ALREADY_USED, "Each full-graph oracle case requires a new process") _PROCESS_ALREADY_USED = True case_dir = Path(case_dir).resolve(strict=True) out_dir = Path(out_dir).resolve() spec = json.loads((case_dir / "spec.json").read_text(encoding="utf-8")) graph_dir = Path(spec["graph_dir"]).resolve(strict=True) steps = int(spec["steps"]) seeds = tuple(int(node) for node in spec["seed_nodes"]) fixed = {"no_input": (100, ()), "single": (500, (1381,)), "five": (500, (1197, 1381, 1511, 82312, 133666))} _check(spec["name"] in fixed and (steps, seeds) == fixed[spec["name"]], "Case is not one of the three fixed trials") _check(float(spec["dt_ms"]) == .1 and np.isclose(float(spec["duration_ms"]), steps * .1, rtol=0, atol=1e-12), "Case clock/duration differs from the fixed protocol") _check(spec.get("precision", "float64") == "float64", "Full-graph oracle is float64 only") _check(spec.get("all_requests_at_tick", 0) == 0 and not spec.get("apl_overlay", False) and not spec.get("training", False), "This oracle does not permit altered input timing, APL, or training") budget_root = Path(spec.get("budget_root", os.path.commonpath([str(case_dir), str(graph_dir), str(out_dir)]))).resolve() budget_limit = int(spec.get("output_budget_bytes", 15 * 1024 ** 3)) _check(budget_limit > 0, "Output budget must be positive") _check(out_dir.is_relative_to(budget_root), "Oracle output must be inside its budget root") out_dir.mkdir(parents=True, exist_ok=False) memmaps = [] last_complete_tick = -1 saved_preferences = None old_dt = None try: _budget(budget_root, budget_limit, reserve=steps * N_NEURONS * 20 + N_EDGES * 8 + (64 << 20)) versions = {name: importlib.metadata.version(name) for name in ("brian2", "numpy", "pandas", "pyarrow", "sympy", "scipy", "cython")} _save(out_dir / "environment.json", {"python": sys.version, "platform": platform.platform(), "packages": versions, "precision": "float64", "codegen": "numpy"}) for name, required in {"brian2": "2.5.1", "numpy": "1.22.3", "sympy": "1.12"}.items(): _check(versions[name] == required, f"{name} must be pinned to {required}") import brian2 as b2 from brian2.core.base import BrianObject ae = _load_checked_module(AE_ORACLE_PATH, AE_ORACLE_SHA256, "phase4af_readonly_ae_helpers") author = _load_checked_module(AUTHOR_PATH, AUTHOR_SHA256, "phase4af_pinned_author") arrays = _graph_inputs(graph_dir) graph_hashes = {name: _digest(graph_dir / name) for name in ( "nodes.npy", "edge_id.npy", "src.npy", "dst.npy", "signed_raw.npy", "ptr.npy", "completeness.csv", "connectivity.parquet")} _save(out_dir / "input-audit.json", {"spec": spec, "spec_sha256": _digest(case_dir / "spec.json"), "graph_hashes": graph_hashes, "author_sha256": AUTHOR_SHA256, "sealed_ae_oracle_sha256": AE_ORACLE_SHA256, "budget_root": str(budget_root), "budget_limit_bytes": budget_limit, "source_order": "canonical CSR; local edge index equals original edge_id", "common_interface_preserved": True, "poi_used": False, "apl_overlay": False, "training": False}) saved_preferences = {key: b2.prefs[key] for key in ( "codegen.target", "core.default_float_dtype", "legacy.refractory_timing")} old_dt = b2.defaultclock.dt b2.start_scope() b2.prefs["codegen.target"] = "numpy" b2.prefs["core.default_float_dtype"] = np.float64 b2.prefs["legacy.refractory_timing"] = False b2.defaultclock.dt = .1 * b2.ms parameters = dict(author.default_params) _save(out_dir / "parameter-checks.json", ae._parameters_match(parameters, spec["model"], b2)) neu, syn, spikes = author.create_model(graph_dir / "completeness.csv", graph_dir / "connectivity.parquet", parameters) _check(len(neu) == N_NEURONS and len(syn) == N_EDGES, "Author model did not load the whole graph") _check(all(np.dtype(neu.variables[key].dtype) == np.float64 for key in ("v", "g")) and np.dtype(syn.variables["w"].dtype) == np.float64, "Author state/weight dtype is not float64") weights = _new_memmap(out_dir / "weights.npy", "= 0) last_ticks.fill(0) last_ticks[seen] = np.rint(last_raw[seen] / .0001).astype(np.int64) ref = np.where(seen, np.maximum(refractory_steps - (tick - last_ticks), 0), 0).astype(np.int32) _check(np.array_equal(ref == 0, available_raw), "Integer refractory history disagrees with Brian") v_history[tick] = v_raw g_history[tick] = g_raw ref_history[tick] = ref active = np.asarray(neu.spikes, dtype=np.int32) _check(not len(active) or np.all(np.diff(active) > 0), "Spike order is not canonical and unique") spike_count[tick] = len(active) total_counts[active] += 1 if len(active): rows = np.empty(len(active), dtype=EVENT_DTYPE) rows["step"] = tick rows["neuron_idx"] = arrays["nodes"][active] event_file.write(memoryview(rows).cast("B")) total_events += len(active) spike_ring[tick % queue_len] = active.copy() due_counts[(tick + delay_steps) % queue_len] = int(outdegree[active].sum(dtype=np.int64)) if tick == 0: _check(np.isin(permitted_seed, active).all(), "An applicable initial request failed to create a real spike") for row in request_rows: request_writer.writerow(row) queue_row = records["rows"][tick] tick_record = {"tick": tick, "spikes": len(active), "requests": int(request_count[tick]), "applicable_requests": int(applicable_count[tick]), "ref_rejected_requests": int(rejected_count[tick]), "due_arrivals": queue_row["count"], "accepted_arrivals": queue_row["accepted_count"], "event_sha256": hashlib.sha256(np.ascontiguousarray(arrays["nodes"][active], dtype="