Seal Phase4AF exact short fullgraph author GPU gate and APL integration audit

This commit is contained in:
Digital Drosophila Archive
2026-09-30 08:14:16 +08:00
parent 1eb4c52cf7
commit 7dfe5d2d87
104 changed files with 11608 additions and 20 deletions
@@ -0,0 +1,44 @@
# Phase 4AF: full-graph author numerical gate
The independent branch is authorized by the human; frozen models are not replaced.
Use the complete canonical 166,700-neuron/25,582,938-edge graph, every original
integer synaptic-contact count, the frozen source transmitter signs and W_syn
0.275 mV/contact. Do not remove isolates, self-loops or any existing edge.
Float64 is the unmodified author's default, not a precision chosen for activity.
The archived original create_model, Brian2 2.5.1/NumPy1.22.3 and its actual scalar
coefficients define the oracle. Use unchanged Phase4AE CUDA math as read-only
dependency, not an approximate solver. Initial state is v=-52mV, g=0, available;
all cells retain the original 22-tick refractory rule and delay is exactly18ticks.
The common deterministic stimulus requests v=threshold+1mV at before_thresholds
only if not refractory. This is not original poi and not physiological input.
Preregister three cases: no request for10ms; neuron1381 requested once at tick0
for50ms; [1197,1381,1511,82312,133666] requested once at tick0 for50ms.
Each CPU case uses one fresh process, zero-time compilation then one continuous
nonzero Network.run. Full end-of-tick V/g/ref are streamed to .npy memory maps;
spikes remain sparse events. GPU compares every tick bitwise against these maps.
No dense spike matrix, no conditioning, no learning and no APL overlay in this
isolated numerical gate. The omission of APL is explicit, not a revised calibrated
brain. Any later APL branch integration requires its own semantic gate.
Observe the actual Brian queue by forwarding the original peek result unchanged.
SHA-256 per tick records raw original edge order and the original postsynaptic
not_refractory acceptance mask. GPU digests the source-ordered expansion before
stable target grouping. Joint hash has the versioned protocol header, little-endian
tick/count, contiguous little-endian int64 IDs followed by uint8 accepted flags.
Different targets may be scheduled in parallel, but each target's accumulation
order and exact incoming old g are preserved. No separate reduction changes.
Compare sparse event sequence, counts, every V/g bit, refractory, request
accepted/rejected and queue edge/guard digests. Stop after the first failed trial;
preserve a failure diagnostic, do not widen tolerance, choose alternate input or
interpret low activity as success. Resource stops are incomplete, not scientific
FAILs. New output budget15GiB; stream states and do not dump all arrivals.
Before and after: verify original frozen inventory, subsequent AA--AE seals and
the background evidence seal. Re-run the five old deterministic-v2 reference trials
into this experiment only. These historical regressions do not certify new model
learning or long-run equivalence. Record source hashes, graph/input hashes,
environment, generated code, code commit, wall time and diagnostic peak VRAM.
All outputs and source are sealed with manifests and SHA-256 after completion.
@@ -0,0 +1,12 @@
64413d57ea8c0222ce40f007078b1b2ce7b79986d846878e4ce408e87446b2bf PROTOCOL.md
2c81cb37598cce2ae0688046fd944bec34ea57824944379bfbf7e778283ab4ad compare_trial.py
9acd8177d94daf92df3435d2df7c854a0b86e8dee65d42f3ec25d084c9fba8f0 config.json
fe221d4470f14a094bc2b02b9763d542c091a2a069c951c5f833a0639c39659b context.py
81885c07c053444cb0b03634b4ddc09b12bef2f237828db2dcf203653ac4b704 finalize.py
895ee595d63ee8650b6aad662f831117af78cb6216437c001c8c7ba69a7b92e7 gpu_full.py
d72003d98f0f1624b12126eff50bd361c43d888f4a64d90ba9247a58976f1961 oracle_full.py
780f26c3c829135a6f67ff03007925649ef777dc15010334830ff1f7fa109519 prepare.py
afb8a0e8513726bd1340db56c2cff8f3615fedd0857dc611420d29e1b858c2c3 run_trial.py
d4bc6905ea6ce4ea04aeeabfc2f99bc111114fdfd0bc54169a40edf4f95c6033 test_protocol.py
a2a19b52c41c8c752692ff524d6f21016c6f2789c5fcabac9c78c8cadd59826e verify_frozen.py
ead78ed89d0daebee5281ef285bbfe9d79c69b5034830609414f7b130d3c5827 manifest.json
@@ -0,0 +1,84 @@
"""Independent file-level exact gate; no reliance on a stored backend PASS."""
import argparse
import csv
import hashlib
import json
import numpy as np
from context import EXP,CFG,check,save,digest
def jsonl(path):
with path.open() as f: return [json.loads(line) for line in f if line.strip()]
def bit_equal(a,b):
return a.shape==b.shape and a.dtype==b.dtype and np.array_equal(np.ascontiguousarray(a).view(np.uint8),np.ascontiguousarray(b).view(np.uint8))
def evaluate(name):
spec=json.loads((EXP/'cases'/name/'spec.json').read_text());steps=spec['steps'];n=CFG['neurons']
cpu=EXP/'oracle'/name;gpu=EXP/'gpu'/name;checks={};details={}
arrays={key:np.load(cpu/filename,mmap_mode='r') for key,filename in (
('v','v-history.float64.npy'),('g','g-history.float64.npy'),('ref','ref-history.int32.npy'))}
expected={'v':np.dtype('float64'),'g':np.dtype('float64'),'ref':np.dtype('int32')}
checks['history_shapes_dtypes']=all(x.shape==(steps,n) and x.dtype==expected[k] for k,x in arrays.items())
assert checks['history_shapes_dtypes']
hashes=jsonl(gpu/'tick-state-hashes.jsonl')
checks['gpu_state_hash_coverage']=len(hashes)==steps and [x['tick'] for x in hashes]==list(range(steps))
assert checks['gpu_state_hash_coverage']
failures=[];finite=True
for tick,row in enumerate(hashes):
for key,value in arrays.items():
a=np.ascontiguousarray(value[tick]);sha=hashlib.sha256(a.tobytes()).hexdigest()
if sha!=row[key+'_sha256']:failures.append(dict(tick=tick,field=key))
if key!='ref' and not np.isfinite(a).all():finite=False
checks['all_tick_raw_state_hashes_exact']=not failures;checks['all_oracle_states_finite']=finite
details['first_hash_mismatches']=failures[:10]
with np.load(cpu/'state.npz',allow_pickle=False) as a,np.load(gpu/'state.npz',allow_pickle=False) as b:
for key in ('step','neuron_idx','counts','v','g','ref'):
checks['exact_'+key]=bit_equal(a[key],b[key])
checks['event_clock_range']=bool(np.all((a['step']>=0)&(a['step']<steps)))
checks['event_neuron_range']=bool(np.all((a['neuron_idx']>=0)&(a['neuron_idx']<n)))
checks['event_sort']=np.array_equal(np.lexsort((a['neuron_idx'],a['step'])),np.arange(len(a['step'])))
checks['event_unique']=len(np.unique(a['step'].astype(np.int64)*n+a['neuron_idx']))==len(a['step'])
checks['cpu_event_counts']=np.array_equal(np.bincount(a['neuron_idx'],minlength=n),a['counts'])
checks['gpu_event_counts']=np.array_equal(np.bincount(b['neuron_idx'],minlength=n),b['counts'])
for key in arrays:checks['cpu_final_'+key]=bit_equal(arrays[key][-1],a[key])
details.update(events=len(a['step']),active_neurons=int(np.count_nonzero(a['counts'])),
max_per_neuron_count=int(a['counts'].max()),
mean_rate_hz=float(len(a['step'])/(n*spec['duration_ms']/1000)),
first_event_step=None if not len(a['step']) else int(a['step'][0]))
ptr=np.load(EXP/'graph/ptr.npy',mmap_mode='r');degrees=np.diff(ptr)
expected_arrivals=np.zeros(steps,np.int64);due=a['step'].astype(np.int64)+18
valid=due<steps
np.add.at(expected_arrivals,due[valid],degrees[a['neuron_idx'][valid]])
cq,gq=jsonl(cpu/'queue.jsonl'),jsonl(gpu/'queue.jsonl')
checks['queue_coverage']=len(cq)==len(gq)==steps and [r['tick'] for r in cq]==list(range(steps))
checks['queue_original_order_exact']=cq==gq
checks['all_presynaptic_edges_due']=np.array_equal(expected_arrivals,np.asarray([r['count'] for r in cq],np.int64))
with (cpu/'requests.csv').open(newline='') as f:cr=list(csv.DictReader(f))
with (gpu/'requests.csv').open(newline='') as f:gr=list(csv.DictReader(f))
checks['request_rows_exact']=cr==gr
checks['request_count']=len(cr)==len(spec['seed_nodes'])
checks['actual_requests_match_protocol']=cr==[
dict(tick='0',neuron_idx=str(node),attempted='1',accepted='1',rejected='0')
for node in sorted(spec['seed_nodes'])]
details['request_rows']=cr
if cq:
details['queue_fields']=list(cq[0])
details['arrival_count']=sum(r['count'] for r in cq)
details['accepted_arrival_count']=sum(r['accepted_count'] for r in cq)
summary=json.loads((gpu/'summary.json').read_text())
checks['backend_summary_pass']=summary['pass_all'] is True
oracle_summary=json.loads((cpu/'summary.json').read_text())
checks['oracle_complete']=oracle_summary['complete'] is True and oracle_summary['steps']==steps and oracle_summary['nodes']==n and oracle_summary['edges']==CFG['edges']
checks={k:bool(v) for k,v in checks.items()}
files=[cpu/'state.npz',gpu/'state.npz',cpu/'queue.jsonl',gpu/'queue.jsonl',gpu/'tick-state-hashes.jsonl']
return dict(case=name,pass_all=all(checks.values()),checks=checks,details=details,
evidence_sha256={str(p.relative_to(EXP)):digest(p) for p in files})
def main():
ap=argparse.ArgumentParser();ap.add_argument('case');a=ap.parse_args();check()
assert a.case in [t['name'] for t in CFG['trials']]
result=evaluate(a.case);save(EXP/'audit'/('completed-'+a.case+'.json'),result)
print(json.dumps(result,ensure_ascii=False),flush=True)
if not result['pass_all']:raise SystemExit(1)
if __name__=='__main__':main()
@@ -0,0 +1,24 @@
{
"phase": "4AF",
"name": "author-fullgraph-lab-v1",
"human_authorization": "允许独立新模型分支测试动力学假设",
"precision": "float64",
"precision_reason": "Original Brian author default; not selected by network activity. Phase4AE float32 is not universally event-equivalent to float64.",
"neurons": 166700,
"edges": 25582938,
"raw_weight_sum": 124177617,
"gpu_uuid": "GPU-79f3a85d-adce-7833-7003-6488140ca596",
"trials": [
{"name": "no_input", "steps": 100, "seed_nodes": []},
{"name": "single", "steps": 500, "seed_nodes": [1381]},
{"name": "five", "steps": 500, "seed_nodes": [1197, 1381, 1511, 82312, 133666]}
],
"all_requests_at_tick": 0,
"apl_overlay": false,
"training": false,
"candidate_generation": false,
"gate": "Every timestep float64 V/g bitwise identical; refractory, sparse events, counts, request delivery, original queue ordered edge/acceptance digests exact.",
"stop_on_first_trial_failure": true,
"new_output_budget_bytes": 16106127360,
"limits": "Only three short trajectories. No long-run/full-protocol equivalence, biological fidelity, olfactory CAL or learning claim. No performance claim."
}
@@ -0,0 +1,54 @@
import hashlib
import importlib.util
import json
from pathlib import Path
MODEL = Path(__file__).resolve().parent
ROOT = Path('/data/digital-drosophila')
EXP = ROOT/'experiments/phase4af-author-fullgraph'
AD = ROOT/'experiments/phase4ad-author-semantics'
AE_MODEL = ROOT/'models/male-cns-lif/author-runtime-lab-v1'
AE_EXP = ROOT/'experiments/phase4ae-author-runtime'
CFG = json.loads((MODEL/'config.json').read_text(encoding='utf-8'))
def digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as f:
for b in iter(lambda: f.read(8<<20), b''): h.update(b)
return h.hexdigest()
def save(path, value):
path = Path(path); path.parent.mkdir(parents=True, exist_ok=True)
with path.open('x', encoding='utf-8') as f:
json.dump(value, f, indent=2, ensure_ascii=False, allow_nan=False); f.write('\n')
def sources():
return {p.name: digest(p) for p in sorted(MODEL.iterdir())
if p.is_file() and p.name not in ('manifest.json', 'checksums.sha256')}
def check():
p = json.loads((EXP/'protocol/preregistered.json').read_text())
assert sources() == p['sources'], 'Source changed after preregistration'
for row in p['inputs']:
assert digest(ROOT/row['path']) == row['sha256'], row['path']
for row in json.loads((EXP/'protocol/graph-case-hashes.json').read_text()):
assert digest(EXP/row['path']) == row['sha256'], row['path']
return p
def integrity():
spec = importlib.util.spec_from_file_location('previous_ae_context', AE_MODEL/'context.py')
previous = importlib.util.module_from_spec(spec); spec.loader.exec_module(previous)
result = previous.integrity()
import sys
sys.path.insert(0, str(ROOT/'models/male-cns-lif/terminal-scope-diagnostic-v1'))
from diagnostic import manifest_check
for p in (AE_MODEL, AE_EXP):
result[str(p.relative_to(ROOT))] = manifest_check(p, digest)
assert all(r['pass'] for r in result.values())
return result
def output_bytes():
return sum(p.stat().st_size for p in EXP.rglob('*') if p.is_file())
def budget_check():
assert output_bytes() < CFG['new_output_budget_bytes'], 'New output budget exceeded'
@@ -0,0 +1,58 @@
"""Seal the fixed short-trial audit, never promote a learned model."""
import json
from context import ROOT,MODEL,EXP,CFG,check,save,digest,integrity,output_bytes,budget_check
from compare_trial import evaluate
def seal(root):
assert not (root/'manifest.json').exists() and not (root/'checksums.sha256').exists()
rows=[dict(path=str(p.relative_to(root)),bytes=p.stat().st_size,sha256=digest(p))
for p in sorted(root.rglob('*')) if p.is_file() and '__pycache__' not in p.parts]
save(root/'manifest.json',dict(root=str(root),files=rows))
with (root/'checksums.sha256').open('x') as f:
for r in rows:f.write(r['sha256']+' '+r['path']+'\n')
f.write(digest(root/'manifest.json')+' manifest.json\n')
def main():
check();budget_check();rows=[];missing=[]
for trial in CFG['trials']:
p=EXP/'audit'/('completed-'+trial['name']+'.json')
if not p.exists():missing.append(trial['name']);continue
stored=json.loads(p.read_text());actual=evaluate(trial['name'])
assert stored==actual,'Saved comparison no longer agrees with raw evidence'
rows.append(actual)
old=json.loads((EXP/'regression/frozen-five.json').read_text())
after=integrity();save(EXP/'regression/old-integrity-after.json',after)
errors=[str(p.relative_to(EXP)) for p in sorted((EXP/'audit').glob('error-*.json'))]
all_present=len(rows)==len(CFG['trials']) and not missing
passed=all_present and not errors and all(r['pass_all'] for r in rows) and old['pass_all'] and all(r['pass'] for r in after.values())
status='PASS-SHORT-FULLGRAPH-NUMERICAL-GATE' if passed else 'FAIL-SHORT-FULLGRAPH-NUMERICAL-GATE'
if errors:status='INCOMPLETE-EXECUTION-ERROR'
metrics={r['case']:json.loads((EXP/'gpu'/r['case']/'metrics.json').read_text()) for r in rows}
result=dict(status=status,pass_all=bool(passed),neurons=CFG['neurons'],edges=CFG['edges'],raw_weight_sum=CFG['raw_weight_sum'],
trials_expected=len(CFG['trials']),trials_compared=len(rows),not_executed=missing,errors=errors,
trials=rows,diagnostic_metrics=metrics,whole_brain_long_run_equivalence=False,
apl_overlay_validated=False,olfactory_representation_validated=False,conditioning=False,
learning_achieved=False,candidate_generated=False,old_frozen_integrity_pass=True,
old_five_reference_pass=old['pass_all'],payload_bytes_before_manifest=output_bytes(),
next_gate='Explicit fixed APL/author semantics integration audit, then fixed DoOR CAL without learning' if passed else 'Locate first numerical/execution discrepancy; no CAL or learning promotion')
save(EXP/'audit/final-status.json',result)
lines=['# Phase 4AF:短时完整图数值对照','',f'状态:`{status}`。','',
'完整166,700节点/25,582,938边,原synaptic-contact总数124,177,617。',
'作者默认float64;新分支只读复用4AE CUDA算术与未修改作者Brian2 create_model。',
'无APL叠加、无Poisson forcing、无训练。只有tick0固定请求,所有原始边/符号/W_syn保留。','',
'| trial | biological ms | events | active neurons | per-tick exact gate |','|---|---:|---:|---:|---|']
for row in rows:
trial=next(x for x in CFG['trials'] if x['name']==row['case']);d=row['details']
lines.append(f"| {row['case']} | {trial['steps']*.1:g} | {d['events']} | {d['active_neurons']} | {row['pass_all']} |")
lines+=['','每tick比较所有细胞原始V/g bit、ref、spike events、实际输入以及原队列edge/guard顺序SHA256。',
'完整CPU状态保留,GPU逐tick状态hash由独立文件审计再次重算核对;未保存dense spike matrix。',
f'未执行:{missing};执行异常:{errors}。',
'GPU wall/VRAM含逐tick同步、CPU拷贝和hash,仅是诊断成本,不是实时性能benchmark。',
'原冻结校验与旧五组deterministic-v2参考回归另列;旧回归PASS不代表新动力学学习PASS。',
'三个短轨迹一致也不证明长期全脑行为、生物学正确、嗅觉区分或学习成功。',
'下一步:'+result['next_gate'],'']
with (EXP/'REPORT.md').open('x',encoding='utf-8') as f:f.write('\n'.join(lines))
check();seal(MODEL);seal(EXP);print(status,flush=True)
if not passed:raise SystemExit(1)
if __name__=='__main__':main()
@@ -0,0 +1,369 @@
"""Float64 full-graph numerical audit using sealed Phase 4AE CUDA operators.
This synchronous diagnostic compares every neuron at every tick with the CPU
memmaps. It does not introduce a new integration rule, APL, training or tuning.
"""
from functools import lru_cache
import csv
import hashlib
import json
import math
from pathlib import Path
import sys
import time
import traceback
import types
import numpy as np
import torch
from context import ROOT
from oracle_full import queue_summary
NEURONS = 166700
EDGES = 25582938
COEFFICIENT_NAMES = ("gd", "c0", "cg", "cv", "threshold", "reset", "seed_volt")
REQUEST_FIELDS = ("tick", "neuron_idx", "attempted", "accepted", "rejected")
AE_MODEL = ROOT / "models/male-cns-lif/author-runtime-lab-v1"
ALLOWED_CASES = {(100, ()), (500, (1381,)), (500, (1197, 1381, 1511, 82312, 133666))}
def require(condition, message):
if not condition:
raise ValueError(message)
def write_json(path, value):
with Path(path).open("x", encoding="utf-8") as output:
json.dump(value, output, indent=2, allow_nan=False)
output.write("\n")
def digest(path):
value = hashlib.sha256()
with Path(path).open("rb") as source:
for block in iter(lambda: source.read(8 << 20), b""):
value.update(block)
return value.hexdigest()
def array_digest(array, dtype):
return hashlib.sha256(np.ascontiguousarray(array, dtype=dtype).tobytes()).hexdigest()
def bitwise_comparison(actual, expected, floating=False):
require(actual.shape == expected.shape and actual.dtype == expected.dtype, "Comparison shape/dtype mismatch")
aa = actual.view(np.uint64) if floating else actual
bb = expected.view(np.uint64) if floating else expected
mismatch = aa != bb
count = int(np.count_nonzero(mismatch))
first = int(np.flatnonzero(mismatch)[0]) if count else None
result = {"bitwise_exact": count == 0, "mismatching_neurons": count,
"first_neuron_idx": first}
if floating:
finite = bool(np.isfinite(actual).all() and np.isfinite(expected).all())
result.update(finite=finite, max_abs_volt=float(np.max(np.abs(actual - expected))) if finite else None)
if first is not None:
result.update(actual_bits_hex=f"{int(aa[first]):016x}", expected_bits_hex=f"{int(bb[first]):016x}")
return result
@lru_cache(maxsize=1)
def sealed_backend():
"""Load reviewed source without writing bytecode into the sealed model."""
source = AE_MODEL / "gpu_backend.py"
module = types.ModuleType("phase4af_sealed_4ae_gpu_backend")
module.__file__ = str(source)
exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__)
return module
@lru_cache(maxsize=2)
def load_graph(graph_directory):
graph_dir = Path(graph_directory)
require(graph_dir.is_absolute(), "graph_dir must be absolute")
dtypes = {"nodes": np.int32, "edge_id": np.int64, "src": np.int32, "dst": np.int32,
"signed_raw": np.int64, "ptr": np.int64}
graph = {}
for name, dtype in dtypes.items():
value = np.load(graph_dir / (name + ".npy"), mmap_mode="r", allow_pickle=False)
require(value.dtype == dtype and value.ndim == 1, "Invalid graph array: " + name)
graph[name] = value
require(graph["nodes"].shape == (NEURONS,) and
np.array_equal(graph["nodes"], np.arange(NEURONS, dtype=np.int32)), "Graph neuron identity changed")
require(graph["ptr"].shape == (NEURONS + 1,) and graph["ptr"][0] == 0 and
graph["ptr"][-1] == EDGES and bool(np.all(np.diff(graph["ptr"]) >= 0)), "Invalid canonical CSR")
for name in ("edge_id", "src", "dst", "signed_raw"):
require(graph[name].shape == (EDGES,), "Graph edge count differs: " + name)
require(np.array_equal(graph["src"], np.repeat(graph["nodes"], np.diff(graph["ptr"]))),
"Canonical source rows disagree with CSR")
for start in range(0, EDGES, 1 << 20):
stop = min(start + (1 << 20), EDGES)
require(np.array_equal(graph["edge_id"][start:stop], np.arange(start, stop, dtype=np.int64)),
"Graph edge IDs are not canonical CSR row positions")
require(bool(np.all((graph["dst"][start:stop] >= 0) & (graph["dst"][start:stop] < NEURONS))),
"Target index outside complete graph")
return graph
def load_inputs(case_dir, oracle_dir):
with (case_dir / "spec.json").open(encoding="utf-8") as source:
spec = json.load(source)
for name in ("name", "steps", "dt_ms", "duration_ms", "seed_nodes", "model", "graph_dir"):
require(name in spec, "Missing case field: " + name)
steps = spec["steps"]
seeds = spec["seed_nodes"]
require(type(steps) is int and isinstance(seeds, list) and
all(type(node) is int for node in seeds) and len(seeds) == len(set(seeds)), "Invalid case steps/seeds")
seeds = tuple(sorted(seeds))
require((steps, seeds) in ALLOWED_CASES, "Case not one of the three fixed full-graph trials")
with (oracle_dir / "summary.json").open(encoding="utf-8") as source:
oracle_summary = json.load(source)
require(oracle_summary.get("complete") is True and oracle_summary.get("steps") == steps and
oracle_summary.get("completed_ticks") == steps and oracle_summary.get("precision") == "float64" and
oracle_summary.get("nodes") == NEURONS and oracle_summary.get("edges") == EDGES,
"CPU oracle did not seal a complete trial with the declared steps/nodes/edges")
require(float(spec["dt_ms"]) == .1 and float(spec["duration_ms"]) == steps * .1,
"Unexpected duration or timestep")
model = spec["model"]
for name in ("resting_potential_mv", "threshold_mv", "reset_potential_mv", "synaptic_delay_ms",
"refractory_ms", "w_syn_mv_per_anatomical_synapse"):
require(name in model and math.isfinite(float(model[name])), "Missing/nonfinite model field: " + name)
require(float(model["synaptic_delay_ms"]) == 1.8 and float(model["refractory_ms"]) == 2.2 and
float(model["w_syn_mv_per_anatomical_synapse"]) == .275, "Declared delay/refractory/W_syn changed")
require(Path(spec["graph_dir"]).is_absolute(), "Case graph_dir must be absolute")
graph = load_graph(str(Path(spec["graph_dir"]).resolve()))
with (oracle_dir / "coefficients.json").open(encoding="utf-8") as source:
coefficients = json.load(source)
for name in COEFFICIENT_NAMES:
require(name in coefficients and math.isfinite(float(coefficients[name])), "Missing/nonfinite oracle coefficient: " + name)
require(coefficients["threshold"] == float(model["threshold_mv"]) * .001 and
coefficients["reset"] == float(model["reset_potential_mv"]) * .001,
"Oracle threshold/reset disagree with model declaration")
histories = {}
for name, dtype in (("v", np.float64), ("g", np.float64), ("ref", np.int32)):
filename = name + "-history." + np.dtype(dtype).name + ".npy"
value = np.load(oracle_dir / filename, mmap_mode="r", allow_pickle=False)
require(value.shape == (steps, NEURONS) and value.dtype == dtype, "Invalid CPU state history: " + name)
histories[name] = value
weights = np.load(oracle_dir / "weights.npy", mmap_mode="r", allow_pickle=False)
require(weights.dtype == np.float64 and weights.shape == (EDGES,) and bool(np.isfinite(weights).all()),
"Invalid actual Brian2 SI weights")
# A sign or row mismatch must not be hidden by copying the oracle weight file.
for start in range(0, EDGES, 1 << 20):
stop = min(start + (1 << 20), EDGES)
require(np.array_equal(np.sign(weights[start:stop]), np.sign(graph["signed_raw"][start:stop])),
"Oracle weight sign/zero pattern differs from graph")
with np.load(oracle_dir / "state.npz", allow_pickle=False) as source:
final = {name: source[name].copy() for name in source.files}
for name, dtype in (("v", np.float64), ("g", np.float64), ("ref", np.int32), ("counts", np.int32)):
require(final[name].dtype == dtype and final[name].shape == (NEURONS,), "Invalid CPU final state: " + name)
require(final["step"].dtype == np.int32 and final["neuron_idx"].dtype == np.int32 and
final["step"].ndim == 1 and final["step"].shape == final["neuron_idx"].shape, "Invalid CPU sparse events")
require(bool(np.all((final["step"] >= 0) & (final["step"] < steps))) and
bool(np.all(np.diff(final["step"]) >= 0)) and
bool(np.all((final["neuron_idx"] >= 0) & (final["neuron_idx"] < NEURONS))), "CPU event index out of range")
with (oracle_dir / "queue.jsonl").open(encoding="utf-8") as source:
queues = [json.loads(line) for line in source if line.strip()]
require(len(queues) == steps and [row["tick"] for row in queues] == list(range(steps)),
"Missing, duplicate or reordered CPU queue records")
with (oracle_dir / "requests.csv").open(encoding="utf-8", newline="") as source:
reader = csv.DictReader(source)
require(tuple(reader.fieldnames or ()) == REQUEST_FIELDS, "CPU request CSV schema mismatch")
request_rows = [{name: int(row[name]) for name in REQUEST_FIELDS} for row in reader]
require(len(request_rows) == len(seeds) and
[row["neuron_idx"] for row in request_rows] == list(seeds) and
all(row["tick"] == 0 and row["attempted"] == 1 and row["accepted"] in (0, 1) and
row["rejected"] == 1 - row["accepted"] for row in request_rows), "CPU request coverage mismatch")
return spec, graph, histories, weights, final, queues, coefficients, seeds, request_rows
def run_gpu(case_dir, out_dir, oracle_dir, build_root):
"""Execute one fixed full graph and compare every state bit, event and queue.
Full CPU histories remain read-only. The first failing tick saves complete
CPU/GPU V/g/ref arrays; every tick saves exact comparison fields and hashes.
"""
case_dir, out_dir, oracle_dir = Path(case_dir), Path(out_dir), Path(oracle_dir)
require(not out_dir.exists(), "Refusing existing output directory: " + str(out_dir))
spec, graph, histories, weights, oracle_final, oracle_queues, coeff, seeds, oracle_requests = load_inputs(case_dir, oracle_dir)
require(torch.cuda.is_available(), "CUDA unavailable")
require(torch.cuda.device_count() == 1 and "4090" in torch.cuda.get_device_name(),
"Only the declared single visible RTX4090 is permitted")
out_dir.mkdir(parents=True, exist_ok=False)
start = time.perf_counter()
tick = -1
rows = []
first_failure = None
try:
backend = sealed_backend()
ext, build_provenance = backend.build_extension(build_root)
build_end = time.perf_counter()
device = torch.device("cuda", torch.cuda.current_device())
torch.cuda.reset_peak_memory_stats(device)
def upload(value, dtype):
# Writable copies prevent any accidental write to read-only source maps.
return torch.from_numpy(np.array(value, copy=True, order="C")).to(device=device, dtype=dtype).contiguous()
indptr = upload(graph["ptr"], torch.int64)
dst = upload(graph["dst"], torch.int32)
gpu_weights = upload(weights, torch.float64)
rest = float(spec["model"]["resting_potential_mv"]) * .001
v = torch.full((NEURONS,), rest, device=device, dtype=torch.float64)
g = torch.zeros(NEURONS, device=device, dtype=torch.float64)
ref = torch.zeros(NEURONS, device=device, dtype=torch.int32)
counts = torch.zeros(NEURONS, device=device, dtype=torch.int32)
requests = torch.zeros(NEURONS, device=device, dtype=torch.uint8)
if seeds:
requests[torch.tensor(seeds, device=device, dtype=torch.int64)] = 1
ring = torch.zeros((19, NEURONS), device=device, dtype=torch.uint8)
events_step, events_node = [], []
requested_total = accepted_request_total = 0
total_due = total_accepted = 0
torch.cuda.synchronize(device)
simulation_start = time.perf_counter()
with (out_dir / "step-comparisons.jsonl").open("x", encoding="utf-8", buffering=1) as comparisons, \
(out_dir / "queue.jsonl").open("x", encoding="utf-8", buffering=1) as queue_output, \
(out_dir / "tick-state-hashes.jsonl").open("x", encoding="utf-8", buffering=1) as hash_output, \
(out_dir / "requests.csv").open("x", encoding="utf-8", newline="", buffering=1) as request_output:
request_writer = csv.DictWriter(request_output, fieldnames=REQUEST_FIELDS)
request_writer.writeheader()
for tick in range(spec["steps"]):
if tick == 1:
requests.zero_()
requested = int(torch.count_nonzero(requests).item())
accepted_requests = int(torch.count_nonzero((requests != 0) & (ref <= 1)).item())
requested_total += requested
accepted_request_total += accepted_requests
request_rows = []
if requested:
requested_nodes = torch.nonzero(requests, as_tuple=False).flatten()
accepted_mask = (ref[requested_nodes] <= 1).to(torch.uint8).cpu().numpy()
for node, accepted in zip(requested_nodes.cpu().numpy(), accepted_mask):
request_rows.append({"tick": tick, "neuron_idx": int(node), "attempted": 1,
"accepted": int(accepted), "rejected": 1 - int(accepted)})
request_writer.writerows(request_rows)
requests_exact = request_rows == [row for row in oracle_requests if row["tick"] == tick]
spikes = ring[tick % 19]
ext.state_step(v, g, ref, requests, spikes, counts,
*[float(coeff[name]) for name in COEFFICIENT_NAMES], 22)
expanded_cpu = np.empty(0, dtype=np.int64)
accepted_cpu = np.empty(0, dtype=np.uint8)
if tick >= 18:
due = ring[(tick - 18) % 19]
sources = torch.nonzero(due, as_tuple=False).flatten().contiguous()
if sources.numel():
degrees = indptr[sources + 1] - indptr[sources]
offsets = (torch.cumsum(degrees, dim=0) - degrees).contiguous()
edge_count = int(degrees.sum().item())
expanded = torch.empty(edge_count, device=device, dtype=torch.int64)
ext.fill_edges(indptr, sources, offsets, expanded)
if edge_count:
destination = dst[expanded]
# Hash the actual GPU-expanded queue BEFORE target grouping.
expanded_cpu = expanded.cpu().numpy().copy()
accepted_cpu = (ref[destination.long()] == 0).to(torch.uint8).cpu().numpy().copy()
order = torch.argsort(destination, stable=True)
ordered_edges = expanded[order].contiguous()
ordered_targets = destination[order]
boundary = torch.ones(edge_count, device=device, dtype=torch.bool)
boundary[1:] = ordered_targets[1:] != ordered_targets[:-1]
starts = torch.nonzero(boundary, as_tuple=False).flatten().contiguous()
targets = ordered_targets[starts].contiguous()
ext.apply_arrivals(ordered_edges, starts, targets, gpu_weights, g, ref)
queue = queue_summary(tick, expanded_cpu, accepted_cpu)
queue_output.write(json.dumps(queue, allow_nan=False) + "\n")
total_due += queue["count"]
total_accepted += queue["accepted_count"]
queue_exact = queue == oracle_queues[tick]
# No external input exists in these fixed cases after the tick-zero requests.
actual = {"v": v.cpu().numpy().copy(), "g": g.cpu().numpy().copy(),
"ref": ref.cpu().numpy().copy()}
state_checks = {name: bitwise_comparison(actual[name], histories[name][tick], name != "ref")
for name in ("v", "g", "ref")}
fired = torch.nonzero(spikes, as_tuple=False).flatten().cpu().numpy().astype(np.int32)
if len(fired):
events_step.append(np.full(len(fired), tick, dtype=np.int32))
events_node.append(fired.copy())
oracle_nodes = oracle_final["neuron_idx"][oracle_final["step"] == tick]
events_exact = bool(np.array_equal(fired, oracle_nodes))
finite = state_checks["v"]["finite"] and state_checks["g"]["finite"]
passed = bool(finite and queue_exact and events_exact and requests_exact and
all(value["bitwise_exact"] for value in state_checks.values()))
row = {"tick": tick, "pass_all": passed, "states": state_checks,
"events_exact": events_exact, "gpu_event_count": len(fired), "cpu_event_count": len(oracle_nodes),
"queue_exact": queue_exact, "gpu_queue": queue, "cpu_queue": oracle_queues[tick],
"requests": requested, "accepted_requests": accepted_requests,
"rejected_requests": requested - accepted_requests,
"request_rows_exact": requests_exact,
"state_sha256": {name: array_digest(actual[name], "<i4" if name == "ref" else "<f8")
for name in ("v", "g", "ref")}}
comparisons.write(json.dumps(row, allow_nan=False) + "\n")
hash_output.write(json.dumps({"tick": tick,
**{name + "_sha256": value for name, value in row["state_sha256"].items()}},
allow_nan=False) + "\n")
rows.append(row)
if not passed and first_failure is None:
first_failure = row
with (out_dir / "first-failure-state.npz").open("xb") as target:
np.savez_compressed(target, tick=np.array(tick, dtype=np.int32),
**{"gpu_" + name: value for name, value in actual.items()},
**{"cpu_" + name: np.asarray(histories[name][tick]).copy() for name in actual},
gpu_event_nodes=fired, cpu_event_nodes=oracle_nodes,
arrival_edge_id=expanded_cpu, arrival_accepted=accepted_cpu)
write_json(out_dir / "first-failure.json", row)
torch.cuda.synchronize(device)
simulation_end = time.perf_counter()
def join(values):
return np.concatenate(values).astype(np.int32, copy=False) if values else np.empty(0, dtype=np.int32)
state = {"step": join(events_step), "neuron_idx": join(events_node), "counts": counts.cpu().numpy(),
"v": v.cpu().numpy(), "g": g.cpu().numpy(), "ref": ref.cpu().numpy()}
with (out_dir / "state.npz").open("xb") as target:
np.savez_compressed(target, **state)
final_checks = {name: bool(np.array_equal(value.view(np.uint64), oracle_final[name].view(np.uint64)))
if name in ("v", "g") else bool(np.array_equal(value, oracle_final[name]))
for name, value in state.items()}
history_final_checks = {name: bool(np.array_equal(oracle_final[name].view(np.uint64), histories[name][-1].view(np.uint64)))
if name in ("v", "g") else bool(np.array_equal(oracle_final[name], histories[name][-1]))
for name in ("v", "g", "ref")}
pass_all = bool(len(rows) == spec["steps"] and all(row["pass_all"] for row in rows) and
all(final_checks.values()) and all(history_final_checks.values()) and
requested_total == len(seeds) and accepted_request_total == len(seeds))
summary = {"case": spec["name"], "pass_all": pass_all, "precision": "float64", "state_units": "SI volts",
"neurons": NEURONS, "edges": EDGES, "steps_expected": spec["steps"], "steps_compared": len(rows),
"all_neurons_compared_every_tick": True, "bitwise_includes_signed_zero": True,
"first_failed_tick": None if first_failure is None else first_failure["tick"],
"failed_tick_count": sum(not row["pass_all"] for row in rows),
"final_state_exact": final_checks, "oracle_final_matches_history": history_final_checks,
"total_spikes": len(state["step"]), "arriving_edges": total_due, "accepted_arriving_edges": total_accepted,
"requested": requested_total, "accepted_requests": accepted_request_total,
"rejected_requests": requested_total - accepted_request_total,
"whole_graph_numerical_comparison": True, "olfactory_representation_validated": False,
"apl_overlay": False, "learning": False, "candidate_generated": False,
"performance_runtime_claim": False, "cpu_sync_each_tick": True,
"first_failure_full_state_saved": first_failure is not None,
"full_gpu_history_saved": False, "sealed_4AE_model": str(AE_MODEL),
"build": build_provenance, "build_or_cache_seconds": build_end - start,
"diagnostic_simulation_seconds": simulation_end - simulation_start,
"wall_seconds": time.perf_counter() - start, "gpu": torch.cuda.get_device_name(device),
"peak_allocated_vram_bytes": int(torch.cuda.max_memory_allocated(device)),
"peak_reserved_vram_bytes": int(torch.cuda.max_memory_reserved(device)),
"input_sha256": {"spec.json": digest(case_dir / "spec.json"),
"oracle/weights.npy": digest(oracle_dir / "weights.npy"),
"oracle/coefficients.json": digest(oracle_dir / "coefficients.json")}}
write_json(out_dir / "summary.json", summary)
write_json(out_dir / "metrics.json", {name: summary[name] for name in
("wall_seconds", "build_or_cache_seconds", "diagnostic_simulation_seconds", "gpu",
"peak_allocated_vram_bytes", "peak_reserved_vram_bytes", "performance_runtime_claim", "cpu_sync_each_tick")})
print(json.dumps({"case": spec["name"], "pass_all": pass_all, "steps": len(rows),
"first_failed_tick": summary["first_failed_tick"], "total_spikes": len(state["step"])}), flush=True)
return summary
except Exception as error:
write_json(out_dir / "error.json", {"type": type(error).__name__, "message": str(error),
"traceback": traceback.format_exc(), "tick_in_progress": tick,
"completed_tick_comparisons": len(rows), "pass_all": False})
raise
@@ -0,0 +1,60 @@
{
"root": "/data/digital-drosophila/models/male-cns-lif/author-fullgraph-lab-v1",
"files": [
{
"path": "PROTOCOL.md",
"bytes": 2992,
"sha256": "64413d57ea8c0222ce40f007078b1b2ce7b79986d846878e4ce408e87446b2bf"
},
{
"path": "compare_trial.py",
"bytes": 5322,
"sha256": "2c81cb37598cce2ae0688046fd944bec34ea57824944379bfbf7e778283ab4ad"
},
{
"path": "config.json",
"bytes": 1183,
"sha256": "9acd8177d94daf92df3435d2df7c854a0b86e8dee65d42f3ec25d084c9fba8f0"
},
{
"path": "context.py",
"bytes": 2171,
"sha256": "fe221d4470f14a094bc2b02b9763d542c091a2a069c951c5f833a0639c39659b"
},
{
"path": "finalize.py",
"bytes": 4373,
"sha256": "81885c07c053444cb0b03634b4ddc09b12bef2f237828db2dcf203653ac4b704"
},
{
"path": "gpu_full.py",
"bytes": 23311,
"sha256": "895ee595d63ee8650b6aad662f831117af78cb6216437c001c8c7ba69a7b92e7"
},
{
"path": "oracle_full.py",
"bytes": 32146,
"sha256": "d72003d98f0f1624b12126eff50bd361c43d888f4a64d90ba9247a58976f1961"
},
{
"path": "prepare.py",
"bytes": 5259,
"sha256": "780f26c3c829135a6f67ff03007925649ef777dc15010334830ff1f7fa109519"
},
{
"path": "run_trial.py",
"bytes": 2027,
"sha256": "afb8a0e8513726bd1340db56c2cff8f3615fedd0857dc611420d29e1b858c2c3"
},
{
"path": "test_protocol.py",
"bytes": 2019,
"sha256": "d4bc6905ea6ce4ea04aeeabfc2f99bc111114fdfd0bc54169a40edf4f95c6033"
},
{
"path": "verify_frozen.py",
"bytes": 2151,
"sha256": "a2a19b52c41c8c752692ff524d6f21016c6f2789c5fcabac9c78c8cadd59826e"
}
]
}
@@ -0,0 +1,540 @@
"""Phase 4AF: a streaming float64 oracle for three fixed short full-graph trials.
Importing this module does not import Brian. queue_summary is shared with the
GPU comparison code. All scientific inputs and the sealed 4AD/4AE code remain
read-only. Every run_case call belongs in a fresh process.
"""
from __future__ import annotations
import csv
import hashlib
import importlib.metadata
import json
import os
import platform
import struct
import sys
import types
from contextlib import ExitStack
from pathlib import Path
import numpy as np
N_NEURONS = 166700
N_EDGES = 25582938
BLOCK_BYTES = 8 << 20
QUEUE_HEADER = b"phase4af-queue-v1\0"
AUTHOR_PATH = Path("/data/digital-drosophila/experiments/phase4ad-author-semantics/external-resources/model.py")
AUTHOR_SHA256 = "fc45837d7122c6ce2a7f3f2f23c515992e4b232aadb919efabb72337fac88e4e"
AE_ORACLE_PATH = Path("/data/digital-drosophila/models/male-cns-lif/author-runtime-lab-v1/oracle_stress.py")
AE_ORACLE_SHA256 = "32eb1c42383068ce52701d2139a376ce155ad7072cde019949f8ca2d55492409"
EVENT_DTYPE = np.dtype([("step", "<i4"), ("neuron_idx", "<i4")])
_PROCESS_ALREADY_USED = False
def queue_summary(tick, edges, accepted, chunk_bytes=BLOCK_BYTES):
"""Hash original queue order without sorting, filtering, or changing inputs.
joint_sha256 is SHA256(header + pack('<qq', tick, count) + all edge bytes
in little-endian int64 order + all uint8 acceptance bytes). The two other
hashes omit the header and contain only their respective byte sequences.
``accepted`` may be a (count,) array or a pure callable(start, stop), which
lets the CPU read the post-threshold guard in bounded chunks. Empty ticks
retain their tick/count header in the joint hash.
"""
edge_view = np.asarray(edges)
if edge_view.ndim != 1 or edge_view.dtype.kind not in "iu":
raise ValueError("Queue edges must be a one-dimensional integer array")
tick_int = int(tick)
if tick_int != tick or tick_int < 0:
raise ValueError("Queue tick must be a nonnegative integer")
count = len(edge_view)
width = int(chunk_bytes) // np.dtype("<i8").itemsize
if width <= 0:
raise ValueError("chunk_bytes must hold at least one int64 edge")
mask_view = None if callable(accepted) else np.asarray(accepted)
if mask_view is not None and mask_view.shape != (count,):
raise ValueError("Queue acceptance mask must match the edge count")
edge_hash, mask_hash = hashlib.sha256(), hashlib.sha256()
joint = hashlib.sha256(QUEUE_HEADER + struct.pack("<qq", tick_int, count))
for begin in range(0, count, width):
original = edge_view[begin:begin + width]
if original.dtype.kind == "u" and np.any(original > np.iinfo(np.int64).max):
raise ValueError("Queue edge ID cannot be encoded as signed int64")
chunk = np.ascontiguousarray(original, dtype="<i8")
data = memoryview(chunk).cast("B")
edge_hash.update(data)
joint.update(data)
accepted_count = 0
for begin in range(0, count, width):
stop = min(count, begin + width)
original = np.asarray(accepted(begin, stop) if callable(accepted) else mask_view[begin:stop])
if original.shape != (stop - begin,) or original.dtype.kind not in "biu":
raise ValueError("Acceptance chunks must be one-dimensional boolean/integer masks")
if np.any((original != 0) & (original != 1)):
raise ValueError("Acceptance values must be zero or one")
chunk = np.ascontiguousarray(original, dtype=np.uint8)
accepted_count += int(chunk.sum(dtype=np.int64))
data = memoryview(chunk).cast("B")
mask_hash.update(data)
joint.update(data)
return {"tick": tick_int, "count": count, "accepted_count": accepted_count,
"ordered_edge_sha256": edge_hash.hexdigest(),
"accepted_mask_sha256": mask_hash.hexdigest(),
"joint_sha256": joint.hexdigest()}
def _save(path, value):
with Path(path).open("x", encoding="utf-8") as handle:
json.dump(value, handle, indent=2, sort_keys=True, allow_nan=False)
handle.write("\n")
def _digest(path):
result = hashlib.sha256()
with Path(path).open("rb") as handle:
for block in iter(lambda: handle.read(BLOCK_BYTES), b""):
result.update(block)
return result.hexdigest()
def _check(condition, message):
if not condition:
raise ValueError(message)
def _tree_bytes(root):
total = 0
for directory, _, filenames in os.walk(root, followlinks=False):
for name in filenames:
total += (Path(directory) / name).stat(follow_symlinks=False).st_size
return total
def _budget(root, limit, reserve=0):
actual = _tree_bytes(root)
if actual + int(reserve) > limit:
raise RuntimeError(f"New-output budget would be exceeded: {actual} + {reserve} > {limit} bytes; inputs were not adjusted")
return actual
def _new_memmap(path, dtype, shape):
if Path(path).exists():
raise FileExistsError(path)
return np.lib.format.open_memmap(path, mode="w+", dtype=np.dtype(dtype), shape=shape)
def _load_checked_module(path, expected_hash, name):
data = Path(path).read_bytes()
actual = hashlib.sha256(data).hexdigest()
_check(actual == expected_hash, f"Pinned source hash mismatch: {path}")
module = types.ModuleType(name)
module.__file__ = str(path)
# Execute checked bytes without producing __pycache__ beside a sealed file.
exec(compile(data, str(path), "exec"), module.__dict__)
return module
def _graph_inputs(graph_dir):
definitions = {
"nodes": (np.int32, (N_NEURONS,)), "edge_id": (np.int64, (N_EDGES,)),
"src": (np.int32, (N_EDGES,)), "dst": (np.int32, (N_EDGES,)),
"signed_raw": (np.int64, (N_EDGES,)), "ptr": (np.int64, (N_NEURONS + 1,)),
}
arrays = {}
for name, (dtype, shape) in definitions.items():
value = np.load(graph_dir / (name + ".npy"), mmap_mode="r", allow_pickle=False)
_check(value.dtype == np.dtype(dtype) and value.shape == shape, f"Incorrect canonical {name} shape/dtype")
arrays[name] = value
_check(np.array_equal(arrays["nodes"], np.arange(N_NEURONS, dtype=np.int32)),
"This full-graph protocol requires canonical global node indices 0..N-1")
ptr = arrays["ptr"]
_check(ptr[0] == 0 and ptr[-1] == N_EDGES and np.all(np.diff(ptr) >= 0), "Invalid canonical CSR pointer")
source_counts = np.zeros(N_NEURONS, dtype=np.int64)
previous_source = -1
width = BLOCK_BYTES // 8
for begin in range(0, N_EDGES, width):
stop = min(N_EDGES, begin + width)
_check(np.array_equal(arrays["edge_id"][begin:stop], np.arange(begin, stop, dtype=np.int64)),
"Full-graph edge identity/order is not the original canonical CSR order")
source, target = arrays["src"][begin:stop], arrays["dst"][begin:stop]
_check(source[0] >= previous_source and np.all(np.diff(source) >= 0), "Sources are not canonically sorted")
_check(source[0] >= 0 and source[-1] < N_NEURONS and np.all((target >= 0) & (target < N_NEURONS)),
"Canonical endpoint outside the full graph")
previous_source = int(source[-1])
source_counts += np.bincount(source, minlength=N_NEURONS)
_check(np.array_equal(source_counts, np.diff(ptr)), "src and CSR pointer disagree")
for name in ("completeness.csv", "connectivity.parquet"):
_check((graph_dir / name).is_file(), f"Missing author input file {name}")
return arrays
class _StreamingQueueProxy:
"""The 4AE forwarding contract with bounded hashing instead of retained lists."""
def __init__(self, original, get_tick, get_available, arrays, due_counts,
spike_ring, delay_steps, queue_file, records):
self._original_queue = original
self._get_tick = get_tick
self._get_available = get_available
self._arrays = arrays
self._due_counts = due_counts
self._spike_ring = spike_ring
self._delay_steps = delay_steps
self._queue_file = queue_file
self._records = records
def peek(self, *args, **kwargs):
original = self._original_queue.peek(*args, **kwargs)
edge_view = np.asarray(original)
tick = self._get_tick()
_check(tick == len(self._records["rows"]), "Queue peek ticks must be unique and consecutive")
_check(len(edge_view) == int(self._due_counts[tick % len(self._due_counts)]),
"Queue arrival count differs from delayed outgoing degree of real spikes")
available = np.asarray(self._get_available(), dtype=bool)
fired_then = np.zeros(N_NEURONS, dtype=bool)
if tick >= self._delay_steps:
fired_then[self._spike_ring[(tick - self._delay_steps) % len(self._spike_ring)]] = True
def accepted_chunk(begin, stop):
local_edges = edge_view[begin:stop]
_check(np.all((local_edges >= 0) & (local_edges < N_EDGES)), "Queue contains an invalid local edge index")
_check(np.all(fired_then[self._arrays["src"][local_edges]]),
"Queue arrival lacks a presynaptic spike at the fixed 18-step delay")
return available[self._arrays["dst"][local_edges]]
# edge_id[k] == k was independently checked for every canonical edge.
# Thus the untouched local queue array is also the original edge-ID stream.
summary = queue_summary(tick, edge_view, accepted_chunk)
self._queue_file.write(json.dumps(summary, sort_keys=True) + "\n")
self._records["rows"].append(summary)
self._records["peek_calls"] += 1
return original
def advance(self, *args, **kwargs):
self._records["advance_calls"] += 1
return self._original_queue.advance(*args, **kwargs)
def __getattr__(self, name):
return getattr(self._original_queue, name)
def run_case(case_dir: Path, out_dir: Path) -> dict:
"""Run exactly one preregistered float64 case in this process.
Outputs: v/g/ref history .npy memmaps; actual weights.npy; sparse state.npz;
queue.jsonl; events.bin with two little-endian int32 fields; request records;
small per-tick statistics and audit JSON. A failed trial retains partial
outputs and error.json and never writes summary.json with complete=True.
"""
global _PROCESS_ALREADY_USED
_check(not _PROCESS_ALREADY_USED, "Each full-graph oracle case requires a new process")
_PROCESS_ALREADY_USED = True
case_dir = Path(case_dir).resolve(strict=True)
out_dir = Path(out_dir).resolve()
spec = json.loads((case_dir / "spec.json").read_text(encoding="utf-8"))
graph_dir = Path(spec["graph_dir"]).resolve(strict=True)
steps = int(spec["steps"])
seeds = tuple(int(node) for node in spec["seed_nodes"])
fixed = {"no_input": (100, ()), "single": (500, (1381,)),
"five": (500, (1197, 1381, 1511, 82312, 133666))}
_check(spec["name"] in fixed and (steps, seeds) == fixed[spec["name"]], "Case is not one of the three fixed trials")
_check(float(spec["dt_ms"]) == .1 and np.isclose(float(spec["duration_ms"]), steps * .1, rtol=0, atol=1e-12),
"Case clock/duration differs from the fixed protocol")
_check(spec.get("precision", "float64") == "float64", "Full-graph oracle is float64 only")
_check(spec.get("all_requests_at_tick", 0) == 0 and not spec.get("apl_overlay", False) and not spec.get("training", False),
"This oracle does not permit altered input timing, APL, or training")
budget_root = Path(spec.get("budget_root", os.path.commonpath([str(case_dir), str(graph_dir), str(out_dir)]))).resolve()
budget_limit = int(spec.get("output_budget_bytes", 15 * 1024 ** 3))
_check(budget_limit > 0, "Output budget must be positive")
_check(out_dir.is_relative_to(budget_root), "Oracle output must be inside its budget root")
out_dir.mkdir(parents=True, exist_ok=False)
memmaps = []
last_complete_tick = -1
saved_preferences = None
old_dt = None
try:
_budget(budget_root, budget_limit, reserve=steps * N_NEURONS * 20 + N_EDGES * 8 + (64 << 20))
versions = {name: importlib.metadata.version(name)
for name in ("brian2", "numpy", "pandas", "pyarrow", "sympy", "scipy", "cython")}
_save(out_dir / "environment.json", {"python": sys.version, "platform": platform.platform(),
"packages": versions, "precision": "float64", "codegen": "numpy"})
for name, required in {"brian2": "2.5.1", "numpy": "1.22.3", "sympy": "1.12"}.items():
_check(versions[name] == required, f"{name} must be pinned to {required}")
import brian2 as b2
from brian2.core.base import BrianObject
ae = _load_checked_module(AE_ORACLE_PATH, AE_ORACLE_SHA256, "phase4af_readonly_ae_helpers")
author = _load_checked_module(AUTHOR_PATH, AUTHOR_SHA256, "phase4af_pinned_author")
arrays = _graph_inputs(graph_dir)
graph_hashes = {name: _digest(graph_dir / name) for name in (
"nodes.npy", "edge_id.npy", "src.npy", "dst.npy", "signed_raw.npy", "ptr.npy",
"completeness.csv", "connectivity.parquet")}
_save(out_dir / "input-audit.json", {"spec": spec, "spec_sha256": _digest(case_dir / "spec.json"),
"graph_hashes": graph_hashes, "author_sha256": AUTHOR_SHA256,
"sealed_ae_oracle_sha256": AE_ORACLE_SHA256,
"budget_root": str(budget_root), "budget_limit_bytes": budget_limit,
"source_order": "canonical CSR; local edge index equals original edge_id",
"common_interface_preserved": True, "poi_used": False, "apl_overlay": False, "training": False})
saved_preferences = {key: b2.prefs[key] for key in (
"codegen.target", "core.default_float_dtype", "legacy.refractory_timing")}
old_dt = b2.defaultclock.dt
b2.start_scope()
b2.prefs["codegen.target"] = "numpy"
b2.prefs["core.default_float_dtype"] = np.float64
b2.prefs["legacy.refractory_timing"] = False
b2.defaultclock.dt = .1 * b2.ms
parameters = dict(author.default_params)
_save(out_dir / "parameter-checks.json", ae._parameters_match(parameters, spec["model"], b2))
neu, syn, spikes = author.create_model(graph_dir / "completeness.csv",
graph_dir / "connectivity.parquet", parameters)
_check(len(neu) == N_NEURONS and len(syn) == N_EDGES, "Author model did not load the whole graph")
_check(all(np.dtype(neu.variables[key].dtype) == np.float64 for key in ("v", "g")) and
np.dtype(syn.variables["w"].dtype) == np.float64, "Author state/weight dtype is not float64")
weights = _new_memmap(out_dir / "weights.npy", "<f8", (N_EDGES,))
memmaps.append(weights)
width = BLOCK_BYTES // 8
w_unit = float(parameters["w_syn"] / b2.volt)
for begin in range(0, N_EDGES, width):
stop = min(N_EDGES, begin + width)
np.testing.assert_array_equal(np.asarray(syn.i[begin:stop]), arrays["src"][begin:stop])
np.testing.assert_array_equal(np.asarray(syn.j[begin:stop]), arrays["dst"][begin:stop])
actual_w = np.asarray(syn.w_[begin:stop], dtype=np.float64)
expected_w = arrays["signed_raw"][begin:stop] * w_unit
_check(np.array_equal(actual_w.view(np.uint64), expected_w.view(np.uint64)),
"Author weights differ from unchanged signed raw times the original SI w_syn")
weights[begin:stop] = actual_w
weights.flush()
v_history = _new_memmap(out_dir / "v-history.float64.npy", "<f8", (steps, N_NEURONS))
g_history = _new_memmap(out_dir / "g-history.float64.npy", "<f8", (steps, N_NEURONS))
ref_history = _new_memmap(out_dir / "ref-history.int32.npy", "<i4", (steps, N_NEURONS))
memmaps.extend([v_history, g_history, ref_history])
_save(out_dir / "output-schema.json", {
"history_shape": [steps, N_NEURONS], "v_file": "v-history.float64.npy",
"g_file": "g-history.float64.npy", "ref_file": "ref-history.int32.npy",
"voltage_unit": "SI volt", "v_g_dtype": "little-endian float64", "ref_dtype": "little-endian int32",
"event_file": "events.bin", "event_record": [["step", "<i4"], ["neuron_idx", "<i4"]],
"event_order": "tick then canonical neuron_idx", "queue_file": "queue.jsonl",
"queue_joint_bytes": "b'phase4af-queue-v1\\0' + struct.pack('<qq',tick,count) + all ordered <i8 edge IDs + all uint8 acceptance bits",
"queue_separate_hashes": "SHA256 of each pure byte sequence without header",
"complete_marker": "summary.json with complete=true; partial memmaps alone are not a completed trial"})
_budget(budget_root, budget_limit, reserve=64 << 20)
delay_steps = int(round(float(parameters["t_dly"] / b2.ms) / .1))
refractory_steps = int(round(float(parameters["t_rfc"] / b2.ms) / .1))
_check(delay_steps == 18 and refractory_steps == 22, "Unexpected fixed delay or refractory duration")
queue_len = delay_steps + 1
due_counts = np.zeros(queue_len, dtype=np.int64)
spike_ring = [np.empty(0, dtype=np.int32) for _ in range(queue_len)]
outdegree = np.diff(arrays["ptr"])
request_count = np.zeros(steps, dtype=np.int32)
applicable_count = np.zeros(steps, dtype=np.int32)
rejected_count = np.zeros(steps, dtype=np.int32)
spike_count = np.zeros(steps, dtype=np.int32)
total_counts = np.zeros(N_NEURONS, dtype=np.int32)
request_seen = np.zeros(steps, dtype=np.uint8)
seed_local = np.asarray(seeds, dtype=np.int32)
permitted_seed = np.empty(0, dtype=np.int32)
request_rows = []
records = {"rows": [], "peek_calls": 0, "advance_calls": 0, "installations": []}
total_events = 0
v_raw = neu.variables["v"].get_value()
g_raw = neu.variables["g"].get_value()
last_raw = neu.variables["lastspike"].get_value()
available_raw = neu.variables["not_refractory"].get_value()
last_ticks = np.zeros(N_NEURONS, dtype=np.int64)
def tick_now():
return int(round(float(neu.clock.t / b2.second) / .0001))
with ExitStack() as stack:
queue_file = stack.enter_context((out_dir / "queue.jsonl").open("x", encoding="utf-8"))
event_file = stack.enter_context((out_dir / "events.bin").open("xb"))
tick_file = stack.enter_context((out_dir / "ticks.jsonl").open("x", encoding="utf-8"))
requests_file = stack.enter_context((out_dir / "requests.csv").open("x", encoding="utf-8", newline=""))
request_writer = csv.DictWriter(requests_file,
fieldnames=["tick", "neuron_idx", "attempted", "accepted", "rejected"])
request_writer.writeheader()
def request_operation():
nonlocal permitted_seed
tick = tick_now()
_check(0 <= tick < steps and not request_seen[tick], "Invalid/duplicate common request tick")
request_seen[tick] = 1
if tick == 0 and len(seed_local):
available = available_raw[seed_local].copy()
permitted_seed = seed_local[available]
request_count[tick] = len(seed_local)
applicable_count[tick] = len(permitted_seed)
rejected_count[tick] = len(seed_local) - len(permitted_seed)
for node, ok in zip(seed_local, available):
request_rows.append({"tick": 0, "neuron_idx": int(node), "attempted": 1,
"accepted": int(ok), "rejected": int(not ok)})
neu.v[permitted_seed] = parameters["v_th"] + 1 * b2.mV
def record_end():
nonlocal last_complete_tick, total_events
tick = tick_now()
_check(tick == last_complete_tick + 1 and tick < steps, "End-state ticks are not consecutive")
_check(request_seen[tick] == 1 and len(records["rows"]) == tick + 1,
"End state lacks its request/queue observation")
_check(np.isfinite(v_raw).all() and np.isfinite(g_raw).all(), "Author state became nonfinite")
seen = np.isfinite(last_raw) & (last_raw >= 0)
last_ticks.fill(0)
last_ticks[seen] = np.rint(last_raw[seen] / .0001).astype(np.int64)
ref = np.where(seen, np.maximum(refractory_steps - (tick - last_ticks), 0), 0).astype(np.int32)
_check(np.array_equal(ref == 0, available_raw), "Integer refractory history disagrees with Brian")
v_history[tick] = v_raw
g_history[tick] = g_raw
ref_history[tick] = ref
active = np.asarray(neu.spikes, dtype=np.int32)
_check(not len(active) or np.all(np.diff(active) > 0), "Spike order is not canonical and unique")
spike_count[tick] = len(active)
total_counts[active] += 1
if len(active):
rows = np.empty(len(active), dtype=EVENT_DTYPE)
rows["step"] = tick
rows["neuron_idx"] = arrays["nodes"][active]
event_file.write(memoryview(rows).cast("B"))
total_events += len(active)
spike_ring[tick % queue_len] = active.copy()
due_counts[(tick + delay_steps) % queue_len] = int(outdegree[active].sum(dtype=np.int64))
if tick == 0:
_check(np.isin(permitted_seed, active).all(), "An applicable initial request failed to create a real spike")
for row in request_rows:
request_writer.writerow(row)
queue_row = records["rows"][tick]
tick_record = {"tick": tick, "spikes": len(active), "requests": int(request_count[tick]),
"applicable_requests": int(applicable_count[tick]), "ref_rejected_requests": int(rejected_count[tick]),
"due_arrivals": queue_row["count"], "accepted_arrivals": queue_row["accepted_count"],
"event_sha256": hashlib.sha256(np.ascontiguousarray(arrays["nodes"][active], dtype="<i4").tobytes()).hexdigest()}
tick_file.write(json.dumps(tick_record, sort_keys=True) + "\n")
# Mapped files already reserve their full logical size. This
# checks the complete new experiment tree, not only this case.
_budget(budget_root, budget_limit, reserve=1 << 20)
last_complete_tick = tick
if (tick + 1) % 50 == 0 or tick + 1 == steps:
for mapped in (v_history, g_history, ref_history):
mapped.flush()
for handle in (queue_file, event_file, tick_file, requests_file):
handle.flush()
if (tick + 1) % 100 == 0 or tick + 1 == steps:
print(json.dumps({"oracle": spec["name"], "completed_ticks": tick + 1,
"spikes": total_events}), flush=True)
class QueueInstaller(BrianObject):
def __init__(self):
super().__init__(clock=neu.clock, when="end", order=1_000_000,
name="zz_fullgraph_queue_installer")
def before_run(self, run_namespace):
installed = []
for code in syn.pre.code_objects:
if "_queue" not in code.namespace or "_queue.peek()" not in str(code.code):
continue
original = code.namespace["_queue"]
if isinstance(original, _StreamingQueueProxy):
original = original._original_queue
code.namespace["_queue"] = _StreamingQueueProxy(
original, tick_now, lambda: available_raw, arrays, due_counts,
spike_ring, delay_steps, queue_file, records)
installed.append(code.name)
_check(len(installed) == 1, "Expected one original synaptic queue consumer")
records["installations"].append(installed[0])
request_op = b2.NetworkOperation(request_operation, clock=neu.clock,
when="before_thresholds", order=0, name="oracle_common_requests")
end_op = b2.NetworkOperation(record_end, clock=neu.clock,
when="end", order=0, name="oracle_streaming_end")
network = b2.Network(neu, syn, spikes, request_op, end_op, QueueInstaller())
_save(out_dir / "schedule.json", {"schedule": list(network.schedule),
"summary": str(network.scheduling_summary())})
network.run(0 * b2.ms, namespace={})
_save(out_dir / "compiled-code.json", ae._code_records(network))
_check(last_complete_tick == -1 and not records["rows"] and not request_seen.any(),
"Zero-duration compilation unexpectedly advanced a trial")
# Exactly one positive-duration call. Flushing files never resets a
# clock, reruns before_run, or changes the in-flight delay queue.
network.run(steps * .1 * b2.ms, namespace={})
_save(out_dir / "generated-code.json", ae._code_records(network))
_check(last_complete_tick == steps - 1 and np.all(request_seen == 1), "Trial stopped before all ticks completed")
_check(records["peek_calls"] == records["advance_calls"] == steps, "Queue observation was incomplete")
_check(total_events == int(spikes.num_spikes) and np.array_equal(total_counts, np.asarray(spikes.count[:])),
"Sparse streamed events disagree with original SpikeMonitor counts")
_check(np.all(np.asarray(neu.rfc[:] / b2.ms) == float(parameters["t_rfc"] / b2.ms)),
"Original refractory configuration changed")
for begin in range(0, N_EDGES, width):
stop = min(N_EDGES, begin + width)
_check(np.array_equal(np.asarray(syn.w_[begin:stop]).view(np.uint64),
np.asarray(weights[begin:stop]).view(np.uint64)),
"Original synaptic weights changed during execution")
events = (np.memmap(out_dir / "events.bin", dtype=EVENT_DTYPE, mode="r", shape=(total_events,))
if total_events else np.empty(0, dtype=EVENT_DTYPE))
_check((out_dir / "events.bin").stat().st_size == total_events * EVENT_DTYPE.itemsize,
"Sparse event file length is inconsistent")
event_block = BLOCK_BYTES // 8
for begin in range(0, total_events, event_block):
stop = min(total_events, begin + event_block)
np.testing.assert_array_equal(events["neuron_idx"][begin:stop],
np.asarray(spikes.i[begin:stop], dtype=np.int32))
native_ticks = np.rint(np.asarray(spikes.t[begin:stop] / b2.second) / .0001).astype(np.int32)
np.testing.assert_array_equal(events["step"][begin:stop], native_ticks)
_budget(budget_root, budget_limit, reserve=total_events * 8 + N_NEURONS * 24 + (8 << 20))
with (out_dir / "state.npz").open("xb") as handle:
np.savez_compressed(handle, step=events["step"], neuron_idx=events["neuron_idx"], counts=total_counts,
v=np.asarray(v_history[-1]), g=np.asarray(g_history[-1]), ref=np.asarray(ref_history[-1]))
with (out_dir / "tick-summary.npz").open("xb") as handle:
np.savez_compressed(handle, request_count=request_count, applicable_count=applicable_count,
rejected_count=rejected_count, spike_count=spike_count,
arrival_count=np.asarray([row["count"] for row in records["rows"]], dtype=np.int64),
accepted_arrival_count=np.asarray([row["accepted_count"] for row in records["rows"]], dtype=np.int64))
coefficients = ae._coefficients(neu, parameters, b2, .0001)
_save(out_dir / "coefficients.json", coefficients)
_save(out_dir / "queue-audit.json", {"peek_calls": records["peek_calls"],
"advance_calls": records["advance_calls"], "installations": records["installations"],
"due_count_matches_presynaptic_spikes_and_original_outdegrees": True,
"acceptance_observed_after_thresholds_before_synaptic_writes": True,
"same_peek_object_returned": True, "all_other_queue_methods_forwarded": True,
"original_order_not_sorted_or_filtered": True, "hash_header_hex": QUEUE_HEADER.hex(),
"edge_id_identity_verified_for_all_edges": True,
"fullgraph_unwrapped_repeat_performed": False,
"prior_evidence": "Phase4AE exact wrapped/unwrapped checks; current bounded-hash adapter preserves the same forwarding contract",
"limit": "Current full-graph streaming proxy is audited by forwarding logic and per-tick counts; no separate unwrapped full-graph run is claimed."})
_check(_digest(AUTHOR_PATH) == AUTHOR_SHA256 and _digest(AE_ORACLE_PATH) == AE_ORACLE_SHA256,
"A sealed author/dependency source changed during execution")
for name, expected in graph_hashes.items():
_check(_digest(graph_dir / name) == expected, f"Prepared graph input changed during execution: {name}")
for mapped in memmaps:
mapped.flush()
current_bytes = _budget(budget_root, budget_limit, reserve=1 << 20)
summary = {"name": spec["name"], "complete": True, "backend": "Brian2 2.5.1 numpy",
"precision": "float64", "nodes": N_NEURONS, "edges": N_EDGES,
"steps": steps, "completed_ticks": last_complete_tick + 1, "spikes": total_events,
"steps_expected": steps, "steps_completed": last_complete_tick + 1, "neurons": N_NEURONS,
"pass_all": True, "pass_all_scope": "CPU input, streaming, and internal self-checks only; GPU equivalence is not assessed here",
"requests": int(request_count.sum()), "applicable_requests": int(applicable_count.sum()),
"ref_rejected_requests": int(rejected_count.sum()),
"due_arrivals": sum(row["count"] for row in records["rows"]),
"accepted_arrivals": sum(row["accepted_count"] for row in records["rows"]),
"single_continuous_run": True, "source_and_graph_inputs_unchanged": True,
"output_budget_root": str(budget_root), "output_bytes_before_summary": current_bytes,
"output_budget_bytes": budget_limit, "weights_sha256": _digest(out_dir / "weights.npy"),
"queue_sha256": _digest(out_dir / "queue.jsonl"), "events_sha256": _digest(out_dir / "events.bin"),
"no_training": True, "no_apl_overlay": True, "no_poi": True,
"scope": "Only this fixed short numerical trajectory; not olfactory CAL, long-run equivalence, physiological validation, or learning.",
"pending_arrivals_limit": "Queue arrivals beyond the final recorded tick are outside this comparison window."}
_save(out_dir / "summary.json", summary)
return summary
except Exception as error:
for mapped in memmaps:
mapped.flush()
_save(out_dir / "error.json", {"complete": False, "last_complete_tick": last_complete_tick,
"type": type(error).__name__, "message": str(error),
"partial_files_preserved": True, "input_or_duration_adjusted": False})
raise
finally:
if saved_preferences is not None:
for key, value in saved_preferences.items():
b2.prefs[key] = value
b2.defaultclock.dt = old_dt
@@ -0,0 +1,71 @@
"""Read-only canonical graph adaptation, before any neural trajectory."""
from datetime import datetime, timezone
import json
import platform
import shutil
import subprocess
import numpy as np
import pandas as pd
import yaml
from context import ROOT, MODEL, EXP, AD, AE_MODEL, AE_EXP, CFG, save, digest, sources, integrity, budget_check
def main():
assert not (EXP/'protocol').exists()
assert shutil.disk_usage(EXP.parent).free > 25*1024**3
for name in ('protocol','graph','cases','oracle','gpu','regression','audit','logs','extension-build'):
(EXP/name).mkdir(parents=True, exist_ok=False)
can = ROOT/'datasets/male-cns-v1.0/processed/neuron-only'
raw_paths = [can/'gpu/csr-indptr.int64.npy', can/'gpu/csr-indices.int32.npy',
can/'gpu/csr-weights.int64.npy', can/'neuron-index/unified-metadata.parquet',
ROOT/'models/male-cns-lif/runtime-v1/phase4c.yaml', AD/'external-resources/model.py',
AD/'manifest.json', AE_MODEL/'manifest.json', AE_EXP/'manifest.json',
AE_MODEL/'gpu_backend.py',AE_MODEL/'author_ext.cpp',AE_MODEL/'author_ext.cu',AE_MODEL/'oracle_stress.py']
save(EXP/'protocol/preregistered.json', dict(timestamp_utc=datetime.now(timezone.utc).isoformat(),
config=CFG, sources=sources(), before_model_execution=True,
inputs=[dict(path=str(p.relative_to(ROOT)),sha256=digest(p)) for p in raw_paths],
parent_commit=subprocess.check_output(['git','-C',str(ROOT),'rev-parse','HEAD'],text=True).strip()))
save(EXP/'regression/old-integrity-before.json',integrity())
ptr = np.load(raw_paths[0], mmap_mode='r'); dst = np.load(raw_paths[1], mmap_mode='r')
raw = np.load(raw_paths[2], mmap_mode='r'); meta = pd.read_parquet(raw_paths[3]).sort_values('neuron_idx').reset_index(drop=True)
n,e = CFG['neurons'],CFG['edges']
assert ptr.dtype == np.int64 and dst.dtype == np.int32 and raw.dtype == np.int64
assert len(meta)==n and np.array_equal(meta.neuron_idx,np.arange(n))
assert ptr.shape==(n+1,) and ptr[0]==0 and ptr[-1]==e and np.all(np.diff(ptr)>=0)
assert dst.shape == raw.shape == (e,) and dst.min()>=0 and dst.max()<n
assert raw.min()>0 and int(raw.sum())==CFG['raw_weight_sum']
policy = yaml.safe_load(raw_paths[4].read_text())['sign_policy']
signs=[]; nt_counts={}
for nt in meta.neurotransmitter:
s='' if pd.isna(nt) else str(nt).strip().lower()
signs.append(policy['unknown_sign'] if not s else policy['unclear_sign'] if s=='unclear' else policy['neurotransmitter_sign'][s])
nt_counts[s or 'NULL'] = nt_counts.get(s or 'NULL',0)+1
signs=np.asarray(signs,np.int8); assert set(signs).issubset({-1,0,1})
src=np.repeat(np.arange(n,dtype=np.int32),np.diff(ptr)); signed=raw*signs[src]
edge_id=np.arange(e,dtype=np.int64);graph=EXP/'graph'
for name,array in dict(nodes=np.arange(n,dtype=np.int32),edge_id=edge_id,src=src,dst=dst,
signed_raw=signed,ptr=ptr,signs=signs).items():
with (graph/(name+'.npy')).open('xb') as f: np.save(f,array,allow_pickle=False)
pd.DataFrame({'local_index':np.arange(n,dtype=np.int32)},index=pd.Index(np.arange(n),name='global_neuron_idx')).to_csv(graph/'completeness.csv')
pd.DataFrame({'Presynaptic_Index':src,'Postsynaptic_Index':dst,
'Excitatory x Connectivity':signed,'edge_id':edge_id},copy=False).to_parquet(graph/'connectivity.parquet',index=False)
idx=np.bincount(dst,minlength=n);out=np.diff(ptr)
assert int(np.count_nonzero(src==dst))==101 and int(np.count_nonzero((idx==0)&(out==0)))==217
save(graph/'audit.json',dict(neurons=n,edges=e,raw_weight_sum=int(raw.sum()),
self_loops=int(np.count_nonzero(src==dst)),isolates=int(np.count_nonzero((idx==0)&(out==0))),
signs={str(int(s)):int(np.count_nonzero(signs==s)) for s in np.unique(signs)},neurotransmitters=nt_counts,
canonical_edge_id='original CSR slot; source sorted; no reordering, filtering, normalization or new edge',
body_mapping_input=str(raw_paths[3]),original_weights_modified=False))
model=json.loads((AD/'protocol/preregistered.json').read_text())['config']['model']
for trial in CFG['trials']:
case=EXP/'cases'/trial['name'];case.mkdir()
save(case/'spec.json',dict(**trial,dt_ms=model['dt_ms'],duration_ms=trial['steps']*model['dt_ms'],model=model,
graph_dir=str(graph),precision=CFG['precision'],all_requests_at_tick=0,apl_overlay=False,training=False,
budget_root=str(EXP),output_budget_bytes=CFG['new_output_budget_bytes']))
meta.iloc[trial['seed_nodes']][['neuron_idx','body_id','annotation_type','neurotransmitter']].to_csv(case/'seed-identities.csv',index=False)
save(EXP/'protocol/environment.json',dict(python=platform.python_version(),platform=platform.platform(),
numpy=np.__version__,free_disk_bytes=shutil.disk_usage(EXP).free))
save(EXP/'protocol/graph-case-hashes.json',[dict(path=str(p.relative_to(EXP)),sha256=digest(p),bytes=p.stat().st_size)
for root in (graph,EXP/'cases') for p in sorted(root.rglob('*')) if p.is_file()])
budget_check();print('PREPARED full canonical graph and3 fixed trials; frozen integrity PASS',flush=True)
if __name__=='__main__': main()
@@ -0,0 +1,41 @@
"""One process per CPU/GPU trial, gate the ordered fixed protocol."""
import argparse
import json
import os
import sys
import time
import traceback
from context import ROOT,MODEL,EXP,CFG,check,save,budget_check
def main():
ap=argparse.ArgumentParser();ap.add_argument('backend',choices=['oracle','gpu']);ap.add_argument('case')
a=ap.parse_args();check();budget_check()
names=[x['name'] for x in CFG['trials']];assert a.case in names
position=names.index(a.case)
if position:
previous=json.loads((EXP/'audit'/('completed-'+names[position-1]+'.json')).read_text())
assert previous['pass_all'], 'Previous fixed trial failed; do not continue'
case=EXP/'cases'/a.case;out=EXP/a.backend/a.case
assert not out.exists(), 'Never overwrite an executed trial'
start=time.perf_counter()
try:
if a.backend=='oracle':
from oracle_full import run_case
result=run_case(case,out)
else:
os.environ['CUDA_VISIBLE_DEVICES']=CFG['gpu_uuid'];os.environ['CUDA_HOME']='/usr/local/cuda'
os.environ['PATH']=str(ROOT/'models/male-cns-lif/.venv/bin')+':/usr/local/cuda/bin:'+os.environ['PATH']
os.environ['MAX_JOBS']='2';os.environ['TORCH_CUDA_ARCH_LIST']='8.9'
sys.path.insert(0,str(ROOT/'models/male-cns-lif/terminal-scope-diagnostic-v1'))
from diagnostic import queues
queues()
from gpu_full import run_gpu
result=run_gpu(case,out,EXP/'oracle'/a.case,EXP/'extension-build')
save(EXP/'audit'/(a.backend+'-'+a.case+'.json'),dict(case=a.case,backend=a.backend,
wall_seconds_including_validation=time.perf_counter()-start,result=result))
budget_check();print('COMPLETED',a.backend,a.case,json.dumps(result),flush=True)
except Exception as exc:
save(EXP/'audit'/('error-'+a.backend+'-'+a.case+'.json'),dict(error=str(exc),type=type(exc).__name__,traceback=traceback.format_exc()))
raise
if __name__=='__main__':main()
@@ -0,0 +1,37 @@
"""Pure audit-format tests; no neural trajectory or GPU required."""
import hashlib
import struct
import unittest
import numpy as np
from oracle_full import queue_summary,QUEUE_HEADER
from compare_trial import bit_equal
class ProtocolTests(unittest.TestCase):
def test_queue_chunk_independence_and_readonly(self):
edges=np.array([0,4,2,2,19000000],dtype=np.int64);mask=np.array([1,0,1,1,0],dtype=np.uint8)
edges.flags.writeable=False;mask.flags.writeable=False
expected=hashlib.sha256(QUEUE_HEADER+struct.pack('<qq',18,len(edges))+edges.astype('<i8').tobytes()+mask.tobytes()).hexdigest()
a=queue_summary(18,edges,mask,chunk_bytes=16)
b=queue_summary(18,edges,lambda x,y:mask[x:y],chunk_bytes=24)
self.assertEqual(a,b);self.assertEqual(a['joint_sha256'],expected)
self.assertEqual(a['accepted_count'],3);self.assertEqual(a['count'],5)
self.assertNotEqual(a['joint_sha256'],queue_summary(19,edges,mask)['joint_sha256'])
def test_empty_coverage(self):
e=np.empty(0,np.int64);m=np.empty(0,np.uint8)
a,b=queue_summary(0,e,m),queue_summary(1,e,m)
self.assertEqual(a['count'],0);self.assertEqual(a['ordered_edge_sha256'],hashlib.sha256(b'').hexdigest())
self.assertNotEqual(a['joint_sha256'],b['joint_sha256'])
def test_order_and_guard_not_ignored(self):
e=np.array([5,1,9],np.int64);m=np.array([1,1,0],np.uint8)
original=queue_summary(20,e,m)
self.assertNotEqual(original['joint_sha256'],queue_summary(20,e[::-1],m)['joint_sha256'])
self.assertNotEqual(original['joint_sha256'],queue_summary(20,e,1-m)['joint_sha256'])
def test_signed_zero_is_not_equal(self):
self.assertFalse(bit_equal(np.array([0.],np.float64),np.array([-0.],np.float64)))
self.assertFalse(bit_equal(np.array([1.],np.float64),np.array([1.],np.float32)))
self.assertTrue(bit_equal(np.array([1.,-0.],np.float64),np.array([1.,-0.],np.float64)))
if __name__=='__main__':unittest.main()
@@ -0,0 +1,33 @@
"""Old deterministic reference regression; new output, unchanged old model."""
import os
import sys
import time
import numpy as np
import pandas as pd
import yaml
from context import ROOT,EXP,CFG,save,check
def main():
check();assert not (EXP/'regression/frozen-five.json').exists()
rtpath=ROOT/'models/male-cns-lif/deterministic-runtime-v2';pc=yaml.safe_load((rtpath/'phase4p.yaml').read_text())
os.environ['CUDA_VISIBLE_DEVICES']=CFG['gpu_uuid'];os.environ['CUDA_HOME']='/usr/local/cuda'
os.environ['PATH']=str(ROOT/'models/male-cns-lif/.venv/bin')+':/usr/local/cuda/bin:'+os.environ['PATH']
os.environ['MAX_JOBS']='2';os.environ['TORCH_CUDA_ARCH_LIST']='8.9'
sys.path.insert(0,str(ROOT/'models/male-cns-lif/terminal-scope-diagnostic-v1'))
from diagnostic import queues
queues();sys.path.insert(0,str(ROOT/'models/male-cns-lif'));sys.path.insert(0,str(rtpath))
import torch,phase4p
assert torch.cuda.mem_get_info()[0]>2*1024**3
torch.cuda.reset_peak_memory_stats();t0=time.perf_counter();rt=phase4p.CanonicalRuntime(pc,rtpath,EXP);rows=[]
for spec in pc['reference_trials']:
queues();base=ROOT/'experiments/phase4p-deterministic-runtime/phase4c-v2/reference'/spec['name']
ev=pd.read_parquet(base/'spike-events.parquet')
ref=dict(step=ev.step.to_numpy(),neuron_idx=ev.neuron_idx.to_numpy(),counts=np.load(base/'per-neuron-spike-counts.int32.npy'),v=np.load(base/'final-voltage.float32.npy'),g=np.load(base/'final-g.float32.npy'))
r=rt.run(np.flatnonzero(rt.pops[spec['population']]),spec['rate_hz'],spec['seed'],spec['duration_ms'])
row=dict(trial=spec['name'],exact={k:bool(np.array_equal(v,r[k])) for k,v in ref.items()},wall_seconds=r['metrics']['wall_seconds'])
rows.append(row);print(row,flush=True)
save(EXP/'regression/frozen-five.json',dict(pass_all=all(all(r['exact'].values()) for r in rows),trials=rows,
wall_seconds=time.perf_counter()-t0,peak_allocated_vram=int(torch.cuda.max_memory_allocated()),oracle='old frozen deterministic-v2 only'))
assert all(all(r['exact'].values()) for r in rows)
if __name__=='__main__':main()