Seal Phase4AF exact short fullgraph author GPU gate and APL integration audit
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
# Phase 4AF: full-graph author numerical gate
|
||||
|
||||
The independent branch is authorized by the human; frozen models are not replaced.
|
||||
Use the complete canonical 166,700-neuron/25,582,938-edge graph, every original
|
||||
integer synaptic-contact count, the frozen source transmitter signs and W_syn
|
||||
0.275 mV/contact. Do not remove isolates, self-loops or any existing edge.
|
||||
|
||||
Float64 is the unmodified author's default, not a precision chosen for activity.
|
||||
The archived original create_model, Brian2 2.5.1/NumPy1.22.3 and its actual scalar
|
||||
coefficients define the oracle. Use unchanged Phase4AE CUDA math as read-only
|
||||
dependency, not an approximate solver. Initial state is v=-52mV, g=0, available;
|
||||
all cells retain the original 22-tick refractory rule and delay is exactly18ticks.
|
||||
The common deterministic stimulus requests v=threshold+1mV at before_thresholds
|
||||
only if not refractory. This is not original poi and not physiological input.
|
||||
|
||||
Preregister three cases: no request for10ms; neuron1381 requested once at tick0
|
||||
for50ms; [1197,1381,1511,82312,133666] requested once at tick0 for50ms.
|
||||
Each CPU case uses one fresh process, zero-time compilation then one continuous
|
||||
nonzero Network.run. Full end-of-tick V/g/ref are streamed to .npy memory maps;
|
||||
spikes remain sparse events. GPU compares every tick bitwise against these maps.
|
||||
No dense spike matrix, no conditioning, no learning and no APL overlay in this
|
||||
isolated numerical gate. The omission of APL is explicit, not a revised calibrated
|
||||
brain. Any later APL branch integration requires its own semantic gate.
|
||||
|
||||
Observe the actual Brian queue by forwarding the original peek result unchanged.
|
||||
SHA-256 per tick records raw original edge order and the original postsynaptic
|
||||
not_refractory acceptance mask. GPU digests the source-ordered expansion before
|
||||
stable target grouping. Joint hash has the versioned protocol header, little-endian
|
||||
tick/count, contiguous little-endian int64 IDs followed by uint8 accepted flags.
|
||||
Different targets may be scheduled in parallel, but each target's accumulation
|
||||
order and exact incoming old g are preserved. No separate reduction changes.
|
||||
|
||||
Compare sparse event sequence, counts, every V/g bit, refractory, request
|
||||
accepted/rejected and queue edge/guard digests. Stop after the first failed trial;
|
||||
preserve a failure diagnostic, do not widen tolerance, choose alternate input or
|
||||
interpret low activity as success. Resource stops are incomplete, not scientific
|
||||
FAILs. New output budget15GiB; stream states and do not dump all arrivals.
|
||||
|
||||
Before and after: verify original frozen inventory, subsequent AA--AE seals and
|
||||
the background evidence seal. Re-run the five old deterministic-v2 reference trials
|
||||
into this experiment only. These historical regressions do not certify new model
|
||||
learning or long-run equivalence. Record source hashes, graph/input hashes,
|
||||
environment, generated code, code commit, wall time and diagnostic peak VRAM.
|
||||
All outputs and source are sealed with manifests and SHA-256 after completion.
|
||||
@@ -0,0 +1,12 @@
|
||||
64413d57ea8c0222ce40f007078b1b2ce7b79986d846878e4ce408e87446b2bf PROTOCOL.md
|
||||
2c81cb37598cce2ae0688046fd944bec34ea57824944379bfbf7e778283ab4ad compare_trial.py
|
||||
9acd8177d94daf92df3435d2df7c854a0b86e8dee65d42f3ec25d084c9fba8f0 config.json
|
||||
fe221d4470f14a094bc2b02b9763d542c091a2a069c951c5f833a0639c39659b context.py
|
||||
81885c07c053444cb0b03634b4ddc09b12bef2f237828db2dcf203653ac4b704 finalize.py
|
||||
895ee595d63ee8650b6aad662f831117af78cb6216437c001c8c7ba69a7b92e7 gpu_full.py
|
||||
d72003d98f0f1624b12126eff50bd361c43d888f4a64d90ba9247a58976f1961 oracle_full.py
|
||||
780f26c3c829135a6f67ff03007925649ef777dc15010334830ff1f7fa109519 prepare.py
|
||||
afb8a0e8513726bd1340db56c2cff8f3615fedd0857dc611420d29e1b858c2c3 run_trial.py
|
||||
d4bc6905ea6ce4ea04aeeabfc2f99bc111114fdfd0bc54169a40edf4f95c6033 test_protocol.py
|
||||
a2a19b52c41c8c752692ff524d6f21016c6f2789c5fcabac9c78c8cadd59826e verify_frozen.py
|
||||
ead78ed89d0daebee5281ef285bbfe9d79c69b5034830609414f7b130d3c5827 manifest.json
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Independent file-level exact gate; no reliance on a stored backend PASS."""
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import numpy as np
|
||||
from context import EXP,CFG,check,save,digest
|
||||
|
||||
def jsonl(path):
|
||||
with path.open() as f: return [json.loads(line) for line in f if line.strip()]
|
||||
|
||||
def bit_equal(a,b):
|
||||
return a.shape==b.shape and a.dtype==b.dtype and np.array_equal(np.ascontiguousarray(a).view(np.uint8),np.ascontiguousarray(b).view(np.uint8))
|
||||
|
||||
def evaluate(name):
|
||||
spec=json.loads((EXP/'cases'/name/'spec.json').read_text());steps=spec['steps'];n=CFG['neurons']
|
||||
cpu=EXP/'oracle'/name;gpu=EXP/'gpu'/name;checks={};details={}
|
||||
arrays={key:np.load(cpu/filename,mmap_mode='r') for key,filename in (
|
||||
('v','v-history.float64.npy'),('g','g-history.float64.npy'),('ref','ref-history.int32.npy'))}
|
||||
expected={'v':np.dtype('float64'),'g':np.dtype('float64'),'ref':np.dtype('int32')}
|
||||
checks['history_shapes_dtypes']=all(x.shape==(steps,n) and x.dtype==expected[k] for k,x in arrays.items())
|
||||
assert checks['history_shapes_dtypes']
|
||||
hashes=jsonl(gpu/'tick-state-hashes.jsonl')
|
||||
checks['gpu_state_hash_coverage']=len(hashes)==steps and [x['tick'] for x in hashes]==list(range(steps))
|
||||
assert checks['gpu_state_hash_coverage']
|
||||
failures=[];finite=True
|
||||
for tick,row in enumerate(hashes):
|
||||
for key,value in arrays.items():
|
||||
a=np.ascontiguousarray(value[tick]);sha=hashlib.sha256(a.tobytes()).hexdigest()
|
||||
if sha!=row[key+'_sha256']:failures.append(dict(tick=tick,field=key))
|
||||
if key!='ref' and not np.isfinite(a).all():finite=False
|
||||
checks['all_tick_raw_state_hashes_exact']=not failures;checks['all_oracle_states_finite']=finite
|
||||
details['first_hash_mismatches']=failures[:10]
|
||||
with np.load(cpu/'state.npz',allow_pickle=False) as a,np.load(gpu/'state.npz',allow_pickle=False) as b:
|
||||
for key in ('step','neuron_idx','counts','v','g','ref'):
|
||||
checks['exact_'+key]=bit_equal(a[key],b[key])
|
||||
checks['event_clock_range']=bool(np.all((a['step']>=0)&(a['step']<steps)))
|
||||
checks['event_neuron_range']=bool(np.all((a['neuron_idx']>=0)&(a['neuron_idx']<n)))
|
||||
checks['event_sort']=np.array_equal(np.lexsort((a['neuron_idx'],a['step'])),np.arange(len(a['step'])))
|
||||
checks['event_unique']=len(np.unique(a['step'].astype(np.int64)*n+a['neuron_idx']))==len(a['step'])
|
||||
checks['cpu_event_counts']=np.array_equal(np.bincount(a['neuron_idx'],minlength=n),a['counts'])
|
||||
checks['gpu_event_counts']=np.array_equal(np.bincount(b['neuron_idx'],minlength=n),b['counts'])
|
||||
for key in arrays:checks['cpu_final_'+key]=bit_equal(arrays[key][-1],a[key])
|
||||
details.update(events=len(a['step']),active_neurons=int(np.count_nonzero(a['counts'])),
|
||||
max_per_neuron_count=int(a['counts'].max()),
|
||||
mean_rate_hz=float(len(a['step'])/(n*spec['duration_ms']/1000)),
|
||||
first_event_step=None if not len(a['step']) else int(a['step'][0]))
|
||||
ptr=np.load(EXP/'graph/ptr.npy',mmap_mode='r');degrees=np.diff(ptr)
|
||||
expected_arrivals=np.zeros(steps,np.int64);due=a['step'].astype(np.int64)+18
|
||||
valid=due<steps
|
||||
np.add.at(expected_arrivals,due[valid],degrees[a['neuron_idx'][valid]])
|
||||
cq,gq=jsonl(cpu/'queue.jsonl'),jsonl(gpu/'queue.jsonl')
|
||||
checks['queue_coverage']=len(cq)==len(gq)==steps and [r['tick'] for r in cq]==list(range(steps))
|
||||
checks['queue_original_order_exact']=cq==gq
|
||||
checks['all_presynaptic_edges_due']=np.array_equal(expected_arrivals,np.asarray([r['count'] for r in cq],np.int64))
|
||||
with (cpu/'requests.csv').open(newline='') as f:cr=list(csv.DictReader(f))
|
||||
with (gpu/'requests.csv').open(newline='') as f:gr=list(csv.DictReader(f))
|
||||
checks['request_rows_exact']=cr==gr
|
||||
checks['request_count']=len(cr)==len(spec['seed_nodes'])
|
||||
checks['actual_requests_match_protocol']=cr==[
|
||||
dict(tick='0',neuron_idx=str(node),attempted='1',accepted='1',rejected='0')
|
||||
for node in sorted(spec['seed_nodes'])]
|
||||
details['request_rows']=cr
|
||||
if cq:
|
||||
details['queue_fields']=list(cq[0])
|
||||
details['arrival_count']=sum(r['count'] for r in cq)
|
||||
details['accepted_arrival_count']=sum(r['accepted_count'] for r in cq)
|
||||
summary=json.loads((gpu/'summary.json').read_text())
|
||||
checks['backend_summary_pass']=summary['pass_all'] is True
|
||||
oracle_summary=json.loads((cpu/'summary.json').read_text())
|
||||
checks['oracle_complete']=oracle_summary['complete'] is True and oracle_summary['steps']==steps and oracle_summary['nodes']==n and oracle_summary['edges']==CFG['edges']
|
||||
checks={k:bool(v) for k,v in checks.items()}
|
||||
files=[cpu/'state.npz',gpu/'state.npz',cpu/'queue.jsonl',gpu/'queue.jsonl',gpu/'tick-state-hashes.jsonl']
|
||||
return dict(case=name,pass_all=all(checks.values()),checks=checks,details=details,
|
||||
evidence_sha256={str(p.relative_to(EXP)):digest(p) for p in files})
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument('case');a=ap.parse_args();check()
|
||||
assert a.case in [t['name'] for t in CFG['trials']]
|
||||
result=evaluate(a.case);save(EXP/'audit'/('completed-'+a.case+'.json'),result)
|
||||
print(json.dumps(result,ensure_ascii=False),flush=True)
|
||||
if not result['pass_all']:raise SystemExit(1)
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"phase": "4AF",
|
||||
"name": "author-fullgraph-lab-v1",
|
||||
"human_authorization": "允许独立新模型分支测试动力学假设",
|
||||
"precision": "float64",
|
||||
"precision_reason": "Original Brian author default; not selected by network activity. Phase4AE float32 is not universally event-equivalent to float64.",
|
||||
"neurons": 166700,
|
||||
"edges": 25582938,
|
||||
"raw_weight_sum": 124177617,
|
||||
"gpu_uuid": "GPU-79f3a85d-adce-7833-7003-6488140ca596",
|
||||
"trials": [
|
||||
{"name": "no_input", "steps": 100, "seed_nodes": []},
|
||||
{"name": "single", "steps": 500, "seed_nodes": [1381]},
|
||||
{"name": "five", "steps": 500, "seed_nodes": [1197, 1381, 1511, 82312, 133666]}
|
||||
],
|
||||
"all_requests_at_tick": 0,
|
||||
"apl_overlay": false,
|
||||
"training": false,
|
||||
"candidate_generation": false,
|
||||
"gate": "Every timestep float64 V/g bitwise identical; refractory, sparse events, counts, request delivery, original queue ordered edge/acceptance digests exact.",
|
||||
"stop_on_first_trial_failure": true,
|
||||
"new_output_budget_bytes": 16106127360,
|
||||
"limits": "Only three short trajectories. No long-run/full-protocol equivalence, biological fidelity, olfactory CAL or learning claim. No performance claim."
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
MODEL = Path(__file__).resolve().parent
|
||||
ROOT = Path('/data/digital-drosophila')
|
||||
EXP = ROOT/'experiments/phase4af-author-fullgraph'
|
||||
AD = ROOT/'experiments/phase4ad-author-semantics'
|
||||
AE_MODEL = ROOT/'models/male-cns-lif/author-runtime-lab-v1'
|
||||
AE_EXP = ROOT/'experiments/phase4ae-author-runtime'
|
||||
CFG = json.loads((MODEL/'config.json').read_text(encoding='utf-8'))
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as f:
|
||||
for b in iter(lambda: f.read(8<<20), b''): h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
def save(path, value):
|
||||
path = Path(path); path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open('x', encoding='utf-8') as f:
|
||||
json.dump(value, f, indent=2, ensure_ascii=False, allow_nan=False); f.write('\n')
|
||||
|
||||
def sources():
|
||||
return {p.name: digest(p) for p in sorted(MODEL.iterdir())
|
||||
if p.is_file() and p.name not in ('manifest.json', 'checksums.sha256')}
|
||||
|
||||
def check():
|
||||
p = json.loads((EXP/'protocol/preregistered.json').read_text())
|
||||
assert sources() == p['sources'], 'Source changed after preregistration'
|
||||
for row in p['inputs']:
|
||||
assert digest(ROOT/row['path']) == row['sha256'], row['path']
|
||||
for row in json.loads((EXP/'protocol/graph-case-hashes.json').read_text()):
|
||||
assert digest(EXP/row['path']) == row['sha256'], row['path']
|
||||
return p
|
||||
|
||||
def integrity():
|
||||
spec = importlib.util.spec_from_file_location('previous_ae_context', AE_MODEL/'context.py')
|
||||
previous = importlib.util.module_from_spec(spec); spec.loader.exec_module(previous)
|
||||
result = previous.integrity()
|
||||
import sys
|
||||
sys.path.insert(0, str(ROOT/'models/male-cns-lif/terminal-scope-diagnostic-v1'))
|
||||
from diagnostic import manifest_check
|
||||
for p in (AE_MODEL, AE_EXP):
|
||||
result[str(p.relative_to(ROOT))] = manifest_check(p, digest)
|
||||
assert all(r['pass'] for r in result.values())
|
||||
return result
|
||||
|
||||
def output_bytes():
|
||||
return sum(p.stat().st_size for p in EXP.rglob('*') if p.is_file())
|
||||
|
||||
def budget_check():
|
||||
assert output_bytes() < CFG['new_output_budget_bytes'], 'New output budget exceeded'
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Seal the fixed short-trial audit, never promote a learned model."""
|
||||
import json
|
||||
from context import ROOT,MODEL,EXP,CFG,check,save,digest,integrity,output_bytes,budget_check
|
||||
from compare_trial import evaluate
|
||||
|
||||
def seal(root):
|
||||
assert not (root/'manifest.json').exists() and not (root/'checksums.sha256').exists()
|
||||
rows=[dict(path=str(p.relative_to(root)),bytes=p.stat().st_size,sha256=digest(p))
|
||||
for p in sorted(root.rglob('*')) if p.is_file() and '__pycache__' not in p.parts]
|
||||
save(root/'manifest.json',dict(root=str(root),files=rows))
|
||||
with (root/'checksums.sha256').open('x') as f:
|
||||
for r in rows:f.write(r['sha256']+' '+r['path']+'\n')
|
||||
f.write(digest(root/'manifest.json')+' manifest.json\n')
|
||||
|
||||
def main():
|
||||
check();budget_check();rows=[];missing=[]
|
||||
for trial in CFG['trials']:
|
||||
p=EXP/'audit'/('completed-'+trial['name']+'.json')
|
||||
if not p.exists():missing.append(trial['name']);continue
|
||||
stored=json.loads(p.read_text());actual=evaluate(trial['name'])
|
||||
assert stored==actual,'Saved comparison no longer agrees with raw evidence'
|
||||
rows.append(actual)
|
||||
old=json.loads((EXP/'regression/frozen-five.json').read_text())
|
||||
after=integrity();save(EXP/'regression/old-integrity-after.json',after)
|
||||
errors=[str(p.relative_to(EXP)) for p in sorted((EXP/'audit').glob('error-*.json'))]
|
||||
all_present=len(rows)==len(CFG['trials']) and not missing
|
||||
passed=all_present and not errors and all(r['pass_all'] for r in rows) and old['pass_all'] and all(r['pass'] for r in after.values())
|
||||
status='PASS-SHORT-FULLGRAPH-NUMERICAL-GATE' if passed else 'FAIL-SHORT-FULLGRAPH-NUMERICAL-GATE'
|
||||
if errors:status='INCOMPLETE-EXECUTION-ERROR'
|
||||
metrics={r['case']:json.loads((EXP/'gpu'/r['case']/'metrics.json').read_text()) for r in rows}
|
||||
result=dict(status=status,pass_all=bool(passed),neurons=CFG['neurons'],edges=CFG['edges'],raw_weight_sum=CFG['raw_weight_sum'],
|
||||
trials_expected=len(CFG['trials']),trials_compared=len(rows),not_executed=missing,errors=errors,
|
||||
trials=rows,diagnostic_metrics=metrics,whole_brain_long_run_equivalence=False,
|
||||
apl_overlay_validated=False,olfactory_representation_validated=False,conditioning=False,
|
||||
learning_achieved=False,candidate_generated=False,old_frozen_integrity_pass=True,
|
||||
old_five_reference_pass=old['pass_all'],payload_bytes_before_manifest=output_bytes(),
|
||||
next_gate='Explicit fixed APL/author semantics integration audit, then fixed DoOR CAL without learning' if passed else 'Locate first numerical/execution discrepancy; no CAL or learning promotion')
|
||||
save(EXP/'audit/final-status.json',result)
|
||||
lines=['# Phase 4AF:短时完整图数值对照','',f'状态:`{status}`。','',
|
||||
'完整166,700节点/25,582,938边,原synaptic-contact总数124,177,617。',
|
||||
'作者默认float64;新分支只读复用4AE CUDA算术与未修改作者Brian2 create_model。',
|
||||
'无APL叠加、无Poisson forcing、无训练。只有tick0固定请求,所有原始边/符号/W_syn保留。','',
|
||||
'| trial | biological ms | events | active neurons | per-tick exact gate |','|---|---:|---:|---:|---|']
|
||||
for row in rows:
|
||||
trial=next(x for x in CFG['trials'] if x['name']==row['case']);d=row['details']
|
||||
lines.append(f"| {row['case']} | {trial['steps']*.1:g} | {d['events']} | {d['active_neurons']} | {row['pass_all']} |")
|
||||
lines+=['','每tick比较所有细胞原始V/g bit、ref、spike events、实际输入以及原队列edge/guard顺序SHA256。',
|
||||
'完整CPU状态保留,GPU逐tick状态hash由独立文件审计再次重算核对;未保存dense spike matrix。',
|
||||
f'未执行:{missing};执行异常:{errors}。',
|
||||
'GPU wall/VRAM含逐tick同步、CPU拷贝和hash,仅是诊断成本,不是实时性能benchmark。',
|
||||
'原冻结校验与旧五组deterministic-v2参考回归另列;旧回归PASS不代表新动力学学习PASS。',
|
||||
'三个短轨迹一致也不证明长期全脑行为、生物学正确、嗅觉区分或学习成功。',
|
||||
'下一步:'+result['next_gate'],'']
|
||||
with (EXP/'REPORT.md').open('x',encoding='utf-8') as f:f.write('\n'.join(lines))
|
||||
check();seal(MODEL);seal(EXP);print(status,flush=True)
|
||||
if not passed:raise SystemExit(1)
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,369 @@
|
||||
"""Float64 full-graph numerical audit using sealed Phase 4AE CUDA operators.
|
||||
|
||||
This synchronous diagnostic compares every neuron at every tick with the CPU
|
||||
memmaps. It does not introduce a new integration rule, APL, training or tuning.
|
||||
"""
|
||||
from functools import lru_cache
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import types
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from context import ROOT
|
||||
from oracle_full import queue_summary
|
||||
|
||||
|
||||
NEURONS = 166700
|
||||
EDGES = 25582938
|
||||
COEFFICIENT_NAMES = ("gd", "c0", "cg", "cv", "threshold", "reset", "seed_volt")
|
||||
REQUEST_FIELDS = ("tick", "neuron_idx", "attempted", "accepted", "rejected")
|
||||
AE_MODEL = ROOT / "models/male-cns-lif/author-runtime-lab-v1"
|
||||
ALLOWED_CASES = {(100, ()), (500, (1381,)), (500, (1197, 1381, 1511, 82312, 133666))}
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
with Path(path).open("x", encoding="utf-8") as output:
|
||||
json.dump(value, output, indent=2, allow_nan=False)
|
||||
output.write("\n")
|
||||
|
||||
|
||||
def digest(path):
|
||||
value = hashlib.sha256()
|
||||
with Path(path).open("rb") as source:
|
||||
for block in iter(lambda: source.read(8 << 20), b""):
|
||||
value.update(block)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def array_digest(array, dtype):
|
||||
return hashlib.sha256(np.ascontiguousarray(array, dtype=dtype).tobytes()).hexdigest()
|
||||
|
||||
|
||||
def bitwise_comparison(actual, expected, floating=False):
|
||||
require(actual.shape == expected.shape and actual.dtype == expected.dtype, "Comparison shape/dtype mismatch")
|
||||
aa = actual.view(np.uint64) if floating else actual
|
||||
bb = expected.view(np.uint64) if floating else expected
|
||||
mismatch = aa != bb
|
||||
count = int(np.count_nonzero(mismatch))
|
||||
first = int(np.flatnonzero(mismatch)[0]) if count else None
|
||||
result = {"bitwise_exact": count == 0, "mismatching_neurons": count,
|
||||
"first_neuron_idx": first}
|
||||
if floating:
|
||||
finite = bool(np.isfinite(actual).all() and np.isfinite(expected).all())
|
||||
result.update(finite=finite, max_abs_volt=float(np.max(np.abs(actual - expected))) if finite else None)
|
||||
if first is not None:
|
||||
result.update(actual_bits_hex=f"{int(aa[first]):016x}", expected_bits_hex=f"{int(bb[first]):016x}")
|
||||
return result
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def sealed_backend():
|
||||
"""Load reviewed source without writing bytecode into the sealed model."""
|
||||
source = AE_MODEL / "gpu_backend.py"
|
||||
module = types.ModuleType("phase4af_sealed_4ae_gpu_backend")
|
||||
module.__file__ = str(source)
|
||||
exec(compile(source.read_bytes(), str(source), "exec"), module.__dict__)
|
||||
return module
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def load_graph(graph_directory):
|
||||
graph_dir = Path(graph_directory)
|
||||
require(graph_dir.is_absolute(), "graph_dir must be absolute")
|
||||
dtypes = {"nodes": np.int32, "edge_id": np.int64, "src": np.int32, "dst": np.int32,
|
||||
"signed_raw": np.int64, "ptr": np.int64}
|
||||
graph = {}
|
||||
for name, dtype in dtypes.items():
|
||||
value = np.load(graph_dir / (name + ".npy"), mmap_mode="r", allow_pickle=False)
|
||||
require(value.dtype == dtype and value.ndim == 1, "Invalid graph array: " + name)
|
||||
graph[name] = value
|
||||
require(graph["nodes"].shape == (NEURONS,) and
|
||||
np.array_equal(graph["nodes"], np.arange(NEURONS, dtype=np.int32)), "Graph neuron identity changed")
|
||||
require(graph["ptr"].shape == (NEURONS + 1,) and graph["ptr"][0] == 0 and
|
||||
graph["ptr"][-1] == EDGES and bool(np.all(np.diff(graph["ptr"]) >= 0)), "Invalid canonical CSR")
|
||||
for name in ("edge_id", "src", "dst", "signed_raw"):
|
||||
require(graph[name].shape == (EDGES,), "Graph edge count differs: " + name)
|
||||
require(np.array_equal(graph["src"], np.repeat(graph["nodes"], np.diff(graph["ptr"]))),
|
||||
"Canonical source rows disagree with CSR")
|
||||
for start in range(0, EDGES, 1 << 20):
|
||||
stop = min(start + (1 << 20), EDGES)
|
||||
require(np.array_equal(graph["edge_id"][start:stop], np.arange(start, stop, dtype=np.int64)),
|
||||
"Graph edge IDs are not canonical CSR row positions")
|
||||
require(bool(np.all((graph["dst"][start:stop] >= 0) & (graph["dst"][start:stop] < NEURONS))),
|
||||
"Target index outside complete graph")
|
||||
return graph
|
||||
|
||||
|
||||
def load_inputs(case_dir, oracle_dir):
|
||||
with (case_dir / "spec.json").open(encoding="utf-8") as source:
|
||||
spec = json.load(source)
|
||||
for name in ("name", "steps", "dt_ms", "duration_ms", "seed_nodes", "model", "graph_dir"):
|
||||
require(name in spec, "Missing case field: " + name)
|
||||
steps = spec["steps"]
|
||||
seeds = spec["seed_nodes"]
|
||||
require(type(steps) is int and isinstance(seeds, list) and
|
||||
all(type(node) is int for node in seeds) and len(seeds) == len(set(seeds)), "Invalid case steps/seeds")
|
||||
seeds = tuple(sorted(seeds))
|
||||
require((steps, seeds) in ALLOWED_CASES, "Case not one of the three fixed full-graph trials")
|
||||
with (oracle_dir / "summary.json").open(encoding="utf-8") as source:
|
||||
oracle_summary = json.load(source)
|
||||
require(oracle_summary.get("complete") is True and oracle_summary.get("steps") == steps and
|
||||
oracle_summary.get("completed_ticks") == steps and oracle_summary.get("precision") == "float64" and
|
||||
oracle_summary.get("nodes") == NEURONS and oracle_summary.get("edges") == EDGES,
|
||||
"CPU oracle did not seal a complete trial with the declared steps/nodes/edges")
|
||||
require(float(spec["dt_ms"]) == .1 and float(spec["duration_ms"]) == steps * .1,
|
||||
"Unexpected duration or timestep")
|
||||
model = spec["model"]
|
||||
for name in ("resting_potential_mv", "threshold_mv", "reset_potential_mv", "synaptic_delay_ms",
|
||||
"refractory_ms", "w_syn_mv_per_anatomical_synapse"):
|
||||
require(name in model and math.isfinite(float(model[name])), "Missing/nonfinite model field: " + name)
|
||||
require(float(model["synaptic_delay_ms"]) == 1.8 and float(model["refractory_ms"]) == 2.2 and
|
||||
float(model["w_syn_mv_per_anatomical_synapse"]) == .275, "Declared delay/refractory/W_syn changed")
|
||||
require(Path(spec["graph_dir"]).is_absolute(), "Case graph_dir must be absolute")
|
||||
graph = load_graph(str(Path(spec["graph_dir"]).resolve()))
|
||||
with (oracle_dir / "coefficients.json").open(encoding="utf-8") as source:
|
||||
coefficients = json.load(source)
|
||||
for name in COEFFICIENT_NAMES:
|
||||
require(name in coefficients and math.isfinite(float(coefficients[name])), "Missing/nonfinite oracle coefficient: " + name)
|
||||
require(coefficients["threshold"] == float(model["threshold_mv"]) * .001 and
|
||||
coefficients["reset"] == float(model["reset_potential_mv"]) * .001,
|
||||
"Oracle threshold/reset disagree with model declaration")
|
||||
histories = {}
|
||||
for name, dtype in (("v", np.float64), ("g", np.float64), ("ref", np.int32)):
|
||||
filename = name + "-history." + np.dtype(dtype).name + ".npy"
|
||||
value = np.load(oracle_dir / filename, mmap_mode="r", allow_pickle=False)
|
||||
require(value.shape == (steps, NEURONS) and value.dtype == dtype, "Invalid CPU state history: " + name)
|
||||
histories[name] = value
|
||||
weights = np.load(oracle_dir / "weights.npy", mmap_mode="r", allow_pickle=False)
|
||||
require(weights.dtype == np.float64 and weights.shape == (EDGES,) and bool(np.isfinite(weights).all()),
|
||||
"Invalid actual Brian2 SI weights")
|
||||
# A sign or row mismatch must not be hidden by copying the oracle weight file.
|
||||
for start in range(0, EDGES, 1 << 20):
|
||||
stop = min(start + (1 << 20), EDGES)
|
||||
require(np.array_equal(np.sign(weights[start:stop]), np.sign(graph["signed_raw"][start:stop])),
|
||||
"Oracle weight sign/zero pattern differs from graph")
|
||||
with np.load(oracle_dir / "state.npz", allow_pickle=False) as source:
|
||||
final = {name: source[name].copy() for name in source.files}
|
||||
for name, dtype in (("v", np.float64), ("g", np.float64), ("ref", np.int32), ("counts", np.int32)):
|
||||
require(final[name].dtype == dtype and final[name].shape == (NEURONS,), "Invalid CPU final state: " + name)
|
||||
require(final["step"].dtype == np.int32 and final["neuron_idx"].dtype == np.int32 and
|
||||
final["step"].ndim == 1 and final["step"].shape == final["neuron_idx"].shape, "Invalid CPU sparse events")
|
||||
require(bool(np.all((final["step"] >= 0) & (final["step"] < steps))) and
|
||||
bool(np.all(np.diff(final["step"]) >= 0)) and
|
||||
bool(np.all((final["neuron_idx"] >= 0) & (final["neuron_idx"] < NEURONS))), "CPU event index out of range")
|
||||
with (oracle_dir / "queue.jsonl").open(encoding="utf-8") as source:
|
||||
queues = [json.loads(line) for line in source if line.strip()]
|
||||
require(len(queues) == steps and [row["tick"] for row in queues] == list(range(steps)),
|
||||
"Missing, duplicate or reordered CPU queue records")
|
||||
with (oracle_dir / "requests.csv").open(encoding="utf-8", newline="") as source:
|
||||
reader = csv.DictReader(source)
|
||||
require(tuple(reader.fieldnames or ()) == REQUEST_FIELDS, "CPU request CSV schema mismatch")
|
||||
request_rows = [{name: int(row[name]) for name in REQUEST_FIELDS} for row in reader]
|
||||
require(len(request_rows) == len(seeds) and
|
||||
[row["neuron_idx"] for row in request_rows] == list(seeds) and
|
||||
all(row["tick"] == 0 and row["attempted"] == 1 and row["accepted"] in (0, 1) and
|
||||
row["rejected"] == 1 - row["accepted"] for row in request_rows), "CPU request coverage mismatch")
|
||||
return spec, graph, histories, weights, final, queues, coefficients, seeds, request_rows
|
||||
|
||||
|
||||
def run_gpu(case_dir, out_dir, oracle_dir, build_root):
|
||||
"""Execute one fixed full graph and compare every state bit, event and queue.
|
||||
|
||||
Full CPU histories remain read-only. The first failing tick saves complete
|
||||
CPU/GPU V/g/ref arrays; every tick saves exact comparison fields and hashes.
|
||||
"""
|
||||
case_dir, out_dir, oracle_dir = Path(case_dir), Path(out_dir), Path(oracle_dir)
|
||||
require(not out_dir.exists(), "Refusing existing output directory: " + str(out_dir))
|
||||
spec, graph, histories, weights, oracle_final, oracle_queues, coeff, seeds, oracle_requests = load_inputs(case_dir, oracle_dir)
|
||||
require(torch.cuda.is_available(), "CUDA unavailable")
|
||||
require(torch.cuda.device_count() == 1 and "4090" in torch.cuda.get_device_name(),
|
||||
"Only the declared single visible RTX4090 is permitted")
|
||||
out_dir.mkdir(parents=True, exist_ok=False)
|
||||
start = time.perf_counter()
|
||||
tick = -1
|
||||
rows = []
|
||||
first_failure = None
|
||||
try:
|
||||
backend = sealed_backend()
|
||||
ext, build_provenance = backend.build_extension(build_root)
|
||||
build_end = time.perf_counter()
|
||||
device = torch.device("cuda", torch.cuda.current_device())
|
||||
torch.cuda.reset_peak_memory_stats(device)
|
||||
|
||||
def upload(value, dtype):
|
||||
# Writable copies prevent any accidental write to read-only source maps.
|
||||
return torch.from_numpy(np.array(value, copy=True, order="C")).to(device=device, dtype=dtype).contiguous()
|
||||
|
||||
indptr = upload(graph["ptr"], torch.int64)
|
||||
dst = upload(graph["dst"], torch.int32)
|
||||
gpu_weights = upload(weights, torch.float64)
|
||||
rest = float(spec["model"]["resting_potential_mv"]) * .001
|
||||
v = torch.full((NEURONS,), rest, device=device, dtype=torch.float64)
|
||||
g = torch.zeros(NEURONS, device=device, dtype=torch.float64)
|
||||
ref = torch.zeros(NEURONS, device=device, dtype=torch.int32)
|
||||
counts = torch.zeros(NEURONS, device=device, dtype=torch.int32)
|
||||
requests = torch.zeros(NEURONS, device=device, dtype=torch.uint8)
|
||||
if seeds:
|
||||
requests[torch.tensor(seeds, device=device, dtype=torch.int64)] = 1
|
||||
ring = torch.zeros((19, NEURONS), device=device, dtype=torch.uint8)
|
||||
events_step, events_node = [], []
|
||||
requested_total = accepted_request_total = 0
|
||||
total_due = total_accepted = 0
|
||||
torch.cuda.synchronize(device)
|
||||
simulation_start = time.perf_counter()
|
||||
with (out_dir / "step-comparisons.jsonl").open("x", encoding="utf-8", buffering=1) as comparisons, \
|
||||
(out_dir / "queue.jsonl").open("x", encoding="utf-8", buffering=1) as queue_output, \
|
||||
(out_dir / "tick-state-hashes.jsonl").open("x", encoding="utf-8", buffering=1) as hash_output, \
|
||||
(out_dir / "requests.csv").open("x", encoding="utf-8", newline="", buffering=1) as request_output:
|
||||
request_writer = csv.DictWriter(request_output, fieldnames=REQUEST_FIELDS)
|
||||
request_writer.writeheader()
|
||||
for tick in range(spec["steps"]):
|
||||
if tick == 1:
|
||||
requests.zero_()
|
||||
requested = int(torch.count_nonzero(requests).item())
|
||||
accepted_requests = int(torch.count_nonzero((requests != 0) & (ref <= 1)).item())
|
||||
requested_total += requested
|
||||
accepted_request_total += accepted_requests
|
||||
request_rows = []
|
||||
if requested:
|
||||
requested_nodes = torch.nonzero(requests, as_tuple=False).flatten()
|
||||
accepted_mask = (ref[requested_nodes] <= 1).to(torch.uint8).cpu().numpy()
|
||||
for node, accepted in zip(requested_nodes.cpu().numpy(), accepted_mask):
|
||||
request_rows.append({"tick": tick, "neuron_idx": int(node), "attempted": 1,
|
||||
"accepted": int(accepted), "rejected": 1 - int(accepted)})
|
||||
request_writer.writerows(request_rows)
|
||||
requests_exact = request_rows == [row for row in oracle_requests if row["tick"] == tick]
|
||||
spikes = ring[tick % 19]
|
||||
ext.state_step(v, g, ref, requests, spikes, counts,
|
||||
*[float(coeff[name]) for name in COEFFICIENT_NAMES], 22)
|
||||
expanded_cpu = np.empty(0, dtype=np.int64)
|
||||
accepted_cpu = np.empty(0, dtype=np.uint8)
|
||||
if tick >= 18:
|
||||
due = ring[(tick - 18) % 19]
|
||||
sources = torch.nonzero(due, as_tuple=False).flatten().contiguous()
|
||||
if sources.numel():
|
||||
degrees = indptr[sources + 1] - indptr[sources]
|
||||
offsets = (torch.cumsum(degrees, dim=0) - degrees).contiguous()
|
||||
edge_count = int(degrees.sum().item())
|
||||
expanded = torch.empty(edge_count, device=device, dtype=torch.int64)
|
||||
ext.fill_edges(indptr, sources, offsets, expanded)
|
||||
if edge_count:
|
||||
destination = dst[expanded]
|
||||
# Hash the actual GPU-expanded queue BEFORE target grouping.
|
||||
expanded_cpu = expanded.cpu().numpy().copy()
|
||||
accepted_cpu = (ref[destination.long()] == 0).to(torch.uint8).cpu().numpy().copy()
|
||||
order = torch.argsort(destination, stable=True)
|
||||
ordered_edges = expanded[order].contiguous()
|
||||
ordered_targets = destination[order]
|
||||
boundary = torch.ones(edge_count, device=device, dtype=torch.bool)
|
||||
boundary[1:] = ordered_targets[1:] != ordered_targets[:-1]
|
||||
starts = torch.nonzero(boundary, as_tuple=False).flatten().contiguous()
|
||||
targets = ordered_targets[starts].contiguous()
|
||||
ext.apply_arrivals(ordered_edges, starts, targets, gpu_weights, g, ref)
|
||||
queue = queue_summary(tick, expanded_cpu, accepted_cpu)
|
||||
queue_output.write(json.dumps(queue, allow_nan=False) + "\n")
|
||||
total_due += queue["count"]
|
||||
total_accepted += queue["accepted_count"]
|
||||
queue_exact = queue == oracle_queues[tick]
|
||||
# No external input exists in these fixed cases after the tick-zero requests.
|
||||
actual = {"v": v.cpu().numpy().copy(), "g": g.cpu().numpy().copy(),
|
||||
"ref": ref.cpu().numpy().copy()}
|
||||
state_checks = {name: bitwise_comparison(actual[name], histories[name][tick], name != "ref")
|
||||
for name in ("v", "g", "ref")}
|
||||
fired = torch.nonzero(spikes, as_tuple=False).flatten().cpu().numpy().astype(np.int32)
|
||||
if len(fired):
|
||||
events_step.append(np.full(len(fired), tick, dtype=np.int32))
|
||||
events_node.append(fired.copy())
|
||||
oracle_nodes = oracle_final["neuron_idx"][oracle_final["step"] == tick]
|
||||
events_exact = bool(np.array_equal(fired, oracle_nodes))
|
||||
finite = state_checks["v"]["finite"] and state_checks["g"]["finite"]
|
||||
passed = bool(finite and queue_exact and events_exact and requests_exact and
|
||||
all(value["bitwise_exact"] for value in state_checks.values()))
|
||||
row = {"tick": tick, "pass_all": passed, "states": state_checks,
|
||||
"events_exact": events_exact, "gpu_event_count": len(fired), "cpu_event_count": len(oracle_nodes),
|
||||
"queue_exact": queue_exact, "gpu_queue": queue, "cpu_queue": oracle_queues[tick],
|
||||
"requests": requested, "accepted_requests": accepted_requests,
|
||||
"rejected_requests": requested - accepted_requests,
|
||||
"request_rows_exact": requests_exact,
|
||||
"state_sha256": {name: array_digest(actual[name], "<i4" if name == "ref" else "<f8")
|
||||
for name in ("v", "g", "ref")}}
|
||||
comparisons.write(json.dumps(row, allow_nan=False) + "\n")
|
||||
hash_output.write(json.dumps({"tick": tick,
|
||||
**{name + "_sha256": value for name, value in row["state_sha256"].items()}},
|
||||
allow_nan=False) + "\n")
|
||||
rows.append(row)
|
||||
if not passed and first_failure is None:
|
||||
first_failure = row
|
||||
with (out_dir / "first-failure-state.npz").open("xb") as target:
|
||||
np.savez_compressed(target, tick=np.array(tick, dtype=np.int32),
|
||||
**{"gpu_" + name: value for name, value in actual.items()},
|
||||
**{"cpu_" + name: np.asarray(histories[name][tick]).copy() for name in actual},
|
||||
gpu_event_nodes=fired, cpu_event_nodes=oracle_nodes,
|
||||
arrival_edge_id=expanded_cpu, arrival_accepted=accepted_cpu)
|
||||
write_json(out_dir / "first-failure.json", row)
|
||||
torch.cuda.synchronize(device)
|
||||
simulation_end = time.perf_counter()
|
||||
|
||||
def join(values):
|
||||
return np.concatenate(values).astype(np.int32, copy=False) if values else np.empty(0, dtype=np.int32)
|
||||
|
||||
state = {"step": join(events_step), "neuron_idx": join(events_node), "counts": counts.cpu().numpy(),
|
||||
"v": v.cpu().numpy(), "g": g.cpu().numpy(), "ref": ref.cpu().numpy()}
|
||||
with (out_dir / "state.npz").open("xb") as target:
|
||||
np.savez_compressed(target, **state)
|
||||
final_checks = {name: bool(np.array_equal(value.view(np.uint64), oracle_final[name].view(np.uint64)))
|
||||
if name in ("v", "g") else bool(np.array_equal(value, oracle_final[name]))
|
||||
for name, value in state.items()}
|
||||
history_final_checks = {name: bool(np.array_equal(oracle_final[name].view(np.uint64), histories[name][-1].view(np.uint64)))
|
||||
if name in ("v", "g") else bool(np.array_equal(oracle_final[name], histories[name][-1]))
|
||||
for name in ("v", "g", "ref")}
|
||||
pass_all = bool(len(rows) == spec["steps"] and all(row["pass_all"] for row in rows) and
|
||||
all(final_checks.values()) and all(history_final_checks.values()) and
|
||||
requested_total == len(seeds) and accepted_request_total == len(seeds))
|
||||
summary = {"case": spec["name"], "pass_all": pass_all, "precision": "float64", "state_units": "SI volts",
|
||||
"neurons": NEURONS, "edges": EDGES, "steps_expected": spec["steps"], "steps_compared": len(rows),
|
||||
"all_neurons_compared_every_tick": True, "bitwise_includes_signed_zero": True,
|
||||
"first_failed_tick": None if first_failure is None else first_failure["tick"],
|
||||
"failed_tick_count": sum(not row["pass_all"] for row in rows),
|
||||
"final_state_exact": final_checks, "oracle_final_matches_history": history_final_checks,
|
||||
"total_spikes": len(state["step"]), "arriving_edges": total_due, "accepted_arriving_edges": total_accepted,
|
||||
"requested": requested_total, "accepted_requests": accepted_request_total,
|
||||
"rejected_requests": requested_total - accepted_request_total,
|
||||
"whole_graph_numerical_comparison": True, "olfactory_representation_validated": False,
|
||||
"apl_overlay": False, "learning": False, "candidate_generated": False,
|
||||
"performance_runtime_claim": False, "cpu_sync_each_tick": True,
|
||||
"first_failure_full_state_saved": first_failure is not None,
|
||||
"full_gpu_history_saved": False, "sealed_4AE_model": str(AE_MODEL),
|
||||
"build": build_provenance, "build_or_cache_seconds": build_end - start,
|
||||
"diagnostic_simulation_seconds": simulation_end - simulation_start,
|
||||
"wall_seconds": time.perf_counter() - start, "gpu": torch.cuda.get_device_name(device),
|
||||
"peak_allocated_vram_bytes": int(torch.cuda.max_memory_allocated(device)),
|
||||
"peak_reserved_vram_bytes": int(torch.cuda.max_memory_reserved(device)),
|
||||
"input_sha256": {"spec.json": digest(case_dir / "spec.json"),
|
||||
"oracle/weights.npy": digest(oracle_dir / "weights.npy"),
|
||||
"oracle/coefficients.json": digest(oracle_dir / "coefficients.json")}}
|
||||
write_json(out_dir / "summary.json", summary)
|
||||
write_json(out_dir / "metrics.json", {name: summary[name] for name in
|
||||
("wall_seconds", "build_or_cache_seconds", "diagnostic_simulation_seconds", "gpu",
|
||||
"peak_allocated_vram_bytes", "peak_reserved_vram_bytes", "performance_runtime_claim", "cpu_sync_each_tick")})
|
||||
print(json.dumps({"case": spec["name"], "pass_all": pass_all, "steps": len(rows),
|
||||
"first_failed_tick": summary["first_failed_tick"], "total_spikes": len(state["step"])}), flush=True)
|
||||
return summary
|
||||
except Exception as error:
|
||||
write_json(out_dir / "error.json", {"type": type(error).__name__, "message": str(error),
|
||||
"traceback": traceback.format_exc(), "tick_in_progress": tick,
|
||||
"completed_tick_comparisons": len(rows), "pass_all": False})
|
||||
raise
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"root": "/data/digital-drosophila/models/male-cns-lif/author-fullgraph-lab-v1",
|
||||
"files": [
|
||||
{
|
||||
"path": "PROTOCOL.md",
|
||||
"bytes": 2992,
|
||||
"sha256": "64413d57ea8c0222ce40f007078b1b2ce7b79986d846878e4ce408e87446b2bf"
|
||||
},
|
||||
{
|
||||
"path": "compare_trial.py",
|
||||
"bytes": 5322,
|
||||
"sha256": "2c81cb37598cce2ae0688046fd944bec34ea57824944379bfbf7e778283ab4ad"
|
||||
},
|
||||
{
|
||||
"path": "config.json",
|
||||
"bytes": 1183,
|
||||
"sha256": "9acd8177d94daf92df3435d2df7c854a0b86e8dee65d42f3ec25d084c9fba8f0"
|
||||
},
|
||||
{
|
||||
"path": "context.py",
|
||||
"bytes": 2171,
|
||||
"sha256": "fe221d4470f14a094bc2b02b9763d542c091a2a069c951c5f833a0639c39659b"
|
||||
},
|
||||
{
|
||||
"path": "finalize.py",
|
||||
"bytes": 4373,
|
||||
"sha256": "81885c07c053444cb0b03634b4ddc09b12bef2f237828db2dcf203653ac4b704"
|
||||
},
|
||||
{
|
||||
"path": "gpu_full.py",
|
||||
"bytes": 23311,
|
||||
"sha256": "895ee595d63ee8650b6aad662f831117af78cb6216437c001c8c7ba69a7b92e7"
|
||||
},
|
||||
{
|
||||
"path": "oracle_full.py",
|
||||
"bytes": 32146,
|
||||
"sha256": "d72003d98f0f1624b12126eff50bd361c43d888f4a64d90ba9247a58976f1961"
|
||||
},
|
||||
{
|
||||
"path": "prepare.py",
|
||||
"bytes": 5259,
|
||||
"sha256": "780f26c3c829135a6f67ff03007925649ef777dc15010334830ff1f7fa109519"
|
||||
},
|
||||
{
|
||||
"path": "run_trial.py",
|
||||
"bytes": 2027,
|
||||
"sha256": "afb8a0e8513726bd1340db56c2cff8f3615fedd0857dc611420d29e1b858c2c3"
|
||||
},
|
||||
{
|
||||
"path": "test_protocol.py",
|
||||
"bytes": 2019,
|
||||
"sha256": "d4bc6905ea6ce4ea04aeeabfc2f99bc111114fdfd0bc54169a40edf4f95c6033"
|
||||
},
|
||||
{
|
||||
"path": "verify_frozen.py",
|
||||
"bytes": 2151,
|
||||
"sha256": "a2a19b52c41c8c752692ff524d6f21016c6f2789c5fcabac9c78c8cadd59826e"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
"""Phase 4AF: a streaming float64 oracle for three fixed short full-graph trials.
|
||||
|
||||
Importing this module does not import Brian. queue_summary is shared with the
|
||||
GPU comparison code. All scientific inputs and the sealed 4AD/4AE code remain
|
||||
read-only. Every run_case call belongs in a fresh process.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import struct
|
||||
import sys
|
||||
import types
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
N_NEURONS = 166700
|
||||
N_EDGES = 25582938
|
||||
BLOCK_BYTES = 8 << 20
|
||||
QUEUE_HEADER = b"phase4af-queue-v1\0"
|
||||
AUTHOR_PATH = Path("/data/digital-drosophila/experiments/phase4ad-author-semantics/external-resources/model.py")
|
||||
AUTHOR_SHA256 = "fc45837d7122c6ce2a7f3f2f23c515992e4b232aadb919efabb72337fac88e4e"
|
||||
AE_ORACLE_PATH = Path("/data/digital-drosophila/models/male-cns-lif/author-runtime-lab-v1/oracle_stress.py")
|
||||
AE_ORACLE_SHA256 = "32eb1c42383068ce52701d2139a376ce155ad7072cde019949f8ca2d55492409"
|
||||
EVENT_DTYPE = np.dtype([("step", "<i4"), ("neuron_idx", "<i4")])
|
||||
_PROCESS_ALREADY_USED = False
|
||||
|
||||
|
||||
def queue_summary(tick, edges, accepted, chunk_bytes=BLOCK_BYTES):
|
||||
"""Hash original queue order without sorting, filtering, or changing inputs.
|
||||
|
||||
joint_sha256 is SHA256(header + pack('<qq', tick, count) + all edge bytes
|
||||
in little-endian int64 order + all uint8 acceptance bytes). The two other
|
||||
hashes omit the header and contain only their respective byte sequences.
|
||||
``accepted`` may be a (count,) array or a pure callable(start, stop), which
|
||||
lets the CPU read the post-threshold guard in bounded chunks. Empty ticks
|
||||
retain their tick/count header in the joint hash.
|
||||
"""
|
||||
edge_view = np.asarray(edges)
|
||||
if edge_view.ndim != 1 or edge_view.dtype.kind not in "iu":
|
||||
raise ValueError("Queue edges must be a one-dimensional integer array")
|
||||
tick_int = int(tick)
|
||||
if tick_int != tick or tick_int < 0:
|
||||
raise ValueError("Queue tick must be a nonnegative integer")
|
||||
count = len(edge_view)
|
||||
width = int(chunk_bytes) // np.dtype("<i8").itemsize
|
||||
if width <= 0:
|
||||
raise ValueError("chunk_bytes must hold at least one int64 edge")
|
||||
mask_view = None if callable(accepted) else np.asarray(accepted)
|
||||
if mask_view is not None and mask_view.shape != (count,):
|
||||
raise ValueError("Queue acceptance mask must match the edge count")
|
||||
edge_hash, mask_hash = hashlib.sha256(), hashlib.sha256()
|
||||
joint = hashlib.sha256(QUEUE_HEADER + struct.pack("<qq", tick_int, count))
|
||||
for begin in range(0, count, width):
|
||||
original = edge_view[begin:begin + width]
|
||||
if original.dtype.kind == "u" and np.any(original > np.iinfo(np.int64).max):
|
||||
raise ValueError("Queue edge ID cannot be encoded as signed int64")
|
||||
chunk = np.ascontiguousarray(original, dtype="<i8")
|
||||
data = memoryview(chunk).cast("B")
|
||||
edge_hash.update(data)
|
||||
joint.update(data)
|
||||
accepted_count = 0
|
||||
for begin in range(0, count, width):
|
||||
stop = min(count, begin + width)
|
||||
original = np.asarray(accepted(begin, stop) if callable(accepted) else mask_view[begin:stop])
|
||||
if original.shape != (stop - begin,) or original.dtype.kind not in "biu":
|
||||
raise ValueError("Acceptance chunks must be one-dimensional boolean/integer masks")
|
||||
if np.any((original != 0) & (original != 1)):
|
||||
raise ValueError("Acceptance values must be zero or one")
|
||||
chunk = np.ascontiguousarray(original, dtype=np.uint8)
|
||||
accepted_count += int(chunk.sum(dtype=np.int64))
|
||||
data = memoryview(chunk).cast("B")
|
||||
mask_hash.update(data)
|
||||
joint.update(data)
|
||||
return {"tick": tick_int, "count": count, "accepted_count": accepted_count,
|
||||
"ordered_edge_sha256": edge_hash.hexdigest(),
|
||||
"accepted_mask_sha256": mask_hash.hexdigest(),
|
||||
"joint_sha256": joint.hexdigest()}
|
||||
|
||||
|
||||
def _save(path, value):
|
||||
with Path(path).open("x", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, indent=2, sort_keys=True, allow_nan=False)
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def _digest(path):
|
||||
result = hashlib.sha256()
|
||||
with Path(path).open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(BLOCK_BYTES), b""):
|
||||
result.update(block)
|
||||
return result.hexdigest()
|
||||
|
||||
|
||||
def _check(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _tree_bytes(root):
|
||||
total = 0
|
||||
for directory, _, filenames in os.walk(root, followlinks=False):
|
||||
for name in filenames:
|
||||
total += (Path(directory) / name).stat(follow_symlinks=False).st_size
|
||||
return total
|
||||
|
||||
|
||||
def _budget(root, limit, reserve=0):
|
||||
actual = _tree_bytes(root)
|
||||
if actual + int(reserve) > limit:
|
||||
raise RuntimeError(f"New-output budget would be exceeded: {actual} + {reserve} > {limit} bytes; inputs were not adjusted")
|
||||
return actual
|
||||
|
||||
|
||||
def _new_memmap(path, dtype, shape):
|
||||
if Path(path).exists():
|
||||
raise FileExistsError(path)
|
||||
return np.lib.format.open_memmap(path, mode="w+", dtype=np.dtype(dtype), shape=shape)
|
||||
|
||||
|
||||
def _load_checked_module(path, expected_hash, name):
|
||||
data = Path(path).read_bytes()
|
||||
actual = hashlib.sha256(data).hexdigest()
|
||||
_check(actual == expected_hash, f"Pinned source hash mismatch: {path}")
|
||||
module = types.ModuleType(name)
|
||||
module.__file__ = str(path)
|
||||
# Execute checked bytes without producing __pycache__ beside a sealed file.
|
||||
exec(compile(data, str(path), "exec"), module.__dict__)
|
||||
return module
|
||||
|
||||
|
||||
def _graph_inputs(graph_dir):
|
||||
definitions = {
|
||||
"nodes": (np.int32, (N_NEURONS,)), "edge_id": (np.int64, (N_EDGES,)),
|
||||
"src": (np.int32, (N_EDGES,)), "dst": (np.int32, (N_EDGES,)),
|
||||
"signed_raw": (np.int64, (N_EDGES,)), "ptr": (np.int64, (N_NEURONS + 1,)),
|
||||
}
|
||||
arrays = {}
|
||||
for name, (dtype, shape) in definitions.items():
|
||||
value = np.load(graph_dir / (name + ".npy"), mmap_mode="r", allow_pickle=False)
|
||||
_check(value.dtype == np.dtype(dtype) and value.shape == shape, f"Incorrect canonical {name} shape/dtype")
|
||||
arrays[name] = value
|
||||
_check(np.array_equal(arrays["nodes"], np.arange(N_NEURONS, dtype=np.int32)),
|
||||
"This full-graph protocol requires canonical global node indices 0..N-1")
|
||||
ptr = arrays["ptr"]
|
||||
_check(ptr[0] == 0 and ptr[-1] == N_EDGES and np.all(np.diff(ptr) >= 0), "Invalid canonical CSR pointer")
|
||||
source_counts = np.zeros(N_NEURONS, dtype=np.int64)
|
||||
previous_source = -1
|
||||
width = BLOCK_BYTES // 8
|
||||
for begin in range(0, N_EDGES, width):
|
||||
stop = min(N_EDGES, begin + width)
|
||||
_check(np.array_equal(arrays["edge_id"][begin:stop], np.arange(begin, stop, dtype=np.int64)),
|
||||
"Full-graph edge identity/order is not the original canonical CSR order")
|
||||
source, target = arrays["src"][begin:stop], arrays["dst"][begin:stop]
|
||||
_check(source[0] >= previous_source and np.all(np.diff(source) >= 0), "Sources are not canonically sorted")
|
||||
_check(source[0] >= 0 and source[-1] < N_NEURONS and np.all((target >= 0) & (target < N_NEURONS)),
|
||||
"Canonical endpoint outside the full graph")
|
||||
previous_source = int(source[-1])
|
||||
source_counts += np.bincount(source, minlength=N_NEURONS)
|
||||
_check(np.array_equal(source_counts, np.diff(ptr)), "src and CSR pointer disagree")
|
||||
for name in ("completeness.csv", "connectivity.parquet"):
|
||||
_check((graph_dir / name).is_file(), f"Missing author input file {name}")
|
||||
return arrays
|
||||
|
||||
|
||||
class _StreamingQueueProxy:
|
||||
"""The 4AE forwarding contract with bounded hashing instead of retained lists."""
|
||||
|
||||
def __init__(self, original, get_tick, get_available, arrays, due_counts,
|
||||
spike_ring, delay_steps, queue_file, records):
|
||||
self._original_queue = original
|
||||
self._get_tick = get_tick
|
||||
self._get_available = get_available
|
||||
self._arrays = arrays
|
||||
self._due_counts = due_counts
|
||||
self._spike_ring = spike_ring
|
||||
self._delay_steps = delay_steps
|
||||
self._queue_file = queue_file
|
||||
self._records = records
|
||||
|
||||
def peek(self, *args, **kwargs):
|
||||
original = self._original_queue.peek(*args, **kwargs)
|
||||
edge_view = np.asarray(original)
|
||||
tick = self._get_tick()
|
||||
_check(tick == len(self._records["rows"]), "Queue peek ticks must be unique and consecutive")
|
||||
_check(len(edge_view) == int(self._due_counts[tick % len(self._due_counts)]),
|
||||
"Queue arrival count differs from delayed outgoing degree of real spikes")
|
||||
available = np.asarray(self._get_available(), dtype=bool)
|
||||
fired_then = np.zeros(N_NEURONS, dtype=bool)
|
||||
if tick >= self._delay_steps:
|
||||
fired_then[self._spike_ring[(tick - self._delay_steps) % len(self._spike_ring)]] = True
|
||||
|
||||
def accepted_chunk(begin, stop):
|
||||
local_edges = edge_view[begin:stop]
|
||||
_check(np.all((local_edges >= 0) & (local_edges < N_EDGES)), "Queue contains an invalid local edge index")
|
||||
_check(np.all(fired_then[self._arrays["src"][local_edges]]),
|
||||
"Queue arrival lacks a presynaptic spike at the fixed 18-step delay")
|
||||
return available[self._arrays["dst"][local_edges]]
|
||||
|
||||
# edge_id[k] == k was independently checked for every canonical edge.
|
||||
# Thus the untouched local queue array is also the original edge-ID stream.
|
||||
summary = queue_summary(tick, edge_view, accepted_chunk)
|
||||
self._queue_file.write(json.dumps(summary, sort_keys=True) + "\n")
|
||||
self._records["rows"].append(summary)
|
||||
self._records["peek_calls"] += 1
|
||||
return original
|
||||
|
||||
def advance(self, *args, **kwargs):
|
||||
self._records["advance_calls"] += 1
|
||||
return self._original_queue.advance(*args, **kwargs)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._original_queue, name)
|
||||
|
||||
|
||||
def run_case(case_dir: Path, out_dir: Path) -> dict:
|
||||
"""Run exactly one preregistered float64 case in this process.
|
||||
|
||||
Outputs: v/g/ref history .npy memmaps; actual weights.npy; sparse state.npz;
|
||||
queue.jsonl; events.bin with two little-endian int32 fields; request records;
|
||||
small per-tick statistics and audit JSON. A failed trial retains partial
|
||||
outputs and error.json and never writes summary.json with complete=True.
|
||||
"""
|
||||
global _PROCESS_ALREADY_USED
|
||||
_check(not _PROCESS_ALREADY_USED, "Each full-graph oracle case requires a new process")
|
||||
_PROCESS_ALREADY_USED = True
|
||||
case_dir = Path(case_dir).resolve(strict=True)
|
||||
out_dir = Path(out_dir).resolve()
|
||||
spec = json.loads((case_dir / "spec.json").read_text(encoding="utf-8"))
|
||||
graph_dir = Path(spec["graph_dir"]).resolve(strict=True)
|
||||
steps = int(spec["steps"])
|
||||
seeds = tuple(int(node) for node in spec["seed_nodes"])
|
||||
fixed = {"no_input": (100, ()), "single": (500, (1381,)),
|
||||
"five": (500, (1197, 1381, 1511, 82312, 133666))}
|
||||
_check(spec["name"] in fixed and (steps, seeds) == fixed[spec["name"]], "Case is not one of the three fixed trials")
|
||||
_check(float(spec["dt_ms"]) == .1 and np.isclose(float(spec["duration_ms"]), steps * .1, rtol=0, atol=1e-12),
|
||||
"Case clock/duration differs from the fixed protocol")
|
||||
_check(spec.get("precision", "float64") == "float64", "Full-graph oracle is float64 only")
|
||||
_check(spec.get("all_requests_at_tick", 0) == 0 and not spec.get("apl_overlay", False) and not spec.get("training", False),
|
||||
"This oracle does not permit altered input timing, APL, or training")
|
||||
budget_root = Path(spec.get("budget_root", os.path.commonpath([str(case_dir), str(graph_dir), str(out_dir)]))).resolve()
|
||||
budget_limit = int(spec.get("output_budget_bytes", 15 * 1024 ** 3))
|
||||
_check(budget_limit > 0, "Output budget must be positive")
|
||||
_check(out_dir.is_relative_to(budget_root), "Oracle output must be inside its budget root")
|
||||
out_dir.mkdir(parents=True, exist_ok=False)
|
||||
memmaps = []
|
||||
last_complete_tick = -1
|
||||
saved_preferences = None
|
||||
old_dt = None
|
||||
try:
|
||||
_budget(budget_root, budget_limit, reserve=steps * N_NEURONS * 20 + N_EDGES * 8 + (64 << 20))
|
||||
versions = {name: importlib.metadata.version(name)
|
||||
for name in ("brian2", "numpy", "pandas", "pyarrow", "sympy", "scipy", "cython")}
|
||||
_save(out_dir / "environment.json", {"python": sys.version, "platform": platform.platform(),
|
||||
"packages": versions, "precision": "float64", "codegen": "numpy"})
|
||||
for name, required in {"brian2": "2.5.1", "numpy": "1.22.3", "sympy": "1.12"}.items():
|
||||
_check(versions[name] == required, f"{name} must be pinned to {required}")
|
||||
import brian2 as b2
|
||||
from brian2.core.base import BrianObject
|
||||
|
||||
ae = _load_checked_module(AE_ORACLE_PATH, AE_ORACLE_SHA256, "phase4af_readonly_ae_helpers")
|
||||
author = _load_checked_module(AUTHOR_PATH, AUTHOR_SHA256, "phase4af_pinned_author")
|
||||
arrays = _graph_inputs(graph_dir)
|
||||
graph_hashes = {name: _digest(graph_dir / name) for name in (
|
||||
"nodes.npy", "edge_id.npy", "src.npy", "dst.npy", "signed_raw.npy", "ptr.npy",
|
||||
"completeness.csv", "connectivity.parquet")}
|
||||
_save(out_dir / "input-audit.json", {"spec": spec, "spec_sha256": _digest(case_dir / "spec.json"),
|
||||
"graph_hashes": graph_hashes, "author_sha256": AUTHOR_SHA256,
|
||||
"sealed_ae_oracle_sha256": AE_ORACLE_SHA256,
|
||||
"budget_root": str(budget_root), "budget_limit_bytes": budget_limit,
|
||||
"source_order": "canonical CSR; local edge index equals original edge_id",
|
||||
"common_interface_preserved": True, "poi_used": False, "apl_overlay": False, "training": False})
|
||||
saved_preferences = {key: b2.prefs[key] for key in (
|
||||
"codegen.target", "core.default_float_dtype", "legacy.refractory_timing")}
|
||||
old_dt = b2.defaultclock.dt
|
||||
b2.start_scope()
|
||||
b2.prefs["codegen.target"] = "numpy"
|
||||
b2.prefs["core.default_float_dtype"] = np.float64
|
||||
b2.prefs["legacy.refractory_timing"] = False
|
||||
b2.defaultclock.dt = .1 * b2.ms
|
||||
parameters = dict(author.default_params)
|
||||
_save(out_dir / "parameter-checks.json", ae._parameters_match(parameters, spec["model"], b2))
|
||||
neu, syn, spikes = author.create_model(graph_dir / "completeness.csv",
|
||||
graph_dir / "connectivity.parquet", parameters)
|
||||
_check(len(neu) == N_NEURONS and len(syn) == N_EDGES, "Author model did not load the whole graph")
|
||||
_check(all(np.dtype(neu.variables[key].dtype) == np.float64 for key in ("v", "g")) and
|
||||
np.dtype(syn.variables["w"].dtype) == np.float64, "Author state/weight dtype is not float64")
|
||||
weights = _new_memmap(out_dir / "weights.npy", "<f8", (N_EDGES,))
|
||||
memmaps.append(weights)
|
||||
width = BLOCK_BYTES // 8
|
||||
w_unit = float(parameters["w_syn"] / b2.volt)
|
||||
for begin in range(0, N_EDGES, width):
|
||||
stop = min(N_EDGES, begin + width)
|
||||
np.testing.assert_array_equal(np.asarray(syn.i[begin:stop]), arrays["src"][begin:stop])
|
||||
np.testing.assert_array_equal(np.asarray(syn.j[begin:stop]), arrays["dst"][begin:stop])
|
||||
actual_w = np.asarray(syn.w_[begin:stop], dtype=np.float64)
|
||||
expected_w = arrays["signed_raw"][begin:stop] * w_unit
|
||||
_check(np.array_equal(actual_w.view(np.uint64), expected_w.view(np.uint64)),
|
||||
"Author weights differ from unchanged signed raw times the original SI w_syn")
|
||||
weights[begin:stop] = actual_w
|
||||
weights.flush()
|
||||
v_history = _new_memmap(out_dir / "v-history.float64.npy", "<f8", (steps, N_NEURONS))
|
||||
g_history = _new_memmap(out_dir / "g-history.float64.npy", "<f8", (steps, N_NEURONS))
|
||||
ref_history = _new_memmap(out_dir / "ref-history.int32.npy", "<i4", (steps, N_NEURONS))
|
||||
memmaps.extend([v_history, g_history, ref_history])
|
||||
_save(out_dir / "output-schema.json", {
|
||||
"history_shape": [steps, N_NEURONS], "v_file": "v-history.float64.npy",
|
||||
"g_file": "g-history.float64.npy", "ref_file": "ref-history.int32.npy",
|
||||
"voltage_unit": "SI volt", "v_g_dtype": "little-endian float64", "ref_dtype": "little-endian int32",
|
||||
"event_file": "events.bin", "event_record": [["step", "<i4"], ["neuron_idx", "<i4"]],
|
||||
"event_order": "tick then canonical neuron_idx", "queue_file": "queue.jsonl",
|
||||
"queue_joint_bytes": "b'phase4af-queue-v1\\0' + struct.pack('<qq',tick,count) + all ordered <i8 edge IDs + all uint8 acceptance bits",
|
||||
"queue_separate_hashes": "SHA256 of each pure byte sequence without header",
|
||||
"complete_marker": "summary.json with complete=true; partial memmaps alone are not a completed trial"})
|
||||
_budget(budget_root, budget_limit, reserve=64 << 20)
|
||||
delay_steps = int(round(float(parameters["t_dly"] / b2.ms) / .1))
|
||||
refractory_steps = int(round(float(parameters["t_rfc"] / b2.ms) / .1))
|
||||
_check(delay_steps == 18 and refractory_steps == 22, "Unexpected fixed delay or refractory duration")
|
||||
queue_len = delay_steps + 1
|
||||
due_counts = np.zeros(queue_len, dtype=np.int64)
|
||||
spike_ring = [np.empty(0, dtype=np.int32) for _ in range(queue_len)]
|
||||
outdegree = np.diff(arrays["ptr"])
|
||||
request_count = np.zeros(steps, dtype=np.int32)
|
||||
applicable_count = np.zeros(steps, dtype=np.int32)
|
||||
rejected_count = np.zeros(steps, dtype=np.int32)
|
||||
spike_count = np.zeros(steps, dtype=np.int32)
|
||||
total_counts = np.zeros(N_NEURONS, dtype=np.int32)
|
||||
request_seen = np.zeros(steps, dtype=np.uint8)
|
||||
seed_local = np.asarray(seeds, dtype=np.int32)
|
||||
permitted_seed = np.empty(0, dtype=np.int32)
|
||||
request_rows = []
|
||||
records = {"rows": [], "peek_calls": 0, "advance_calls": 0, "installations": []}
|
||||
total_events = 0
|
||||
v_raw = neu.variables["v"].get_value()
|
||||
g_raw = neu.variables["g"].get_value()
|
||||
last_raw = neu.variables["lastspike"].get_value()
|
||||
available_raw = neu.variables["not_refractory"].get_value()
|
||||
last_ticks = np.zeros(N_NEURONS, dtype=np.int64)
|
||||
|
||||
def tick_now():
|
||||
return int(round(float(neu.clock.t / b2.second) / .0001))
|
||||
|
||||
with ExitStack() as stack:
|
||||
queue_file = stack.enter_context((out_dir / "queue.jsonl").open("x", encoding="utf-8"))
|
||||
event_file = stack.enter_context((out_dir / "events.bin").open("xb"))
|
||||
tick_file = stack.enter_context((out_dir / "ticks.jsonl").open("x", encoding="utf-8"))
|
||||
requests_file = stack.enter_context((out_dir / "requests.csv").open("x", encoding="utf-8", newline=""))
|
||||
request_writer = csv.DictWriter(requests_file,
|
||||
fieldnames=["tick", "neuron_idx", "attempted", "accepted", "rejected"])
|
||||
request_writer.writeheader()
|
||||
|
||||
def request_operation():
|
||||
nonlocal permitted_seed
|
||||
tick = tick_now()
|
||||
_check(0 <= tick < steps and not request_seen[tick], "Invalid/duplicate common request tick")
|
||||
request_seen[tick] = 1
|
||||
if tick == 0 and len(seed_local):
|
||||
available = available_raw[seed_local].copy()
|
||||
permitted_seed = seed_local[available]
|
||||
request_count[tick] = len(seed_local)
|
||||
applicable_count[tick] = len(permitted_seed)
|
||||
rejected_count[tick] = len(seed_local) - len(permitted_seed)
|
||||
for node, ok in zip(seed_local, available):
|
||||
request_rows.append({"tick": 0, "neuron_idx": int(node), "attempted": 1,
|
||||
"accepted": int(ok), "rejected": int(not ok)})
|
||||
neu.v[permitted_seed] = parameters["v_th"] + 1 * b2.mV
|
||||
|
||||
def record_end():
|
||||
nonlocal last_complete_tick, total_events
|
||||
tick = tick_now()
|
||||
_check(tick == last_complete_tick + 1 and tick < steps, "End-state ticks are not consecutive")
|
||||
_check(request_seen[tick] == 1 and len(records["rows"]) == tick + 1,
|
||||
"End state lacks its request/queue observation")
|
||||
_check(np.isfinite(v_raw).all() and np.isfinite(g_raw).all(), "Author state became nonfinite")
|
||||
seen = np.isfinite(last_raw) & (last_raw >= 0)
|
||||
last_ticks.fill(0)
|
||||
last_ticks[seen] = np.rint(last_raw[seen] / .0001).astype(np.int64)
|
||||
ref = np.where(seen, np.maximum(refractory_steps - (tick - last_ticks), 0), 0).astype(np.int32)
|
||||
_check(np.array_equal(ref == 0, available_raw), "Integer refractory history disagrees with Brian")
|
||||
v_history[tick] = v_raw
|
||||
g_history[tick] = g_raw
|
||||
ref_history[tick] = ref
|
||||
active = np.asarray(neu.spikes, dtype=np.int32)
|
||||
_check(not len(active) or np.all(np.diff(active) > 0), "Spike order is not canonical and unique")
|
||||
spike_count[tick] = len(active)
|
||||
total_counts[active] += 1
|
||||
if len(active):
|
||||
rows = np.empty(len(active), dtype=EVENT_DTYPE)
|
||||
rows["step"] = tick
|
||||
rows["neuron_idx"] = arrays["nodes"][active]
|
||||
event_file.write(memoryview(rows).cast("B"))
|
||||
total_events += len(active)
|
||||
spike_ring[tick % queue_len] = active.copy()
|
||||
due_counts[(tick + delay_steps) % queue_len] = int(outdegree[active].sum(dtype=np.int64))
|
||||
if tick == 0:
|
||||
_check(np.isin(permitted_seed, active).all(), "An applicable initial request failed to create a real spike")
|
||||
for row in request_rows:
|
||||
request_writer.writerow(row)
|
||||
queue_row = records["rows"][tick]
|
||||
tick_record = {"tick": tick, "spikes": len(active), "requests": int(request_count[tick]),
|
||||
"applicable_requests": int(applicable_count[tick]), "ref_rejected_requests": int(rejected_count[tick]),
|
||||
"due_arrivals": queue_row["count"], "accepted_arrivals": queue_row["accepted_count"],
|
||||
"event_sha256": hashlib.sha256(np.ascontiguousarray(arrays["nodes"][active], dtype="<i4").tobytes()).hexdigest()}
|
||||
tick_file.write(json.dumps(tick_record, sort_keys=True) + "\n")
|
||||
# Mapped files already reserve their full logical size. This
|
||||
# checks the complete new experiment tree, not only this case.
|
||||
_budget(budget_root, budget_limit, reserve=1 << 20)
|
||||
last_complete_tick = tick
|
||||
if (tick + 1) % 50 == 0 or tick + 1 == steps:
|
||||
for mapped in (v_history, g_history, ref_history):
|
||||
mapped.flush()
|
||||
for handle in (queue_file, event_file, tick_file, requests_file):
|
||||
handle.flush()
|
||||
if (tick + 1) % 100 == 0 or tick + 1 == steps:
|
||||
print(json.dumps({"oracle": spec["name"], "completed_ticks": tick + 1,
|
||||
"spikes": total_events}), flush=True)
|
||||
|
||||
class QueueInstaller(BrianObject):
|
||||
def __init__(self):
|
||||
super().__init__(clock=neu.clock, when="end", order=1_000_000,
|
||||
name="zz_fullgraph_queue_installer")
|
||||
|
||||
def before_run(self, run_namespace):
|
||||
installed = []
|
||||
for code in syn.pre.code_objects:
|
||||
if "_queue" not in code.namespace or "_queue.peek()" not in str(code.code):
|
||||
continue
|
||||
original = code.namespace["_queue"]
|
||||
if isinstance(original, _StreamingQueueProxy):
|
||||
original = original._original_queue
|
||||
code.namespace["_queue"] = _StreamingQueueProxy(
|
||||
original, tick_now, lambda: available_raw, arrays, due_counts,
|
||||
spike_ring, delay_steps, queue_file, records)
|
||||
installed.append(code.name)
|
||||
_check(len(installed) == 1, "Expected one original synaptic queue consumer")
|
||||
records["installations"].append(installed[0])
|
||||
|
||||
request_op = b2.NetworkOperation(request_operation, clock=neu.clock,
|
||||
when="before_thresholds", order=0, name="oracle_common_requests")
|
||||
end_op = b2.NetworkOperation(record_end, clock=neu.clock,
|
||||
when="end", order=0, name="oracle_streaming_end")
|
||||
network = b2.Network(neu, syn, spikes, request_op, end_op, QueueInstaller())
|
||||
_save(out_dir / "schedule.json", {"schedule": list(network.schedule),
|
||||
"summary": str(network.scheduling_summary())})
|
||||
network.run(0 * b2.ms, namespace={})
|
||||
_save(out_dir / "compiled-code.json", ae._code_records(network))
|
||||
_check(last_complete_tick == -1 and not records["rows"] and not request_seen.any(),
|
||||
"Zero-duration compilation unexpectedly advanced a trial")
|
||||
# Exactly one positive-duration call. Flushing files never resets a
|
||||
# clock, reruns before_run, or changes the in-flight delay queue.
|
||||
network.run(steps * .1 * b2.ms, namespace={})
|
||||
_save(out_dir / "generated-code.json", ae._code_records(network))
|
||||
|
||||
_check(last_complete_tick == steps - 1 and np.all(request_seen == 1), "Trial stopped before all ticks completed")
|
||||
_check(records["peek_calls"] == records["advance_calls"] == steps, "Queue observation was incomplete")
|
||||
_check(total_events == int(spikes.num_spikes) and np.array_equal(total_counts, np.asarray(spikes.count[:])),
|
||||
"Sparse streamed events disagree with original SpikeMonitor counts")
|
||||
_check(np.all(np.asarray(neu.rfc[:] / b2.ms) == float(parameters["t_rfc"] / b2.ms)),
|
||||
"Original refractory configuration changed")
|
||||
for begin in range(0, N_EDGES, width):
|
||||
stop = min(N_EDGES, begin + width)
|
||||
_check(np.array_equal(np.asarray(syn.w_[begin:stop]).view(np.uint64),
|
||||
np.asarray(weights[begin:stop]).view(np.uint64)),
|
||||
"Original synaptic weights changed during execution")
|
||||
events = (np.memmap(out_dir / "events.bin", dtype=EVENT_DTYPE, mode="r", shape=(total_events,))
|
||||
if total_events else np.empty(0, dtype=EVENT_DTYPE))
|
||||
_check((out_dir / "events.bin").stat().st_size == total_events * EVENT_DTYPE.itemsize,
|
||||
"Sparse event file length is inconsistent")
|
||||
event_block = BLOCK_BYTES // 8
|
||||
for begin in range(0, total_events, event_block):
|
||||
stop = min(total_events, begin + event_block)
|
||||
np.testing.assert_array_equal(events["neuron_idx"][begin:stop],
|
||||
np.asarray(spikes.i[begin:stop], dtype=np.int32))
|
||||
native_ticks = np.rint(np.asarray(spikes.t[begin:stop] / b2.second) / .0001).astype(np.int32)
|
||||
np.testing.assert_array_equal(events["step"][begin:stop], native_ticks)
|
||||
_budget(budget_root, budget_limit, reserve=total_events * 8 + N_NEURONS * 24 + (8 << 20))
|
||||
with (out_dir / "state.npz").open("xb") as handle:
|
||||
np.savez_compressed(handle, step=events["step"], neuron_idx=events["neuron_idx"], counts=total_counts,
|
||||
v=np.asarray(v_history[-1]), g=np.asarray(g_history[-1]), ref=np.asarray(ref_history[-1]))
|
||||
with (out_dir / "tick-summary.npz").open("xb") as handle:
|
||||
np.savez_compressed(handle, request_count=request_count, applicable_count=applicable_count,
|
||||
rejected_count=rejected_count, spike_count=spike_count,
|
||||
arrival_count=np.asarray([row["count"] for row in records["rows"]], dtype=np.int64),
|
||||
accepted_arrival_count=np.asarray([row["accepted_count"] for row in records["rows"]], dtype=np.int64))
|
||||
coefficients = ae._coefficients(neu, parameters, b2, .0001)
|
||||
_save(out_dir / "coefficients.json", coefficients)
|
||||
_save(out_dir / "queue-audit.json", {"peek_calls": records["peek_calls"],
|
||||
"advance_calls": records["advance_calls"], "installations": records["installations"],
|
||||
"due_count_matches_presynaptic_spikes_and_original_outdegrees": True,
|
||||
"acceptance_observed_after_thresholds_before_synaptic_writes": True,
|
||||
"same_peek_object_returned": True, "all_other_queue_methods_forwarded": True,
|
||||
"original_order_not_sorted_or_filtered": True, "hash_header_hex": QUEUE_HEADER.hex(),
|
||||
"edge_id_identity_verified_for_all_edges": True,
|
||||
"fullgraph_unwrapped_repeat_performed": False,
|
||||
"prior_evidence": "Phase4AE exact wrapped/unwrapped checks; current bounded-hash adapter preserves the same forwarding contract",
|
||||
"limit": "Current full-graph streaming proxy is audited by forwarding logic and per-tick counts; no separate unwrapped full-graph run is claimed."})
|
||||
_check(_digest(AUTHOR_PATH) == AUTHOR_SHA256 and _digest(AE_ORACLE_PATH) == AE_ORACLE_SHA256,
|
||||
"A sealed author/dependency source changed during execution")
|
||||
for name, expected in graph_hashes.items():
|
||||
_check(_digest(graph_dir / name) == expected, f"Prepared graph input changed during execution: {name}")
|
||||
for mapped in memmaps:
|
||||
mapped.flush()
|
||||
current_bytes = _budget(budget_root, budget_limit, reserve=1 << 20)
|
||||
summary = {"name": spec["name"], "complete": True, "backend": "Brian2 2.5.1 numpy",
|
||||
"precision": "float64", "nodes": N_NEURONS, "edges": N_EDGES,
|
||||
"steps": steps, "completed_ticks": last_complete_tick + 1, "spikes": total_events,
|
||||
"steps_expected": steps, "steps_completed": last_complete_tick + 1, "neurons": N_NEURONS,
|
||||
"pass_all": True, "pass_all_scope": "CPU input, streaming, and internal self-checks only; GPU equivalence is not assessed here",
|
||||
"requests": int(request_count.sum()), "applicable_requests": int(applicable_count.sum()),
|
||||
"ref_rejected_requests": int(rejected_count.sum()),
|
||||
"due_arrivals": sum(row["count"] for row in records["rows"]),
|
||||
"accepted_arrivals": sum(row["accepted_count"] for row in records["rows"]),
|
||||
"single_continuous_run": True, "source_and_graph_inputs_unchanged": True,
|
||||
"output_budget_root": str(budget_root), "output_bytes_before_summary": current_bytes,
|
||||
"output_budget_bytes": budget_limit, "weights_sha256": _digest(out_dir / "weights.npy"),
|
||||
"queue_sha256": _digest(out_dir / "queue.jsonl"), "events_sha256": _digest(out_dir / "events.bin"),
|
||||
"no_training": True, "no_apl_overlay": True, "no_poi": True,
|
||||
"scope": "Only this fixed short numerical trajectory; not olfactory CAL, long-run equivalence, physiological validation, or learning.",
|
||||
"pending_arrivals_limit": "Queue arrivals beyond the final recorded tick are outside this comparison window."}
|
||||
_save(out_dir / "summary.json", summary)
|
||||
return summary
|
||||
except Exception as error:
|
||||
for mapped in memmaps:
|
||||
mapped.flush()
|
||||
_save(out_dir / "error.json", {"complete": False, "last_complete_tick": last_complete_tick,
|
||||
"type": type(error).__name__, "message": str(error),
|
||||
"partial_files_preserved": True, "input_or_duration_adjusted": False})
|
||||
raise
|
||||
finally:
|
||||
if saved_preferences is not None:
|
||||
for key, value in saved_preferences.items():
|
||||
b2.prefs[key] = value
|
||||
b2.defaultclock.dt = old_dt
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Read-only canonical graph adaptation, before any neural trajectory."""
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import yaml
|
||||
from context import ROOT, MODEL, EXP, AD, AE_MODEL, AE_EXP, CFG, save, digest, sources, integrity, budget_check
|
||||
|
||||
def main():
|
||||
assert not (EXP/'protocol').exists()
|
||||
assert shutil.disk_usage(EXP.parent).free > 25*1024**3
|
||||
for name in ('protocol','graph','cases','oracle','gpu','regression','audit','logs','extension-build'):
|
||||
(EXP/name).mkdir(parents=True, exist_ok=False)
|
||||
can = ROOT/'datasets/male-cns-v1.0/processed/neuron-only'
|
||||
raw_paths = [can/'gpu/csr-indptr.int64.npy', can/'gpu/csr-indices.int32.npy',
|
||||
can/'gpu/csr-weights.int64.npy', can/'neuron-index/unified-metadata.parquet',
|
||||
ROOT/'models/male-cns-lif/runtime-v1/phase4c.yaml', AD/'external-resources/model.py',
|
||||
AD/'manifest.json', AE_MODEL/'manifest.json', AE_EXP/'manifest.json',
|
||||
AE_MODEL/'gpu_backend.py',AE_MODEL/'author_ext.cpp',AE_MODEL/'author_ext.cu',AE_MODEL/'oracle_stress.py']
|
||||
save(EXP/'protocol/preregistered.json', dict(timestamp_utc=datetime.now(timezone.utc).isoformat(),
|
||||
config=CFG, sources=sources(), before_model_execution=True,
|
||||
inputs=[dict(path=str(p.relative_to(ROOT)),sha256=digest(p)) for p in raw_paths],
|
||||
parent_commit=subprocess.check_output(['git','-C',str(ROOT),'rev-parse','HEAD'],text=True).strip()))
|
||||
save(EXP/'regression/old-integrity-before.json',integrity())
|
||||
ptr = np.load(raw_paths[0], mmap_mode='r'); dst = np.load(raw_paths[1], mmap_mode='r')
|
||||
raw = np.load(raw_paths[2], mmap_mode='r'); meta = pd.read_parquet(raw_paths[3]).sort_values('neuron_idx').reset_index(drop=True)
|
||||
n,e = CFG['neurons'],CFG['edges']
|
||||
assert ptr.dtype == np.int64 and dst.dtype == np.int32 and raw.dtype == np.int64
|
||||
assert len(meta)==n and np.array_equal(meta.neuron_idx,np.arange(n))
|
||||
assert ptr.shape==(n+1,) and ptr[0]==0 and ptr[-1]==e and np.all(np.diff(ptr)>=0)
|
||||
assert dst.shape == raw.shape == (e,) and dst.min()>=0 and dst.max()<n
|
||||
assert raw.min()>0 and int(raw.sum())==CFG['raw_weight_sum']
|
||||
policy = yaml.safe_load(raw_paths[4].read_text())['sign_policy']
|
||||
signs=[]; nt_counts={}
|
||||
for nt in meta.neurotransmitter:
|
||||
s='' if pd.isna(nt) else str(nt).strip().lower()
|
||||
signs.append(policy['unknown_sign'] if not s else policy['unclear_sign'] if s=='unclear' else policy['neurotransmitter_sign'][s])
|
||||
nt_counts[s or 'NULL'] = nt_counts.get(s or 'NULL',0)+1
|
||||
signs=np.asarray(signs,np.int8); assert set(signs).issubset({-1,0,1})
|
||||
src=np.repeat(np.arange(n,dtype=np.int32),np.diff(ptr)); signed=raw*signs[src]
|
||||
edge_id=np.arange(e,dtype=np.int64);graph=EXP/'graph'
|
||||
for name,array in dict(nodes=np.arange(n,dtype=np.int32),edge_id=edge_id,src=src,dst=dst,
|
||||
signed_raw=signed,ptr=ptr,signs=signs).items():
|
||||
with (graph/(name+'.npy')).open('xb') as f: np.save(f,array,allow_pickle=False)
|
||||
pd.DataFrame({'local_index':np.arange(n,dtype=np.int32)},index=pd.Index(np.arange(n),name='global_neuron_idx')).to_csv(graph/'completeness.csv')
|
||||
pd.DataFrame({'Presynaptic_Index':src,'Postsynaptic_Index':dst,
|
||||
'Excitatory x Connectivity':signed,'edge_id':edge_id},copy=False).to_parquet(graph/'connectivity.parquet',index=False)
|
||||
idx=np.bincount(dst,minlength=n);out=np.diff(ptr)
|
||||
assert int(np.count_nonzero(src==dst))==101 and int(np.count_nonzero((idx==0)&(out==0)))==217
|
||||
save(graph/'audit.json',dict(neurons=n,edges=e,raw_weight_sum=int(raw.sum()),
|
||||
self_loops=int(np.count_nonzero(src==dst)),isolates=int(np.count_nonzero((idx==0)&(out==0))),
|
||||
signs={str(int(s)):int(np.count_nonzero(signs==s)) for s in np.unique(signs)},neurotransmitters=nt_counts,
|
||||
canonical_edge_id='original CSR slot; source sorted; no reordering, filtering, normalization or new edge',
|
||||
body_mapping_input=str(raw_paths[3]),original_weights_modified=False))
|
||||
model=json.loads((AD/'protocol/preregistered.json').read_text())['config']['model']
|
||||
for trial in CFG['trials']:
|
||||
case=EXP/'cases'/trial['name'];case.mkdir()
|
||||
save(case/'spec.json',dict(**trial,dt_ms=model['dt_ms'],duration_ms=trial['steps']*model['dt_ms'],model=model,
|
||||
graph_dir=str(graph),precision=CFG['precision'],all_requests_at_tick=0,apl_overlay=False,training=False,
|
||||
budget_root=str(EXP),output_budget_bytes=CFG['new_output_budget_bytes']))
|
||||
meta.iloc[trial['seed_nodes']][['neuron_idx','body_id','annotation_type','neurotransmitter']].to_csv(case/'seed-identities.csv',index=False)
|
||||
save(EXP/'protocol/environment.json',dict(python=platform.python_version(),platform=platform.platform(),
|
||||
numpy=np.__version__,free_disk_bytes=shutil.disk_usage(EXP).free))
|
||||
save(EXP/'protocol/graph-case-hashes.json',[dict(path=str(p.relative_to(EXP)),sha256=digest(p),bytes=p.stat().st_size)
|
||||
for root in (graph,EXP/'cases') for p in sorted(root.rglob('*')) if p.is_file()])
|
||||
budget_check();print('PREPARED full canonical graph and3 fixed trials; frozen integrity PASS',flush=True)
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""One process per CPU/GPU trial, gate the ordered fixed protocol."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from context import ROOT,MODEL,EXP,CFG,check,save,budget_check
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument('backend',choices=['oracle','gpu']);ap.add_argument('case')
|
||||
a=ap.parse_args();check();budget_check()
|
||||
names=[x['name'] for x in CFG['trials']];assert a.case in names
|
||||
position=names.index(a.case)
|
||||
if position:
|
||||
previous=json.loads((EXP/'audit'/('completed-'+names[position-1]+'.json')).read_text())
|
||||
assert previous['pass_all'], 'Previous fixed trial failed; do not continue'
|
||||
case=EXP/'cases'/a.case;out=EXP/a.backend/a.case
|
||||
assert not out.exists(), 'Never overwrite an executed trial'
|
||||
start=time.perf_counter()
|
||||
try:
|
||||
if a.backend=='oracle':
|
||||
from oracle_full import run_case
|
||||
result=run_case(case,out)
|
||||
else:
|
||||
os.environ['CUDA_VISIBLE_DEVICES']=CFG['gpu_uuid'];os.environ['CUDA_HOME']='/usr/local/cuda'
|
||||
os.environ['PATH']=str(ROOT/'models/male-cns-lif/.venv/bin')+':/usr/local/cuda/bin:'+os.environ['PATH']
|
||||
os.environ['MAX_JOBS']='2';os.environ['TORCH_CUDA_ARCH_LIST']='8.9'
|
||||
sys.path.insert(0,str(ROOT/'models/male-cns-lif/terminal-scope-diagnostic-v1'))
|
||||
from diagnostic import queues
|
||||
queues()
|
||||
from gpu_full import run_gpu
|
||||
result=run_gpu(case,out,EXP/'oracle'/a.case,EXP/'extension-build')
|
||||
save(EXP/'audit'/(a.backend+'-'+a.case+'.json'),dict(case=a.case,backend=a.backend,
|
||||
wall_seconds_including_validation=time.perf_counter()-start,result=result))
|
||||
budget_check();print('COMPLETED',a.backend,a.case,json.dumps(result),flush=True)
|
||||
except Exception as exc:
|
||||
save(EXP/'audit'/('error-'+a.backend+'-'+a.case+'.json'),dict(error=str(exc),type=type(exc).__name__,traceback=traceback.format_exc()))
|
||||
raise
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Pure audit-format tests; no neural trajectory or GPU required."""
|
||||
import hashlib
|
||||
import struct
|
||||
import unittest
|
||||
import numpy as np
|
||||
from oracle_full import queue_summary,QUEUE_HEADER
|
||||
from compare_trial import bit_equal
|
||||
|
||||
class ProtocolTests(unittest.TestCase):
|
||||
def test_queue_chunk_independence_and_readonly(self):
|
||||
edges=np.array([0,4,2,2,19000000],dtype=np.int64);mask=np.array([1,0,1,1,0],dtype=np.uint8)
|
||||
edges.flags.writeable=False;mask.flags.writeable=False
|
||||
expected=hashlib.sha256(QUEUE_HEADER+struct.pack('<qq',18,len(edges))+edges.astype('<i8').tobytes()+mask.tobytes()).hexdigest()
|
||||
a=queue_summary(18,edges,mask,chunk_bytes=16)
|
||||
b=queue_summary(18,edges,lambda x,y:mask[x:y],chunk_bytes=24)
|
||||
self.assertEqual(a,b);self.assertEqual(a['joint_sha256'],expected)
|
||||
self.assertEqual(a['accepted_count'],3);self.assertEqual(a['count'],5)
|
||||
self.assertNotEqual(a['joint_sha256'],queue_summary(19,edges,mask)['joint_sha256'])
|
||||
|
||||
def test_empty_coverage(self):
|
||||
e=np.empty(0,np.int64);m=np.empty(0,np.uint8)
|
||||
a,b=queue_summary(0,e,m),queue_summary(1,e,m)
|
||||
self.assertEqual(a['count'],0);self.assertEqual(a['ordered_edge_sha256'],hashlib.sha256(b'').hexdigest())
|
||||
self.assertNotEqual(a['joint_sha256'],b['joint_sha256'])
|
||||
|
||||
def test_order_and_guard_not_ignored(self):
|
||||
e=np.array([5,1,9],np.int64);m=np.array([1,1,0],np.uint8)
|
||||
original=queue_summary(20,e,m)
|
||||
self.assertNotEqual(original['joint_sha256'],queue_summary(20,e[::-1],m)['joint_sha256'])
|
||||
self.assertNotEqual(original['joint_sha256'],queue_summary(20,e,1-m)['joint_sha256'])
|
||||
|
||||
def test_signed_zero_is_not_equal(self):
|
||||
self.assertFalse(bit_equal(np.array([0.],np.float64),np.array([-0.],np.float64)))
|
||||
self.assertFalse(bit_equal(np.array([1.],np.float64),np.array([1.],np.float32)))
|
||||
self.assertTrue(bit_equal(np.array([1.,-0.],np.float64),np.array([1.,-0.],np.float64)))
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Old deterministic reference regression; new output, unchanged old model."""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import yaml
|
||||
from context import ROOT,EXP,CFG,save,check
|
||||
|
||||
def main():
|
||||
check();assert not (EXP/'regression/frozen-five.json').exists()
|
||||
rtpath=ROOT/'models/male-cns-lif/deterministic-runtime-v2';pc=yaml.safe_load((rtpath/'phase4p.yaml').read_text())
|
||||
os.environ['CUDA_VISIBLE_DEVICES']=CFG['gpu_uuid'];os.environ['CUDA_HOME']='/usr/local/cuda'
|
||||
os.environ['PATH']=str(ROOT/'models/male-cns-lif/.venv/bin')+':/usr/local/cuda/bin:'+os.environ['PATH']
|
||||
os.environ['MAX_JOBS']='2';os.environ['TORCH_CUDA_ARCH_LIST']='8.9'
|
||||
sys.path.insert(0,str(ROOT/'models/male-cns-lif/terminal-scope-diagnostic-v1'))
|
||||
from diagnostic import queues
|
||||
queues();sys.path.insert(0,str(ROOT/'models/male-cns-lif'));sys.path.insert(0,str(rtpath))
|
||||
import torch,phase4p
|
||||
assert torch.cuda.mem_get_info()[0]>2*1024**3
|
||||
torch.cuda.reset_peak_memory_stats();t0=time.perf_counter();rt=phase4p.CanonicalRuntime(pc,rtpath,EXP);rows=[]
|
||||
for spec in pc['reference_trials']:
|
||||
queues();base=ROOT/'experiments/phase4p-deterministic-runtime/phase4c-v2/reference'/spec['name']
|
||||
ev=pd.read_parquet(base/'spike-events.parquet')
|
||||
ref=dict(step=ev.step.to_numpy(),neuron_idx=ev.neuron_idx.to_numpy(),counts=np.load(base/'per-neuron-spike-counts.int32.npy'),v=np.load(base/'final-voltage.float32.npy'),g=np.load(base/'final-g.float32.npy'))
|
||||
r=rt.run(np.flatnonzero(rt.pops[spec['population']]),spec['rate_hz'],spec['seed'],spec['duration_ms'])
|
||||
row=dict(trial=spec['name'],exact={k:bool(np.array_equal(v,r[k])) for k,v in ref.items()},wall_seconds=r['metrics']['wall_seconds'])
|
||||
rows.append(row);print(row,flush=True)
|
||||
save(EXP/'regression/frozen-five.json',dict(pass_all=all(all(r['exact'].values()) for r in rows),trials=rows,
|
||||
wall_seconds=time.perf_counter()-t0,peak_allocated_vram=int(torch.cuda.max_memory_allocated()),oracle='old frozen deterministic-v2 only'))
|
||||
assert all(all(r['exact'].values()) for r in rows)
|
||||
|
||||
if __name__=='__main__':main()
|
||||
Reference in New Issue
Block a user