Digital Drosophila Archive
509 lines
33 KiB
Python
509 lines
33 KiB
Python
"""Fail-closed byte-exact audit of the shared-APL closed-loop fixtures.
|
|
|
|
Comparison is independent of either backend's stored PASS. No tolerance is
|
|
used, timing fields are never compared, and reports contain no wall timestamp.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
|
|
import numpy as np
|
|
|
|
|
|
N_GLOBAL = 166700
|
|
STATE_KEYS = ('step', 'neuron_idx', 'counts', 'v', 'g', 'ref')
|
|
REQUEST_FIELDS = ('tick', 'neuron_idx', 'attempted', 'accepted', 'rejected')
|
|
HASH_FIELDS = {'tick', 'v_sha256', 'g_sha256', 'ref_sha256'}
|
|
QUEUE_FIELDS = {'tick', 'count', 'accepted_count', 'ordered_edge_sha256',
|
|
'accepted_mask_sha256', 'joint_sha256'}
|
|
BUNDLE_FILES = ('input_matrix.npy', 'output_matrix.npy', 'kc_globals.npy', 'apl_globals.npy')
|
|
COEFFICIENTS = ('gd', 'c0', 'cg', 'cv', 'threshold', 'reset', 'seed_volt')
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def file_hash(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as source:
|
|
for block in iter(lambda: source.read(8 << 20), b''):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def raw_hash(array, dtype=None):
|
|
return hashlib.sha256(np.ascontiguousarray(array, dtype=dtype).tobytes()).hexdigest()
|
|
|
|
|
|
def save(path, report):
|
|
if path is None:
|
|
return
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open('x', encoding='utf-8') as target:
|
|
json.dump(report, target, indent=2, sort_keys=True, allow_nan=False)
|
|
target.write('\n')
|
|
|
|
|
|
def _read_json(path, used):
|
|
used.add(Path(path))
|
|
return json.loads(Path(path).read_text(encoding='utf-8'))
|
|
|
|
|
|
def _read_jsonl(path, used):
|
|
used.add(Path(path))
|
|
with Path(path).open(encoding='utf-8') as source:
|
|
return [json.loads(line) for line in source if line.strip()]
|
|
|
|
|
|
def _read_array(path, used):
|
|
used.add(Path(path))
|
|
return np.load(path, mmap_mode='r', allow_pickle=False)
|
|
|
|
|
|
def _check(checks, name, condition):
|
|
checks[name] = bool(condition)
|
|
|
|
|
|
def _bit_mask(a, b):
|
|
if a.dtype.kind == 'f':
|
|
unsigned = np.dtype('uint'+str(a.dtype.itemsize*8))
|
|
return np.ascontiguousarray(a).view(unsigned) != np.ascontiguousarray(b).view(unsigned)
|
|
return a != b
|
|
|
|
|
|
def _compare_array(name, a, b, checks, differences, dtype=None, shape=None):
|
|
a, b = np.asarray(a), np.asarray(b)
|
|
schema = a.shape == b.shape and a.dtype == b.dtype and \
|
|
(dtype is None or a.dtype == np.dtype(dtype)) and (shape is None or a.shape == shape)
|
|
_check(checks, name+'.schema', schema)
|
|
if not schema:
|
|
differences[name] = {'left_shape': list(a.shape), 'right_shape': list(b.shape),
|
|
'left_dtype': str(a.dtype), 'right_dtype': str(b.dtype), 'expected_shape': None if shape is None else list(shape),
|
|
'expected_dtype': None if dtype is None else str(np.dtype(dtype))}
|
|
_check(checks, name+'.bitwise_exact', False)
|
|
return False
|
|
finite = bool(np.isfinite(a).all() and np.isfinite(b).all())
|
|
_check(checks, name+'.finite', finite)
|
|
mask = _bit_mask(a, b)
|
|
exact = not bool(np.any(mask))
|
|
_check(checks, name+'.bitwise_exact', exact)
|
|
if not exact or not finite:
|
|
flat = np.flatnonzero(mask.ravel())
|
|
detail = {'mismatching_elements': int(len(flat)),
|
|
'first_positions': [list(map(int, np.unravel_index(int(index), a.shape))) for index in flat[:5]],
|
|
'left_nonfinite': int(np.count_nonzero(~np.isfinite(a))),
|
|
'right_nonfinite': int(np.count_nonzero(~np.isfinite(b)))}
|
|
if finite and a.size:
|
|
error = np.abs(a.astype(np.float64)-b.astype(np.float64))
|
|
detail.update(max_abs_error=float(error.max()), mean_abs_error=float(error.mean()))
|
|
if a.dtype.kind == 'f' and len(flat):
|
|
unsigned = np.dtype('uint'+str(a.dtype.itemsize*8))
|
|
av = np.ascontiguousarray(a).view(unsigned).ravel()
|
|
bv = np.ascontiguousarray(b).view(unsigned).ravel()
|
|
detail['first_bit_patterns'] = [{'left': hex(int(av[index])), 'right': hex(int(bv[index]))} for index in flat[:5]]
|
|
differences[name] = detail
|
|
return exact and finite
|
|
|
|
|
|
def _sha(value):
|
|
return isinstance(value, str) and re.fullmatch('[0-9a-f]{64}', value) is not None
|
|
|
|
|
|
def _read_requests(path, steps, used):
|
|
used.add(Path(path))
|
|
with Path(path).open(encoding='utf-8', newline='') as source:
|
|
reader = csv.DictReader(source)
|
|
require(tuple(reader.fieldnames or ()) == REQUEST_FIELDS, 'Request CSV schema mismatch: '+str(path))
|
|
rows = [{key: int(row[key]) for key in REQUEST_FIELDS} for row in reader]
|
|
previous = (-1, -1)
|
|
for row in rows:
|
|
key = row['tick'], row['neuron_idx']
|
|
require(previous < key and 0 <= key[0] < steps and 0 <= key[1] < N_GLOBAL,
|
|
'Request rows must be sorted, unique, and in range: '+str(path))
|
|
require(row['attempted'] == 1 and row['accepted'] in (0, 1) and row['rejected'] == 1-row['accepted'],
|
|
'Request accounting is incomplete: '+str(path))
|
|
previous = key
|
|
return rows
|
|
|
|
|
|
def _load_run(directory, role, n, edges, steps, record_history, name, used, checks):
|
|
directory = Path(directory)
|
|
summary = _read_json(directory/'summary.json', used)
|
|
_check(checks, role+'.summary_complete', summary.get('complete') is True and summary.get('pass_all') is True and
|
|
summary.get('steps') == steps and summary.get('completed_ticks') == steps and
|
|
summary.get('nodes') == n and summary.get('edges') == edges and summary.get('precision') == 'float64')
|
|
is_cpu = role == 'cpu'
|
|
_check(checks, role+'.case_name', summary.get('name' if is_cpu else 'case') == name)
|
|
_check(checks, role+'.history_mode', summary.get('history_recorded' if is_cpu else 'record_history') is record_history)
|
|
_check(checks, role+'.apl_precision', summary.get('APL_precision' if is_cpu else 'apl_precision') == 'float32')
|
|
_check(checks, role+'.no_error_marker', not (directory/'error.json').exists())
|
|
used.add(directory/'state.npz')
|
|
with np.load(directory/'state.npz', allow_pickle=False) as source:
|
|
require(set(source.files) == set(STATE_KEYS), 'State NPZ keys changed: '+str(directory))
|
|
state = {key: source[key].copy() for key in STATE_KEYS}
|
|
for key, dtype in (('v', 'float64'), ('g', 'float64'), ('ref', 'int32'), ('counts', 'int32')):
|
|
require(state[key].shape == (n,) and state[key].dtype == np.dtype(dtype), 'Invalid final '+key+': '+str(directory))
|
|
require(state['step'].dtype == state['neuron_idx'].dtype == np.int32 and state['step'].ndim == 1 and
|
|
state['step'].shape == state['neuron_idx'].shape, 'Invalid sparse event schema: '+str(directory))
|
|
require(bool(np.all((state['step'] >= 0) & (state['step'] < steps))) and
|
|
bool(np.all((state['neuron_idx'] >= 0) & (state['neuron_idx'] < N_GLOBAL))), 'Sparse event range error')
|
|
event_keys = state['step'].astype(np.int64)*N_GLOBAL+state['neuron_idx']
|
|
_check(checks, role+'.events_sorted_unique', len(event_keys) < 2 or np.all(np.diff(event_keys) > 0))
|
|
_check(checks, role+'.count_nonnegative_total', np.all(state['counts'] >= 0) and int(state['counts'].sum()) == len(event_keys))
|
|
_check(checks, role+'.ref_range', np.all((state['ref'] >= 0) & (state['ref'] <= 22)))
|
|
_check(checks, role+'.reported_events', summary.get('spikes' if is_cpu else 'total_spikes') == len(event_keys))
|
|
hashes = _read_jsonl(directory/'tick-state-hashes.jsonl', used)
|
|
require(len(hashes) == steps and [row.get('tick') for row in hashes] == list(range(steps)), 'Incomplete tick hash coverage')
|
|
require(all(set(row) == HASH_FIELDS and all(_sha(row[key]) for key in HASH_FIELDS-{'tick'}) for row in hashes),
|
|
'Invalid tick state hash schema')
|
|
queues = _read_jsonl(directory/'queue.jsonl', used)
|
|
require(len(queues) == steps and [row.get('tick') for row in queues] == list(range(steps)), 'Incomplete queue coverage')
|
|
require(all(set(row) == QUEUE_FIELDS and type(row['count']) is int and type(row['accepted_count']) is int and
|
|
0 <= row['accepted_count'] <= row['count'] <= edges and
|
|
all(_sha(row[key]) for key in QUEUE_FIELDS-{'tick', 'count', 'accepted_count'}) for row in queues),
|
|
'Invalid queue row schema/count/hash')
|
|
for key in ('v', 'g', 'ref'):
|
|
_check(checks, role+'.last_hash_'+key, hashes[-1][key+'_sha256'] == raw_hash(state[key], '<i4' if key == 'ref' else '<f8'))
|
|
requests = _read_requests(directory/'requests.csv', steps, used)
|
|
requested, accepted = len(requests), sum(row['accepted'] for row in requests)
|
|
_check(checks, role+'.request_summary', summary.get('requests' if is_cpu else 'requested') == requested and
|
|
summary.get('applicable_requests' if is_cpu else 'accepted_requests') == accepted and
|
|
summary.get('ref_rejected_requests' if is_cpu else 'rejected_requests') == requested-accepted)
|
|
apl = {field: _read_array(directory/('apl-'+field+'.float32.npy'), used) for field in ('drive', 'state', 'inhibition')}
|
|
for field, width in (('drive', 2), ('state', 2), ('inhibition', 4064)):
|
|
require(apl[field].dtype == np.float32 and apl[field].shape == (steps, width), 'Invalid APL '+field+' history schema')
|
|
_check(checks, role+'.apl_positive_'+field, np.all(apl[field] >= 0))
|
|
histories, pre = {}, {}
|
|
for field, dtype in (('v', 'float64'), ('g', 'float64'), ('ref', 'int32')):
|
|
files = (directory/(field+'-history.'+dtype+'.npy'), directory/('preAPL-'+field+'.'+dtype+'.npy'))
|
|
if record_history:
|
|
histories[field], pre[field] = (_read_array(path, used) for path in files)
|
|
require(all(array.dtype == np.dtype(dtype) and array.shape == (steps, n) for array in (histories[field], pre[field])),
|
|
'Small pre/post history schema mismatch')
|
|
_check(checks, role+'.history_hashes_'+field, all(row[field+'_sha256'] == raw_hash(histories[field][tick])
|
|
for tick, row in enumerate(hashes)))
|
|
_check(checks, role+'.history_final_'+field, not np.any(_bit_mask(histories[field][-1], state[field])))
|
|
else:
|
|
_check(checks, role+'.no_dense_'+field, not any(path.exists() for path in files))
|
|
if is_cpu:
|
|
component = _read_json(directory/'apl-worker-handshake.json', used)['handshake']
|
|
else:
|
|
component = _read_json(directory/'apl-component.json', used)
|
|
_check(checks, role+'.component_summary_consistent', summary.get('apl_component') == component)
|
|
return dict(directory=directory, summary=summary, state=state, hashes=hashes, queues=queues,
|
|
requests=requests, apl=apl, histories=histories, pre=pre, component=component)
|
|
|
|
|
|
def _pair_outputs(left, right, checks, differences, n, steps, record_history):
|
|
for field in STATE_KEYS:
|
|
shape = None if field in ('step', 'neuron_idx') else (n,)
|
|
dtype = 'float64' if field in ('v', 'g') else 'int32'
|
|
_compare_array('state.'+field, left['state'][field], right['state'][field], checks, differences, dtype, shape)
|
|
for field, width in (('drive', 2), ('state', 2), ('inhibition', 4064)):
|
|
_compare_array('apl.'+field, left['apl'][field], right['apl'][field], checks, differences, 'float32', (steps, width))
|
|
for field in ('hashes', 'queues', 'requests', 'component'):
|
|
same = left[field] == right[field]
|
|
_check(checks, field+'.parsed_exact', same)
|
|
if not same:
|
|
if isinstance(left[field], list):
|
|
first = next((i for i, (a, b) in enumerate(zip(left[field], right[field])) if a != b), None)
|
|
differences[field] = {'left_count': len(left[field]), 'right_count': len(right[field]),
|
|
'first_differing_row': first,
|
|
'left_row': None if first is None else left[field][first],
|
|
'right_row': None if first is None else right[field][first]}
|
|
else:
|
|
differences[field] = {'left': left[field], 'right': right[field]}
|
|
if record_history:
|
|
for stage in ('histories', 'pre'):
|
|
for field, dtype in (('v', 'float64'), ('g', 'float64'), ('ref', 'int32')):
|
|
_compare_array(stage+'.'+field, left[stage][field], right[stage][field], checks, differences, dtype, (steps, n))
|
|
|
|
|
|
def _queue_digest(tick, ids, accepted):
|
|
ids = np.ascontiguousarray(ids, dtype='<i8').tobytes()
|
|
mask = np.ascontiguousarray(accepted, dtype=np.uint8).tobytes()
|
|
return {'tick': tick, 'count': len(mask), 'accepted_count': int(np.count_nonzero(accepted)),
|
|
'ordered_edge_sha256': hashlib.sha256(ids).hexdigest(),
|
|
'accepted_mask_sha256': hashlib.sha256(mask).hexdigest(),
|
|
'joint_sha256': hashlib.sha256(b'phase4af-queue-v1\0'+struct.pack('<qq', tick, len(mask))+ids+mask).hexdigest()}
|
|
|
|
|
|
def _independent_event_accounting(run, role, graph, requested, steps, checks, differences):
|
|
nodes, ptr = graph['nodes'], graph['ptr']
|
|
n = len(nodes)
|
|
state = run['state']
|
|
local = np.searchsorted(nodes, state['neuron_idx'])
|
|
require(np.all(local < n) and np.array_equal(nodes[local], state['neuron_idx']), 'Actual spike outside graph identity')
|
|
_check(checks, role+'.events_per_neuron_counts', np.array_equal(np.bincount(local, minlength=n), state['counts']))
|
|
actual_request_pairs = [(row['tick'], row['neuron_idx']) for row in run['requests']]
|
|
_check(checks, role+'.requests_match_frozen_input', actual_request_pairs == requested)
|
|
require(actual_request_pairs == requested, 'Actual request population/time differs from frozen input')
|
|
bounds = np.searchsorted(state['step'], np.arange(steps+1))
|
|
request_ticks = np.asarray([row['tick'] for row in run['requests']], np.int32)
|
|
rbounds = np.searchsorted(request_ticks, np.arange(steps+1))
|
|
request_nodes = np.asarray([row['neuron_idx'] for row in run['requests']], np.int32)
|
|
request_local = np.searchsorted(nodes, request_nodes)
|
|
require(np.all(request_local < n) and np.array_equal(nodes[request_local], request_nodes), 'Request outside graph')
|
|
last = np.full(n, -1_000_000_000, np.int64)
|
|
degrees = np.diff(ptr)
|
|
expected_counts = np.zeros(steps, np.int64)
|
|
arrival_tick = state['step'].astype(np.int64)+18
|
|
valid = arrival_tick < steps
|
|
np.add.at(expected_counts, arrival_tick[valid], degrees[local[valid]])
|
|
_check(checks, role+'.delayed_all_edges_count', np.array_equal(expected_counts, [row['count'] for row in run['queues']]))
|
|
request_ok, delivered_ok, ref_ok, queue_ok, reset_ok = True, True, True, True, True
|
|
first = {}
|
|
for tick in range(steps):
|
|
fired = local[bounds[tick]:bounds[tick+1]]
|
|
lo, hi = rbounds[tick:tick+2]
|
|
targets = request_local[lo:hi]
|
|
accepted = (tick-last[targets]) >= 22
|
|
for index, ok in zip(range(lo, hi), accepted):
|
|
row = run['requests'][index]
|
|
if row['accepted'] != int(ok) or row['rejected'] != int(not ok):
|
|
request_ok = False
|
|
first.setdefault('request_ref_mismatch', {'tick': tick, 'neuron_idx': row['neuron_idx']})
|
|
if not np.isin(targets[accepted], fired).all() or np.isin(targets[~accepted], fired).any():
|
|
delivered_ok = False
|
|
first.setdefault('request_delivery_mismatch', tick)
|
|
last[fired] = tick
|
|
ref = np.maximum(22-(tick-last), 0).astype(np.int32)
|
|
if raw_hash(ref, '<i4') != run['hashes'][tick]['ref_sha256']:
|
|
ref_ok = False
|
|
first.setdefault('ref_hash_mismatch', tick)
|
|
if run['pre']:
|
|
if len(fired) and not np.array_equal(np.ascontiguousarray(run['pre']['g'][tick, fired]).view(np.uint64), np.zeros(len(fired), np.uint64)):
|
|
reset_ok = False
|
|
first.setdefault('pre_apl_spike_reset_mismatch', tick)
|
|
due = local[bounds[tick-18]:bounds[tick-17]] if tick >= 18 else np.empty(0, np.int64)
|
|
chunks = [np.arange(ptr[source], ptr[source+1], dtype=np.int64) for source in due]
|
|
indices = np.concatenate(chunks) if chunks else np.empty(0, np.int64)
|
|
expected_queue = _queue_digest(tick, graph['edge_id'][indices], (ref[graph['dst'][indices]] == 0).astype(np.uint8))
|
|
if expected_queue != run['queues'][tick]:
|
|
queue_ok = False
|
|
first.setdefault('queue_reconstruction_mismatch', tick)
|
|
_check(checks, role+'.request_refractory_gate', request_ok)
|
|
_check(checks, role+'.accepted_spikes_rejected_absent', delivered_ok)
|
|
_check(checks, role+'.all_ref_hashes_from_actual_events', ref_ok)
|
|
if run['pre']:
|
|
_check(checks, role+'.small_queue_independent_reconstruction', queue_ok)
|
|
_check(checks, role+'.spike_g_reset_before_apl', reset_ok)
|
|
if first:
|
|
differences[role+'.independent_accounting'] = first
|
|
|
|
|
|
def _check_apl_application(run, role, nodes, kc, checks, differences):
|
|
if not run['pre']:
|
|
return
|
|
common, local_kc, bundle_kc = np.intersect1d(nodes, kc, assume_unique=True, return_indices=True)
|
|
_compare_array(role+'.apl_keeps_voltage', run['pre']['v'], run['histories']['v'], checks, differences, 'float64')
|
|
_compare_array(role+'.apl_keeps_ref', run['pre']['ref'], run['histories']['ref'], checks, differences, 'int32')
|
|
expected = np.array(run['pre']['g'], copy=True)
|
|
delta = run['apl']['inhibition'][:, bundle_kc].astype(np.float64)*np.float64(.001)
|
|
expected[:, local_kc] = expected[:, local_kc]-delta
|
|
_compare_array(role+'.apl_unconditional_g_write', expected, run['histories']['g'], checks, differences, 'float64')
|
|
|
|
|
|
def _finish(report, checks, differences, used, out_json):
|
|
report['checks'] = {name: bool(value) for name, value in checks.items()}
|
|
report['differences'] = differences
|
|
report['pass_all'] = bool(checks) and all(checks.values()) and 'error' not in report
|
|
report['failed_checks'] = sorted(name for name, value in checks.items() if not value)
|
|
report['evidence_sha256'] = {str(path): file_hash(path) for path in sorted(used) if path.is_file()}
|
|
save(out_json, report)
|
|
return report
|
|
|
|
|
|
def compare(case_dir, gpu_dir, cpu_dir, out_json=None):
|
|
"""Strict independent closed-loop comparison; out_json=None is read-only."""
|
|
case_dir, gpu_dir, cpu_dir = Path(case_dir), Path(gpu_dir), Path(cpu_dir)
|
|
if out_json is not None:
|
|
require(not Path(out_json).exists(), 'Refusing existing comparison report')
|
|
checks, differences, used = {}, {}, set()
|
|
report = {'kind': 'shared-APL-component closed-loop byte-exact comparison', 'pass_all': False,
|
|
'case_dir': str(case_dir), 'gpu_dir': str(gpu_dir), 'cpu_dir': str(cpu_dir),
|
|
'tolerance': None, 'signed_zero_included': True, 'timing_fields_compared': False,
|
|
'independent_CPU_all_components_claim': False}
|
|
try:
|
|
spec = _read_json(case_dir/'spec.json', used)
|
|
steps, history = spec['steps'], spec['record_history']
|
|
require(type(steps) is int and steps > 0 and type(history) is bool and spec['dt_ms'] == .1,
|
|
'Invalid fixed case clock/history schema')
|
|
graph_dir, bundle_dir = Path(spec['graph_dir']), Path(spec['apl_bundle_dir'])
|
|
graph = {field: _read_array(graph_dir/(field+'.npy'), used) for field in
|
|
('nodes', 'edge_id', 'src', 'dst', 'ptr', 'signed_raw', 'weights')}
|
|
nodes = graph['nodes']
|
|
n, edges = len(nodes), len(graph['edge_id'])
|
|
require(n > 0 and edges > 0 and nodes.dtype == np.int32 and nodes.ndim == 1 and np.all((nodes >= 0) & (nodes < N_GLOBAL)) and
|
|
np.all(np.diff(nodes) > 0), 'Invalid canonical node identities')
|
|
require(graph['ptr'].dtype == np.int64 and graph['ptr'].shape == (n+1,) and graph['ptr'][0] == 0 and
|
|
graph['ptr'][-1] == edges and np.all(np.diff(graph['ptr']) >= 0), 'Invalid CSR')
|
|
for key, dtype in (('edge_id', 'int64'), ('src', 'int32'), ('dst', 'int32'), ('signed_raw', 'int64'), ('weights', 'float64')):
|
|
require(graph[key].dtype == np.dtype(dtype) and graph[key].shape == (edges,), 'Invalid graph '+key)
|
|
require(np.array_equal(graph['src'], np.repeat(np.arange(n, dtype=np.int32), np.diff(graph['ptr']))) and
|
|
np.all((graph['dst'] >= 0) & (graph['dst'] < n)), 'Invalid source/destination row mapping')
|
|
require(history == (n < 512) and (n < 512 or n == N_GLOBAL), 'Invalid small/full history declaration')
|
|
require(np.all(graph['edge_id'] >= 0) and np.all(np.diff(graph['edge_id']) > 0),
|
|
'Original canonical edge IDs must be unique and increasing')
|
|
if n == N_GLOBAL:
|
|
require(edges == 25582938 and np.array_equal(nodes, np.arange(n, dtype=np.int32)) and
|
|
np.array_equal(graph['edge_id'], np.arange(edges, dtype=np.int64)), 'Full canonical graph identity changed')
|
|
bundle = {name: _read_array(bundle_dir/name, used) for name in BUNDLE_FILES}
|
|
kc, apl_ids = bundle['kc_globals.npy'], bundle['apl_globals.npy']
|
|
require(kc.dtype == apl_ids.dtype == np.int32 and kc.shape == (4064,) and apl_ids.shape == (2,), 'Invalid APL/KC identities')
|
|
require(np.all((kc >= 0) & (kc < N_GLOBAL)) and np.all(np.diff(kc) > 0) and
|
|
np.all((apl_ids >= 0) & (apl_ids < N_GLOBAL)) and np.all(np.diff(apl_ids) > 0) and
|
|
not np.intersect1d(kc, apl_ids).size, 'APL/KC identities overlap or are invalid')
|
|
for key, shape in (('input_matrix.npy', (2, N_GLOBAL)), ('output_matrix.npy', (2, 4064))):
|
|
require(bundle[key].dtype == np.float32 and bundle[key].shape == shape and
|
|
np.isfinite(bundle[key]).all() and np.all(bundle[key] >= 0), 'Invalid normalized APL matrix')
|
|
gpu = _load_run(gpu_dir, 'gpu', n, edges, steps, history, spec['name'], used, checks)
|
|
cpu = _load_run(cpu_dir, 'cpu', n, edges, steps, history, spec['name'], used, checks)
|
|
_pair_outputs(gpu, cpu, checks, differences, n, steps, history)
|
|
used.add(case_dir/'requests.npz')
|
|
with np.load(case_dir/'requests.npz', allow_pickle=False) as source:
|
|
require(set(source.files) == {'tick', 'neuron_idx'} and source['tick'].dtype == source['neuron_idx'].dtype == np.int32,
|
|
'Fixed request NPZ schema changed')
|
|
require(source['tick'].shape == source['neuron_idx'].shape and source['tick'].ndim == 1, 'Invalid sparse request shape')
|
|
requested = list(zip(map(int, source['tick']), map(int, source['neuron_idx'])))
|
|
for role, run in (('gpu', gpu), ('cpu', cpu)):
|
|
_independent_event_accounting(run, role, graph, requested, steps, checks, differences)
|
|
_check_apl_application(run, role, nodes, kc, checks, differences)
|
|
|
|
# Independently derive the effective clone. Original graph weights stay read-only.
|
|
cpu_weights = _read_array(cpu_dir/'weights.npy', used)
|
|
require(cpu_weights.dtype == np.float64 and cpu_weights.shape == (edges,), 'CPU effective weight schema mismatch')
|
|
expected_hash, base_hash, mask_hash = hashlib.sha256(), hashlib.sha256(), hashlib.sha256()
|
|
weight_ok, mask_count = True, 0
|
|
for start in range(0, edges, 1 << 20):
|
|
stop = min(edges, start+(1 << 20))
|
|
original = np.asarray(graph['weights'][start:stop])
|
|
require(np.isfinite(original).all(), 'Nonfinite original weights')
|
|
raw_expected = graph['signed_raw'][start:stop]*(.275*.001)
|
|
require(not np.any(_bit_mask(original, raw_expected)), 'Original SI weights differ from unaltered signed contacts')
|
|
mask = np.isin(nodes[graph['src'][start:stop]], apl_ids)
|
|
expected = original.copy()
|
|
expected[mask] = np.float64(0)
|
|
mask_count += int(mask.sum())
|
|
expected_hash.update(np.ascontiguousarray(expected, dtype='<f8').tobytes())
|
|
base_hash.update(np.ascontiguousarray(original, dtype='<f8').tobytes())
|
|
mask_hash.update(np.ascontiguousarray(graph['edge_id'][start:stop][mask], dtype='<i8').tobytes())
|
|
weight_ok &= not np.any(_bit_mask(expected, cpu_weights[start:stop]))
|
|
expected_sha = expected_hash.hexdigest()
|
|
_check(checks, 'cpu.effective_weights_independent_bytes', weight_ok)
|
|
for role, run in (('gpu', gpu), ('cpu', cpu)):
|
|
_check(checks, role+'.effective_weights_raw_sha', run['summary'].get('effective_weights_raw_sha256') == expected_sha)
|
|
_check(checks, role+'.apl_fast_edge_count', run['summary'].get('apl_fast_edges_zeroed') == mask_count)
|
|
_check(checks, role+'.kc_target_count', run['summary'].get('kc_targets_present') == len(np.intersect1d(nodes, kc)))
|
|
mask_audit = _read_json(cpu_dir/'apl-mask-audit.json', used)
|
|
_check(checks, 'cpu.mask_provenance', mask_audit['count'] == mask_count and
|
|
mask_audit['ordered_original_edge_id_sha256'] == mask_hash.hexdigest() and
|
|
mask_audit['base_weight_raw_bytes_sha256'] == base_hash.hexdigest() and
|
|
mask_audit['effective_weights_raw_sha256'] == expected_sha and mask_audit['base_graph_files_modified'] is False)
|
|
|
|
case_hashes = {name: file_hash(case_dir/name) for name in ('spec.json', 'requests.npz')}
|
|
if (case_dir/'arrays.npz').exists():
|
|
case_hashes['arrays.npz'] = file_hash(case_dir/'arrays.npz')
|
|
used.add(case_dir/'arrays.npz')
|
|
baseline_coeff = _read_json(Path(spec['coefficients_path']), used)
|
|
actual_coeff = _read_json(cpu_dir/'coefficients.json', used)
|
|
_check(checks, 'cpu.actual_author_coefficients_exact', all(float(actual_coeff[key]).hex() == float(baseline_coeff[key]).hex()
|
|
for key in COEFFICIENTS))
|
|
audit = _read_json(cpu_dir/'input-audit.json', used)
|
|
expected_graph_hashes = {name: file_hash(graph_dir/name) for name in
|
|
tuple(field+'.npy' for field in graph)+('completeness.csv', 'connectivity.parquet')}
|
|
bundle_hashes = {name: file_hash(bundle_dir/name) for name in BUNDLE_FILES}
|
|
_check(checks, 'cpu.read_actual_case', audit['spec'] == spec and audit['case_hashes'] == case_hashes)
|
|
_check(checks, 'cpu.graph_hashes', audit['graph_hashes'] == expected_graph_hashes)
|
|
_check(checks, 'cpu.bundle_hashes', audit['bundle_hashes'] == bundle_hashes)
|
|
_check(checks, 'cpu.no_candidate_trace_input', audit.get('candidate_trace_used_as_input') is False and
|
|
cpu['summary'].get('candidate_trace_used_as_input') is False)
|
|
_check(checks, 'cpu.worker_source_hash', audit['legacy_apl_worker_sha256'] == file_hash(spec['legacy_apl_worker']))
|
|
_check(checks, 'cpu.coefficient_input_hash', audit['coefficients_sha256'] == file_hash(spec['coefficients_path']))
|
|
gpu_hashes = dict(case_hashes, **{'base_weights.npy': file_hash(graph_dir/'weights.npy'),
|
|
'coefficients.json': file_hash(spec['coefficients_path'])})
|
|
_check(checks, 'gpu.input_hashes', gpu['summary']['input_sha256'] == gpu_hashes)
|
|
for role, run in (('gpu', gpu), ('cpu', cpu)):
|
|
component = run['component']
|
|
_check(checks, role+'.actual_component_bundle_bytes', component['files_sha256'] == bundle_hashes and
|
|
component['array_sha256'] == {name: raw_hash(value) for name, value in bundle.items()})
|
|
_check(checks, role+'.component_dimensions_clock', component.get('ok') is True and
|
|
component['protocol'] == 'phase4ag-apl-v1' and component['n_neurons'] == N_GLOBAL and
|
|
component['n_apl'] == 2 and component['n_kc'] == 4064 and component['dt_ms'] == .1)
|
|
decay = np.float32(math.exp(-.1/20.0))
|
|
gain = np.float32(.01)
|
|
_check(checks, role+'.frozen_apl_constants', component['decay_f32_bits'] == f'{int(decay.view(np.uint32)):08x}' and
|
|
component['gain_f32_bits'] == f'{int(gain.view(np.uint32)):08x}')
|
|
env = component['environment']
|
|
_check(checks, role+'.component_precision_environment', env['cuda_matmul_allow_tf32'] is False and
|
|
env['float32_matmul_precision'] == 'highest' and env['precision_configuration_changed_by_component'] is False and
|
|
'4090' in env['device_name'])
|
|
environment = _read_json(cpu_dir/'environment.json', used)
|
|
_check(checks, 'cpu.pinned_environment', environment['packages']['brian2'] == '2.5.1' and
|
|
environment['packages']['numpy'] == '1.22.3' and environment['packages']['sympy'] == '1.12' and
|
|
environment['Torch_imported_into_Brian'] is False and environment['author_precision'] == 'float64')
|
|
queue_audit = _read_json(cpu_dir/'queue-audit.json', used)
|
|
_check(checks, 'cpu.queue_observer_coverage', queue_audit['peek_calls'] == queue_audit['advance_calls'] == steps and
|
|
queue_audit['same_peek_object_returned'] is True and queue_audit['zero_weight_and_refractory_rejected_edges_retained'] is True)
|
|
_check(checks, 'cpu.sidecar_complete', cpu['summary']['APL_worker_exit_status'] == 0 and cpu['summary']['APL_worker_ticks'] == steps)
|
|
report.update(case=spec['name'], steps=steps, nodes=n, edges=edges, record_history=history,
|
|
total_spikes=len(gpu['state']['step']), active_neurons=int(np.count_nonzero(gpu['state']['counts'])),
|
|
requested=len(requested), accepted_requests=sum(row['accepted'] for row in gpu['requests']),
|
|
rejected_requests=sum(row['rejected'] for row in gpu['requests']),
|
|
arriving_edges=sum(row['count'] for row in gpu['queues']),
|
|
accepted_arriving_edges=sum(row['accepted_count'] for row in gpu['queues']),
|
|
apl_fast_edges_zeroed=mask_count, effective_weights_raw_sha256=expected_sha,
|
|
component_environment=gpu['component']['environment'],
|
|
limits=['APL Torch primitives are shared; original AST component validation is a separate required gate.',
|
|
'Small cases preserve and independently hash all pre/post states; full cases compare all-tick instrumented state hashes without saving dense whole-brain histories.',
|
|
'For full cases intermediate finite checks are performed by the instrumented backends; this file independently inspects final states and all APL traces.',
|
|
'This is exact agreement for the declared trajectory, not physiological validity or learning.'])
|
|
except Exception as error:
|
|
report['error'] = {'type': type(error).__name__, 'message': str(error)}
|
|
return _finish(report, checks, differences, used, out_json)
|
|
|
|
|
|
def compare_gpu_repeats(first, second, out_json=None):
|
|
"""Repeat equality only; the first run's independent oracle gate is separate."""
|
|
first, second = Path(first), Path(second)
|
|
if out_json is not None:
|
|
require(not Path(out_json).exists(), 'Refusing existing repeat report')
|
|
checks, differences, used = {}, {}, set()
|
|
report = {'kind': 'fresh-GPU-run exact repeat comparison', 'first': str(first), 'second': str(second),
|
|
'pass_all': False, 'tolerance': None, 'signed_zero_included': True, 'timing_fields_compared': False,
|
|
'scope': 'Scientific outputs, recording integrity and input hashes; independent case/graph audit belongs to compare()'}
|
|
try:
|
|
summary = _read_json(first/'summary.json', used)
|
|
n, edges, steps, name, history = (summary[key] for key in ('nodes', 'edges', 'steps', 'case', 'record_history'))
|
|
require(type(n) is int and type(edges) is int and type(steps) is int and n > 0 and edges > 0 and steps > 0 and
|
|
type(history) is bool and history == (n < 512), 'Invalid repeat dimensions')
|
|
left = _load_run(first, 'first_gpu', n, edges, steps, history, name, used, checks)
|
|
right = _load_run(second, 'second_gpu', n, edges, steps, history, name, used, checks)
|
|
_pair_outputs(left, right, checks, differences, n, steps, history)
|
|
stable = ('case', 'complete', 'pass_all', 'steps', 'completed_ticks', 'nodes', 'edges', 'precision',
|
|
'apl_precision', 'state_units', 'total_spikes', 'requested', 'accepted_requests', 'rejected_requests',
|
|
'arriving_edges', 'accepted_arriving_edges', 'counts_match_sparse_events', 'all_finite', 'record_history',
|
|
'apl_fast_edges_zeroed', 'kc_targets_present', 'effective_weights_raw_sha256', 'apl_schedule',
|
|
'apl_si_conversion', 'build', 'sealed_ae_backend_sha256', 'learning', 'input_sha256')
|
|
for key in stable:
|
|
_check(checks, 'summary.'+key, key in left['summary'] and key in right['summary'] and
|
|
left['summary'][key] == right['summary'][key])
|
|
if not checks['summary.'+key]:
|
|
differences['summary.'+key] = {'first': left['summary'].get(key), 'second': right['summary'].get(key)}
|
|
report.update(case=name, steps=steps, nodes=n, edges=edges, record_history=history,
|
|
total_spikes=len(left['state']['step']),
|
|
explicit_exclusions=['wall_seconds', 'diagnostic_simulation_seconds', 'build_or_cache_seconds',
|
|
'peak_allocated_vram_bytes', 'peak_reserved_vram_bytes'])
|
|
except Exception as error:
|
|
report['error'] = {'type': type(error).__name__, 'message': str(error)}
|
|
return _finish(report, checks, differences, used, out_json)
|