"""Bounded replay benchmark. No brain simulation, training, or weight installation.""" from __future__ import annotations import argparse from datetime import datetime, timezone import hashlib import json import math import os from pathlib import Path import statistics import subprocess import sys import time import traceback import urllib.request import numpy as np from gpu_backend import CudaEligibility, prepare_events REG_SHA = 'b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740' IDENTITY_KEYS = ('kc', 'candidate_edge_id', 'candidate_sources', 'candidate_targets', 'candidate_compartments', 'base_candidate') OUTPUTS = ('sampled_trace', 'final_trace', 'snapshots', 'delta_after') def sha(path): h = hashlib.sha256() with Path(path).open('rb') as stream: for chunk in iter(lambda: stream.read(8 << 20), b''): h.update(chunk) return h.hexdigest() def save(path, value): with Path(path).open('x', encoding='utf-8') as stream: json.dump(value, stream, indent=2, sort_keys=True, allow_nan=False) stream.write('\n') def read(path): return json.loads(Path(path).read_text()) def evidence(a): return dict(dtype=a.dtype.str, shape=list(a.shape), sha256=hashlib.sha256(a.tobytes()).hexdigest()) def contained(root, relative): p = root / relative assert not p.is_symlink() and p.resolve().is_relative_to(root.resolve()), 'Escaping fixture path' return p def idle_queues(): result = {} for port in (8188, 8189, 8190): with urllib.request.urlopen(f'http://192.168.10.51:{port}/queue', timeout=5) as response: q = json.load(response) assert set(('queue_running', 'queue_pending')).issubset(q) result[str(port)] = {key: len(q[key]) for key in ('queue_running', 'queue_pending')} assert not any(result[str(port)].values()), 'ComfyUI has work; benchmark will not compete' return result def gpu_snapshot(uuid): text = subprocess.check_output(['nvidia-smi', '--id=' + uuid, '--query-gpu=uuid,name,pci.bus_id,memory.total,memory.used,utilization.gpu,driver_version', '--format=csv,noheader,nounits'], text=True).strip() fields = [s.strip() for s in text.split(',')] assert fields[0] == uuid return dict(zip(('uuid', 'name', 'pci_bus_id', 'memory_total_mib', 'memory_used_mib', 'utilization_percent', 'driver_version'), fields)) def stats(values): return dict(n=len(values), min=min(values), median=statistics.median(values), mean=statistics.mean(values), max=max(values)) def compare(result, arrays): report = {} for key in OUTPUTS: actual, expected = getattr(result, key), arrays['reference_' + key] exact = actual.dtype == expected.dtype and actual.shape == expected.shape and actual.tobytes() == expected.tobytes() diff = np.abs(actual.astype(np.float64) - expected.astype(np.float64)) report[key] = dict(bitwise_exact=exact, actual=evidence(actual), expected=evidence(expected), max_abs_error=float(np.max(diff)) if diff.size else 0., mean_abs_error=float(np.mean(diff)) if diff.size else 0.) return report def load_fixtures(manifests): fixtures = [] common_identity = None for manifest_path in manifests: m = read(manifest_path) assert m.get('complete') is True and m['registration_sha256'] == REG_SHA root = manifest_path.parent for f in m['fixtures']: npz = contained(root, f['npz']) command_path = contained(root, f['command']) assert sha(npz) == f['npz_sha256'] and sha(command_path) == f['command_sha256'] with np.load(npz, allow_pickle=False) as z: a = {k: z[k].copy() for k in z.files} identity = {k: evidence(a[k]) for k in IDENTITY_KEYS} if common_identity is None: common_identity = identity assert common_identity == identity, 'Full KC/mask/base identity differs across fixtures' assert len(a['kc']) == 4064 and len(a['candidate_edge_id']) == 14232 sources_path = npz.parent / 'reference-sources.json' assert sha(sources_path) == m['files'][sources_path.relative_to(root).as_posix()]['sha256'] source_pins = read(sources_path) expected_sources = dict( conditioning_update_source='96efff678cff910654eb08e86f1a17789e4e5589e340345f419c7027a7d547e2', eligibility_bridge_source='174fb8d14619189e74b095e77a6db18b14c93046dc431c0b55710a5aa2dd2e61', phase4j_source='abb4574da85f00bd9ef61024c65993ef81d42651ff8d8d63c15dd69c6222471e') assert set(source_pins) == set(expected_sources) for role, info in source_pins.items(): assert info['sha256'] == expected_sources[role] and sha(info['path']) == info['sha256'], 'Reference source changed' # Extract constants from the authenticated original bridge, without import/exec. import ast bridge = source_pins['eligibility_bridge_source'] assert bridge['sha256'] == '174fb8d14619189e74b095e77a6db18b14c93046dc431c0b55710a5aa2dd2e61' constants = {} for node in ast.parse(Path(bridge['path']).read_text()).body: if isinstance(node, ast.Assign): for target in node.targets: if isinstance(target, ast.Name) and target.id in ('DT_MS', 'TAU_MS', 'ETA_MV', 'FLOOR_FRACTION', 'POLARITY'): constants[target.id] = ast.literal_eval(node.value) assert constants['POLARITY'] == -1 rule = dict(eta_mv=constants['ETA_MV'], floor_fraction=constants['FLOOR_FRACTION'], decay=math.exp(-constants['DT_MS'] / constants['TAU_MS'])) fixtures.append(dict(group=root.name, record=f, arrays=a, command=read(command_path), rule=rule, input_path=str(npz), manifest_sha256=sha(manifest_path))) assert fixtures return fixtures, common_identity def check_rejections(manifests, backend, rule): reports = [] for path in manifests: for case in read(path).get('invalid_cases', []): root = path.parent npz, command = contained(root, case['npz']), contained(root, case['command']) assert sha(npz) == case['npz_sha256'] and sha(command) == case['command_sha256'] with np.load(npz, allow_pickle=False) as z: a = {k: z[k].copy() for k in z.files} c = read(command) commands = [] if c['us_tick'] is None else [c['us_tick']] stage = 'prepare' try: prepared = prepare_events(a['step'], a['neuron_idx'], a['kc'], c['steps'], commands, n_neurons=c['n_neurons']) stage = 'run' backend.run(prepared, a['candidate_sources'], a['candidate_compartments'], c['compartment'], a['delta_before'], a['base_candidate'], d=c['d_command'], frozen=c['frozen'], **rule) except ValueError as error: assert stage == case['stage'], 'Rejected at a different validation boundary' reports.append(dict(case_name=case['case_name'], rejected=True, stage=stage, expected_exception=case['expected_exception'], message=str(error))) else: raise AssertionError('Candidate accepted invalid case: ' + case['case_name']) return reports def main(args): assert os.environ.get('CUDA_VISIBLE_DEVICES') == args.uuid, 'Use one explicit physical UUID per process' assert args.output.is_absolute() and not args.output.exists() args.output.mkdir() started = time.perf_counter() try: queues_before = idle_queues() gpu_before = gpu_snapshot(args.uuid) initial_manifest_pins = {str(p): sha(p) for p in args.manifest} fixtures, common_identity = load_fixtures(args.manifest) backend = CudaEligibility(device_index=0) cp = backend.cp props = cp.cuda.runtime.getDeviceProperties(0) bus = cp.cuda.runtime.deviceGetPCIBusId(0) if isinstance(bus, bytes): bus = bus.decode() assert bus.lower()[-10:] == gpu_before['pci_bus_id'].lower()[-10:], 'Physical GPU binding differs' device = dict(**gpu_before, runtime_pci_bus_id=bus, compute_capability=backend.device.compute_capability, initialization_seconds=backend.initialization_seconds, cupy_version=cp.__version__, numpy_version=np.__version__, python_version=sys.version, runtime_version=cp.cuda.runtime.runtimeGetVersion(), driver_version_runtime=cp.cuda.runtime.driverGetVersion(), runtime_name=props['name'].decode() if isinstance(props['name'], bytes) else props['name']) source = Path(__file__).resolve().parent commit = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() assert subprocess.check_output(['git', '-C', str(source), 'status', '--porcelain'], text=True) == '', 'Candidate source worktree dirty' save(args.output / 'authority.json', dict(device=device, queues_before=queues_before, candidate_commit=commit, common_full_mapping_identity=common_identity, fixture_manifests=initial_manifest_pins, repeats=20, warmups=3, all_source_files={p.name: sha(p) for p in sorted(source.iterdir()) if p.is_file()}, scope='eligibility/delta replay only; no brain, training, or installation', original_experiment_resumed=False, created_utc=datetime.now(timezone.utc).isoformat())) reports = [] peak_reserved = 0 for i, fixture in enumerate(fixtures): assert time.perf_counter() - started < 900, 'Benchmark wall budget exhausted' idle_queues() a, c, rule = fixture['arrays'], fixture['command'], fixture['rule'] arrays_before = {k: evidence(v) for k, v in a.items()} f_id = fixture['group'] + '--' + fixture['record']['fixture_id'] def run_once(resident=False): begin = time.perf_counter() prepared = prepare_events(a['step'], a['neuron_idx'], a['kc'], c['steps'], a['reference_command_steps'], n_neurons=c['n_neurons']) result = backend.run(prepared, a['candidate_sources'], a['candidate_compartments'], c['compartment'], a['delta_before'], a['base_candidate'], d=c['d_command'], frozen=c['frozen'], **rule, kernel_warmup=3 if resident else 0, kernel_repeats=20 if resident else 1) comparison = compare(result, a) elapsed = time.perf_counter() - begin if not all(r['bitwise_exact'] for r in comparison.values()): save(args.output / (f_id + '-mismatch.json'), comparison) raise AssertionError('GPU bitwise mismatch: ' + f_id) return result, comparison, elapsed cold, comp, cold_wall = run_once() for _ in range(3): run_once() samples = [] for _ in range(20): result, comp, elapsed = run_once() samples.append(dict(**result.timing, replay_wall_including_preparation_and_comparison_seconds=elapsed)) resident, _, resident_wall = run_once(resident=True) peak_reserved = max(peak_reserved, cp.get_default_memory_pool().total_bytes()) assert arrays_before == {k: evidence(v) for k, v in a.items()}, 'Input arrays mutated' report = dict(fixture_id=f_id, input_path=fixture['input_path'], fixture_manifest_sha256=fixture['manifest_sha256'], command=c, rule=rule, first_call_timing=cold.timing, first_call_wall_seconds=cold_wall, raw_warm_samples=samples, resident_timing=resident.timing, summaries={key: stats([s[key] for s in samples]) for key in ('preparation_seconds', 'kernel_seconds', 'end_to_end_seconds', 'replay_wall_including_preparation_and_comparison_seconds')}, comparison=comp, input_unchanged=True, all_repeats_bitwise_exact=True, reference_timing=fixture['record'].get('reference_timing'), audit=result.audit) save(args.output / (f_id + '.json'), report) reports.append(report) print(json.dumps(dict(fixture=f_id, exact=True, median_ms=1000 * report['summaries']['replay_wall_including_preparation_and_comparison_seconds']['median'])), flush=True) # Recheck original exported files at the end, not just at read time. assert initial_manifest_pins == {str(p): sha(p) for p in args.manifest}, 'Fixture manifest changed' load_fixtures(args.manifest) rejections = check_rejections(args.manifest, backend, fixtures[0]['rule']) save(args.output / 'invalid-input-rejections.json', rejections) assert commit == subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() assert subprocess.check_output(['git', '-C', str(source), 'status', '--porcelain'], text=True) == '' result = dict(status='PASS-ELIGIBILITY-UPDATE-REPLAY-EQUIVALENCE', fixtures=len(reports), device=device, candidate_commit=commit, all_fixture_repeats_bitwise_exact=True, observed_cuda_pool_reserved_peak_bytes=peak_reserved, gpu_after=gpu_snapshot(args.uuid), queues_after=idle_queues(), wall_seconds=time.perf_counter()-started, original_experiment_resumed=False, brain_simulation_tested=False, rejected_invalid_cases=len(rejections), full_training_speedup_measured=False, learning_success_claimed=False, coverage_limit='available prefix + fixed synthetic edge cases, not all registered conditions', files={p.name: sha(p) for p in sorted(args.output.iterdir()) if p.is_file()}) save(args.output / 'summary.json', result) print(json.dumps(result), flush=True) except Exception: save(args.output / 'failure.json', dict(exit_code=1, traceback=traceback.format_exc(), wall_seconds=time.perf_counter()-started, original_experiment_resumed=False)) raise if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--manifest', type=Path, action='append', required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--uuid', required=True) main(parser.parse_args())