"""Tiny temporary CPU fixtures, never scientific events or a real brain run.""" import copy import json from pathlib import Path import tempfile from types import SimpleNamespace import unittest from unittest.mock import patch import numpy as np import conditioning_epoch as m import conditioning_update as update from test_conditioning_update import BRIDGE, SOURCE def save(path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, sort_keys=True, allow_nan=False), encoding='utf-8') class TinyFixture: def __init__(self, root): self.root = root self.inputs = root / 'inputs'; self.inputs.mkdir() self.case = self.inputs / 'case'; self.case.mkdir() self.graph = self.inputs / 'graph'; self.graph.mkdir() self.apl = self.inputs / 'apl'; self.apl.mkdir() self.checkpoint = root / 'runtime'; self.checkpoint.mkdir() self.authority_dir = root / 'authority'; self.authority_dir.mkdir() self.output_parent = root / 'outputs'; self.output_parent.mkdir() self.authority_path = self.authority_dir / 'epoch.json' self.mask = np.array([1, 3], np.int64) self.kc = np.array([1, 3], np.int32) self.base = np.array([.275, .55], np.float32) self.before = np.zeros(2, np.float32) graph = dict(nodes=np.arange(5, dtype=np.int32), edge_id=np.arange(6, dtype=np.int64), src=np.array([0, 1, 1, 3, 3, 4], np.int32), dst=np.array([1, 2, 4, 2, 4, 0], np.int32), signed_raw=np.ones(6, np.int64), ptr=np.array([0, 1, 3, 3, 5, 6], np.int64), weights=np.full(6, .000275)) for name, array in graph.items(): np.save(self.graph / (name + '.npy'), array) for name, array in dict(input_matrix=np.zeros((2, 5), np.float32), output_matrix=np.zeros((2, 2), np.float32), kc_globals=self.kc, apl_globals=np.array([0, 4], np.int32)).items(): np.save(self.apl / (name + '.npy'), array) np.save(self.inputs / 'kc.npy', self.kc) np.save(self.inputs / 'mask.npy', self.mask) np.save(self.inputs / 'base.npy', self.base) np.savez_compressed(self.inputs / 'mapping.npz', edge_id=self.mask, source_neuron_idx=graph['src'][self.mask], target_neuron_idx=graph['dst'][self.mask], target_compartment_verbatim=np.array(["y5B'2a", 'a1'])) np.savez_compressed(self.inputs / 'before.npz', edge_id=self.mask, plastic_delta_mv=self.before) save(self.inputs / 'coeff.json', {'synthetic': True}) source_root = Path(m.__file__).resolve().parent source_files = dict(runtime_light_source=source_root / 'runtime_light.py', runtime_continuation_source=source_root / 'runtime_continuation.py', conditioning_update_source=source_root / 'conditioning_update.py', eligibility_bridge_source=BRIDGE.resolve(), phase4j_source=SOURCE.resolve(), am_overlay_source=source_root.parent / 'wholebrain-plastic-interface-v1/overlay.py') metadata = dict(schema='phase4an-overlay-v1', overlay_name='zero', overlay_sha256=m._digest(self.inputs / 'before.npz'), mask_path=str(self.inputs / 'mask.npy'), mask_sha256=m._digest(self.inputs / 'mask.npy'), base_candidate_path=str(self.inputs / 'base.npy'), base_candidate_sha256=m._digest(self.inputs / 'base.npy'), aj_model_dir=str(self.graph), am_overlay_path=str(source_files['am_overlay_source']), am_overlay_sha256=m._digest(source_files['am_overlay_source'])) save(self.inputs / 'before.json', metadata) spec = dict(name='synthetic', steps=4, max_steps=4, dt_ms=.1, duration_ms=.4, apl_delay_steps=18, record_history=False, within_trial_weight_updates=False, graph_dir=str(self.graph), apl_bundle_dir=str(self.apl), coefficients_path=str(self.inputs / 'coeff.json')) save(self.case / 'spec.json', spec) np.savez_compressed(self.case / 'requests.npz', tick=np.array([0], np.int32), neuron_idx=np.array([1], np.int32)) state = dict(step=np.array([0], np.int32), neuron_idx=np.array([1], np.int32), counts=np.array([0, 1, 0, 0, 0], np.int32), v=np.full(5, -.052), g=np.zeros(5), ref=np.zeros(5, np.int32)) np.savez_compressed(self.checkpoint / 'state.npz', **state) (self.checkpoint / 'overlay-input.npz').write_bytes((self.inputs / 'before.npz').read_bytes()) (self.checkpoint / 'overlay-manifest.json').write_bytes((self.inputs / 'before.json').read_bytes()) summary = dict(nodes=5, edges=6, diagnostic_mode='light', effective_weights_raw_sha256='e' * 64) save(self.checkpoint / 'summary.json', summary) provenance = dict(input_sha256={str(path): m._digest(path) for path in (self.case / 'spec.json', self.case / 'requests.npz', self.inputs / 'before.npz', self.inputs / 'before.json')}) save(self.checkpoint / 'runtime-provenance.json', provenance) save(self.checkpoint / 'overlay-audit.json', dict(delta_raw_sha256=self.raw(self.before), effective_weights_raw_sha256='e' * 64)) paths = dict(spec=self.case / 'spec.json', requests=self.case / 'requests.npz', overlay_before=self.inputs / 'before.npz', overlay_manifest_before=self.inputs / 'before.json', state=self.checkpoint / 'state.npz', kc=self.inputs / 'kc.npy', canonical_kc=self.apl / 'kc_globals.npy', candidate_map=self.inputs / 'mapping.npz', mask=self.inputs / 'mask.npy', base_candidate=self.inputs / 'base.npy', **source_files) self.value = dict(schema=m.SCHEMA, epoch_id='synthetic/epoch-0', diagnostic_mode='light', command=dict(steps=4, compartment="y5B'2a", us_tick=1, d_command=1., frozen=False, trace_backend='numpy'), checkpoint_dir=str(self.checkpoint), checkpoint_manifest_sha256=None, files={role: dict(path=str(path), sha256=m._digest(path)) for role, path in paths.items()}, frozen_inputs={str(path): m._digest(path) for path in [ *(self.graph / name for name in m.GRAPH_FILES), *(self.apl / name for name in m.APL_FILES), self.inputs / 'coeff.json']}) self.reseal() @staticmethod def raw(array): return __import__('hashlib').sha256(array.tobytes()).hexdigest() def reseal(self): files = {path.name: m._digest(path) for path in self.checkpoint.iterdir() if path.name != 'checkpoint-manifest.json'} save(self.checkpoint / 'checkpoint-manifest.json', dict(files_sha256=files)) self.value['checkpoint_manifest_sha256'] = m._digest(self.checkpoint / 'checkpoint-manifest.json') for entry in self.value['files'].values(): entry['sha256'] = m._digest(entry['path']) for path in self.value['frozen_inputs']: self.value['frozen_inputs'][path] = m._digest(path) save(self.authority_path, self.value) def load_checkpoint(self, directory, pin): self.asserted_loads += 1 directory = Path(directory) m._require(m._digest(directory / 'checkpoint-manifest.json') == pin, 'synthetic seal pin mismatch') manifest = m._read(directory / 'checkpoint-manifest.json') actual = {path.name: m._digest(path) for path in directory.iterdir() if path.name != 'checkpoint-manifest.json'} m._require(manifest['files_sha256'] == actual, 'synthetic sealed inventory mismatch') return dict(fresh=True, global_tick_offset=0, steps=4, summary=m._read(directory / 'summary.json'), provenance=m._read(directory / 'runtime-provenance.json'), manifest=manifest, state=m._npz(directory / 'state.npz', ('step', 'neuron_idx', 'counts', 'v', 'g', 'ref')), overlay=m._read(directory / 'overlay-audit.json')) def load_overlay(self, file, manifest): meta = m._read(manifest) m._require(meta['overlay_sha256'] == m._digest(file), 'synthetic overlay SHA mismatch') value = m._npz(file, ('edge_id', 'plastic_delta_mv')) value['base_candidate_mv'] = np.load(meta['base_candidate_path'], allow_pickle=False) if meta['overlay_name'] == 'learned': expected = {'schema', 'overlay_name', 'overlay_sha256', 'mask_path', 'mask_sha256', 'base_candidate_path', 'base_candidate_sha256', 'aj_model_dir', 'am_overlay_path', 'am_overlay_sha256', 'parent_snapshot_sha256', 'eligibility_v2_source_sha256'} m._require(set(meta) == expected, 'learned manifest fields differ from real runtime contract') return None, value, meta, {} def run(self, out=None): self.asserted_loads = 0 parsed = m._authority(self.authority_path, m._digest(self.authority_path)) c = SimpleNamespace(load_checkpoint=self.load_checkpoint) r = SimpleNamespace(PINS={}, validate_spec=lambda value, _: value, load_overlay=self.load_overlay, _raw=self.raw) return m._coordinate(*parsed, self.output_parent / 'result' if out is None else out, (c, r, update), (5, 6, 2, 2)) class EpochPostprocessing(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.f = TinyFixture(Path(self.tmp.name).resolve()) def tearDown(self): self.tmp.cleanup() def test_exclusive_complete_output_and_distinct_parent_hashes(self): chain = self.f.run() output = self.f.output_parent / 'result' self.assertEqual({p.name for p in output.iterdir()}, {'delta-after.npz', 'overlay-manifest.json', 'update-receipt.json', 'chain-receipt.json'}) meta = m._read(output / 'overlay-manifest.json') self.assertEqual(meta['parent_snapshot_sha256'], self.f.value['files']['overlay_before']['sha256']) self.assertNotEqual(meta['parent_snapshot_sha256'], self.f.value['checkpoint_manifest_sha256']) self.assertEqual(chain['before_overlay_manifest_sha256'], self.f.value['files']['overlay_manifest_before']['sha256']) self.assertEqual(chain['source_neural_checkpoint_manifest_sha256'], self.f.value['checkpoint_manifest_sha256']) self.assertEqual(self.f.asserted_loads, 2) self.assertFalse(chain['weights_installed']) self.assertFalse(chain['registration_authority_independently_proven']) after = m._npz(output / 'delta-after.npz', ('edge_id', 'plastic_delta_mv')) self.assertLess(after['plastic_delta_mv'][0], 0) self.assertEqual(after['plastic_delta_mv'][1].view(np.uint32), 0) with self.assertRaisesRegex(ValueError, 'Output must be new'): self.f.run() def test_every_external_sha_before_processing(self): with self.assertRaisesRegex(ValueError, 'Authority differs'): m._authority(self.f.authority_path, '0' * 64) with (self.f.inputs / 'mapping.npz').open('ab') as stream: stream.write(b'changed') with self.assertRaisesRegex(ValueError, 'SHA mismatch'): self.f.run() self.assertFalse((self.f.output_parent / 'result').exists()) def test_sealed_event_substitution_and_unbound_case_are_rejected(self): path = self.f.checkpoint / 'runtime-provenance.json' provenance = m._read(path) provenance['input_sha256'][str(self.f.case / 'requests.npz')] = 'f' * 64 save(path, provenance) self.f.reseal() with self.assertRaisesRegex(ValueError, 'did not bind authorized requests'): self.f.run() def test_changed_runtime_overlay_cannot_match_only_its_own_seal(self): (self.f.checkpoint / 'overlay-input.npz').write_bytes(b'different') self.f.reseal() with self.assertRaisesRegex(ValueError, 'used a different overlay'): self.f.run() def test_candidate_endpoint_and_canonical_kc_must_really_match(self): path = self.f.inputs / 'mapping.npz' mapping = m._npz(path, ('edge_id', 'source_neuron_idx', 'target_neuron_idx', 'target_compartment_verbatim')) mapping['target_neuron_idx'][0] = 4 np.savez_compressed(path, **mapping) self.f.reseal() with self.assertRaisesRegex(ValueError, 'endpoints differ'): self.f.run() mapping['target_neuron_idx'][0] = 2 np.savez_compressed(path, **mapping) np.save(self.f.inputs / 'kc.npy', np.array([1, 4], np.int32)) self.f.reseal() with self.assertRaisesRegex(ValueError, 'KC differs'): self.f.run() def test_command_steps_and_full_frozen_inventory_required(self): self.f.value['command']['steps'] = 5 self.f.reseal() with self.assertRaisesRegex(ValueError, 'Command steps differ'): self.f.run() self.f.value['command']['steps'] = 4 del self.f.value['frozen_inputs'][str(self.f.graph / 'signed_raw.npy')] self.f.reseal() with self.assertRaisesRegex(ValueError, 'authority is incomplete'): self.f.run() def test_protected_directories_and_existing_output_fail_before_write(self): for path in (self.f.checkpoint / 'new', self.f.inputs / 'new', self.f.graph / 'new', self.f.authority_dir / 'new', self.f.checkpoint): with self.subTest(path=path), self.assertRaisesRegex(ValueError, 'Output must be new'): self.f.run(path) def test_partial_is_preserved_and_not_retryable(self): with patch.object(self.f, 'load_overlay', side_effect=[ self.f.load_overlay(self.f.inputs / 'before.npz', self.f.inputs / 'before.json'), ValueError('synthetic output validation failed')]): with self.assertRaisesRegex(ValueError, 'output validation'): self.f.run() output = self.f.output_parent / 'result' self.assertTrue((output / 'postprocess-error.json').is_file()) self.assertTrue((output / 'delta-after.npz').is_file()) self.assertFalse((output / 'chain-receipt.json').exists()) with self.assertRaisesRegex(ValueError, 'Output must be new'): self.f.run() def test_actual_pinned_dependencies_load_without_gpu(self): _, paths, pins, _, _ = m._authority(self.f.authority_path, m._digest(self.f.authority_path)) c, r, u = m._production_dependencies(paths, pins) self.assertEqual(Path(r.__file__).resolve(), paths['runtime_light_source']) self.assertEqual(c.N, 166700) self.assertEqual(u.BRIDGE_SHA256, update.BRIDGE_SHA256) for role in m.SOURCE_PINS: self.assertEqual(m._digest(paths[role]), m.SOURCE_PINS[role]) def test_production_dimensions_cannot_be_set_by_authority(self): self.assertEqual(m.DIMENSIONS, (166700, 25582938, 4064, 14232)) self.f.value['small_test_mode'] = True self.f.reseal() with self.assertRaisesRegex(ValueError, 'fields/schema'): m.postprocess_epoch(self.f.authority_path, m._digest(self.f.authority_path), self.f.output_parent / 'result') def test_public_entry_rejects_tiny_population_even_with_valid_fixture_seal(self): self.f.asserted_loads = 0 c = SimpleNamespace(load_checkpoint=self.f.load_checkpoint) r = SimpleNamespace(PINS={}, validate_spec=lambda value, _: value, load_overlay=self.f.load_overlay, _raw=self.f.raw) with patch.object(m, '_production_dependencies', return_value=(c, r, update)): with self.assertRaisesRegex(ValueError, 'Runtime population/mode'): m.postprocess_epoch(self.f.authority_path, m._digest(self.f.authority_path), self.f.output_parent / 'result') self.assertFalse((self.f.output_parent / 'result').exists()) def test_authorized_event_file_cannot_be_an_unsealed_copy(self): other = self.f.inputs / 'state-copy.npz' other.write_bytes((self.f.checkpoint / 'state.npz').read_bytes()) self.f.value['files']['state'] = dict(path=str(other), sha256=m._digest(other)) self.f.reseal() with self.assertRaisesRegex(ValueError, 'Events must come from this checkpoint'): self.f.run() def test_late_input_change_preserves_partial_but_never_complete_chain(self): original = self.f.load_checkpoint calls = 0 def late_change(directory, pin): nonlocal calls calls += 1 result = original(directory, pin) if calls == 2: with (self.f.graph / 'weights.npy').open('ab') as stream: stream.write(b'changed after outputs were written') return result with patch.object(self.f, 'load_checkpoint', side_effect=late_change): with self.assertRaisesRegex(ValueError, 'SHA mismatch'): self.f.run() output = self.f.output_parent / 'result' self.assertTrue((output / 'postprocess-error.json').is_file()) self.assertFalse((output / 'chain-receipt.json').exists()) def test_zero_d_and_frozen_return_identity_without_installation(self): for index, (d, frozen) in enumerate(((0., False), (1., True))): self.f.value['command'].update(d_command=d, frozen=frozen) self.f.reseal() self.f.run(self.f.output_parent / str(index)) value = m._npz(self.f.output_parent / str(index) / 'delta-after.npz', ('edge_id', 'plastic_delta_mv')) self.assertEqual(value['plastic_delta_mv'].tobytes(), self.f.before.tobytes()) if __name__ == '__main__': unittest.main()