Add bounded same-state update validation and functional gates
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,573 @@
|
||||
"""Synthetic CPU admission/state/AST tests only. No GPU or scientific input."""
|
||||
import ast
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace, ModuleType
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
import updated_continuation as m
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
AJ = ROOT / 'apl-timing-lab-v1/gpu_timing.py'
|
||||
|
||||
|
||||
def save(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value, sort_keys=True, allow_nan=False), encoding='utf-8')
|
||||
|
||||
|
||||
def pin(path):
|
||||
return dict(path=str(path), sha256=m._digest(path))
|
||||
|
||||
|
||||
class ArrayTarget:
|
||||
def __init__(self, a):
|
||||
self.a = np.array(a, copy=True)
|
||||
|
||||
def copy_(self, a):
|
||||
self.a[...] = a
|
||||
|
||||
def any(self):
|
||||
return np.asarray(self.a.any())
|
||||
|
||||
def detach(self):
|
||||
return self
|
||||
|
||||
def cpu(self):
|
||||
return self
|
||||
|
||||
def numpy(self):
|
||||
return self.a
|
||||
|
||||
|
||||
def tiny_parent():
|
||||
parent = dict(global_next_tick=41, state=dict(v=np.array([-.051, -.047], np.float64),
|
||||
g=np.array([-0., .00013], np.float64), ref=np.array([22, 3], np.int32)),
|
||||
terminal=dict(spike_ring=np.zeros((19, 2), np.uint8), next_tick=np.asarray(5, np.int64),
|
||||
apl_pending=np.arange(36, dtype=np.float32).reshape(18, 2), apl_state=np.array([.5, 1.], np.float32)))
|
||||
parent['terminal']['spike_ring'][4, 1] = 1
|
||||
return parent
|
||||
|
||||
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def runtime(self):
|
||||
obj = object.__new__(m.UpdatedRuntime)
|
||||
obj.parent, obj.installations, obj.restorations = tiny_parent(), 1, 0
|
||||
obj.authority = {'value': {'expected_effective_weight_after_sha256': 'a'*64}}
|
||||
obj.torch = SimpleNamespace(float64=np.float64, float32=np.float32, int32=np.int32, uint8=np.uint8)
|
||||
return obj
|
||||
|
||||
def targets(self):
|
||||
state = [ArrayTarget(np.zeros(2, dtype)) for dtype in (np.float64, np.float64, np.int32, np.int32)]
|
||||
return [*state, ArrayTarget(np.zeros((19, 2), np.uint8)), ArrayTarget(np.zeros((18, 2), np.float32)),
|
||||
SimpleNamespace(state=ArrayTarget(np.zeros(2, np.float32))), lambda a, dtype: np.array(a, dtype=dtype)]
|
||||
|
||||
def test_restore_all_bytes_and_no_retroactive_g_rescale(self):
|
||||
obj, args = self.runtime(), self.targets()
|
||||
before = {name:a.tobytes() for name,a in obj.parent['state'].items()}
|
||||
obj.restore(*args)
|
||||
self.assertEqual(args[1].a.tobytes(), before['g'])
|
||||
self.assertEqual(args[0].a.tobytes(), before['v'])
|
||||
self.assertEqual(args[2].a.tobytes(), before['ref'])
|
||||
self.assertFalse(args[3].a.any())
|
||||
self.assertTrue(m._same(args[4].a, m.C.rotate_spike_ring(obj.parent['terminal']['spike_ring'], 5)))
|
||||
self.assertTrue(m._same(args[5].a, obj.parent['terminal']['apl_pending']))
|
||||
self.assertTrue(m._same(args[6].state.a, obj.parent['terminal']['apl_state']))
|
||||
self.assertFalse(obj.restore_audit['existing_g_rescaled'])
|
||||
self.assertTrue(obj.restore_audit['pending_spike_arrivals_use_current_installed_weight'])
|
||||
self.assertEqual(before, {name:a.tobytes() for name,a in obj.parent['state'].items()})
|
||||
with self.assertRaisesRegex(ValueError, 'once'):
|
||||
obj.restore(*args)
|
||||
|
||||
def test_restore_rejects_nonzero_counts_or_before_installation(self):
|
||||
obj, args = self.runtime(), self.targets()
|
||||
obj.installations = 0
|
||||
with self.assertRaises(ValueError): obj.restore(*args)
|
||||
obj.installations = 1
|
||||
args[3].a[0] = 1
|
||||
with self.assertRaisesRegex(ValueError, 'counts'): obj.restore(*args)
|
||||
|
||||
def test_copy_corruption_is_not_hidden_by_receipt(self):
|
||||
obj, args = self.runtime(), self.targets()
|
||||
args[1].copy_ = lambda value: args[1].a.fill(0.)
|
||||
with self.assertRaisesRegex(ValueError, 'state bytes: g'): obj.restore(*args)
|
||||
|
||||
def test_authorized_new_hash_not_parent_old_hash_and_no_double_install(self):
|
||||
obj = self.runtime(); obj.installations = 0
|
||||
def synthetic_original_install(target, *args, **kwargs):
|
||||
target.installations += 1
|
||||
target.effective_sha = 'a'*64
|
||||
obj.parent['summary'] = {'effective_weights_raw_sha256':'b'*64}
|
||||
with patch.object(m.R._Runtime, 'install', synthetic_original_install):
|
||||
obj.install(None)
|
||||
self.assertEqual(obj.installations, 1)
|
||||
with self.assertRaisesRegex(ValueError, 'Double'): obj.install(None)
|
||||
obj.installations = 0
|
||||
obj.authority['value']['expected_effective_weight_after_sha256'] = 'c'*64
|
||||
with patch.object(m.R._Runtime, 'install', synthetic_original_install):
|
||||
with self.assertRaisesRegex(ValueError, 'authorized after'): obj.install(None)
|
||||
|
||||
def test_actual_source_AST_keeps_only_frozen_restore_and_admission(self):
|
||||
for mode in ('reference', 'light'):
|
||||
aj = ModuleType('synthetic_aj'); aj.__file__ = str(AJ)
|
||||
old = dict(aj.__dict__)
|
||||
runtime = SimpleNamespace(write_json=lambda *_:None, audit_sequence=lambda *_:None)
|
||||
function, audit = m.compile_updated(aj, runtime, mode)
|
||||
self.assertTrue(audit['full_AST_rollback_identical'])
|
||||
self.assertTrue(audit['kernels_sorting_and_local_ring_math_unchanged'])
|
||||
self.assertEqual(old, aj.__dict__)
|
||||
self.assertIs(function.__globals__['_runtime'], runtime)
|
||||
self.assertIs(function.__globals__['_continuation'], runtime)
|
||||
self.assertEqual(function.__globals__['__file__'], str(Path(m.__file__).resolve()))
|
||||
|
||||
def test_pending_source_uses_new_weight_and_old_APL_sample(self):
|
||||
# Independent numerical toy: a source emitted before boundary arrives
|
||||
# afterwards; no captured edge weight is stored in the source ring.
|
||||
for cut in (5, 17, 18, 19, 20, 41):
|
||||
source_tick = cut - 1
|
||||
ring = np.zeros((19, 2), np.uint8); ring[source_tick % 19, 0] = 1
|
||||
child = m.C.rotate_spike_ring(ring, cut)
|
||||
g = np.float64(.00013)
|
||||
delta = np.float32(-.00275)
|
||||
base = np.float64(.000275)
|
||||
after = np.add(base, np.multiply(np.float64(delta), np.float64(.001)))
|
||||
arrived = []
|
||||
for tick in range(19):
|
||||
child[tick % 19] = 0
|
||||
if cut+tick >= 18 and child[(tick-18) % 19, 0]:
|
||||
g = np.add(g, after); arrived.append(tick)
|
||||
self.assertEqual(arrived, [17])
|
||||
self.assertEqual(g.tobytes(), np.add(np.float64(.00013), after).tobytes())
|
||||
self.assertNotEqual(g.tobytes(), np.add(np.float64(.00013), base).tobytes())
|
||||
initial = np.arange(36, dtype=np.float32).reshape(18, 2)
|
||||
generated = np.ones((5, 2), np.float32)
|
||||
joined = np.concatenate((initial, generated))
|
||||
self.assertTrue(m.C.audit_carried_sequence(generated, joined[:5], joined[5:], initial)['pass_all'])
|
||||
source = (ROOT/'author-runtime-lab-v1/author_ext.cu').read_text()
|
||||
self.assertIn('weights[edges[j]]', source)
|
||||
|
||||
|
||||
class AdmissionFixture:
|
||||
"""Private tiny input scaffolding. Actual production loaders are mocked."""
|
||||
def __init__(self, root):
|
||||
self.root=root; self.case=root/'case'; self.case.mkdir()
|
||||
self.parentdir=root/'parent'; self.parentdir.mkdir()
|
||||
self.graph=root/'graph'; self.graph.mkdir(); self.apl=root/'apl'; self.apl.mkdir()
|
||||
self.coeff=root/'coeff.json'; save(self.coeff, {})
|
||||
for name in m.GRAPH_FILES: (self.graph/name).write_bytes(b'synthetic graph '+name.encode())
|
||||
for name in m.APL_FILES: (self.apl/name).write_bytes(b'synthetic apl '+name.encode())
|
||||
self.before=root/'before.npz'; self.before.write_bytes(b'before snapshot')
|
||||
self.bm=root/'before.json'; save(self.bm, {'synthetic':True})
|
||||
self.after=root/'after.npz'; self.after.write_bytes(b'after snapshot')
|
||||
self.am=root/'after.json'; save(self.am, {'synthetic':False})
|
||||
(self.parentdir/'overlay-input.npz').write_bytes(self.before.read_bytes())
|
||||
(self.parentdir/'overlay-manifest.json').write_bytes(self.bm.read_bytes())
|
||||
save(self.parentdir/m.MANIFEST, {'synthetic':True})
|
||||
self.parentpin=m._digest(self.parentdir/m.MANIFEST)
|
||||
self.spec=dict(steps=5,max_steps=5,dt_ms=.1,duration_ms=.5,apl_delay_steps=18,
|
||||
record_history=False,within_trial_weight_updates=False,model={'fixed':1},apl={'fixed':2},
|
||||
graph_dir=str(self.graph),apl_bundle_dir=str(self.apl),coefficients_path=str(self.coeff),
|
||||
parent_checkpoint_manifest_sha256=self.parentpin,global_tick_offset=41)
|
||||
save(self.case/'spec.json',self.spec)
|
||||
np.savez_compressed(self.case/'requests.npz',tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32))
|
||||
lab=ROOT/'olfactory-conditioning-lab-v1'
|
||||
self.sources={name:lab/(name.removesuffix('_source')+'.py') for name in m.SOURCE_PINS}
|
||||
self.sources['phase4j_source']=ROOT/'models/male-cns-lif/phase4j.py'
|
||||
if not self.sources['phase4j_source'].exists(): self.sources['phase4j_source']=ROOT/'mb-physiology-v1/phase4j.py'
|
||||
self.sources['am_overlay_source']=ROOT/'wholebrain-plastic-interface-v1/overlay.py'
|
||||
self.sources['eligibility_bridge_source']=ROOT/'wholebrain-plastic-interface-v1/eligibility_bridge.py'
|
||||
self.sources['updated_continuation_source']=Path(m.__file__).resolve()
|
||||
self.sources['cpu_validator_source']=Path(m.__file__).resolve().parent/'validate_update_cpu.py'
|
||||
frozen={str(p):m._digest(p) for p in [*(self.graph/n for n in m.GRAPH_FILES),*(self.apl/n for n in m.APL_FILES),self.coeff]}
|
||||
frozen.update({str((ROOT/relative).resolve()):sha for relative,sha in m.R.PINS.items()})
|
||||
self.value=dict(schema=m.SCHEMA,boundary_id='synthetic',registration_sha256='a'*64,
|
||||
transition='epoch_update',diagnostic_mode='light',input_policy='empty_idle',
|
||||
case={'spec':pin(self.case/'spec.json'),'requests':pin(self.case/'requests.npz')},
|
||||
parent={'directory':str(self.parentdir),'checkpoint_manifest_sha256':self.parentpin},global_tick_offset=41,
|
||||
overlay_before={'file':pin(self.before),'manifest':pin(self.bm)},overlay_after={'file':pin(self.after),'manifest':pin(self.am)},
|
||||
expected_effective_weight_before_sha256='b'*64,expected_effective_weight_after_sha256='c'*64,
|
||||
update=None,cpu_validation=None,source_files={k:pin(p) for k,p in self.sources.items()},frozen_inputs=frozen)
|
||||
self.path=root/'authority.json'; self.refresh()
|
||||
self.plan={'edge_id':np.array([1],np.int64),'plastic_delta_mv':np.array([0],np.float32),'base_candidate_mv':np.array([.275],np.float32)}
|
||||
self.meta=dict(mask_path=str(root/'mask.npy'),mask_sha256='m'*64,base_candidate_path=str(root/'base.npy'),
|
||||
base_candidate_sha256='d'*64,aj_model_dir=str(ROOT/'apl-timing-lab-v1'),
|
||||
am_overlay_path=str(self.sources['am_overlay_source']),am_overlay_sha256=m.R.AM_OVERLAY_SHA)
|
||||
self.parent=dict(fresh=True,global_tick_offset=0,global_next_tick=41,steps=41,directory=self.parentdir,
|
||||
manifest_sha256=self.parentpin,manifest={'schema':'phase4an-checkpoint-v1'},
|
||||
summary={'effective_weights_raw_sha256':'b'*64,'input_sha256':{'coefficients.json':m._digest(self.coeff),'base_weights.npy':m._digest(self.graph/'weights.npy')}},
|
||||
overlay={'delta_raw_sha256':m.R._raw(self.plan['plastic_delta_mv'])},ancestry_directories=[self.parentdir])
|
||||
self.helper=object()
|
||||
|
||||
def refresh(self): save(self.path,self.value)
|
||||
|
||||
def admit(self, effective=None):
|
||||
with patch.object(m,'load_checkpoint',return_value=self.parent), patch.object(m.C,'_parent_spec',return_value=self.spec), \
|
||||
patch.object(m.R,'load_overlay',return_value=(self.helper,self.plan,self.meta,{})), \
|
||||
patch.object(m,'_expected_effective',side_effect=['b'*64,'c'*64] if effective is None else effective), \
|
||||
patch.object(m,'_epoch_update',return_value={'pass_all':True}) as ep:
|
||||
result=m.validate_authority(self.path,m._digest(self.path))
|
||||
return result,ep.call_count
|
||||
|
||||
|
||||
class AdmissionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory(); self.f=AdmissionFixture(Path(self.tmp.name).resolve())
|
||||
def tearDown(self): self.tmp.cleanup()
|
||||
|
||||
def test_all_actual_frozen_sources_and_return_contract(self):
|
||||
result,calls=self.f.admit()
|
||||
self.assertEqual(calls,1)
|
||||
for key in ('value','authority','authority_sha256','paths','pins','parent','case_dir','spec','helper',
|
||||
'sources','before_plan','after_plan','before_meta','after_meta'):
|
||||
self.assertIn(key,result)
|
||||
self.assertEqual(result['update_root_authority_sha256'],m._digest(self.f.path))
|
||||
self.assertNotIn('torch',result)
|
||||
|
||||
def test_wrong_absolute_clock_and_wrong_case_parent(self):
|
||||
for field,new in (('global_tick_offset',40),('parent_checkpoint_manifest_sha256','f'*64)):
|
||||
with self.subTest(field=field):
|
||||
original=copy.deepcopy(self.f.spec)
|
||||
self.f.spec[field]=new;save(self.f.case/'spec.json',self.f.spec)
|
||||
self.f.value['case']['spec']=pin(self.f.case/'spec.json');self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'Case parent/clock'):self.f.admit()
|
||||
self.f.spec=original;save(self.f.case/'spec.json',original)
|
||||
self.f.value['case']['spec']=pin(self.f.case/'spec.json');self.f.refresh()
|
||||
self.f.value['global_tick_offset']=42;self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'absolute continuation clock'):self.f.admit()
|
||||
|
||||
def test_second_update_is_rejected(self):
|
||||
self.f.parent['fresh']=False
|
||||
with self.assertRaisesRegex(ValueError,'only once'):self.f.admit()
|
||||
|
||||
def test_authority_hash_and_actual_before_copy_not_self_asserted(self):
|
||||
with self.assertRaisesRegex(ValueError,'changed'):
|
||||
m.validate_authority(self.f.path,'e'*64)
|
||||
(self.f.parentdir/'overlay-input.npz').write_bytes(b'wrong bytes')
|
||||
with self.assertRaisesRegex(ValueError,'Before snapshot'):self.f.admit()
|
||||
|
||||
def test_empty_idle_rejects_requests_probe_does_not_claim_semantics(self):
|
||||
np.savez_compressed(self.f.case/'requests.npz',tick=np.array([0],np.int32),neuron_idx=np.array([1],np.int32))
|
||||
self.f.value['case']['requests']=pin(self.f.case/'requests.npz');self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'empty_idle'):self.f.admit()
|
||||
self.f.value['input_policy']='externally_pinned_CS_probe';self.f.refresh()
|
||||
self.assertEqual(self.f.admit()[0]['value']['input_policy'],'externally_pinned_CS_probe')
|
||||
|
||||
def test_extra_fields_or_source_replacement_fail(self):
|
||||
self.f.value['trust_me']=True;self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'schema'):self.f.admit()
|
||||
del self.f.value['trust_me']
|
||||
self.f.value['source_files']['conditioning_update_source']['sha256']='f'*64;self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'Frozen source'):self.f.admit()
|
||||
|
||||
def test_missing_compiled_pin_not_accepted(self):
|
||||
del self.f.value['frozen_inputs'][str((ROOT/'author-runtime-lab-v1/author_ext.cu').resolve())];self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'dependency authority'):self.f.admit()
|
||||
|
||||
def test_identical_CPU_validator_copy_is_not_the_actual_executor_path(self):
|
||||
copied=self.f.root/'validator-copy.py'
|
||||
copied.write_bytes(self.f.sources['cpu_validator_source'].read_bytes())
|
||||
self.f.value['source_files']['cpu_validator_source']=pin(copied);self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'validator source path'):self.f.admit()
|
||||
|
||||
def idle(self):
|
||||
self.f.value.update(transition='idle',update=None,cpu_validation=None,
|
||||
expected_effective_weight_after_sha256='b'*64,overlay_after=copy.deepcopy(self.f.value['overlay_before']))
|
||||
self.f.parent.update(fresh=False,manifest={'schema':m.CHECKPOINT_SCHEMA},input_policy='empty_idle',
|
||||
update_root_authority_sha256='e'*64)
|
||||
self.f.refresh()
|
||||
|
||||
def test_idle_retains_original_update_root_no_E_or_cpu_recompute(self):
|
||||
self.idle()
|
||||
result,calls=self.f.admit(['b'*64,'b'*64])
|
||||
self.assertEqual(calls,0)
|
||||
self.assertEqual(result['update_root_authority_sha256'],'e'*64)
|
||||
self.assertIsNone(result['paths']['epoch_authority'])
|
||||
self.assertIsNone(result['paths']['cpu_validation'])
|
||||
|
||||
def test_probe_parent_cannot_flow_back_into_idle_or_another_probe(self):
|
||||
self.idle();self.f.parent['input_policy']='externally_pinned_CS_probe'
|
||||
for policy in ('empty_idle','externally_pinned_CS_probe'):
|
||||
self.f.value['input_policy']=policy;self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'probe branch'):self.f.admit(['b'*64,'b'*64])
|
||||
|
||||
def test_idle_cannot_authorize_a_second_snapshot_change(self):
|
||||
self.idle()
|
||||
self.f.value['overlay_after']={'file':pin(self.f.after),'manifest':pin(self.f.am)};self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'Idle changed'):self.f.admit(['b'*64,'b'*64])
|
||||
self.idle();self.f.value['update']={};self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'no new update'):self.f.admit(['b'*64,'b'*64])
|
||||
|
||||
def test_wrong_expected_weight_hash(self):
|
||||
self.f.value['expected_effective_weight_after_sha256']='e'*64;self.f.refresh()
|
||||
with self.assertRaisesRegex(ValueError,'After effective'):self.f.admit()
|
||||
|
||||
def test_output_and_build_reject_entire_ancestry_before_Torch(self):
|
||||
admitted,_=self.f.admit()
|
||||
ancestor=self.f.root/'ancestor';ancestor.mkdir()
|
||||
admitted['parent']['ancestry_directories'].append(ancestor)
|
||||
for output,build in ((ancestor/'out',self.f.root/'build'),(self.f.root/'out',ancestor/'build')):
|
||||
with patch.object(m,'validate_authority',return_value=admitted),patch.object(m.R,'_load') as loader:
|
||||
with self.assertRaises(ValueError):m.run(self.f.path,m._digest(self.f.path),output,build)
|
||||
loader.assert_not_called()
|
||||
self.assertFalse(output.exists());self.assertFalse(build.exists())
|
||||
|
||||
def test_dtype_sorting_and_request_schema_reject(self):
|
||||
for ticks,nodes in ((np.array([0],np.int64),np.array([1],np.int32)),
|
||||
(np.array([1,0],np.int32),np.array([1,1],np.int32)),
|
||||
(np.array([5],np.int32),np.array([1],np.int32))):
|
||||
np.savez_compressed(self.f.case/'requests.npz',tick=ticks,neuron_idx=nodes)
|
||||
with self.assertRaises(ValueError):m._requests(self.f.case/'requests.npz',self.f.spec,'externally_pinned_CS_probe')
|
||||
|
||||
|
||||
class CheckpointDispatchTests(unittest.TestCase):
|
||||
def test_old_checkpoint_is_honestly_dispatched_to_old_C(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);save(root/m.MANIFEST,{'schema':'phase4an-checkpoint-v1'})
|
||||
with patch.object(m.C,'load_checkpoint',return_value={'fresh':True}) as old:
|
||||
self.assertEqual(m.load_checkpoint(root,m._digest(root/m.MANIFEST)),{'fresh':True})
|
||||
old.assert_called_once()
|
||||
|
||||
def test_new_checkpoint_cannot_copy_old_provenance(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp)
|
||||
save(root/'runtime-provenance.json',{'schema':'phase4an-continuation-runtime-v1','complete':True})
|
||||
with self.assertRaisesRegex(ValueError,'genuine new'):m._record(root,{root})
|
||||
|
||||
def test_new_checkpoint_cycle_and_wrong_external_pin(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp).resolve();save(root/m.MANIFEST,{'schema':m.CHECKPOINT_SCHEMA})
|
||||
with self.assertRaisesRegex(ValueError,'External checkpoint'):m.load_checkpoint(root,'e'*64)
|
||||
with self.assertRaisesRegex(ValueError,'cycle'):m.load_checkpoint(root,m._digest(root/m.MANIFEST),_seen={root})
|
||||
|
||||
|
||||
class NewCheckpointFixture:
|
||||
def __init__(self, f):
|
||||
self.f=f;self.admitted,_=f.admit();self.out=f.root/'new-output';self.out.mkdir()
|
||||
self.parent=self.admitted['parent'];self.parent.update(tiny_parent())
|
||||
self.state=dict(step=np.array([0],np.int32),neuron_idx=np.array([1],np.int32),counts=np.array([0,1],np.int32),
|
||||
v=np.array([-.05,-.052],np.float64),g=np.array([0.,-.003],np.float64),ref=np.array([0,18],np.int32))
|
||||
# Production APL dimensions, tiny canonical neural dimension only in tests.
|
||||
self.parent['terminal']['apl_pending']=np.zeros((18,4064),np.float32)
|
||||
self.parent['terminal']['apl_pending'][1,0]=np.float32(.5)
|
||||
ring=m.C.rotate_spike_ring(self.parent['terminal']['spike_ring'],5)
|
||||
for tick in range(5):ring[tick]=0
|
||||
ring[0,1]=1
|
||||
self.terminal=dict(spike_ring=ring,apl_pending=np.zeros((18,4064),np.float32),
|
||||
apl_state=np.array([1.,2.],np.float32),next_tick=np.asarray(5,np.int64))
|
||||
np.savez_compressed(self.out/'state.npz',**self.state)
|
||||
np.savez_compressed(self.out/'terminal-queues.npz',**self.terminal)
|
||||
np.save(self.out/'apl-pending-tail.float32.npy',self.terminal['apl_pending'])
|
||||
save(self.out/'apl-timing-audit.json',dict(pending_raw_sha256=m.R._raw(self.terminal['apl_pending']),
|
||||
apl_delay_steps=18,brain_tail_advanced=False,final_apl_state_u32_bits=self.terminal['apl_state'].view(np.uint32).tolist(),
|
||||
global_tick_offset=41,global_next_tick=46,initial_queue_raw_sha256=m.R._raw(self.parent['terminal']['apl_pending'])))
|
||||
self.summary=dict(complete=True,pass_all=True,all_finite=True,steps=5,completed_ticks=5,nodes=2,edges=6,
|
||||
precision='float64',apl_precision='float32',within_trial_weight_updates=False,checkpoint_continuation_used=True,
|
||||
global_tick_offset=41,global_next_tick=46,explicit_steps_executed=5,max_steps=5,hidden_early_stop=False,
|
||||
entire_neural_trial_executed=True,record_history=False,counts_scope='segment_only_not_lifetime',event_tick_scope='segment_local',
|
||||
weight_update_at_boundary_authorized=True,frozen_same_weight_retention_claimed=False,
|
||||
parent_checkpoint_manifest_sha256=self.parent['manifest_sha256'],diagnostic_mode='light',
|
||||
boundary_authority_sha256=self.admitted['authority_sha256'],update_root_authority_sha256=self.admitted['update_root_authority_sha256'],
|
||||
updated_boundary_schema=m.SCHEMA,boundary_transition='epoch_update',input_policy='empty_idle',effective_weights_raw_sha256='c'*64,
|
||||
input_sha256={'spec.json':m._digest(f.case/'spec.json'),'requests.npz':m._digest(f.case/'requests.npz'),
|
||||
'base_weights.npy':m._digest(f.graph/'weights.npy'),'coefficients.json':m._digest(f.coeff)})
|
||||
save(self.out/'summary.json',self.summary)
|
||||
save(self.out/'finite-audit.json',dict(all_finite=True,first_failure_tick=-1,finite_observed_ticks=5))
|
||||
save(self.out/'overlay-audit.json',dict(complete=True,weights_updated_during_run=False,effective_weights_raw_sha256='c'*64,
|
||||
delta_raw_sha256=m.R._raw(f.plan['plastic_delta_mv']),mask_sha256=m.R.MASK_SHA,am_helper_sha256=m.R.AM_OVERLAY_SHA))
|
||||
(self.out/'overlay-input.npz').write_bytes(f.after.read_bytes())
|
||||
(self.out/'overlay-manifest.json').write_bytes(f.am.read_bytes())
|
||||
_,audit=m.C.derive_continued(AJ.read_bytes(),'light')
|
||||
self.provenance=dict(schema=m.PROVENANCE_SCHEMA,complete=True,mode='light',source_file=str(Path(m.__file__).resolve()),
|
||||
source_sha256=m._digest(m.__file__),authority_file=str(f.path),authority_sha256=self.admitted['authority_sha256'],ast=audit,
|
||||
input_sha256=self.admitted['pins'],old_globals_patched=False,weights_updated_during_run=False,U_recomputed_here=False,
|
||||
learning_pass_claimed=False,parent_checkpoint_manifest_sha256=f.parentpin,global_tick_offset=41,global_next_tick=46,
|
||||
external_CPU_validation_receipt_bound=True,external_registration_process_and_probe_semantics_verified_here=False)
|
||||
restored=dict(v=self.parent['state']['v'],g=self.parent['state']['g'],ref=self.parent['state']['ref'],
|
||||
spike_ring=m.C.rotate_spike_ring(self.parent['terminal']['spike_ring'],5),apl_pending=self.parent['terminal']['apl_pending'],
|
||||
apl_state=self.parent['terminal']['apl_state'])
|
||||
save(self.out/'boundary-restore-audit.json',dict(restored_raw_sha256={k:m.R._raw(a) for k,a in restored.items()},
|
||||
all_state_and_queue_bytes_exact=True,existing_g_rescaled=False,retroactive_arrival_reweighting=False,
|
||||
pending_spike_arrivals_use_current_installed_weight=True,pending_APL_samples_preserved=True,
|
||||
boundary_installations=1,restorations=1,learning_pass_claimed=False))
|
||||
self.receipt=dict(schema='conditioning-updated-receipt-v1',complete=True,authority_file=str(f.path),
|
||||
authority_sha256=self.admitted['authority_sha256'],parent_checkpoint_dir=str(f.parentdir),parent_checkpoint_manifest_sha256=f.parentpin,
|
||||
global_tick_offset=41,global_next_tick=46,segment_steps=5,update_root_authority_sha256=self.admitted['update_root_authority_sha256'],
|
||||
input_policy='empty_idle',weights_updated_during_run=False,counts_scope='segment_only_not_lifetime',
|
||||
checkpoint_continuation_verified_by_GPU_comparison=False,learning_pass_claimed=False)
|
||||
self.rebind()
|
||||
|
||||
def rebind(self):
|
||||
for key,name in (('summary_sha256','summary.json'),('finite_audit_sha256','finite-audit.json'),
|
||||
('terminal_queues_sha256','terminal-queues.npz'),('overlay_audit_sha256','overlay-audit.json'),('state_sha256','state.npz')):
|
||||
self.provenance[key]=m._digest(self.out/name)
|
||||
save(self.out/'runtime-provenance.json',self.provenance)
|
||||
payload=m.C._tree_hashes(self.out);payload.pop(m.RECEIPT,None)
|
||||
self.receipt['payload_sha256']=payload;save(self.out/m.RECEIPT,self.receipt)
|
||||
|
||||
def record(self):
|
||||
with patch.object(m,'validate_authority',return_value=self.admitted),patch.object(m.C,'N',2),patch.object(m.C,'E',6):
|
||||
return m._record(self.out,{self.out})
|
||||
|
||||
|
||||
class NewCheckpointTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.f=NewCheckpointFixture(AdmissionFixture(Path(self.tmp.name).resolve()))
|
||||
def tearDown(self):self.tmp.cleanup()
|
||||
|
||||
def test_full_payload_exclusive_seal_round_trip(self):
|
||||
self.assertEqual(self.f.record()['global_next_tick'],46)
|
||||
with patch.object(m,'validate_authority',return_value=self.f.admitted),patch.object(m.C,'N',2),patch.object(m.C,'E',6):
|
||||
sealed=m.seal_checkpoint(self.f.out)
|
||||
loaded=m.load_checkpoint(self.f.out,sealed['sha256'])
|
||||
self.assertEqual(loaded['manifest']['schema'],m.CHECKPOINT_SCHEMA)
|
||||
self.assertTrue(m._same(loaded['state']['g'],self.f.state['g']))
|
||||
self.assertFalse(loaded['manifest']['checkpoint_continuation_verified_by_GPU_comparison'])
|
||||
with self.assertRaisesRegex(ValueError,'already sealed'):m.seal_checkpoint(self.f.out)
|
||||
(self.f.out/'extra.json').write_text('{}')
|
||||
with self.assertRaisesRegex(ValueError,'inventory'):m.load_checkpoint(self.f.out,sealed['sha256'])
|
||||
|
||||
def test_rehashed_wrong_parent_and_clock_still_rejected(self):
|
||||
self.f.provenance['parent_checkpoint_manifest_sha256']='f'*64;self.f.rebind()
|
||||
with self.assertRaisesRegex(ValueError,'boundary/clock'):self.f.record()
|
||||
self.f.provenance['parent_checkpoint_manifest_sha256']=self.f.f.parentpin
|
||||
self.f.summary['global_next_tick']=47;save(self.f.out/'summary.json',self.f.summary);self.f.rebind()
|
||||
with self.assertRaisesRegex(ValueError,'global_next_tick'):self.f.record()
|
||||
|
||||
def test_rehashed_changed_unconsumed_queue_rejected(self):
|
||||
self.f.terminal['spike_ring'][18,0]^=np.uint8(1)
|
||||
np.savez_compressed(self.f.out/'terminal-queues.npz',**self.f.terminal);self.f.rebind()
|
||||
with self.assertRaisesRegex(ValueError,'Unwritten carried'):self.f.record()
|
||||
|
||||
def test_rehashed_restoration_g_receipt_not_trusted(self):
|
||||
path=self.f.out/'boundary-restore-audit.json';value=m.R._read_json(path)
|
||||
value['restored_raw_sha256']['g']='f'*64;save(path,value);self.f.rebind()
|
||||
with self.assertRaisesRegex(ValueError,'Boundary restore'):self.f.record()
|
||||
|
||||
def test_actual_state_dtype_rejected_after_rehash(self):
|
||||
self.f.state['g']=self.f.state['g'].astype(np.float32)
|
||||
np.savez_compressed(self.f.out/'state.npz',**self.f.state);self.f.rebind()
|
||||
with self.assertRaisesRegex(ValueError,'parent g'):self.f.record()
|
||||
|
||||
|
||||
class EpochChainFixture:
|
||||
"""Full candidate cardinality and actual metadata bindings; no U execution.
|
||||
|
||||
Q is mocked explicitly: its frozen pure arithmetic has separate tests.
|
||||
This fixture exercises E/CPU file chain and supplied full-array authority.
|
||||
"""
|
||||
def __init__(self,root):
|
||||
self.f=AdmissionFixture(root);f=self.f
|
||||
self.update=root/'postprocess';self.update.mkdir()
|
||||
self.before={'edge_id':np.arange(14232,dtype=np.int64),'plastic_delta_mv':np.zeros(14232,np.float32),
|
||||
'base_candidate_mv':np.full(14232,.275,np.float32)}
|
||||
self.after={k:a.copy() for k,a in self.before.items()};self.after['plastic_delta_mv'][0]=np.float32(-.00275)
|
||||
kc=np.arange(4064,dtype=np.int32);source=(np.arange(14232)%4064).astype(np.int32);target=np.full(14232,4500,np.int32)
|
||||
comp=np.full(14232,"y5B'2a")
|
||||
np.save(f.apl/'kc_globals.npy',kc);np.save(root/'kc.npy',kc);np.save(root/'mask.npy',self.before['edge_id'])
|
||||
np.save(root/'base.npy',self.before['base_candidate_mv']);np.save(f.graph/'src.npy',source);np.save(f.graph/'dst.npy',target)
|
||||
np.savez_compressed(root/'map.npz',edge_id=self.before['edge_id'],source_neuron_idx=source,target_neuron_idx=target,target_compartment_verbatim=comp)
|
||||
np.savez_compressed(f.before,edge_id=self.before['edge_id'],plastic_delta_mv=self.before['plastic_delta_mv'])
|
||||
self.bm={**f.meta,'mask_sha256':m._digest(root/'mask.npy'),'base_candidate_sha256':m._digest(root/'base.npy')}
|
||||
save(f.bm,self.bm)
|
||||
self.afterfile=self.update/'delta-after.npz';np.savez_compressed(self.afterfile,edge_id=self.after['edge_id'],plastic_delta_mv=self.after['plastic_delta_mv'])
|
||||
self.am={**self.bm,'parent_snapshot_sha256':m._digest(f.before)};save(self.update/'overlay-manifest.json',self.am)
|
||||
self.state=dict(step=np.array([0],np.int32),neuron_idx=np.array([0],np.int32),counts=np.array([1],np.int32),
|
||||
v=np.array([-.05]),g=np.array([.001]),ref=np.array([18],np.int32))
|
||||
np.savez_compressed(f.parentdir/'state.npz',**self.state)
|
||||
self.parent=dict(fresh=True,global_tick_offset=0,steps=4,directory=f.parentdir,manifest_sha256=f.parentpin,state=self.state,
|
||||
summary={'diagnostic_mode':'light','effective_weights_raw_sha256':'b'*64},
|
||||
manifest={'files_sha256':{'state.npz':m._digest(f.parentdir/'state.npz')}},provenance={'input_sha256':{}})
|
||||
self.epaths=dict(spec=f.case/'spec.json',requests=f.case/'requests.npz',overlay_before=f.before,overlay_manifest_before=f.bm,
|
||||
state=f.parentdir/'state.npz',kc=root/'kc.npy',canonical_kc=f.apl/'kc_globals.npy',candidate_map=root/'map.npz',
|
||||
mask=root/'mask.npy',base_candidate=root/'base.npy',**{key:f.sources[key] for key in m.E_SOURCE_ROLES})
|
||||
self.epath=root/'epoch-authority.json'
|
||||
frozen={str(p):m._digest(p) for p in [*(f.graph/n for n in m.GRAPH_FILES),*(f.apl/n for n in m.APL_FILES),f.coeff]}
|
||||
frozen.update({str((ROOT/relative).resolve()):sha for relative,sha in m.R.PINS.items()})
|
||||
command=dict(steps=4,compartment="y5B'2a",us_tick=3,d_command=1.,frozen=False,trace_backend='numpy')
|
||||
self.evalue=dict(schema='phase4an-epoch-postprocess-authority-v1',epoch_id='synthetic-e0',diagnostic_mode='light',command=command,
|
||||
checkpoint_dir=str(f.parentdir),checkpoint_manifest_sha256=f.parentpin,files={role:pin(path) for role,path in self.epaths.items()},frozen_inputs=frozen)
|
||||
save(self.epath,self.evalue)
|
||||
self.epins={str(self.epath):m._digest(self.epath),str(f.parentdir/m.MANIFEST):f.parentpin,
|
||||
**{str(p):m._digest(p) for p in self.epaths.values()},**frozen}
|
||||
self.parent['provenance']['input_sha256']={str(self.epaths[k]):m._digest(self.epaths[k]) for k in ('spec','requests','overlay_before','overlay_manifest_before')}
|
||||
arrays=dict(step=self.state['step'],neuron_idx=self.state['neuron_idx'],kc=kc,candidate_edge_id=self.before['edge_id'],
|
||||
candidate_sources=source,candidate_targets=target,candidate_compartments=comp,delta_before=self.before['plastic_delta_mv'],base_candidate=self.before['base_candidate_mv'])
|
||||
self.receipt=dict(input_arrays={k:m._evidence(a) for k,a in arrays.items()},command=dict(steps=4,n_neurons=5000,
|
||||
compartment="y5B'2a",us_tick=3,D_scheduled=1.,frozen=False,trace_backend='numpy'),payload_sha256='d'*64)
|
||||
save(self.update/'update-receipt.json',self.receipt)
|
||||
self.chain=dict(schema='phase4an-epoch-postprocess-chain-v1',complete=True,epoch_id='synthetic-e0',authority_file=str(self.epath),
|
||||
authority_sha256=m._digest(self.epath),coordinator_source_sha256=m.SOURCE_PINS['conditioning_epoch_source'],command=command,
|
||||
diagnostic_mode='light',source_neural_checkpoint_dir=str(f.parentdir),source_neural_checkpoint_manifest_sha256=f.parentpin,
|
||||
before_overlay_file_sha256=m._digest(f.before),before_overlay_manifest_sha256=m._digest(f.bm),
|
||||
after_overlay_file_sha256=m._digest(self.afterfile),after_overlay_manifest_sha256=m._digest(self.update/'overlay-manifest.json'),
|
||||
source_state_file_sha256=m._digest(self.epaths['state']),source_spec_sha256=m._digest(self.epaths['spec']),
|
||||
source_requests_sha256=m._digest(self.epaths['requests']),input_files_sha256=self.epins,
|
||||
output_files_sha256={name:m._digest(self.update/name) for name in m.UPDATE_FILES if name!='chain-receipt.json'},
|
||||
runtime_delta_before_raw_sha256=m.R._raw(self.before['plastic_delta_mv']),runtime_effective_weights_raw_sha256='b'*64,
|
||||
update_receipt_payload_sha256='d'*64,runtime_learned_manifest_loader_verified=True,input_and_checkpoint_bytes_reverified=True,
|
||||
candidate_endpoints_equal_graph=True,KC_equals_supplied_canonical_membership=True,weights_installed=False,
|
||||
brain_executed=False,learning_pass_claimed=False,candidate_generated=False)
|
||||
save(self.update/'chain-receipt.json',self.chain)
|
||||
self.paths=dict(before_overlay=f.before,before_manifest=f.bm,after_overlay=self.afterfile,after_manifest=self.update/'overlay-manifest.json')
|
||||
self.value=dict(update={'epoch_authority':pin(self.epath),'files':{name:pin(self.update/name) for name in m.UPDATE_FILES}})
|
||||
self.cp=root/'cpu-validation.json'
|
||||
self.cpu=dict(schema='conditioning-update-cpu-validation-v1',complete=True,epoch_authority_sha256=m._digest(self.epath),
|
||||
parent_checkpoint_manifest_sha256=f.parentpin,update_files_sha256={name:m._digest(self.update/name) for name in m.UPDATE_FILES},
|
||||
before_overlay_sha256=m._digest(f.before),before_manifest_sha256=m._digest(f.bm),after_overlay_sha256=m._digest(self.afterfile),
|
||||
after_manifest_sha256=m._digest(self.update/'overlay-manifest.json'),before_delta_raw_sha256=m.R._raw(self.before['plastic_delta_mv']),
|
||||
after_delta_raw_sha256=m.R._raw(self.after['plastic_delta_mv']),update_receipt_payload_sha256='d'*64,
|
||||
validator_source=pin(f.sources['cpu_validator_source']),frozen_U_source_sha256=m.SOURCE_PINS['conditioning_update_source'],
|
||||
frozen_U_verify_receipt_recomputed=True,actual_sealed_events_used=True,full_candidate_mapping_verified=True,
|
||||
brain_executed=False,learning_pass_claimed=False)
|
||||
save(self.cp,self.cpu);self.value['cpu_validation']=pin(self.cp)
|
||||
|
||||
def run(self):
|
||||
pins={str(p):m._digest(p) for p in [*self.paths.values(),*self.f.sources.values()]}
|
||||
with patch.object(m,'_Q',return_value=SimpleNamespace(audit_delta_binding=lambda *a,**k:{'pass_all':True})), \
|
||||
patch.object(m.C,'_parent_spec',return_value=self.f.spec),patch.object(m.C,'N',5000):
|
||||
return m._epoch_update(self.value,self.paths,self.f.sources,pins,self.parent,self.before,self.after,self.bm,self.am)
|
||||
|
||||
|
||||
class EpochChainTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.f=EpochChainFixture(Path(self.tmp.name).resolve())
|
||||
def tearDown(self):self.tmp.cleanup()
|
||||
|
||||
def test_full_four_files_actual_inputs_and_cpu_schema_bind(self):
|
||||
self.assertTrue(self.f.run()['pass_all'])
|
||||
|
||||
def test_chain_wrong_parent_rehashed_still_rejected(self):
|
||||
self.f.chain['source_neural_checkpoint_manifest_sha256']='e'*64
|
||||
save(self.f.update/'chain-receipt.json',self.f.chain)
|
||||
self.f.value['update']['files']['chain-receipt.json']=pin(self.f.update/'chain-receipt.json')
|
||||
with self.assertRaisesRegex(ValueError,'Original E chain'):self.f.run()
|
||||
|
||||
def test_cpu_no_recomputation_or_wrong_after_raw_rejected(self):
|
||||
for key,value in (('frozen_U_verify_receipt_recomputed',False),('after_delta_raw_sha256','e'*64)):
|
||||
original=copy.deepcopy(self.f.cpu);self.f.cpu[key]=value;save(self.f.cp,self.f.cpu)
|
||||
self.f.value['cpu_validation']=pin(self.f.cp)
|
||||
with self.assertRaisesRegex(ValueError,'CPU recomputation'):self.f.run()
|
||||
self.f.cpu=original
|
||||
|
||||
def test_update_four_file_hash_changed_or_extra_file(self):
|
||||
(self.f.update/'extra.json').write_text('{}')
|
||||
with self.assertRaisesRegex(ValueError,'four-file'):self.f.run()
|
||||
|
||||
def test_actual_array_not_receipt_booleans_is_checked(self):
|
||||
self.f.parent['state']['step'][0]=1
|
||||
with self.assertRaisesRegex(ValueError,'actual array binding'):self.f.run()
|
||||
|
||||
def test_candidate_endpoint_change_rejected_even_with_same_shape(self):
|
||||
mapping=m.C._npz(self.f.epaths['candidate_map'],('edge_id','source_neuron_idx','target_neuron_idx','target_compartment_verbatim'))
|
||||
mapping['source_neuron_idx'][0]=22
|
||||
np.savez_compressed(self.f.epaths['candidate_map'],**mapping)
|
||||
with self.assertRaisesRegex(ValueError,'changed'):self.f.run()
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in New Issue
Block a user