Seal Phase 4AN exact continuation engineering evidence
This commit is contained in:
Digital Drosophila Archive
+227
@@ -0,0 +1,227 @@
|
||||
"""Synthetic/mocked dispatch + temporary-array tests; no GPU or brain runs."""
|
||||
import copy
|
||||
from contextlib import ExitStack
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
import engineering_analysis as a
|
||||
import engineering_plan as p
|
||||
|
||||
|
||||
PIN = 'a' * 64
|
||||
WEIGHT = 'b' * 64
|
||||
|
||||
|
||||
def raw(value):
|
||||
return hashlib.sha256(value.tobytes()).hexdigest()
|
||||
|
||||
|
||||
def record(row):
|
||||
n, k, steps, start = 3, 2, row['steps'], row['start']
|
||||
state = dict(step=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32), counts=np.zeros(n, np.int32),
|
||||
v=np.zeros(n, np.float64), g=np.zeros(n, np.float64), ref=np.zeros(n, np.int32))
|
||||
reference = row['mode'] == 'reference'
|
||||
summary = dict(complete=True, pass_all=True, all_finite=True, counts_match_sparse_events=True,
|
||||
steps=steps, completed_ticks=steps, nodes=n, edges=3, requested=0, total_spikes=0, accepted_requests=0,
|
||||
rejected_requests=0, arriving_edges=0, accepted_arriving_edges=0, apl_fast_edges_zeroed=1, kc_targets_present=k,
|
||||
precision='float64', apl_precision='float32', apl_delay_steps=18, diagnostic_mode=row['mode'],
|
||||
effective_weights_raw_sha256=WEIGHT, within_trial_weight_updates=False, learning_pass_claimed=False,
|
||||
checkpoint_continuation_used=start > 0, global_tick_offset=start, global_next_tick=start + steps,
|
||||
wall_seconds=1., diagnostic_simulation_seconds=.5, build_or_cache_seconds=.2,
|
||||
peak_allocated_vram_bytes=1000, peak_reserved_vram_bytes=2000)
|
||||
return dict(source=row['row_id'], state=state, steps=steps, global_tick_offset=start, summary=summary,
|
||||
terminal=dict(spike_ring=np.zeros((19, n), np.uint8), apl_state=np.zeros(2, np.float32),
|
||||
apl_pending=np.zeros((18, k), np.float32), next_tick=np.asarray(steps, np.int64)),
|
||||
requests=np.empty((0, 5), np.int64),
|
||||
traces={name: np.zeros((steps, 2), np.float32) for name in a.compare.TRACE_FILES} if reference else {},
|
||||
state_hashes=[dict(tick=t, v_sha256=raw(state['v']), g_sha256=raw(state['g']), ref_sha256=raw(state['ref'])) for t in range(steps)] if reference else None,
|
||||
queues=[dict(tick=t, count=0, accepted_count=0, ordered_edge_sha256='c' * 64,
|
||||
accepted_mask_sha256='c' * 64, joint_sha256='d' * 64) for t in range(steps)] if reference else None)
|
||||
|
||||
|
||||
def fixture():
|
||||
plan = p.build_plan(); exp = Path(tempfile.gettempdir()).resolve() / 'synthetic-project' / 'experiments/phase4an-engineering-validation'
|
||||
reg = dict(schema='phase4an-engineering-registration-v1', plan=plan, training=False,
|
||||
cases={r['row_id']: dict(case_dir=str(exp / 'cases' / r['row_id'])) for r in plan['rows']},
|
||||
source_sha256={}, input_sha256={}, selected_case=dict(AM_outputs={'zero': 'mock', 'static25': 'mock'}),
|
||||
overlays={'zero': {}, 'static25': {}})
|
||||
verified = {r['row_id']: dict(schema='phase4an-engineering-runtime-record-v1', complete=True, training=False,
|
||||
row=copy.deepcopy(r), registration_sha256=PIN, checkpoint_manifest_sha256=PIN,
|
||||
runtime_and_immediate_seal_wall_seconds=2., storage={'state.npz': {'file_bytes': 10}, 'terminal-queues.npz': {'file_bytes': 20}})
|
||||
for r in plan['rows']}
|
||||
records = {r['row_id']: record(r) for r in plan['rows']}
|
||||
reader = Mock(); reader.finish.return_value = []; reader.close.return_value = None
|
||||
context = dict(reader=reader, records=records,
|
||||
legacy={stage: record(next(r for r in plan['rows'] if r['overlay'] == stage and r['kind'] == 'fresh' and r['mode'] == 'reference')) for stage in p.OVERLAYS},
|
||||
graph=dict(ptr=np.array([0, 1, 2, 3], np.int64), dst=np.array([1, 2, 0], np.int32), edge_id=np.arange(3, dtype=np.int64)),
|
||||
requests=dict(tick=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32)), storage={r['row_id']: 600 for r in plan['rows']})
|
||||
return reg, verified, context
|
||||
|
||||
|
||||
class Orchestration(unittest.TestCase):
|
||||
def mocked(self, reg, verified, context, *, fail=None, live=True, queue_exact=True, joint=True):
|
||||
calls = []
|
||||
def ordinary(name):
|
||||
def f(*args):
|
||||
calls.append(name)
|
||||
return dict(numerical_exact=name != fail, checks={}, observed_scope={'mock_only': True})
|
||||
return f
|
||||
def chain(mono, parts):
|
||||
calls.append('chain_exact')
|
||||
reference = mono['summary']['diagnostic_mode'] == 'reference'
|
||||
return dict(numerical_exact=fail != ('reference_chain' if reference else 'light_chain'),
|
||||
pass_all=reference and live, nonvacuous_temporal_coverage=reference and live,
|
||||
status='PASS' if reference else 'INCOMPLETE-COVERAGE',
|
||||
coverage={'carried_APL_nonzero_applied_values': 1 if live and reference else 0,
|
||||
'boundaries': [{'pending_source_spike_events': 1, 'carried_APL_nonzero_pending_values': 1 if joint else 0}]} if reference else None)
|
||||
with ExitStack() as stack:
|
||||
for name, value in dict(N=3, E=3, K=2, FAST_APL_EDGES=1).items():
|
||||
stack.enter_context(patch.object(a, name, value))
|
||||
stack.enter_context(patch.object(a, '_load_context', return_value=context))
|
||||
stack.enter_context(patch.object(a, '_materialize', return_value=(np.ones(3, np.float64), {'effective_weights_raw_sha256': WEIGHT})))
|
||||
stack.enter_context(patch.object(a.compare, 'compare_fresh', side_effect=ordinary('fresh')))
|
||||
stack.enter_context(patch.object(a.scopes, 'compare_slice', side_effect=ordinary('slice')))
|
||||
stack.enter_context(patch.object(a.scopes, 'compare_checkpoint', side_effect=ordinary('checkpoint')))
|
||||
stack.enter_context(patch.object(a.scopes, 'compare_parent_carry', side_effect=ordinary('carry')))
|
||||
stack.enter_context(patch.object(a, '_pair', side_effect=ordinary('pair')))
|
||||
stack.enter_context(patch.object(a.compare, 'compare_segments', side_effect=chain))
|
||||
queue_mock = stack.enter_context(patch.object(a.queue_audit, 'audit_queues', return_value=dict(
|
||||
queue_ref_request_exact=queue_exact, accepted_nonzero_crossing_witness_verified=live and queue_exact)))
|
||||
result = a.analyze(reg, PIN, verified)
|
||||
return result, calls, queue_mock
|
||||
|
||||
def test_executes_all_descriptors_and_both_independent_queues(self):
|
||||
reg, verified, ctx = fixture()
|
||||
report, calls, queued = self.mocked(reg, verified, ctx)
|
||||
expected = [r['row_id'] + '/' + str(i) for r in reg['plan']['rows'] for i, _ in enumerate(r['expected_comparisons'])]
|
||||
self.assertEqual([r['descriptor_id'] for r in report['comparisons']], expected)
|
||||
self.assertEqual(len(calls), len(expected))
|
||||
self.assertEqual(report['evaluated_comparisons'], 74)
|
||||
self.assertEqual(queued.call_count, 2)
|
||||
self.assertEqual(queued.call_args.kwargs['cuts'], [5, 18, 19, 500])
|
||||
self.assertEqual(report['status'], a.SUCCESS)
|
||||
self.assertFalse(report['learning_pass_claimed'])
|
||||
self.assertEqual(len(report['metrics']), 32)
|
||||
self.assertTrue(all(row['gpu_utilization'] is None for row in report['metrics']))
|
||||
self.assertTrue(all(r['light_standalone_coverage'] is None for r in report['overlay_reports'].values()))
|
||||
json.dumps(report, allow_nan=False)
|
||||
ctx['reader'].finish.assert_called_once()
|
||||
ctx['reader'].close.assert_called_once()
|
||||
|
||||
def test_numerical_failure_still_evaluates_every_descriptor(self):
|
||||
for failure in ('fresh', 'slice', 'checkpoint', 'carry', 'pair', 'reference_chain', 'light_chain'):
|
||||
reg, verified, ctx = fixture()
|
||||
with self.subTest(failure=failure):
|
||||
report, calls, _ = self.mocked(reg, verified, ctx, fail=failure)
|
||||
self.assertEqual(len(calls), 74)
|
||||
self.assertEqual(report['status'], 'FAIL-NUMERICAL')
|
||||
self.assertFalse(report['pass_all'])
|
||||
|
||||
def test_queue_failure_and_absent_live_coverage_are_distinct(self):
|
||||
for arguments, expected in [({'queue_exact': False}, 'FAIL-NUMERICAL'),
|
||||
({'live': False}, 'INCOMPLETE-COVERAGE'), ({'joint': False}, 'INCOMPLETE-COVERAGE')]:
|
||||
reg, verified, ctx = fixture()
|
||||
report, _, _ = self.mocked(reg, verified, ctx, **arguments)
|
||||
self.assertEqual(report['status'], expected)
|
||||
self.assertFalse(report['pass_all'])
|
||||
|
||||
def test_receipt_inventory_alias_and_descriptor_removal_fail_closed(self):
|
||||
for mutation in ('receipt', 'descriptor', 'pin', 'case', 'row_alias'):
|
||||
reg, verified, ctx = fixture(); identity = reg['plan']['rows'][0]['row_id']
|
||||
if mutation == 'receipt':
|
||||
verified.pop(identity)
|
||||
elif mutation == 'descriptor':
|
||||
reg['plan']['rows'][0]['expected_comparisons'] = []
|
||||
elif mutation == 'pin':
|
||||
verified[identity]['registration_sha256'] = 'f' * 64
|
||||
elif mutation == 'row_alias':
|
||||
verified[identity]['row']['steps'] = True
|
||||
else:
|
||||
reg['cases'][identity]['case_dir'] += '-another'
|
||||
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
|
||||
self.mocked(reg, verified, ctx)
|
||||
|
||||
def test_summary_uses_actual_AJ_requested_names_and_checks_total_accounting(self):
|
||||
for key in ('requested', 'accepted_requests', 'arriving_edges', 'accepted_arriving_edges', 'edges', 'effective_weights_raw_sha256'):
|
||||
reg, verified, ctx = fixture(); identity = reg['plan']['rows'][0]['row_id']
|
||||
ctx['records'][identity]['summary'][key] = 'e' * 64 if key.endswith('sha256') else 1
|
||||
with self.subTest(key=key):
|
||||
report, _, _ = self.mocked(reg, verified, ctx)
|
||||
self.assertEqual(report['status'], 'FAIL-NUMERICAL')
|
||||
self.assertIn(identity, report['failed_run_checks'])
|
||||
|
||||
def test_identical_bytes_give_deterministic_report_and_unknown_is_null(self):
|
||||
reg, verified, ctx = fixture(); identity = reg['plan']['rows'][0]['row_id']
|
||||
ctx['records'][identity]['summary'].pop('peak_reserved_vram_bytes')
|
||||
one, _, _ = self.mocked(reg, verified, ctx)
|
||||
two, _, _ = self.mocked(reg, verified, ctx)
|
||||
self.assertEqual(a._canonical(one), a._canonical(two))
|
||||
row = next(r for r in one['metrics'] if r['row_id'] == identity)
|
||||
self.assertIsNone(row['peak_reserved_vram_bytes'])
|
||||
|
||||
def test_AM_reference_full_comparison_is_not_hidden_by_light(self):
|
||||
reg, _, ctx = fixture()
|
||||
row = reg['plan']['rows'][0]; descriptor = row['expected_comparisons'][0]
|
||||
ctx['records'][row['row_id']]['traces']['drive'][10, 0] = np.float32(1)
|
||||
report = a._dispatch(descriptor, ctx['records'], ctx['legacy'], row['overlay'])
|
||||
self.assertFalse(report['numerical_exact'])
|
||||
self.assertIn('pair.trace.drive', report['failed_checks'])
|
||||
|
||||
def test_missing_loaded_population_or_reference_inventory_rejected(self):
|
||||
reg, verified, ctx = fixture(); ctx['records'].pop(next(iter(ctx['records'])))
|
||||
with self.assertRaises(ValueError):
|
||||
self.mocked(reg, verified, ctx)
|
||||
|
||||
|
||||
class FileAndWeightInputs(unittest.TestCase):
|
||||
def test_reader_rehashes_actual_content_and_never_uses_stat_cache(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder).resolve() / 'data.bin'; path.write_bytes(b'AAAA')
|
||||
reader = a._Reader({str(path): a._sha_file(path)})
|
||||
reader.bind(path); path.write_bytes(b'BBBB')
|
||||
with self.assertRaises(ValueError):
|
||||
reader.bind(path)
|
||||
with self.assertRaises(ValueError):
|
||||
reader.finish()
|
||||
|
||||
def test_original_pinned_overlay_math_signed_zero_and_nonzero_only_bridge(self):
|
||||
helper = Path(__file__).resolve().parents[1] / 'wholebrain-plastic-interface-v1/overlay.py'
|
||||
self.assertEqual(a._sha_file(helper), a.AM_OVERLAY_SHA)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
folder = Path(folder).resolve()
|
||||
mask = folder / 'mask.npy'; base = folder / 'base.npy'
|
||||
np.save(mask, np.array([0], np.int64)); np.save(base, np.array([1.5], np.float32))
|
||||
original = np.array([.002, -0., -0., .009], np.float64)
|
||||
graph = dict(weights=original, edge_id=np.arange(4, dtype=np.int64), apl_out_mask=np.array([False, True, False, False]))
|
||||
before = original.tobytes()
|
||||
for stage in ('zero', 'static25'):
|
||||
delta = np.array([-0. if stage == 'zero' else -.375], np.float32)
|
||||
overlay = folder / (stage + '.npz'); manifest = folder / (stage + '.json')
|
||||
np.savez_compressed(overlay, edge_id=np.array([0], np.int64), plastic_delta_mv=delta)
|
||||
meta = dict(schema='phase4an-overlay-v1', overlay_name=stage, am_overlay_path=str(helper), am_overlay_sha256=a.AM_OVERLAY_SHA,
|
||||
mask_path=str(mask), mask_sha256=a._sha_file(mask), base_candidate_path=str(base), base_candidate_sha256=a._sha_file(base),
|
||||
overlay_sha256=a._sha_file(overlay))
|
||||
manifest.write_text(json.dumps(meta), encoding='utf-8')
|
||||
paths = [mask, base, overlay, manifest, helper]
|
||||
reader = a._Reader({str(path): a._sha_file(path) for path in paths})
|
||||
reg = {'overlays': {stage: dict(overlay_file=str(overlay), overlay_manifest=str(manifest))}}
|
||||
with patch.object(a, 'MASK_SHA', meta['mask_sha256']), patch.object(a, 'CANDIDATES', 1), patch.object(a, 'FAST_APL_EDGES', 1):
|
||||
actual, info = a._materialize(reader, reg, graph, stage)
|
||||
expected = original.copy(); expected[1] = 0.
|
||||
if stage == 'static25':
|
||||
expected[0] = np.add(original[0], np.multiply(delta[0].astype(np.float64), np.float64(.001)))
|
||||
self.assertEqual(actual.tobytes(), expected.tobytes())
|
||||
self.assertEqual(info['effective_weights_raw_sha256'], raw(expected))
|
||||
self.assertEqual(info['installed_nonzero_count'], 0 if stage == 'zero' else 1)
|
||||
self.assertEqual(original.tobytes(), before)
|
||||
reader.finish(); reader.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user