Digital Drosophila Archive
300 lines
20 KiB
Python
300 lines
20 KiB
Python
"""Synthetic file-chain/dispatch tests; never a brain or historical audit."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from contextlib import ExitStack
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
|
|
import audit_execution as T
|
|
|
|
|
|
def write(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, sort_keys=True), encoding='utf-8')
|
|
|
|
|
|
def npz(path, **values):
|
|
with path.open('wb') as stream:
|
|
np.savez(stream, **values)
|
|
|
|
|
|
class PlanTests(unittest.TestCase):
|
|
def test_exact_fixed_process_budget_and_kinds(self):
|
|
plan = T.PLAN.build_plan(); values = T.expected_processes(plan)
|
|
self.assertEqual(len(values), 31)
|
|
self.assertEqual(sum(k in ('updated', 'oracle', 'unchanged', 'brain') for k in values.values()), 23)
|
|
self.assertEqual(sum(k in ('update', 'validate_cpu') for k in values.values()), 7)
|
|
self.assertEqual(list(values.values()).count('sample'), 1)
|
|
self.assertEqual(sum(c['steps'] for c in plan['calls']), 125200)
|
|
|
|
def test_duplicate_call_and_wrong_budget_rejected(self):
|
|
plan = T.PLAN.build_plan(); plan['calls'].append(copy.deepcopy(plan['calls'][0]))
|
|
with self.assertRaisesRegex(ValueError, 'Duplicate'): T.expected_processes(plan)
|
|
plan = T.PLAN.build_plan(); plan['counts']['total_children'] += 1
|
|
with self.assertRaisesRegex(ValueError, 'inventory'): T.expected_processes(plan)
|
|
|
|
def test_D1_zero_coverage_does_not_deny_mechanical_witness(self):
|
|
pending = {'D1': {'accepted_changed_edge_witness_verified': False},
|
|
'mechanical': {'accepted_changed_edge_witness_verified': True}}
|
|
exact, coverage, status = T.outcome({'all_numeric': True}, pending)
|
|
self.assertTrue(exact and coverage)
|
|
self.assertEqual(status, 'PASS-EXACT-UPDATED-BOUNDARY-ENGINEERING-NOT-LEARNING')
|
|
pending['mechanical']['accepted_changed_edge_witness_verified'] = False
|
|
self.assertEqual(T.outcome({'all_numeric': True}, pending)[2], 'INCOMPLETE-CHANGED-EDGE-COVERAGE')
|
|
self.assertEqual(T.outcome({'all_numeric': False}, pending)[2], 'FAIL-NUMERICAL-OR-LINEAGE')
|
|
with self.assertRaises(ValueError): T.outcome({}, pending)
|
|
with self.assertRaises(ValueError): T.outcome({'impostor_bool': 1}, pending)
|
|
|
|
|
|
class ProcessTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(); self.exp = Path(self.temp.name).resolve() / 'phase4aq-synthetic'
|
|
self.patch = patch.object(T, 'EXP', self.exp); self.patch.start()
|
|
self.name = 'performance--natural-light-fresh'; self.kind = 'brain'
|
|
self.regfile = self.exp / 'protocol/preregistered.json'; write(self.regfile, {'synthetic': True})
|
|
self.reg_sha = T.digest(self.regfile)
|
|
self.h = T._helpers()
|
|
self.reg = dict(source_sha256={str(T.MODEL / 'isolated_execution.py'): T.digest(T.MODEL / 'isolated_execution.py')},
|
|
input_sha256={}, prepared_file_sha256={})
|
|
case = self.exp / 'cases' / self.name; write(case / 'spec.json', {'synthetic': True})
|
|
(case / 'requests.npz').write_bytes(b'not-scientific-requests')
|
|
overlay = self.exp / 'inputs/before.npz'; overlay.parent.mkdir(); overlay.write_bytes(b'not-scientific-overlay')
|
|
om = self.exp / 'inputs/before.json'; write(om, {})
|
|
self.output = self.exp / 'runs' / self.name; write(self.output / 'summary.json', {'synthetic': True})
|
|
self.jobpath = self.exp / 'jobs' / (self.name + '.json')
|
|
self.workerpath = self.exp / 'receipts' / (self.name + '.worker.json')
|
|
self.processpath = self.exp / 'receipts' / (self.name + '.process.json')
|
|
self.stdout = self.exp / 'logs' / (self.name + '.stdout.log'); self.stdout.parent.mkdir()
|
|
self.stdout.write_bytes(b'fixture'); self.stderr = self.stdout.with_name(self.name + '.stderr.log'); self.stderr.write_bytes(b'')
|
|
self.sources = {n: dict(path=str(T.LAB / (n + '.py')), sha256=self.h.SOURCE_PINS[n])
|
|
for n in ('runtime_light', 'runtime_continuation')}
|
|
inputs = T._closure(self.reg, self.reg_sha)
|
|
inputs.update({str(p): T.digest(p) for p in (case / 'spec.json', case / 'requests.npz', overlay, om)})
|
|
self.job = dict(schema='phase4ap-isolated-worker-job-v1', job_id=self.name, kind='brain', registration_sha256=self.reg_sha,
|
|
worker_sha256=T.W.AP_SHA, case_dir=str(case), overlay_file=str(overlay), overlay_manifest=str(om),
|
|
output_dir=str(self.output), receipt_file=str(self.workerpath), input_sha256=inputs, sources=self.sources)
|
|
self.seal_records()
|
|
|
|
def tearDown(self):
|
|
self.patch.stop(); self.temp.cleanup()
|
|
|
|
def seal_records(self):
|
|
write(self.jobpath, self.job); job_sha = T.digest(self.jobpath)
|
|
attempt = Path(str(self.workerpath) + '.attempt.json')
|
|
write(attempt, dict(complete=False, job_sha256=job_sha, no_automatic_retry=True))
|
|
hashes = {k: v['sha256'] for k, v in T.tree(self.output).items()}
|
|
pins = {**self.job['input_sha256'], str(self.jobpath): job_sha, str(T.W.AP): T.W.AP_SHA,
|
|
**{v['path']: v['sha256'] for v in self.sources.values()}}
|
|
receipt = dict(schema='phase4ap-isolated-worker-receipt-v1', complete=True, job_id=self.name, kind='brain',
|
|
registration_sha256=self.reg_sha, worker_sha256=T.W.AP_SHA, job_sha256=job_sha,
|
|
process_exit_independently_observed=False, scientific_gate_pass_claimed=False, source_files=self.sources,
|
|
verified_input_sha256=pins, output_dir=str(self.output), output_files_sha256=hashes,
|
|
attempt_sha256=T.digest(attempt), result={'synthetic': True})
|
|
write(self.workerpath, receipt)
|
|
process = dict(schema='conditioning-execution-process-v1', name=self.name, kind='brain', registration_sha256=self.reg_sha,
|
|
complete=True, exit_code=0, timed_out=False, job_sha256=job_sha, worker_receipt_sha256=T.digest(self.workerpath),
|
|
stdout_sha256=T.digest(self.stdout), stderr_sha256=T.digest(self.stderr),
|
|
argv=['/synthetic/python', '-B', str(T.W.AP), 'brain', '--job', str(self.jobpath), '--job-sha256', job_sha],
|
|
output_bytes=sum(v['bytes'] for v in T.tree(self.output).values()),
|
|
output_inventory_sha256=hashlib.sha256(json.dumps(hashes, sort_keys=True).encode()).hexdigest(),
|
|
child_wall_seconds=.1, end_to_end_seconds=.2, validation_wall_seconds=.1)
|
|
write(self.processpath, process)
|
|
|
|
def verify(self):
|
|
return T.verify_process(self.name, self.kind, self.reg, self.reg_sha, self.processpath, self.h)
|
|
|
|
def test_actual_file_chain_not_working_memory_flags(self):
|
|
result = self.verify()
|
|
self.assertEqual(result['kind'], 'brain'); self.assertEqual(result['output'], self.output)
|
|
|
|
def test_nonzero_exit_and_bool_exit_rejected(self):
|
|
original = T.read(self.processpath)
|
|
for value in (1, False, None):
|
|
changed = copy.deepcopy(original); changed['exit_code'] = value; write(self.processpath, changed)
|
|
with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'recorded process exit'): self.verify()
|
|
|
|
def test_log_mutation_and_unknown_output_rejected(self):
|
|
self.stderr.write_bytes(b'changed')
|
|
with self.assertRaisesRegex(ValueError, 'file chain'): self.verify()
|
|
self.stderr.write_bytes(b''); (self.output / 'unreported.json').write_text('{}')
|
|
with self.assertRaisesRegex(ValueError, 'inventory'): self.verify()
|
|
|
|
def test_missing_registration_source_closure_even_rehashed_rejected(self):
|
|
del self.job['input_sha256'][str(T.MODEL / 'isolated_execution.py')]; self.seal_records()
|
|
with self.assertRaisesRegex(ValueError, 'closure omitted'): self.verify()
|
|
|
|
def test_extra_self_authorized_input_even_rehashed_rejected(self):
|
|
extra = self.exp / 'arbitrary.txt'; extra.write_text('extra')
|
|
self.job['input_sha256'][str(extra)] = T.digest(extra); self.seal_records()
|
|
with self.assertRaisesRegex(ValueError, 'undeclared input'): self.verify()
|
|
|
|
def test_wrong_command_and_process_path_rejected(self):
|
|
p = T.read(self.processpath); p['argv'][2] = '/unregistered/worker.py'; write(self.processpath, p)
|
|
with self.assertRaisesRegex(ValueError, 'command differs'): self.verify()
|
|
with self.assertRaisesRegex(ValueError, 'fixed process record'):
|
|
T.verify_process(self.name, self.kind, self.reg, self.reg_sha, self.stdout, self.h)
|
|
|
|
def test_changed_source_snapshot_and_attempt_rejected(self):
|
|
worker = T.read(self.workerpath); worker['attempt_sha256'] = '0' * 64; write(self.workerpath, worker)
|
|
p = T.read(self.processpath); p['worker_receipt_sha256'] = T.digest(self.workerpath); write(self.processpath, p)
|
|
with self.assertRaisesRegex(ValueError, 'Attempt record changed'): self.verify()
|
|
|
|
|
|
class ComparisonTests(unittest.TestCase):
|
|
def test_light_missing_perstep_is_reported_not_fabricated(self):
|
|
state = {k: np.zeros(3, np.int32) for k in T.B.C.STATE_KEYS}
|
|
terminal = {k: np.zeros(3, np.int32) for k in ('spike_ring', 'apl_state', 'apl_pending', 'next_tick')}
|
|
left = dict(steps=1, global_tick_offset=0, state=state, terminal=terminal, requests=np.empty((0, 5), np.int64),
|
|
summary={'effective_weights_raw_sha256': '1'*64}, traces={}, state_hashes=None, queues=None)
|
|
right = copy.deepcopy(left)
|
|
with patch.object(T.B, '_complete'):
|
|
report = T._pair(left, right)
|
|
self.assertTrue(report['numerical_exact']); self.assertFalse(any(report['observed'].values()))
|
|
right['state']['v'][0] = 1
|
|
report = T._pair(left, right)
|
|
self.assertFalse(report['numerical_exact'])
|
|
self.assertIn('pair.state.v', report['failed_checks'])
|
|
|
|
def test_imported_parent_fields_cannot_be_rebound(self):
|
|
reg = {'parent_records': {'p': {'directory': '/original', 'checkpoint_sha256': '1'*64}}}
|
|
complete = {'parents': {'p': {**reg['parent_records']['p'], 'cpu_validation': '/new/check.json'}}}
|
|
self.assertEqual(T._parent(reg, complete, 'p')['directory'], '/original')
|
|
complete['parents']['p']['directory'] = '/other'
|
|
with self.assertRaisesRegex(ValueError, 'Imported parent'): T._parent(reg, complete, 'p')
|
|
|
|
def test_output_tree_nonfiles_and_pins_are_actual_bytes(self):
|
|
with tempfile.TemporaryDirectory() as name:
|
|
p = Path(name).resolve() / 'payload'; p.write_bytes(b'abc')
|
|
T.verify_pins({str(p): T.digest(p)})
|
|
with self.assertRaisesRegex(ValueError, 'Pinned file changed'): T.verify_pins({str(p): '0'*64})
|
|
|
|
|
|
class OrchestrationTests(unittest.TestCase):
|
|
"""Dispatch fixture: deliberately mocked physics/loaders, no scientific PASS."""
|
|
def fixture(self, exp):
|
|
plan = T.PLAN.build_plan(); parents = {}; imports = {}; records = {}; verified = {}; templates = {}
|
|
delta = dict(edge_id=np.array([0], np.int64), plastic_delta_mv=np.array([0], np.float32))
|
|
def parent(directory, name):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
write(directory / 'summary.json', {'synthetic': True})
|
|
write(directory / 'checkpoint-manifest.json', {'synthetic': True})
|
|
update = exp / 'updates' / name; update.mkdir(parents=True, exist_ok=True)
|
|
npz(update / 'delta-after.npz', **delta); write(update / 'update-receipt.json', {'same': 'synthetic-U'})
|
|
write(update / 'overlay-manifest.json', {'synthetic': True})
|
|
auth = exp / 'authorities' / (name + '.epoch.json'); write(auth, {'synthetic': True})
|
|
return dict(directory=str(directory), checkpoint_sha256=T.digest(directory / 'checkpoint-manifest.json'),
|
|
epoch_authority=T.A.pin(auth), update_dir=str(update),
|
|
after_overlay=dict(overlay_file=str(update / 'delta-after.npz'), overlay_manifest=str(update / 'overlay-manifest.json')))
|
|
for arm, desc in plan['imported_parents'].items():
|
|
p = parent(exp / 'old-AP' / arm, 'old-' + arm)
|
|
desc['checkpoint_manifest'] = str(Path(p['directory']) / 'checkpoint-manifest.json')
|
|
desc['sha256'] = p['checkpoint_sha256']; parents[desc['role']] = p; imports[arm] = p
|
|
registered_parents = copy.deepcopy(parents)
|
|
for call in plan['calls']:
|
|
name = call['call_id']; out = exp / 'runs' / name
|
|
if call['runner'] != 'boundary_oracle':
|
|
p = parent(out, name); parents[name] = p
|
|
else:
|
|
out.mkdir(parents=True); write(out / 'summary.json', {'synthetic': True})
|
|
write(out / 'oracle-manifest.json', dict(schema='conditioning-live-oracle-files-v1', complete=True,
|
|
files_sha256={'summary.json': T.digest(out / 'summary.json')}))
|
|
group = call['case_group']; template = exp / 'protocol/case-templates' / (group + '.json')
|
|
spec = {'synthetic': group, 'steps': call['steps']}
|
|
if group not in templates: write(template, spec); templates[group] = str(template)
|
|
if call['parent'] is not None:
|
|
spec.update(parent_checkpoint_manifest_sha256=parents[call['parent']]['checkpoint_sha256'],
|
|
global_tick_offset=call['global_tick_offset'])
|
|
case = exp / 'cases' / group; write(case / 'spec.json', spec)
|
|
npz(case / 'requests.npz', tick=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32))
|
|
record = dict(source=str(out), steps=call['steps'], global_tick_offset=call['global_tick_offset'],
|
|
summary=dict(nodes=1, edges=1, diagnostic_mode=call['diagnostic_mode'], apl_component={'environment': {}}),
|
|
state={'v': np.array([0.], np.float64)})
|
|
if call['runner'] != 'boundary_oracle': record['manifest_sha256'] = parents[name]['checkpoint_sha256']
|
|
records[str(out)] = record
|
|
cycle = dict(complete=True, call=call, source_registration_sha256='placeholder', output_inventory={})
|
|
write(exp / 'receipts' / (name + '.cycle.json'), cycle)
|
|
for arm, p in imports.items():
|
|
records[p['directory']] = dict(source=p['directory'], summary={}, manifest_sha256=p['checkpoint_sha256'])
|
|
reg = dict(schema='conditioning-execution-registration-v1', engineering_only=True, scientific_acquisition_authorized=False,
|
|
plan=plan, source_sha256={}, input_sha256={}, prepared_file_sha256={}, parent_records=registered_parents,
|
|
case_templates=templates)
|
|
regfile = exp / 'protocol/preregistered.json'; write(regfile, reg); sha = T.digest(regfile)
|
|
kinds = T.expected_processes(plan)
|
|
for name, kind in kinds.items():
|
|
out = exp / ('updates' if kind == 'update' else 'runs') / name.removesuffix('--original-E')
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
if not any(out.iterdir()): write(out / 'payload.json', {'synthetic': True})
|
|
job = {'kind': kind}
|
|
call = next((c for c in plan['calls'] if c['call_id'] == name), None)
|
|
if call is not None:
|
|
case = exp / 'cases' / call['case_group']
|
|
if kind in ('brain', 'unchanged'): job['case_dir'] = str(case)
|
|
elif kind == 'oracle': job['oracle_case'] = str(case)
|
|
else:
|
|
p = parents[call['parent']]
|
|
auth = dict(case=dict(spec=T.A.pin(case / 'spec.json'), requests=T.A.pin(case / 'requests.npz')),
|
|
boundary_id=name, registration_sha256=sha, diagnostic_mode=call['diagnostic_mode'],
|
|
input_policy=call['input_policy'], transition=call['transition'],
|
|
parent=dict(directory=p['directory'], checkpoint_manifest_sha256=p['checkpoint_sha256']),
|
|
expected_effective_weight_before_sha256='a'*64, expected_effective_weight_after_sha256='b'*64)
|
|
path = exp / 'authorities' / (name + '.json'); write(path, auth); job['authority_file'] = str(path)
|
|
cyclefile = exp / 'receipts' / (name + '.cycle.json'); cycle = T.read(cyclefile)
|
|
cycle['source_registration_sha256'] = sha; write(cyclefile, cycle)
|
|
result = {}
|
|
if kind in ('brain', 'unchanged', 'updated'): result['checkpoint_manifest_sha256'] = parents[name]['checkpoint_sha256']
|
|
if kind == 'update':
|
|
write(out / 'chain-receipt.json', {'synthetic': True}); result['chain_receipt_sha256'] = T.digest(out / 'chain-receipt.json')
|
|
paths = {key: exp / folder / (name + suffix) for key, folder, suffix in
|
|
(('process', 'receipts', '.process.json'), ('worker', 'receipts', '.worker.json'), ('job', 'jobs', '.json'))}
|
|
for p in paths.values(): write(p, {'synthetic': name})
|
|
verified[name] = dict(name=name, kind=kind, job=job, paths=paths, output=out,
|
|
output_inventory=T.tree(out), receipt={'result': result, 'environment': {}}, input_pins={},
|
|
evidence_sha256={str(p): T.digest(p) for p in paths.values()})
|
|
complete = dict(complete=True, registration_sha256=sha, completed_calls=[c['call_id'] for c in plan['calls']],
|
|
processes={k: str(v['paths']['process']) for k, v in verified.items()}, parents=parents,
|
|
scientific_learning_pass_claimed=False, candidate_generated=False)
|
|
write(exp / 'audit/execution-complete.json', complete)
|
|
return sha, records, verified
|
|
|
|
def test_all_predetermined_calls_compared_and_numeric_failure_not_hidden(self):
|
|
with tempfile.TemporaryDirectory() as name, ExitStack() as stack:
|
|
exp = Path(name).resolve() / 'phase4aq-mock'; sha, records, verified = self.fixture(exp)
|
|
stack.enter_context(patch.object(T, 'EXP', exp))
|
|
stack.enter_context(patch.object(T, 'N', 1)); stack.enter_context(patch.object(T, 'EDGES', 1))
|
|
stack.enter_context(patch.object(T.PLAN, 'validate_plan'))
|
|
stack.enter_context(patch.object(T, '_helpers', return_value=SimpleNamespace(_verify_precision=lambda *a: None)))
|
|
verifier = stack.enter_context(patch.object(T, 'verify_process', side_effect=lambda n, *args: verified[n]))
|
|
stack.enter_context(patch.object(T.D, 'load_checkpoint')); stack.enter_context(patch.object(T.D.C, 'load_checkpoint'))
|
|
stack.enter_context(patch.object(T.B, 'load_record', side_effect=lambda d, **kw: records[str(d)]))
|
|
stack.enter_context(patch.object(T.B, '_complete')); stack.enter_context(patch.object(T.B, 'close_record'))
|
|
stack.enter_context(patch.object(T, '_update_check', return_value={'pass_all': True}))
|
|
live = stack.enter_context(patch.object(T.B, 'compare_boundary', return_value={'numerical_exact': True}))
|
|
pair = stack.enter_context(patch.object(T, '_pair', return_value={'numerical_exact': True}))
|
|
pending = dict(conditional_ref_request_queue_exact=True, reference_queue_and_ref_coverage_complete=True,
|
|
accepted_changed_edge_witness_verified=True)
|
|
stack.enter_context(patch.object(T, '_pending', return_value=pending))
|
|
stack.enter_context(patch.object(T, '_lineage', return_value={'pass_all': True}))
|
|
report = T.audit(sha, save_report=False)
|
|
self.assertTrue(report['pass_all']); self.assertEqual(report['actual_new_brain_calls'], 23)
|
|
self.assertEqual(report['actual_new_brain_ticks'], 125200); self.assertEqual(report['actual_processes'], 31)
|
|
self.assertEqual(verifier.call_count, 31); self.assertEqual(live.call_count, 4); self.assertEqual(pair.call_count, 5)
|
|
self.assertFalse(report['brain_executed_here']); self.assertFalse(report['learning_pass_claimed'])
|
|
self.assertFalse((exp / 'audit/numerical-comparison.json').exists())
|
|
pair.return_value = {'numerical_exact': False}
|
|
report = T.audit(sha)
|
|
self.assertEqual(report['status'], 'FAIL-NUMERICAL-OR-LINEAGE'); self.assertFalse(report['pass_all'])
|
|
self.assertTrue((exp / 'audit/numerical-comparison.json').is_file())
|
|
with self.assertRaises(FileExistsError): T.audit(sha)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|