qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish sealed updated-boundary audit and truthful learning status

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-09-30 20:17:50 +08:00
parent a9c5918445
commit 3e30934a55
574 changed files with 218421 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
registry/phase4aq-conditioning-execution/source-snapshot/test_audit_execution.py
+299
View File
@@ -0,0 +1,299 @@
"""Synthetic file-chain/dispatch tests; never a brain or historical audit."""
from __future__ import annotations
import copy
import hashlib
import json
from pathlib import Path
import tempfile
from contextlib import ExitStack
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import numpy as np
import audit_execution as T
def write(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, sort_keys=True), encoding='utf-8')
def npz(path, **values):
with path.open('wb') as stream:
np.savez(stream, **values)
class PlanTests(unittest.TestCase):
def test_exact_fixed_process_budget_and_kinds(self):
plan = T.PLAN.build_plan(); values = T.expected_processes(plan)
self.assertEqual(len(values), 31)
self.assertEqual(sum(k in ('updated', 'oracle', 'unchanged', 'brain') for k in values.values()), 23)
self.assertEqual(sum(k in ('update', 'validate_cpu') for k in values.values()), 7)
self.assertEqual(list(values.values()).count('sample'), 1)
self.assertEqual(sum(c['steps'] for c in plan['calls']), 125200)
def test_duplicate_call_and_wrong_budget_rejected(self):
plan = T.PLAN.build_plan(); plan['calls'].append(copy.deepcopy(plan['calls'][0]))
with self.assertRaisesRegex(ValueError, 'Duplicate'): T.expected_processes(plan)
plan = T.PLAN.build_plan(); plan['counts']['total_children'] += 1
with self.assertRaisesRegex(ValueError, 'inventory'): T.expected_processes(plan)
def test_D1_zero_coverage_does_not_deny_mechanical_witness(self):
pending = {'D1': {'accepted_changed_edge_witness_verified': False},
'mechanical': {'accepted_changed_edge_witness_verified': True}}
exact, coverage, status = T.outcome({'all_numeric': True}, pending)
self.assertTrue(exact and coverage)
self.assertEqual(status, 'PASS-EXACT-UPDATED-BOUNDARY-ENGINEERING-NOT-LEARNING')
pending['mechanical']['accepted_changed_edge_witness_verified'] = False
self.assertEqual(T.outcome({'all_numeric': True}, pending)[2], 'INCOMPLETE-CHANGED-EDGE-COVERAGE')
self.assertEqual(T.outcome({'all_numeric': False}, pending)[2], 'FAIL-NUMERICAL-OR-LINEAGE')
with self.assertRaises(ValueError): T.outcome({}, pending)
with self.assertRaises(ValueError): T.outcome({'impostor_bool': 1}, pending)
class ProcessTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(); self.exp = Path(self.temp.name).resolve() / 'phase4aq-synthetic'
self.patch = patch.object(T, 'EXP', self.exp); self.patch.start()
self.name = 'performance--natural-light-fresh'; self.kind = 'brain'
self.regfile = self.exp / 'protocol/preregistered.json'; write(self.regfile, {'synthetic': True})
self.reg_sha = T.digest(self.regfile)
self.h = T._helpers()
self.reg = dict(source_sha256={str(T.MODEL / 'isolated_execution.py'): T.digest(T.MODEL / 'isolated_execution.py')},
input_sha256={}, prepared_file_sha256={})
case = self.exp / 'cases' / self.name; write(case / 'spec.json', {'synthetic': True})
(case / 'requests.npz').write_bytes(b'not-scientific-requests')
overlay = self.exp / 'inputs/before.npz'; overlay.parent.mkdir(); overlay.write_bytes(b'not-scientific-overlay')
om = self.exp / 'inputs/before.json'; write(om, {})
self.output = self.exp / 'runs' / self.name; write(self.output / 'summary.json', {'synthetic': True})
self.jobpath = self.exp / 'jobs' / (self.name + '.json')
self.workerpath = self.exp / 'receipts' / (self.name + '.worker.json')
self.processpath = self.exp / 'receipts' / (self.name + '.process.json')
self.stdout = self.exp / 'logs' / (self.name + '.stdout.log'); self.stdout.parent.mkdir()
self.stdout.write_bytes(b'fixture'); self.stderr = self.stdout.with_name(self.name + '.stderr.log'); self.stderr.write_bytes(b'')
self.sources = {n: dict(path=str(T.LAB / (n + '.py')), sha256=self.h.SOURCE_PINS[n])
for n in ('runtime_light', 'runtime_continuation')}
inputs = T._closure(self.reg, self.reg_sha)
inputs.update({str(p): T.digest(p) for p in (case / 'spec.json', case / 'requests.npz', overlay, om)})
self.job = dict(schema='phase4ap-isolated-worker-job-v1', job_id=self.name, kind='brain', registration_sha256=self.reg_sha,
worker_sha256=T.W.AP_SHA, case_dir=str(case), overlay_file=str(overlay), overlay_manifest=str(om),
output_dir=str(self.output), receipt_file=str(self.workerpath), input_sha256=inputs, sources=self.sources)
self.seal_records()
def tearDown(self):
self.patch.stop(); self.temp.cleanup()
def seal_records(self):
write(self.jobpath, self.job); job_sha = T.digest(self.jobpath)
attempt = Path(str(self.workerpath) + '.attempt.json')
write(attempt, dict(complete=False, job_sha256=job_sha, no_automatic_retry=True))
hashes = {k: v['sha256'] for k, v in T.tree(self.output).items()}
pins = {**self.job['input_sha256'], str(self.jobpath): job_sha, str(T.W.AP): T.W.AP_SHA,
**{v['path']: v['sha256'] for v in self.sources.values()}}
receipt = dict(schema='phase4ap-isolated-worker-receipt-v1', complete=True, job_id=self.name, kind='brain',
registration_sha256=self.reg_sha, worker_sha256=T.W.AP_SHA, job_sha256=job_sha,
process_exit_independently_observed=False, scientific_gate_pass_claimed=False, source_files=self.sources,
verified_input_sha256=pins, output_dir=str(self.output), output_files_sha256=hashes,
attempt_sha256=T.digest(attempt), result={'synthetic': True})
write(self.workerpath, receipt)
process = dict(schema='conditioning-execution-process-v1', name=self.name, kind='brain', registration_sha256=self.reg_sha,
complete=True, exit_code=0, timed_out=False, job_sha256=job_sha, worker_receipt_sha256=T.digest(self.workerpath),
stdout_sha256=T.digest(self.stdout), stderr_sha256=T.digest(self.stderr),
argv=['/synthetic/python', '-B', str(T.W.AP), 'brain', '--job', str(self.jobpath), '--job-sha256', job_sha],
output_bytes=sum(v['bytes'] for v in T.tree(self.output).values()),
output_inventory_sha256=hashlib.sha256(json.dumps(hashes, sort_keys=True).encode()).hexdigest(),
child_wall_seconds=.1, end_to_end_seconds=.2, validation_wall_seconds=.1)
write(self.processpath, process)
def verify(self):
return T.verify_process(self.name, self.kind, self.reg, self.reg_sha, self.processpath, self.h)
def test_actual_file_chain_not_working_memory_flags(self):
result = self.verify()
self.assertEqual(result['kind'], 'brain'); self.assertEqual(result['output'], self.output)
def test_nonzero_exit_and_bool_exit_rejected(self):
original = T.read(self.processpath)
for value in (1, False, None):
changed = copy.deepcopy(original); changed['exit_code'] = value; write(self.processpath, changed)
with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'recorded process exit'): self.verify()
def test_log_mutation_and_unknown_output_rejected(self):
self.stderr.write_bytes(b'changed')
with self.assertRaisesRegex(ValueError, 'file chain'): self.verify()
self.stderr.write_bytes(b''); (self.output / 'unreported.json').write_text('{}')
with self.assertRaisesRegex(ValueError, 'inventory'): self.verify()
def test_missing_registration_source_closure_even_rehashed_rejected(self):
del self.job['input_sha256'][str(T.MODEL / 'isolated_execution.py')]; self.seal_records()
with self.assertRaisesRegex(ValueError, 'closure omitted'): self.verify()
def test_extra_self_authorized_input_even_rehashed_rejected(self):
extra = self.exp / 'arbitrary.txt'; extra.write_text('extra')
self.job['input_sha256'][str(extra)] = T.digest(extra); self.seal_records()
with self.assertRaisesRegex(ValueError, 'undeclared input'): self.verify()
def test_wrong_command_and_process_path_rejected(self):
p = T.read(self.processpath); p['argv'][2] = '/unregistered/worker.py'; write(self.processpath, p)
with self.assertRaisesRegex(ValueError, 'command differs'): self.verify()
with self.assertRaisesRegex(ValueError, 'fixed process record'):
T.verify_process(self.name, self.kind, self.reg, self.reg_sha, self.stdout, self.h)
def test_changed_source_snapshot_and_attempt_rejected(self):
worker = T.read(self.workerpath); worker['attempt_sha256'] = '0' * 64; write(self.workerpath, worker)
p = T.read(self.processpath); p['worker_receipt_sha256'] = T.digest(self.workerpath); write(self.processpath, p)
with self.assertRaisesRegex(ValueError, 'Attempt record changed'): self.verify()
class ComparisonTests(unittest.TestCase):
def test_light_missing_perstep_is_reported_not_fabricated(self):
state = {k: np.zeros(3, np.int32) for k in T.B.C.STATE_KEYS}
terminal = {k: np.zeros(3, np.int32) for k in ('spike_ring', 'apl_state', 'apl_pending', 'next_tick')}
left = dict(steps=1, global_tick_offset=0, state=state, terminal=terminal, requests=np.empty((0, 5), np.int64),
summary={'effective_weights_raw_sha256': '1'*64}, traces={}, state_hashes=None, queues=None)
right = copy.deepcopy(left)
with patch.object(T.B, '_complete'):
report = T._pair(left, right)
self.assertTrue(report['numerical_exact']); self.assertFalse(any(report['observed'].values()))
right['state']['v'][0] = 1
report = T._pair(left, right)
self.assertFalse(report['numerical_exact'])
self.assertIn('pair.state.v', report['failed_checks'])
def test_imported_parent_fields_cannot_be_rebound(self):
reg = {'parent_records': {'p': {'directory': '/original', 'checkpoint_sha256': '1'*64}}}
complete = {'parents': {'p': {**reg['parent_records']['p'], 'cpu_validation': '/new/check.json'}}}
self.assertEqual(T._parent(reg, complete, 'p')['directory'], '/original')
complete['parents']['p']['directory'] = '/other'
with self.assertRaisesRegex(ValueError, 'Imported parent'): T._parent(reg, complete, 'p')
def test_output_tree_nonfiles_and_pins_are_actual_bytes(self):
with tempfile.TemporaryDirectory() as name:
p = Path(name).resolve() / 'payload'; p.write_bytes(b'abc')
T.verify_pins({str(p): T.digest(p)})
with self.assertRaisesRegex(ValueError, 'Pinned file changed'): T.verify_pins({str(p): '0'*64})
class OrchestrationTests(unittest.TestCase):
"""Dispatch fixture: deliberately mocked physics/loaders, no scientific PASS."""
def fixture(self, exp):
plan = T.PLAN.build_plan(); parents = {}; imports = {}; records = {}; verified = {}; templates = {}
delta = dict(edge_id=np.array([0], np.int64), plastic_delta_mv=np.array([0], np.float32))
def parent(directory, name):
directory.mkdir(parents=True, exist_ok=True)
write(directory / 'summary.json', {'synthetic': True})
write(directory / 'checkpoint-manifest.json', {'synthetic': True})
update = exp / 'updates' / name; update.mkdir(parents=True, exist_ok=True)
npz(update / 'delta-after.npz', **delta); write(update / 'update-receipt.json', {'same': 'synthetic-U'})
write(update / 'overlay-manifest.json', {'synthetic': True})
auth = exp / 'authorities' / (name + '.epoch.json'); write(auth, {'synthetic': True})
return dict(directory=str(directory), checkpoint_sha256=T.digest(directory / 'checkpoint-manifest.json'),
epoch_authority=T.A.pin(auth), update_dir=str(update),
after_overlay=dict(overlay_file=str(update / 'delta-after.npz'), overlay_manifest=str(update / 'overlay-manifest.json')))
for arm, desc in plan['imported_parents'].items():
p = parent(exp / 'old-AP' / arm, 'old-' + arm)
desc['checkpoint_manifest'] = str(Path(p['directory']) / 'checkpoint-manifest.json')
desc['sha256'] = p['checkpoint_sha256']; parents[desc['role']] = p; imports[arm] = p
registered_parents = copy.deepcopy(parents)
for call in plan['calls']:
name = call['call_id']; out = exp / 'runs' / name
if call['runner'] != 'boundary_oracle':
p = parent(out, name); parents[name] = p
else:
out.mkdir(parents=True); write(out / 'summary.json', {'synthetic': True})
write(out / 'oracle-manifest.json', dict(schema='conditioning-live-oracle-files-v1', complete=True,
files_sha256={'summary.json': T.digest(out / 'summary.json')}))
group = call['case_group']; template = exp / 'protocol/case-templates' / (group + '.json')
spec = {'synthetic': group, 'steps': call['steps']}
if group not in templates: write(template, spec); templates[group] = str(template)
if call['parent'] is not None:
spec.update(parent_checkpoint_manifest_sha256=parents[call['parent']]['checkpoint_sha256'],
global_tick_offset=call['global_tick_offset'])
case = exp / 'cases' / group; write(case / 'spec.json', spec)
npz(case / 'requests.npz', tick=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32))
record = dict(source=str(out), steps=call['steps'], global_tick_offset=call['global_tick_offset'],
summary=dict(nodes=1, edges=1, diagnostic_mode=call['diagnostic_mode'], apl_component={'environment': {}}),
state={'v': np.array([0.], np.float64)})
if call['runner'] != 'boundary_oracle': record['manifest_sha256'] = parents[name]['checkpoint_sha256']
records[str(out)] = record
cycle = dict(complete=True, call=call, source_registration_sha256='placeholder', output_inventory={})
write(exp / 'receipts' / (name + '.cycle.json'), cycle)
for arm, p in imports.items():
records[p['directory']] = dict(source=p['directory'], summary={}, manifest_sha256=p['checkpoint_sha256'])
reg = dict(schema='conditioning-execution-registration-v1', engineering_only=True, scientific_acquisition_authorized=False,
plan=plan, source_sha256={}, input_sha256={}, prepared_file_sha256={}, parent_records=registered_parents,
case_templates=templates)
regfile = exp / 'protocol/preregistered.json'; write(regfile, reg); sha = T.digest(regfile)
kinds = T.expected_processes(plan)
for name, kind in kinds.items():
out = exp / ('updates' if kind == 'update' else 'runs') / name.removesuffix('--original-E')
out.mkdir(parents=True, exist_ok=True)
if not any(out.iterdir()): write(out / 'payload.json', {'synthetic': True})
job = {'kind': kind}
call = next((c for c in plan['calls'] if c['call_id'] == name), None)
if call is not None:
case = exp / 'cases' / call['case_group']
if kind in ('brain', 'unchanged'): job['case_dir'] = str(case)
elif kind == 'oracle': job['oracle_case'] = str(case)
else:
p = parents[call['parent']]
auth = dict(case=dict(spec=T.A.pin(case / 'spec.json'), requests=T.A.pin(case / 'requests.npz')),
boundary_id=name, registration_sha256=sha, diagnostic_mode=call['diagnostic_mode'],
input_policy=call['input_policy'], transition=call['transition'],
parent=dict(directory=p['directory'], checkpoint_manifest_sha256=p['checkpoint_sha256']),
expected_effective_weight_before_sha256='a'*64, expected_effective_weight_after_sha256='b'*64)
path = exp / 'authorities' / (name + '.json'); write(path, auth); job['authority_file'] = str(path)
cyclefile = exp / 'receipts' / (name + '.cycle.json'); cycle = T.read(cyclefile)
cycle['source_registration_sha256'] = sha; write(cyclefile, cycle)
result = {}
if kind in ('brain', 'unchanged', 'updated'): result['checkpoint_manifest_sha256'] = parents[name]['checkpoint_sha256']
if kind == 'update':
write(out / 'chain-receipt.json', {'synthetic': True}); result['chain_receipt_sha256'] = T.digest(out / 'chain-receipt.json')
paths = {key: exp / folder / (name + suffix) for key, folder, suffix in
(('process', 'receipts', '.process.json'), ('worker', 'receipts', '.worker.json'), ('job', 'jobs', '.json'))}
for p in paths.values(): write(p, {'synthetic': name})
verified[name] = dict(name=name, kind=kind, job=job, paths=paths, output=out,
output_inventory=T.tree(out), receipt={'result': result, 'environment': {}}, input_pins={},
evidence_sha256={str(p): T.digest(p) for p in paths.values()})
complete = dict(complete=True, registration_sha256=sha, completed_calls=[c['call_id'] for c in plan['calls']],
processes={k: str(v['paths']['process']) for k, v in verified.items()}, parents=parents,
scientific_learning_pass_claimed=False, candidate_generated=False)
write(exp / 'audit/execution-complete.json', complete)
return sha, records, verified
def test_all_predetermined_calls_compared_and_numeric_failure_not_hidden(self):
with tempfile.TemporaryDirectory() as name, ExitStack() as stack:
exp = Path(name).resolve() / 'phase4aq-mock'; sha, records, verified = self.fixture(exp)
stack.enter_context(patch.object(T, 'EXP', exp))
stack.enter_context(patch.object(T, 'N', 1)); stack.enter_context(patch.object(T, 'EDGES', 1))
stack.enter_context(patch.object(T.PLAN, 'validate_plan'))
stack.enter_context(patch.object(T, '_helpers', return_value=SimpleNamespace(_verify_precision=lambda *a: None)))
verifier = stack.enter_context(patch.object(T, 'verify_process', side_effect=lambda n, *args: verified[n]))
stack.enter_context(patch.object(T.D, 'load_checkpoint')); stack.enter_context(patch.object(T.D.C, 'load_checkpoint'))
stack.enter_context(patch.object(T.B, 'load_record', side_effect=lambda d, **kw: records[str(d)]))
stack.enter_context(patch.object(T.B, '_complete')); stack.enter_context(patch.object(T.B, 'close_record'))
stack.enter_context(patch.object(T, '_update_check', return_value={'pass_all': True}))
live = stack.enter_context(patch.object(T.B, 'compare_boundary', return_value={'numerical_exact': True}))
pair = stack.enter_context(patch.object(T, '_pair', return_value={'numerical_exact': True}))
pending = dict(conditional_ref_request_queue_exact=True, reference_queue_and_ref_coverage_complete=True,
accepted_changed_edge_witness_verified=True)
stack.enter_context(patch.object(T, '_pending', return_value=pending))
stack.enter_context(patch.object(T, '_lineage', return_value={'pass_all': True}))
report = T.audit(sha, save_report=False)
self.assertTrue(report['pass_all']); self.assertEqual(report['actual_new_brain_calls'], 23)
self.assertEqual(report['actual_new_brain_ticks'], 125200); self.assertEqual(report['actual_processes'], 31)
self.assertEqual(verifier.call_count, 31); self.assertEqual(live.call_count, 4); self.assertEqual(pair.call_count, 5)
self.assertFalse(report['brain_executed_here']); self.assertFalse(report['learning_pass_claimed'])
self.assertFalse((exp / 'audit/numerical-comparison.json').exists())
pair.return_value = {'numerical_exact': False}
report = T.audit(sha)
self.assertEqual(report['status'], 'FAIL-NUMERICAL-OR-LINEAGE'); self.assertFalse(report['pass_all'])
self.assertTrue((exp / 'audit/numerical-comparison.json').is_file())
with self.assertRaises(FileExistsError): T.audit(sha)
if __name__ == '__main__':
unittest.main()