Digital Drosophila Archive
291 lines
15 KiB
Python
291 lines
15 KiB
Python
"""CPU-only temporary evidence trees; no real AP/brain execution."""
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import publish_phase4ap as P
|
|
|
|
|
|
_candidates = (
|
|
Path(__file__).resolve().parents[1] / 'pn-recruitment-audit-v1/publication.py',
|
|
Path(__file__).resolve().parents[2] / P.HELPER_REL,
|
|
)
|
|
HELPER = next((path for path in _candidates if path.is_file()), _candidates[0])
|
|
|
|
|
|
def put(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
raw = value if isinstance(value, bytes) else (json.dumps(value, sort_keys=True) + '\n').encode()
|
|
path.write_bytes(raw)
|
|
return P.digest(path)
|
|
|
|
|
|
def read(path):
|
|
return json.loads(path.read_text(encoding='utf-8'))
|
|
|
|
|
|
def seal(directory):
|
|
files = [r for r in P.file_rows(directory) if r['path'] not in ('manifest.json', 'checksums.sha256')]
|
|
put(directory / 'manifest.json', dict(root=str(directory), files=files))
|
|
raw = ''.join(row['sha256'] + ' ' + row['path'] + '\n' for row in files)
|
|
put(directory / 'checksums.sha256', (raw + P.digest(directory / 'manifest.json') + ' manifest.json\n').encode())
|
|
|
|
|
|
class PublicationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name).resolve()
|
|
self.exp, self.registry, self.model = (self.root / rel for rel in (P.EXP_REL, P.REGISTRY_REL, P.MODEL_REL))
|
|
put(self.root / P.HELPER_REL, HELPER.read_bytes())
|
|
|
|
def fixture(self, status='PASS-EXACT-PROCESS-ISOLATED-DELTA-INTERFACE-NOT-LEARNING'):
|
|
sources, prepared = {}, {}
|
|
for name in P.SOURCE_FILES:
|
|
raw = ('# synthetic only: ' + name).encode()
|
|
checksum = put(self.model / name, raw)
|
|
snapshot = self.exp / P.SNAPSHOT / name
|
|
put(snapshot, raw)
|
|
sources[str(self.model / name)] = prepared[str(snapshot)] = checksum
|
|
slots = []
|
|
for arm in ('D1', 'D0', 'frozen'):
|
|
for mode in ('reference', 'light'):
|
|
for epoch in (0, 1):
|
|
name = f'{arm}--{mode}--epoch-{epoch:03d}'
|
|
imported = name == P.FIRST
|
|
base = self.root / (P.AO_REL if imported else P.EXP_REL)
|
|
slots.append(dict(row_id=name, steps=1200, origin='imported_sealed_AO_neural_trajectory' if imported else 'new_AP_neural_execution',
|
|
neural_dir=str(base / 'runs' / name), case_dir=str(base / 'cases' / name),
|
|
original_case_dir=str(self.root / P.AO_REL / 'cases' / name)))
|
|
self.reg = dict(schema='phase4ap-process-isolated-interface-v1', source_sha256=sources,
|
|
prepared_file_sha256=prepared, source_code_commit='5' * 40, slots=slots,
|
|
covered_slots=12, new_neural_calls=11, covered_ticks=14400, new_neural_ticks=13200, new_sampler_calls=0,
|
|
imported_slot=P.FIRST, imported_checkpoint_sha256='e' * 64,
|
|
anchors={str(self.root / P.AO_REL / 'runs' / P.FIRST / 'checkpoint-manifest.json'): 'e' * 64},
|
|
original_failed_workflow_reclassified=False, original_AO_process_complete=False,
|
|
inherited_neural_checkpoint_complete=True, full_conditioning=False, learning_success_claimed=False,
|
|
within_trial_updates=False, retention_transition_supported=False)
|
|
self.analysis = dict(status=status, covered_slots=12, new_neural_calls=11,
|
|
imported_neural_trajectories=1, new_sampler_calls=0, original_AO_status=P.AO_STATUS,
|
|
learned_candidate=False, learning_success_claimed=False, full_conditioning_executed=False)
|
|
self.analysis_path = P.ANALYSES[1 if status.startswith('FAIL') else 0]
|
|
self.final = {k: v for k, v in self.analysis.items() if k != 'original_AO_status'}
|
|
self.final['AO_failure_preserved'] = True
|
|
put(self.exp / 'logs/first.log', b'complete evidence, not learning\n')
|
|
put(self.exp / 'runs/new/state.npz', b'bulk remains here')
|
|
put(self.exp / 'runs/new/hash.jsonl', b'{}\n')
|
|
put(self.exp / 'extension-build/source.py', b'build source not evidence')
|
|
put(self.exp / 'large.json', b' ' * (P.MAX_BYTES + 1))
|
|
put(self.exp / 'protocol/full-trial-plan.json', {'not_exported_even_if_small': True})
|
|
self.refresh()
|
|
|
|
def refresh(self):
|
|
reg_sha = put(self.exp / P.REGISTRATION, self.reg)
|
|
self.analysis['registration_sha256'] = reg_sha
|
|
self.final['registration_sha256'] = reg_sha
|
|
self.final['analysis_sha256'] = put(self.exp / self.analysis_path, self.analysis)
|
|
put(self.exp / P.FINAL, self.final)
|
|
seal(self.exp)
|
|
self.anchors = (reg_sha, P.digest(self.exp / 'manifest.json'), P.digest(self.exp / P.FINAL))
|
|
|
|
def publish(self, anchors=None):
|
|
return P.publish(*(anchors or self.anchors), root=self.root)
|
|
|
|
def test_success_independent_seal_exact_copy_and_omitted_inventory(self):
|
|
self.fixture()
|
|
prior = P.file_rows(self.exp)
|
|
result = self.publish()
|
|
self.assertTrue(result['publication_complete'])
|
|
self.assertEqual(result['source_count'], 5)
|
|
self.assertFalse(result['complete_bulk_backup'])
|
|
helper = P.load_helper(self.root / P.HELPER_REL)
|
|
registry_seal = helper.verify_seal(self.registry)
|
|
self.assertEqual(registry_seal['manifest_sha256'], result['registry_manifest_sha256'])
|
|
self.assertEqual(result['receipt_sha256'], P.digest(self.registry / 'publication-receipt.json'))
|
|
receipt = read(self.registry / 'publication-receipt.json')
|
|
self.assertEqual(receipt['source_origin']['declared_final_counts']['new_neural_calls'], 11)
|
|
self.assertEqual(receipt['source_origin']['original_AO_status'], P.AO_STATUS)
|
|
self.assertFalse(receipt['source_origin']['imported_bulk_independently_reverified'])
|
|
self.assertFalse(receipt['learning_achieved'])
|
|
for row in receipt['files']:
|
|
self.assertEqual(P.digest(self.registry / row['path']), row['sha256'])
|
|
inventory = read(self.registry / 'artifact-inventory.json')['files']
|
|
self.assertEqual({r['path'] for r in prior}, {r['path'] for r in inventory})
|
|
for row in inventory:
|
|
self.assertEqual(row['sha256'], P.digest(self.exp / row['path']))
|
|
indexed = {r['path']: r for r in inventory}
|
|
for name in ('runs/new/state.npz', 'runs/new/hash.jsonl', 'extension-build/source.py',
|
|
'large.json', 'protocol/full-trial-plan.json'):
|
|
self.assertEqual(indexed[name]['disposition'], 'omitted')
|
|
self.assertTrue((self.registry / 'source-snapshot/coordinator.py').is_file())
|
|
self.assertEqual((self.registry / 'evidence/manifest.json').read_bytes(), (self.exp / 'manifest.json').read_bytes())
|
|
self.assertEqual((self.registry / 'evidence/checksums.sha256').read_bytes(), (self.exp / 'checksums.sha256').read_bytes())
|
|
self.assertTrue(all(p.stat().st_size <= P.MAX_BYTES for p in self.registry.rglob('*') if p.is_file()))
|
|
self.assertEqual(prior, P.file_rows(self.exp))
|
|
|
|
def test_incomplete_and_failed_status_are_not_upgraded(self):
|
|
for status in ('INCOMPLETE-NO-NONZERO-UPDATE-COVERAGE', 'FAIL-CHILD-PROCESS'):
|
|
with self.subTest(status=status):
|
|
# Separate exclusive trees; no deletion/retry of a publication.
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
self.root = Path(temp).resolve()
|
|
self.exp, self.registry, self.model = (self.root / rel for rel in (P.EXP_REL, P.REGISTRY_REL, P.MODEL_REL))
|
|
put(self.root / P.HELPER_REL, HELPER.read_bytes())
|
|
self.fixture(status)
|
|
if status.startswith('FAIL'):
|
|
for value in (self.final, self.analysis):
|
|
value.update(covered_slots=1, new_neural_calls=0)
|
|
self.refresh()
|
|
result = self.publish()
|
|
self.assertEqual(result['declared_audit_status'], status)
|
|
self.assertFalse(result['scientific_gate_certified_by_publication'])
|
|
|
|
def test_three_anchors_are_required_and_rechecked(self):
|
|
self.fixture()
|
|
for index in range(3):
|
|
values = list(self.anchors); values[index] = '0' * 64
|
|
with self.subTest(index=index), self.assertRaisesRegex(ValueError, 'External anchor mismatch'):
|
|
self.publish(values)
|
|
with self.assertRaisesRegex(ValueError, 'mandatory'):
|
|
self.publish(['', self.anchors[1], self.anchors[2]])
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
def test_missing_final_or_unsealed_tree_refuses_publication(self):
|
|
self.fixture()
|
|
(self.exp / P.FINAL).unlink()
|
|
with self.assertRaises(ValueError):
|
|
self.publish()
|
|
put(self.exp / P.FINAL, self.final)
|
|
(self.exp / 'checksums.sha256').unlink()
|
|
with self.assertRaises(ValueError):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
def test_existing_registry_is_untouched(self):
|
|
self.fixture()
|
|
marker = self.registry / 'keep.txt'; put(marker, b'keep')
|
|
with self.assertRaisesRegex(ValueError, 'never overwrite'):
|
|
self.publish()
|
|
self.assertEqual(marker.read_bytes(), b'keep')
|
|
self.assertEqual(len(list(self.registry.iterdir())), 1)
|
|
|
|
def test_source_mutation_and_extra_or_missing_inventory_fail(self):
|
|
self.fixture()
|
|
put(self.model / 'coordinator.py', b'# different')
|
|
with self.assertRaisesRegex(ValueError, 'differ'):
|
|
self.publish()
|
|
put(self.model / 'coordinator.py', (self.exp / P.SNAPSHOT / 'coordinator.py').read_bytes())
|
|
put(self.exp / P.SNAPSHOT / 'extra.py', b'# extra')
|
|
seal(self.exp)
|
|
anchors = (self.anchors[0], P.digest(self.exp / 'manifest.json'), self.anchors[2])
|
|
with self.assertRaisesRegex(ValueError, 'inventory differs'):
|
|
self.publish(anchors)
|
|
|
|
def test_resealed_snapshot_disagrees_with_registered_source(self):
|
|
self.fixture()
|
|
put(self.exp / P.SNAPSHOT / 'coordinator.py', b'# altered snapshot')
|
|
seal(self.exp)
|
|
with self.assertRaisesRegex(ValueError, 'differ'):
|
|
self.publish((self.anchors[0], P.digest(self.exp / 'manifest.json'), self.anchors[2]))
|
|
|
|
def test_import_cannot_be_reclassified_as_new_or_AO_success(self):
|
|
self.fixture()
|
|
self.reg['slots'][0]['origin'] = 'new_AP_neural_execution'; self.refresh()
|
|
with self.assertRaisesRegex(ValueError, 'slot provenance'):
|
|
self.publish()
|
|
self.reg['slots'][0]['origin'] = 'imported_sealed_AO_neural_trajectory'
|
|
self.analysis['original_AO_status'] = 'PASS'; self.refresh()
|
|
with self.assertRaisesRegex(ValueError, 'reclassify AO'):
|
|
self.publish()
|
|
|
|
def test_missing_duplicate_slot_and_source_escape_fail(self):
|
|
self.fixture()
|
|
self.reg['slots'][-1] = dict(self.reg['slots'][0]); self.refresh()
|
|
with self.assertRaisesRegex(ValueError, 'coverage/order'):
|
|
self.publish()
|
|
self.reg['slots'][-1]['row_id'] = 'frozen--light--epoch-001'
|
|
self.reg['slots'][-1]['origin'] = 'new_AP_neural_execution'
|
|
self.reg['slots'][-1]['neural_dir'] = str(self.exp / 'runs/frozen--light--epoch-001')
|
|
self.reg['slots'][-1]['case_dir'] = str(self.exp / 'cases/frozen--light--epoch-001')
|
|
self.reg['slots'][-1]['original_case_dir'] = str(self.root / P.AO_REL / 'cases/frozen--light--epoch-001')
|
|
key = str(self.model / 'coordinator.py')
|
|
self.reg['source_sha256'][str(self.root / 'elsewhere/coordinator.py')] = self.reg['source_sha256'].pop(key)
|
|
self.refresh()
|
|
with self.assertRaisesRegex(ValueError, 'exactly five'):
|
|
self.publish()
|
|
|
|
def test_learning_claim_or_inconsistent_declared_count_rejected(self):
|
|
self.fixture()
|
|
self.analysis['learning_success_claimed'] = True; self.refresh()
|
|
with self.assertRaisesRegex(ValueError, 'No-learning'):
|
|
self.publish()
|
|
self.analysis['learning_success_claimed'] = False
|
|
self.analysis['new_neural_calls'] = 12; self.refresh()
|
|
with self.assertRaisesRegex(ValueError, 'declared count'):
|
|
self.publish()
|
|
|
|
def test_final_analysis_link_and_helper_pin_rejected(self):
|
|
self.fixture()
|
|
self.final['analysis_sha256'] = '0' * 64
|
|
put(self.exp / P.FINAL, self.final); seal(self.exp)
|
|
anchors = (self.anchors[0], P.digest(self.exp / 'manifest.json'), P.digest(self.exp / P.FINAL))
|
|
with self.assertRaisesRegex(ValueError, 'analysis SHA'):
|
|
self.publish(anchors)
|
|
put(self.root / P.HELPER_REL, b'# not the pinned helper')
|
|
with self.assertRaisesRegex(ValueError, 'helper SHA mismatch'):
|
|
self.publish(anchors)
|
|
|
|
def test_unlisted_payload_or_corrupt_checksums_refused(self):
|
|
self.fixture()
|
|
put(self.exp / 'unlisted.txt', b'not registered')
|
|
with self.assertRaisesRegex(ValueError, 'unlisted'):
|
|
self.publish()
|
|
(self.exp / 'unlisted.txt').unlink()
|
|
with (self.exp / 'checksums.sha256').open('ab') as stream:
|
|
stream.write(b'bad\n')
|
|
with self.assertRaisesRegex(ValueError, 'Malformed checksums'):
|
|
self.publish()
|
|
|
|
def test_postcopy_bulk_change_is_caught_without_deleting_partial_registry(self):
|
|
self.fixture()
|
|
helper = P.load_helper(self.root / P.HELPER_REL)
|
|
original = helper._copy_verified
|
|
changed = False
|
|
def copy_and_mutate(*args):
|
|
nonlocal changed
|
|
original(*args)
|
|
if not changed:
|
|
changed = True
|
|
put(self.exp / 'runs/new/state.npz', b'mutated omitted bulk')
|
|
with mock.patch.object(helper, '_copy_verified', side_effect=copy_and_mutate), mock.patch.object(P, 'load_helper', return_value=helper):
|
|
with self.assertRaisesRegex(ValueError, 'Payload byte count|Payload SHA'):
|
|
self.publish()
|
|
self.assertTrue(read(self.registry / 'publication-failure.json')['partial_outputs_preserved'])
|
|
self.assertFalse((self.registry / 'publication-receipt.json').exists())
|
|
with self.assertRaisesRegex(ValueError, 'never overwrite'):
|
|
self.publish()
|
|
|
|
def test_external_manifest_race_and_registry_mutation_detected(self):
|
|
self.fixture()
|
|
helper = P.load_helper(self.root / P.HELPER_REL)
|
|
original = helper.verify_seal
|
|
def new_seal(directory):
|
|
put(self.exp / 'logs/first.log', b'changed before full verification')
|
|
seal(self.exp)
|
|
return original(directory)
|
|
with mock.patch.object(helper, 'verify_seal', side_effect=new_seal), mock.patch.object(P, 'load_helper', return_value=helper):
|
|
with self.assertRaisesRegex(ValueError, 'differs from external'):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
self.anchors = (self.anchors[0], P.digest(self.exp / 'manifest.json'), self.anchors[2])
|
|
self.publish()
|
|
put(self.registry / 'source-snapshot/coordinator.py', b'changed')
|
|
with self.assertRaisesRegex(ValueError, 'Payload byte count|Payload SHA'):
|
|
helper.verify_seal(self.registry)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|