Publish sealed updated-boundary audit and truthful learning status
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,147 @@
|
||||
"""Synthetic seal checks only; no historical integrity or brain execution."""
|
||||
import copy
|
||||
from contextlib import contextmanager
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
import finalize_execution as F
|
||||
|
||||
|
||||
class FinalizeTests(unittest.TestCase):
|
||||
@contextmanager
|
||||
def complete_fixture(self):
|
||||
"""Temporary evidence + mocked auditors/child; no real integrity run."""
|
||||
helper = Path(__file__).parents[1]/'pn-recruitment-audit-v1/publication.py'
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
for folder in ('protocol', 'audit', 'regression'):
|
||||
(root/folder).mkdir()
|
||||
reg = dict(schema='conditioning-execution-registration-v1', engineering_only=True,
|
||||
scientific_acquisition_authorized=False, source_sha256={}, input_sha256={},
|
||||
prepared_file_sha256={}, source_code_commit='a'*40)
|
||||
F.save(root/'protocol/preregistered.json', reg)
|
||||
regsha = F.digest(root/'protocol/preregistered.json')
|
||||
numerical = dict(schema='conditioning-execution-numerical-audit-v1', complete=True,
|
||||
registration_sha256=regsha, actual_new_brain_calls=23, actual_processes=31,
|
||||
actual_new_brain_ticks=125200, learning_pass_claimed=False, candidate_generated=False,
|
||||
status='PASS-EXACT-UPDATED-BOUNDARY-ENGINEERING-NOT-LEARNING', pass_all=True,
|
||||
process_evidence={'synthetic': {'worker_sha256':'b'*64}})
|
||||
budget = dict(schema='synthetic-budget-report-not-scientific', complete=True,
|
||||
registration_sha256=regsha, learning_pass_claimed=False, measured_fixture_seconds=1.)
|
||||
integrity = dict(complete=True, AP={'pass_all':True}, prior={'synthetic_old_roots':'unchanged'},
|
||||
anchors={'synthetic_reference':'c'*64}, old_experiments_reexecuted=False,
|
||||
old_files_modified=False, five_frozen_reference_trials_rerun_here=False,
|
||||
scientific_learning_pass_claimed=False)
|
||||
files = (root/'audit/numerical-comparison.json', root/'audit/budget-report.json',
|
||||
root/'regression/integrity-after.json')
|
||||
for file, value in zip(files, (numerical, budget, integrity)):
|
||||
F.save(file, value)
|
||||
numerical_module = SimpleNamespace(audit=Mock(return_value=copy.deepcopy(numerical)),
|
||||
same_json=lambda a,b: json.dumps(a,sort_keys=True,allow_nan=False)==json.dumps(b,sort_keys=True,allow_nan=False))
|
||||
budget_module = SimpleNamespace(report=Mock(return_value=copy.deepcopy(budget)))
|
||||
def child(command, **kwargs):
|
||||
self.assertEqual(command, [F.sys.executable,'-B',str(F.MODEL/'prepare_execution.py'),
|
||||
'integrity','--output',str(root/'regression/integrity-at-seal.json')])
|
||||
self.assertEqual(kwargs, dict(cwd=F.MODEL,capture_output=True,text=True,timeout=1800))
|
||||
F.save(root/'regression/integrity-at-seal.json', integrity)
|
||||
return SimpleNamespace(returncode=0, stdout='synthetic child', stderr='')
|
||||
with patch.object(F,'EXP',root),patch.object(F,'HELPER',helper), \
|
||||
patch.dict(F.sys.modules,{'audit_execution':numerical_module,'budget_report':budget_module}), \
|
||||
patch.object(F.subprocess,'run',side_effect=child) as process:
|
||||
yield SimpleNamespace(root=root, numerical=numerical, integrity=integrity,
|
||||
audit_module=numerical_module,budget_module=budget_module,process=process,
|
||||
args=(regsha,*(F.digest(p) for p in files)))
|
||||
|
||||
def test_source_helper_matches_actual_local_frozen_file(self):
|
||||
path = Path(__file__).parents[1]/'pn-recruitment-audit-v1/publication.py'
|
||||
with patch.object(F, 'HELPER', path):
|
||||
module = F.helper()
|
||||
self.assertTrue(callable(module.verify_seal))
|
||||
|
||||
def test_exact_seal_detects_mutated_and_extra_payload(self):
|
||||
path = Path(__file__).parents[1]/'pn-recruitment-audit-v1/publication.py'
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(F, 'HELPER', path):
|
||||
root = Path(tmp)
|
||||
F.save(root/'one.json', {'learning':False})
|
||||
result = F.seal(root)
|
||||
self.assertTrue(result['pass_all'])
|
||||
self.assertEqual(result['payload_file_count'], 1)
|
||||
with self.assertRaisesRegex(ValueError, 'Never replace'):
|
||||
F.seal(root)
|
||||
F.save(root/'unlisted.json', {})
|
||||
with self.assertRaisesRegex(ValueError, 'unlisted/missing'):
|
||||
F.helper().verify_seal(root)
|
||||
|
||||
def test_refuses_empty_and_bytecode_trees(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
with self.assertRaisesRegex(ValueError, 'Empty'):
|
||||
F.seal(root)
|
||||
(root/'__pycache__').mkdir()
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||
F.seal(root)
|
||||
|
||||
def test_stopped_workflow_never_gets_normal_final(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(F, 'EXP', Path(tmp)):
|
||||
(Path(tmp)/'audit').mkdir()
|
||||
F.save(Path(tmp)/'audit/stopped.json', {'complete':False})
|
||||
with self.assertRaisesRegex(ValueError, 'separate explicit failure audit'):
|
||||
F.finalize('a'*64, 'b'*64, 'c'*64, 'd'*64)
|
||||
|
||||
def test_full_mock_finalize_recomputes_and_seals_actual_independent_report(self):
|
||||
with self.complete_fixture() as fixture:
|
||||
result = F.finalize(*fixture.args)
|
||||
fixture.audit_module.audit.assert_called_once_with(fixture.args[0],save_report=False)
|
||||
fixture.budget_module.report.assert_called_once_with(fixture.args[0],save_report=False)
|
||||
fixture.process.assert_called_once()
|
||||
final = F.read(fixture.root/'audit/final-status.json')
|
||||
self.assertEqual(result['status'],'PASS-EXACT-UPDATED-BOUNDARY-ENGINEERING-NOT-LEARNING')
|
||||
self.assertEqual(final['integrity_at_seal_sha256'],F.digest(fixture.root/'regression/integrity-at-seal.json'))
|
||||
self.assertTrue(final['numerical_and_budget_recomputed_at_seal'])
|
||||
self.assertFalse(final['scientific_acquisition_executed'])
|
||||
self.assertFalse(final['learning_pass_claimed']); self.assertFalse(final['candidate_generated'])
|
||||
self.assertEqual(final['historical_five_trial_regression'],'AM sealed evidence preserved; not rerun here')
|
||||
sealed = F.helper().verify_seal(fixture.root)
|
||||
self.assertTrue(sealed['pass_all'])
|
||||
payload = {row['path'] for row in F.read(fixture.root/'manifest.json')['files']}
|
||||
self.assertTrue({'audit/numerical-comparison.json','audit/budget-report.json',
|
||||
'regression/integrity-after.json','regression/integrity-at-seal.json',
|
||||
'audit/final-status.json'}.issubset(payload))
|
||||
|
||||
def test_recomputed_numerical_or_budget_mutation_rejected_before_integrity_and_seal(self):
|
||||
for kind in ('numerical','budget'):
|
||||
with self.subTest(kind=kind),self.complete_fixture() as fixture:
|
||||
if kind=='numerical':
|
||||
changed=copy.deepcopy(fixture.numerical)
|
||||
changed['process_evidence']['synthetic']['worker_sha256']='d'*64
|
||||
fixture.audit_module.audit.return_value=changed
|
||||
else:
|
||||
changed=copy.deepcopy(fixture.budget_module.report.return_value)
|
||||
changed['measured_fixture_seconds']=2.
|
||||
fixture.budget_module.report.return_value=changed
|
||||
with self.assertRaisesRegex(ValueError,'evidence changed after the independent audit'):
|
||||
F.finalize(*fixture.args)
|
||||
fixture.process.assert_not_called()
|
||||
for name in ('audit/final-status.json','manifest.json','checksums.sha256'):
|
||||
self.assertFalse((fixture.root/name).exists())
|
||||
|
||||
def test_independent_integrity_failure_missing_output_or_changed_prior_refuses_seal(self):
|
||||
for mode in ('nonzero_exit','missing_file','changed_prior'):
|
||||
with self.subTest(mode=mode),self.complete_fixture() as fixture:
|
||||
def child(*args,**kwargs):
|
||||
if mode=='changed_prior':
|
||||
changed=copy.deepcopy(fixture.integrity); changed['prior']['synthetic_old_roots']='changed'
|
||||
F.save(fixture.root/'regression/integrity-at-seal.json',changed)
|
||||
return SimpleNamespace(returncode=1 if mode=='nonzero_exit' else 0,stdout='',stderr='synthetic failure')
|
||||
fixture.process.side_effect=child
|
||||
with self.assertRaisesRegex(ValueError,'integrity-at-seal process failed|Frozen integrity changed'):
|
||||
F.finalize(*fixture.args)
|
||||
for name in ('audit/final-status.json','manifest.json','checksums.sha256'):
|
||||
self.assertFalse((fixture.root/name).exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user