Digital Drosophila Archive
261 lines
14 KiB
Python
261 lines
14 KiB
Python
"""Temporary synthetic seals only; never publish/register any real evidence."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import publish_phase4ak as publication
|
|
|
|
|
|
HELPER = Path(__file__).resolve().parents[1] / 'pn-recruitment-audit-v1/publication.py'
|
|
if not HELPER.is_file():
|
|
HELPER = publication.ROOT / publication.HELPER_RELATIVE
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def write(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, indent=2), encoding='utf-8')
|
|
|
|
|
|
def seal(root):
|
|
rows = [dict(path=path.relative_to(root).as_posix(), bytes=path.stat().st_size, sha256=sha(path))
|
|
for path in sorted(root.rglob('*')) if path.is_file() and
|
|
path.name not in ('manifest.json', 'checksums.sha256') and '__pycache__' not in path.parts]
|
|
write(root / 'manifest.json', dict(root=str(root), files=rows))
|
|
lines = [row['sha256'] + ' ' + row['path'] for row in rows]
|
|
lines.append(sha(root / 'manifest.json') + ' manifest.json')
|
|
(root / 'checksums.sha256').write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
|
|
|
|
|
class PublicationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory(prefix='phase4ak-publication-synthetic-')
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.model = self.root / publication.MODEL_RELATIVE
|
|
self.experiment = self.root / publication.EXPERIMENT_RELATIVE
|
|
self.registry = self.root / publication.REGISTRY_RELATIVE
|
|
self.model.mkdir(parents=True)
|
|
self.experiment.mkdir(parents=True)
|
|
(self.model / 'module.py').write_text('# synthetic, not a model\n', encoding='utf-8')
|
|
self.status = dict(status='FAIL-SEED-VALIDATION-REPRESENTATION', numerical_gate_pass=True,
|
|
seed_validation_pass=False, novel_to_aj_pass=False, novel_dynamics_executed=False,
|
|
novel_unlock_policy_violation=False, old_frozen_integrity_pass=True, frozen_five_reference_pass=True,
|
|
planned_primary_cases=120, actual_primary_cases=60, aggregated_valid_primary_cases=60,
|
|
actual_checked_repeats=12, candidate_generated=False, learning_achieved=False,
|
|
full_scientific_goal_achieved=False, unchanged_sealed_runtime=True, dynamics_changed=False,
|
|
old_heldout_dynamic_results_parsed=False, biological_delay_measured=False,
|
|
shared_APL_arithmetic_oracle_limitation=True, APL_nonKC_fast_edges_unreplaced=1212,
|
|
novel_scope='Unused for AG/AJ calibration, not historically never simulated')
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
write(self.experiment / 'regression/frozen-five.json', dict(pass_all=True, trials=[{'synthetic': i} for i in range(5)]))
|
|
write(self.experiment / 'regression/old-integrity-after.json', {'synthetic-old-tree': {'pass': True}})
|
|
write(self.experiment / 'nested/case/summary.json', {'synthetic': True})
|
|
(self.experiment / 'REPORT.md').write_text('Synthetic failed hypothesis.\n', encoding='utf-8')
|
|
(self.experiment / 'tests.log').write_text('Synthetic log\n', encoding='utf-8')
|
|
(self.experiment / 'bulk.npy').write_bytes(b'synthetic bulk bytes, not an array')
|
|
seal(self.model)
|
|
seal(self.experiment)
|
|
|
|
def publish(self):
|
|
return publication.publish(self.root, HELPER)
|
|
|
|
def test_failed_hypothesis_is_published_without_scientific_pass(self):
|
|
before = {str(path): sha(path) for base in (self.model, self.experiment)
|
|
for path in base.rglob('*') if path.is_file()}
|
|
result = self.publish()
|
|
self.assertTrue(result['publication_complete'])
|
|
self.assertEqual(result['declared_audit_status'], self.status['status'])
|
|
receipt = json.loads((self.registry / 'publication-receipt.json').read_text(encoding='utf-8'))
|
|
self.assertFalse(receipt['declared_gate_results']['seed_validation_pass'])
|
|
self.assertFalse(receipt['scientific_gate_certified_by_publication'])
|
|
self.assertTrue(receipt['publication_success_is_not_scientific_pass'])
|
|
self.assertTrue((self.registry / 'evidence/nested/case/summary.json').is_file())
|
|
self.assertTrue((self.registry / 'model/checksums.sha256').is_file())
|
|
self.assertFalse((self.registry / 'model/module.py').exists())
|
|
self.assertFalse((self.registry / 'evidence/bulk.npy').exists())
|
|
for row in receipt['files']:
|
|
self.assertEqual(sha(self.registry / row['path']), row['sha256'])
|
|
self.assertEqual(before, {str(path): sha(path) for base in (self.model, self.experiment)
|
|
for path in base.rglob('*') if path.is_file()})
|
|
|
|
def test_incomplete_final_status_remains_incomplete(self):
|
|
self.status.update(status='INCOMPLETE-VALIDATION-EXECUTION', numerical_gate_pass=False)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
seal(self.experiment)
|
|
self.assertEqual(self.publish()['declared_audit_status'], self.status['status'])
|
|
|
|
def test_pass_requires_consistent_declared_gate_flags_and_coverage(self):
|
|
self.status['status'] = 'PASS-FROZEN-D18-INDEPENDENT-VALIDATION'
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
seal(self.experiment)
|
|
with self.assertRaisesRegex(ValueError, 'Declared PASS contradicts'):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
self.status.update(seed_validation_pass=True, novel_to_aj_pass=True, novel_dynamics_executed=True,
|
|
actual_primary_cases=120, aggregated_valid_primary_cases=120, actual_checked_repeats=24)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
seal(self.experiment)
|
|
self.assertEqual(self.publish()['declared_audit_status'], self.status['status'])
|
|
|
|
def test_explicit_frozen_failure_is_recorded_not_hidden(self):
|
|
self.status.update(status='FAIL-FROZEN-INTEGRITY-OR-REGRESSION', old_frozen_integrity_pass=False)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
write(self.experiment / 'regression/old-integrity-after.json', {'synthetic-old-tree': {'pass': False}})
|
|
seal(self.experiment)
|
|
self.publish()
|
|
receipt = json.loads((self.registry / 'publication-receipt.json').read_text(encoding='utf-8'))
|
|
self.assertFalse(receipt['frozen_integrity_report']['declared_pass_by_root']['synthetic-old-tree'])
|
|
|
|
def test_existing_registry_never_overwritten_or_resumed(self):
|
|
self.registry.mkdir(parents=True)
|
|
sentinel = self.registry / 'keep.txt'
|
|
sentinel.write_bytes(b'keep these bytes')
|
|
with self.assertRaisesRegex(ValueError, 'Registry already exists'):
|
|
self.publish()
|
|
self.assertEqual(sentinel.read_bytes(), b'keep these bytes')
|
|
self.assertEqual(list(self.registry.iterdir()), [sentinel])
|
|
|
|
def test_missing_reference_preserved_as_absent_for_incomplete_not_fabricated_fail(self):
|
|
(self.experiment / 'regression/frozen-five.json').unlink()
|
|
self.status.update(status='INCOMPLETE-VALIDATION-EXECUTION', frozen_five_reference_pass=False)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
seal(self.experiment)
|
|
result = self.publish()
|
|
self.assertEqual(result['declared_audit_status'], 'INCOMPLETE-VALIDATION-EXECUTION')
|
|
receipt = json.loads((self.registry / 'publication-receipt.json').read_text(encoding='utf-8'))
|
|
self.assertFalse(receipt['frozen_reference_report']['present'])
|
|
self.assertIsNone(receipt['frozen_reference_report']['declared_pass_all'])
|
|
self.assertIsNone(receipt['frozen_reference_report']['reported_trial_count'])
|
|
self.assertFalse((self.registry / 'evidence/regression/frozen-five.json').exists())
|
|
|
|
def test_missing_integrity_report_blocks_before_destination(self):
|
|
(self.experiment / 'regression/old-integrity-after.json').unlink()
|
|
seal(self.experiment)
|
|
with self.assertRaisesRegex(ValueError, 'Required finalized evidence absent'):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
def test_explicit_reference_failure_is_copied_without_regrading(self):
|
|
self.status.update(status='FAIL-FROZEN-INTEGRITY-OR-REGRESSION', frozen_five_reference_pass=False)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
write(self.experiment / 'regression/frozen-five.json', dict(pass_all=False,
|
|
trials=[dict(synthetic=0, pass_all=False, message='retained exact diagnostic')]))
|
|
seal(self.experiment)
|
|
self.publish()
|
|
receipt = json.loads((self.registry / 'publication-receipt.json').read_text(encoding='utf-8'))
|
|
self.assertTrue(receipt['frozen_reference_report']['present'])
|
|
self.assertFalse(receipt['frozen_reference_report']['declared_pass_all'])
|
|
self.assertEqual(receipt['frozen_reference_report']['reported_trial_count'], 1)
|
|
self.assertEqual((self.registry / 'evidence/regression/frozen-five.json').read_bytes(),
|
|
(self.experiment / 'regression/frozen-five.json').read_bytes())
|
|
|
|
def test_novel_representation_failure_keeps_its_own_status(self):
|
|
self.status.update(status='FAIL-NOVEL-ODOR-REPRESENTATION', seed_validation_pass=True,
|
|
novel_dynamics_executed=True, actual_primary_cases=120, aggregated_valid_primary_cases=120,
|
|
actual_checked_repeats=24)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
seal(self.experiment)
|
|
self.assertEqual(self.publish()['declared_audit_status'], self.status['status'])
|
|
|
|
def test_policy_failure_is_not_relabelled_or_hidden(self):
|
|
self.status.update(status='FAIL-NUMERICAL-OR-EXECUTION-POLICY', numerical_gate_pass=False,
|
|
novel_dynamics_executed=True, novel_unlock_policy_violation=True,
|
|
actual_primary_cases=121)
|
|
write(self.experiment / 'audit/final-status.json', self.status)
|
|
seal(self.experiment)
|
|
self.publish()
|
|
receipt = json.loads((self.registry / 'publication-receipt.json').read_text(encoding='utf-8'))
|
|
self.assertTrue(receipt['declared_gate_results']['novel_unlock_policy_violation'])
|
|
self.assertEqual(receipt['declared_validation_counts']['actual_primary_cases'], 121)
|
|
self.assertEqual(receipt['declared_audit_status'], self.status['status'])
|
|
|
|
def test_frozen_report_contradiction_and_string_boolean_fail_closed(self):
|
|
for changed in ({'frozen_five_reference_pass': False}, {'old_frozen_integrity_pass': False},
|
|
{'seed_validation_pass': 'false'}, {'learning_achieved': True}):
|
|
with self.subTest(changed=changed):
|
|
value = dict(self.status, **changed)
|
|
write(self.experiment / 'audit/final-status.json', value)
|
|
seal(self.experiment)
|
|
with self.assertRaises(ValueError):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
|
|
def test_bulk_corruption_blocks_even_though_bulk_is_not_copied(self):
|
|
path = self.experiment / 'bulk.npy'
|
|
path.write_bytes(b'x' * path.stat().st_size)
|
|
with self.assertRaisesRegex(ValueError, 'Payload SHA-256 mismatch'):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
def test_unlisted_payload_and_bad_manifest_self_hash_fail(self):
|
|
extra = self.experiment / 'unlisted.csv'
|
|
extra.write_text('unlisted\n')
|
|
with self.assertRaisesRegex(ValueError, 'unlisted/missing'):
|
|
self.publish()
|
|
extra.unlink()
|
|
controls = self.experiment / 'checksums.sha256'
|
|
lines = controls.read_text().splitlines()
|
|
lines[-1] = '0' * 64 + ' manifest.json'
|
|
controls.write_text('\n'.join(lines) + '\n')
|
|
with self.assertRaisesRegex(ValueError, 'manifest self-checksum'):
|
|
self.publish()
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
def test_large_json_verified_but_not_copied(self):
|
|
path = self.experiment / 'large.json'
|
|
path.write_bytes(b' ' * (publication.MAX_EVIDENCE_BYTES + 1))
|
|
seal(self.experiment)
|
|
self.publish()
|
|
self.assertFalse((self.registry / 'evidence/large.json').exists())
|
|
receipt = json.loads((self.registry / 'publication-receipt.json').read_text(encoding='utf-8'))
|
|
self.assertEqual(next(r for r in receipt['experiment_payload_not_copied'] if r['path'] == 'large.json')['reason'],
|
|
'exceeds_4_MiB')
|
|
|
|
def test_interrupted_copy_retained_and_cannot_resume(self):
|
|
with mock.patch.object(publication, '_copy', side_effect=OSError('synthetic interrupted copy')):
|
|
with self.assertRaisesRegex(OSError, 'synthetic interrupted copy'):
|
|
self.publish()
|
|
self.assertTrue((self.registry / 'publication-failure.json').is_file())
|
|
self.assertFalse((self.registry / 'publication-receipt.json').exists())
|
|
with self.assertRaisesRegex(ValueError, 'Registry already exists'):
|
|
self.publish()
|
|
|
|
def test_after_copy_full_verification_detects_bulk_change(self):
|
|
original = publication._copy
|
|
changed = False
|
|
|
|
def copy_then_change(helper, source, destination, size, checksum):
|
|
nonlocal changed
|
|
original(helper, source, destination, size, checksum)
|
|
if not changed:
|
|
changed = True
|
|
(self.experiment / 'bulk.npy').write_bytes(b'changed after initial seal check')
|
|
|
|
with mock.patch.object(publication, '_copy', side_effect=copy_then_change):
|
|
with self.assertRaises(ValueError):
|
|
self.publish()
|
|
self.assertTrue((self.registry / 'publication-failure.json').is_file())
|
|
self.assertFalse((self.registry / 'publication-receipt.json').exists())
|
|
|
|
def test_helper_pin_checked_without_modifying_old_globals_or_cache(self):
|
|
helper = publication.load_helpers(HELPER)
|
|
self.assertEqual(helper.MODEL_RELATIVE, Path('models/male-cns-lif/pn-recruitment-audit-v1'))
|
|
self.assertEqual(helper.FINAL_STATUS, 'PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT')
|
|
wrong = self.root / 'wrong-helper.py'
|
|
wrong.write_bytes(b'raise RuntimeError("must never execute")\n')
|
|
with self.assertRaisesRegex(ValueError, 'helper hash mismatch'):
|
|
publication.publish(self.root, wrong)
|
|
self.assertFalse(self.registry.exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|