Seal Phase 4AK frozen D18 independent validation
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,239 @@
|
||||
"""Pure in-memory SFTP plus temporary local destinations, never a real server."""
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
|
||||
import fetch_phase4ak as fetcher
|
||||
|
||||
|
||||
def data(value):
|
||||
return (json.dumps(value, sort_keys=True, allow_nan=False) + '\n').encode()
|
||||
|
||||
|
||||
def row(name, content):
|
||||
return dict(path=name, bytes=len(content), sha256=fetcher.sha(content))
|
||||
|
||||
|
||||
def sealed(root, payload):
|
||||
rows = [row(name, content) for name, content in sorted(payload.items())]
|
||||
manifest = data(dict(root=root, files=rows))
|
||||
checksums = '\n'.join([entry['sha256'] + ' ' + entry['path'] for entry in rows] +
|
||||
[fetcher.sha(manifest) + ' manifest.json']).encode() + b'\n'
|
||||
return dict(payload, **{'manifest.json': manifest, 'checksums.sha256': checksums}), dict(
|
||||
pass_all=True, root=root, payload_file_count=len(rows), payload_bytes=sum(x['bytes'] for x in rows),
|
||||
manifest_sha256=fetcher.sha(manifest), checksums_sha256=fetcher.sha(checksums))
|
||||
|
||||
|
||||
class FakeSFTP:
|
||||
def __init__(self, files):
|
||||
self.files = files
|
||||
self.opened = []
|
||||
self.hook = None
|
||||
self.symlinks = set()
|
||||
|
||||
def lstat(self, name):
|
||||
if name in self.symlinks:
|
||||
return types.SimpleNamespace(st_mode=stat.S_IFLNK | 0o777, st_size=1)
|
||||
if name in self.files:
|
||||
return types.SimpleNamespace(st_mode=stat.S_IFREG | 0o644, st_size=len(self.files[name]))
|
||||
if any(path.startswith(name.rstrip('/') + '/') for path in self.files):
|
||||
return types.SimpleNamespace(st_mode=stat.S_IFDIR | 0o755, st_size=0)
|
||||
raise FileNotFoundError(name)
|
||||
|
||||
def open(self, name, mode):
|
||||
if mode != 'rb':
|
||||
raise AssertionError('Remote write attempted')
|
||||
self.opened.append(name)
|
||||
if self.hook:
|
||||
self.hook(name)
|
||||
return io.BytesIO(self.files[name])
|
||||
|
||||
|
||||
def fixture(extra_experiment=None, extra_model=None):
|
||||
status = dict(status='INCOMPLETE-VALIDATION-EXECUTION', numerical_gate_pass=False,
|
||||
planned_primary_cases=120, candidate_generated=False, learning_achieved=False)
|
||||
mp = {'module.py': b'# synthetic, not model execution\n', 'config.json': data({'synthetic': True}),
|
||||
'never-download.npy': b'raw model bulk'}
|
||||
mp.update(extra_model or {})
|
||||
ep = {'audit/final-status.json': data(status), 'regression/old-integrity-after.json': data({'tree': {'pass': True}}),
|
||||
'REPORT.md': b'Synthetic incomplete evidence.\n', 'tests.log': b'Synthetic test log.\n',
|
||||
'gpu/example/state.npz': b'large trajectory placeholder not requested',
|
||||
'gpu/example/tick-state-hashes.jsonl': b'non-whitelisted suffix'}
|
||||
ep.update(extra_experiment or {})
|
||||
model, model_proof = sealed(fetcher.MODEL_ROOT, mp)
|
||||
experiment, exp_proof = sealed(fetcher.REMOTE_ROOT + '/' + fetcher.EXPERIMENT_RELATIVE, ep)
|
||||
registry = {}
|
||||
index = []
|
||||
for name, content in experiment.items():
|
||||
if Path(name).suffix.lower() in fetcher.EVIDENCE_SUFFIXES and len(content) <= fetcher.MAX_FILE_BYTES:
|
||||
dest = 'evidence/' + name
|
||||
registry[dest] = content
|
||||
index.append(dict(source=fetcher.EXPERIMENT_RELATIVE + '/' + name,
|
||||
copy=dest, bytes=len(content), sha256=fetcher.sha(content)))
|
||||
for name in fetcher.CONTROLS:
|
||||
dest = 'model/' + name; content = model[name]
|
||||
registry[dest] = content
|
||||
index.append(dict(source=fetcher.MODEL_RELATIVE + '/' + name,
|
||||
copy=dest, bytes=len(content), sha256=fetcher.sha(content)))
|
||||
registry['seal-verification.json'] = data(dict(model=model_proof, experiment=exp_proof))
|
||||
registry['evidence-index.json'] = data(index)
|
||||
receipt = dict(publication_complete=True, registry=fetcher.REGISTRY_RELATIVE, project_root=fetcher.REMOTE_ROOT,
|
||||
source_trees_verified_before_and_after_copy=True, publication_success_is_not_scientific_pass=True,
|
||||
complete_bulk_backup=False, scientific_gate_executed_by_publisher=False,
|
||||
scientific_gate_certified_by_publication=False, candidate_generated=False, learning_achieved=False,
|
||||
model_manifest_sha256=model_proof['manifest_sha256'], experiment_manifest_sha256=exp_proof['manifest_sha256'],
|
||||
files=[row(name, content) for name, content in sorted(registry.items())],
|
||||
limits=dict(max_selected_evidence_file_bytes=fetcher.MAX_FILE_BYTES, selected_suffixes=sorted(fetcher.EVIDENCE_SUFFIXES)),
|
||||
declared_audit_status=status['status'], declared_gate_results={'numerical_gate_pass': False},
|
||||
declared_validation_counts={'planned_primary_cases': 120})
|
||||
registry['publication-receipt.json'] = data(receipt)
|
||||
files = {fetcher.MODEL_ROOT + '/' + name: content for name, content in model.items()}
|
||||
files.update({fetcher.REGISTRY_ROOT + '/' + name: content for name, content in registry.items()})
|
||||
return FakeSFTP(files), fetcher.sha(registry['publication-receipt.json'])
|
||||
|
||||
|
||||
class FetchTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(prefix='phase4ak-fetch-synthetic-')
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.destination = Path(self.temp.name).resolve() / 'report'
|
||||
self.sftp, self.receipt_sha = fixture()
|
||||
|
||||
def receipt(self):
|
||||
return fetcher.json_bytes(self.sftp.files[fetcher.REGISTRY_ROOT + '/publication-receipt.json'])
|
||||
|
||||
def resign(self, receipt):
|
||||
raw = data(receipt)
|
||||
self.sftp.files[fetcher.REGISTRY_ROOT + '/publication-receipt.json'] = raw
|
||||
self.receipt_sha = fetcher.sha(raw)
|
||||
|
||||
def get(self, **kwargs):
|
||||
return fetcher.fetch(self.sftp, self.destination, self.receipt_sha, **kwargs)
|
||||
|
||||
def test_only_receipt_small_files_and_model_source_with_hashes(self):
|
||||
result = self.get()
|
||||
self.assertTrue(result['fetch_complete'])
|
||||
self.assertTrue(result['both_manifest_self_checksums_verified'])
|
||||
self.assertTrue(result['fetch_success_is_not_scientific_pass'])
|
||||
self.assertFalse(result['complete_bulk_backup'])
|
||||
self.assertFalse(result['all_remote_experiment_bulk_rehashed_locally'])
|
||||
self.assertEqual(result['declared_audit_status'], 'INCOMPLETE-VALIDATION-EXECUTION')
|
||||
self.assertTrue((self.destination / 'audit/final-status.json').is_file())
|
||||
self.assertTrue((self.destination / 'model/module.py').is_file())
|
||||
self.assertTrue((self.destination / 'registry/publication-receipt.json').is_file())
|
||||
self.assertFalse((self.destination / 'gpu/example/state.npz').exists())
|
||||
self.assertFalse((self.destination / 'model/never-download.npy').exists())
|
||||
self.assertFalse(any('interpretation' in path or 'phase4aj' in path or path.endswith(('.npy', '.npz', '.jsonl'))
|
||||
for path in self.sftp.opened))
|
||||
for entry in result['files']:
|
||||
self.assertEqual(fetcher.sha((self.destination / entry['local']).read_bytes()), entry['sha256'])
|
||||
|
||||
def test_out_of_band_sha_is_mandatory_and_checked_before_other_remote_reads(self):
|
||||
for checksum in ('', 'g' * 64, '0' * 64):
|
||||
self.sftp.opened.clear()
|
||||
with self.subTest(checksum=checksum), self.assertRaises(ValueError):
|
||||
fetcher.fetch(self.sftp, self.destination, checksum)
|
||||
self.assertFalse(self.destination.exists())
|
||||
self.assertTrue(all(path.endswith('/publication-receipt.json') for path in self.sftp.opened))
|
||||
|
||||
def test_no_overwrite_or_resume_even_empty_existing_directory(self):
|
||||
self.destination.mkdir()
|
||||
with self.assertRaisesRegex(ValueError, 'exists'):
|
||||
self.get()
|
||||
self.assertFalse(self.sftp.opened)
|
||||
self.assertEqual(list(self.destination.iterdir()), [])
|
||||
|
||||
def test_malformed_or_non_small_receipt_paths_fail_without_download(self):
|
||||
for name, size in (('../escape.json', 1), ('evidence/bulk.npy', 1), ('interpretation-metrics.json', 1),
|
||||
('evidence/huge.json', fetcher.MAX_FILE_BYTES + 1), ('evidence/C:/escape.json', 1)):
|
||||
with self.subTest(name=name):
|
||||
self.sftp, self.receipt_sha = fixture()
|
||||
receipt = self.receipt()
|
||||
receipt['files'].append(dict(path=name, bytes=size, sha256='a' * 64))
|
||||
self.resign(receipt)
|
||||
with self.assertRaises(ValueError):
|
||||
self.get()
|
||||
self.assertFalse(self.destination.exists())
|
||||
self.assertEqual(self.sftp.opened, [fetcher.REGISTRY_ROOT + '/publication-receipt.json'])
|
||||
|
||||
def test_manifest_self_checksum_is_not_trusted_from_receipt_alone(self):
|
||||
name = 'evidence/checksums.sha256'; path = fetcher.REGISTRY_ROOT + '/' + name
|
||||
lines = self.sftp.files[path].decode().splitlines()
|
||||
lines[-1] = '0' * 64 + ' manifest.json'
|
||||
changed = ('\n'.join(lines) + '\n').encode()
|
||||
self.sftp.files[path] = changed
|
||||
receipt = self.receipt()
|
||||
next(entry for entry in receipt['files'] if entry['path'] == name).update(bytes=len(changed), sha256=fetcher.sha(changed))
|
||||
self.resign(receipt)
|
||||
with self.assertRaisesRegex(ValueError, 'self-SHA'):
|
||||
self.get()
|
||||
self.assertFalse(self.destination.exists())
|
||||
|
||||
def test_model_original_seal_must_equal_registry_anchor(self):
|
||||
self.sftp.files[fetcher.MODEL_ROOT + '/manifest.json'] += b' '
|
||||
with self.assertRaises(ValueError):
|
||||
self.get()
|
||||
self.assertFalse(self.destination.exists())
|
||||
|
||||
def test_corrupt_selected_payload_preserves_partial_and_blocks_resume(self):
|
||||
path = fetcher.REGISTRY_ROOT + '/evidence/REPORT.md'
|
||||
self.sftp.files[path] = b'x' * len(self.sftp.files[path])
|
||||
with self.assertRaisesRegex(ValueError, 'SHA'):
|
||||
self.get()
|
||||
self.assertTrue((self.destination / 'fetch-failure.json').is_file())
|
||||
self.assertFalse((self.destination / 'fetch-verification.json').exists())
|
||||
with self.assertRaisesRegex(ValueError, 'exists'):
|
||||
self.get()
|
||||
|
||||
def test_model_source_hash_is_verified_without_fetching_other_payloads(self):
|
||||
path = fetcher.MODEL_ROOT + '/module.py'
|
||||
self.sftp.files[path] = b'x' * len(self.sftp.files[path])
|
||||
with self.assertRaisesRegex(ValueError, 'SHA'):
|
||||
self.get()
|
||||
self.assertTrue((self.destination / 'fetch-failure.json').is_file())
|
||||
self.assertFalse(any(path.endswith('never-download.npy') for path in self.sftp.opened))
|
||||
|
||||
def test_local_path_collision_and_total_budget_fail_before_destination(self):
|
||||
self.sftp, self.receipt_sha = fixture(extra_experiment={'model/module.py.json': data({'not_source': True}),
|
||||
'model/config.json': data({'collision': True})})
|
||||
with self.assertRaisesRegex(ValueError, 'collision'):
|
||||
self.get()
|
||||
self.assertFalse(self.destination.exists())
|
||||
self.sftp, self.receipt_sha = fixture()
|
||||
with self.assertRaisesRegex(ValueError, 'total download budget'):
|
||||
self.get(max_total_bytes=1)
|
||||
self.assertFalse(self.destination.exists())
|
||||
|
||||
def test_remote_symlink_rejected_without_open(self):
|
||||
path = fetcher.REGISTRY_ROOT + '/publication-receipt.json'
|
||||
self.sftp.symlinks.add(path)
|
||||
with self.assertRaisesRegex(ValueError, 'symlink'):
|
||||
self.get()
|
||||
self.assertFalse(self.sftp.opened)
|
||||
|
||||
def test_anchor_changed_during_copy_fails_final_check(self):
|
||||
receipt_path = fetcher.REGISTRY_ROOT + '/publication-receipt.json'
|
||||
def change_on_source(name):
|
||||
if name == fetcher.MODEL_ROOT + '/module.py':
|
||||
self.sftp.files[receipt_path] += b' '
|
||||
self.sftp.hook = change_on_source
|
||||
with self.assertRaises(ValueError):
|
||||
self.get()
|
||||
self.assertTrue((self.destination / 'fetch-failure.json').is_file())
|
||||
self.assertFalse((self.destination / 'fetch-verification.json').exists())
|
||||
|
||||
def test_no_import_network_and_cli_rejects_unknown_hosts(self):
|
||||
source = Path(fetcher.__file__).read_text(encoding='utf-8')
|
||||
self.assertIn('paramiko.RejectPolicy()', source)
|
||||
self.assertIn('client.load_system_host_keys(str(args.known_hosts))', source)
|
||||
self.assertIn("default=Path.home() / '.ssh/known_hosts'", source)
|
||||
self.assertNotIn('AutoAddPolicy', source)
|
||||
self.assertNotIn('interpretation-metrics.json', source)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user