Add post-exit normal acquisition audit with frozen gate regression
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,228 @@
|
||||
"""Temporary-file accounting tests only; no experiment, runtime or metrics run."""
|
||||
import ast
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import terminal_ledger as T
|
||||
|
||||
FROZEN_SHA = '1f981f61cd26950e7d4b5393be70b82753ea7e721c0bb75d05f8fd1e92a453fb'
|
||||
FROZEN_CANDIDATES = (
|
||||
Path(__file__).resolve().parents[1] / 'normal-conditioning-v1/append_ledger.py',
|
||||
Path('/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/append_ledger.py'),
|
||||
)
|
||||
FROZEN = next((p for p in FROZEN_CANDIDATES if p.is_file() and
|
||||
hashlib.sha256(p.read_bytes()).hexdigest() == FROZEN_SHA), None)
|
||||
if FROZEN is None:
|
||||
raise RuntimeError('No explicitly supported frozen ledger location has the required SHA-256')
|
||||
|
||||
|
||||
def write_json(p, value):
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes((T.canonical(value) + '\n').encode())
|
||||
|
||||
|
||||
def all_hashes(root):
|
||||
return {str(p.relative_to(root)): T.identity(p) for p in root.rglob('*') if p.is_file()}
|
||||
|
||||
|
||||
class Fixture:
|
||||
"""Two actual successful toy ledger entries, not scientific output."""
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
self.exp, self.model, self.supervisor = (root / name for name in ('exp', 'model', 'supervisor'))
|
||||
for p in (self.exp, self.model, self.supervisor): p.mkdir()
|
||||
shutil.copyfile(FROZEN, self.model / 'append_ledger.py')
|
||||
for name in ('run_normal.py', 'normal_runner.py'):
|
||||
(self.model / name).write_text('# synthetic non-executable source\n', encoding='utf-8')
|
||||
self.external = root / 'external.dat'; self.external.write_bytes(b'read-only synthetic input')
|
||||
self.source = {str(p): T.identity(p)['sha256'] for p in self.model.iterdir()}
|
||||
snapshot = self.exp / 'protocol/source-snapshot'; snapshot.mkdir(parents=True)
|
||||
for p in self.model.iterdir(): shutil.copyfile(p, snapshot / p.name)
|
||||
inventory = dict(train_calls=0, probe_calls=1, brain_calls=1, original_E_calls=0,
|
||||
training_cpu_validation_calls=0)
|
||||
plan = dict(schema='normal-conditioning-plan-v1', inventory=inventory,
|
||||
tasks=[dict(task_id='synthetic/probe', kind='probe')])
|
||||
write_json(self.exp / 'protocol/normal-plan.json', plan)
|
||||
prepared = {str(p): T.identity(p)['sha256'] for p in (self.exp / 'protocol').rglob('*') if p.is_file()}
|
||||
self.reg = dict(schema='normal-conditioning-registration-v1', scientific_scope='normal-acquisition-only',
|
||||
subsequent_assays_authorized=False, administrative_supervisor_directory=str(self.supervisor),
|
||||
source_sha256=self.source, input_sha256={str(self.external): T.identity(self.external)['sha256']},
|
||||
prepared_sha256=prepared, normal_inventory=inventory,
|
||||
budget=dict(output_cap_bytes=16 << 20, minimum_free_reserve_bytes=0))
|
||||
regpath = self.exp / 'protocol/preregistered.json'; write_json(regpath, self.reg)
|
||||
self.sha = T.identity(regpath)['sha256']
|
||||
initial = {Path(p).relative_to(self.exp).as_posix(): T.identity(p) for p in prepared}
|
||||
initial['protocol/preregistered.json'] = T.identity(regpath)
|
||||
ledger_module = types.ModuleType('_toy_frozen_ledger')
|
||||
exec(compile(FROZEN.read_bytes(), str(FROZEN), 'exec'), ledger_module.__dict__)
|
||||
self.ledger = ledger_module.Ledger(self.exp, self.sha, initial, 16 << 20, 0)
|
||||
self.ids = T._expected_entries(plan, self.reg)
|
||||
for n, name in enumerate(self.ids):
|
||||
job = self.exp / 'jobs' / (name + '.json')
|
||||
write_json(job, dict(registration_sha256=self.sha, toy=True, entry_id=name))
|
||||
self.ledger.begin(name, job, dict(scope='synthetic receipt'))
|
||||
receipt = self.exp / 'receipts' / (name + '.json')
|
||||
write_json(receipt, dict(entry_id=name, registration_sha256=self.sha,
|
||||
job_sha256=T.identity(job)['sha256'], complete=True, exit_code=0,
|
||||
timed_out=False, exit_observed_by_parent=True))
|
||||
output = self.exp / ('payload-' + str(n) + '.dat'); output.write_bytes(b'known payload')
|
||||
self.ledger.append(name, {str(p): T.identity(p) for p in (regpath, job, self.external)},
|
||||
{str(p): T.identity(p) for p in (job, receipt, output)},
|
||||
dict(job_file=str(job), process_receipt_file=str(receipt),
|
||||
scope='synthetic accounting only', observed_returncode=0))
|
||||
build = self.exp / 'extension-build'; build.mkdir(); (build / 'mock.bin').write_bytes(b'build')
|
||||
self.original = self.ledger.final_verify(self.ids)
|
||||
assert self.original['pass_all'] is True
|
||||
self.completion = dict(complete=True, registration_sha256=self.sha, inventory=inventory,
|
||||
ledger=self.original, learning_success_claimed=False)
|
||||
write_json(self.exp / T.COMPLETION, self.completion)
|
||||
self.supervisor_values = {
|
||||
'launch-attempt.json': dict(registration_sha256=self.sha, complete=False,
|
||||
terminal_state='UNKNOWN', automatic_retry=False),
|
||||
'launch.json': dict(registration_sha256=self.sha, supervisor_pid=123,
|
||||
launch_observed=True, final_exit_observed=False, not_evidence_of_completion=True,
|
||||
argv=[sys.executable, '-B', str(self.model/'run_normal.py'), 'supervise', '--registration-sha256', self.sha]),
|
||||
'supervisor-start.json': dict(registration_sha256=self.sha, pid=123, complete=False),
|
||||
'exit.json': dict(registration_sha256=self.sha, complete=True, exit_code=0,
|
||||
final_exit_observed=True, error=None, learning_success_claimed=False,
|
||||
scientific_analysis_required=True,
|
||||
argv=[sys.executable, '-B', str(self.model/'normal_runner.py'), '--registration-sha256', self.sha]),
|
||||
}
|
||||
for name, value in self.supervisor_values.items(): write_json(self.supervisor/name, value)
|
||||
for name in ('stdout.log', 'stderr.log'): (self.supervisor/name).write_bytes(b'synthetic terminal log\n')
|
||||
self.start_pins = {name: T.identity(self.supervisor/name) for name in T.START_FILES}
|
||||
|
||||
def run(self):
|
||||
return T.terminal_inventory(self.exp, self.sha, self.model, supervisor_dir=self.supervisor,
|
||||
supervisor_start_pins=self.start_pins)
|
||||
|
||||
|
||||
class TerminalLedgerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name).resolve()
|
||||
self.f = Fixture(self.root)
|
||||
|
||||
def tearDown(self): self.temporary.cleanup()
|
||||
|
||||
def test_authentic_reports_preserved_and_exact_tree_inventory_read_only(self):
|
||||
before = all_hashes(self.root); value = self.f.run()
|
||||
self.assertEqual(before, all_hashes(self.root))
|
||||
self.assertTrue(value['terminal_reconciliation_pass'])
|
||||
self.assertEqual(value['recorded_final_ledger'], self.f.original)
|
||||
self.assertEqual(value['final_ledger'], self.f.original)
|
||||
self.assertFalse(value['current_final_ledger']['pass_all'])
|
||||
self.assertEqual(value['current_final_ledger']['extra_files'], [T.COMPLETION])
|
||||
for key in ('actual_total_bytes', 'immutable_and_ledger_bytes'):
|
||||
self.assertEqual(value['current_final_ledger'][key],
|
||||
self.f.original[key] + T.identity(self.f.exp/T.COMPLETION)['bytes'])
|
||||
for p in self.root.rglob('*'):
|
||||
if p.is_file(): self.assertEqual(value['verified_inventory'][str(p)], T.identity(p))
|
||||
for key in ('historical_seals_verified_here', 'metrics_executed', 'scientific_learning_pass_claimed',
|
||||
'candidate_generated', 'files_written'):
|
||||
self.assertIs(value[key], False)
|
||||
self.assertIs(value['external_OS_quiescence_required'], True)
|
||||
|
||||
def test_schema_and_filesystem_task_key_match_actual_frozen_source(self):
|
||||
model = FROZEN.parent
|
||||
plan = ast.parse((model/'normal_plan.py').read_bytes())
|
||||
schema = next(ast.literal_eval(node.value) for node in plan.body if isinstance(node, ast.Assign)
|
||||
and any(isinstance(target, ast.Name) and target.id == 'SCHEMA' for target in node.targets))
|
||||
self.assertEqual(schema, 'normal-conditioning-plan-v1')
|
||||
source = ast.parse((model/'normal_io.py').read_bytes())
|
||||
function = next(node for node in source.body if isinstance(node, ast.FunctionDef) and node.name == 'task_key')
|
||||
namespace = dict(hashlib=hashlib)
|
||||
exec(compile(ast.Module(body=[function], type_ignores=[]), '<frozen task_key only>', 'exec'), namespace)
|
||||
key = namespace['task_key'](dict(task_id='synthetic/probe'))
|
||||
self.assertEqual(self.f.ids, ['input-generation', key+'--brain'])
|
||||
|
||||
def test_missing_completion_rejects_without_creating_controls(self):
|
||||
(self.f.exp/T.COMPLETION).unlink()
|
||||
shutil.rmtree(self.f.exp/'.append-ledger')
|
||||
before = all_hashes(self.root)
|
||||
with self.assertRaises(ValueError): self.f.run()
|
||||
self.assertEqual(before, all_hashes(self.root))
|
||||
self.assertFalse((self.f.exp/'.append-ledger').exists())
|
||||
|
||||
def test_missing_header_cannot_auto_initialize(self):
|
||||
shutil.rmtree(self.f.exp/'.append-ledger')
|
||||
before = all_hashes(self.root)
|
||||
with self.assertRaisesRegex(ValueError, 'automatic creation forbidden'): self.f.run()
|
||||
self.assertEqual(before, all_hashes(self.root))
|
||||
|
||||
def test_missing_exit_or_live_lock_refused(self):
|
||||
(self.f.supervisor/'exit.json').unlink()
|
||||
with self.assertRaisesRegex(ValueError, 'incomplete/extra'): self.f.run()
|
||||
write_json(self.f.supervisor/'exit.json', self.f.supervisor_values['exit.json'])
|
||||
(self.f.exp/'.append-ledger/writer.lock').write_bytes(b'live')
|
||||
with self.assertRaisesRegex(ValueError, 'Live/stopped/incomplete'): self.f.run()
|
||||
|
||||
def test_external_startup_pin_required_and_tamper_rejected(self):
|
||||
pins = self.f.start_pins
|
||||
self.f.start_pins = dict(pins); self.f.start_pins.pop('launch.json')
|
||||
with self.assertRaisesRegex(ValueError, 'three external'): self.f.run()
|
||||
self.f.start_pins = pins
|
||||
target = self.f.supervisor/'launch.json'; target.write_bytes(target.read_bytes()+b' ')
|
||||
with self.assertRaisesRegex(ValueError, 'conflicts with authority'): self.f.run()
|
||||
|
||||
def test_exit_code_bool_or_failed_or_wrong_command_refused(self):
|
||||
for code in (False, None, 1):
|
||||
record = dict(self.f.supervisor_values['exit.json'], exit_code=code)
|
||||
write_json(self.f.supervisor/'exit.json', record)
|
||||
with self.assertRaisesRegex(ValueError, 'parent-observed'): self.f.run()
|
||||
record = copy.deepcopy(self.f.supervisor_values['exit.json']); record['argv'][2] = str(self.f.model/'run_normal.py')
|
||||
write_json(self.f.supervisor/'exit.json', record)
|
||||
with self.assertRaisesRegex(ValueError, 'argv binding'): self.f.run()
|
||||
|
||||
def test_extra_file_and_changed_build_rejected(self):
|
||||
extra = self.f.exp/'unregistered.dat'; extra.write_bytes(b'x')
|
||||
with self.assertRaisesRegex(ValueError, 'Only the post-ledger'): self.f.run()
|
||||
extra.unlink(); (self.f.exp/'extension-build/mock.bin').write_bytes(b'BUILD')
|
||||
with self.assertRaisesRegex(ValueError, 'Non-completion'): self.f.run()
|
||||
|
||||
def test_historical_payload_and_registered_source_mutation_rejected(self):
|
||||
target = self.f.exp/'payload-0.dat'; original = target.read_bytes(); target.write_bytes(b'x'*len(original))
|
||||
with self.assertRaises(ValueError): self.f.run()
|
||||
target.write_bytes(original)
|
||||
(self.f.model/'normal_runner.py').write_bytes(b'changed source')
|
||||
with self.assertRaisesRegex(ValueError, 'conflicts with authority'): self.f.run()
|
||||
|
||||
def test_historical_pass_and_counters_cannot_be_rewritten(self):
|
||||
for key, replacement in (('pass_all', False), ('actual_total_bytes', 0), ('head_sha256', '0'*64)):
|
||||
completion = copy.deepcopy(self.f.completion); completion['ledger'][key] = replacement
|
||||
write_json(self.f.exp/T.COMPLETION, completion)
|
||||
with self.assertRaises(ValueError): self.f.run()
|
||||
self.assertEqual(self.f.original, self.f.completion['ledger'])
|
||||
|
||||
def test_supervisor_log_change_during_read_rejected_at_end(self):
|
||||
real = T.identity; log = self.f.supervisor/'stdout.log'; seen = 0
|
||||
def changing(p):
|
||||
nonlocal seen
|
||||
result = real(p)
|
||||
if Path(p) == log:
|
||||
seen += 1
|
||||
if seen == 1: log.write_bytes(b'written after initial audit read')
|
||||
return result
|
||||
with mock.patch.object(T, 'identity', side_effect=changing):
|
||||
with self.assertRaisesRegex(ValueError, 'Supervisor files changed'): self.f.run()
|
||||
|
||||
def test_pending_start_or_stopped_marker_not_a_terminal_result(self):
|
||||
name = 'unexpected-start'; job = self.f.exp/'jobs/unexpected.json'; write_json(job, {'toy': True})
|
||||
self.f.ledger.begin(name, job)
|
||||
with self.assertRaises(ValueError): self.f.run()
|
||||
|
||||
def test_duplicate_json_keys_rejected(self):
|
||||
target = self.f.supervisor/'exit.json'; raw = target.read_text(encoding='utf-8')
|
||||
target.write_text(raw[:-2] + ',"complete":true}\n', encoding='utf-8')
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate JSON key'): self.f.run()
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
Reference in New Issue
Block a user