Add read-only normal-run progress audit and verified startup observation
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,168 @@
|
||||
"""Temporary metadata fixtures only; never starts a model or reads its arrays."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
spec = importlib.util.spec_from_file_location('_progress_subject', Path(__file__).with_name('progress_normal.py'))
|
||||
P = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(P)
|
||||
|
||||
|
||||
def write(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
raw = (json.dumps(value, sort_keys=True) + '\n').encode()
|
||||
path.write_bytes(raw)
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
class Fixture:
|
||||
def __init__(self, root, register=True):
|
||||
self.exp, self.control = root / 'EXP', root / 'CONTROL'
|
||||
self.exp.mkdir(); self.control.mkdir()
|
||||
self.inventory = dict(train_calls=1, probe_calls=1, brain_calls=2, original_E_calls=1, training_cpu_validation_calls=1)
|
||||
self.count = 0
|
||||
if register:
|
||||
self.sha = write(self.exp / 'protocol/preregistered.json', dict(schema='normal-conditioning-registration-v1',
|
||||
scientific_scope='normal-acquisition-only', subsequent_assays_authorized=False, normal_inventory=self.inventory))
|
||||
write(self.exp / '.append-ledger/header.json', dict(schema='normal-append-ledger-v1', root=str(self.exp),
|
||||
registration_sha256=self.sha, initial_inventory={'protocol/preregistered.json': {'sha256': self.sha}}))
|
||||
|
||||
def start(self, entry_id):
|
||||
job = self.exp / 'jobs' / (entry_id + '.json')
|
||||
jobsha = write(job, dict(kind='sample' if entry_id == 'input-generation' else 'fixture'))
|
||||
value = dict(schema='normal-append-ledger-v1-start', entry_id=entry_id, registration_sha256=self.sha,
|
||||
next_sequence=self.count + 1, prev_entry_sha256=None, terminal_state='unknown', retry_authorized=False,
|
||||
job=dict(path=str(job), sha256=jobsha))
|
||||
write(self.exp / '.append-ledger/started' / (entry_id + '.json'), value)
|
||||
return jobsha
|
||||
|
||||
def process(self, entry_id, jobsha, code=0, complete=True, timed_out=False):
|
||||
path = self.exp / 'receipts' / entry_id / 'process.json'
|
||||
digest = write(path, dict(entry_id=entry_id, registration_sha256=self.sha, job_sha256=jobsha,
|
||||
complete=complete, exit_code=code, timed_out=timed_out, exit_observed_by_parent=code is not None or timed_out))
|
||||
return dict(path=str(path), relative_path=path.relative_to(self.exp).as_posix(), sha256=digest, bytes=path.stat().st_size)
|
||||
|
||||
def result(self, task):
|
||||
key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest()
|
||||
path = self.exp / 'results' / (key + '.json')
|
||||
digest = write(path, dict(schema='normal-task-result-v1', registration_sha256=self.sha, complete=True, task=task))
|
||||
return dict(path=str(path), relative_path=path.relative_to(self.exp).as_posix(), sha256=digest, bytes=path.stat().st_size)
|
||||
|
||||
def commit(self, entry_id, task=None):
|
||||
jobsha = self.start(entry_id)
|
||||
outputs = [self.process(entry_id, jobsha)]
|
||||
if task is not None:
|
||||
outputs.append(self.result(task))
|
||||
self.count += 1
|
||||
return write(self.exp / f'.append-ledger/entries/{self.count:016d}.json', dict(
|
||||
schema='normal-append-ledger-v1-entry', entry_id=entry_id, sequence=self.count, registration_sha256=self.sha,
|
||||
scientific_gate_claimed=False, new_outputs=outputs, metadata=dict(observed_returncode=0,
|
||||
job_file=str(self.exp / 'jobs' / (entry_id + '.json')),
|
||||
process_receipt_file=str(self.exp / 'receipts' / entry_id / 'process.json'))))
|
||||
|
||||
def task(self, kind):
|
||||
task = dict(task_id='block/normal/paired/' + ('000' if kind == 'train' else 'checkpoint-00/probe-A'),
|
||||
kind=kind, block_id='block', arm='paired')
|
||||
key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest()
|
||||
return task, key
|
||||
|
||||
def read(self):
|
||||
return P.read_progress(self.exp, self.control)
|
||||
|
||||
|
||||
class ProgressTests(unittest.TestCase):
|
||||
def test_no_registration_not_learning_or_inferred_preparation_liveness(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp), register=False)
|
||||
value = f.read()
|
||||
self.assertEqual(value['status'], 'PREPARING-OR-NOT-REGISTERED')
|
||||
self.assertFalse(value['registered'] or value['OS_liveness_checked'] or value['learning_pass_claimed'])
|
||||
|
||||
def test_registered_and_launch_pid_are_not_completion(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp))
|
||||
self.assertEqual(f.read()['status'], 'REGISTERED-NOT-STARTED')
|
||||
write(f.control / 'launch.json', dict(registration_sha256=f.sha, supervisor_pid=123,
|
||||
launch_observed=True, final_exit_observed=False, not_evidence_of_completion=True))
|
||||
value = f.read()
|
||||
self.assertEqual(value['status'], 'LAUNCH-RECORDED-EXIT-UNKNOWN')
|
||||
self.assertEqual(value['expected_process_entries'], 5)
|
||||
self.assertEqual(value['committed_process_entries'], 0)
|
||||
|
||||
def test_sampler_started_vs_committed_and_latest_actual_exit(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp)); f.start('input-generation')
|
||||
self.assertEqual(f.read()['status'], 'SAMPLER-STARTED-EXIT-UNKNOWN')
|
||||
f.commit('input-generation')
|
||||
value = f.read()
|
||||
self.assertTrue(value['sampler_committed'])
|
||||
self.assertEqual(value['latest_entry']['entry_id'], 'input-generation')
|
||||
self.assertTrue(value['latest_entry']['process']['sha_checked_against_latest_entry'])
|
||||
self.assertFalse(value['integrity_verified'])
|
||||
|
||||
def test_written_result_never_substitutes_for_verify_commit(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp)); f.commit('input-generation')
|
||||
task, key = f.task('train')
|
||||
f.commit(key + '--brain'); f.commit(key + '--update')
|
||||
jobsha = f.start(key + '--verify'); f.result(task)
|
||||
before = f.read()
|
||||
self.assertEqual(before['training_epochs_completed'], 0)
|
||||
self.assertEqual(before['result_files_present'], 1)
|
||||
self.assertEqual(len(before['result_files_without_commit']), 1)
|
||||
f.process(key + '--verify', jobsha)
|
||||
self.assertEqual(f.read()['status'], 'CHILD-EXITED-ZERO-NOT-COMMITTED')
|
||||
f.commit(key + '--verify', task)
|
||||
after = f.read()
|
||||
self.assertEqual(after['training_epochs_completed'], 1)
|
||||
self.assertEqual(after['result_files_without_commit'], [])
|
||||
|
||||
def test_probe_and_train_commits_separate_and_no_neural_files_read(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp)); f.commit('input-generation')
|
||||
train, key = f.task('train')
|
||||
f.commit(key + '--brain'); f.commit(key + '--update'); f.commit(key + '--verify', train)
|
||||
probe, key = f.task('probe'); f.commit(key + '--brain', probe)
|
||||
state = f.exp / 'runs' / key / 'state.npz'; state.parent.mkdir(parents=True); state.write_bytes(b'not-an-array')
|
||||
value = f.read()
|
||||
self.assertEqual((value['training_epochs_completed'], value['probes_completed']), (1, 1))
|
||||
self.assertEqual(value['task_results_ledger_owned'], 2)
|
||||
self.assertEqual(value['latest_entry']['task']['kind'], 'probe')
|
||||
self.assertFalse(value['scientific_metrics_evaluated'] or value['model_executed'])
|
||||
|
||||
def test_failed_child_and_real_supervisor_exit_and_stop_record(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp)); jobsha = f.start('input-generation')
|
||||
f.process('input-generation', jobsha, code=7, complete=False)
|
||||
self.assertEqual(f.read()['status'], 'CHILD-FAILED-UNCOMMITTED')
|
||||
write(f.control / 'exit.json', dict(registration_sha256=f.sha, final_exit_observed=True, exit_code=1, complete=False))
|
||||
self.assertEqual(f.read()['status'], 'PARENT-EXITED-NONZERO')
|
||||
write(f.exp / 'audit/stopped.json', dict(registration_sha256=f.sha, complete=False, error='fixture'))
|
||||
self.assertEqual(f.read()['status'], 'STOPPED-RECORD-PRESENT')
|
||||
|
||||
def test_missing_fields_bad_sequence_or_registration_are_unknown(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp)); f.commit('input-generation')
|
||||
value = P.read_progress(f.exp, f.control, '0' * 64)
|
||||
self.assertEqual(value['status'], 'UNKNOWN-METADATA-INCONSISTENT')
|
||||
path = f.exp / '.append-ledger/entries/0000000000000001.json'
|
||||
row, _ = P.read_json(path); del row['metadata']['observed_returncode']; write(path, row)
|
||||
self.assertEqual(f.read()['status'], 'UNKNOWN-METADATA-INCONSISTENT')
|
||||
path.rename(path.with_name('0000000000000002.json'))
|
||||
self.assertEqual(f.read()['status'], 'UNKNOWN-METADATA-INCONSISTENT')
|
||||
|
||||
def test_latest_receipt_mutation_is_unknown_not_metadata_success(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
f = Fixture(Path(temp)); f.commit('input-generation')
|
||||
path = f.exp / 'receipts/input-generation/process.json'
|
||||
row, _ = P.read_json(path); row['exit_code'] = 9; row['complete'] = False; write(path, row)
|
||||
value = f.read()
|
||||
self.assertEqual(value['status'], 'UNKNOWN-METADATA-INCONSISTENT')
|
||||
self.assertFalse(value['completion_counts_available'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user