From 1e429fd50d7d91784614d1af50102c2b608163fa Mon Sep 17 00:00:00 2001 From: Digital Drosophila Archive Date: Wed, 30 Sep 2026 21:17:50 +0800 Subject: [PATCH] Add read-only normal-run progress audit and verified startup observation --- docs/progress/phase4ar-normal-acquisition.md | 16 ++ .../phase4ar-startup-observation.json | 29 ++ .../progress_normal.py | 271 ++++++++++++++++++ .../test_progress_normal.py | 168 +++++++++++ 4 files changed, 484 insertions(+) create mode 100644 registry/phase4ar-normal-conditioning-active/phase4ar-startup-observation.json create mode 100644 tools/normal-conditioning-audit/progress_normal.py create mode 100644 tools/normal-conditioning-audit/test_progress_normal.py diff --git a/docs/progress/phase4ar-normal-acquisition.md b/docs/progress/phase4ar-normal-acquisition.md index 1b98db4..ff88aa4 100644 --- a/docs/progress/phase4ar-normal-acquisition.md +++ b/docs/progress/phase4ar-normal-acquisition.md @@ -7,6 +7,11 @@ 4,080 份固定 CS 输入已生成并由真实进程退出记录及账本确认。 独立后台执行器已启动；尚未据此宣称训练完成或学习成功。 +启动后观察到 4 个训练 epoch 已完成原更新及独立 CPU 校验、6 次独立 +probe 已提交；后续仍在运行。这是有时间边界的启动快照，不是当前实时 +累计数，也不是科学 gate 结果。首轮实际 CPU 校验确认使用封存 spike events +重算原 U 更新，父进程记录退出码为 0。 + - 模型：`/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/` - 实验：`/data/digital-drosophila/experiments/phase4ar-normal-conditioning/` - 管理日志：`/data/digital-drosophila/logs/phase4ar-normal-conditioning/` @@ -50,3 +55,14 @@ delayed reward 或任何游戏、动作解码器、RL、视觉、Web/3D。 完整性核验与 CPU 测试是工程证据，不是新数值回归，更不是学习证据。 最终必须另行进行全产物审计、固定学习 gate 分析及旧阶段完整性复核。 仓库同步源码/文档/小型证据，不等于大体量实验数据已经异机备份。 + +## 后续只读检查 + +已安排本线程每小时检查，仅在实质进展、异常、完成或需人工操作时提醒。 +不自动重试、修补已注册源码、进入后续 assay 或声明学习成功。 + +独立只读工具位于 `tools/normal-conditioning-audit/progress_normal.py`， +8 项临时数据测试在服务器通过。它读取实际提交账本与最新退出记录， +不读取大体量脑输出；其进度结果不是完整性报告或科学结论。 +正在执行的 child 在没有真实退出记录时明确保持 UNKNOWN，不能由 PID +或一个提前写出的 results 文件推定成功。完整 run 完成后仍需最终独立审计。 diff --git a/registry/phase4ar-normal-conditioning-active/phase4ar-startup-observation.json b/registry/phase4ar-normal-conditioning-active/phase4ar-startup-observation.json new file mode 100644 index 0000000..2bbd626 --- /dev/null +++ b/registry/phase4ar-normal-conditioning-active/phase4ar-startup-observation.json @@ -0,0 +1,29 @@ +{ + "schema": "phase4ar-startup-observation-v1", + "status": "RUNNING-NOT-LEARNING-PASS", + "registration_sha256": "b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740", + "scientific_source_commit": "80aeb2093db655329774eff7b32fb6e417b8b40b", + "historical_integrity_before_complete": true, + "CPU_source_tests": {"tests": 73, "failures": 0, "errors": 0, "skips": 0}, + "metadata_reader_tests": {"tests": 8, "failures": 0, "errors": 0, "skips": 0, "wall_seconds": 0.032}, + "progress_tool_sha256": "081d5de098c9a249a13394f9669c76d79dd421ca28765397d3a9d8bbbde051d0", + "progress_tool_test_sha256": "f1a34c8d0933cb7226cbeea8afed7321df771e6ad8b1979072af13a5e5bb3c9c", + "observed_committed_process_entries": 19, + "observed_completed_training_epochs": 4, + "observed_completed_probes": 6, + "sampler_committed": true, + "first_epoch_independent_CPU_validation": { + "task_id": "physiological__PAM15__reinforce_A__rep01/normal/paired/000", + "actual_parent_exit_code": 0, + "frozen_U_verify_receipt_recomputed": true, + "actual_sealed_events_used": true, + "brain_finite": true + }, + "learning_success_claimed": false, + "candidate_generated": false, + "snapshot_scope": "startup metadata observations only, not complete final integrity or current real-time progress", + "experiment_snapshot_is_final_seal": false, + "automatic_retry_authorized": false, + "heartbeat_id": "phase-4ar", + "heartbeat_scope": "hourly read-only progress, meaningful changes only; never execute or retry experiments" +} diff --git a/tools/normal-conditioning-audit/progress_normal.py b/tools/normal-conditioning-audit/progress_normal.py new file mode 100644 index 0000000..01e9066 --- /dev/null +++ b/tools/normal-conditioning-audit/progress_normal.py @@ -0,0 +1,271 @@ +"""Read-only metadata progress, NOT integrity, a scientific gate, or liveness. + +One CLI invocation reads registration, small ledger-entry JSON files and file +names. Only the newest committed entry and pending child's small process/result +receipts are inspected. No model imports, array loads, neural execution, full +artifact SHA walk, PID inference, retries, writes or persistent cache. +""" +from __future__ import annotations + +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re + + +ENTRY_ID = re.compile(r'(?:input-generation|task-[0-9a-f]{64}--(?:brain|update|verify))\Z') +RESULT_NAME = re.compile(r'task-[0-9a-f]{64}\.json\Z') +SEQUENCE_NAME = re.compile(r'[0-9]{16}\.json\Z') +SHA = re.compile(r'[0-9a-f]{64}\Z') +MAX_METADATA_BYTES = 64 * 1024 * 1024 + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def safe(path): + path = Path(path) + require(path.is_absolute() and '..' not in path.parts and not any(p.is_symlink() or + (hasattr(p, 'is_junction') and p.is_junction()) for p in (path, *path.parents)), 'Unsafe linked/nonabsolute path') + return path + + +def read_json(path): + path = safe(path) + require(path.is_file() and path.stat().st_size <= MAX_METADATA_BYTES, 'Missing/oversized metadata: ' + str(path)) + raw = path.read_bytes() + def unique(items): + result = {} + for key, value in items: + require(key not in result, 'Duplicate JSON field'); result[key] = value + return result + value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON metadata'))) + require(type(value) is dict, 'Metadata object required') + return value, raw + + +def directory_files(path): + path = safe(path) + if not path.exists(): + return [] + require(path.is_dir(), 'Expected metadata directory') + rows = sorted(path.iterdir()) + for item in rows: + require(safe(item).is_file(), 'Unexpected directory/nonregular metadata child: ' + str(item)) + return rows + + +def named_record(path, regsha): + if not path.exists(): + return None + value, _ = read_json(path) + require(value.get('registration_sha256') == regsha, 'Record registration differs: ' + str(path)) + return value + + +def process_record(exp, entry_id, regsha, *, expected_sha=None, job_sha=None): + path = exp / 'receipts' / entry_id / 'process.json' + if not path.exists(): + return dict(path=str(path), present=False, actual_exit='UNKNOWN') + value, raw = read_json(path) + require(value.get('entry_id') == entry_id and value.get('registration_sha256') == regsha and + type(value.get('complete')) is bool and type(value.get('timed_out')) is bool and + type(value.get('exit_observed_by_parent')) is bool and + (value.get('exit_code') is None or type(value.get('exit_code')) is int), 'Incomplete parent process receipt') + require(SHA.fullmatch(value.get('job_sha256', '')) and (job_sha is None or value['job_sha256'] == job_sha), 'Parent job binding differs') + require(expected_sha is None or hashlib.sha256(raw).hexdigest() == expected_sha, 'Latest committed process receipt SHA differs') + code = value['exit_code'] + require(code is None or value['exit_observed_by_parent'] is True, 'Exit code lacks parent observation') + require(not value['complete'] or (type(code) is int and code == 0 and not value['timed_out']), 'Completion contradicts recorded exit') + return dict(path=str(path), present=True, exit_code=code, timed_out=value['timed_out'], + exit_observed_by_parent=value['exit_observed_by_parent'], complete=value['complete'], + actual_exit='RECORDED-BY-PARENT' if code is not None else 'UNKNOWN', + OS_liveness_checked=False, sha_checked_against_latest_entry=expected_sha is not None) + + +def _progress(exp, control, expected_registration_sha): + regpath = exp / 'protocol/preregistered.json' + if not regpath.exists(): + return dict(status='PREPARING-OR-NOT-REGISTERED', registered=False, + reason='No complete registration file is available; preparation activity is not inferred') + reg, raw = read_json(regpath) + regsha = hashlib.sha256(raw).hexdigest() + require(expected_registration_sha is None or regsha == expected_registration_sha, 'External registration SHA differs') + require(reg.get('schema') == 'normal-conditioning-registration-v1' and + reg.get('scientific_scope') == 'normal-acquisition-only' and reg.get('subsequent_assays_authorized') is False, + 'Unexpected registration schema/scope') + inventory = reg['normal_inventory'] + for field in ('train_calls', 'probe_calls', 'brain_calls', 'original_E_calls', 'training_cpu_validation_calls'): + require(type(inventory[field]) is int and inventory[field] >= 0, 'Missing registered inventory: ' + field) + require(inventory['brain_calls'] == inventory['train_calls'] + inventory['probe_calls'] and + inventory['original_E_calls'] == inventory['training_cpu_validation_calls'] == inventory['train_calls'], + 'Registered normal inventory is inconsistent') + expected_processes = 1 + inventory['brain_calls'] + inventory['original_E_calls'] + inventory['training_cpu_validation_calls'] + ledger = exp / '.append-ledger' + files = directory_files(ledger / 'entries') + starts = directory_files(ledger / 'started') + results = directory_files(exp / 'results') + require(all(RESULT_NAME.fullmatch(p.name) for p in results), 'Unrecognized/partial result file') + if ledger.exists(): + header, _ = read_json(ledger / 'header.json') + require(header.get('schema') == 'normal-append-ledger-v1' and header.get('root') == str(exp) and + header.get('registration_sha256') == regsha and + header.get('initial_inventory', {}).get('protocol/preregistered.json', {}).get('sha256') == regsha, + 'Ledger header registration binding differs') + seen, committed_results, counts = set(), set(), Counter() + latest = None + for number, path in enumerate(files, 1): + require(SEQUENCE_NAME.fullmatch(path.name) and path.name == f'{number:016d}.json', 'Noncontiguous/partial ledger sequence') + row, _ = read_json(path) + entry_id = row.get('entry_id') + require(row.get('schema') == 'normal-append-ledger-v1-entry' and type(row.get('sequence')) is int and + row['sequence'] == number and row.get('registration_sha256') == regsha and + type(entry_id) is str and ENTRY_ID.fullmatch(entry_id) and entry_id not in seen and + row.get('scientific_gate_claimed') is False, 'Ledger entry identity/schema differs') + metadata = row['metadata'] + require(type(metadata.get('observed_returncode')) is int and metadata['observed_returncode'] == 0 and + metadata.get('job_file') == str(exp / 'jobs' / (entry_id + '.json')) and + metadata.get('process_receipt_file') == str(exp / 'receipts' / entry_id / 'process.json'), 'Ledger process metadata differs') + outgoing = row['new_outputs'] + require(type(outgoing) is list and len({x['path'] for x in outgoing}) == len(outgoing), 'Invalid ledger output rows') + process_rows = [x for x in outgoing if x['path'] == metadata['process_receipt_file']] + require(len(process_rows) == 1 and SHA.fullmatch(process_rows[0]['sha256']), 'Committed process receipt is not ledger owned') + result_rows = [x for x in outgoing if Path(x['path']).parent == exp / 'results'] + require(len(result_rows) <= 1, 'Multiple task results in one process entry') + if entry_id == 'input-generation': + require(number == 1 and not result_rows, 'Sampler must be the first committed process') + counts['sampler'] += 1 + else: + require('input-generation' in seen, 'Task commit precedes sampler commit') + key, phase = entry_id.rsplit('--', 1) + counts[phase] += 1 + if phase in ('update', 'verify'): + require(key + '--brain' in seen and (phase != 'verify' or key + '--update' in seen), 'Training commit lacks earlier brain/update') + if result_rows: + expected = str(exp / 'results' / (key + '.json')) + require(result_rows[0]['path'] == expected and expected not in committed_results and phase in ('brain', 'verify'), + 'Unexpected/repeated committed task result') + committed_results.add(expected) + counts['probe_complete' if phase == 'brain' else 'train_complete'] += 1 + else: + require(phase != 'verify', 'Committed training verification lacks its task result') + seen.add(entry_id) + latest = dict(path=path, row=row, process=process_rows[0], result=result_rows[0] if result_rows else None) + require(len(files) <= expected_processes and counts['train_complete'] <= inventory['train_calls'] and + counts['probe_complete'] <= inventory['probe_calls'], 'Recorded progress exceeds registered inventory') + start_ids = {p.stem for p in starts} + require(len(start_ids) == len(starts) and all(p.suffix == '.json' and ENTRY_ID.fullmatch(p.stem) for p in starts) and + seen <= start_ids, 'Missing/invalid ledger start inventory') + pending = sorted(start_ids - seen) + require(len(pending) <= 1, 'More than one uncommitted child in serial ledger') + result_paths = {str(p) for p in results} + require(committed_results <= result_paths, 'Ledger-owned result file is missing') + latest_report = None + if latest is not None: + row = latest['row']; entry_id = row['entry_id'] + start, _ = read_json(ledger / 'started' / (entry_id + '.json')) + require(start['entry_id'] == entry_id and start['registration_sha256'] == regsha and start['next_sequence'] == len(files), 'Latest start binding differs') + receipt = process_record(exp, entry_id, regsha, expected_sha=latest['process']['sha256'], job_sha=start['job']['sha256']) + require(receipt['present'] and receipt['complete'] and receipt['exit_code'] == 0 and not receipt['timed_out'], 'Latest committed process is not recorded successful') + latest_report = dict(sequence=len(files), entry_id=entry_id, process=receipt) + if latest['result'] is not None: + record, raw_result = read_json(latest['result']['path']) + task = record['task']; kind = 'train' if entry_id.endswith('--verify') else 'probe' + key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest() + require(hashlib.sha256(raw_result).hexdigest() == latest['result']['sha256'] and + record.get('schema') == 'normal-task-result-v1' and record.get('registration_sha256') == regsha and + record.get('complete') is True and task['kind'] == kind and entry_id.startswith(key + '--'), 'Latest task result binding differs') + latest_report['task'] = {k: task[k] for k in ('task_id', 'kind', 'block_id', 'arm')} + pending_report = None + if pending: + entry_id = pending[0] + start, _ = read_json(ledger / 'started' / (entry_id + '.json')) + require(start.get('schema') == 'normal-append-ledger-v1-start' and start.get('entry_id') == entry_id and + start.get('registration_sha256') == regsha and start.get('next_sequence') == len(files) + 1 and + start.get('terminal_state') == 'unknown' and start.get('retry_authorized') is False, 'Pending start identity differs') + pending_report = dict(entry_id=entry_id, process=process_record(exp, entry_id, regsha, job_sha=start['job']['sha256'])) + control_records = {name: named_record(control / filename, regsha) for name, filename in + (('launch_attempt', 'launch-attempt.json'), ('launch', 'launch.json'), ('supervisor_start', 'supervisor-start.json'), ('exit', 'exit.json'))} + launch = control_records['launch'] + if launch is not None: + require(launch.get('launch_observed') is True and launch.get('final_exit_observed') is False and + launch.get('not_evidence_of_completion') is True and type(launch.get('supervisor_pid')) is int, 'Incomplete launch observation') + attempt = control_records['launch_attempt'] + if attempt is not None: + require(attempt.get('complete') is False and attempt.get('terminal_state') == 'UNKNOWN' and + attempt.get('automatic_retry') is False, 'Incomplete launch attempt record') + supervisor = control_records['supervisor_start'] + if supervisor is not None: + require(supervisor.get('complete') is False and type(supervisor.get('pid')) is int, 'Incomplete supervisor start record') + stopped = named_record(exp / 'audit/stopped.json', regsha) + completion = named_record(exp / 'audit/execution-complete.json', regsha) + exit_record = control_records['exit'] + status = 'REGISTERED-NOT-STARTED' + if attempt is not None: + status = 'LAUNCH-ATTEMPT-RECORDED-EXIT-UNKNOWN' + if control_records['launch'] is not None or control_records['supervisor_start'] is not None: + status = 'LAUNCH-RECORDED-EXIT-UNKNOWN' + if files: + status = 'BETWEEN-COMMITS-OR-PARENT-STATE-UNKNOWN' + if pending_report: + receipt = pending_report['process'] + status = 'SAMPLER-STARTED-EXIT-UNKNOWN' if pending[0] == 'input-generation' else 'CHILD-STARTED-UNCOMMITTED-EXIT-UNKNOWN' + if receipt['present']: + if receipt['timed_out'] or (receipt['exit_code'] is not None and receipt['exit_code'] != 0): status = 'CHILD-FAILED-UNCOMMITTED' + elif receipt['complete']: status = 'CHILD-EXITED-ZERO-NOT-COMMITTED' + elif receipt['exit_code'] == 0: status = 'CHILD-EXITED-ZERO-NOT-VALIDATED' + if completion is not None: + require(completion.get('complete') is True and completion.get('inventory') == inventory and + len(files) == expected_processes and not pending, 'Execution completion contradicts ledger inventory') + status = 'EXECUTION-COMPLETION-RECORDED-PARENT-EXIT-UNKNOWN' + if exit_record is not None: + require(type(exit_record.get('final_exit_observed')) is bool and type(exit_record.get('complete')) is bool and + (exit_record.get('exit_code') is None or type(exit_record.get('exit_code')) is int), 'Incomplete supervisor exit record') + code = exit_record['exit_code'] + require((code is None) == (exit_record['final_exit_observed'] is False) and exit_record['complete'] is (code == 0), 'Contradictory supervisor exit record') + status = 'SUPERVISOR-ERROR-EXIT-UNKNOWN' if code is None else ('PARENT-EXITED-ZERO' if code == 0 else 'PARENT-EXITED-NONZERO') + if stopped is not None: + require(stopped.get('complete') is False, 'Malformed stopped record') + status = 'STOPPED-RECORD-PRESENT' + return dict(status=status, registered=True, registration_sha256=regsha, registered_inventory=inventory, + expected_process_entries=expected_processes, committed_process_entries=len(files), + sampler_committed=counts['sampler'] == 1, brain_entries=counts['brain'], update_entries=counts['update'], + training_epochs_completed=counts['train_complete'], probes_completed=counts['probe_complete'], + result_files_present=len(results), task_results_ledger_owned=len(committed_results), + result_files_without_commit=sorted(result_paths - committed_results), latest_entry=latest_report, + uncommitted_child=pending_report, control_records=control_records, stopped_record=stopped, + execution_completion_record_present=completion is not None, + registered_workflow_inventory_recorded_complete=(len(files) == expected_processes and + counts['train_complete'] == inventory['train_calls'] and counts['probe_complete'] == inventory['probe_calls'] and not pending), + ledger_writer_lock_present=(ledger / 'writer.lock').exists(), + entries_appeared_during_read=len(directory_files(ledger / 'entries')) != len(files)) + + +def read_progress(exp, control, registration_sha256=None): + """Unknown/inconsistent metadata never become completed work or learning.""" + base = dict(schema='normal-fast-progress-v1', scope='READ-ONLY-METADATA-NOT-INTEGRITY', + integrity_verified=False, historical_receipts_reverified=False, OS_liveness_checked=False, + learning_pass_claimed=False, scientific_metrics_evaluated=False, candidate_generated=False, + writes_performed=False, model_executed=False, automatic_retry_authorized=False, + snapshot_scope='Non-atomic observation; concurrent publication can require reading again') + try: + exp, control = safe(exp), safe(control) + require(registration_sha256 is None or SHA.fullmatch(registration_sha256), 'Invalid external registration SHA') + return dict(base, **_progress(exp, control, registration_sha256)) + except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError) as error: + return dict(base, status='UNKNOWN-METADATA-INCONSISTENT', error=type(error).__name__ + ': ' + str(error), + completion_counts_available=False) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--exp', required=True) + parser.add_argument('--control', required=True) + parser.add_argument('--registration-sha256') + args = parser.parse_args() + print(json.dumps(read_progress(args.exp, args.control, args.registration_sha256), ensure_ascii=False, sort_keys=True, indent=2)) diff --git a/tools/normal-conditioning-audit/test_progress_normal.py b/tools/normal-conditioning-audit/test_progress_normal.py new file mode 100644 index 0000000..079e14e --- /dev/null +++ b/tools/normal-conditioning-audit/test_progress_normal.py @@ -0,0 +1,168 @@ +"""Temporary metadata fixtures only; never starts a model or reads its arrays.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('_progress_subject', Path(__file__).with_name('progress_normal.py')) +P = importlib.util.module_from_spec(spec) +spec.loader.exec_module(P) + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + raw = (json.dumps(value, sort_keys=True) + '\n').encode() + path.write_bytes(raw) + return hashlib.sha256(raw).hexdigest() + + +class Fixture: + def __init__(self, root, register=True): + self.exp, self.control = root / 'EXP', root / 'CONTROL' + self.exp.mkdir(); self.control.mkdir() + self.inventory = dict(train_calls=1, probe_calls=1, brain_calls=2, original_E_calls=1, training_cpu_validation_calls=1) + self.count = 0 + if register: + self.sha = write(self.exp / 'protocol/preregistered.json', dict(schema='normal-conditioning-registration-v1', + scientific_scope='normal-acquisition-only', subsequent_assays_authorized=False, normal_inventory=self.inventory)) + write(self.exp / '.append-ledger/header.json', dict(schema='normal-append-ledger-v1', root=str(self.exp), + registration_sha256=self.sha, initial_inventory={'protocol/preregistered.json': {'sha256': self.sha}})) + + def start(self, entry_id): + job = self.exp / 'jobs' / (entry_id + '.json') + jobsha = write(job, dict(kind='sample' if entry_id == 'input-generation' else 'fixture')) + value = dict(schema='normal-append-ledger-v1-start', entry_id=entry_id, registration_sha256=self.sha, + next_sequence=self.count + 1, prev_entry_sha256=None, terminal_state='unknown', retry_authorized=False, + job=dict(path=str(job), sha256=jobsha)) + write(self.exp / '.append-ledger/started' / (entry_id + '.json'), value) + return jobsha + + def process(self, entry_id, jobsha, code=0, complete=True, timed_out=False): + path = self.exp / 'receipts' / entry_id / 'process.json' + digest = write(path, dict(entry_id=entry_id, registration_sha256=self.sha, job_sha256=jobsha, + complete=complete, exit_code=code, timed_out=timed_out, exit_observed_by_parent=code is not None or timed_out)) + return dict(path=str(path), relative_path=path.relative_to(self.exp).as_posix(), sha256=digest, bytes=path.stat().st_size) + + def result(self, task): + key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest() + path = self.exp / 'results' / (key + '.json') + digest = write(path, dict(schema='normal-task-result-v1', registration_sha256=self.sha, complete=True, task=task)) + return dict(path=str(path), relative_path=path.relative_to(self.exp).as_posix(), sha256=digest, bytes=path.stat().st_size) + + def commit(self, entry_id, task=None): + jobsha = self.start(entry_id) + outputs = [self.process(entry_id, jobsha)] + if task is not None: + outputs.append(self.result(task)) + self.count += 1 + return write(self.exp / f'.append-ledger/entries/{self.count:016d}.json', dict( + schema='normal-append-ledger-v1-entry', entry_id=entry_id, sequence=self.count, registration_sha256=self.sha, + scientific_gate_claimed=False, new_outputs=outputs, metadata=dict(observed_returncode=0, + job_file=str(self.exp / 'jobs' / (entry_id + '.json')), + process_receipt_file=str(self.exp / 'receipts' / entry_id / 'process.json')))) + + def task(self, kind): + task = dict(task_id='block/normal/paired/' + ('000' if kind == 'train' else 'checkpoint-00/probe-A'), + kind=kind, block_id='block', arm='paired') + key = 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest() + return task, key + + def read(self): + return P.read_progress(self.exp, self.control) + + +class ProgressTests(unittest.TestCase): + def test_no_registration_not_learning_or_inferred_preparation_liveness(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp), register=False) + value = f.read() + self.assertEqual(value['status'], 'PREPARING-OR-NOT-REGISTERED') + self.assertFalse(value['registered'] or value['OS_liveness_checked'] or value['learning_pass_claimed']) + + def test_registered_and_launch_pid_are_not_completion(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)) + self.assertEqual(f.read()['status'], 'REGISTERED-NOT-STARTED') + write(f.control / 'launch.json', dict(registration_sha256=f.sha, supervisor_pid=123, + launch_observed=True, final_exit_observed=False, not_evidence_of_completion=True)) + value = f.read() + self.assertEqual(value['status'], 'LAUNCH-RECORDED-EXIT-UNKNOWN') + self.assertEqual(value['expected_process_entries'], 5) + self.assertEqual(value['committed_process_entries'], 0) + + def test_sampler_started_vs_committed_and_latest_actual_exit(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)); f.start('input-generation') + self.assertEqual(f.read()['status'], 'SAMPLER-STARTED-EXIT-UNKNOWN') + f.commit('input-generation') + value = f.read() + self.assertTrue(value['sampler_committed']) + self.assertEqual(value['latest_entry']['entry_id'], 'input-generation') + self.assertTrue(value['latest_entry']['process']['sha_checked_against_latest_entry']) + self.assertFalse(value['integrity_verified']) + + def test_written_result_never_substitutes_for_verify_commit(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)); f.commit('input-generation') + task, key = f.task('train') + f.commit(key + '--brain'); f.commit(key + '--update') + jobsha = f.start(key + '--verify'); f.result(task) + before = f.read() + self.assertEqual(before['training_epochs_completed'], 0) + self.assertEqual(before['result_files_present'], 1) + self.assertEqual(len(before['result_files_without_commit']), 1) + f.process(key + '--verify', jobsha) + self.assertEqual(f.read()['status'], 'CHILD-EXITED-ZERO-NOT-COMMITTED') + f.commit(key + '--verify', task) + after = f.read() + self.assertEqual(after['training_epochs_completed'], 1) + self.assertEqual(after['result_files_without_commit'], []) + + def test_probe_and_train_commits_separate_and_no_neural_files_read(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)); f.commit('input-generation') + train, key = f.task('train') + f.commit(key + '--brain'); f.commit(key + '--update'); f.commit(key + '--verify', train) + probe, key = f.task('probe'); f.commit(key + '--brain', probe) + state = f.exp / 'runs' / key / 'state.npz'; state.parent.mkdir(parents=True); state.write_bytes(b'not-an-array') + value = f.read() + self.assertEqual((value['training_epochs_completed'], value['probes_completed']), (1, 1)) + self.assertEqual(value['task_results_ledger_owned'], 2) + self.assertEqual(value['latest_entry']['task']['kind'], 'probe') + self.assertFalse(value['scientific_metrics_evaluated'] or value['model_executed']) + + def test_failed_child_and_real_supervisor_exit_and_stop_record(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)); jobsha = f.start('input-generation') + f.process('input-generation', jobsha, code=7, complete=False) + self.assertEqual(f.read()['status'], 'CHILD-FAILED-UNCOMMITTED') + write(f.control / 'exit.json', dict(registration_sha256=f.sha, final_exit_observed=True, exit_code=1, complete=False)) + self.assertEqual(f.read()['status'], 'PARENT-EXITED-NONZERO') + write(f.exp / 'audit/stopped.json', dict(registration_sha256=f.sha, complete=False, error='fixture')) + self.assertEqual(f.read()['status'], 'STOPPED-RECORD-PRESENT') + + def test_missing_fields_bad_sequence_or_registration_are_unknown(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)); f.commit('input-generation') + value = P.read_progress(f.exp, f.control, '0' * 64) + self.assertEqual(value['status'], 'UNKNOWN-METADATA-INCONSISTENT') + path = f.exp / '.append-ledger/entries/0000000000000001.json' + row, _ = P.read_json(path); del row['metadata']['observed_returncode']; write(path, row) + self.assertEqual(f.read()['status'], 'UNKNOWN-METADATA-INCONSISTENT') + path.rename(path.with_name('0000000000000002.json')) + self.assertEqual(f.read()['status'], 'UNKNOWN-METADATA-INCONSISTENT') + + def test_latest_receipt_mutation_is_unknown_not_metadata_success(self): + with tempfile.TemporaryDirectory() as temp: + f = Fixture(Path(temp)); f.commit('input-generation') + path = f.exp / 'receipts/input-generation/process.json' + row, _ = P.read_json(path); row['exit_code'] = 9; row['complete'] = False; write(path, row) + value = f.read() + self.assertEqual(value['status'], 'UNKNOWN-METADATA-INCONSISTENT') + self.assertFalse(value['completion_counts_available']) + + +if __name__ == '__main__': + unittest.main()