qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Seal AO interface failure and verified isolated postprocessing diagnosis

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-09-30 17:57:02 +08:00
parent 4ff241bdc9
commit 8e9f8f4d56
126 changed files with 92124 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/phase4ao-publication/publish_phase4ao.py
+291
View File
@@ -0,0 +1,291 @@
"""Publish the sealed AO interface evidence, not a backup or learning verdict.
All three external anchors are mandatory. The original complete experiment is
verified before and after copying; only this new exclusive registry is written.
The registry then receives its own independently verified small-file seal.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
from pathlib import Path, PurePosixPath
import re
import sys
ROOT = Path('/data/digital-drosophila')
MODEL_REL = Path('models/male-cns-lif/olfactory-conditioning-lab-v1')
EXP_REL = Path('experiments/phase4ao-conditioning-interface')
REGISTRY_REL = Path('registry/phase4ao-conditioning-interface')
HELPER_REL = Path('models/male-cns-lif/pn-recruitment-audit-v1/publication.py')
HELPER_SHA = '7a79c499098b74163e7bb16adabcbc54e7069dafa204d0b99c7f4c03d79f8016'
REGISTRATION = 'protocol/preregistered.json'
FINAL = 'audit/final-status.json'
ANALYSIS = 'audit/interface-analysis.json'
FAILURE_ANALYSIS = 'audit/failure-analysis.json'
FULL_PLAN = 'protocol/full-trial-plan.json'
SNAPSHOT = 'protocol/source-snapshot/'
MAX_BYTES = 4 * 1024 ** 2
SUFFIXES = frozenset(('.json', '.md', '.log', '.sha256'))
SHA = re.compile(r'^[0-9a-f]{64}$')
def require(ok, message):
if not ok:
raise ValueError(message)
def digest(path):
value = hashlib.sha256()
with Path(path).open('rb') as handle:
for block in iter(lambda: handle.read(8 << 20), b''):
value.update(block)
return value.hexdigest()
def load_helper(path):
raw = Path(path).read_bytes()
require(hashlib.sha256(raw).hexdigest() == HELPER_SHA, 'Frozen seal helper SHA mismatch')
spec = importlib.util.spec_from_file_location('phase4ao_pinned_seal_helper', path)
module = importlib.util.module_from_spec(spec)
# Execute only the exact pinned helper; never import old experiment runners
# or change any helper globals. No import-generated __pycache__ is written.
exec(compile(raw, str(path), 'exec'), module.__dict__)
return module
def write_json(path, value):
raw = (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False,
allow_nan=False) + '\n').encode('utf-8')
require(len(raw) <= MAX_BYTES, 'Generated evidence exceeds the 4 MiB limit')
with Path(path).open('xb') as handle:
handle.write(raw)
def small_json(helper, experiment, rows, name):
require(name in rows and rows[name]['bytes'] <= MAX_BYTES, 'Missing/oversize required JSON: ' + name)
path = helper._regular_file(experiment, name)
require(digest(path) == rows[name]['sha256'], 'Evidence changed: ' + name)
value = helper._json_bytes(path.read_bytes())
require(isinstance(value, dict), 'Expected JSON object: ' + name)
return value
def source_identity(helper, root, experiment, rows, registration, registration_sha):
model = root / MODEL_REL
source_pins = registration.get('source_sha256')
prepared = registration.get('prepared_file_sha256')
require(isinstance(source_pins, dict) and source_pins and isinstance(prepared, dict),
'Missing registered source/prepared inventory')
files, sealed_names = [], set()
for text, checksum in sorted(source_pins.items()):
require(isinstance(text, str) and isinstance(checksum, str) and SHA.fullmatch(checksum),
'Malformed registered source identity')
path = Path(text)
require(path.is_absolute() and path.parent == model and path.suffix in ('.py', '.md') and
path.name not in ('', '.', '..') and '..' not in path.parts,
'Registered source escapes the fixed model directory')
original = helper._regular_file(root, path.relative_to(root).as_posix())
name = SNAPSHOT + path.name
require(name not in sealed_names and name in rows, 'Missing/duplicate registered snapshot')
sealed_names.add(name)
snapshot = helper._regular_file(experiment, name)
row = rows[name]
require(prepared.get(str(snapshot)) == checksum and row['sha256'] == checksum and
row['bytes'] <= MAX_BYTES and original.stat().st_size == row['bytes'] and
digest(original) == checksum and digest(snapshot) == checksum,
'Model, registration, prepared inventory and sealed snapshot differ: ' + path.name)
files.append(dict(model_source=path.relative_to(root).as_posix(), sealed_source=name,
copy='source-snapshot/' + path.name, bytes=row['bytes'], sha256=checksum))
actual = {name for name in rows if name.startswith(SNAPSHOT)}
require(actual == sealed_names, 'Source snapshot inventory differs from registration')
# Prepared AO files must also be the same sealed payload, not merely listed
# in a registration whose sources happen to match.
for text, checksum in prepared.items():
require(isinstance(text, str) and isinstance(checksum, str) and SHA.fullmatch(checksum),
'Malformed prepared-file identity')
path = Path(text)
require(path.is_absolute() and path.is_relative_to(experiment) and '..' not in path.parts,
'Prepared file escapes this experiment')
name = path.relative_to(experiment).as_posix()
helper._relative_name(name)
require(name in rows and rows[name]['sha256'] == checksum, 'Prepared file differs from seal: ' + name)
require(FULL_PLAN in rows and registration.get('full_trial_plan_file_sha256') == rows[FULL_PLAN]['sha256'] and
prepared.get(str(experiment / FULL_PLAN)) == rows[FULL_PLAN]['sha256'],
'Full plan identity is not bound to the registration')
require(type(registration.get('committed_source_blobs_verified')) is int and
registration['committed_source_blobs_verified'] == len(files) and
isinstance(registration.get('git_code_commit'), str) and
re.fullmatch(r'[0-9a-f]{40}', registration['git_code_commit']), 'Missing registered source version')
return dict(registration_sha256=registration_sha, source_count=len(files), files=files,
git_code_commit=registration['git_code_commit'],
original_snapshot_registration_exact=True,
git_blobs_reverified_by_publisher=False, model_directory_sealed=False)
def selection(row):
name = row['path']
parsed = PurePosixPath(name)
if name == FULL_PLAN:
return None, 'full_trial_plan_intentionally_not_exported'
if 'extension-build' in parsed.parts or 'build' in parsed.parts or '__pycache__' in parsed.parts:
return None, 'build_or_cache_artifact'
if row['bytes'] > MAX_BYTES:
return None, 'exceeds_4_MiB'
if name.startswith(SNAPSHOT):
return 'source-snapshot/' + parsed.name, None
if parsed.suffix.lower() in SUFFIXES:
return 'evidence/' + name, None
return None, 'binary_or_non_selected_extension'
def _check_roots(root, paths):
for path in paths:
current = root
for part in path.relative_to(root).parts:
current /= part
require(not current.is_symlink(), 'Symlink in publication path: ' + str(current))
def _registry_seal(helper, registry):
files = [dict(path=p.relative_to(registry).as_posix(), bytes=p.stat().st_size, sha256=digest(p))
for p in sorted(registry.rglob('*')) if p.is_file()]
write_json(registry / 'manifest.json', dict(root=str(registry), files=files))
raw = ''.join(row['sha256'] + ' ' + row['path'] + '\n' for row in files)
raw += digest(registry / 'manifest.json') + ' manifest.json\n'
require(len(raw.encode('utf-8')) <= MAX_BYTES, 'Registry checksums exceed small-file limit')
with (registry / 'checksums.sha256').open('xb') as handle:
handle.write(raw.encode('utf-8'))
return helper.verify_seal(registry)
def publish(registration_sha256, experiment_manifest_sha256, final_sha256, *, root=ROOT):
"""Create only the fixed AO registry; verify integrity, not science.
The root override exists for isolated synthetic tests. Existing and partial
registries are never overwritten, removed, or resumed.
"""
require(sys.dont_write_bytecode, 'Use Python -B')
for value in (registration_sha256, experiment_manifest_sha256, final_sha256):
require(isinstance(value, str) and SHA.fullmatch(value), 'Three external SHA-256 anchors are mandatory')
root = Path(root)
require(root.is_absolute() and root.is_dir() and not root.is_symlink(), 'Invalid project root')
root = root.resolve(strict=True)
experiment, registry, helper_path = (root / part for part in (EXP_REL, REGISTRY_REL, HELPER_REL))
_check_roots(root, (experiment, root / MODEL_REL, registry, helper_path))
require(not registry.exists(), 'Registry already exists; no overwrite or resume')
helper = load_helper(helper_path)
publisher_sha = digest(__file__)
for relative, expected in ((REGISTRATION, registration_sha256), ('manifest.json', experiment_manifest_sha256),
(FINAL, final_sha256)):
require(digest(helper._regular_file(experiment, relative)) == expected, 'External anchor mismatch: ' + relative)
verified = helper.verify_seal(experiment)
rows = {row['path']: row for row in verified['files']}
require(verified['manifest_sha256'] == experiment_manifest_sha256 and
rows.get(REGISTRATION, {}).get('sha256') == registration_sha256 and
rows.get(FINAL, {}).get('sha256') == final_sha256,
'Verified seal differs from external registration/manifest/final anchors')
registration = small_json(helper, experiment, rows, REGISTRATION)
require(registration.get('schema') == 'phase4ao-conditioning-interface-registration-v1', 'Wrong phase registration')
final = small_json(helper, experiment, rows, FINAL)
status = final.get('status')
require(isinstance(status, str) and re.fullmatch(r'(PASS|FAIL|INCOMPLETE)(-[A-Z0-9]+)*', status),
'Unknown finalized status family')
require(final.get('registration_sha256') == registration_sha256, 'Final status names another registration')
for flag in ('learned_candidate', 'learning_success_claimed', 'full_conditioning_executed'):
require(final.get(flag) is False, 'Final scope flag is not explicitly false: ' + flag)
analysis_sha = final.get('analysis_sha256')
require(isinstance(analysis_sha, str) and SHA.fullmatch(analysis_sha), 'Missing finalized analysis SHA')
linked = [name for name in (ANALYSIS, FAILURE_ANALYSIS)
if name in rows and rows[name]['sha256'] == analysis_sha]
require(len(linked) == 1, 'Final analysis link must identify one sealed interface/failure analysis')
analysis_path = linked[0]
if 'analysis_path' in final:
require(final['analysis_path'] == analysis_path, 'Explicit analysis path disagrees with SHA')
analysis = small_json(helper, experiment, rows, analysis_path)
require(analysis.get('status') == status and analysis.get('registration_sha256') == registration_sha256,
'Final and analysis identities differ')
sources = source_identity(helper, root, experiment, rows, registration, registration_sha256)
inventory, plans = [], []
for row in verified['files'] + helper._control_rows(experiment, verified):
target, reason = selection(row)
inventory.append(dict(**row, disposition='copied' if target else 'omitted', copy=target, reason=reason))
if target:
plans.append((row, target))
targets = [target for _, target in plans]
require(len(set(targets)) == len(targets), 'Publication destination collision')
required = {REGISTRATION, FINAL, 'manifest.json', 'checksums.sha256'}
required.add(analysis_path)
require(all('evidence/' + name in targets for name in required), 'Mandatory evidence is missing/oversize')
require(all(row['copy'] in targets for row in sources['files']), 'A registered source is not exportable')
registry.parent.mkdir(parents=True, exist_ok=True)
registry.mkdir(exist_ok=False)
try:
records = []
for row, target in plans:
source = helper._regular_file(experiment, row['path'])
helper._copy_verified(source, registry / target, row['bytes'], row['sha256'])
records.append(dict(source=source.relative_to(root).as_posix(), copy=target,
bytes=row['bytes'], sha256=row['sha256']))
after = helper.verify_seal(experiment)
require(after == verified, 'Experiment changed during publication')
require(source_identity(helper, root, experiment, rows, registration, registration_sha256) == sources,
'Registered source changed during publication')
require(digest(helper_path) == HELPER_SHA and digest(__file__) == publisher_sha, 'Publication/helper source changed')
write_json(registry / 'seal-verification.json', {k: v for k, v in verified.items() if k != 'files'})
write_json(registry / 'evidence-index.json', records)
write_json(registry / 'source-identity.json', sources)
write_json(registry / 'artifact-inventory.json', dict(
experiment_manifest_sha256=experiment_manifest_sha256, files=inventory,
omitted_files=sum(row['disposition'] == 'omitted' for row in inventory),
omitted_bytes=sum(row['bytes'] for row in inventory if row['disposition'] == 'omitted'),
inventory_covers_complete_experiment_and_controls=True, complete_bulk_backup=False))
receipt_files = [dict(path=p.relative_to(registry).as_posix(), bytes=p.stat().st_size, sha256=digest(p))
for p in sorted(registry.rglob('*')) if p.is_file()]
receipt = dict(schema='phase4ao-publication-v1', publication_complete=True,
registry=REGISTRY_REL.as_posix(), project_root=str(root), declared_audit_status=status,
registration_sha256=registration_sha256, experiment_manifest_sha256=experiment_manifest_sha256,
experiment_checksums_sha256=verified['checksums_sha256'], final_sha256=final_sha256,
analysis_sha256=analysis_sha, analysis_path=analysis_path,
helper_sha256=HELPER_SHA, publisher_sha256=publisher_sha,
helper_globals_modified=False, source_count=sources['source_count'],
copied_evidence_files=len(records), copied_evidence_bytes=sum(row['bytes'] for row in records),
files=receipt_files, bulk_remains_at=EXP_REL.as_posix(), complete_bulk_backup=False,
model_directory_sealed=False, scientific_gate_executed_by_publisher=False,
scientific_gate_certified_by_publication=False, candidate_generated=False, learning_achieved=False,
existing_files_modified=False, new_brain_run=False,
scope='Complete source experiment byte seal verified; small evidence copied. Original status preserved, not reassessed.',
receipt_self_hash='Returned externally; registry manifest separately seals this receipt.',
registry_controls='manifest.json and checksums.sha256 are created after this receipt and independently verified.',
limits=dict(max_selected_file_bytes=MAX_BYTES, selected_suffixes=sorted(SUFFIXES),
full_trial_plan_always_omitted=True, registered_source_snapshot_exception=True))
write_json(registry / 'publication-receipt.json', receipt)
small_seal = _registry_seal(helper, registry)
return dict(publication_complete=True, registry=str(registry), declared_audit_status=status,
registration_sha256=registration_sha256, receipt_sha256=digest(registry / 'publication-receipt.json'),
registry_manifest_sha256=small_seal['manifest_sha256'], registry_checksums_sha256=small_seal['checksums_sha256'],
registry_payload_files=small_seal['payload_file_count'], source_count=sources['source_count'],
copied_evidence_files=len(records), complete_bulk_backup=False,
scientific_gate_executed_by_publisher=False, scientific_gate_certified_by_publication=False)
except Exception as error:
if not (registry / 'publication-failure.json').exists():
write_json(registry / 'publication-failure.json', dict(publication_complete=False,
error_type=type(error).__name__, error=str(error), partial_outputs_preserved=True,
existing_files_modified=False))
raise
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--registration-sha256', required=True)
parser.add_argument('--experiment-manifest-sha256', required=True)
parser.add_argument('--final-sha256', required=True)
args = parser.parse_args()
result = publish(args.registration_sha256, args.experiment_manifest_sha256, args.final_sha256)
print(json.dumps(result, ensure_ascii=False, sort_keys=True, allow_nan=False), flush=True)
if __name__ == '__main__':
main()