Files
digital-drosophila/tools/phase4am-publication/publish_phase4am.py
T
2026-09-30 16:11:14 +08:00

355 lines
24 KiB
Python

"""Exclusive small-evidence publication after both Phase 4AM seals verify.
Publication verifies and copies bytes. It never runs a brain, re-evaluates a
scientific gate, converts FAIL/INCOMPLETE into PASS, or changes frozen sources.
The source-pinned Phase 4AI verification/copy helpers are executed unchanged.
"""
from __future__ import annotations
import hashlib
import importlib.util
import json
import math
from pathlib import Path, PurePosixPath
ROOT = Path('/data/digital-drosophila')
MODEL_RELATIVE = Path('models/male-cns-lif/wholebrain-plastic-interface-v1')
EXPERIMENT_RELATIVE = Path('experiments/phase4am-wholebrain-plastic-interface')
REGISTRY_RELATIVE = Path('registry/phase4am-wholebrain-plastic-interface')
HELPER_RELATIVE = Path('models/male-cns-lif/pn-recruitment-audit-v1/publication.py')
HELPER_SHA256 = '7a79c499098b74163e7bb16adabcbc54e7069dafa204d0b99c7f4c03d79f8016'
MAX_EVIDENCE_BYTES = 4 * 1024 ** 2
EVIDENCE_SUFFIXES = frozenset(('.json', '.csv', '.md', '.log', '.sha256'))
FINAL_STATUS = 'PASS-EXACT-WHOLEBRAIN-PLASTIC-INTERFACE-NOT-LEARNING'
# Finalize currently seals only this completed diagnostic status. A partial
# run is not promoted; future failure sealing needs an explicit new contract.
FINAL_STATUSES = frozenset((FINAL_STATUS,))
REQUIRED_EVIDENCE = ('audit/final-status.json', 'regression/old-integrity-after.json',
'audit/scientific-summary.json', 'protocol/preregistered.json',
'audit/zero-complete.json', 'audit/static25-complete.json')
REFERENCE_EVIDENCE = 'regression/frozen-five.json'
GATE_FIELDS = ('zero_overlay_exact', 'static25_repeat_exact', 'old_integrity_pass',
'frozen_five_reference_pass', 'frozen_eligibility_and_mask_unchanged', 'base_weights_unchanged')
COUNT_FIELDS = ('primary_trials_per_stage', 'repeats_per_stage')
COHORTS = ('seed_validation', 'novel_to_aj')
MODES = ('physiological', 'matched_energy')
STAGES = ('zero', 'static25')
POPULATIONS = ('KC', 'MBON', 'mask_recipient_MBON', 'DAN', 'MBON_downstream', 'descending', 'motor')
REFERENCE_NAMES = ('auditory-10hz', 'auditory-100hz', 'olfactory-50hz',
'mechanosensory-50hz', 'gustatory-50hz')
EXPECTED_CONFIG = dict(phase='4AM',
purpose='Exact D18 trial-boundary plastic-overlay interface and closed-loop transfer gate',
neurons=166700, edges=25582938, candidate_edges=14232, primary_trials=120, technical_repeats=24,
steps=1000, dt_ms=.1, apl_delay_steps=18, stages=list(STAGES), max_fractional_depression=.25,
virtual_gate_tick=900, gpu_uuid='GPU-79f3a85d-adce-7833-7003-6488140ca596', output_budget_bytes=17179869184,
training=False, candidate_generation=False, static_test_is_learning=False,
precision_bridge='base float64 SI plus separate float64(delta float32 mV) times 0.001; zero rows untouched',
future_learning='Original frozen eligibility-v2 and additive trial-boundary update; artificial scalar D remains separate from actual DAN events')
SCIENCE_SCOPE = 'Valid interface and recording; reported static response is not an associative learning PASS'
def require(condition, message):
if not condition:
raise ValueError(message)
def load_helpers(path):
"""Instantiate the unchanged module from its exact pinned source bytes.
Using module_from_spec plus the source bytes avoids writing/importing a pyc
inside the sealed old model, even if the publisher was invoked without -B.
No old module global is redirected or modified after module initialization.
"""
path = Path(path)
require(path.is_absolute() and path.is_file() and not path.is_symlink(), 'Invalid frozen helper path')
data = path.read_bytes()
require(hashlib.sha256(data).hexdigest() == HELPER_SHA256, 'Frozen AI publication helper hash mismatch')
spec = importlib.util.spec_from_file_location('phase4am_frozen_ai_publication_helpers', path)
require(spec is not None and spec.loader is not None, 'Unable to define frozen helper module')
module = importlib.util.module_from_spec(spec)
exec(compile(data, str(path), 'exec'), module.__dict__)
require(module._digest(path) == HELPER_SHA256, 'Helper source changed while loading')
return module
def _copy(helper, source, destination, size, checksum):
"""Separate local delegation point for interruption-only synthetic tests."""
helper._copy_verified(source, destination, size, checksum)
def _write_small_json(helper, path, value):
data = (json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + '\n').encode('utf-8')
require(len(data) <= MAX_EVIDENCE_BYTES, 'Generated registry evidence exceeds 4 MiB: ' + str(path))
with Path(path).open('xb') as stream:
stream.write(data)
require(Path(path).stat().st_size == len(data), 'Generated registry evidence size changed')
def _small_json(helper, experiment, rows, relative):
require(relative in rows, 'Required finalized evidence absent from seal: ' + relative)
require(rows[relative]['bytes'] <= MAX_EVIDENCE_BYTES, 'Required finalized evidence exceeds 4 MiB')
return helper._json_bytes((experiment / relative).read_bytes())
def _planned_inventory(registration, config):
require(registration.get('config') == config and registration.get('training') is False and
registration.get('candidate_generated') is False and registration.get('before_new_brain_trajectories') is True,
'Preregistration configuration/scope mismatch')
planned = registration.get('planned')
require(isinstance(planned, list) and len(planned) == 144, 'Planned inventory must contain 144 records')
primary, repeated = {}, {}
for row in planned:
require(isinstance(row, dict) and isinstance(row.get('name'), str) and row['name'] and
PurePosixPath(row['name']).name == row['name'] and row['name'] not in ('.', '..') and
not any(c in row['name'] for c in ('\\', ':', '\0', '\r', '\n')), 'Invalid planned case name')
require(row.get('cohort') in COHORTS and row.get('mode') in MODES and type(row.get('seed')) is int and
isinstance(row.get('odor_id'), str) and row['odor_id'] and type(row.get('repeat')) is int and
row['repeat'] in (0, 1), 'Invalid planned cohort/mode/seed/repeat')
target = primary if row['repeat'] == 0 else repeated
require(row['name'] not in target, 'Duplicate planned case/repeat')
target[row['name']] = row
require(len(primary) == 120 and len(repeated) == 24, 'Planned primary/repeat coverage mismatch')
seeds = {row['seed'] for row in primary.values()}
require(len(seeds) == 5, 'Planned inventory requires five seeds')
identities = {(r['cohort'], r['mode'], r['odor_id'], r['seed']) for r in primary.values()}
require(len(identities) == 120, 'Duplicate planned scientific identity')
for cohort in COHORTS:
odors_by_mode = []
for mode in MODES:
block = [r for r in primary.values() if r['cohort'] == cohort and r['mode'] == mode]
odors = {r['odor_id'] for r in block}
require(len(block) == 30 and len(odors) == 6 and
{(r['odor_id'], r['seed']) for r in block} == {(o, s) for o in odors for s in seeds},
'Incomplete planned cohort-mode inventory')
odors_by_mode.append(odors)
require(odors_by_mode[0] == odors_by_mode[1], 'Modes use different odor identities')
require(len({r['odor_id'] for r in primary.values()}) == 12, 'Cohorts must retain twelve distinct frozen odors')
expected_repeats = {name for name, r in primary.items() if r['seed'] == min(seeds)}
require(set(repeated) == expected_repeats and all(repeated[name] == dict(primary[name], repeat=1)
for name in repeated), 'Repeats do not match the complete first-seed inventory')
return planned, primary
def _stage_coverage(helper, experiment, rows, stage, planned):
receipt = _small_json(helper, experiment, rows, 'audit/' + stage + '-complete.json')
require(receipt.get('stage') == stage and receipt.get('pass_all') is True and receipt.get('training') is False and
all(type(receipt.get(k)) is int and receipt[k] == v for k, v in
(('trials', 144), ('primary_trials', 120), ('technical_repeats', 24))), 'Incomplete stage gate receipt: ' + stage)
completed = receipt.get('completed')
require(isinstance(completed, list) and len(completed) == 144, 'Stage receipt lacks 144 completed records')
project = experiment.parents[1]
for plan, entry in zip(planned, completed):
require(isinstance(entry, dict) and set(entry) == {'name', 'repeat', 'output_hashes_sha256', 'comparison_sha256'}
and entry['name'] == plan['name'] and type(entry['repeat']) is int and entry['repeat'] == plan['repeat'],
'Stage receipt inventory order/identity mismatch')
prefix = 'gpu/' + stage + '/' + plan['name'] + '-r' + str(plan['repeat']) + '/'
for filename, key in (('output-hashes.json', 'output_hashes_sha256'), ('comparison.json', 'comparison_sha256')):
require(prefix + filename in rows and rows[prefix + filename]['sha256'] == entry[key],
'Stage receipt does not bind sealed output evidence: ' + prefix + filename)
report = _small_json(helper, experiment, rows, prefix + 'comparison.json')
mode = 'zero' if stage == 'zero' else ('repeat' if plan['repeat'] else 'self')
folder = experiment / prefix.rstrip('/')
case = project / 'experiments/phase4ak-apl-timing-validation/cases' / plan['cohort'] / plan['name']
left = (project / 'experiments/phase4ak-apl-timing-validation/gpu' / plan['cohort'] /
(plan['name'] + '-r' + str(plan['repeat'])) if stage == 'zero' else
(experiment / 'gpu' / stage / (plan['name'] + '-r0') if plan['repeat'] else folder))
require(report.get('pass_all') is True and report.get('overlay_name') == stage and report.get('mode') == mode and
report.get('case_dir') == str(case) and report.get('left') == str(left) and report.get('right') == str(folder),
'Stage comparison evidence has wrong case/stage/repeat identity')
inventory = _small_json(helper, experiment, rows, prefix + 'output-hashes.json')
require(isinstance(inventory, list) and inventory, 'Empty trial output inventory')
expected = {name[len(prefix):]: {**r, 'path': name[len(prefix):]} for name, r in rows.items()
if name.startswith(prefix) and name != prefix + 'output-hashes.json'}
actual = {}
for item in inventory:
require(isinstance(item, dict) and set(item) == {'path', 'bytes', 'sha256'} and
item['path'] not in actual, 'Invalid/duplicate trial output inventory')
actual[item['path']] = item
require(actual == expected, 'Trial output inventory contradicts complete experiment seal')
return dict(pass_all=True, trials=144, primary_trials=120, technical_repeats=24,
receipt_sha256=rows['audit/' + stage + '-complete.json']['sha256'])
def _status_evidence(helper, experiment, rows, model):
"""Verify declarations and their sealed coverage, never recompute science."""
evidence = {name: _small_json(helper, experiment, rows, name) for name in REQUIRED_EVIDENCE}
status = evidence['audit/final-status.json']
science = evidence['audit/scientific-summary.json']
integrity = evidence['regression/old-integrity-after.json']
config = helper._json_bytes((model / 'config.json').read_bytes())
require(isinstance(config, dict) and set(config) == set(EXPECTED_CONFIG) and all(
type(config[k]) is type(v) and config[k] == v for k, v in EXPECTED_CONFIG.items()),
'Changed Phase 4AM configuration/design contract')
require(isinstance(status, dict) and status.get('status') in FINAL_STATUSES, 'Unrecognized Phase 4AM final status')
require(all(status.get(key) is True for key in GATE_FIELDS), 'Declared PASS contradicts finalized gate flags')
require(type(status.get('primary_trials_per_stage')) is int and status['primary_trials_per_stage'] == 120 and
type(status.get('repeats_per_stage')) is int and status['repeats_per_stage'] == 24, 'Invalid finalized coverage')
require(status.get('training') is False and status.get('learned_candidate') is False, 'Unexpected finalized scope claim')
for record in (status, science):
require(isinstance(record, dict) and all(record.get(k, False) is False for k in
('learning_achieved', 'candidate_generated', 'conditioning_executed', 'full_scientific_goal_achieved')),
'Unexpected learning/candidate scope claim')
require(status.get('scientific_summary_sha256') == rows['audit/scientific-summary.json']['sha256'],
'Final status scientific-summary SHA mismatch')
require(isinstance(science, dict) and science.get('status') == status['status'] and
science.get('zero_overlay_exact') is True and science.get('static25_repeat_exact') is True and
science.get('training') is False and science.get('learned_candidate') is False and
science.get('conditional_AL_not_a_nonzero_arithmetic_oracle') is True and science.get('scope') == SCIENCE_SCOPE,
'Scientific summary scope/status contradiction')
require(type(science.get('primary_trials_per_stage')) is int and science['primary_trials_per_stage'] == 120 and
type(science.get('technical_repeats_per_stage')) is int and science['technical_repeats_per_stage'] == 24,
'Scientific summary coverage mismatch')
require('summaries/per-MBON-static-effects.parquet' in rows and science.get('per_MBON_sha256') ==
rows['summaries/per-MBON-static-effects.parquet']['sha256'], 'Scientific per-MBON payload SHA mismatch')
require(isinstance(integrity, dict) and integrity and all(isinstance(r, dict) and r.get('pass') is True
for r in integrity.values()), 'Final status contradicts its frozen-integrity report')
reference = _small_json(helper, experiment, rows, REFERENCE_EVIDENCE)
require(isinstance(reference, dict), 'Invalid frozen reference report')
trials = reference.get('trials')
require(reference.get('pass_all') is True and isinstance(trials, list) and
[r.get('trial') for r in trials if isinstance(r, dict)] == list(REFERENCE_NAMES) and len(trials) == 5 and
all(isinstance(r.get('exact'), dict) and set(r['exact']) == {'step', 'neuron_idx', 'counts', 'v', 'g'} and
all(value is True for value in r['exact'].values()) for r in trials),
'Declared frozen-five PASS requires five exact trial records with the original ordered names')
blocks = science.get('blocks')
require(isinstance(blocks, dict) and set(blocks) == {c + '/' + m for c in COHORTS for m in MODES},
'Missing/extra cohort-mode blocks')
for block in blocks.values():
require(isinstance(block, dict) and type(block.get('trials')) is int and block['trials'] == 30 and
isinstance(block.get('populations'), dict) and set(block['populations']) == set(POPULATIONS),
'Incomplete cohort-mode population block')
for metrics in block['populations'].values():
require(isinstance(metrics, dict) and set(metrics) == {'trials_event_change', 'trials_count_change',
'mean_count_l1', 'mean_recruited_cells', 'mean_lost_cells'}, 'Invalid population metric fields')
for key in ('trials_event_change', 'trials_count_change'):
require(type(metrics[key]) is int and 0 <= metrics[key] <= 30, 'Invalid diagnostic count')
for key in ('mean_count_l1', 'mean_recruited_cells', 'mean_lost_cells'):
require(type(metrics[key]) in (int, float) and math.isfinite(metrics[key]) and metrics[key] >= 0,
'Nonfinite/negative population metric')
require(metrics['trials_count_change'] <= metrics['trials_event_change'], 'Count change without event change')
# Zero effects remain publishable: this is not an effect-size gate.
planned, primary = _planned_inventory(evidence['protocol/preregistered.json'], config)
timing = science.get('timing')
require(isinstance(timing, list) and len(timing) == 240 and all(isinstance(r, dict) for r in timing) and
{(r.get('stage'), r.get('case')) for r in timing} == {(s, n) for s in STAGES for n in primary},
'Scientific timing inventory lacks all 240 primary runs')
for item in timing:
require(all(type(item.get(k)) in (int, float) and math.isfinite(item[k]) and item[k] >= 0
for k in ('wall_seconds', 'simulation_seconds')), 'Invalid diagnostic timing value')
stages = {stage: _stage_coverage(helper, experiment, rows, stage, planned) for stage in STAGES}
return status, reference, integrity, stages
def publish(root=ROOT, helper_path=None):
"""Publish the fixed AM registry exclusively; root override is for tests.
Only sealed small experiment evidence and the two model seal controls are
copied. Every bulk payload is nevertheless verified before and after copying.
An interrupted registry is preserved and cannot be overwritten or resumed.
"""
root = Path(root)
require(root.is_absolute() and root.is_dir() and not root.is_symlink(), 'Invalid project root')
root = root.resolve(strict=True)
model, experiment, registry = (root / part for part in
(MODEL_RELATIVE, EXPERIMENT_RELATIVE, REGISTRY_RELATIVE))
require(not registry.exists() and not registry.is_symlink(), 'Registry already exists; no overwrite/resume')
helper_path = root / HELPER_RELATIVE if helper_path is None else Path(helper_path)
helper = load_helpers(helper_path)
for path in (model, experiment, registry.parent):
current = root
for part in path.relative_to(root).parts:
current = current / part
require(not current.is_symlink(), 'Symlink in publication root path: ' + str(current))
verified = {'model': helper.verify_seal(model), 'experiment': helper.verify_seal(experiment)}
experiment_rows = {row['path']: row for row in verified['experiment']['files']}
status, reference, integrity, stages = _status_evidence(helper, experiment, experiment_rows, model)
plans, omitted = [], []
for row in verified['experiment']['files'] + helper._control_rows(experiment, verified['experiment']):
if PurePosixPath(row['path']).suffix.lower() in EVIDENCE_SUFFIXES and row['bytes'] <= MAX_EVIDENCE_BYTES:
plans.append((experiment, row, 'evidence/' + row['path']))
else:
omitted.append(dict(path=row['path'], bytes=row['bytes'], sha256=row['sha256'],
reason='exceeds_4_MiB' if row['bytes'] > MAX_EVIDENCE_BYTES else 'bulk_or_non_evidence_extension'))
for row in helper._control_rows(model, verified['model']):
require(row['bytes'] <= MAX_EVIDENCE_BYTES, 'Model seal control exceeds small-evidence limit')
plans.append((model, row, 'model/' + row['path']))
destinations = {target for _, _, target in plans}
required_copies = ['manifest.json', 'checksums.sha256', *REQUIRED_EVIDENCE]
if reference is not None:
required_copies.append(REFERENCE_EVIDENCE)
for relative in required_copies:
require('evidence/' + relative in destinations, 'Required experiment evidence cannot be copied: ' + relative)
require(len(destinations) == len(plans), 'Duplicate planned evidence destination')
registry.parent.mkdir(parents=True, exist_ok=True)
registry.mkdir(exist_ok=False)
try:
records = []
for source_root, row, relative in plans:
source = helper._regular_file(source_root.resolve(), row['path'])
_copy(helper, source, registry / relative, row['bytes'], row['sha256'])
records.append(dict(source=source.relative_to(root).as_posix(), copy=relative,
bytes=row['bytes'], sha256=row['sha256']))
after = {'model': helper.verify_seal(model), 'experiment': helper.verify_seal(experiment)}
require(after == verified, 'A sealed tree changed during publication')
require(helper._digest(helper_path) == HELPER_SHA256, 'Frozen helper changed during publication')
proof = {name: {key: value for key, value in report.items() if key != 'files'}
for name, report in verified.items()}
_write_small_json(helper, registry / 'seal-verification.json', proof)
_write_small_json(helper, registry / 'evidence-index.json', records)
files = [dict(path=path.relative_to(registry).as_posix(), bytes=path.stat().st_size,
sha256=helper._digest(path)) for path in sorted(registry.rglob('*')) if path.is_file()]
receipt = dict(publication_complete=True, registry=registry.relative_to(root).as_posix(),
project_root=str(root), declared_audit_status=status['status'],
declared_gate_results={key: status[key] for key in GATE_FIELDS},
declared_audit_counts={key: status[key] for key in COUNT_FIELDS},
declared_diagnostic_scope='Exact zero-overlay D18 identity and fixed whole-brain static25 interface; no training or associative learning',
declared_stage_coverage=stages, declared_scientific_summary_scope=SCIENCE_SCOPE,
static_effect_size_gate_defined=False,
scientific_summary_sha256=status['scientific_summary_sha256'],
frozen_reference_report=dict(present=reference is not None,
path='evidence/' + REFERENCE_EVIDENCE if reference is not None else None,
sha256=experiment_rows[REFERENCE_EVIDENCE]['sha256'] if reference is not None else None,
declared_pass_all=reference['pass_all'] if reference is not None else None,
reported_trial_count=len(reference['trials']) if reference is not None else None,
missing_is_not_a_success_or_fabricated_trial_failure=True),
frozen_integrity_report=dict(path='evidence/regression/old-integrity-after.json',
sha256=experiment_rows['regression/old-integrity-after.json']['sha256'],
declared_pass_by_root={name: row['pass'] for name, row in integrity.items()}),
frozen_helper_source=str(helper_path), frozen_helper_sha256=HELPER_SHA256,
publisher_sha256=helper._digest(Path(__file__)), helper_globals_redirected=False,
model_manifest_sha256=verified['model']['manifest_sha256'],
experiment_manifest_sha256=verified['experiment']['manifest_sha256'],
source_trees_verified_before_and_after_copy=True,
copied_evidence_files=len(records), copied_evidence_bytes=sum(row['bytes'] for row in records),
files=files, experiment_payload_not_copied=omitted,
model_payload_not_duplicated=True, model_payload_remains_at=MODEL_RELATIVE.as_posix(),
bulk_remains_at=EXPERIMENT_RELATIVE.as_posix(), complete_bulk_backup=False,
scientific_gate_executed_by_publisher=False, scientific_gate_certified_by_publication=False,
publication_success_is_not_scientific_pass=True, candidate_generated=False,
learning_achieved=False, full_scientific_goal_achieved=False,
old_heldout_dynamic_results_parsed=False, existing_files_modified=False,
receipt_self_hash='Not self-referential; returned by publish() and printed by main()',
limits=dict(max_selected_evidence_file_bytes=MAX_EVIDENCE_BYTES,
selected_suffixes=sorted(EVIDENCE_SUFFIXES)))
receipt_path = registry / 'publication-receipt.json'
_write_small_json(helper, receipt_path, receipt)
return dict(publication_complete=True, registry=str(registry), declared_audit_status=status['status'],
copied_evidence_files=len(records), receipt_sha256=helper._digest(receipt_path),
scientific_gate_executed_by_publisher=False, publication_success_is_not_scientific_pass=True)
except Exception as error:
failure = registry / 'publication-failure.json'
if not failure.exists():
_write_small_json(helper, failure, dict(publication_complete=False, type=type(error).__name__,
message=str(error), partial_outputs_preserved=True, existing_files_modified=False))
raise
def main():
print(json.dumps(publish(), ensure_ascii=False, allow_nan=False), flush=True)
if __name__ == '__main__':
main()