Digital Drosophila Archive
560 lines
35 KiB
Python
560 lines
35 KiB
Python
"""AP: process-isolated delta interface, 1 imported + 11 new neural trials.
|
|
|
|
Old AO remains sealed FAIL. This coordinator never calls E in its process and
|
|
does not mutate any frozen module globals. Neural and update workers use true
|
|
exec subprocesses. Source, plan and sampled input authority precede execution.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
import numpy as np
|
|
|
|
MODEL = Path(__file__).resolve().parent
|
|
LAB = MODEL.parent / 'olfactory-conditioning-lab-v1'
|
|
sys.path.insert(0, str(LAB))
|
|
import conditioning_preflight_prepare as OLD
|
|
import conditioning_preflight_workflow as AOW
|
|
import engineering_prepare as P
|
|
import engineering_workflow as ENV
|
|
import engineering_compare as COMP
|
|
import conditioning_preflight_compare as Q
|
|
import runtime_light as R
|
|
import runtime_continuation as C
|
|
|
|
ROOT = Path('/data/digital-drosophila')
|
|
EXP = ROOT / 'experiments/phase4ap-conditioning-interface-v2'
|
|
AO = ROOT / 'experiments/phase4ao-conditioning-interface'
|
|
DIAG = ROOT / 'experiments/phase4ao-postprocess-isolation'
|
|
FIRST = 'D1--reference--epoch-000'
|
|
ANCHORS = {
|
|
str(AO / 'protocol/preregistered.json'): '9e2ed05879ce279da7dc0b4cac09a44d67665c230b56db94e2ca39c645bdc250',
|
|
str(AO / 'manifest.json'): 'dcfb24338a39ce0bb4d0bf5fe3c92a1907fb63536b4c57856c2f40203996f979',
|
|
str(AO / 'audit/final-status.json'): 'f8fff2cb7d4eb0abfc9653c989781c0310b6ad7ec42cce4949bae1a15fd793d8',
|
|
str(AO / 'runs' / FIRST / C.MANIFEST): 'e2ad8068bcc8450840ff4c5fb5fbe6a39a1960cd5869b5e16096fb579df28a3a',
|
|
str(AO / 'receipts' / (FIRST + '.authority.json')): '9c86c505d6156ca4ba0496105a7fde0556c0a05bfbd68dec6e7ec7c1865cdb60',
|
|
str(DIAG / 'manifest.json'): '202efff603a05016b3f530df9eb1d3c52e1c4119dbc4372ad7b75c6ca69013a2',
|
|
str(DIAG / 'diagnostic-receipt.json'): 'cefe84ac1642285b2c815a1f2a03677be064dd38125024f638eda4892cf477b3',
|
|
}
|
|
AO_REG = ANCHORS[str(AO / 'protocol/preregistered.json')]
|
|
BUDGET = dict(output_budget_bytes=4 * 1024**3, minimum_free_reserve_bytes=10 * 1024**3,
|
|
child_timeout_seconds=1800)
|
|
require, read, save, digest = P.require, P.read, P.save, P.digest
|
|
|
|
|
|
def same(a, b): return OLD.same(a, b)
|
|
def npz(p): return AOW.npz(p)
|
|
|
|
|
|
def check_pins(pins):
|
|
OLD.verify_pins(pins)
|
|
|
|
|
|
def helper():
|
|
return P.load_module('phase4ap_seal_helper', ROOT / P.AI_HELPER_REL, P.AI_HELPER_SHA)
|
|
|
|
|
|
def previous_integrity():
|
|
check_pins(ANCHORS)
|
|
h = helper()
|
|
result = dict(AO=h.verify_seal(AO), diagnostic=h.verify_seal(DIAG), AN=h.verify_seal(OLD.AN))
|
|
old = P.read_registration(OLD.AN_REG)
|
|
result['AM'] = P.verify_authority(old['anchors'])
|
|
result['old_roots'] = P.verify_old_integrity(old)
|
|
require(read(AO / 'audit/final-status.json')['status'] == 'FAIL-POSTPROCESS-MODULE-IDENTITY', 'AO failure must remain unchanged')
|
|
return result
|
|
|
|
|
|
def derive_spec(original, budget_root, cap):
|
|
"""The only new-case differences from AO are administrative budget fields."""
|
|
require(type(budget_root) is str and Path(budget_root).is_absolute() and type(cap) is int and cap > 0,
|
|
'Explicit new absolute budget/cap required')
|
|
require(budget_root != original['budget_root'], 'New experiment cannot reuse the stopped budget root')
|
|
spec = copy.deepcopy(original)
|
|
spec.update(budget_root=budget_root, output_budget_bytes=cap)
|
|
changes = {k: dict(before=original[k], after=spec[k]) for k in original if not same(original[k], spec[k])}
|
|
require(set(changes) <= {'budget_root', 'output_budget_bytes'}, 'Frozen neural/input spec changed')
|
|
return spec, dict(changes=changes, all_other_fields_exact=True, no_neural_parameter_change=True)
|
|
|
|
|
|
def source_inventory():
|
|
names = ('coordinator.py', 'test_coordinator.py', 'isolated_workers.py', 'test_isolated_workers.py', 'PROTOCOL.md')
|
|
return {str(MODEL / n): digest(MODEL / n) for n in names}
|
|
|
|
|
|
def prepare():
|
|
require(sys.dont_write_bytecode and EXP.is_dir() and not any(EXP.iterdir()), 'Use -B and the new empty AP directory')
|
|
before = previous_integrity()
|
|
old = OLD.read_registration(AO_REG)
|
|
generator = AOW.generated_inputs(AO_REG)
|
|
checkpoint = C.load_checkpoint(AO / 'runs' / FIRST, ANCHORS[str(AO / 'runs' / FIRST / C.MANIFEST)])
|
|
require(checkpoint['fresh'] and checkpoint['steps'] == 1200, 'Imported trajectory is not complete/fresh')
|
|
source = source_inventory()
|
|
commit = subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD']).decode().strip()
|
|
require(len(commit) == 40, 'Git code commit missing')
|
|
for path, sha in source.items():
|
|
blob = subprocess.check_output(['git', '-C', str(ROOT), 'show', commit + ':' + Path(path).relative_to(ROOT).as_posix()])
|
|
require(hashlib.sha256(blob).hexdigest() == sha, 'AP sources must be committed before registration')
|
|
pins = dict(ANCHORS)
|
|
for field in ('source_sha256', 'input_sha256', 'prepared_file_sha256'):
|
|
pins.update(old[field])
|
|
pins.update(generator['inputs_sha256'])
|
|
pins[str(AO / 'receipts/input-generation.json')] = digest(AO / 'receipts/input-generation.json')
|
|
pins[str(AO / 'receipts/input-generation.process.json')] = digest(AO / 'receipts/input-generation.process.json')
|
|
for p in (DIAG / 'postprocess').iterdir():
|
|
pins[str(p)] = digest(p)
|
|
check_pins(pins)
|
|
for n in ('protocol/source-snapshot', 'cases', 'runs', 'updates', 'jobs', 'receipts', 'logs', 'audit', 'regression'):
|
|
(EXP / n).mkdir(parents=True)
|
|
for path in source:
|
|
shutil.copyfile(path, EXP / 'protocol/source-snapshot' / Path(path).name)
|
|
save(EXP / 'regression/integrity-before.json', before)
|
|
slots = []
|
|
for row in old['plan']['calls']:
|
|
row = copy.deepcopy(row)
|
|
imported = row['row_id'] == FIRST
|
|
original_case = Path(old['cases'][row['row_id']])
|
|
if imported:
|
|
case, run = original_case, AO / 'runs' / FIRST
|
|
else:
|
|
case, run = EXP / 'cases' / row['row_id'], EXP / 'runs' / row['row_id']
|
|
case.mkdir()
|
|
spec, change = derive_spec(read(original_case / 'spec.json'), str(EXP), BUDGET['output_budget_bytes'])
|
|
R.validate_spec(spec, case)
|
|
save(case / 'spec.json', spec)
|
|
save(case / 'derivation.json', dict(original_spec=str(original_case / 'spec.json'),
|
|
original_spec_sha256=digest(original_case / 'spec.json'), **change))
|
|
with (original_case / 'requests.npz').open('rb') as src, (case / 'requests.npz').open('xb') as dst:
|
|
shutil.copyfileobj(src, dst)
|
|
require(digest(case/'requests.npz') == digest(original_case/'requests.npz'), 'Saved requests copy differs')
|
|
row.update(origin='imported_sealed_AO_neural_trajectory' if imported else 'new_AP_neural_execution',
|
|
case_dir=str(case), neural_dir=str(run), original_case_dir=str(original_case))
|
|
slots.append(row)
|
|
prepared = {str(p): digest(p) for sub in ('protocol/source-snapshot', 'cases', 'regression')
|
|
for p in (EXP / sub).rglob('*') if p.is_file()}
|
|
reg = dict(schema='phase4ap-process-isolated-interface-v1', anchors=ANCHORS, source_sha256=source,
|
|
input_sha256=pins, prepared_file_sha256=prepared, source_code_commit=commit, budget=BUDGET,
|
|
slots=slots, selected_epochs=old['plan']['selected_epochs'], compartment=old['plan']['compartment_verbatim'],
|
|
dan_type=old['plan']['dan_type'], initial_overlay=old['initial_overlay'], authority_template=str(AO / 'receipts' / (FIRST + '.authority.json')),
|
|
imported_slot=FIRST, imported_checkpoint_sha256=ANCHORS[str(AO / 'runs' / FIRST / C.MANIFEST)],
|
|
covered_slots=12, new_neural_calls=11, covered_ticks=14400, new_neural_ticks=13200, new_sampler_calls=0,
|
|
original_failed_workflow_reclassified=False, inherited_neural_checkpoint_complete=True,
|
|
original_AO_process_complete=False, first_postprocess='fresh exec CPU E-only process, compare to sealed independent diagnostic',
|
|
neural_reset='fresh each epoch; carry delta only', update_trace_backend='torch', within_trial_updates=False,
|
|
scheduled_D='artificial scalar not multiplied by actual DAN acceptance',
|
|
success='six exact reference/light pairs, six exact zero-weight cross-arm pairs, all real v2 updates recomputed, D0/frozen exact positive-zero, nonzero D1 coverage and epoch 2 exact installation',
|
|
no_nonzero_coverage_status='INCOMPLETE-NO-NONZERO-UPDATE-COVERAGE',
|
|
failure_policy='stop and preserve partials; no automatic child retries, seed search or parameter rescue',
|
|
full_conditioning=False, retention_transition_supported=False, learning_success_claimed=False,
|
|
light_intermediate_states_observed=False, future_hashes_fabricated=False)
|
|
check_pins({**pins, **source, **prepared})
|
|
save(EXP / 'protocol/preregistered.json', reg)
|
|
print('REGISTRATION_SHA256', digest(EXP / 'protocol/preregistered.json'), flush=True)
|
|
|
|
|
|
def registration(sha):
|
|
require(P.sha(sha) and digest(EXP / 'protocol/preregistered.json') == sha, 'Registration differs from external SHA')
|
|
reg = read(EXP / 'protocol/preregistered.json')
|
|
require(reg['schema'] == 'phase4ap-process-isolated-interface-v1' and reg['anchors'] == ANCHORS and reg['budget'] == BUDGET,
|
|
'Wrong AP registration')
|
|
require(len(reg['slots']) == 12 and sum(r['origin'] == 'new_AP_neural_execution' for r in reg['slots']) == 11,
|
|
'AP must cover exactly one import plus eleven new neural calls')
|
|
check_pins({**reg['input_sha256'], **reg['source_sha256'], **reg['prepared_file_sha256']})
|
|
return reg
|
|
|
|
|
|
def slot(reg, name):
|
|
matches = [r for r in reg['slots'] if r['row_id'] == name]
|
|
require(len(matches) == 1, 'Unknown/nonunique slot')
|
|
return matches[0]
|
|
|
|
|
|
def paths(row):
|
|
name = row['row_id']
|
|
require(type(name) is str and name and all(c.isalnum() or c in '-_' for c in name), 'Unsafe slot identity')
|
|
return dict(neural=Path(row['neural_dir']), case=Path(row['case_dir']), update=EXP / 'updates' / name,
|
|
authority=EXP / 'jobs' / (name + '.authority.json'), record=EXP / 'receipts' / (name + '.slot.json'))
|
|
|
|
|
|
def admission(reg):
|
|
require(not any((EXP / x).exists() for x in ('audit/stopped.json', 'audit/execution-complete.json', 'manifest.json')),
|
|
'Stopped/completed AP must not be resumed automatically')
|
|
all_paths = list(EXP.rglob('*'))
|
|
require(not any(p.is_symlink() for p in all_paths), 'Linked output is forbidden')
|
|
used = sum(p.stat().st_size for p in all_paths if p.is_file())
|
|
require(used <= reg['budget']['output_budget_bytes'] and shutil.disk_usage(EXP).free >=
|
|
reg['budget']['minimum_free_reserve_bytes'] + max(0, reg['budget']['output_budget_bytes'] - used), 'AP resource cap exceeded')
|
|
|
|
|
|
def before_overlay(reg, reg_sha, row):
|
|
previous = row['previous_delta_row_id']
|
|
if previous is None: return reg['initial_overlay']
|
|
parent = slot(reg, previous)
|
|
require(parent['arm'] == row['arm'] and parent['diagnostic_mode'] == row['diagnostic_mode'] and
|
|
parent['input_epoch_index'] + 1 == row['input_epoch_index'], 'Invalid delta ancestry')
|
|
verify_slot(reg, reg_sha, parent)
|
|
directory = paths(parent)['update']
|
|
return dict(overlay_file=str(directory / 'delta-after.npz'), overlay_manifest=str(directory / 'overlay-manifest.json'))
|
|
|
|
|
|
def authority_for(reg, row, checkpoint_sha, overlay):
|
|
value = copy.deepcopy(read(reg['authority_template']))
|
|
p = paths(row)
|
|
value.update(epoch_id=row['epoch_id'], diagnostic_mode=row['diagnostic_mode'], checkpoint_dir=str(p['neural']),
|
|
checkpoint_manifest_sha256=checkpoint_sha)
|
|
value['command'] = dict(steps=row['steps'], compartment=reg['compartment'],
|
|
us_tick=reg['selected_epochs'][row['input_epoch_index']]['us']['tick'], d_command=row['d_command'],
|
|
frozen=row['frozen'], trace_backend=reg['update_trace_backend'])
|
|
changed = dict(spec=p['case'] / 'spec.json', requests=p['case'] / 'requests.npz', state=p['neural'] / 'state.npz',
|
|
overlay_before=overlay['overlay_file'], overlay_manifest_before=overlay['overlay_manifest'])
|
|
for role, path in changed.items():
|
|
value['files'][role] = dict(path=str(path), sha256=digest(path))
|
|
return value
|
|
|
|
|
|
def job_for(reg, reg_sha, row, kind, overlay=None):
|
|
require(kind in ('brain', 'update'), 'Unknown child kind')
|
|
p = paths(row); name = row['row_id'] + '.' + kind
|
|
worker = MODEL / 'isolated_workers.py'
|
|
pins = {**reg['input_sha256'], **reg['source_sha256'], **reg['prepared_file_sha256']}
|
|
base = dict(schema='phase4ap-isolated-worker-job-v1', job_id=name, kind=kind,
|
|
registration_sha256=reg_sha, worker_sha256=reg['source_sha256'][str(worker)],
|
|
output_dir=str(p['neural'] if kind == 'brain' else p['update']),
|
|
receipt_file=str(EXP / 'receipts' / (name + '.worker.json')))
|
|
if kind == 'brain':
|
|
require(row['origin'] == 'new_AP_neural_execution', 'Imported neural trajectory must not be executed again')
|
|
for path in (p['case'] / 'spec.json', p['case'] / 'requests.npz', overlay['overlay_file'], overlay['overlay_manifest']):
|
|
pins[str(path)] = digest(path)
|
|
base.update(case_dir=str(p['case']), build_root=str(EXP / 'extension-build'), **overlay,
|
|
diagnostic_mode=row['diagnostic_mode'], admission=dict(gpu_uuid=ENV.GPU,
|
|
comfy_queue_urls=[f'http://192.168.10.51:{port}/queue' for port in (8188, 8189, 8190)]),
|
|
sources={role: dict(path=str(LAB / (role + '.py')), sha256=reg['input_sha256'][str(LAB / (role + '.py'))])
|
|
for role in ('runtime_light', 'runtime_continuation')})
|
|
else:
|
|
authority = read(p['authority'])
|
|
pins[str(p['authority'])] = digest(p['authority'])
|
|
pins.update({v['path']: v['sha256'] for v in authority['files'].values()})
|
|
pins.update(authority['frozen_inputs'])
|
|
pins[str(p['neural'] / C.MANIFEST)] = authority['checkpoint_manifest_sha256']
|
|
base.update(authority_file=str(p['authority']), authority_sha256=digest(p['authority']),
|
|
sources={'conditioning_epoch': dict(path=str(LAB / 'conditioning_epoch.py'),
|
|
sha256=reg['input_sha256'][str(LAB / 'conditioning_epoch.py')])})
|
|
base['input_sha256'] = pins
|
|
return base
|
|
|
|
|
|
def process_locations(row, kind):
|
|
name = row['row_id'] + '.' + kind
|
|
return dict(job=EXP / 'jobs' / (name + '.json'), worker=EXP / 'receipts' / (name + '.worker.json'),
|
|
process=EXP / 'receipts' / (name + '.process.json'), stdout=EXP / 'logs' / (name + '.stdout.log'),
|
|
stderr=EXP / 'logs' / (name + '.stderr.log'))
|
|
|
|
|
|
def launch(reg, reg_sha, row, kind, job):
|
|
loc = process_locations(row, kind)
|
|
require(not any(p.exists() for p in loc.values()), 'Partial worker cannot be retried')
|
|
admission(reg)
|
|
save(loc['job'], job)
|
|
command = [sys.executable, '-B', str(MODEL / 'isolated_workers.py'), kind, '--job', str(loc['job']),
|
|
'--job-sha256', digest(loc['job'])]
|
|
start = time.perf_counter()
|
|
env = dict(os.environ)
|
|
if kind == 'update': env['CUDA_VISIBLE_DEVICES'] = ''
|
|
timed_out = False
|
|
with loc['stdout'].open('xb') as out, loc['stderr'].open('xb') as err:
|
|
try:
|
|
result = subprocess.run(command, cwd=MODEL, env=env, stdout=out, stderr=err,
|
|
timeout=reg['budget']['child_timeout_seconds'])
|
|
exit_code = result.returncode
|
|
except subprocess.TimeoutExpired:
|
|
timed_out, exit_code = True, None
|
|
complete = not timed_out and exit_code == 0 and loc['worker'].is_file()
|
|
receipt = dict(complete=complete, registration_sha256=reg_sha, row_id=row['row_id'], kind=kind,
|
|
exit_code=exit_code, timed_out=timed_out, argv=command, job_sha256=digest(loc['job']),
|
|
worker_receipt_sha256=digest(loc['worker']) if loc['worker'].is_file() else None, stdout_sha256=digest(loc['stdout']),
|
|
stderr_sha256=digest(loc['stderr']), wall_seconds=time.perf_counter()-start)
|
|
save(loc['process'], receipt)
|
|
require(complete, 'Isolated child failed or timed out; preserve partials')
|
|
return verify_process(reg, reg_sha, row, kind, job)
|
|
|
|
|
|
def verify_process(reg, reg_sha, row, kind, expected_job):
|
|
loc = process_locations(row, kind)
|
|
process = read(loc['process'])
|
|
require(process['complete'] is True and process['exit_code'] == 0 and process['registration_sha256'] == reg_sha and
|
|
process['row_id'] == row['row_id'] and process['kind'] == kind, 'Missing actual complete child process')
|
|
for k, f in (('job_sha256','job'), ('worker_receipt_sha256','worker'), ('stdout_sha256','stdout'), ('stderr_sha256','stderr')):
|
|
require(process[k] == digest(loc[f]), 'Child evidence changed: ' + k)
|
|
require(same(read(loc['job']), expected_job), 'Job differs from registered slot and actual parent')
|
|
worker = read(loc['worker'])
|
|
require(worker['schema'] == 'phase4ap-isolated-worker-receipt-v1' and worker['complete'] is True and
|
|
worker['job_sha256'] == process['job_sha256'] and worker['registration_sha256'] == reg_sha and
|
|
worker['job_id'] == expected_job['job_id'] and worker['kind'] == kind and
|
|
worker['worker_sha256'] == expected_job['worker_sha256'] and
|
|
worker['output_dir'] == expected_job['output_dir'] and worker['receipt_file'] == str(loc['worker']) and
|
|
worker['source_files'] == expected_job['sources'], 'Worker is not bound to this job')
|
|
attempt = Path(str(loc['worker']) + '.attempt.json')
|
|
require(digest(attempt) == worker['attempt_sha256'], 'Worker attempt record changed')
|
|
expected_pins = {**expected_job['input_sha256'], str(loc['job']): process['job_sha256'],
|
|
str(MODEL/'isolated_workers.py'): expected_job['worker_sha256']}
|
|
require(worker['verified_input_sha256'] == expected_pins, 'Worker did not verify the complete job inputs')
|
|
output = Path(expected_job['output_dir'])
|
|
descendants = list(output.rglob('*'))
|
|
require(not any(p.is_symlink() for p in descendants), 'Linked worker output forbidden')
|
|
actual_outputs = {p.relative_to(output).as_posix(): digest(p) for p in descendants if p.is_file()}
|
|
require(actual_outputs and actual_outputs == worker['output_files_sha256'], 'Worker output inventory changed')
|
|
if kind == 'brain':
|
|
require(worker['result']['complete_fresh_trial'] is True and
|
|
worker['result']['steps'] == row['steps'] and
|
|
worker['result']['checkpoint_manifest_sha256'] == digest(output/C.MANIFEST), 'Brain worker checkpoint mismatch')
|
|
ENV.verify_environment(worker['environment'], read(output/'summary.json')['apl_component']['environment'])
|
|
else:
|
|
result = worker['result']; chain = read(output/'chain-receipt.json')
|
|
require(result['brain_executed'] is False and result['weights_installed'] is False and
|
|
result['chain_receipt_sha256'] == digest(output/'chain-receipt.json') and
|
|
chain['authority_sha256'] == expected_job['authority_sha256'], 'Update worker chain mismatch')
|
|
for field in ('source_neural_checkpoint_manifest_sha256', 'before_overlay_file_sha256',
|
|
'after_overlay_file_sha256', 'before_overlay_manifest_sha256', 'after_overlay_manifest_sha256'):
|
|
require(result[field] == chain[field], 'Update worker result differs: ' + field)
|
|
require(chain['after_overlay_file_sha256'] == digest(output/'delta-after.npz') and
|
|
chain['after_overlay_manifest_sha256'] == digest(output/'overlay-manifest.json'), 'Update after-overlay payload differs')
|
|
require(worker['environment']['CUDA_hidden'] is True and worker['environment']['CUDA_initialized'] is False,
|
|
'Update worker was not CPU-isolated')
|
|
check_pins(expected_job['input_sha256'])
|
|
return worker
|
|
|
|
|
|
def verify_slot(reg, reg_sha, row):
|
|
p = paths(row); record = read(p['record'])
|
|
require(record['complete'] is True and record['registration_sha256'] == reg_sha and same(record['row'], row), 'Slot record mismatch')
|
|
for name, sha in record['files_sha256'].items(): require(digest(name) == sha, 'Slot output changed')
|
|
checkpoint = C.load_checkpoint(p['neural'], record['checkpoint_manifest_sha256'])
|
|
require(checkpoint['fresh'] and checkpoint['steps'] == row['steps'], 'Slot is not a fresh neural epoch')
|
|
if row['origin'] == 'imported_sealed_AO_neural_trajectory':
|
|
require(record['checkpoint_manifest_sha256'] == reg['imported_checkpoint_sha256'] and
|
|
record['new_neural_execution'] is False and row['row_id'] == FIRST, 'Imported provenance mismatch')
|
|
else:
|
|
require(record['new_neural_execution'] is True, 'New runtime marked as import')
|
|
overlay = before_overlay(reg, reg_sha, row)
|
|
require(record['before_overlay_sha256'] == digest(overlay['overlay_file']), 'Slot parent delta changed')
|
|
if row['row_id'] != FIRST:
|
|
worker = verify_process(reg, reg_sha, row, 'brain', job_for(reg, reg_sha, row, 'brain', overlay))
|
|
require(worker['result']['checkpoint_manifest_sha256'] == record['checkpoint_manifest_sha256'], 'Slot checkpoint differs from actual brain')
|
|
authority = authority_for(reg, row, record['checkpoint_manifest_sha256'], overlay)
|
|
require(same(authority, read(p['authority'])), 'Slot authority differs from registered trial')
|
|
verify_process(reg, reg_sha, row, 'update', job_for(reg, reg_sha, row, 'update'))
|
|
chain = read(p['update']/'chain-receipt.json')
|
|
require(chain['authority_sha256'] == digest(p['authority']) and
|
|
chain['source_neural_checkpoint_manifest_sha256'] == record['checkpoint_manifest_sha256'] and
|
|
chain['before_overlay_file_sha256'] == digest(overlay['overlay_file']), 'Slot update ancestry differs')
|
|
verify_delta_payload(overlay, p['update'], read(p['update']/'update-receipt.json'))
|
|
return record
|
|
|
|
|
|
def verify_delta_payload(overlay, directory, update):
|
|
_, after, _, _ = R.load_overlay(directory/'delta-after.npz', directory/'overlay-manifest.json')
|
|
binding = Q.audit_delta_binding(npz(overlay['overlay_file'])['plastic_delta_mv'], after['plastic_delta_mv'],
|
|
after['plastic_delta_mv'], update_receipt=update, parent_snapshot_sha256=digest(overlay['overlay_file']),
|
|
child_manifest=read(directory/'overlay-manifest.json'))
|
|
require(binding['pass_all'], 'Actual delta binding failed')
|
|
return binding, after['plastic_delta_mv']
|
|
|
|
|
|
def verify_imported_delta(delta, update):
|
|
prior = npz(DIAG/'postprocess/delta-after.npz')['plastic_delta_mv']
|
|
require(COMP.bit_equal(prior, delta) and same(update, read(DIAG/'postprocess/update-receipt.json')),
|
|
'Imported trajectory postprocessing differs from sealed diagnostic')
|
|
|
|
|
|
def perform_slot(reg, reg_sha, row):
|
|
p = paths(row)
|
|
require(not p['record'].exists() and not p['authority'].exists() and not p['update'].exists(), 'Partial slot cannot be retried')
|
|
overlay = before_overlay(reg, reg_sha, row)
|
|
started = time.perf_counter()
|
|
if row['origin'] == 'imported_sealed_AO_neural_trajectory':
|
|
checkpoint_sha = reg['imported_checkpoint_sha256']
|
|
C.load_checkpoint(p['neural'], checkpoint_sha)
|
|
else:
|
|
worker = launch(reg, reg_sha, row, 'brain', job_for(reg, reg_sha, row, 'brain', overlay))
|
|
checkpoint_sha = worker['result']['checkpoint_manifest_sha256']
|
|
C.load_checkpoint(p['neural'], checkpoint_sha)
|
|
ENV.verify_environment(worker['environment'], read(p['neural'] / 'summary.json')['apl_component']['environment'])
|
|
authority = authority_for(reg, row, checkpoint_sha, overlay)
|
|
save(p['authority'], authority)
|
|
launch(reg, reg_sha, row, 'update', job_for(reg, reg_sha, row, 'update'))
|
|
update = read(p['update'] / 'update-receipt.json')
|
|
AOW.replay_update(authority, update) # CPU-only original U verification, never E in this process.
|
|
binding, delta = verify_delta_payload(overlay, p['update'], update)
|
|
if row['row_id'] == FIRST:
|
|
verify_imported_delta(delta, update)
|
|
save(EXP / 'receipts' / (row['row_id'] + '.binding.json'), binding)
|
|
files = [p['authority'], *p['update'].iterdir(), EXP / 'receipts' / (row['row_id'] + '.binding.json')]
|
|
for kind in (('update',) if row['row_id'] == FIRST else ('brain', 'update')):
|
|
files.extend(process_locations(row, kind).values())
|
|
files.append(Path(str(process_locations(row, kind)['worker']) + '.attempt.json'))
|
|
registration(reg_sha)
|
|
save(p['record'], dict(complete=True, registration_sha256=reg_sha, row=row,
|
|
new_neural_execution=row['row_id'] != FIRST, original_AO_workflow_pass_claimed=False,
|
|
checkpoint_manifest_sha256=checkpoint_sha, before_overlay_sha256=digest(overlay['overlay_file']),
|
|
files_sha256={str(f): digest(f) for f in files}, wall_seconds=time.perf_counter()-started,
|
|
update_recomputed=True, learning_success_claimed=False))
|
|
verify_slot(reg, reg_sha, row)
|
|
|
|
|
|
def run(reg_sha):
|
|
reg = registration(reg_sha); admission(reg)
|
|
completed = []
|
|
try:
|
|
for row in reg['slots']:
|
|
admission(reg)
|
|
print('START', row['row_id'], row['origin'], flush=True)
|
|
perform_slot(reg, reg_sha, row)
|
|
completed.append(row['row_id'])
|
|
print('COMPLETE', row['row_id'], flush=True)
|
|
save(EXP / 'audit/execution-complete.json', dict(registration_sha256=reg_sha, complete=True,
|
|
completed=completed, covered_slots=12, new_brain_calls=11, new_ticks=13200, covered_ticks=14400,
|
|
imported_trajectory_count=1, new_sampler_calls=0, learning_success_claimed=False))
|
|
except Exception as error:
|
|
save(EXP / 'audit/stopped.json', dict(complete=False, registration_sha256=reg_sha,
|
|
error_type=type(error).__name__, error=str(error), traceback=traceback.format_exc(),
|
|
completed_slots=completed, no_automatic_retry=True, partials_preserved=True))
|
|
raise
|
|
|
|
|
|
def expected_weight(reg, reg_sha, row):
|
|
overlay = before_overlay(reg, reg_sha, row)
|
|
h, plan, _, _ = R.load_overlay(overlay['overlay_file'], overlay['overlay_manifest'])
|
|
spec = read(paths(row)['case'] / 'spec.json'); arrays = {}
|
|
try:
|
|
for n in ('nodes', 'weights', 'edge_id', 'src'):
|
|
arrays[n] = np.load(Path(spec['graph_dir']) / (n+'.npy'), mmap_mode='r', allow_pickle=False)
|
|
require(np.array_equal(arrays['nodes'], np.arange(166700, dtype=np.int32)), 'Not canonical fullgraph nodes')
|
|
mask = np.isin(arrays['src'], np.load(Path(spec['apl_bundle_dir']) / 'apl_globals.npy', allow_pickle=False))
|
|
require(int(mask.sum()) == 5845, 'Fast APL mask changed')
|
|
mapping = h.map_installation(arrays['weights'], arrays['edge_id'], mask, plan)
|
|
require(mapping['missing_candidate_count'] == 0, 'Incomplete mask mapping')
|
|
return h.expected_hashes(arrays['weights'], mask, mapping)[1]
|
|
finally:
|
|
for a in arrays.values(): a._mmap.close()
|
|
|
|
|
|
def analyze(reg_sha):
|
|
reg = registration(reg_sha)
|
|
complete = read(EXP / 'audit/execution-complete.json')
|
|
require(not (EXP/'audit/stopped.json').exists() and complete['complete'] and complete['registration_sha256'] == reg_sha and
|
|
complete['completed'] == [r['row_id'] for r in reg['slots']], 'Complete AP coverage is missing')
|
|
records, deltas, summaries = {}, {}, []
|
|
for row in reg['slots']:
|
|
p=paths(row); records[row['row_id']] = verify_slot(reg, reg_sha, row)
|
|
overlay = before_overlay(reg, reg_sha, row)
|
|
if row['row_id'] != FIRST:
|
|
worker = verify_process(reg, reg_sha, row, 'brain', job_for(reg, reg_sha, row, 'brain', overlay))
|
|
require(worker['result']['checkpoint_manifest_sha256'] == records[row['row_id']]['checkpoint_manifest_sha256'], 'Brain worker checkpoint mismatch')
|
|
authority = authority_for(reg, row, records[row['row_id']]['checkpoint_manifest_sha256'], overlay)
|
|
require(same(authority, read(p['authority'])), 'Actual command/input authority differs from registered row')
|
|
verify_process(reg, reg_sha, row, 'update', job_for(reg, reg_sha, row, 'update'))
|
|
update = read(p['update']/'update-receipt.json'); AOW.replay_update(authority, update)
|
|
chain = read(p['update']/'chain-receipt.json')
|
|
require(chain['authority_sha256'] == digest(p['authority']) and chain['source_neural_checkpoint_manifest_sha256'] ==
|
|
records[row['row_id']]['checkpoint_manifest_sha256'], 'Update chain differs from actual checkpoint')
|
|
_, delta = verify_delta_payload(overlay, p['update'], update)
|
|
if row['row_id'] == FIRST: verify_imported_delta(delta, update)
|
|
deltas[row['row_id']]=delta
|
|
if row['arm'] in ('D0','frozen'): require(not np.any(delta.view(np.uint32)), 'Zero update controls are not exact positive-zero')
|
|
loaded=COMP.load_run(p['neural'])
|
|
try:
|
|
us=npz(AO/f"inputs/generated/epoch-{row['input_epoch_index']:03d}/us_requests.npz")
|
|
keys=us['tick'].astype(np.int64)*166700+us['neuron_idx']
|
|
actual=loaded['requests']; selected=actual[np.isin(actual[:,0]*166700+actual[:,1], keys)]
|
|
require(len(selected)==len(keys), 'US delivery accounting incomplete')
|
|
dan=npz(AO/'inputs/static/populations.npz')[reg['dan_type']]
|
|
s=loaded['summary']
|
|
summaries.append(dict(row_id=row['row_id'], origin=row['origin'], spikes=s['total_spikes'],
|
|
active_neurons=int(np.count_nonzero(loaded['state']['counts'])), all_finite=s['all_finite'],
|
|
peak_vram=s['peak_allocated_vram_bytes'], simulation_seconds=s['diagnostic_simulation_seconds'],
|
|
runtime_wall_seconds=s['wall_seconds'], slot_wall_seconds=records[row['row_id']]['wall_seconds'],
|
|
updates=update['counts'], DAN=dict(requested=len(selected),accepted=int(selected[:,3].sum()),
|
|
rejected=int(selected[:,4].sum()), actual_spikes=int(loaded['state']['counts'][dan].sum()),
|
|
scheduled_D=row['d_command'], actual_DAN_gates_update=False)))
|
|
finally: COMP.close_run(loaded)
|
|
pairs=[]; cross=[]
|
|
for arm in ('D1','D0','frozen'):
|
|
for epoch in range(2):
|
|
names=[f'{arm}--{mode}--epoch-{epoch:03d}' for mode in ('reference','light')]
|
|
runs=[COMP.load_run(paths(slot(reg,n))['neural']) for n in names]
|
|
try:
|
|
requests=npz(AO/f'inputs/generated/epoch-{epoch:03d}/requests.npz')
|
|
expected=expected_weight(reg,reg_sha,slot(reg,names[0]))
|
|
report=Q.compare_epoch_pair(*runs,expected_requests=np.column_stack((requests['tick'],requests['neuron_idx'])),
|
|
expected_steps=1200,expected_effective_weight_sha256=expected)
|
|
require(report['pass_all'] and COMP.bit_equal(deltas[names[0]],deltas[names[1]]), 'Reference/light regression failed')
|
|
pairs.append(dict(arm=arm,epoch=epoch,exact_delta_after=True,expected_effective_weight_sha256=expected,comparison=report))
|
|
finally:
|
|
for r in runs:COMP.close_run(r)
|
|
for mode in ('reference','light'):
|
|
for left,right,epoch in (('D0','frozen',0),('D0','frozen',1),('D1','D0',0)):
|
|
runs=[COMP.load_run(paths(slot(reg,f'{a}--{mode}--epoch-{epoch:03d}'))['neural']) for a in (left,right)]
|
|
try:
|
|
result=COMP.compare_fresh(*runs); require(result['numerical_exact'],'Zero-weight cross-arm behavior differs')
|
|
cross.append(dict(left=left,right=right,epoch=epoch,mode=mode,comparison=result))
|
|
finally:
|
|
for r in runs:COMP.close_run(r)
|
|
installations={}
|
|
for mode in ('reference','light'):
|
|
first=f'D1--{mode}--epoch-000'; second=f'D1--{mode}--epoch-001'
|
|
second_row=slot(reg,second); before=before_overlay(reg,reg_sha,second_row)
|
|
audit=read(paths(second_row)['neural']/'overlay-audit.json')
|
|
require(audit['delta_raw_sha256']==R._raw(deltas[first]) and
|
|
records[second]['before_overlay_sha256']==digest(before['overlay_file']), 'Second epoch did not install first delta exactly')
|
|
installations[mode]=dict(exact=True,nonzero_edges=int(np.count_nonzero(deltas[first])))
|
|
coverage=all(s['updates']['actual_bitwise_changed_edges']>0 for s in summaries if s['row_id'].startswith('D1--'))
|
|
return dict(status='PASS-EXACT-PROCESS-ISOLATED-DELTA-INTERFACE-NOT-LEARNING' if coverage else 'INCOMPLETE-NO-NONZERO-UPDATE-COVERAGE',
|
|
registration_sha256=reg_sha, covered_slots=12,new_neural_calls=11,imported_neural_trajectories=1,new_sampler_calls=0,
|
|
new_ticks=13200,covered_ticks=14400,pairs=pairs,cross_arm=cross,rows=summaries,epoch2_installations=installations,
|
|
D0_frozen_exact_zero=True,nonzero_D1_coverage=coverage,original_AO_status='FAIL-POSTPROCESS-MODULE-IDENTITY',
|
|
retention_transition_supported=False,full_conditioning_executed=False,learning_success_claimed=False,learned_candidate=False,
|
|
scope='One replicate, two A epochs, one PAM15 compartment; exact interface coverage only. No A/B behavioral evaluation or learning success.',
|
|
light_intermediate_states_observed=False)
|
|
|
|
|
|
def finalize(reg_sha):
|
|
require(not (EXP/'manifest.json').exists() and not (EXP/'audit/final-status.json').exists(), 'Never overwrite a final or partial seal')
|
|
report=analyze(reg_sha)
|
|
save(EXP/'audit/interface-analysis.json',report)
|
|
after=previous_integrity();save(EXP/'regression/integrity-after.json',after)
|
|
registration(reg_sha)
|
|
final=dict(status=report['status'],registration_sha256=reg_sha,analysis_sha256=digest(EXP/'audit/interface-analysis.json'),
|
|
covered_slots=12,new_neural_calls=11,imported_neural_trajectories=1,new_sampler_calls=0,
|
|
old_integrity_pass=True,old_integrity_roots=len(after['old_roots']),AO_failure_preserved=True,
|
|
source_integrity_pass=True,old_five_reference_trials='Sealed AM evidence retained; not rerun by this interface gate',
|
|
learned_candidate=False,learning_success_claimed=False,full_conditioning_executed=False)
|
|
save(EXP/'audit/final-status.json',final)
|
|
all_paths=sorted(EXP.rglob('*'));require(not any(p.is_symlink() or '__pycache__' in p.parts for p in all_paths),'Unsafe seal payload')
|
|
files=[dict(path=p.relative_to(EXP).as_posix(),bytes=p.stat().st_size,sha256=digest(p)) for p in all_paths if p.is_file()]
|
|
save(EXP/'manifest.json',dict(root=str(EXP),files=files))
|
|
with (EXP/'checksums.sha256').open('x',encoding='utf-8',newline='\n') as out:
|
|
out.writelines(f['sha256']+' '+f['path']+'\n' for f in files)
|
|
out.write(digest(EXP/'manifest.json')+' manifest.json\n')
|
|
helper().verify_seal(EXP)
|
|
print(json.dumps(dict(status=final['status'],manifest_sha256=digest(EXP/'manifest.json'),
|
|
final_sha256=digest(EXP/'audit/final-status.json'))),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
require(sys.dont_write_bytecode,'Use scientific Python -B')
|
|
parser=argparse.ArgumentParser();parser.add_argument('action',choices=('prepare','run','finalize'))
|
|
parser.add_argument('--registration-sha256');args=parser.parse_args()
|
|
if args.action=='prepare':prepare()
|
|
elif args.action=='run':run(args.registration_sha256)
|
|
else:finalize(args.registration_sha256)
|