Digital Drosophila Archive
241 lines
14 KiB
Python
241 lines
14 KiB
Python
"""Fresh exec workers for the NEW boundary-validation branch, not learning.
|
|
|
|
The unchanged AP worker supplies read-only path/hash/environment helpers.
|
|
One worker imports exactly one new scientific entry point. The process parent
|
|
must authenticate registration, record actual exit status and independently
|
|
check the result. A worker receipt cannot prove its own future successful exit.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
import time
|
|
|
|
AP_SHA = 'e47651a8ea99d7c44115107ff6b5dc71312183609ae11c891db65bb21ed61e30'
|
|
MODEL = Path(__file__).resolve().parent
|
|
AP = MODEL.parent / 'conditioning-interface-v2' / 'isolated_workers.py'
|
|
SCHEMA = 'conditioning-isolated-execution-job-v1'
|
|
C_PATH = MODEL.parent/'olfactory-conditioning-lab-v1/runtime_continuation.py'
|
|
C_SHA = '0b8e8e0a6a23a079dc62a88355ce355d53b3ae22f6665519a8f6aef2dcfc5aba'
|
|
COMMON = {'schema', 'job_id', 'kind', 'registration_sha256', 'worker_sha256',
|
|
'source_sha256', 'input_sha256', 'output_dir', 'receipt_file', 'budget'}
|
|
FIELDS = {
|
|
'updated': {'authority_file', 'authority_sha256', 'build_root', 'diagnostic_mode'},
|
|
'oracle': {'authority_file', 'authority_sha256', 'build_root', 'oracle_case',
|
|
'oracle_spec_sha256', 'oracle_requests_sha256'},
|
|
'validate_cpu': {'authority_file', 'authority_sha256', 'update_dir', 'chain_sha256'},
|
|
'unchanged': {'authority_file', 'authority_sha256', 'case_dir', 'build_root', 'diagnostic_mode',
|
|
'overlay_file', 'overlay_manifest', 'checkpoint_dir', 'checkpoint_sha256'},
|
|
'sample': {'authority_file', 'authority_sha256'},
|
|
}
|
|
SOURCES = {
|
|
'updated': {'updated_continuation.py'},
|
|
'oracle': {'updated_continuation.py', 'boundary_oracle.py'},
|
|
'validate_cpu': {'validate_update_cpu.py'},
|
|
'unchanged': set(),
|
|
'sample': {'boundary_sampler.py','boundary_plan.py'},
|
|
}
|
|
|
|
|
|
def ap_helpers():
|
|
raw = AP.read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != AP_SHA:
|
|
raise ValueError('Frozen AP helper SHA differs')
|
|
name = '_conditioning_execution_AP_helpers'
|
|
if name in sys.modules:
|
|
raise ValueError('Fresh exec helper binding required')
|
|
spec = importlib.util.spec_from_file_location(name, AP)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
exec(compile(raw, str(AP), 'exec'), module.__dict__)
|
|
return module
|
|
|
|
|
|
def validate_job(h, job_file, expected_sha):
|
|
job_file = h.absolute_path(str(job_file), kind='file')
|
|
h.require(h.valid_sha(expected_sha) and h.digest(job_file) == expected_sha, 'External job pin differs')
|
|
job = h.read_json(job_file)
|
|
kind = job.get('kind')
|
|
h.require(kind in FIELDS and set(job) == COMMON | FIELDS[kind] and job['schema'] == SCHEMA,
|
|
'Wrong job schema/fields')
|
|
h.require(type(job['job_id']) is str and job['job_id'] and
|
|
all(c.isascii() and (c.isalnum() or c in '-_') for c in job['job_id']), 'Unsafe job ID')
|
|
h.require(h.valid_sha(job['registration_sha256']) and job['worker_sha256'] == h.digest(__file__),
|
|
'Missing registration/worker binding')
|
|
h.require(type(job['source_sha256']) is dict and set(job['source_sha256']) == SOURCES[kind] and
|
|
type(job['input_sha256']) is dict and job['input_sha256'], 'Wrong source/input inventory')
|
|
pins = {job_file: expected_sha, Path(__file__).resolve(): job['worker_sha256'], AP: AP_SHA}
|
|
for path, sha in job['input_sha256'].items():
|
|
h._put(pins, h.absolute_path(path, kind='file'), sha)
|
|
for name, sha in job['source_sha256'].items():
|
|
h._put(pins, h.absolute_path(str(MODEL / name), kind='file'), sha)
|
|
authority = h.absolute_path(job['authority_file'], kind='file')
|
|
h._required(pins, authority, job['authority_sha256'])
|
|
if kind=='sample':
|
|
h.require(job['authority_sha256']==job['registration_sha256'],'Sampler registration and authority differ')
|
|
registered=h.read_json(authority)
|
|
h.require(registered.get('schema')=='conditioning-execution-registration-v1','Wrong sampler registration')
|
|
for field in ('source_sha256','input_sha256','prepared_file_sha256'):
|
|
for path,sha in registered[field].items():h._required(pins,path,sha)
|
|
output, receipt = (h.absolute_path(job[name]) for name in ('output_dir', 'receipt_file'))
|
|
attempt = Path(str(receipt) + '.attempt.json')
|
|
h.require(output.parent.is_dir() and receipt.parent.is_dir() and
|
|
not any(p.exists() for p in (output, receipt, attempt)), 'Output/receipt exists; no automatic retry')
|
|
protected = {p.parent for p in pins}
|
|
h.require(not any(output.is_relative_to(p) or receipt.is_relative_to(p) for p in protected) and
|
|
not receipt.is_relative_to(output) and not output.is_relative_to(receipt.parent),
|
|
'Output/receipt overlaps protected input or each other')
|
|
paths = dict(job=job_file, authority=authority, output=output, receipt=receipt, attempt=attempt)
|
|
if kind in ('updated', 'oracle', 'unchanged'):
|
|
build = h.absolute_path(job['build_root'])
|
|
h.require(not any(build.is_relative_to(p) for p in protected) and not build.is_relative_to(output) and
|
|
not output.is_relative_to(build), 'Build overlaps protected data')
|
|
paths['build'] = build
|
|
if kind in ('updated', 'unchanged'):
|
|
h.require(job['diagnostic_mode'] in ('light', 'reference'), 'Unknown diagnostic mode')
|
|
if kind == 'unchanged':
|
|
h._required(pins, C_PATH, C_SHA)
|
|
paths['case'] = h.absolute_path(job['case_dir'], kind='dir')
|
|
paths['checkpoint'] = h.absolute_path(job['checkpoint_dir'], kind='dir')
|
|
h._required(pins, paths['checkpoint']/'checkpoint-manifest.json', job['checkpoint_sha256'])
|
|
for role in ('overlay_file', 'overlay_manifest'):
|
|
paths[role] = h._required(pins, job[role])
|
|
for name in ('spec.json', 'requests.npz'): h._required(pins, paths['case']/name)
|
|
h.require(not output.is_relative_to(paths['checkpoint']) and not paths['build'].is_relative_to(paths['checkpoint']),
|
|
'Execution overlaps frozen parent')
|
|
elif kind == 'oracle':
|
|
paths['case'] = h.absolute_path(job['oracle_case'], kind='dir')
|
|
for name, key in (('spec.json', 'oracle_spec_sha256'), ('requests.npz', 'oracle_requests_sha256')):
|
|
h._required(pins, paths['case'] / name, job[key])
|
|
elif kind == 'validate_cpu':
|
|
paths['update'] = h.absolute_path(job['update_dir'], kind='dir')
|
|
h._required(pins, paths['update'] / 'chain-receipt.json', job['chain_sha256'])
|
|
h.require(not output.is_relative_to(paths['update']), 'CPU receipt overlaps source update')
|
|
budget = job['budget']
|
|
h.require(type(budget) is dict and set(budget) == {'root', 'output_cap_bytes', 'minimum_free_reserve_bytes'},
|
|
'Explicit bounded resource plan required')
|
|
paths['budget'] = h.absolute_path(budget['root'], kind='dir')
|
|
h.require(paths['budget'].name.startswith('phase4') and output.is_relative_to(paths['budget']) and
|
|
receipt.is_relative_to(paths['budget']) and
|
|
(kind in ('validate_cpu','sample') or paths['build'].is_relative_to(paths['budget'])), 'Execution leaves assigned experiment')
|
|
for name in ('output_cap_bytes', 'minimum_free_reserve_bytes'):
|
|
h.require(type(budget[name]) is int and budget[name] > 0, 'Positive resource limit required')
|
|
h.verify_pins(pins)
|
|
return job, paths, pins
|
|
|
|
|
|
def budget_check(h, job, paths):
|
|
used = 0
|
|
for path in paths['budget'].rglob('*'):
|
|
h.require(not path.is_symlink(), 'Linked output forbidden')
|
|
if path.is_file():
|
|
used += path.stat().st_size
|
|
else:
|
|
h.require(path.is_dir(), 'Nonregular output forbidden')
|
|
cap = job['budget']['output_cap_bytes']
|
|
free = shutil.disk_usage(paths['budget']).free
|
|
h.require(used <= cap and free >= job['budget']['minimum_free_reserve_bytes'] + max(0, cap - used),
|
|
'Resource budget exceeded; do not change protocol or retry')
|
|
return dict(output_bytes=used, free_bytes=free, output_cap_bytes=cap)
|
|
|
|
|
|
def bind_unchanged_authority(h, job, paths, pins):
|
|
admitted=h.read_json(paths['authority'])
|
|
h.require(admitted.get('schema')=='conditioning-updated-boundary-v1' and admitted.get('transition')=='epoch_update' and
|
|
admitted['diagnostic_mode']==job['diagnostic_mode'] and admitted['registration_sha256']==job['registration_sha256'] and
|
|
admitted['expected_effective_weight_before_sha256']==admitted['expected_effective_weight_after_sha256'],
|
|
'Old C comparison requires its unchanged-weight registered authority')
|
|
h.require(admitted['parent']==dict(directory=str(paths['checkpoint']),checkpoint_manifest_sha256=job['checkpoint_sha256']),
|
|
'Old C job parent differs from authority')
|
|
for pair,role in ((admitted['case']['spec'],'spec.json'),(admitted['case']['requests'],'requests.npz')):
|
|
h.require(pair['path']==str(paths['case']/role) and pins[paths['case']/role]==pair['sha256'],
|
|
'Old C case differs from authority')
|
|
for role,key in (('file','overlay_file'),('manifest','overlay_manifest')):
|
|
pair=admitted['overlay_before'][role]
|
|
h.require(pair['path']==str(paths[key]) and pins[paths[key]]==pair['sha256'],
|
|
'Old C must use actual before snapshot, not equal after data')
|
|
def visit(value):
|
|
if type(value)is dict:
|
|
if set(value)=={'path','sha256'}:h._required(pins,value['path'],value['sha256'])
|
|
else:
|
|
for key,item in value.items():
|
|
if key=='frozen_inputs':
|
|
for path,sha in item.items():h._required(pins,path,sha)
|
|
else:visit(item)
|
|
elif type(value)is list:
|
|
for item in value:visit(item)
|
|
visit(admitted)
|
|
return admitted
|
|
|
|
|
|
def execute(job_file, expected_sha):
|
|
started = time.perf_counter()
|
|
h = ap_helpers()
|
|
h.require(sys.dont_write_bytecode, 'Use Python -B')
|
|
job, paths, pins = validate_job(h, job_file, expected_sha)
|
|
kind = job['kind']
|
|
h._clean_process('update' if kind == 'validate_cpu' else 'brain')
|
|
before = budget_check(h, job, paths)
|
|
h.save_json(paths['attempt'], dict(schema='conditioning-execution-attempt-v1', complete=False,
|
|
job_sha256=expected_sha, pid=os.getpid(), started_utc=datetime.now(timezone.utc).isoformat(),
|
|
no_automatic_retry=True))
|
|
admission = None if kind == 'validate_cpu' else h._admit_brain()
|
|
if kind == 'unchanged':
|
|
module = h._load_source('_conditioning_execution_entry', C_PATH, C_SHA)
|
|
bind_unchanged_authority(h, job, paths, pins)
|
|
result = module.run_continued(paths['case'], paths['output'], paths['build'], paths['overlay_file'],
|
|
paths['overlay_manifest'], paths['checkpoint'], mode=job['diagnostic_mode'])
|
|
checked = module.load_checkpoint(paths['output'], result['checkpoint_manifest_sha256'])
|
|
h.require(checked['summary'] == {k:v for k,v in result.items() if k!='checkpoint_manifest_sha256'},
|
|
'Original C returned checkpoint differs')
|
|
else:
|
|
entry = {'updated': 'updated_continuation.py', 'oracle': 'boundary_oracle.py',
|
|
'validate_cpu': 'validate_update_cpu.py', 'sample': 'boundary_sampler.py'}[kind]
|
|
module = h._load_source('_conditioning_execution_entry', MODEL / entry, job['source_sha256'][entry])
|
|
if kind == 'updated':
|
|
result = module.run(paths['authority'], job['authority_sha256'], paths['output'], paths['build'],
|
|
mode=job['diagnostic_mode'])
|
|
elif kind == 'oracle':
|
|
result = module.run(paths['authority'], job['authority_sha256'], job['source_sha256']['updated_continuation.py'],
|
|
paths['case'], job['oracle_spec_sha256'], job['oracle_requests_sha256'], paths['output'], paths['build'])
|
|
elif kind == 'validate_cpu':
|
|
paths['output'].mkdir()
|
|
result = module.validate(paths['authority'], job['authority_sha256'], paths['update'],
|
|
job['chain_sha256'], paths['output'] / 'validation.json')
|
|
elif kind == 'sample':
|
|
result = module.run(paths['authority'], job['authority_sha256'], paths['output'])
|
|
environment = h._environment('update' if kind == 'validate_cpu' else 'brain')
|
|
if kind not in ('validate_cpu','sample'):
|
|
h._verify_precision(environment, result['apl_component']['environment'])
|
|
outputs = h.tree_hashes(paths['output'])
|
|
h.verify_pins(pins)
|
|
after = budget_check(h, job, paths)
|
|
h.require(outputs == h.tree_hashes(paths['output']), 'Outputs changed during verification')
|
|
receipt = dict(schema='conditioning-isolated-execution-receipt-v1', complete=True,
|
|
job_id=job['job_id'], kind=kind, registration_sha256=job['registration_sha256'], job_sha256=expected_sha,
|
|
worker_sha256=job['worker_sha256'], attempt_sha256=h.digest(paths['attempt']),
|
|
input_sha256={str(p): sha for p, sha in pins.items()}, output_files_sha256=outputs,
|
|
output_dir=str(paths['output']), result=result, environment=environment, admission=admission,
|
|
resource_before=before, resource_after=after, wall_seconds=time.perf_counter()-started,
|
|
wall_scope='entry through verification; parent separately measures spawn/exit/parent validation',
|
|
process_exit_independently_observed=False, registration_authority_independently_proven=False,
|
|
scientific_learning_pass_claimed=False, candidate_generated=False, old_source_globals_patched=False)
|
|
h.save_json(paths['receipt'], receipt)
|
|
return receipt
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--job', required=True)
|
|
parser.add_argument('--job-sha256', required=True)
|
|
args = parser.parse_args()
|
|
receipt = execute(args.job, args.job_sha256)
|
|
print(json.dumps(dict(complete=True, job_id=receipt['job_id'], kind=receipt['kind'])), flush=True)
|