Digital Drosophila Archive
152 lines
7.7 KiB
Python
152 lines
7.7 KiB
Python
"""Ordered fixed-D18 validation; representation failure never becomes execution failure.
|
|
|
|
Only the two predeclared first-odor/first-seed mode probes use Brian per cohort.
|
|
Their GPU r0 outputs are reused in the full 60-case block. No novel-to-AJ
|
|
trajectory can start until the complete seed block and its repeats pass.
|
|
"""
|
|
import argparse
|
|
import subprocess
|
|
import time
|
|
import traceback
|
|
|
|
from context import ROOT, MODEL, EXP, CFG, check, save, read, budget
|
|
import runtime_bridge as bridge
|
|
|
|
|
|
SCIENCE = ROOT / 'models/male-cns-lif/.venv/bin/python'
|
|
ORACLE = ROOT / 'experiments/phase4ad-author-semantics/environment/.venv/bin/python'
|
|
|
|
|
|
class RepresentationGateFailed(RuntimeError):
|
|
"""Completed valid trajectories did not pass the predeclared representation gate."""
|
|
|
|
|
|
def child(backend, group, name, repeat=0):
|
|
# Do not resolve either interpreter: the venv invocation path is meaningful.
|
|
interpreter = ORACLE if backend == 'oracle' else SCIENCE
|
|
bridge.require(interpreter.is_file(), 'Missing fixed runtime interpreter: ' + str(interpreter))
|
|
bridge.authorize_case(EXP, CFG, group, name, backend, repeat)
|
|
log = EXP / 'logs' / f'{backend}-{group}-{name}-r{repeat}.log'
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
command = [str(interpreter), '-B', str(MODEL / 'run_case.py'), backend, group, name, '--repeat', str(repeat)]
|
|
print('START', backend, group, name, repeat, flush=True)
|
|
started = time.perf_counter()
|
|
with log.open('x', encoding='utf-8') as stream:
|
|
result = subprocess.run(command, stdout=stream, stderr=subprocess.STDOUT, timeout=1800, check=False)
|
|
print('FINISH', backend, group, name, repeat, 'exit', result.returncode,
|
|
'wall', round(time.perf_counter() - started, 2), flush=True)
|
|
bridge.require(result.returncode == 0, f'{backend} {group}/{name} failed; preserve partial evidence and inspect {log}')
|
|
|
|
|
|
def control(row):
|
|
name = row['name']
|
|
child('gpu', 'controls', name)
|
|
case = EXP / 'cases/controls' / name
|
|
output = EXP / 'gpu/controls' / (name + '-r0')
|
|
report = bridge.control_identity(output, row['source_output'], case, row['source_case'],
|
|
EXP / 'audit' / ('control-' + name + '.json'))
|
|
bridge.require(report['pass_all'], 'Fresh fixed-D18 control differs from sealed AJ: ' + name)
|
|
child('component', 'controls', name)
|
|
return dict(case=name, group='controls', pass_all=True, fresh_gpu_runs=1, new_oracle_runs=0)
|
|
|
|
|
|
def probe(row):
|
|
name, group = row['name'], row['cohort']
|
|
child('gpu', group, name)
|
|
child('oracle', group, name)
|
|
result = bridge.compare(EXP / 'cases' / group / name, EXP / 'gpu' / group / (name + '-r0'),
|
|
EXP / 'oracle' / group / name, EXP / 'audit' / ('compare-' + group + '-' + name + '.json'))
|
|
bridge.require(result['pass_all'], 'Closed-loop fixed-D18 comparison failed: ' + name)
|
|
child('component', group, name)
|
|
return dict(case=name, group=group, pass_all=True, fresh_gpu_runs=1, new_oracle_runs=1)
|
|
|
|
|
|
def validation_block(cohort, records):
|
|
from validation import summarize
|
|
rows = bridge.cohort_rows(records, CFG, cohort)
|
|
probes = {row['name'] for row in bridge.probe_rows(records, CFG, cohort)}
|
|
repeated = []
|
|
for row in rows:
|
|
name = row['name']
|
|
if name not in probes:
|
|
child('gpu', cohort, name)
|
|
else:
|
|
comparison = read(EXP / 'audit' / ('compare-' + cohort + '-' + name + '.json'))
|
|
bridge.require(comparison.get('pass_all') is True and comparison.get('case') == name,
|
|
'Missing successful probe evidence for reused r0: ' + name)
|
|
if row['seed'] == CFG['validation']['seeds'][0]:
|
|
child('gpu', cohort, name, 1)
|
|
result = bridge.compare_gpu_repeats(EXP / 'gpu' / cohort / (name + '-r0'),
|
|
EXP / 'gpu' / cohort / (name + '-r1'), EXP / 'audit' / ('repeat-' + cohort + '-' + name + '.json'))
|
|
bridge.require(result['pass_all'], 'Fixed-input GPU repeat diverged: ' + name)
|
|
repeated.append(name)
|
|
bridge.require(len(rows) == 60 and len(repeated) == 12, 'Incomplete registered base/repeat block')
|
|
result = summarize(cohort)
|
|
return result, repeated
|
|
|
|
|
|
def run(stage):
|
|
bridge.require(stage in bridge.STAGES, 'Unknown stage')
|
|
check(); budget()
|
|
bridge.ensure_unstopped(EXP)
|
|
receipt = EXP / 'audit' / (stage + '-gate.json')
|
|
bridge.require(not receipt.exists() and not (EXP / 'audit' / ('stopped-' + stage + '.json')).exists(),
|
|
'Stage already has completion/stop evidence; do not rerun or overwrite')
|
|
controls, records = bridge.inventories(EXP, CFG)
|
|
try:
|
|
if stage == 'controls':
|
|
rows = [control(row) for row in controls]
|
|
result = dict(stage=stage, pass_all=True, cases=rows, scope='Same-input fresh D18 GPU identity with sealed AJ; no new model or Brian run')
|
|
else:
|
|
bridge.gate_pass(EXP, 'controls')
|
|
cohort = 'seed_validation' if stage.startswith('seed-') else 'novel_to_aj'
|
|
if cohort == 'novel_to_aj':
|
|
bridge.gate_pass(EXP, 'seed-validation', representation=True, cfg=CFG)
|
|
if stage.endswith('-probes'):
|
|
rows = [probe(row) for row in bridge.probe_rows(records, CFG, cohort)]
|
|
result = dict(stage=stage, pass_all=True, cases=rows,
|
|
scope='Two fixed first-odor first-seed mode probes; independent LIF with shared unchanged Torch APL component')
|
|
else:
|
|
bridge.gate_pass(EXP, 'seed-probes' if cohort == 'seed_validation' else 'novel-probes')
|
|
summary, repeats = validation_block(cohort, records)
|
|
execution = summary.get('execution_gate_pass') is True
|
|
representation = summary.get('representation_gate_pass') is True
|
|
if execution:
|
|
bridge.validate_summary_coverage(summary, EXP, CFG, cohort)
|
|
result = dict(stage=stage, cohort=cohort, pass_all=execution,
|
|
execution_gate_pass=execution, representation_gate_pass=representation,
|
|
base_count=60, repeat_count=len(repeats), repeat_cases=repeats,
|
|
summary='validation/' + cohort + '/summary.json',
|
|
summary_sha256=bridge.digest(EXP / 'validation' / cohort / 'summary.json'),
|
|
validation_inventory_sha256=bridge.digest(EXP / 'protocol/validation-cases.json'),
|
|
scope='Complete fixed representation block; no pooled rescue, learning, tuning, or historically-unseen-odor claim')
|
|
save(receipt, result)
|
|
if stage.endswith('-validation'):
|
|
bridge.require(result['execution_gate_pass'], 'Validation execution/numerical/integrity gate failed')
|
|
if not result['representation_gate_pass']:
|
|
raise RepresentationGateFailed('Completed fixed validation block failed representation; no later cohort is authorized')
|
|
budget()
|
|
return result
|
|
except Exception as error:
|
|
save(EXP / 'audit' / ('stopped-' + stage + '.json'), dict(stage=stage,
|
|
type=type(error).__name__, message=str(error), traceback=traceback.format_exc(),
|
|
stop_kind='representation_failure' if isinstance(error, RepresentationGateFailed) else 'execution_or_numerical_failure',
|
|
representation_failure_is_execution_failure=False,
|
|
later_stages_not_permitted=True, partial_outputs_preserved=True,
|
|
parameters_changed=False, replacement_seeds_or_odors_selected=False))
|
|
raise
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('stage', choices=bridge.STAGES)
|
|
args = parser.parse_args()
|
|
try:
|
|
run(args.stage)
|
|
except RepresentationGateFailed:
|
|
raise SystemExit(2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|