Digital Drosophila Archive
200 lines
11 KiB
Python
200 lines
11 KiB
Python
"""Pure normal-only adapter of the AO-pinned, master-20260930 inventory.
|
|
|
|
No sampling, brain execution, file writes, biological identity certification,
|
|
or execution authorization. Import reads and SHA-checks six frozen sources.
|
|
The trial argument must reproduce the ORIGINAL AO saved-file bytes under its
|
|
documented JSON encoding. The evaluation argument is a new exact expansion of
|
|
the old pinned policy, NOT an independently archived AO evaluation artifact.
|
|
Physical inputs, actual DAN mapping, checkpoints, model and process receipts
|
|
remain the registering executor's responsibility. No scientific gate changes.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import builtins
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import types
|
|
|
|
|
|
SCHEMA = "normal-conditioning-plan-v1"
|
|
MASTER = 20260930
|
|
AO_TRIAL_FILE_SHA256 = "a7bda850ee87df4d9f647994e1a1a4074dfea5793184edb1822ab3d8c02e8216"
|
|
AO_REGISTRATION_SHA256 = "9e2ed05879ce279da7dc0b4cac09a44d67665c230b56db94e2ca39c645bdc250"
|
|
AP_REGISTRATION_SHA256 = "02a03b536a9f14196ad4b92fa9565b592732c6c863e72eeaee89b3b0ea8ed37d"
|
|
AO_TRIAL_PATH = "/data/digital-drosophila/experiments/phase4ao-conditioning-interface/protocol/full-trial-plan.json"
|
|
FROZEN_DIRECTORY = Path(__file__).resolve().parent.parent / "olfactory-conditioning-lab-v1"
|
|
SOURCE_PINS = {
|
|
"seed_registry.py": "0da033b132095ec3087b5d9f64e53c2e391fa50ab0b7653b31296eeb84799b7e",
|
|
"trial_plan.py": "6fed3f494d075339a7e5b001d43b2598640b189e038952fbd4d41200bacac587",
|
|
"evaluation_plan.py": "991c813ff3322b230947f302b12b04ec382c3b632817f94b698101e4b17efa7d",
|
|
"evaluation_policy.py": "21331a9ff3567dc6f6399e16bb91ef486d74231f5db51799beb6dd1bf71d022b",
|
|
"PROTOCOL_DRAFT.md": "8ebecb1f3d012ddc29a1cb727269dd277629c53565e09978bf9dbf7140cb8508",
|
|
"EVALUATION_POLICY_DRAFT.md": "a84170a61564715a1532f20df4c6a330f0f9725c383db48b8f3e1a4b5fba0bc8",
|
|
}
|
|
|
|
|
|
def _require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def _canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"),
|
|
ensure_ascii=False, allow_nan=False)
|
|
|
|
|
|
def _sha(value):
|
|
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _load_frozen():
|
|
"""Private module graph; never change sys.path or ordinary module caches."""
|
|
sources = {}
|
|
for filename, expected in SOURCE_PINS.items():
|
|
raw = (FROZEN_DIRECTORY / filename).read_bytes()
|
|
_require(hashlib.sha256(raw).hexdigest() == expected,
|
|
"Frozen source pin mismatch: " + filename)
|
|
sources[filename] = raw
|
|
modules = {}
|
|
ordinary_import = builtins.__import__
|
|
|
|
def private_import(name, globals=None, locals=None, fromlist=(), level=0):
|
|
if level == 0 and name in modules:
|
|
return modules[name]
|
|
return ordinary_import(name, globals, locals, fromlist, level)
|
|
|
|
for name in ("seed_registry", "trial_plan", "evaluation_plan", "evaluation_policy"):
|
|
module = types.ModuleType("_normal_frozen_" + name)
|
|
module.__file__ = str(FROZEN_DIRECTORY / (name + ".py"))
|
|
module.__dict__["__builtins__"] = dict(vars(builtins), __import__=private_import)
|
|
exec(compile(sources[name + ".py"], module.__file__, "exec"), module.__dict__)
|
|
modules[name] = module
|
|
return modules
|
|
|
|
|
|
_FROZEN = _load_frozen()
|
|
|
|
|
|
def _validate_original_trial(trial):
|
|
_require(type(trial) is dict and set(trial) ==
|
|
{"plan", "seed_manifest", "budget", "payload_sha256"}, "Full original trial payload required")
|
|
_require(type(trial["plan"]["master"]) is int and trial["plan"]["master"] == MASTER,
|
|
"Original AO master is fixed; no seed selection")
|
|
# This is the exact LF serialization used by AO engineering_prepare.save.
|
|
# Its registered file pin is stronger than reconstructing with a new seed.
|
|
encoded = json.dumps(trial, sort_keys=True, indent=2, ensure_ascii=False,
|
|
allow_nan=False) + "\n"
|
|
_require(hashlib.sha256(encoded.encode("utf-8")).hexdigest() == AO_TRIAL_FILE_SHA256,
|
|
"Trial does not reproduce the AO registered full-trial-plan file")
|
|
_require(_sha({k: v for k, v in trial.items() if k != "payload_sha256"}) ==
|
|
trial["payload_sha256"], "Original trial self-hash mismatch")
|
|
|
|
|
|
def _expand_validated_trial(trial):
|
|
# Private frozen assembly is safe AFTER the exact archived-file pin above;
|
|
# avoid the old public APIs' redundant 68,000-epoch trial reconstruction.
|
|
policy = _FROZEN["evaluation_policy"]._assemble(trial)
|
|
return _FROZEN["evaluation_plan"]._assemble(trial, policy["config"])
|
|
|
|
|
|
def expand_pinned_evaluation(full_trial_payload):
|
|
"""Expand the fixed historical policy; no new choices or sampled inputs."""
|
|
_validate_original_trial(full_trial_payload)
|
|
return _expand_validated_trial(full_trial_payload)
|
|
|
|
|
|
def build_normal_plan(full_trial_payload, full_evaluation_payload):
|
|
"""Return 33,920 ordered tasks; caller supplies both complete authorities."""
|
|
trial, evaluation = full_trial_payload, full_evaluation_payload
|
|
_validate_original_trial(trial)
|
|
expected_evaluation = _expand_validated_trial(trial)
|
|
_require(_canonical(evaluation) == _canonical(expected_evaluation),
|
|
"Evaluation must exactly expand the AO-pinned evaluation policy")
|
|
snapshots = {(s["block_id"], s["arm"], s["completed_epochs"]): s
|
|
for s in evaluation["snapshots"] if s["assay"] == "normal"}
|
|
probes = {(p["snapshot_id"], p["cue"]): p
|
|
for p in evaluation["probes"] if p["assay"] == "normal"}
|
|
tasks, inputs, blocks = [], [], []
|
|
for block in trial["plan"]["blocks"]:
|
|
common = dict(block_id=block["block_id"], mode=block["mode"], dan_type=block["dan_type"],
|
|
compartment_verbatim=block["compartment_verbatim"],
|
|
reinforced_cue=block["initial_reinforced_cue"], replicate=block["replicate"],
|
|
assay="normal", steps=1200, artifact_validation_required=True)
|
|
blocks.append(dict(common, DAN_target_binding=block["DAN_target_binding"]))
|
|
for cue in ("A", "B"):
|
|
sid = block["block_id"] + "/normal/paired/checkpoint-00"
|
|
inputs.append(dict(block_id=block["block_id"], cue=cue, **probes[(sid, cue)]["input"]))
|
|
suite = next(s for s in block["suites"] if s["assay"] == "normal")
|
|
for arm in suite["arms"]:
|
|
name = arm["arm"]
|
|
delta = dict(arm["delta_start"])
|
|
|
|
def append_probes(completed):
|
|
snapshot = snapshots.get((block["block_id"], name, completed))
|
|
if snapshot is None:
|
|
return
|
|
_require(snapshot["delta_source"] == delta, "Snapshot delta dependency differs")
|
|
for cue in ("A", "B"):
|
|
recipe = probes[(snapshot["snapshot_id"], cue)]
|
|
tasks.append(dict(common, kind="probe", arm=name, task_id=recipe["probe_id"],
|
|
probe_id=recipe["probe_id"], snapshot_id=snapshot["snapshot_id"],
|
|
checkpoint=snapshot["checkpoint"], completed_epochs=completed,
|
|
evaluation_input_id=recipe["input"]["input_id"], delta_dependency=dict(delta),
|
|
delta_output_id=None, neural_state_policy="common_frozen_reset_independent_probe",
|
|
carry_between_epochs=[], original_E_required=False, cpu_validation_required=False,
|
|
feedback_to_training_or_idle=False, recipe=recipe, snapshot_binding=snapshot,
|
|
source_binding=dict(payload_sha256=evaluation["payload_sha256"],
|
|
record_id=recipe["probe_id"], record_sha256=_sha(recipe))))
|
|
|
|
append_probes(0)
|
|
for completed, epoch in enumerate(arm["epochs"], 1):
|
|
tasks.append(dict(common, kind="train", arm=name, task_id=epoch["epoch_id"],
|
|
epoch_id=epoch["epoch_id"], epoch_index=epoch["epoch_index"],
|
|
delta_dependency=dict(delta), delta_output_id=epoch["epoch_id"] + "/delta-after",
|
|
neural_state_policy=epoch["initial_neural_state"],
|
|
carry_between_epochs=list(arm["carry_between_epochs"]),
|
|
original_E_required=True, cpu_validation_required=True,
|
|
feedback_to_training_or_idle=True, recipe=epoch,
|
|
source_binding=dict(payload_sha256=trial["payload_sha256"],
|
|
record_id=epoch["epoch_id"], record_sha256=_sha(epoch))))
|
|
delta = dict(kind="epoch_delta_after", source=epoch["epoch_id"] + "/delta-after")
|
|
append_probes(completed)
|
|
counts = Counter(row["kind"] for row in tasks)
|
|
_require(counts == {"train": 30000, "probe": 3920} and len(inputs) == 80 and len(blocks) == 40,
|
|
"Normal inventory changed")
|
|
_require(len({t["task_id"] for t in tasks}) == len(tasks), "Task IDs must be unique")
|
|
result = dict(schema=SCHEMA, master=MASTER, assay="normal", blocks=blocks,
|
|
tasks=tasks, evaluation_inputs=inputs,
|
|
inventory=dict(blocks=40, training_replicates_per_cell=5, arms_per_block=7,
|
|
nominal_reinforcements=50, checkpoints=[0, 1, 2, 5, 10, 20, 50],
|
|
train_calls=30000, probe_calls=3920, brain_calls=33920,
|
|
original_E_calls=30000, training_cpu_validation_calls=30000,
|
|
brain_steps=40704000, biological_seconds=4070.4, evaluation_inputs=80,
|
|
baseline_execution="execute_each_snapshot", baseline_output_deduplication=False),
|
|
provenance=dict(trial_payload_sha256=trial["payload_sha256"],
|
|
seed_manifest_sha256=trial["seed_manifest"]["payload_sha256"],
|
|
evaluation_payload_sha256=evaluation["payload_sha256"],
|
|
evaluation_config_sha256=evaluation["config_sha256"],
|
|
ao_trial_file_sha256=AO_TRIAL_FILE_SHA256, ao_trial_path=AO_TRIAL_PATH,
|
|
ao_registration_sha256=AO_REGISTRATION_SHA256, ap_registration_sha256=AP_REGISTRATION_SHA256,
|
|
frozen_source_sha256=SOURCE_PINS,
|
|
evaluation_authority="exact expansion of AO-pinned policy; not a historical saved evaluation artifact",
|
|
external_inputs_mapping_processes_and_physical_source_files_verified=False),
|
|
execution_authorized=False, inputs_generated=False, new_seed_streams=0,
|
|
learning_pass_claimed=False, subsequent_assays_included=False)
|
|
result["payload_sha256"] = _sha(result)
|
|
# No returned nested structure aliases either caller argument or module pins.
|
|
return json.loads(_canonical(result))
|
|
|
|
|
|
def validate_normal_plan(payload, full_trial_payload, full_evaluation_payload):
|
|
"""Fail closed even if a modified plan has been given a fresh self-hash."""
|
|
_require(type(payload) is dict and "payload_sha256" in payload, "Normal payload required")
|
|
_require(_sha({k: v for k, v in payload.items() if k != "payload_sha256"}) ==
|
|
payload["payload_sha256"], "Normal plan self-hash mismatch")
|
|
expected = build_normal_plan(full_trial_payload, full_evaluation_payload)
|
|
_require(_canonical(payload) == _canonical(expected), "Normal plan exact reconstruction mismatch")
|
|
return True
|