Digital Drosophila Archive
261 lines
16 KiB
Python
261 lines
16 KiB
Python
"""Pure, UNREGISTERED evaluation inventory; no files, RNG draws or execution.
|
|
|
|
build_evaluation_plan(trial_payload, config) requires the complete existing
|
|
trial_plan payload, including its caller-chosen master and seed manifest. It
|
|
reuses that payload's 80 evaluation inputs; repeated probes are paired
|
|
observations, never extra independent training replicates. No defaults select
|
|
CS offset, reversal curve checkpoints, control snapshot boundaries, or baseline
|
|
execution policy. See required_choices(). A complete inventory is not approval,
|
|
an executed snapshot, an externally verified source pin, or a scientific gate.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
|
|
import seed_registry
|
|
import trial_plan
|
|
|
|
|
|
SCHEMA = "phase4an-evaluation-plan-draft-v1"
|
|
CONFIG_SCHEMA = "phase4an-evaluation-config-draft-v1"
|
|
CONFIG_FIELDS = frozenset(("schema", "probe_steps", "cs_start_tick",
|
|
"cs_stop_tick_exclusive", "reversal_checkpoints", "baseline_execution",
|
|
"snapshot_binding_policy", "snapshot_bindings"))
|
|
BINDING_FIELDS = frozenset(("block_id", "assay", "arm", "checkpoint", "completed_epochs"))
|
|
|
|
|
|
def _require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def _canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"),
|
|
ensure_ascii=False, allow_nan=False)
|
|
|
|
|
|
def _sha(value):
|
|
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def required_choices():
|
|
"""Describe mandatory choices, without filling or approving any of them."""
|
|
return {
|
|
"probe_steps": "Explicit integer 1200: the draft's fixed 120 ms metric window.",
|
|
"cs_start_tick": "Explicit integer 0: stimulus onset defines probe latency.",
|
|
"cs_stop_tick_exclusive": "Explicit integer in [1,1200]; draft does not settle evaluation CS offset.",
|
|
"reversal_checkpoints": "Explicit sorted subset of [0,1,2,5,10,20,50], including 0 and 50.",
|
|
"baseline_execution": "Only execute_each_snapshot is implemented; declare it explicitly. No output deduplication.",
|
|
"snapshot_binding_policy": "Explicit completed_epochs; nominal control labels are not actual update counts.",
|
|
"snapshot_bindings": "Every block/assay/arm/checkpoint, in trial-plan order, with explicit completed_epochs. No control alignment is guessed.",
|
|
"authority": "Provide the complete existing trial payload; no master seed is chosen here. Source/input pins remain coordinator responsibilities.",
|
|
}
|
|
|
|
|
|
def _configuration(config):
|
|
_require(type(config) is dict, "Explicit evaluation config is required; see required_choices()")
|
|
_require(set(config) == CONFIG_FIELDS,
|
|
"Evaluation config fields: missing=" + str(sorted(CONFIG_FIELDS - set(config))) +
|
|
"; unknown=" + str(sorted(set(config) - CONFIG_FIELDS)))
|
|
_require(config["schema"] == CONFIG_SCHEMA, "Wrong evaluation config schema")
|
|
_require(type(config["probe_steps"]) is int and config["probe_steps"] == 1200,
|
|
"Explicit probe_steps must be integer 1200")
|
|
_require(type(config["cs_start_tick"]) is int and config["cs_start_tick"] == 0,
|
|
"Explicit CS onset must be integer tick 0")
|
|
stop = config["cs_stop_tick_exclusive"]
|
|
_require(type(stop) is int and 0 < stop <= 1200,
|
|
"Evaluation CS stop tick is required; no training-CS default is inferred")
|
|
checks = config["reversal_checkpoints"]
|
|
_require(type(checks) is list and len(checks) >= 2 and
|
|
all(type(v) is int and v in trial_plan.PROBE_CHECKPOINTS for v in checks) and
|
|
checks == sorted(set(checks)) and checks[0] == 0 and checks[-1] == 50,
|
|
"Explicit reversal checkpoints must include 0/50, sorted, unique, and within the draft curve")
|
|
_require(config["baseline_execution"] == "execute_each_snapshot",
|
|
"Declare execute_each_snapshot explicitly; baseline reuse is not implemented or inferred")
|
|
_require(config["snapshot_binding_policy"] == "explicit_completed_epochs",
|
|
"Explicit control snapshot epoch bindings are mandatory")
|
|
_require(type(config["snapshot_bindings"]) is list, "snapshot_bindings must be an explicit ordered list")
|
|
return json.loads(_canonical(config))
|
|
|
|
|
|
def _binding_inventory(plan, config):
|
|
expected, specs = [], []
|
|
for block in plan["blocks"]:
|
|
for suite in block["suites"]:
|
|
assay = suite["assay"]
|
|
points = config["reversal_checkpoints"] if assay == "reversal" else block["evaluation_checkpoints"]
|
|
for arm in suite["arms"]:
|
|
for point in points:
|
|
expected.append((block["block_id"], assay, arm["arm"], point))
|
|
specs.append((block, suite, arm))
|
|
bindings = config["snapshot_bindings"]
|
|
_require(len(bindings) == len(expected), "Missing/extra snapshot bindings: require every block, arm and checkpoint")
|
|
previous = {}
|
|
result = []
|
|
for binding, identity, spec in zip(bindings, expected, specs):
|
|
_require(type(binding) is dict and set(binding) == BINDING_FIELDS, "Invalid snapshot binding fields")
|
|
actual = tuple(binding[key] for key in ("block_id", "assay", "arm", "checkpoint"))
|
|
_require(all(type(v) is str for v in actual[:3]) and type(actual[3]) is int and actual == identity,
|
|
"Snapshot bindings must match exact canonical coverage/order/types")
|
|
block, suite, arm = spec
|
|
count = binding["completed_epochs"]
|
|
_require(type(count) is int and 0 <= count <= len(arm["epochs"]), "Invalid completed_epochs")
|
|
point = identity[-1]
|
|
_require((count == 0) == (point == 0), "Checkpoint zero is before the arm's first epoch")
|
|
_require(point != 50 or count == len(arm["epochs"]), "Terminal checkpoint must include the entire arm, including neutral/blank epochs")
|
|
key = identity[:3]
|
|
_require(key not in previous or count > previous[key], "Snapshot epochs must advance strictly")
|
|
previous[key] = count
|
|
us = sum(e["us"]["requested"] for e in arm["epochs"][:count])
|
|
updates = sum(e["update_enabled"] and e["D_command"] != 0 for e in arm["epochs"][:count])
|
|
if arm["arm"] in ("paired", "reversal", "no_reversal"):
|
|
_require(us == point and updates == point,
|
|
"Paired/reversal checkpoint must contain exactly its declared reinforcement opportunities")
|
|
result.append((binding, block, suite, arm, us, updates))
|
|
return result
|
|
|
|
|
|
def _streams(trial):
|
|
"""Reconstruct evaluation semantic keys, not just plausible seed strings."""
|
|
manifest = trial["seed_manifest"]
|
|
master = trial["plan"]["master"]
|
|
independent = {(r["split"], tuple(r["key"])): r for r in manifest["independent_streams"]}
|
|
non_eval = {r["seed"] for r in manifest["independent_streams"] if r["split"] != "evaluation"}
|
|
used, result = set(), {}
|
|
for block in trial["plan"]["blocks"]:
|
|
_require([v["cue"] for v in block["evaluation_streams"]] == ["A", "B"], "Both original evaluation cues are required")
|
|
for row in block["evaluation_streams"]:
|
|
cue = row["cue"]
|
|
key = (block["block_id"], "evaluation", "CS", cue, "00", "base-probe")
|
|
seed, derivation = seed_registry.seed_for(master, "evaluation", key)
|
|
registered = independent.get(("evaluation", key))
|
|
_require(registered == {"split": "evaluation", "key": list(key), "seed": seed,
|
|
"derivation_sha256": derivation}, "Missing exact evaluation registry key")
|
|
stream_id = "evaluation:" + _sha([trial_plan.SCHEMA, "stream-id", master, "evaluation", list(key)])
|
|
_require(type(row["seed"]) is int and row["seed"] == seed and row["seed_bits"] == 63 and
|
|
row["stream_id"] == stream_id and row["odor"] == trial_plan.ODORS[cue] and row["role"] == "CS",
|
|
"Evaluation seed/key/odor lineage mismatch")
|
|
_require(seed not in used and seed not in non_eval, "Evaluation seed overlaps another independent stream")
|
|
used.add(seed)
|
|
result[(block["block_id"], cue)] = dict(stream_id=stream_id, split="evaluation",
|
|
semantic_key=list(key), seed=seed, seed_bits=63, derivation_sha256=derivation,
|
|
input_id=block["block_id"] + "/evaluation/input-" + cue,
|
|
reuse="paired repeated probes within this training replicate; not additional independent seeds")
|
|
_require(len(result) == manifest["split_counts"]["evaluation"] == 80, "Exactly the existing 80 evaluation streams are required")
|
|
return result
|
|
|
|
|
|
def _snapshot_id(block_id, assay, arm, checkpoint):
|
|
return f"{block_id}/{assay}/{arm}/checkpoint-{checkpoint:02d}"
|
|
|
|
|
|
def _assemble(trial, config):
|
|
"""Internal assembly; public entry points first validate full trial authority."""
|
|
config = _configuration(config)
|
|
records = _binding_inventory(trial["plan"], config)
|
|
streams = _streams(trial)
|
|
snapshots, probes = [], []
|
|
|
|
def emit(block, snapshot):
|
|
common = dict(block_id=block["block_id"], mode=block["mode"], dan_type=block["dan_type"],
|
|
compartment_verbatim=block["compartment_verbatim"], training_replicate=block["replicate"],
|
|
original_reinforced_cue=block["initial_reinforced_cue"], original_neutral_cue=block["initial_neutral_cue"],
|
|
original_S_orientation="original reinforced minus original neutral, never flipped for reversal")
|
|
snapshot = dict(common, **snapshot)
|
|
snapshots.append(snapshot)
|
|
for cue in trial_plan.CUES:
|
|
probes.append(dict(common, probe_id=snapshot["snapshot_id"] + "/probe-" + cue,
|
|
snapshot_id=snapshot["snapshot_id"], baseline_snapshot_id=snapshot["baseline_snapshot_id"],
|
|
assay=snapshot["assay"], arm=snapshot["arm"], checkpoint=snapshot["checkpoint"],
|
|
elapsed_idle_ticks=snapshot["elapsed_idle_ticks"], cue=cue, odor=trial_plan.ODORS[cue],
|
|
steps=config["probe_steps"], dt_ms=0.1, cs_start_tick=config["cs_start_tick"],
|
|
cs_stop_tick_exclusive=config["cs_stop_tick_exclusive"],
|
|
input=streams[(block["block_id"], cue)], execution="independent fork; discard probe terminal state",
|
|
US_requested=False, D_command=0.0, weight_updates_enabled=False,
|
|
feedback_to_training_or_idle=False, adds_training_replicate=False))
|
|
|
|
for binding, block, suite, arm, us, updates in records:
|
|
block_id, assay, name = block["block_id"], suite["assay"], arm["arm"]
|
|
point, count = binding["checkpoint"], binding["completed_epochs"]
|
|
last = None if count == 0 else arm["epochs"][count - 1]["epoch_id"]
|
|
delta_source = (dict(arm["delta_start"]) if count == 0 else
|
|
{"kind": "epoch_delta_after", "source": last + "/delta-after"})
|
|
baseline = _snapshot_id(block_id, "normal" if assay == "reversal" else assay,
|
|
"paired" if assay == "reversal" else name, 0)
|
|
emit(block, dict(snapshot_id=_snapshot_id(block_id, assay, name, point), assay=assay, arm=name,
|
|
checkpoint=point, checkpoint_semantics="nominal reinforcement progress; control epoch boundary explicitly supplied",
|
|
completed_epochs=count, last_completed_epoch_id=last,
|
|
actual_US_requests_completed=us, enabled_nonzero_D_update_opportunities_completed=updates,
|
|
arm_reinforced_cue=arm["reinforced_cue"], delta_source=delta_source,
|
|
neural_state_source={"kind": "common_frozen_reset", "from_training_terminal_state": False},
|
|
baseline_snapshot_id=baseline, elapsed_idle_ticks=None,
|
|
probe_count=2, snapshot_mutated_by_planning=False))
|
|
|
|
for block in trial["plan"]["blocks"]:
|
|
block_id = block["block_id"]
|
|
normal = {a["arm"]: a for a in block["suites"][0]["arms"]}
|
|
for branch in block["retention"]["branches"]:
|
|
name = branch.split("/")[1]
|
|
terminal_epoch = normal[name]["epochs"][-1]["epoch_id"]
|
|
lineage = block_id + "/retention/" + name + "/idle-10s"
|
|
for seconds in block["retention"]["checkpoint_seconds"]:
|
|
ticks = seconds * 10000
|
|
emit(block, dict(snapshot_id=f"{block_id}/retention/{name}/idle-{seconds:02d}s",
|
|
assay="retention", arm=name, checkpoint=None, completed_epochs=len(normal[name]["epochs"]),
|
|
last_completed_epoch_id=terminal_epoch, delta_source={"kind": "epoch_delta_after", "source": terminal_epoch + "/delta-after"},
|
|
neural_state_source={"kind": "actual_terminal_checkpoint" if seconds == 0 else "continuous_idle_checkpoint",
|
|
"terminal_epoch_id": terminal_epoch, "idle_lineage_id": lineage,
|
|
"elapsed_ticks": ticks, "total_idle_ticks": 100000,
|
|
"one_second_is_prefix_of_same_ten_second_idle": True,
|
|
"carry": ["v", "g", "ref", "spike_ring", "apl_state", "apl_pending", "absolute_tick", "delta", "relevant_rng_state"],
|
|
"reset_or_analytic_time_advance": False},
|
|
baseline_snapshot_id=_snapshot_id(block_id, "normal", name, 0),
|
|
elapsed_idle_ticks=ticks, probe_count=2, snapshot_mutated_by_planning=False))
|
|
|
|
ids = {row["snapshot_id"] for row in snapshots}
|
|
_require(len(ids) == len(snapshots) and all(p["baseline_snapshot_id"] in ids for p in probes),
|
|
"Duplicate snapshot or missing original pre baseline")
|
|
by_assay = Counter(p["assay"] for p in probes)
|
|
steps = len(probes) * config["probe_steps"]
|
|
result = dict(schema=SCHEMA, status="DRAFT-NOT-REGISTERED", config=config,
|
|
trial_payload_sha256=trial["payload_sha256"], seed_manifest_sha256=trial["seed_manifest"]["payload_sha256"],
|
|
config_sha256=_sha(config), snapshots=snapshots, probes=probes,
|
|
budget=dict(snapshot_count=len(snapshots), evaluation_probes=len(probes),
|
|
probes_by_assay=dict(sorted(by_assay.items())), evaluation_steps=steps,
|
|
evaluation_biological_seconds=steps / 10000,
|
|
preexisting_training_control_reversal_epochs=trial["budget"]["training_control_reversal_epochs"],
|
|
preexisting_minimum_steps_excluding_evaluation=trial["budget"]["minimum_steps_excluding_evaluation"],
|
|
planned_scientific_steps_including_evaluation=trial["budget"]["minimum_steps_excluding_evaluation"] + steps,
|
|
planned_scientific_biological_seconds=(trial["budget"]["minimum_steps_excluding_evaluation"] + steps) / 10000,
|
|
independent_evaluation_inputs=80, independent_training_blocks=40,
|
|
evaluation_probes_are_training_replicates=False, baseline_output_deduplication=False,
|
|
scope="conditional full draft inventory, not an executed count or wall-time/storage estimate",
|
|
excludes=["implementation validation", "frozen regressions", "encoding", "I/O", "resource reserve"]),
|
|
evaluation_inventory_structurally_complete=True, scientific_gate_added=False,
|
|
protocol_approval_claimed=False, execution_authorized=False, training_executed=False,
|
|
inputs_generated=False, input_generator_implementation_verified=False, candidate_generated=False,
|
|
learning_pass_claimed=False, external_source_and_snapshot_authority_verified=False,
|
|
subsequent_assays_condition="not authorized here; full planned inventory remains subject to the draft's acquisition gate")
|
|
result["payload_sha256"] = _sha(result)
|
|
# No returned nested object aliases caller-owned trial/config structures.
|
|
return json.loads(_canonical(result))
|
|
|
|
|
|
def build_evaluation_plan(trial_payload, config):
|
|
"""Build a complete draft inventory, with mandatory explicit choices."""
|
|
_configuration(config) # Report missing choices before expensive reconstruction.
|
|
trial_plan.validate_trial_plan(trial_payload)
|
|
return _assemble(trial_payload, config)
|
|
|
|
|
|
def validate_evaluation_plan(payload, trial_payload, config):
|
|
"""Reconstruct every probe/binding; an updated self-hash cannot excuse drift."""
|
|
_require(type(payload) is dict and "payload_sha256" in payload, "Invalid evaluation payload")
|
|
_require(payload["payload_sha256"] == _sha({k: v for k, v in payload.items() if k != "payload_sha256"}),
|
|
"Evaluation payload SHA mismatch")
|
|
expected = build_evaluation_plan(trial_payload, config)
|
|
_require(_canonical(payload) == _canonical(expected), "Evaluation coverage, binding, type, seed, or order differs from exact reconstruction")
|
|
return True
|