"""Pure, UNREGISTERED evaluation inventory; no files, RNG draws or execution. build_evaluation_plan(trial_payload, config) requires the complete existing trial_plan payload, including its caller-chosen master and seed manifest. It reuses that payload's 80 evaluation inputs; repeated probes are paired observations, never extra independent training replicates. No defaults select CS offset, reversal curve checkpoints, control snapshot boundaries, or baseline execution policy. See required_choices(). A complete inventory is not approval, an executed snapshot, an externally verified source pin, or a scientific gate. """ from __future__ import annotations from collections import Counter import hashlib import json import seed_registry import trial_plan SCHEMA = "phase4an-evaluation-plan-draft-v1" CONFIG_SCHEMA = "phase4an-evaluation-config-draft-v1" CONFIG_FIELDS = frozenset(("schema", "probe_steps", "cs_start_tick", "cs_stop_tick_exclusive", "reversal_checkpoints", "baseline_execution", "snapshot_binding_policy", "snapshot_bindings")) BINDING_FIELDS = frozenset(("block_id", "assay", "arm", "checkpoint", "completed_epochs")) def _require(condition, message): if not condition: raise ValueError(message) def _canonical(value): return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False) def _sha(value): return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() def required_choices(): """Describe mandatory choices, without filling or approving any of them.""" return { "probe_steps": "Explicit integer 1200: the draft's fixed 120 ms metric window.", "cs_start_tick": "Explicit integer 0: stimulus onset defines probe latency.", "cs_stop_tick_exclusive": "Explicit integer in [1,1200]; draft does not settle evaluation CS offset.", "reversal_checkpoints": "Explicit sorted subset of [0,1,2,5,10,20,50], including 0 and 50.", "baseline_execution": "Only execute_each_snapshot is implemented; declare it explicitly. No output deduplication.", "snapshot_binding_policy": "Explicit completed_epochs; nominal control labels are not actual update counts.", "snapshot_bindings": "Every block/assay/arm/checkpoint, in trial-plan order, with explicit completed_epochs. No control alignment is guessed.", "authority": "Provide the complete existing trial payload; no master seed is chosen here. Source/input pins remain coordinator responsibilities.", } def _configuration(config): _require(type(config) is dict, "Explicit evaluation config is required; see required_choices()") _require(set(config) == CONFIG_FIELDS, "Evaluation config fields: missing=" + str(sorted(CONFIG_FIELDS - set(config))) + "; unknown=" + str(sorted(set(config) - CONFIG_FIELDS))) _require(config["schema"] == CONFIG_SCHEMA, "Wrong evaluation config schema") _require(type(config["probe_steps"]) is int and config["probe_steps"] == 1200, "Explicit probe_steps must be integer 1200") _require(type(config["cs_start_tick"]) is int and config["cs_start_tick"] == 0, "Explicit CS onset must be integer tick 0") stop = config["cs_stop_tick_exclusive"] _require(type(stop) is int and 0 < stop <= 1200, "Evaluation CS stop tick is required; no training-CS default is inferred") checks = config["reversal_checkpoints"] _require(type(checks) is list and len(checks) >= 2 and all(type(v) is int and v in trial_plan.PROBE_CHECKPOINTS for v in checks) and checks == sorted(set(checks)) and checks[0] == 0 and checks[-1] == 50, "Explicit reversal checkpoints must include 0/50, sorted, unique, and within the draft curve") _require(config["baseline_execution"] == "execute_each_snapshot", "Declare execute_each_snapshot explicitly; baseline reuse is not implemented or inferred") _require(config["snapshot_binding_policy"] == "explicit_completed_epochs", "Explicit control snapshot epoch bindings are mandatory") _require(type(config["snapshot_bindings"]) is list, "snapshot_bindings must be an explicit ordered list") return json.loads(_canonical(config)) def _binding_inventory(plan, config): expected, specs = [], [] for block in plan["blocks"]: for suite in block["suites"]: assay = suite["assay"] points = config["reversal_checkpoints"] if assay == "reversal" else block["evaluation_checkpoints"] for arm in suite["arms"]: for point in points: expected.append((block["block_id"], assay, arm["arm"], point)) specs.append((block, suite, arm)) bindings = config["snapshot_bindings"] _require(len(bindings) == len(expected), "Missing/extra snapshot bindings: require every block, arm and checkpoint") previous = {} result = [] for binding, identity, spec in zip(bindings, expected, specs): _require(type(binding) is dict and set(binding) == BINDING_FIELDS, "Invalid snapshot binding fields") actual = tuple(binding[key] for key in ("block_id", "assay", "arm", "checkpoint")) _require(all(type(v) is str for v in actual[:3]) and type(actual[3]) is int and actual == identity, "Snapshot bindings must match exact canonical coverage/order/types") block, suite, arm = spec count = binding["completed_epochs"] _require(type(count) is int and 0 <= count <= len(arm["epochs"]), "Invalid completed_epochs") point = identity[-1] _require((count == 0) == (point == 0), "Checkpoint zero is before the arm's first epoch") _require(point != 50 or count == len(arm["epochs"]), "Terminal checkpoint must include the entire arm, including neutral/blank epochs") key = identity[:3] _require(key not in previous or count > previous[key], "Snapshot epochs must advance strictly") previous[key] = count us = sum(e["us"]["requested"] for e in arm["epochs"][:count]) updates = sum(e["update_enabled"] and e["D_command"] != 0 for e in arm["epochs"][:count]) if arm["arm"] in ("paired", "reversal", "no_reversal"): _require(us == point and updates == point, "Paired/reversal checkpoint must contain exactly its declared reinforcement opportunities") result.append((binding, block, suite, arm, us, updates)) return result def _streams(trial): """Reconstruct evaluation semantic keys, not just plausible seed strings.""" manifest = trial["seed_manifest"] master = trial["plan"]["master"] independent = {(r["split"], tuple(r["key"])): r for r in manifest["independent_streams"]} non_eval = {r["seed"] for r in manifest["independent_streams"] if r["split"] != "evaluation"} used, result = set(), {} for block in trial["plan"]["blocks"]: _require([v["cue"] for v in block["evaluation_streams"]] == ["A", "B"], "Both original evaluation cues are required") for row in block["evaluation_streams"]: cue = row["cue"] key = (block["block_id"], "evaluation", "CS", cue, "00", "base-probe") seed, derivation = seed_registry.seed_for(master, "evaluation", key) registered = independent.get(("evaluation", key)) _require(registered == {"split": "evaluation", "key": list(key), "seed": seed, "derivation_sha256": derivation}, "Missing exact evaluation registry key") stream_id = "evaluation:" + _sha([trial_plan.SCHEMA, "stream-id", master, "evaluation", list(key)]) _require(type(row["seed"]) is int and row["seed"] == seed and row["seed_bits"] == 63 and row["stream_id"] == stream_id and row["odor"] == trial_plan.ODORS[cue] and row["role"] == "CS", "Evaluation seed/key/odor lineage mismatch") _require(seed not in used and seed not in non_eval, "Evaluation seed overlaps another independent stream") used.add(seed) result[(block["block_id"], cue)] = dict(stream_id=stream_id, split="evaluation", semantic_key=list(key), seed=seed, seed_bits=63, derivation_sha256=derivation, input_id=block["block_id"] + "/evaluation/input-" + cue, reuse="paired repeated probes within this training replicate; not additional independent seeds") _require(len(result) == manifest["split_counts"]["evaluation"] == 80, "Exactly the existing 80 evaluation streams are required") return result def _snapshot_id(block_id, assay, arm, checkpoint): return f"{block_id}/{assay}/{arm}/checkpoint-{checkpoint:02d}" def _assemble(trial, config): """Internal assembly; public entry points first validate full trial authority.""" config = _configuration(config) records = _binding_inventory(trial["plan"], config) streams = _streams(trial) snapshots, probes = [], [] def emit(block, snapshot): common = dict(block_id=block["block_id"], mode=block["mode"], dan_type=block["dan_type"], compartment_verbatim=block["compartment_verbatim"], training_replicate=block["replicate"], original_reinforced_cue=block["initial_reinforced_cue"], original_neutral_cue=block["initial_neutral_cue"], original_S_orientation="original reinforced minus original neutral, never flipped for reversal") snapshot = dict(common, **snapshot) snapshots.append(snapshot) for cue in trial_plan.CUES: probes.append(dict(common, probe_id=snapshot["snapshot_id"] + "/probe-" + cue, snapshot_id=snapshot["snapshot_id"], baseline_snapshot_id=snapshot["baseline_snapshot_id"], assay=snapshot["assay"], arm=snapshot["arm"], checkpoint=snapshot["checkpoint"], elapsed_idle_ticks=snapshot["elapsed_idle_ticks"], cue=cue, odor=trial_plan.ODORS[cue], steps=config["probe_steps"], dt_ms=0.1, cs_start_tick=config["cs_start_tick"], cs_stop_tick_exclusive=config["cs_stop_tick_exclusive"], input=streams[(block["block_id"], cue)], execution="independent fork; discard probe terminal state", US_requested=False, D_command=0.0, weight_updates_enabled=False, feedback_to_training_or_idle=False, adds_training_replicate=False)) for binding, block, suite, arm, us, updates in records: block_id, assay, name = block["block_id"], suite["assay"], arm["arm"] point, count = binding["checkpoint"], binding["completed_epochs"] last = None if count == 0 else arm["epochs"][count - 1]["epoch_id"] delta_source = (dict(arm["delta_start"]) if count == 0 else {"kind": "epoch_delta_after", "source": last + "/delta-after"}) baseline = _snapshot_id(block_id, "normal" if assay == "reversal" else assay, "paired" if assay == "reversal" else name, 0) emit(block, dict(snapshot_id=_snapshot_id(block_id, assay, name, point), assay=assay, arm=name, checkpoint=point, checkpoint_semantics="nominal reinforcement progress; control epoch boundary explicitly supplied", completed_epochs=count, last_completed_epoch_id=last, actual_US_requests_completed=us, enabled_nonzero_D_update_opportunities_completed=updates, arm_reinforced_cue=arm["reinforced_cue"], delta_source=delta_source, neural_state_source={"kind": "common_frozen_reset", "from_training_terminal_state": False}, baseline_snapshot_id=baseline, elapsed_idle_ticks=None, probe_count=2, snapshot_mutated_by_planning=False)) for block in trial["plan"]["blocks"]: block_id = block["block_id"] normal = {a["arm"]: a for a in block["suites"][0]["arms"]} for branch in block["retention"]["branches"]: name = branch.split("/")[1] terminal_epoch = normal[name]["epochs"][-1]["epoch_id"] lineage = block_id + "/retention/" + name + "/idle-10s" for seconds in block["retention"]["checkpoint_seconds"]: ticks = seconds * 10000 emit(block, dict(snapshot_id=f"{block_id}/retention/{name}/idle-{seconds:02d}s", assay="retention", arm=name, checkpoint=None, completed_epochs=len(normal[name]["epochs"]), last_completed_epoch_id=terminal_epoch, delta_source={"kind": "epoch_delta_after", "source": terminal_epoch + "/delta-after"}, neural_state_source={"kind": "actual_terminal_checkpoint" if seconds == 0 else "continuous_idle_checkpoint", "terminal_epoch_id": terminal_epoch, "idle_lineage_id": lineage, "elapsed_ticks": ticks, "total_idle_ticks": 100000, "one_second_is_prefix_of_same_ten_second_idle": True, "carry": ["v", "g", "ref", "spike_ring", "apl_state", "apl_pending", "absolute_tick", "delta", "relevant_rng_state"], "reset_or_analytic_time_advance": False}, baseline_snapshot_id=_snapshot_id(block_id, "normal", name, 0), elapsed_idle_ticks=ticks, probe_count=2, snapshot_mutated_by_planning=False)) ids = {row["snapshot_id"] for row in snapshots} _require(len(ids) == len(snapshots) and all(p["baseline_snapshot_id"] in ids for p in probes), "Duplicate snapshot or missing original pre baseline") by_assay = Counter(p["assay"] for p in probes) steps = len(probes) * config["probe_steps"] result = dict(schema=SCHEMA, status="DRAFT-NOT-REGISTERED", config=config, trial_payload_sha256=trial["payload_sha256"], seed_manifest_sha256=trial["seed_manifest"]["payload_sha256"], config_sha256=_sha(config), snapshots=snapshots, probes=probes, budget=dict(snapshot_count=len(snapshots), evaluation_probes=len(probes), probes_by_assay=dict(sorted(by_assay.items())), evaluation_steps=steps, evaluation_biological_seconds=steps / 10000, preexisting_training_control_reversal_epochs=trial["budget"]["training_control_reversal_epochs"], preexisting_minimum_steps_excluding_evaluation=trial["budget"]["minimum_steps_excluding_evaluation"], planned_scientific_steps_including_evaluation=trial["budget"]["minimum_steps_excluding_evaluation"] + steps, planned_scientific_biological_seconds=(trial["budget"]["minimum_steps_excluding_evaluation"] + steps) / 10000, independent_evaluation_inputs=80, independent_training_blocks=40, evaluation_probes_are_training_replicates=False, baseline_output_deduplication=False, scope="conditional full draft inventory, not an executed count or wall-time/storage estimate", excludes=["implementation validation", "frozen regressions", "encoding", "I/O", "resource reserve"]), evaluation_inventory_structurally_complete=True, scientific_gate_added=False, protocol_approval_claimed=False, execution_authorized=False, training_executed=False, inputs_generated=False, input_generator_implementation_verified=False, candidate_generated=False, learning_pass_claimed=False, external_source_and_snapshot_authority_verified=False, subsequent_assays_condition="not authorized here; full planned inventory remains subject to the draft's acquisition gate") result["payload_sha256"] = _sha(result) # No returned nested object aliases caller-owned trial/config structures. return json.loads(_canonical(result)) def build_evaluation_plan(trial_payload, config): """Build a complete draft inventory, with mandatory explicit choices.""" _configuration(config) # Report missing choices before expensive reconstruction. trial_plan.validate_trial_plan(trial_payload) return _assemble(trial_payload, config) def validate_evaluation_plan(payload, trial_payload, config): """Reconstruct every probe/binding; an updated self-hash cannot excuse drift.""" _require(type(payload) is dict and "payload_sha256" in payload, "Invalid evaluation payload") _require(payload["payload_sha256"] == _sha({k: v for k, v in payload.items() if k != "payload_sha256"}), "Evaluation payload SHA mismatch") expected = build_evaluation_plan(trial_payload, config) _require(_canonical(payload) == _canonical(expected), "Evaluation coverage, binding, type, seed, or order differs from exact reconstruction") return True