"""Frozen-D18 validation cohorts; no simulation, fitting, or old heldout read. Only the requested cohort's new trajectories are opened. The suite owns the seed-validation -> novel-to-AJ unlock. Legacy modules are source-hash pinned, loaded without writing bytecode, and never have their globals rebound. """ from __future__ import annotations import hashlib import json from pathlib import Path import re import sys import types import numpy as np AG_SHA256 = "edb9cb2d890471fd07afd88c959ef3540b0cd40af490fac3f26396b5e7f5bf9c" AJ_SHA256 = "fb15ddeaa108305c8de697bbae36b23ab0414a7c58db51c66dce9feb4a048a0d" PANEL_SHA256 = "907f16a56e024a8e2112b5751e53b0e6037b329fa110098af3d31b119c10a1a0" COHORTS = ("seed_validation", "novel_to_aj") SPLITS = dict(zip(COHORTS, ("calibration", "heldout"))) MODES = ("physiological", "matched_energy") MEDIAN = 2.556107997894287 _DEFAULT_OUTPUT = object() def require(ok, message): if not ok: raise ValueError(message) def read(path): return json.loads(Path(path).read_text(encoding="utf-8")) def digest(path): h = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(8 << 20), b""): h.update(block) return h.hexdigest() def load_frozen(path, expected, name): path = Path(path) source = path.read_bytes() require(hashlib.sha256(source).hexdigest() == expected, "Frozen source SHA mismatch: " + str(path)) module = types.ModuleType(name) module.__file__ = str(path) sys.modules[name] = module exec(compile(source, str(path), "exec"), module.__dict__) return module def validate_design(cfg): design = cfg["validation"] seeds = design["seeds"] require(isinstance(seeds, list) and len(seeds) == 5 and len(set(seeds)) == 5 and all(type(seed) is int and 0 <= seed < 2**63 for seed in seeds), "Need five explicitly audited seed integers") require(design["cohorts"] == list(COHORTS) and design["modes"] == list(MODES) and design["odor_count"] == 6 and design["duration_ms"] == 100.0 and design["repeat_first_seed_each_odor"] is True, "Validation design changed") require(type(cfg["apl_delay_steps"]) is int and cfg["apl_delay_steps"] == 18 and cfg["neurons"] == 166700 and cfg["edges"] == 25582938 and cfg["training"] is False, "Frozen D18/fullgraph/nonlearning scope changed") gates = cfg["gates"] require(gates["sparse_min_fraction"] == .03 and gates["sparse_max_fraction"] == .09 and gates["max_fraction_strict"] == .5 and gates["min_fraction_strict"] == 0 and gates["minimum_separable_odor_pairs"] == 3 and all(gates[key] is True for key in ("same_jaccard_gt_between", "between_euclidean_gt_within", "both_modes_required", "finite_required", "repeat_exact_required")), "Original complete representation/integrity gates changed") return seeds def validate_inventory(records, cfg, panel_rows, legacy): """Static 120-row inventory, preserving original panel row order in each half.""" seeds = validate_design(cfg) require(isinstance(records, list) and len(records) == 120, "Inventory must contain exactly 120 base cases") require(len(panel_rows) == 12 and [row["split"] for row in panel_rows] == ["calibration"] * 6 + ["heldout"] * 6, "Frozen static panel order/splits changed") all_ids = [row["odor_id"] for row in panel_rows] require(all(isinstance(oid, str) and oid for oid in all_ids) and len(set(all_ids)) == 12 and set(all_ids[:6]) == legacy.ORIGINAL_CAL_IDS, "Frozen panel identities invalid") require(float(np.median([row["mapped_response_sum"] for row in panel_rows])) == MEDIAN, "Matched-energy reference must remain the original full-12 median") result = {} names, keys = set(), set() for row in records: require(row["cohort"] in COHORTS and row["mode"] in MODES, "Unknown cohort/mode") require(type(row["odor_index"]) is int and 0 <= row["odor_index"] < 6 and type(row["seed"]) is int and row["seed"] in seeds, "Invalid odor index/seed") require(isinstance(row["name"], str) and re.fullmatch(r"[A-Za-z0-9_-]+", row["name"]), "Unsafe case name") namekey = (row["cohort"], row["name"]) key = (row["cohort"], row["mode"], row["odor_index"], row["seed"]) require(namekey not in names and key not in keys, "Duplicate case name or experimental cell") names.add(namekey); keys.add(key) offset = 0 if row["cohort"] == "seed_validation" else 6 require(row["source_panel_split"] == SPLITS[row["cohort"]] and row["odor_id"] == all_ids[offset + row["odor_index"]], "Cohort identity/order/source split mismatch") if "source_panel_row" in row: require(type(row["source_panel_row"]) is int and row["source_panel_row"] == offset + row["odor_index"], "Inventory original panel row mismatch") require(type(row["targets"]) is int and 0 < row["targets"] <= cfg["neurons"] and type(row["requests"]) is int and row["requests"] >= 0, "Invalid inventory target/request counts") expected = {(cohort, mode, oi, seed) for cohort in COHORTS for mode in MODES for oi in range(6) for seed in seeds} require(keys == expected, "Inventory missing or replacing a predeclared cell") for index, cohort in enumerate(COHORTS): rows = [row for row in records if row["cohort"] == cohort] rows.sort(key=lambda row: (MODES.index(row["mode"]), row["odor_index"], seeds.index(row["seed"]))) result[cohort] = {"records": rows, "odor_ids": all_ids[index * 6:(index + 1) * 6]} return result def validate_encoder(encoder, inventory, cfg): require(encoder["panel_count"] == 12 and encoder["cal_count"] == 6 and encoder["novel_to_aj_count"] == 6 and encoder["median_mapped_response_sum"] == MEDIAN and encoder["seeds"] == cfg["validation"]["seeds"] and encoder["requested_before_any_dynamics"] is True and encoder["historical_never_simulated_claim"] is False, "Frozen encoder/reference or validation scope changed") require(isinstance(encoder["seeds_static_audit_sha256"], str) and re.fullmatch(r"[0-9a-f]{64}", encoder["seeds_static_audit_sha256"]), "Missing seed audit SHA") expected = {cohort: [dict(odor_id=oid, odor_index=index, source_panel_split=SPLITS[cohort]) for index, oid in enumerate(inventory[cohort]["odor_ids"])] for cohort in COHORTS} actual = encoder["cohorts_static_panel_rows"] require(set(actual) == set(COHORTS), "Encoder cohort set changed") for ci, cohort in enumerate(COHORTS): require(len(actual[cohort]) == 6, "Encoder cohort panel count changed") for index, (row, wanted) in enumerate(zip(actual[cohort], expected[cohort])): require(all(row.get(key) == value for key, value in wanted.items()), "Encoder cohort panel rows differ from frozen identities") if "source_panel_row" in row: require(type(row["source_panel_row"]) is int and row["source_panel_row"] == ci * 6 + index, "Encoder original panel row mismatch") def validate_case(case, row, cfg, frozen_spec): spec = read(case / "spec.json") require(spec["name"] == row["name"] and spec["kind"] == "fullgraph" and spec["validation_cohort"] == row["cohort"] and spec["source_panel_split"] == row["source_panel_split"] and all(spec[key] == row[key] for key in ("mode", "odor_id", "odor_index", "seed")), "Spec differs from validation inventory") require(type(spec["apl_delay_steps"]) is int and spec["apl_delay_steps"] == 18 and spec["steps"] == 1000 and spec["dt_ms"] == .1 and spec["duration_ms"] == 100.0 and spec["training"] is False and spec["record_history"] is False, "Fixed D18/window/nonlearning schema changed") if "source_panel_row" in row or "source_panel_row" in spec: expected_row = (0 if row["cohort"] == "seed_validation" else 6) + row["odor_index"] require(type(spec.get("source_panel_row")) is int and spec["source_panel_row"] == expected_row and row.get("source_panel_row", expected_row) == expected_row, "Spec original panel row mismatch") for key in ("model", "graph_dir", "apl_bundle_dir", "apl", "coefficients_path", "science_python", "legacy_apl_worker", "precision", "apl_precision", "dt_ms", "duration_ms", "steps"): require(spec[key] == frozen_spec[key], "Frozen physical field changed: " + key) with np.load(case / "pn-drive.npz", allow_pickle=False) as source: require(set(source.files) == {"targets", "rates_hz"}, "PN drive schema changed") targets, rates = source["targets"], source["rates_hz"] require(targets.dtype == np.int32 and targets.ndim == 1 and len(targets) == row["targets"] and len(np.unique(targets)) == len(targets) and np.all((targets >= 0) & (targets < cfg["neurons"])) and rates.dtype == np.float32 and rates.shape == targets.shape and np.isfinite(rates).all() and np.all(rates > 0), "Invalid PN drive identities/rates; no sorting or zero-channel insertion is allowed") expected_events = float(rates.sum()) * spec["steps"] * .1 / 1000. require(spec["target_count"] == row["targets"] and spec["rate_sum_hz"] == float(rates.sum()) and spec["expected_external_events"] == expected_events and np.isfinite(expected_events) and expected_events > 0, "PN rates/spec event expectation disagree") if "expected_external_events" in row: require(row["expected_external_events"] == expected_events, "Inventory expected events disagree") if "rates_sum" in row: require(row["rates_sum"] == float(rates.sum()), "Inventory rate sum disagrees") with np.load(case / "requests.npz", allow_pickle=False) as source: require(set(source.files) == {"tick", "neuron_idx"} and len(source["tick"]) == row["requests"] and np.isin(source["neuron_idx"], targets).all(), "Request count/targets differ from prepared input") return spec def read_validation_trial(legacy, timing, output, case, spec, kc_globals, cfg): """Reuse all sealed checks; replace only the returned descriptive record.""" cohort = spec["validation_cohort"] require(cohort in COHORTS and spec["source_panel_split"] == SPLITS[cohort] and type(spec["apl_delay_steps"]) is int and spec["apl_delay_steps"] == 18, "Wrong validation cohort or delay") trial = legacy.read_trial(output, case, spec, kc_globals, cfg["neurons"], cfg["edges"]) try: trial["timing"] = timing.validate_timing(output, spec, trial) except Exception: close_trial(trial) raise trial["record"] = dict(trial["record"], split=cohort, cohort=cohort, validation_cohort=cohort, source_panel_split=spec["source_panel_split"], historical_never_simulated_claim=False, used_for_parameter_selection=False) if "source_panel_row" in spec: trial["record"]["source_panel_row"] = spec["source_panel_row"] return trial def close_trial(trial): if trial is None: return arrays = list(trial.get("apl", {}).values()) arrays += [trial.get("timing", {}).get(key) for key in ("applied", "pending")] for value in arrays: mmap = getattr(value, "_mmap", None) if mmap is not None and not mmap.closed: mmap.close() def evidence_hashes(output, case, name, repeat): files = ("state.npz", "summary.json", "tick-state-hashes.jsonl", "queue.jsonl", "requests.csv", "apl-component.json", "apl-drive.float32.npy", "apl-state.float32.npy", "apl-inhibition.float32.npy", "apl-applied.float32.npy", "apl-pending-tail.float32.npy", "apl-timing-audit.json") return {"case": name, "repeat": repeat, "input_sha256": {key: digest(case / key) for key in ("spec.json", "requests.npz", "pn-drive.npz")}, "output_sha256": {key: digest(output / key) for key in files}} def collect_cohort(cohort, records, odors, cfg, exp, kc_globals, frozen_spec, legacy, timing): """Aggregate one block even when its representation gate fails.""" seeds = cfg["validation"]["seeds"] failures, runs, rows, repeats, timings, proof, modes = [], {}, [], [], [], [], {} for row in records: name, first, spec = row["name"], None, None case = exp / "cases" / cohort / name try: spec = validate_case(case, row, cfg, frozen_spec) except Exception as error: failures.append(dict(stage="case_input", case=name, type=type(error).__name__, message=str(error))) for repeat_index in ([0, 1] if row["seed"] == seeds[0] else [0]): trial = None try: require(spec is not None, "Case input validation unavailable") output = exp / "gpu" / cohort / (name + "-r" + str(repeat_index)) trial = read_validation_trial(legacy, timing, output, case, spec, kc_globals, cfg) require(trial["record"]["requested"] == row["requests"], "Inventory/actual request count differs") entry = evidence_hashes(output, case, name, repeat_index) proof.append(entry) timings.append(dict(case=name, repeat=repeat_index, **trial["timing"]["report"])) if repeat_index == 0: first = trial runs[(row["mode"], row["odor_id"], row["seed"])] = {key: trial[key] for key in ("kc_counts", "expected_external_events", "external_events")} rows.append(trial["record"]) else: require(first is not None, "Baseline unavailable for exact repeat") report = dict(case=name, **timing.timing_repeat(first, trial, legacy)) repeats.append(report) require(report["pass_all"] is True, "Exact repeat differs: " + ", ".join( key for key, value in report["checks"].items() if not value)) except Exception as error: failures.append(dict(stage="raw_trial" if repeat_index == 0 else "repeat_trial", case=name, repeat=repeat_index, type=type(error).__name__, message=str(error))) finally: if trial is not first: close_trial(trial) close_trial(first) for mode in MODES: expected = {(mode, odor, seed) for odor in odors for seed in seeds} if expected.issubset(runs): try: report = legacy.summarize_mode(mode, odors, seeds, runs, cfg["gates"]) require(report["conditions"] == legacy.evaluate_gates(report, cfg["gates"]), "Original gate conjunction mismatch") require(report["within_distance_count"] == 60 and report["between_distance_count"] == 75 and report["pair_count"] == 15 and all(pair["within_distance_count"] == 20 and pair["between_distance_count"] == 5 for pair in report["pairs"]), "Pair-specific denominator changed") modes[mode] = report if not report["gate"]: failures.append(dict(stage="representation_gate", mode=mode, failed_conditions=[key for key, value in report["conditions"].items() if not value])) except Exception as error: failures.append(dict(stage="representation_calculation", mode=mode, type=type(error).__name__, message=str(error))) numerical = bool(len(runs) == 60 and len(timings) == 72 and not any(row["stage"] in ("case_input", "raw_trial", "repeat_trial") for row in failures)) repeated = bool(len(repeats) == 12 and all(row["pass_all"] is True for row in repeats)) execution = bool(numerical and repeated and not any(row["stage"] == "representation_calculation" for row in failures)) represented = bool(len(modes) == 2 and all(row["gate"] is True for row in modes.values())) return dict(cohort=cohort, source_panel_split=SPLITS[cohort], pass_all=bool(execution and represented), execution_gate_pass=execution, numerical_integrity_pass=numerical, exact_repeat_pass=repeated, representation_gate_pass=represented, original_representation_gate_pass=represented, expected_base_trials=60, valid_base_trials=len(runs), expected_repeats=12, checked_repeats=len(repeats), checked_timing_trials=len(timings), modes=modes, runs=rows, repeats=repeats, timing_checks=timings, failures=failures, input_output_provenance=proof) def summarize(cohort, out_json=_DEFAULT_OUTPUT): """Default exclusive write; None recomputes deterministically without writes.""" require(cohort in COHORTS, "Unknown validation cohort") import context as ctx target = ctx.EXP / "validation" / cohort / "summary.json" if out_json is _DEFAULT_OUTPUT else ( None if out_json is None else Path(out_json)) require(target is None or not target.exists(), "Refusing existing cohort summary") result = dict(cohort=cohort, source_panel_split=SPLITS[cohort], pass_all=False, execution_gate_pass=False, numerical_integrity_pass=False, exact_repeat_pass=False, representation_gate_pass=False, original_representation_gate_pass=False, expected_base_trials=60, valid_base_trials=0, expected_repeats=12, checked_repeats=0, checked_timing_trials=0, modes={}, runs=[], repeats=[], timing_checks=[], failures=[], input_output_provenance=[]) proof, odors, encoder = {}, [], None try: ctx.check() legacy = load_frozen(ctx.AG_MODEL / "calibration.py", AG_SHA256, "phase4ak_frozen_ag_calibration") timing = load_frozen(ctx.AJ_MODEL / "calibration.py", AJ_SHA256, "phase4ak_frozen_aj_calibration") inventory_path = ctx.EXP / "protocol/validation-cases.json" encoder_path = ctx.EXP / "protocol/encoder-audit.json" panel_path = ctx.ROOT / "experiments/phase4l-olfactory-grounding/odor-encoder/frozen-odor-panel.parquet" require(digest(panel_path) == PANEL_SHA256, "Frozen static panel SHA changed") import pandas as pd panel = pd.read_parquet(panel_path).to_dict(orient="records") inventory = validate_inventory(read(inventory_path), ctx.CFG, panel, legacy) encoder = read(encoder_path) validate_encoder(encoder, inventory, ctx.CFG) seed_audit = ctx.MODEL / "seed-audit.json" require(digest(seed_audit) == encoder["seeds_static_audit_sha256"], "Seed-audit receipt/encoder mismatch") kc_path = ctx.AG / "apl-bundle/kc_globals.npy" kc = np.load(kc_path, allow_pickle=False) require(kc.dtype == np.int32 and kc.shape == (4064,) and np.all(np.diff(kc) > 0) and np.all((kc >= 0) & (kc < ctx.CFG["neurons"])), "Frozen KC identity invalid") base_path = ctx.AJ / "cases/delay18/cal-physiological-00-s51101/spec.json" base = read(base_path) require(base["apl_delay_steps"] == 18 and base["training"] is False, "Wrong frozen physical reference") proof = {"validation_inventory": digest(inventory_path), "encoder_audit": digest(encoder_path), "static_panel": digest(panel_path), "seed_audit": digest(seed_audit), "KC_identity": digest(kc_path), "frozen_D18_spec": digest(base_path), "AG_calibration_source": AG_SHA256, "AJ_calibration_source": AJ_SHA256} odors = inventory[cohort]["odor_ids"] result = collect_cohort(cohort, inventory[cohort]["records"], odors, ctx.CFG, ctx.EXP, kc, base, legacy, timing) except Exception as error: result["failures"].append(dict(stage="prerequisites", type=type(error).__name__, message=str(error))) result.update(phase="4AK", apl_delay_steps=18, seeds=ctx.CFG["validation"]["seeds"], duration_ms=100.0, odor_ids=odors, encoder_audit=encoder, prerequisite_input_sha256=proof, representation_failure_is_not_numerical_failure=True, old_module_globals_modified=False, old_heldout_dynamic_access=False, historical_never_simulated_claim=False, requested_cohort_only_dynamic_read=True, suite_owns_sequential_unlock=True, shared_apl_component_oracle_limit=True, independent_oracle_equivalence_claim=False, seed_namespace_claim="Declared static-audit scope only; not an unqualified all-history independence claim", biological_delay_measurement_claimed=False, candidate_generated=False, training=False, metric_source="SHA-pinned AG complete phase4l gate; SHA-pinned AJ timing/repeat checks") # Serializability/finiteness must be established even for read-only finalization. json.dumps(result, allow_nan=False, sort_keys=True) if target is not None: outputs = {target: result} outputs.update({target.parent / (mode + ".json"): report for mode, report in result["modes"].items()}) require(all(not path.exists() for path in outputs), "Refusing existing cohort/mode output") for path, value in outputs.items(): ctx.save(path, value) return result if __name__ == "__main__": require(len(sys.argv) == 2, "Usage: validation.py seed_validation|novel_to_aj") answer = summarize(sys.argv[1]) print(json.dumps({key: answer[key] for key in ("cohort", "pass_all", "execution_gate_pass", "representation_gate_pass")}))