Digital Drosophila Archive
345 lines
21 KiB
Python
345 lines
21 KiB
Python
"""Frozen-D18 validation cohorts; no simulation, fitting, or old heldout read.
|
|
|
|
Only the requested cohort's new trajectories are opened. The suite owns the
|
|
seed-validation -> novel-to-AJ unlock. Legacy modules are source-hash pinned,
|
|
loaded without writing bytecode, and never have their globals rebound.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
import types
|
|
|
|
import numpy as np
|
|
|
|
AG_SHA256 = "edb9cb2d890471fd07afd88c959ef3540b0cd40af490fac3f26396b5e7f5bf9c"
|
|
AJ_SHA256 = "fb15ddeaa108305c8de697bbae36b23ab0414a7c58db51c66dce9feb4a048a0d"
|
|
PANEL_SHA256 = "907f16a56e024a8e2112b5751e53b0e6037b329fa110098af3d31b119c10a1a0"
|
|
COHORTS = ("seed_validation", "novel_to_aj")
|
|
SPLITS = dict(zip(COHORTS, ("calibration", "heldout")))
|
|
MODES = ("physiological", "matched_energy")
|
|
MEDIAN = 2.556107997894287
|
|
_DEFAULT_OUTPUT = object()
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def read(path):
|
|
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open("rb") as stream:
|
|
for block in iter(lambda: stream.read(8 << 20), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def load_frozen(path, expected, name):
|
|
path = Path(path)
|
|
source = path.read_bytes()
|
|
require(hashlib.sha256(source).hexdigest() == expected, "Frozen source SHA mismatch: " + str(path))
|
|
module = types.ModuleType(name)
|
|
module.__file__ = str(path)
|
|
sys.modules[name] = module
|
|
exec(compile(source, str(path), "exec"), module.__dict__)
|
|
return module
|
|
|
|
|
|
def validate_design(cfg):
|
|
design = cfg["validation"]
|
|
seeds = design["seeds"]
|
|
require(isinstance(seeds, list) and len(seeds) == 5 and len(set(seeds)) == 5 and
|
|
all(type(seed) is int and 0 <= seed < 2**63 for seed in seeds), "Need five explicitly audited seed integers")
|
|
require(design["cohorts"] == list(COHORTS) and design["modes"] == list(MODES) and
|
|
design["odor_count"] == 6 and design["duration_ms"] == 100.0 and
|
|
design["repeat_first_seed_each_odor"] is True, "Validation design changed")
|
|
require(type(cfg["apl_delay_steps"]) is int and cfg["apl_delay_steps"] == 18 and
|
|
cfg["neurons"] == 166700 and cfg["edges"] == 25582938 and cfg["training"] is False,
|
|
"Frozen D18/fullgraph/nonlearning scope changed")
|
|
gates = cfg["gates"]
|
|
require(gates["sparse_min_fraction"] == .03 and gates["sparse_max_fraction"] == .09 and
|
|
gates["max_fraction_strict"] == .5 and gates["min_fraction_strict"] == 0 and
|
|
gates["minimum_separable_odor_pairs"] == 3 and all(gates[key] is True for key in
|
|
("same_jaccard_gt_between", "between_euclidean_gt_within", "both_modes_required",
|
|
"finite_required", "repeat_exact_required")), "Original complete representation/integrity gates changed")
|
|
return seeds
|
|
|
|
|
|
def validate_inventory(records, cfg, panel_rows, legacy):
|
|
"""Static 120-row inventory, preserving original panel row order in each half."""
|
|
seeds = validate_design(cfg)
|
|
require(isinstance(records, list) and len(records) == 120, "Inventory must contain exactly 120 base cases")
|
|
require(len(panel_rows) == 12 and [row["split"] for row in panel_rows] == ["calibration"] * 6 + ["heldout"] * 6,
|
|
"Frozen static panel order/splits changed")
|
|
all_ids = [row["odor_id"] for row in panel_rows]
|
|
require(all(isinstance(oid, str) and oid for oid in all_ids) and len(set(all_ids)) == 12 and
|
|
set(all_ids[:6]) == legacy.ORIGINAL_CAL_IDS, "Frozen panel identities invalid")
|
|
require(float(np.median([row["mapped_response_sum"] for row in panel_rows])) == MEDIAN,
|
|
"Matched-energy reference must remain the original full-12 median")
|
|
result = {}
|
|
names, keys = set(), set()
|
|
for row in records:
|
|
require(row["cohort"] in COHORTS and row["mode"] in MODES, "Unknown cohort/mode")
|
|
require(type(row["odor_index"]) is int and 0 <= row["odor_index"] < 6 and
|
|
type(row["seed"]) is int and row["seed"] in seeds, "Invalid odor index/seed")
|
|
require(isinstance(row["name"], str) and re.fullmatch(r"[A-Za-z0-9_-]+", row["name"]), "Unsafe case name")
|
|
namekey = (row["cohort"], row["name"])
|
|
key = (row["cohort"], row["mode"], row["odor_index"], row["seed"])
|
|
require(namekey not in names and key not in keys, "Duplicate case name or experimental cell")
|
|
names.add(namekey); keys.add(key)
|
|
offset = 0 if row["cohort"] == "seed_validation" else 6
|
|
require(row["source_panel_split"] == SPLITS[row["cohort"]] and
|
|
row["odor_id"] == all_ids[offset + row["odor_index"]], "Cohort identity/order/source split mismatch")
|
|
if "source_panel_row" in row:
|
|
require(type(row["source_panel_row"]) is int and row["source_panel_row"] == offset + row["odor_index"],
|
|
"Inventory original panel row mismatch")
|
|
require(type(row["targets"]) is int and 0 < row["targets"] <= cfg["neurons"] and
|
|
type(row["requests"]) is int and row["requests"] >= 0, "Invalid inventory target/request counts")
|
|
expected = {(cohort, mode, oi, seed) for cohort in COHORTS for mode in MODES for oi in range(6) for seed in seeds}
|
|
require(keys == expected, "Inventory missing or replacing a predeclared cell")
|
|
for index, cohort in enumerate(COHORTS):
|
|
rows = [row for row in records if row["cohort"] == cohort]
|
|
rows.sort(key=lambda row: (MODES.index(row["mode"]), row["odor_index"], seeds.index(row["seed"])))
|
|
result[cohort] = {"records": rows, "odor_ids": all_ids[index * 6:(index + 1) * 6]}
|
|
return result
|
|
|
|
|
|
def validate_encoder(encoder, inventory, cfg):
|
|
require(encoder["panel_count"] == 12 and encoder["cal_count"] == 6 and encoder["novel_to_aj_count"] == 6 and
|
|
encoder["median_mapped_response_sum"] == MEDIAN and encoder["seeds"] == cfg["validation"]["seeds"] and
|
|
encoder["requested_before_any_dynamics"] is True and encoder["historical_never_simulated_claim"] is False,
|
|
"Frozen encoder/reference or validation scope changed")
|
|
require(isinstance(encoder["seeds_static_audit_sha256"], str) and
|
|
re.fullmatch(r"[0-9a-f]{64}", encoder["seeds_static_audit_sha256"]), "Missing seed audit SHA")
|
|
expected = {cohort: [dict(odor_id=oid, odor_index=index, source_panel_split=SPLITS[cohort])
|
|
for index, oid in enumerate(inventory[cohort]["odor_ids"])] for cohort in COHORTS}
|
|
actual = encoder["cohorts_static_panel_rows"]
|
|
require(set(actual) == set(COHORTS), "Encoder cohort set changed")
|
|
for ci, cohort in enumerate(COHORTS):
|
|
require(len(actual[cohort]) == 6, "Encoder cohort panel count changed")
|
|
for index, (row, wanted) in enumerate(zip(actual[cohort], expected[cohort])):
|
|
require(all(row.get(key) == value for key, value in wanted.items()),
|
|
"Encoder cohort panel rows differ from frozen identities")
|
|
if "source_panel_row" in row:
|
|
require(type(row["source_panel_row"]) is int and row["source_panel_row"] == ci * 6 + index,
|
|
"Encoder original panel row mismatch")
|
|
|
|
|
|
def validate_case(case, row, cfg, frozen_spec):
|
|
spec = read(case / "spec.json")
|
|
require(spec["name"] == row["name"] and spec["kind"] == "fullgraph" and
|
|
spec["validation_cohort"] == row["cohort"] and spec["source_panel_split"] == row["source_panel_split"] and
|
|
all(spec[key] == row[key] for key in ("mode", "odor_id", "odor_index", "seed")), "Spec differs from validation inventory")
|
|
require(type(spec["apl_delay_steps"]) is int and spec["apl_delay_steps"] == 18 and spec["steps"] == 1000 and
|
|
spec["dt_ms"] == .1 and spec["duration_ms"] == 100.0 and spec["training"] is False and
|
|
spec["record_history"] is False, "Fixed D18/window/nonlearning schema changed")
|
|
if "source_panel_row" in row or "source_panel_row" in spec:
|
|
expected_row = (0 if row["cohort"] == "seed_validation" else 6) + row["odor_index"]
|
|
require(type(spec.get("source_panel_row")) is int and spec["source_panel_row"] == expected_row and
|
|
row.get("source_panel_row", expected_row) == expected_row, "Spec original panel row mismatch")
|
|
for key in ("model", "graph_dir", "apl_bundle_dir", "apl", "coefficients_path", "science_python",
|
|
"legacy_apl_worker", "precision", "apl_precision", "dt_ms", "duration_ms", "steps"):
|
|
require(spec[key] == frozen_spec[key], "Frozen physical field changed: " + key)
|
|
with np.load(case / "pn-drive.npz", allow_pickle=False) as source:
|
|
require(set(source.files) == {"targets", "rates_hz"}, "PN drive schema changed")
|
|
targets, rates = source["targets"], source["rates_hz"]
|
|
require(targets.dtype == np.int32 and targets.ndim == 1 and len(targets) == row["targets"] and
|
|
len(np.unique(targets)) == len(targets) and np.all((targets >= 0) & (targets < cfg["neurons"])) and
|
|
rates.dtype == np.float32 and rates.shape == targets.shape and np.isfinite(rates).all() and np.all(rates > 0),
|
|
"Invalid PN drive identities/rates; no sorting or zero-channel insertion is allowed")
|
|
expected_events = float(rates.sum()) * spec["steps"] * .1 / 1000.
|
|
require(spec["target_count"] == row["targets"] and spec["rate_sum_hz"] == float(rates.sum()) and
|
|
spec["expected_external_events"] == expected_events and np.isfinite(expected_events) and expected_events > 0,
|
|
"PN rates/spec event expectation disagree")
|
|
if "expected_external_events" in row:
|
|
require(row["expected_external_events"] == expected_events, "Inventory expected events disagree")
|
|
if "rates_sum" in row:
|
|
require(row["rates_sum"] == float(rates.sum()), "Inventory rate sum disagrees")
|
|
with np.load(case / "requests.npz", allow_pickle=False) as source:
|
|
require(set(source.files) == {"tick", "neuron_idx"} and len(source["tick"]) == row["requests"] and
|
|
np.isin(source["neuron_idx"], targets).all(), "Request count/targets differ from prepared input")
|
|
return spec
|
|
|
|
|
|
def read_validation_trial(legacy, timing, output, case, spec, kc_globals, cfg):
|
|
"""Reuse all sealed checks; replace only the returned descriptive record."""
|
|
cohort = spec["validation_cohort"]
|
|
require(cohort in COHORTS and spec["source_panel_split"] == SPLITS[cohort] and
|
|
type(spec["apl_delay_steps"]) is int and spec["apl_delay_steps"] == 18, "Wrong validation cohort or delay")
|
|
trial = legacy.read_trial(output, case, spec, kc_globals, cfg["neurons"], cfg["edges"])
|
|
try:
|
|
trial["timing"] = timing.validate_timing(output, spec, trial)
|
|
except Exception:
|
|
close_trial(trial)
|
|
raise
|
|
trial["record"] = dict(trial["record"], split=cohort, cohort=cohort, validation_cohort=cohort,
|
|
source_panel_split=spec["source_panel_split"], historical_never_simulated_claim=False,
|
|
used_for_parameter_selection=False)
|
|
if "source_panel_row" in spec:
|
|
trial["record"]["source_panel_row"] = spec["source_panel_row"]
|
|
return trial
|
|
|
|
|
|
def close_trial(trial):
|
|
if trial is None:
|
|
return
|
|
arrays = list(trial.get("apl", {}).values())
|
|
arrays += [trial.get("timing", {}).get(key) for key in ("applied", "pending")]
|
|
for value in arrays:
|
|
mmap = getattr(value, "_mmap", None)
|
|
if mmap is not None and not mmap.closed:
|
|
mmap.close()
|
|
|
|
|
|
def evidence_hashes(output, case, name, repeat):
|
|
files = ("state.npz", "summary.json", "tick-state-hashes.jsonl", "queue.jsonl", "requests.csv", "apl-component.json",
|
|
"apl-drive.float32.npy", "apl-state.float32.npy", "apl-inhibition.float32.npy",
|
|
"apl-applied.float32.npy", "apl-pending-tail.float32.npy", "apl-timing-audit.json")
|
|
return {"case": name, "repeat": repeat,
|
|
"input_sha256": {key: digest(case / key) for key in ("spec.json", "requests.npz", "pn-drive.npz")},
|
|
"output_sha256": {key: digest(output / key) for key in files}}
|
|
|
|
|
|
def collect_cohort(cohort, records, odors, cfg, exp, kc_globals, frozen_spec, legacy, timing):
|
|
"""Aggregate one block even when its representation gate fails."""
|
|
seeds = cfg["validation"]["seeds"]
|
|
failures, runs, rows, repeats, timings, proof, modes = [], {}, [], [], [], [], {}
|
|
for row in records:
|
|
name, first, spec = row["name"], None, None
|
|
case = exp / "cases" / cohort / name
|
|
try:
|
|
spec = validate_case(case, row, cfg, frozen_spec)
|
|
except Exception as error:
|
|
failures.append(dict(stage="case_input", case=name, type=type(error).__name__, message=str(error)))
|
|
for repeat_index in ([0, 1] if row["seed"] == seeds[0] else [0]):
|
|
trial = None
|
|
try:
|
|
require(spec is not None, "Case input validation unavailable")
|
|
output = exp / "gpu" / cohort / (name + "-r" + str(repeat_index))
|
|
trial = read_validation_trial(legacy, timing, output, case, spec, kc_globals, cfg)
|
|
require(trial["record"]["requested"] == row["requests"], "Inventory/actual request count differs")
|
|
entry = evidence_hashes(output, case, name, repeat_index)
|
|
proof.append(entry)
|
|
timings.append(dict(case=name, repeat=repeat_index, **trial["timing"]["report"]))
|
|
if repeat_index == 0:
|
|
first = trial
|
|
runs[(row["mode"], row["odor_id"], row["seed"])] = {key: trial[key] for key in
|
|
("kc_counts", "expected_external_events", "external_events")}
|
|
rows.append(trial["record"])
|
|
else:
|
|
require(first is not None, "Baseline unavailable for exact repeat")
|
|
report = dict(case=name, **timing.timing_repeat(first, trial, legacy))
|
|
repeats.append(report)
|
|
require(report["pass_all"] is True, "Exact repeat differs: " + ", ".join(
|
|
key for key, value in report["checks"].items() if not value))
|
|
except Exception as error:
|
|
failures.append(dict(stage="raw_trial" if repeat_index == 0 else "repeat_trial", case=name,
|
|
repeat=repeat_index, type=type(error).__name__, message=str(error)))
|
|
finally:
|
|
if trial is not first:
|
|
close_trial(trial)
|
|
close_trial(first)
|
|
for mode in MODES:
|
|
expected = {(mode, odor, seed) for odor in odors for seed in seeds}
|
|
if expected.issubset(runs):
|
|
try:
|
|
report = legacy.summarize_mode(mode, odors, seeds, runs, cfg["gates"])
|
|
require(report["conditions"] == legacy.evaluate_gates(report, cfg["gates"]), "Original gate conjunction mismatch")
|
|
require(report["within_distance_count"] == 60 and report["between_distance_count"] == 75 and
|
|
report["pair_count"] == 15 and all(pair["within_distance_count"] == 20 and
|
|
pair["between_distance_count"] == 5 for pair in report["pairs"]), "Pair-specific denominator changed")
|
|
modes[mode] = report
|
|
if not report["gate"]:
|
|
failures.append(dict(stage="representation_gate", mode=mode,
|
|
failed_conditions=[key for key, value in report["conditions"].items() if not value]))
|
|
except Exception as error:
|
|
failures.append(dict(stage="representation_calculation", mode=mode, type=type(error).__name__, message=str(error)))
|
|
numerical = bool(len(runs) == 60 and len(timings) == 72 and not any(row["stage"] in
|
|
("case_input", "raw_trial", "repeat_trial") for row in failures))
|
|
repeated = bool(len(repeats) == 12 and all(row["pass_all"] is True for row in repeats))
|
|
execution = bool(numerical and repeated and not any(row["stage"] == "representation_calculation" for row in failures))
|
|
represented = bool(len(modes) == 2 and all(row["gate"] is True for row in modes.values()))
|
|
return dict(cohort=cohort, source_panel_split=SPLITS[cohort], pass_all=bool(execution and represented),
|
|
execution_gate_pass=execution, numerical_integrity_pass=numerical, exact_repeat_pass=repeated,
|
|
representation_gate_pass=represented, original_representation_gate_pass=represented,
|
|
expected_base_trials=60, valid_base_trials=len(runs), expected_repeats=12, checked_repeats=len(repeats),
|
|
checked_timing_trials=len(timings), modes=modes, runs=rows, repeats=repeats, timing_checks=timings,
|
|
failures=failures, input_output_provenance=proof)
|
|
|
|
|
|
def summarize(cohort, out_json=_DEFAULT_OUTPUT):
|
|
"""Default exclusive write; None recomputes deterministically without writes."""
|
|
require(cohort in COHORTS, "Unknown validation cohort")
|
|
import context as ctx
|
|
target = ctx.EXP / "validation" / cohort / "summary.json" if out_json is _DEFAULT_OUTPUT else (
|
|
None if out_json is None else Path(out_json))
|
|
require(target is None or not target.exists(), "Refusing existing cohort summary")
|
|
result = dict(cohort=cohort, source_panel_split=SPLITS[cohort], pass_all=False, execution_gate_pass=False,
|
|
numerical_integrity_pass=False, exact_repeat_pass=False, representation_gate_pass=False,
|
|
original_representation_gate_pass=False, expected_base_trials=60, valid_base_trials=0,
|
|
expected_repeats=12, checked_repeats=0, checked_timing_trials=0,
|
|
modes={}, runs=[], repeats=[], timing_checks=[], failures=[], input_output_provenance=[])
|
|
proof, odors, encoder = {}, [], None
|
|
try:
|
|
ctx.check()
|
|
legacy = load_frozen(ctx.AG_MODEL / "calibration.py", AG_SHA256, "phase4ak_frozen_ag_calibration")
|
|
timing = load_frozen(ctx.AJ_MODEL / "calibration.py", AJ_SHA256, "phase4ak_frozen_aj_calibration")
|
|
inventory_path = ctx.EXP / "protocol/validation-cases.json"
|
|
encoder_path = ctx.EXP / "protocol/encoder-audit.json"
|
|
panel_path = ctx.ROOT / "experiments/phase4l-olfactory-grounding/odor-encoder/frozen-odor-panel.parquet"
|
|
require(digest(panel_path) == PANEL_SHA256, "Frozen static panel SHA changed")
|
|
import pandas as pd
|
|
panel = pd.read_parquet(panel_path).to_dict(orient="records")
|
|
inventory = validate_inventory(read(inventory_path), ctx.CFG, panel, legacy)
|
|
encoder = read(encoder_path)
|
|
validate_encoder(encoder, inventory, ctx.CFG)
|
|
seed_audit = ctx.MODEL / "seed-audit.json"
|
|
require(digest(seed_audit) == encoder["seeds_static_audit_sha256"], "Seed-audit receipt/encoder mismatch")
|
|
kc_path = ctx.AG / "apl-bundle/kc_globals.npy"
|
|
kc = np.load(kc_path, allow_pickle=False)
|
|
require(kc.dtype == np.int32 and kc.shape == (4064,) and np.all(np.diff(kc) > 0) and
|
|
np.all((kc >= 0) & (kc < ctx.CFG["neurons"])), "Frozen KC identity invalid")
|
|
base_path = ctx.AJ / "cases/delay18/cal-physiological-00-s51101/spec.json"
|
|
base = read(base_path)
|
|
require(base["apl_delay_steps"] == 18 and base["training"] is False, "Wrong frozen physical reference")
|
|
proof = {"validation_inventory": digest(inventory_path), "encoder_audit": digest(encoder_path),
|
|
"static_panel": digest(panel_path), "seed_audit": digest(seed_audit),
|
|
"KC_identity": digest(kc_path), "frozen_D18_spec": digest(base_path),
|
|
"AG_calibration_source": AG_SHA256, "AJ_calibration_source": AJ_SHA256}
|
|
odors = inventory[cohort]["odor_ids"]
|
|
result = collect_cohort(cohort, inventory[cohort]["records"], odors, ctx.CFG, ctx.EXP, kc, base, legacy, timing)
|
|
except Exception as error:
|
|
result["failures"].append(dict(stage="prerequisites", type=type(error).__name__, message=str(error)))
|
|
result.update(phase="4AK", apl_delay_steps=18, seeds=ctx.CFG["validation"]["seeds"], duration_ms=100.0,
|
|
odor_ids=odors, encoder_audit=encoder, prerequisite_input_sha256=proof,
|
|
representation_failure_is_not_numerical_failure=True, old_module_globals_modified=False,
|
|
old_heldout_dynamic_access=False, historical_never_simulated_claim=False,
|
|
requested_cohort_only_dynamic_read=True, suite_owns_sequential_unlock=True,
|
|
shared_apl_component_oracle_limit=True, independent_oracle_equivalence_claim=False,
|
|
seed_namespace_claim="Declared static-audit scope only; not an unqualified all-history independence claim",
|
|
biological_delay_measurement_claimed=False, candidate_generated=False, training=False,
|
|
metric_source="SHA-pinned AG complete phase4l gate; SHA-pinned AJ timing/repeat checks")
|
|
# Serializability/finiteness must be established even for read-only finalization.
|
|
json.dumps(result, allow_nan=False, sort_keys=True)
|
|
if target is not None:
|
|
outputs = {target: result}
|
|
outputs.update({target.parent / (mode + ".json"): report for mode, report in result["modes"].items()})
|
|
require(all(not path.exists() for path in outputs), "Refusing existing cohort/mode output")
|
|
for path, value in outputs.items():
|
|
ctx.save(path, value)
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
require(len(sys.argv) == 2, "Usage: validation.py seed_validation|novel_to_aj")
|
|
answer = summarize(sys.argv[1])
|
|
print(json.dumps({key: answer[key] for key in ("cohort", "pass_all", "execution_gate_pass", "representation_gate_pass")}))
|