Digital Drosophila Archive
299 lines
17 KiB
Python
299 lines
17 KiB
Python
"""Synthetic-only validation tests: no historical outputs or brain execution."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import types
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
import validation as val
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
LEGACY = val.load_frozen(HERE.parent / "author-apl-lab-v2/calibration.py", val.AG_SHA256, "ak_test_ag")
|
|
TIMING = val.load_frozen(HERE.parent / "apl-timing-lab-v1/calibration.py", val.AJ_SHA256, "ak_test_aj")
|
|
|
|
|
|
def config():
|
|
cfg = val.read(HERE / "config.json")
|
|
cfg["validation"]["seeds"] = [1001, 1002, 1003, 1004, 1005] # Synthetic, not proposed scientific seeds.
|
|
return cfg
|
|
|
|
|
|
def panel():
|
|
return [dict(odor_id=oid, odor_name=oid, split="calibration" if i < 6 else "heldout",
|
|
mapped_response_sum=val.MEDIAN)
|
|
for i, oid in enumerate(sorted(LEGACY.ORIGINAL_CAL_IDS) + ["synthetic-new-" + str(i) for i in range(6)])]
|
|
|
|
|
|
def records(cfg):
|
|
rows = []
|
|
pp = panel()
|
|
for ci, cohort in enumerate(val.COHORTS):
|
|
for mode in val.MODES:
|
|
for oi in range(6):
|
|
for seed in cfg["validation"]["seeds"]:
|
|
rows.append(dict(name=f"test-{mode}-{oi}-{seed}", cohort=cohort,
|
|
source_panel_split=val.SPLITS[cohort], mode=mode, odor_id=pp[6 * ci + oi]["odor_id"],
|
|
odor_index=oi, seed=seed, targets=1, requests=0))
|
|
return rows
|
|
|
|
|
|
def encoder(cfg, inventory):
|
|
return dict(panel_count=12, cal_count=6, novel_to_aj_count=6, median_mapped_response_sum=val.MEDIAN,
|
|
seeds=cfg["validation"]["seeds"], seeds_static_audit_sha256="a" * 64,
|
|
cohorts_static_panel_rows={cohort: [dict(odor_id=oid, odor_index=i, source_panel_split=val.SPLITS[cohort])
|
|
for i, oid in enumerate(inventory[cohort]["odor_ids"])] for cohort in val.COHORTS},
|
|
requested_before_any_dynamics=True, historical_never_simulated_claim=False)
|
|
|
|
|
|
def save(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("x", encoding="utf-8") as stream:
|
|
json.dump(value, stream, allow_nan=False)
|
|
|
|
|
|
def make_raw_trial(root):
|
|
"""Tiny neuron state with the real full-width APL trace schema."""
|
|
output, case, bundle = root / "out", root / "case", root / "bundle"
|
|
output.mkdir(); case.mkdir(); bundle.mkdir()
|
|
steps, neurons = 20, 2
|
|
cfg = dict(neurons=neurons, edges=0)
|
|
for filename in ("input_matrix.npy", "output_matrix.npy", "kc_globals.npy", "apl_globals.npy"):
|
|
np.save(bundle / filename, np.zeros(1, np.int32))
|
|
spec = dict(name="synthetic", steps=steps, validation_cohort="novel_to_aj", source_panel_split="heldout",
|
|
apl_delay_steps=18, mode="physiological", odor_id="synthetic", odor_name="synthetic", seed=1001,
|
|
expected_external_events=1., rate_sum_hz=50., apl_bundle_dir=str(bundle))
|
|
save(case / "spec.json", spec)
|
|
np.savez(case / "requests.npz", tick=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32))
|
|
state = {"step": np.empty(0, np.int32), "neuron_idx": np.empty(0, np.int32),
|
|
"counts": np.zeros(neurons, np.int32), "v": np.zeros(neurons, np.float64),
|
|
"g": np.zeros(neurons, np.float64), "ref": np.zeros(neurons, np.int32)}
|
|
np.savez(output / "state.npz", **state)
|
|
hashes = [dict(tick=t, **{key + "_sha256": LEGACY.raw_hash(state[key]) for key in ("v", "g", "ref")}) for t in range(steps)]
|
|
queues = [dict(tick=t, count=0, accepted_count=0, ordered_edge_sha256="0" * 64,
|
|
accepted_mask_sha256="0" * 64, joint_sha256="0" * 64) for t in range(steps)]
|
|
for filename, rows in (("tick-state-hashes.jsonl", hashes), ("queue.jsonl", queues)):
|
|
(output / filename).write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
|
|
(output / "requests.csv").write_text("tick,neuron_idx,attempted,accepted,rejected\n", encoding="utf-8")
|
|
for filename, width in LEGACY.APL_FILES.items():
|
|
np.save(output / filename, np.zeros((steps, width), np.float32))
|
|
np.save(output / "apl-applied.float32.npy", np.zeros((steps, 4064), np.float32))
|
|
pending = np.zeros((18, 4064), np.float32)
|
|
np.save(output / "apl-pending-tail.float32.npy", pending)
|
|
audit = dict(pass_all=True, apl_delay_steps=18, steps=steps, pending_generated_count=18,
|
|
pending_initial_zero_prefix=0, pending_application_tick_start=steps,
|
|
pending_application_tick_stop_exclusive=steps + 18, pending_generated_tick_start=steps - 18,
|
|
pending_generated_tick_stop_exclusive=steps, initial_padding_positive_zero=True,
|
|
sample_bits_preserved=True, brain_tail_advanced=False, generated_trace="apl-inhibition.float32.npy",
|
|
applied_trace="apl-applied.float32.npy", pending_trace="apl-pending-tail.float32.npy",
|
|
pending_raw_sha256=LEGACY.raw_hash(pending), pending_queue_drained=False,
|
|
finite_generated_sequence_only=True, future_apl_decay_or_feedback_generated=False,
|
|
closed_loop_branch_integrals_claimed_equal=False, biological_delay_measurement_claimed=False,
|
|
final_apl_state_float32=[0., 0.], final_apl_state_u32_bits=[0, 0], final_apl_state_nonzero=False)
|
|
save(output / "apl-timing-audit.json", audit)
|
|
component = dict(ok=True, n_neurons=neurons, n_kc=2, n_apl=2, dt_ms=.1,
|
|
files_sha256={name: val.digest(bundle / name) for name in
|
|
("input_matrix.npy", "output_matrix.npy", "kc_globals.npy", "apl_globals.npy")})
|
|
save(output / "apl-component.json", component)
|
|
save(output / "summary.json", dict(complete=True, pass_all=True, all_finite=True, steps=steps,
|
|
completed_ticks=steps, nodes=neurons, edges=0, precision="float64", apl_precision="float32",
|
|
case=spec["name"], record_history=False, learning=False, requested=0, accepted_requests=0,
|
|
rejected_requests=0, total_spikes=0, apl_component=component, apl_delay_steps=18, apl_timing=audit,
|
|
input_sha256={name: val.digest(case / name) for name in ("spec.json", "requests.npz")}))
|
|
return output, case, spec, cfg
|
|
|
|
|
|
def fake_trial(row, cohort, silent=False):
|
|
counts = np.zeros(1000, np.int32)
|
|
if not silent:
|
|
counts[row["odor_index"] * 50:row["odor_index"] * 50 + 50] = 1
|
|
return dict(kc_counts=counts, expected_external_events=1., external_events=1,
|
|
record=dict(case=row["name"], mode=row["mode"], seed=row["seed"], odor_id=row["odor_id"],
|
|
requested=row["requests"], split=cohort, cohort=cohort, source_panel_split=val.SPLITS[cohort]),
|
|
timing=dict(report=dict(pass_all=True)), apl={})
|
|
|
|
|
|
class ValidationTests(unittest.TestCase):
|
|
def test_source_pins_and_globals_are_unchanged(self):
|
|
self.assertEqual(LEGACY.ORIGINAL_SEEDS, (51101, 51102, 51103, 51104, 51105))
|
|
self.assertFalse(hasattr(LEGACY, "EXP"))
|
|
self.assertFalse(hasattr(TIMING, "EXP"))
|
|
with self.assertRaisesRegex(ValueError, "SHA"):
|
|
val.load_frozen(HERE.parent / "author-apl-lab-v2/calibration.py", "0" * 64, "bad_pin")
|
|
|
|
def test_inventory_accepts_configured_seeds_and_keeps_static_halves(self):
|
|
cfg = config(); inv = val.validate_inventory(records(cfg), cfg, panel(), LEGACY)
|
|
self.assertEqual([len(inv[c]["records"]) for c in val.COHORTS], [60, 60])
|
|
self.assertEqual(inv["novel_to_aj"]["odor_ids"], ["synthetic-new-" + str(i) for i in range(6)])
|
|
val.validate_encoder(encoder(cfg, inv), inv, cfg)
|
|
self.assertEqual(LEGACY.ORIGINAL_SEEDS[0], 51101)
|
|
|
|
def test_inventory_rejects_missing_duplicate_mislabelled_and_unaudited_cells(self):
|
|
cfg = config()
|
|
variants = []
|
|
rr = records(cfg); variants.append(rr[:-1])
|
|
rr = records(cfg); rr[-1] = copy.deepcopy(rr[-2]); variants.append(rr)
|
|
for field, value in (("source_panel_split", "heldout"), ("odor_id", "wrong"),
|
|
("seed", 51101), ("name", "../outside"), ("odor_index", True), ("targets", 0)):
|
|
rr = records(cfg); rr[0][field] = value; variants.append(rr)
|
|
for rr in variants:
|
|
with self.subTest(first=rr[0]), self.assertRaises(ValueError):
|
|
val.validate_inventory(rr, cfg, panel(), LEGACY)
|
|
|
|
def test_no_gate_relaxation_or_subset_median(self):
|
|
for key, value in (("minimum_separable_odor_pairs", 1), ("sparse_min_fraction", .01),
|
|
("repeat_exact_required", False)):
|
|
cfg = config(); cfg["gates"][key] = value
|
|
with self.subTest(key=key), self.assertRaises(ValueError):
|
|
val.validate_inventory(records(cfg), cfg, panel(), LEGACY)
|
|
pp = panel()
|
|
for row in pp:
|
|
row["mapped_response_sum"] = 1.
|
|
with self.assertRaisesRegex(ValueError, "median"):
|
|
val.validate_inventory(records(config()), config(), pp, LEGACY)
|
|
|
|
def test_encoder_rejects_history_claim_or_changed_panel_rows(self):
|
|
cfg = config(); inv = val.validate_inventory(records(cfg), cfg, panel(), LEGACY)
|
|
for key, value in (("historical_never_simulated_claim", True), ("requested_before_any_dynamics", False),
|
|
("seeds_static_audit_sha256", ""), ("cohorts_static_panel_rows", {})):
|
|
ee = encoder(cfg, inv); ee[key] = value
|
|
with self.subTest(key=key), self.assertRaises(ValueError):
|
|
val.validate_encoder(ee, inv, cfg)
|
|
|
|
def test_encoder_actual_prepare_panel_row_extension(self):
|
|
cfg = config(); inv = val.validate_inventory(records(cfg), cfg, panel(), LEGACY)
|
|
ee = encoder(cfg, inv)
|
|
for ci, cohort in enumerate(val.COHORTS):
|
|
for oi, row in enumerate(ee["cohorts_static_panel_rows"][cohort]):
|
|
row["source_panel_row"] = ci * 6 + oi
|
|
val.validate_encoder(ee, inv, cfg)
|
|
ee["cohorts_static_panel_rows"]["novel_to_aj"][0]["source_panel_row"] = 0
|
|
with self.assertRaisesRegex(ValueError, "panel row"):
|
|
val.validate_encoder(ee, inv, cfg)
|
|
|
|
def test_fullgraph_case_matches_static_physics_and_exact_input_expectation(self):
|
|
cfg = config(); row = records(cfg)[60]
|
|
row.update(source_panel_row=6, expected_external_events=10., rates_sum=100.)
|
|
spec = dict(name=row["name"], kind="fullgraph", validation_cohort=row["cohort"],
|
|
source_panel_split=row["source_panel_split"], source_panel_row=6,
|
|
**{key: row[key] for key in ("mode", "odor_id", "odor_index", "seed")},
|
|
apl_delay_steps=18, steps=1000, dt_ms=.1, duration_ms=100., training=False, record_history=False,
|
|
model={"resting_potential_mv": -52.}, graph_dir="unchanged-graph", apl_bundle_dir="unchanged-bundle",
|
|
apl={"gain": .01}, coefficients_path="unchanged-coefficients", science_python="unchanged-entry",
|
|
legacy_apl_worker="unchanged-worker", precision="float64", apl_precision="float32",
|
|
target_count=1, rate_sum_hz=100., expected_external_events=10.)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
case = Path(tmp)
|
|
save(case / "spec.json", spec)
|
|
np.savez(case / "pn-drive.npz", targets=np.array([10], np.int32), rates_hz=np.array([100.], np.float32))
|
|
np.savez(case / "requests.npz", tick=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32))
|
|
self.assertEqual(val.validate_case(case, row, cfg, spec), spec)
|
|
wrong = copy.deepcopy(spec); wrong["model"]["resting_potential_mv"] = -51.
|
|
with self.assertRaisesRegex(ValueError, "physical field"):
|
|
val.validate_case(case, row, cfg, wrong)
|
|
wrong_row = dict(row, targets=2)
|
|
with self.assertRaisesRegex(ValueError, "PN drive"):
|
|
val.validate_case(case, wrong_row, cfg, spec)
|
|
|
|
def test_real_legacy_reader_record_is_relabelled_without_globals_or_numerical_changes(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
output, case, spec, cfg = make_raw_trial(Path(tmp))
|
|
trial = val.read_validation_trial(LEGACY, TIMING, output, case, spec, np.arange(2, dtype=np.int32), cfg)
|
|
try:
|
|
self.assertEqual(trial["record"]["split"], "novel_to_aj")
|
|
self.assertEqual(trial["record"]["source_panel_split"], "heldout")
|
|
self.assertFalse(trial["record"]["historical_never_simulated_claim"])
|
|
self.assertTrue(trial["timing"]["report"]["pass_all"])
|
|
self.assertEqual(LEGACY.ORIGINAL_SEEDS[0], 51101)
|
|
finally:
|
|
val.close_trial(trial)
|
|
|
|
def test_real_reader_still_rejects_missing_state_and_delay_zero(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
output, case, spec, cfg = make_raw_trial(Path(tmp))
|
|
bad = dict(spec, apl_delay_steps=0)
|
|
with self.assertRaisesRegex(ValueError, "delay"):
|
|
val.read_validation_trial(LEGACY, TIMING, output, case, bad, np.arange(2, dtype=np.int32), cfg)
|
|
(output / "state.npz").unlink()
|
|
with self.assertRaises(FileNotFoundError):
|
|
val.read_validation_trial(LEGACY, TIMING, output, case, spec, np.arange(2, dtype=np.int32), cfg)
|
|
|
|
def collect(self, silent=False, broken_repeat=False, missing=False):
|
|
cfg = config(); inv = val.validate_inventory(records(cfg), cfg, panel(), LEGACY)
|
|
cohort = "seed_validation"; rows = inv[cohort]["records"]
|
|
byname = {row["name"]: row for row in rows}
|
|
accessed = []
|
|
def load(legacy, timing, output, case, spec, kc, config_arg):
|
|
accessed.append(str(output))
|
|
if missing and spec["name"] == rows[0]["name"]:
|
|
raise FileNotFoundError("synthetic missing evidence")
|
|
return fake_trial(byname[spec["name"]], cohort, silent)
|
|
timing = types.SimpleNamespace(timing_repeat=lambda *args: dict(pass_all=not broken_repeat,
|
|
checks=dict(apl_applied=not broken_repeat)))
|
|
with patch.object(val, "validate_case", side_effect=lambda case, row, *args: row), \
|
|
patch.object(val, "read_validation_trial", side_effect=load), \
|
|
patch.object(val, "evidence_hashes", return_value={"synthetic": True}):
|
|
result = val.collect_cohort(cohort, rows, inv[cohort]["odor_ids"], cfg, Path("synthetic"),
|
|
np.arange(2, dtype=np.int32), {}, LEGACY, timing)
|
|
return result, accessed
|
|
|
|
def test_full_60_plus_12_and_pair_specific_gate_without_other_cohort_read(self):
|
|
result, accessed = self.collect()
|
|
self.assertTrue(result["pass_all"])
|
|
self.assertEqual((result["valid_base_trials"], result["checked_repeats"], result["checked_timing_trials"]), (60, 12, 72))
|
|
self.assertEqual(len(accessed), 72)
|
|
self.assertFalse(any("novel_to_aj" in path for path in accessed))
|
|
for report in result["modes"].values():
|
|
self.assertEqual((report["pair_count"], report["within_distance_count"], report["between_distance_count"]), (15, 60, 75))
|
|
json.dumps(result, allow_nan=False)
|
|
|
|
def test_representation_failure_keeps_all_runs_and_is_not_execution_failure(self):
|
|
result, accessed = self.collect(silent=True)
|
|
self.assertTrue(result["execution_gate_pass"])
|
|
self.assertFalse(result["representation_gate_pass"])
|
|
self.assertFalse(result["pass_all"])
|
|
self.assertEqual(len(accessed), 72)
|
|
self.assertTrue(all(row["stage"] == "representation_gate" for row in result["failures"]))
|
|
|
|
def test_missing_evidence_and_repeat_failure_never_pass_execution(self):
|
|
for options in (dict(missing=True), dict(broken_repeat=True)):
|
|
with self.subTest(options=options):
|
|
result, _ = self.collect(**options)
|
|
self.assertFalse(result["execution_gate_pass"])
|
|
self.assertFalse(result["pass_all"])
|
|
self.assertTrue(result["failures"])
|
|
|
|
def test_none_recompute_is_deterministic_and_writes_nothing(self):
|
|
cfg = config(); rr = records(cfg); inv = val.validate_inventory(rr, cfg, panel(), LEGACY)
|
|
ee = encoder(cfg, inv)
|
|
ctx = types.SimpleNamespace(ROOT=Path("synthetic-root"), MODEL=Path("synthetic-model"), EXP=Path("synthetic-exp"),
|
|
AG_MODEL=Path("ag-model"), AJ_MODEL=Path("aj-model"), AG=Path("ag"), AJ=Path("aj"), CFG=cfg,
|
|
check=lambda: {}, save=lambda *args: self.fail("Read-only summarize wrote output"))
|
|
collected, _ = self.collect()
|
|
def read_fixture(path):
|
|
if path.name == "validation-cases.json": return rr
|
|
if path.name == "encoder-audit.json": return ee
|
|
if path.name == "spec.json": return dict(apl_delay_steps=18, training=False)
|
|
raise AssertionError("Unexpected input read: " + str(path))
|
|
def hash_fixture(path):
|
|
return val.PANEL_SHA256 if path.name == "frozen-odor-panel.parquet" else "a" * 64
|
|
with patch.dict("sys.modules", context=ctx), patch.object(val, "read", side_effect=read_fixture), \
|
|
patch.object(val, "digest", side_effect=hash_fixture), \
|
|
patch.object(val, "load_frozen", side_effect=[LEGACY, TIMING, LEGACY, TIMING]), \
|
|
patch.object(pd, "read_parquet", return_value=pd.DataFrame(panel())), \
|
|
patch.object(np, "load", return_value=np.arange(4064, dtype=np.int32)), \
|
|
patch.object(val, "collect_cohort", side_effect=lambda *args: copy.deepcopy(collected)):
|
|
first = val.summarize("seed_validation", None)
|
|
second = val.summarize("seed_validation", None)
|
|
self.assertEqual(first, second)
|
|
self.assertTrue(first["pass_all"])
|
|
json.dumps(first, allow_nan=False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|