Seal Phase 4AM whole-brain plastic interface audit without learning claim
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,220 @@
|
||||
"""Synthetic AM workflow identity/dispatch tests; no old evidence or network."""
|
||||
import builtins
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from types import ModuleType
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import run_suite as runner
|
||||
|
||||
|
||||
class WorkflowIdentityTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name).resolve()
|
||||
self.exp = self.root / "experiments/new-am"
|
||||
self.ak = self.root / "experiments/fake-ak"
|
||||
self.patches = patch.multiple(runner, ROOT=self.root, EXP=self.exp, AK=self.ak)
|
||||
self.patches.start()
|
||||
self.addCleanup(self.patches.stop)
|
||||
|
||||
def fixture(self, stage="zero", repeat=0, name="synthetic-odor"):
|
||||
row = dict(name=name, repeat=repeat, cohort="seed_validation")
|
||||
case = self.ak / "cases" / row["cohort"] / name
|
||||
case.mkdir(parents=True, exist_ok=True)
|
||||
(case / "spec.json").write_bytes(b'{"synthetic":true}\n')
|
||||
# This is a hash-only mock input, never parsed as scientific NPZ data.
|
||||
(case / "requests.npz").write_bytes(b"synthetic-request-bytes")
|
||||
overlays = self.exp / "overlays"
|
||||
overlays.mkdir(parents=True, exist_ok=True)
|
||||
(overlays / (stage + ".npz")).write_bytes((stage + "-synthetic-overlay").encode())
|
||||
(overlays / (stage + ".json")).write_bytes((stage + "-synthetic-manifest").encode())
|
||||
folder = self.exp / "gpu" / stage / (name + "-r" + str(repeat))
|
||||
folder.mkdir(parents=True, exist_ok=False)
|
||||
summary = dict(case=name, complete=True, pass_all=True, input_sha256={
|
||||
key: runner.digest(case / key) for key in ("spec.json", "requests.npz")})
|
||||
overlay = dict(complete=True, overlay_name=stage, input={
|
||||
"overlay_sha256": runner.digest(overlays / (stage + ".npz")),
|
||||
"manifest_sha256": runner.digest(overlays / (stage + ".json"))})
|
||||
if stage == "zero":
|
||||
mode = "zero"
|
||||
left = self.ak / "gpu" / row["cohort"] / (name + "-r" + str(repeat))
|
||||
elif repeat == 1:
|
||||
mode = "repeat"
|
||||
left = self.exp / "gpu" / stage / (name + "-r0")
|
||||
else:
|
||||
mode, left = "self", folder
|
||||
report = dict(pass_all=True, overlay_name=stage, case_dir=str(case),
|
||||
right=str(folder), left=str(left), mode=mode)
|
||||
for key, value in (("summary.json", summary), ("overlay-audit.json", overlay),
|
||||
("comparison.json", report)):
|
||||
runner.save(folder / key, value)
|
||||
runner.save(folder / "output-hashes.json", runner.output_inventory(folder))
|
||||
return row, case, folder, report
|
||||
|
||||
@staticmethod
|
||||
def replace_json(path, value):
|
||||
# Mutations are deliberate corruptions of NEW temporary test fixtures.
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
|
||||
def corrupt(self, folder, filename, field, value):
|
||||
original = runner.read(folder / filename)
|
||||
changed = copy.deepcopy(original)
|
||||
target = changed
|
||||
for key in field[:-1]:
|
||||
target = target[key]
|
||||
target[field[-1]] = value
|
||||
self.replace_json(folder / filename, changed)
|
||||
return original
|
||||
|
||||
def test_three_legitimate_identities_and_zero_repeat(self):
|
||||
for stage, repeat in (("zero", 0), ("zero", 1), ("static25", 0), ("static25", 1)):
|
||||
with self.subTest(stage=stage, repeat=repeat):
|
||||
row, _, folder, _ = self.fixture(stage, repeat)
|
||||
runner.verify_identity(folder, stage, row)
|
||||
runner.verify_completed(folder, stage, row)
|
||||
|
||||
def test_wrong_case_and_cohort_fail_even_with_valid_hash_inventory(self):
|
||||
row, _, folder, _ = self.fixture()
|
||||
for altered in (dict(row, name="other-odor"), dict(row, cohort="novel_to_aj")):
|
||||
with self.subTest(altered=altered), self.assertRaises((AssertionError, FileNotFoundError)):
|
||||
runner.verify_identity(folder, "zero", altered)
|
||||
self.corrupt(folder, "summary.json", ("case",), "other-odor")
|
||||
self.replace_json(folder / "output-hashes.json", runner.output_inventory(folder))
|
||||
with self.assertRaises(AssertionError):
|
||||
runner.verify_completed(folder, "zero", row)
|
||||
|
||||
def test_wrong_stage_and_repeat_path_are_rejected(self):
|
||||
row, _, folder, _ = self.fixture()
|
||||
for stage, wrong_row in (("static25", row), ("zero", dict(row, repeat=1)),
|
||||
("unknown", row), ("zero", dict(row, repeat=2))):
|
||||
with self.subTest(stage=stage, row=wrong_row), self.assertRaises(AssertionError):
|
||||
runner.verify_identity(folder, stage, wrong_row)
|
||||
|
||||
def test_overlay_stage_identity_and_two_input_hashes_are_required(self):
|
||||
row, _, folder, _ = self.fixture("static25")
|
||||
for field, value in ((("overlay_name",), "zero"), (("complete",), False),
|
||||
(("input", "overlay_sha256"), "0" * 64),
|
||||
(("input", "manifest_sha256"), "0" * 64)):
|
||||
original = self.corrupt(folder, "overlay-audit.json", field, value)
|
||||
with self.subTest(field=field), self.assertRaises(AssertionError):
|
||||
runner.verify_identity(folder, "static25", row)
|
||||
self.replace_json(folder / "overlay-audit.json", original)
|
||||
|
||||
def test_summary_completion_and_actual_case_hashes_are_required(self):
|
||||
row, _, folder, _ = self.fixture()
|
||||
for field, value in ((("complete",), False), (("pass_all",), False),
|
||||
(("input_sha256", "spec.json"), "0" * 64),
|
||||
(("input_sha256", "requests.npz"), "0" * 64)):
|
||||
original = self.corrupt(folder, "summary.json", field, value)
|
||||
with self.subTest(field=field), self.assertRaises(AssertionError):
|
||||
runner.verify_identity(folder, "zero", row)
|
||||
self.replace_json(folder / "summary.json", original)
|
||||
|
||||
def test_comparison_case_paths_mode_and_stage_are_required(self):
|
||||
for stage, repeat in (("zero", 0), ("static25", 0), ("static25", 1)):
|
||||
row, _, folder, _ = self.fixture(stage, repeat)
|
||||
for field, value in ((("case_dir",), str(self.root / "wrong-case")),
|
||||
(("right",), str(self.root / "wrong-output")),
|
||||
(("left",), str(self.root / "wrong-reference")),
|
||||
(("overlay_name",), "not-the-stage"),
|
||||
(("mode",), "invalid-mode"), (("pass_all",), False)):
|
||||
original = self.corrupt(folder, "comparison.json", field, value)
|
||||
with self.subTest(stage=stage, repeat=repeat, field=field), self.assertRaises(AssertionError):
|
||||
runner.verify_identity(folder, stage, row)
|
||||
self.replace_json(folder / "comparison.json", original)
|
||||
|
||||
def test_self_report_cannot_substitute_for_repeat(self):
|
||||
row, _, folder, _ = self.fixture("static25", 1)
|
||||
report = runner.read(folder / "comparison.json")
|
||||
report.update(mode="self", left=str(folder))
|
||||
self.replace_json(folder / "comparison.json", report)
|
||||
self.replace_json(folder / "output-hashes.json", runner.output_inventory(folder))
|
||||
with self.assertRaises(AssertionError):
|
||||
runner.verify_completed(folder, "static25", row)
|
||||
|
||||
def test_recompute_must_match_full_stored_report(self):
|
||||
row, _, folder, report = self.fixture("static25", 1)
|
||||
with patch.object(runner, "compare_expected", return_value=copy.deepcopy(report)) as compare:
|
||||
runner.verify_completed(folder, "static25", row, recompute=True)
|
||||
compare.assert_called_once_with("static25", row, folder)
|
||||
for result in (dict(report, pass_all=False), dict(report, newly_detected_difference=True)):
|
||||
with patch.object(runner, "compare_expected", return_value=result), self.assertRaises(AssertionError):
|
||||
runner.verify_completed(folder, "static25", row, recompute=True)
|
||||
|
||||
def test_partial_folder_is_preserved_without_attempted_comparison(self):
|
||||
row, _, folder, _ = self.fixture()
|
||||
(folder / "output-hashes.json").unlink() # Deliberately incomplete NEW fixture.
|
||||
before = {p.name: p.read_bytes() for p in folder.iterdir()}
|
||||
with patch.object(runner, "compare_expected") as compare, self.assertRaises(FileNotFoundError):
|
||||
runner.verify_completed(folder, "zero", row, recompute=True)
|
||||
compare.assert_not_called()
|
||||
self.assertEqual(before, {p.name: p.read_bytes() for p in folder.iterdir()})
|
||||
|
||||
def test_compare_expected_zero_dispatches_to_corresponding_ak_repeat(self):
|
||||
self.assert_dispatch("zero", 0)
|
||||
self.assert_dispatch("zero", 1)
|
||||
|
||||
def test_compare_expected_static_repeat_dispatches_to_r0(self):
|
||||
self.assert_dispatch("static25", 1)
|
||||
|
||||
def test_compare_expected_static_primary_uses_self_validation(self):
|
||||
self.assert_dispatch("static25", 0)
|
||||
|
||||
def assert_dispatch(self, stage, repeat):
|
||||
fake = ModuleType("comparison")
|
||||
fake.compare_runs = Mock(return_value={"marker": "pair"})
|
||||
fake.validate_run = Mock(return_value={"marker": "single"})
|
||||
row = dict(name="synthetic", cohort="novel_to_aj", repeat=repeat)
|
||||
folder = self.exp / "gpu" / stage / (row["name"] + "-r" + str(repeat))
|
||||
case = self.ak / "cases" / row["cohort"] / row["name"]
|
||||
with patch.dict(sys.modules, {"comparison": fake}):
|
||||
result = runner.compare_expected(stage, row, folder)
|
||||
if stage == "zero":
|
||||
reference = self.ak / "gpu" / row["cohort"] / (row["name"] + "-r" + str(repeat))
|
||||
fake.compare_runs.assert_called_once_with(case, reference, folder, mode="zero")
|
||||
fake.validate_run.assert_not_called()
|
||||
self.assertEqual(result, {"marker": "pair"})
|
||||
elif repeat:
|
||||
reference = self.exp / "gpu" / stage / (row["name"] + "-r0")
|
||||
fake.compare_runs.assert_called_once_with(case, reference, folder, mode="repeat")
|
||||
fake.validate_run.assert_not_called()
|
||||
self.assertEqual(result, {"marker": "pair"})
|
||||
else:
|
||||
fake.validate_run.assert_called_once_with(case, folder)
|
||||
fake.compare_runs.assert_not_called()
|
||||
self.assertEqual(result, {"marker": "single"})
|
||||
# No output_path was passed: the comparator cannot write into evidence.
|
||||
|
||||
def test_preflight_sets_cuda_environment_without_torch_or_network(self):
|
||||
imported = builtins.__import__
|
||||
|
||||
def guarded_import(name, *args, **kwargs):
|
||||
if name == "torch" or name.startswith("torch."):
|
||||
self.fail("preflight imported Torch")
|
||||
return imported(name, *args, **kwargs)
|
||||
|
||||
def response(url, timeout):
|
||||
self.assertEqual(os.environ["CUDA_VISIBLE_DEVICES"], runner.CFG["gpu_uuid"])
|
||||
self.assertEqual(os.environ["CUDA_HOME"], "/usr/local/cuda")
|
||||
self.assertEqual(os.environ["TORCH_CUDA_ARCH_LIST"], "8.9")
|
||||
self.assertEqual(timeout, 5)
|
||||
return io.BytesIO(b'{"queue_running":[],"queue_pending":[]}')
|
||||
|
||||
with patch.dict(os.environ, {"PATH": "synthetic-original-path"}), \
|
||||
patch("builtins.__import__", side_effect=guarded_import), \
|
||||
patch.object(runner.urllib.request, "urlopen", side_effect=response) as urlopen:
|
||||
runner.preflight()
|
||||
self.assertEqual(urlopen.call_count, 3)
|
||||
self.assertTrue(os.environ["PATH"].endswith("synthetic-original-path"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user