"""Synthetic AM workflow identity/dispatch tests; no old evidence or network.""" import builtins import copy import io import json import os from pathlib import Path import sys import tempfile from types import ModuleType import unittest from unittest.mock import Mock, patch import run_suite as runner class WorkflowIdentityTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name).resolve() self.exp = self.root / "experiments/new-am" self.ak = self.root / "experiments/fake-ak" self.patches = patch.multiple(runner, ROOT=self.root, EXP=self.exp, AK=self.ak) self.patches.start() self.addCleanup(self.patches.stop) def fixture(self, stage="zero", repeat=0, name="synthetic-odor"): row = dict(name=name, repeat=repeat, cohort="seed_validation") case = self.ak / "cases" / row["cohort"] / name case.mkdir(parents=True, exist_ok=True) (case / "spec.json").write_bytes(b'{"synthetic":true}\n') # This is a hash-only mock input, never parsed as scientific NPZ data. (case / "requests.npz").write_bytes(b"synthetic-request-bytes") overlays = self.exp / "overlays" overlays.mkdir(parents=True, exist_ok=True) (overlays / (stage + ".npz")).write_bytes((stage + "-synthetic-overlay").encode()) (overlays / (stage + ".json")).write_bytes((stage + "-synthetic-manifest").encode()) folder = self.exp / "gpu" / stage / (name + "-r" + str(repeat)) folder.mkdir(parents=True, exist_ok=False) summary = dict(case=name, complete=True, pass_all=True, input_sha256={ key: runner.digest(case / key) for key in ("spec.json", "requests.npz")}) overlay = dict(complete=True, overlay_name=stage, input={ "overlay_sha256": runner.digest(overlays / (stage + ".npz")), "manifest_sha256": runner.digest(overlays / (stage + ".json"))}) if stage == "zero": mode = "zero" left = self.ak / "gpu" / row["cohort"] / (name + "-r" + str(repeat)) elif repeat == 1: mode = "repeat" left = self.exp / "gpu" / stage / (name + "-r0") else: mode, left = "self", folder report = dict(pass_all=True, overlay_name=stage, case_dir=str(case), right=str(folder), left=str(left), mode=mode) for key, value in (("summary.json", summary), ("overlay-audit.json", overlay), ("comparison.json", report)): runner.save(folder / key, value) runner.save(folder / "output-hashes.json", runner.output_inventory(folder)) return row, case, folder, report @staticmethod def replace_json(path, value): # Mutations are deliberate corruptions of NEW temporary test fixtures. path.write_text(json.dumps(value), encoding="utf-8") def corrupt(self, folder, filename, field, value): original = runner.read(folder / filename) changed = copy.deepcopy(original) target = changed for key in field[:-1]: target = target[key] target[field[-1]] = value self.replace_json(folder / filename, changed) return original def test_three_legitimate_identities_and_zero_repeat(self): for stage, repeat in (("zero", 0), ("zero", 1), ("static25", 0), ("static25", 1)): with self.subTest(stage=stage, repeat=repeat): row, _, folder, _ = self.fixture(stage, repeat) runner.verify_identity(folder, stage, row) runner.verify_completed(folder, stage, row) def test_wrong_case_and_cohort_fail_even_with_valid_hash_inventory(self): row, _, folder, _ = self.fixture() for altered in (dict(row, name="other-odor"), dict(row, cohort="novel_to_aj")): with self.subTest(altered=altered), self.assertRaises((AssertionError, FileNotFoundError)): runner.verify_identity(folder, "zero", altered) self.corrupt(folder, "summary.json", ("case",), "other-odor") self.replace_json(folder / "output-hashes.json", runner.output_inventory(folder)) with self.assertRaises(AssertionError): runner.verify_completed(folder, "zero", row) def test_wrong_stage_and_repeat_path_are_rejected(self): row, _, folder, _ = self.fixture() for stage, wrong_row in (("static25", row), ("zero", dict(row, repeat=1)), ("unknown", row), ("zero", dict(row, repeat=2))): with self.subTest(stage=stage, row=wrong_row), self.assertRaises(AssertionError): runner.verify_identity(folder, stage, wrong_row) def test_overlay_stage_identity_and_two_input_hashes_are_required(self): row, _, folder, _ = self.fixture("static25") for field, value in ((("overlay_name",), "zero"), (("complete",), False), (("input", "overlay_sha256"), "0" * 64), (("input", "manifest_sha256"), "0" * 64)): original = self.corrupt(folder, "overlay-audit.json", field, value) with self.subTest(field=field), self.assertRaises(AssertionError): runner.verify_identity(folder, "static25", row) self.replace_json(folder / "overlay-audit.json", original) def test_summary_completion_and_actual_case_hashes_are_required(self): row, _, folder, _ = self.fixture() for field, value in ((("complete",), False), (("pass_all",), False), (("input_sha256", "spec.json"), "0" * 64), (("input_sha256", "requests.npz"), "0" * 64)): original = self.corrupt(folder, "summary.json", field, value) with self.subTest(field=field), self.assertRaises(AssertionError): runner.verify_identity(folder, "zero", row) self.replace_json(folder / "summary.json", original) def test_comparison_case_paths_mode_and_stage_are_required(self): for stage, repeat in (("zero", 0), ("static25", 0), ("static25", 1)): row, _, folder, _ = self.fixture(stage, repeat) for field, value in ((("case_dir",), str(self.root / "wrong-case")), (("right",), str(self.root / "wrong-output")), (("left",), str(self.root / "wrong-reference")), (("overlay_name",), "not-the-stage"), (("mode",), "invalid-mode"), (("pass_all",), False)): original = self.corrupt(folder, "comparison.json", field, value) with self.subTest(stage=stage, repeat=repeat, field=field), self.assertRaises(AssertionError): runner.verify_identity(folder, stage, row) self.replace_json(folder / "comparison.json", original) def test_self_report_cannot_substitute_for_repeat(self): row, _, folder, _ = self.fixture("static25", 1) report = runner.read(folder / "comparison.json") report.update(mode="self", left=str(folder)) self.replace_json(folder / "comparison.json", report) self.replace_json(folder / "output-hashes.json", runner.output_inventory(folder)) with self.assertRaises(AssertionError): runner.verify_completed(folder, "static25", row) def test_recompute_must_match_full_stored_report(self): row, _, folder, report = self.fixture("static25", 1) with patch.object(runner, "compare_expected", return_value=copy.deepcopy(report)) as compare: runner.verify_completed(folder, "static25", row, recompute=True) compare.assert_called_once_with("static25", row, folder) for result in (dict(report, pass_all=False), dict(report, newly_detected_difference=True)): with patch.object(runner, "compare_expected", return_value=result), self.assertRaises(AssertionError): runner.verify_completed(folder, "static25", row, recompute=True) def test_partial_folder_is_preserved_without_attempted_comparison(self): row, _, folder, _ = self.fixture() (folder / "output-hashes.json").unlink() # Deliberately incomplete NEW fixture. before = {p.name: p.read_bytes() for p in folder.iterdir()} with patch.object(runner, "compare_expected") as compare, self.assertRaises(FileNotFoundError): runner.verify_completed(folder, "zero", row, recompute=True) compare.assert_not_called() self.assertEqual(before, {p.name: p.read_bytes() for p in folder.iterdir()}) def test_compare_expected_zero_dispatches_to_corresponding_ak_repeat(self): self.assert_dispatch("zero", 0) self.assert_dispatch("zero", 1) def test_compare_expected_static_repeat_dispatches_to_r0(self): self.assert_dispatch("static25", 1) def test_compare_expected_static_primary_uses_self_validation(self): self.assert_dispatch("static25", 0) def assert_dispatch(self, stage, repeat): fake = ModuleType("comparison") fake.compare_runs = Mock(return_value={"marker": "pair"}) fake.validate_run = Mock(return_value={"marker": "single"}) row = dict(name="synthetic", cohort="novel_to_aj", repeat=repeat) folder = self.exp / "gpu" / stage / (row["name"] + "-r" + str(repeat)) case = self.ak / "cases" / row["cohort"] / row["name"] with patch.dict(sys.modules, {"comparison": fake}): result = runner.compare_expected(stage, row, folder) if stage == "zero": reference = self.ak / "gpu" / row["cohort"] / (row["name"] + "-r" + str(repeat)) fake.compare_runs.assert_called_once_with(case, reference, folder, mode="zero") fake.validate_run.assert_not_called() self.assertEqual(result, {"marker": "pair"}) elif repeat: reference = self.exp / "gpu" / stage / (row["name"] + "-r0") fake.compare_runs.assert_called_once_with(case, reference, folder, mode="repeat") fake.validate_run.assert_not_called() self.assertEqual(result, {"marker": "pair"}) else: fake.validate_run.assert_called_once_with(case, folder) fake.compare_runs.assert_not_called() self.assertEqual(result, {"marker": "single"}) # No output_path was passed: the comparator cannot write into evidence. def test_preflight_sets_cuda_environment_without_torch_or_network(self): imported = builtins.__import__ def guarded_import(name, *args, **kwargs): if name == "torch" or name.startswith("torch."): self.fail("preflight imported Torch") return imported(name, *args, **kwargs) def response(url, timeout): self.assertEqual(os.environ["CUDA_VISIBLE_DEVICES"], runner.CFG["gpu_uuid"]) self.assertEqual(os.environ["CUDA_HOME"], "/usr/local/cuda") self.assertEqual(os.environ["TORCH_CUDA_ARCH_LIST"], "8.9") self.assertEqual(timeout, 5) return io.BytesIO(b'{"queue_running":[],"queue_pending":[]}') with patch.dict(os.environ, {"PATH": "synthetic-original-path"}), \ patch("builtins.__import__", side_effect=guarded_import), \ patch.object(runner.urllib.request, "urlopen", side_effect=response) as urlopen: runner.preflight() self.assertEqual(urlopen.call_count, 3) self.assertTrue(os.environ["PATH"].endswith("synthetic-original-path")) if __name__ == "__main__": unittest.main()