Digital Drosophila Archive
221 lines
12 KiB
Python
221 lines
12 KiB
Python
"""Synthetic AM workflow identity/dispatch tests; no old evidence or network."""
|
|
import builtins
|
|
import copy
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
from types import ModuleType
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
import run_suite as runner
|
|
|
|
|
|
class WorkflowIdentityTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name).resolve()
|
|
self.exp = self.root / "experiments/new-am"
|
|
self.ak = self.root / "experiments/fake-ak"
|
|
self.patches = patch.multiple(runner, ROOT=self.root, EXP=self.exp, AK=self.ak)
|
|
self.patches.start()
|
|
self.addCleanup(self.patches.stop)
|
|
|
|
def fixture(self, stage="zero", repeat=0, name="synthetic-odor"):
|
|
row = dict(name=name, repeat=repeat, cohort="seed_validation")
|
|
case = self.ak / "cases" / row["cohort"] / name
|
|
case.mkdir(parents=True, exist_ok=True)
|
|
(case / "spec.json").write_bytes(b'{"synthetic":true}\n')
|
|
# This is a hash-only mock input, never parsed as scientific NPZ data.
|
|
(case / "requests.npz").write_bytes(b"synthetic-request-bytes")
|
|
overlays = self.exp / "overlays"
|
|
overlays.mkdir(parents=True, exist_ok=True)
|
|
(overlays / (stage + ".npz")).write_bytes((stage + "-synthetic-overlay").encode())
|
|
(overlays / (stage + ".json")).write_bytes((stage + "-synthetic-manifest").encode())
|
|
folder = self.exp / "gpu" / stage / (name + "-r" + str(repeat))
|
|
folder.mkdir(parents=True, exist_ok=False)
|
|
summary = dict(case=name, complete=True, pass_all=True, input_sha256={
|
|
key: runner.digest(case / key) for key in ("spec.json", "requests.npz")})
|
|
overlay = dict(complete=True, overlay_name=stage, input={
|
|
"overlay_sha256": runner.digest(overlays / (stage + ".npz")),
|
|
"manifest_sha256": runner.digest(overlays / (stage + ".json"))})
|
|
if stage == "zero":
|
|
mode = "zero"
|
|
left = self.ak / "gpu" / row["cohort"] / (name + "-r" + str(repeat))
|
|
elif repeat == 1:
|
|
mode = "repeat"
|
|
left = self.exp / "gpu" / stage / (name + "-r0")
|
|
else:
|
|
mode, left = "self", folder
|
|
report = dict(pass_all=True, overlay_name=stage, case_dir=str(case),
|
|
right=str(folder), left=str(left), mode=mode)
|
|
for key, value in (("summary.json", summary), ("overlay-audit.json", overlay),
|
|
("comparison.json", report)):
|
|
runner.save(folder / key, value)
|
|
runner.save(folder / "output-hashes.json", runner.output_inventory(folder))
|
|
return row, case, folder, report
|
|
|
|
@staticmethod
|
|
def replace_json(path, value):
|
|
# Mutations are deliberate corruptions of NEW temporary test fixtures.
|
|
path.write_text(json.dumps(value), encoding="utf-8")
|
|
|
|
def corrupt(self, folder, filename, field, value):
|
|
original = runner.read(folder / filename)
|
|
changed = copy.deepcopy(original)
|
|
target = changed
|
|
for key in field[:-1]:
|
|
target = target[key]
|
|
target[field[-1]] = value
|
|
self.replace_json(folder / filename, changed)
|
|
return original
|
|
|
|
def test_three_legitimate_identities_and_zero_repeat(self):
|
|
for stage, repeat in (("zero", 0), ("zero", 1), ("static25", 0), ("static25", 1)):
|
|
with self.subTest(stage=stage, repeat=repeat):
|
|
row, _, folder, _ = self.fixture(stage, repeat)
|
|
runner.verify_identity(folder, stage, row)
|
|
runner.verify_completed(folder, stage, row)
|
|
|
|
def test_wrong_case_and_cohort_fail_even_with_valid_hash_inventory(self):
|
|
row, _, folder, _ = self.fixture()
|
|
for altered in (dict(row, name="other-odor"), dict(row, cohort="novel_to_aj")):
|
|
with self.subTest(altered=altered), self.assertRaises((AssertionError, FileNotFoundError)):
|
|
runner.verify_identity(folder, "zero", altered)
|
|
self.corrupt(folder, "summary.json", ("case",), "other-odor")
|
|
self.replace_json(folder / "output-hashes.json", runner.output_inventory(folder))
|
|
with self.assertRaises(AssertionError):
|
|
runner.verify_completed(folder, "zero", row)
|
|
|
|
def test_wrong_stage_and_repeat_path_are_rejected(self):
|
|
row, _, folder, _ = self.fixture()
|
|
for stage, wrong_row in (("static25", row), ("zero", dict(row, repeat=1)),
|
|
("unknown", row), ("zero", dict(row, repeat=2))):
|
|
with self.subTest(stage=stage, row=wrong_row), self.assertRaises(AssertionError):
|
|
runner.verify_identity(folder, stage, wrong_row)
|
|
|
|
def test_overlay_stage_identity_and_two_input_hashes_are_required(self):
|
|
row, _, folder, _ = self.fixture("static25")
|
|
for field, value in ((("overlay_name",), "zero"), (("complete",), False),
|
|
(("input", "overlay_sha256"), "0" * 64),
|
|
(("input", "manifest_sha256"), "0" * 64)):
|
|
original = self.corrupt(folder, "overlay-audit.json", field, value)
|
|
with self.subTest(field=field), self.assertRaises(AssertionError):
|
|
runner.verify_identity(folder, "static25", row)
|
|
self.replace_json(folder / "overlay-audit.json", original)
|
|
|
|
def test_summary_completion_and_actual_case_hashes_are_required(self):
|
|
row, _, folder, _ = self.fixture()
|
|
for field, value in ((("complete",), False), (("pass_all",), False),
|
|
(("input_sha256", "spec.json"), "0" * 64),
|
|
(("input_sha256", "requests.npz"), "0" * 64)):
|
|
original = self.corrupt(folder, "summary.json", field, value)
|
|
with self.subTest(field=field), self.assertRaises(AssertionError):
|
|
runner.verify_identity(folder, "zero", row)
|
|
self.replace_json(folder / "summary.json", original)
|
|
|
|
def test_comparison_case_paths_mode_and_stage_are_required(self):
|
|
for stage, repeat in (("zero", 0), ("static25", 0), ("static25", 1)):
|
|
row, _, folder, _ = self.fixture(stage, repeat)
|
|
for field, value in ((("case_dir",), str(self.root / "wrong-case")),
|
|
(("right",), str(self.root / "wrong-output")),
|
|
(("left",), str(self.root / "wrong-reference")),
|
|
(("overlay_name",), "not-the-stage"),
|
|
(("mode",), "invalid-mode"), (("pass_all",), False)):
|
|
original = self.corrupt(folder, "comparison.json", field, value)
|
|
with self.subTest(stage=stage, repeat=repeat, field=field), self.assertRaises(AssertionError):
|
|
runner.verify_identity(folder, stage, row)
|
|
self.replace_json(folder / "comparison.json", original)
|
|
|
|
def test_self_report_cannot_substitute_for_repeat(self):
|
|
row, _, folder, _ = self.fixture("static25", 1)
|
|
report = runner.read(folder / "comparison.json")
|
|
report.update(mode="self", left=str(folder))
|
|
self.replace_json(folder / "comparison.json", report)
|
|
self.replace_json(folder / "output-hashes.json", runner.output_inventory(folder))
|
|
with self.assertRaises(AssertionError):
|
|
runner.verify_completed(folder, "static25", row)
|
|
|
|
def test_recompute_must_match_full_stored_report(self):
|
|
row, _, folder, report = self.fixture("static25", 1)
|
|
with patch.object(runner, "compare_expected", return_value=copy.deepcopy(report)) as compare:
|
|
runner.verify_completed(folder, "static25", row, recompute=True)
|
|
compare.assert_called_once_with("static25", row, folder)
|
|
for result in (dict(report, pass_all=False), dict(report, newly_detected_difference=True)):
|
|
with patch.object(runner, "compare_expected", return_value=result), self.assertRaises(AssertionError):
|
|
runner.verify_completed(folder, "static25", row, recompute=True)
|
|
|
|
def test_partial_folder_is_preserved_without_attempted_comparison(self):
|
|
row, _, folder, _ = self.fixture()
|
|
(folder / "output-hashes.json").unlink() # Deliberately incomplete NEW fixture.
|
|
before = {p.name: p.read_bytes() for p in folder.iterdir()}
|
|
with patch.object(runner, "compare_expected") as compare, self.assertRaises(FileNotFoundError):
|
|
runner.verify_completed(folder, "zero", row, recompute=True)
|
|
compare.assert_not_called()
|
|
self.assertEqual(before, {p.name: p.read_bytes() for p in folder.iterdir()})
|
|
|
|
def test_compare_expected_zero_dispatches_to_corresponding_ak_repeat(self):
|
|
self.assert_dispatch("zero", 0)
|
|
self.assert_dispatch("zero", 1)
|
|
|
|
def test_compare_expected_static_repeat_dispatches_to_r0(self):
|
|
self.assert_dispatch("static25", 1)
|
|
|
|
def test_compare_expected_static_primary_uses_self_validation(self):
|
|
self.assert_dispatch("static25", 0)
|
|
|
|
def assert_dispatch(self, stage, repeat):
|
|
fake = ModuleType("comparison")
|
|
fake.compare_runs = Mock(return_value={"marker": "pair"})
|
|
fake.validate_run = Mock(return_value={"marker": "single"})
|
|
row = dict(name="synthetic", cohort="novel_to_aj", repeat=repeat)
|
|
folder = self.exp / "gpu" / stage / (row["name"] + "-r" + str(repeat))
|
|
case = self.ak / "cases" / row["cohort"] / row["name"]
|
|
with patch.dict(sys.modules, {"comparison": fake}):
|
|
result = runner.compare_expected(stage, row, folder)
|
|
if stage == "zero":
|
|
reference = self.ak / "gpu" / row["cohort"] / (row["name"] + "-r" + str(repeat))
|
|
fake.compare_runs.assert_called_once_with(case, reference, folder, mode="zero")
|
|
fake.validate_run.assert_not_called()
|
|
self.assertEqual(result, {"marker": "pair"})
|
|
elif repeat:
|
|
reference = self.exp / "gpu" / stage / (row["name"] + "-r0")
|
|
fake.compare_runs.assert_called_once_with(case, reference, folder, mode="repeat")
|
|
fake.validate_run.assert_not_called()
|
|
self.assertEqual(result, {"marker": "pair"})
|
|
else:
|
|
fake.validate_run.assert_called_once_with(case, folder)
|
|
fake.compare_runs.assert_not_called()
|
|
self.assertEqual(result, {"marker": "single"})
|
|
# No output_path was passed: the comparator cannot write into evidence.
|
|
|
|
def test_preflight_sets_cuda_environment_without_torch_or_network(self):
|
|
imported = builtins.__import__
|
|
|
|
def guarded_import(name, *args, **kwargs):
|
|
if name == "torch" or name.startswith("torch."):
|
|
self.fail("preflight imported Torch")
|
|
return imported(name, *args, **kwargs)
|
|
|
|
def response(url, timeout):
|
|
self.assertEqual(os.environ["CUDA_VISIBLE_DEVICES"], runner.CFG["gpu_uuid"])
|
|
self.assertEqual(os.environ["CUDA_HOME"], "/usr/local/cuda")
|
|
self.assertEqual(os.environ["TORCH_CUDA_ARCH_LIST"], "8.9")
|
|
self.assertEqual(timeout, 5)
|
|
return io.BytesIO(b'{"queue_running":[],"queue_pending":[]}')
|
|
|
|
with patch.dict(os.environ, {"PATH": "synthetic-original-path"}), \
|
|
patch("builtins.__import__", side_effect=guarded_import), \
|
|
patch.object(runner.urllib.request, "urlopen", side_effect=response) as urlopen:
|
|
runner.preflight()
|
|
self.assertEqual(urlopen.call_count, 3)
|
|
self.assertTrue(os.environ["PATH"].endswith("synthetic-original-path"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|