Digital Drosophila Archive
347 lines
18 KiB
Python
347 lines
18 KiB
Python
"""Synthetic/read-only-loader tests only: never invoke GPU or real events."""
|
|
import ast
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from types import ModuleType, SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import isolated_workers as w
|
|
|
|
|
|
LAB = Path(__file__).resolve().parent.parent / 'olfactory-conditioning-lab-v1'
|
|
AM = LAB.parent / 'wholebrain-plastic-interface-v1/overlay.py'
|
|
|
|
|
|
def write_json(path, value):
|
|
path.write_text(json.dumps(value, sort_keys=True, allow_nan=False), encoding='utf-8')
|
|
|
|
|
|
class Fixture:
|
|
def __init__(self, root, mode):
|
|
self.root = root
|
|
for name in ('data', 'jobs', 'outputs', 'receipts', 'cache'):
|
|
(root / name).mkdir()
|
|
self.data, self.job_file = root / 'data', root / 'jobs/job.json'
|
|
for name in ('spec.json', 'requests.npz', 'before.npz', 'before.json', 'mask.npy', 'base.npy', 'coeff.json'):
|
|
(self.data / name).write_bytes(b'fixture bytes, not a scientific array\n')
|
|
self.output, self.receipt = root / 'outputs/new', root / 'receipts/new.json'
|
|
source_roles = ('runtime_light', 'runtime_continuation') if mode == 'brain' else ('conditioning_epoch',)
|
|
self.job = dict(schema=w.SCHEMA, job_id='synthetic-row.' + mode, kind=mode, registration_sha256='a' * 64,
|
|
worker_sha256=w.digest(w.__file__), output_dir=str(self.output), receipt_file=str(self.receipt),
|
|
sources={role: dict(path=str(LAB / (role + '.py')), sha256=w.SOURCE_PINS[role]) for role in source_roles},
|
|
input_sha256={})
|
|
if mode == 'brain':
|
|
graph, apl = self.data / 'graph', self.data / 'apl'
|
|
graph.mkdir(); apl.mkdir()
|
|
for name in w.GRAPH_FILES:
|
|
(graph / name).write_bytes(b'not a real graph\n')
|
|
for name in w.APL_FILES:
|
|
(apl / name).write_bytes(b'not a real APL array\n')
|
|
write_json(self.data / 'spec.json', dict(graph_dir=str(graph), apl_bundle_dir=str(apl),
|
|
coefficients_path=str(self.data / 'coeff.json')))
|
|
write_json(self.data / 'before.json', dict(mask_path=str(self.data / 'mask.npy'),
|
|
mask_sha256=w.digest(self.data / 'mask.npy'), base_candidate_path=str(self.data / 'base.npy'),
|
|
base_candidate_sha256=w.digest(self.data / 'base.npy'), am_overlay_path=str(AM),
|
|
am_overlay_sha256=w.digest(AM)))
|
|
self.job.update(case_dir=str(self.data), build_root=str(root / 'cache/build'),
|
|
overlay_file=str(self.data / 'before.npz'), overlay_manifest=str(self.data / 'before.json'),
|
|
diagnostic_mode='reference', admission=dict(gpu_uuid=w.GPU, comfy_queue_urls=list(w.COMFY_URLS)))
|
|
self.job['input_sha256'][str(AM)] = w.digest(AM)
|
|
else:
|
|
(self.data / 'checkpoint').mkdir()
|
|
(self.data / 'checkpoint/checkpoint-manifest.json').write_text('{}', encoding='utf-8')
|
|
write_json(self.data / 'authority.json', dict(files={'spec': dict(path=str(self.data / 'spec.json'),
|
|
sha256=w.digest(self.data / 'spec.json'))}, frozen_inputs={str(self.data / 'before.npz'): w.digest(self.data / 'before.npz')},
|
|
checkpoint_dir=str(self.data / 'checkpoint'),
|
|
checkpoint_manifest_sha256=w.digest(self.data / 'checkpoint/checkpoint-manifest.json')))
|
|
self.job.update(authority_file=str(self.data / 'authority.json'), authority_sha256=w.digest(self.data / 'authority.json'))
|
|
self.job['input_sha256'].update({str(p): w.digest(p) for p in self.data.rglob('*') if p.is_file()})
|
|
self.save()
|
|
|
|
def save(self):
|
|
write_json(self.job_file, self.job)
|
|
return w.digest(self.job_file)
|
|
|
|
def validate(self):
|
|
return w.validate_job(self.job_file, w.digest(self.job_file), self.job['kind'])
|
|
|
|
|
|
class JobTests(unittest.TestCase):
|
|
def fixture(self, mode='update'):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(tmp.cleanup)
|
|
return Fixture(Path(tmp.name), mode)
|
|
|
|
def test_top_level_stdlib_only_and_hard_pins_actual(self):
|
|
tree = ast.parse(Path(w.__file__).read_text(encoding='utf-8'))
|
|
for node in tree.body:
|
|
if isinstance(node, ast.Import):
|
|
names = [item.name.split('.')[0] for item in node.names]
|
|
elif isinstance(node, ast.ImportFrom):
|
|
names = [node.module.split('.')[0]]
|
|
else:
|
|
continue
|
|
self.assertTrue(set(names) <= sys.stdlib_module_names | {'__future__'}, names)
|
|
for name, expected in w.SOURCE_PINS.items():
|
|
self.assertEqual(w.digest(LAB / (name + '.py')), expected)
|
|
|
|
def test_valid_job_schema_for_both_modes_no_execution(self):
|
|
for mode in ('brain', 'update'):
|
|
with self.subTest(mode=mode):
|
|
f = self.fixture(mode)
|
|
job, paths, pins, sources = f.validate()
|
|
self.assertEqual(job['kind'], mode)
|
|
self.assertEqual(paths['output'], f.output)
|
|
self.assertIn(f.job_file, pins)
|
|
self.assertEqual(set(sources), {'runtime_light', 'runtime_continuation'} if mode == 'brain' else {'conditioning_epoch'})
|
|
self.assertFalse(f.output.exists())
|
|
self.assertFalse(f.receipt.exists())
|
|
|
|
def test_external_job_hash_and_file_mutation(self):
|
|
f = self.fixture()
|
|
original = w.digest(f.job_file)
|
|
f.job['job_id'] = 'changed'; f.save()
|
|
with self.assertRaisesRegex(ValueError, 'job SHA'):
|
|
w.validate_job(f.job_file, original, 'update')
|
|
(f.data / 'spec.json').write_bytes(b'mutation')
|
|
with self.assertRaisesRegex(ValueError, 'Pinned file changed'):
|
|
f.validate()
|
|
self.assertFalse(f.output.exists())
|
|
|
|
def test_unknown_mode_bool_unknown_fields_and_bad_source(self):
|
|
for mutation in (
|
|
lambda j: j.update(kind=True), lambda j: j.update(extra=True),
|
|
lambda j: j.update(job_id='../escape'), lambda j: j.update(job_id='a..b'),
|
|
lambda j: j.update(authority_sha256=True),
|
|
lambda j: j['sources']['conditioning_epoch'].update(sha256='f' * 64),
|
|
lambda j: j['sources'].update(runtime_light=dict(path=str(LAB / 'runtime_light.py'), sha256=w.SOURCE_PINS['runtime_light'])),
|
|
):
|
|
f = self.fixture(); mutation(f.job); f.save()
|
|
with self.assertRaises((ValueError, KeyError)):
|
|
w.validate_job(f.job_file, w.digest(f.job_file), 'update')
|
|
|
|
def test_same_sha_does_not_authorize_relocated_input(self):
|
|
f = self.fixture()
|
|
twin = f.root / 'jobs/twin.json'
|
|
twin.write_bytes((f.data / 'authority.json').read_bytes())
|
|
f.job['authority_file'] = str(twin); f.save()
|
|
with self.assertRaisesRegex(ValueError, 'Required external input pin missing'):
|
|
f.validate()
|
|
|
|
def test_required_graph_and_authority_pins_cannot_be_omitted(self):
|
|
f = self.fixture('brain')
|
|
del f.job['input_sha256'][str(f.data / 'graph/ptr.npy')]; f.save()
|
|
with self.assertRaisesRegex(ValueError, 'Required external input pin missing'):
|
|
f.validate()
|
|
f = self.fixture()
|
|
del f.job['input_sha256'][str(f.data / 'spec.json')]; f.save()
|
|
with self.assertRaisesRegex(ValueError, 'Required external input pin missing'):
|
|
f.validate()
|
|
|
|
def test_receipt_output_cache_paths_are_exclusive_and_protected(self):
|
|
cases = [('output_dir', 'data/new'), ('receipt_file', 'data/new.json'),
|
|
('receipt_file', 'outputs/new/receipt.json'), ('build_root', 'data/cache'),
|
|
('build_root', 'outputs/new/cache')]
|
|
for key, suffix in cases:
|
|
with self.subTest(key=key, suffix=suffix):
|
|
f = self.fixture('brain')
|
|
f.job[key] = str(f.root / suffix); f.save()
|
|
with self.assertRaises(ValueError):
|
|
f.validate()
|
|
for what in ('output', 'receipt', 'attempt'):
|
|
f = self.fixture()
|
|
target = {'output': f.output, 'receipt': f.receipt, 'attempt': Path(str(f.receipt) + '.attempt.json')}[what]
|
|
if what == 'output': target.mkdir()
|
|
else: target.write_text('existing')
|
|
with self.assertRaisesRegex(ValueError, 'retry forbidden'):
|
|
f.validate()
|
|
|
|
def test_path_link_escape_rejected(self):
|
|
f = self.fixture()
|
|
link = f.root / 'linked'
|
|
try:
|
|
link.symlink_to(f.data, target_is_directory=True)
|
|
except OSError as error:
|
|
self.skipTest('Local symlink privilege unavailable: ' + str(error))
|
|
f.job['output_dir'] = str(link / 'new'); f.save()
|
|
with self.assertRaisesRegex(ValueError, 'Linked/junction'):
|
|
f.validate()
|
|
|
|
def test_wrong_gpu_or_missing_comfy_admission_rejected(self):
|
|
for mutation in (lambda j: j['admission'].update(gpu_uuid='GPU-other'),
|
|
lambda j: j['admission'].update(comfy_queue_urls=[])):
|
|
f = self.fixture('brain'); mutation(f.job); f.save()
|
|
with self.assertRaisesRegex(ValueError, 'admission differs'):
|
|
f.validate()
|
|
|
|
def test_update_hides_cuda_before_scientific_import(self):
|
|
# The unittest host may legitimately have other suites' runtimes
|
|
# imported. The production contract is a fresh exec child, not that
|
|
# host; exercise the real boundary instead of clearing host modules.
|
|
code = "import os,sys; import isolated_workers as w; os.environ['CUDA_VISIBLE_DEVICES']=w.GPU; w._clean_process('update'); assert os.environ['CUDA_VISIBLE_DEVICES']==''; e=w._environment('update'); assert e['CUDA_hidden'] and not e['CUDA_initialized']; assert 'torch' not in sys.modules; print('hidden-before-import')"
|
|
result = subprocess.run([sys.executable, '-B', '-c', code], cwd=Path(w.__file__).parent,
|
|
capture_output=True, text=True, timeout=20)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn('hidden-before-import', result.stdout)
|
|
|
|
def test_fake_preimport_is_rejected_not_removed(self):
|
|
bad = ModuleType('phase4an_sealed_am_overlay')
|
|
with patch.dict(sys.modules, {'phase4an_sealed_am_overlay': bad}):
|
|
with self.assertRaisesRegex(ValueError, 'fresh exec'):
|
|
w._clean_process('update')
|
|
self.assertIs(sys.modules['phase4an_sealed_am_overlay'], bad)
|
|
|
|
def test_brain_configures_only_admitted_cuda_and_original_venv_path(self):
|
|
code = "import os,sys; import isolated_workers as w; exe=sys.executable; w._clean_process('brain'); assert sys.executable==exe; assert os.environ['CUDA_VISIBLE_DEVICES']==w.GPU; assert os.environ['PATH'].split(os.pathsep)[:2]==[w.SCIENCE_BIN,'/usr/local/cuda/bin']; assert os.environ['TORCH_CUDA_ARCH_LIST']=='8.9'; assert 'torch' not in sys.modules; print('brain-env-no-GPU-call')"
|
|
result = subprocess.run([sys.executable, '-B', '-c', code], cwd=Path(w.__file__).parent,
|
|
capture_output=True, text=True, timeout=20)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn('brain-env-no-GPU-call', result.stdout)
|
|
|
|
def test_failed_or_mutated_worker_never_receipts_and_attempt_blocks_retry(self):
|
|
for failure in ('raise', 'mutate'):
|
|
f = self.fixture()
|
|
def fake(job, paths, pins, sources):
|
|
paths['output'].mkdir()
|
|
(paths['output'] / 'partial.txt').write_text('preserved')
|
|
if failure == 'raise':
|
|
raise RuntimeError('synthetic child failure')
|
|
(f.data / 'before.npz').write_bytes(b'changed after validation')
|
|
return {'environment': {'synthetic': True}}
|
|
with patch.object(w, '_clean_process'), patch.object(w, '_run_update', side_effect=fake):
|
|
with self.assertRaises((RuntimeError, ValueError)):
|
|
w.execute_job('update', f.job_file, w.digest(f.job_file))
|
|
self.assertFalse(f.receipt.exists())
|
|
self.assertTrue((f.output / 'partial.txt').is_file())
|
|
self.assertTrue(Path(str(f.receipt) + '.attempt.json').is_file())
|
|
with self.assertRaisesRegex(ValueError, 'retry forbidden'):
|
|
f.validate()
|
|
|
|
def test_success_receipt_binds_inputs_outputs_but_not_process_exit(self):
|
|
f = self.fixture()
|
|
def fake(job, paths, pins, sources):
|
|
paths['output'].mkdir()
|
|
write_json(paths['output'] / 'synthetic-only.json', {'not_scientific': True})
|
|
return {'environment': {'synthetic': True}}
|
|
with patch.object(w, '_clean_process'), patch.object(w, '_run_update', side_effect=fake):
|
|
result = w.execute_job('update', f.job_file, w.digest(f.job_file))
|
|
self.assertTrue(result['complete'])
|
|
self.assertFalse(result['process_exit_independently_observed'])
|
|
self.assertFalse(result['learning_success_claimed'])
|
|
self.assertEqual(result, w.read_json(f.receipt))
|
|
self.assertEqual(result['output_files_sha256'], {'synthetic-only.json': w.digest(f.output / 'synthetic-only.json')})
|
|
self.assertEqual(result['verified_input_sha256'][str(f.job_file)], w.digest(f.job_file))
|
|
self.assertNotIn(str(f.receipt), result['output_files_sha256'])
|
|
|
|
def test_comfy_busy_and_malformed_fail_without_gpu_query(self):
|
|
for data in ({'queue_running': [1], 'queue_pending': []}, {'queue_running': [], 'queue_pending': [1]}, {}):
|
|
response = SimpleNamespace(geturl=lambda: w.COMFY_URLS[0], read=lambda n: json.dumps(data).encode())
|
|
class Open:
|
|
def __enter__(self): return response
|
|
def __exit__(self, *args): return False
|
|
with patch.object(w.urllib.request, 'urlopen', return_value=Open()), patch.object(w, '_gpu_inventory') as gpu:
|
|
with self.assertRaisesRegex(ValueError, 'ComfyUI busy'):
|
|
w._admit_brain()
|
|
gpu.assert_not_called()
|
|
|
|
def test_actual_source_byte_mutation_rejected_before_execution(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = Path(tmp) / 'changed.py'
|
|
path.write_bytes((LAB / 'conditioning_epoch.py').read_bytes() + b'\n# mutation\n')
|
|
with self.assertRaisesRegex(ValueError, 'Unverified source'):
|
|
w._load_source('_test_never_exec', path, w.SOURCE_PINS['conditioning_epoch'])
|
|
self.assertNotIn('_test_never_exec', sys.modules)
|
|
|
|
|
|
class ActualLoaderSubprocessTests(unittest.TestCase):
|
|
def run_python(self, code):
|
|
result = subprocess.run([sys.executable, '-B', '-c', code], cwd=LAB,
|
|
text=True, capture_output=True, timeout=30)
|
|
self.assertEqual(result.returncode, 0, result.stdout + '\n' + result.stderr)
|
|
return result.stdout
|
|
|
|
@staticmethod
|
|
def clean_code():
|
|
return """
|
|
import pathlib,sys,json
|
|
import conditioning_epoch as E
|
|
p=pathlib.Path.cwd()
|
|
c=E._load_source(p/'runtime_continuation.py',E.SOURCE_PINS['runtime_continuation_source'],'continuation')
|
|
h=c.R._load('phase4an_sealed_am_overlay',p.parent/'wholebrain-plastic-interface-v1/overlay.py',E.SOURCE_PINS['am_overlay_source'])
|
|
assert c.R._load('phase4an_sealed_am_overlay',p.parent/'wholebrain-plastic-interface-v1/overlay.py',E.SOURCE_PINS['am_overlay_source']) is h
|
|
assert 'runtime_light' not in sys.modules and 'torch' not in sys.modules
|
|
print('clean-E-real-helper-PASS')
|
|
"""
|
|
|
|
def test_real_E_only_loader_reuses_same_helper_without_torch(self):
|
|
self.assertIn('clean-E-real-helper-PASS', self.run_python(self.clean_code()))
|
|
|
|
def test_old_mixed_runtime_failure_and_true_exec_child_isolation(self):
|
|
code = """
|
|
import pathlib,sys,subprocess
|
|
import runtime_light as R
|
|
import conditioning_epoch as E
|
|
p=pathlib.Path.cwd()
|
|
h=R._load('phase4an_sealed_am_overlay',p.parent/'wholebrain-plastic-interface-v1/overlay.py',R.AM_OVERLAY_SHA)
|
|
c=E._load_source(p/'runtime_continuation.py',E.SOURCE_PINS['runtime_continuation_source'],'continuation')
|
|
assert R is not c.R
|
|
try:
|
|
c.R._load('phase4an_sealed_am_overlay',p.parent/'wholebrain-plastic-interface-v1/overlay.py',R.AM_OVERLAY_SHA)
|
|
except ValueError as e:
|
|
assert str(e)=='Existing unverified module binding: phase4an_sealed_am_overlay'
|
|
else: raise AssertionError('old loader collision not reproduced')
|
|
assert sys.modules['phase4an_sealed_am_overlay'] is h
|
|
""" + '\nr=subprocess.run([sys.executable,"-B","-c",' + repr(self.clean_code()) + '],capture_output=True,text=True,timeout=20)\nassert r.returncode==0,r.stderr\nprint(r.stdout)'
|
|
self.assertIn('clean-E-real-helper-PASS', self.run_python(code))
|
|
|
|
def test_real_same_runtime_loader_does_not_accept_poisoned_helper(self):
|
|
code = """
|
|
import pathlib,sys,types
|
|
import conditioning_epoch as E
|
|
p=pathlib.Path.cwd()
|
|
c=E._load_source(p/'runtime_continuation.py',E.SOURCE_PINS['runtime_continuation_source'],'continuation')
|
|
bad=types.ModuleType('phase4an_sealed_am_overlay');sys.modules[bad.__name__]=bad
|
|
try:
|
|
c.R._load(bad.__name__,p.parent/'wholebrain-plastic-interface-v1/overlay.py',E.SOURCE_PINS['am_overlay_source'])
|
|
except ValueError as e: assert 'Existing unverified module binding' in str(e)
|
|
else: raise AssertionError('poisoned helper accepted')
|
|
assert sys.modules[bad.__name__] is bad
|
|
print('poison-rejected')
|
|
"""
|
|
self.assertIn('poison-rejected', self.run_python(code))
|
|
|
|
def test_C_read_only_loader_after_ordinary_R_does_not_itself_load_overlay(self):
|
|
code = """
|
|
import pathlib,sys
|
|
import runtime_light as R
|
|
p=pathlib.Path.cwd()
|
|
h=R._load('phase4an_sealed_am_overlay',p.parent/'wholebrain-plastic-interface-v1/overlay.py',R.AM_OVERLAY_SHA)
|
|
import runtime_continuation as C
|
|
assert 'phase4an_sealed_am_overlay' not in C.R._LOADED
|
|
assert sys.modules['phase4an_sealed_am_overlay'] is h
|
|
assert 'torch' not in sys.modules
|
|
print('checkpoint-reader-import-only-PASS')
|
|
"""
|
|
self.assertIn('checkpoint-reader-import-only-PASS', self.run_python(code))
|
|
|
|
def test_cli_bad_external_pin_exits_nonzero_no_outputs(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
f = Fixture(Path(tmp), 'update')
|
|
result = subprocess.run([sys.executable, '-B', str(Path(w.__file__).resolve()), 'update',
|
|
'--job', str(f.job_file), '--job-sha256', 'f' * 64], capture_output=True, text=True, timeout=20)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn('External job SHA mismatch', result.stderr)
|
|
self.assertFalse(f.output.exists())
|
|
self.assertFalse(f.receipt.exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|