Digital Drosophila Archive
275 lines
17 KiB
Python
275 lines
17 KiB
Python
"""CPU arrays and mocked artifact I/O; these are not scientific observations."""
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
from pathlib import Path
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
|
|
spec = importlib.util.spec_from_file_location('_test_normal_metrics', Path(__file__).with_name('normal_metrics.py'))
|
|
N = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(N)
|
|
|
|
|
|
def fixture():
|
|
nodes = np.arange(6, dtype=np.int32)
|
|
pops = {k: np.asarray(v, np.int32) for k, v in dict(MBON=[0, 1], MBON01=[0], MBON12=[1],
|
|
legacyDownstream=[0, 1, 2, 3], nonMBONDownstream=[2, 3]).items()}
|
|
blocks, probes, arms = {}, {}, {}
|
|
seed = 1
|
|
for mode in N.M.MODES:
|
|
for dan in N.M.DAN_TYPES:
|
|
for cue in ('A', 'B'):
|
|
for rep in range(1, 6):
|
|
identity = f'{mode}__{dan}__reinforce_{cue}__rep{rep:02d}'
|
|
sources = []
|
|
for split in ('training', 'design'):
|
|
sources.append(dict(stream_id=identity + '/' + split, seed=seed, split=split)); seed += 1
|
|
evaluation = {}
|
|
for c in ('A', 'B'):
|
|
evaluation[N.ODORS[c]] = dict(stream_id=identity + '/evaluation/' + c, seed=seed, split='evaluation'); seed += 1
|
|
blocks[identity] = dict(mode=mode, dan_type=dan, compartment_verbatim=N.G.DAN_COMPARTMENTS[dan],
|
|
conditioned_cue=N.ODORS[cue], replicate=rep, training_sources=sources, evaluation_sources=evaluation,
|
|
evaluation_input_sha256={N.ODORS[c]: N.sha([identity, c]) for c in ('A', 'B')})
|
|
for arm in N.ARM_NAMES:
|
|
arms[(identity, arm)] = [dict(result=[identity, arm, e], requested=e < 50 and arm != 'cs_only')
|
|
for e in range(150 if arm == 'unpaired' else 100)]
|
|
for point in N.POINTS:
|
|
for c in ('A', 'B'):
|
|
probes[(identity, arm, point, c)] = dict(block=identity, arm=arm, point=point, cue=c,
|
|
active=arm == 'paired' and point > 0 and c == cue and rep <= 3, dan=dan)
|
|
membership = N.M.validate_populations(nodes, pops)
|
|
registry = dict(schema=N.G.REGISTRY_SCHEMA, protocol_sha256='1' * 64, population_binding_sha256='2' * 64,
|
|
nodes_raw_sha256=N.M._hash(nodes), population_membership_sha256=membership['membership_sha256'], blocks=blocks)
|
|
authority = dict(schema=N.G.AUTHORITY_SCHEMA, registry_sha256=N.G.descriptor_sha256(registry),
|
|
verification_receipt_sha256='3' * 64, execution_inventory_sha256='4' * 64, population_binding_sha256='2' * 64)
|
|
return nodes, pops, registry, authority, probes, arms
|
|
|
|
|
|
def state_for(record):
|
|
ids = np.asarray(sorted([1 if record['dan'] == 'PAM15' else 0, 2]) if record['active'] else [], np.int32)
|
|
state = dict(step=np.full(len(ids), 10, np.int32), neuron_idx=ids,
|
|
counts=np.bincount(ids, minlength=6).astype(np.int32), v=np.zeros(6, np.float64),
|
|
g=np.zeros(6, np.float64), ref=np.zeros(6, np.int32))
|
|
if record['active']:
|
|
state['g'][ids[0]] = .1
|
|
return state
|
|
|
|
|
|
class MockReader:
|
|
def __init__(self, values):
|
|
self.values = values
|
|
self.inventory = {str(p): dict(bytes=len(N.canonical(v)), sha256=N.sha(v)) for p, v in values.items()}
|
|
def json(self, path): return copy.deepcopy(self.values[str(path)])
|
|
def check(self, path): return self.inventory[str(path)]
|
|
def path(self, path): return Path(path), self.inventory[str(path)]
|
|
|
|
|
|
def process_fixture():
|
|
exp = Path.cwd() / 'mock-normal-evidence'
|
|
entry_id = 'input-generation'; regsha = '1' * 64
|
|
job_path = exp / 'jobs/input-generation.json'
|
|
proc_path = exp / 'receipts/input-generation/process.json'
|
|
start_path = exp / '.append-ledger/started/input-generation.json'
|
|
entry_path = exp / '.append-ledger/entries/0000000000000001.json'
|
|
values = {str(job_path): dict(kind='sample', registration_sha256=regsha, input_sha256={})}
|
|
for label in ('stdout', 'stderr'):
|
|
values[str(exp / f'logs/input-generation.{label}.log')] = {'log': label}
|
|
values[str(proc_path)] = dict(entry_id=entry_id, registration_sha256=regsha, job_sha256=N.sha(values[str(job_path)]),
|
|
complete=True, exit_code=0, timed_out=False, exit_observed_by_parent=True,
|
|
stdout_sha256=N.sha({'log': 'stdout'}), stderr_sha256=N.sha({'log': 'stderr'}))
|
|
r = MockReader(values)
|
|
row = lambda p: dict(path=str(p), relative_path=Path(p).relative_to(exp).as_posix(), **r.inventory[str(p)])
|
|
values[str(start_path)] = dict(entry_id=entry_id, registration_sha256=regsha, next_sequence=1,
|
|
prev_entry_sha256=None, retry_authorized=False, job=row(job_path))
|
|
values[str(entry_path)] = dict(entry_id=entry_id, registration_sha256=regsha, sequence=1,
|
|
prev_entry_sha256=None, start_sha256=N.sha(values[str(start_path)]), consumed_inputs=[row(job_path)],
|
|
new_outputs=[row(p) for p in list(values) if p != str(start_path)],
|
|
metadata=dict(job_file=str(job_path), process_receipt_file=str(proc_path), observed_returncode=0))
|
|
final = dict(schema='normal-append-ledger-v1-final-audit', registration_sha256=regsha, pass_all=True,
|
|
entry_ids=[entry_id], expected_entry_ids=[entry_id], expected_order_exact=True,
|
|
extra_files=[], missing_files=[], changed_files=[], unsafe_files=[], errors=[], uncommitted_entry_ids=[],
|
|
uncommitted_state_unknown=False, head_sha256=N.sha(values[str(entry_path)]))
|
|
return exp, values, final, proc_path, entry_path
|
|
|
|
|
|
def case_fixture(no_cs=False):
|
|
exp = Path.cwd() / 'mock-normal-case'; population_file = str(exp / 'population.npz')
|
|
seed = (1 << 62) + 17
|
|
task = dict(task_id='real-recipe/epoch-000', kind='train', mode='physiological', arm='dan_only' if no_cs else 'paired',
|
|
dan_type='PAM15', recipe=dict(cs=None if no_cs else dict(cue='A', seed=seed, source_stream_id='independent-original'),
|
|
us=dict(requested=True), D_command=1.0))
|
|
case = exp / 'cases' / N.task_key(task)
|
|
template = dict(kind='fullgraph', steps=1200, dt_ms=.1, apl_delay_steps=18, precision='float64',
|
|
apl_precision='float32', record_history=False, seed=98765, synaptic_tau=5.0, cs_input_seed=999)
|
|
spec = {k: v for k, v in template.items() if k != 'cs_input_seed'}
|
|
spec.update(name=N.task_key(task), steps=1200, duration_ms=120., max_steps=1200, budget_root=str(exp),
|
|
output_budget_bytes=123, mode=task['mode'], within_trial_weight_updates=False,
|
|
outer_protocol='normal-acquisition-v1', outer_task_id=task['task_id'], outer_arm=task['arm'])
|
|
descriptor = dict(mode=task['mode'], cue='A', seed=seed, steps=1200, cs_start_tick=0,
|
|
cs_stop_tick_exclusive=1000, source_stream_id='independent-original', table_id='physiological--A')
|
|
csid = None if no_cs else N.sha(descriptor)
|
|
arrays = {}
|
|
source = dict(tick=np.asarray([] if no_cs else [10], np.int32), neuron_idx=np.asarray([] if no_cs else [3], np.int32))
|
|
dan = np.asarray([4, 5, 6, 7], np.int32)
|
|
us = dict(tick=np.full(4, 900, np.int32), neuron_idx=dan)
|
|
combined = {k: np.concatenate((source[k], us[k])) for k in source}
|
|
for name, value in (('cs_requests.npz', source), ('us_requests.npz', us), ('requests.npz', combined),
|
|
('source-mask.npz', dict(source_mask=np.asarray(([1] if not no_cs else []) + [2]*4, np.uint8)))):
|
|
arrays[str(case / name)] = value
|
|
values = {str(case / 'spec.json'): spec, population_file: {'artifact': 'actual-static-populations'}}
|
|
source_pin = None; sampler = dict(inputs={})
|
|
if not no_cs:
|
|
source_path = exp / f'inputs/generated/{csid}.npz'; audit_path = exp / f'inputs/generated/{csid}.json'
|
|
values[str(source_path)] = {'array-artifact': 'mock-sparse-CS'}
|
|
arrays[str(source_path)] = source
|
|
source_pin = dict(path=str(source_path), sha256=N.sha(values[str(source_path)]))
|
|
values[str(audit_path)] = dict(descriptor=descriptor, requests_sha256=source_pin['sha256'], audit=dict(seed=seed,
|
|
generator_initial_seed=seed, seed_bits=63, seed_conversion='exact integer; no truncation/modulo/offset',
|
|
private_generator=True, matrix_draw_calls=1, uniform_matrix_shape=[1200, 1], CS_window=[0, 1000]))
|
|
sampler['inputs'][csid] = dict(requests=source_pin, audit=dict(path=str(audit_path), sha256=N.sha(values[str(audit_path)])))
|
|
values[str(case / 'input-binding.json')] = dict(task_id=task['task_id'], CS_descriptor_id=csid, CS_source=source_pin,
|
|
DAN_type='PAM15', population_file=dict(path=population_file, sha256=N.sha(values[population_file])),
|
|
DAN_membership_sha256=hashlib.sha256(dan.tobytes()).hexdigest(), composition=dict(policy='error', cross_source_conflict_count=0))
|
|
reader = MockReader(values)
|
|
reader.arrays = lambda p: {k: v.copy() for k, v in arrays[str(p)].items()}
|
|
args = dict(template=template, inputs=dict(task_inputs={task['task_id']: csid}, descriptors={} if no_cs else {csid: descriptor}),
|
|
sampler=sampler, populations=dict(PAM15=dan), population_file=population_file, output_cap=123)
|
|
return reader, exp, task, dict(case_dir=str(case)), args, values, arrays, case
|
|
|
|
|
|
class NormalMetricTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.nodes, cls.pops, cls.registry, cls.authority, cls.probes, cls.arms = fixture()
|
|
cls.full_reports = {}
|
|
original = N.G.evaluate_acquisition
|
|
def capture(*args, **kwargs):
|
|
result = original(*args, **kwargs)
|
|
cls.full_reports[(result['mode'], result['dan_type'], result['conditioned_cue'])] = result
|
|
return result
|
|
with patch.object(N.G, 'evaluate_acquisition', side_effect=capture):
|
|
cls.report = N.scientific_reports(cls.probes, cls.arms, registry=cls.registry, authority=cls.authority,
|
|
nodes=cls.nodes, populations=cls.pops, load_state=state_for)
|
|
|
|
def test_full_fixed_curves_and_eight_same_replicate_gates(self):
|
|
self.assertEqual(len(self.report['cells']), 8)
|
|
self.assertEqual(len(self.report['curves']), 1960)
|
|
self.assertTrue(self.report['cohort']['all_functional_predicates_pass'])
|
|
self.assertTrue(all(r['successful_same_replicate_conjunctions'] == 3 for r in self.report['cells']))
|
|
self.assertTrue(all('acquisition_metric_inputs' not in r for r in self.report['cells']))
|
|
self.assertTrue(all(not r['scientific_learning_pass_claimed'] for r in self.report['cells']))
|
|
self.assertEqual({r['checkpoint'] for r in self.report['curves']}, set(N.POINTS))
|
|
self.assertTrue(all(r['metrics']['rawS'] == 0 for r in self.report['curves'] if r['checkpoint'] == 0))
|
|
self.assertTrue(all(not isinstance(v, list) for r in self.report['curves']
|
|
for cue in r['metrics']['cues'].values() for group in cue['populations'].values() for v in group.values()))
|
|
|
|
def test_legacy_mapping_exactly_matches_frozen_wrapper(self):
|
|
expected = N.G.legacy_h_cohort_audit(list(self.full_reports.values()), registry=self.registry,
|
|
authority=self.authority, declared_mapping=N.G.H_MAPPING)
|
|
actual = N.legacy_audit(self.report['cells'], N.G.H_MAPPING)
|
|
self.assertEqual(actual, expected)
|
|
self.assertEqual(len(actual['groups']), 4)
|
|
self.assertTrue(all(set(g['control_sources']) == {'PAM15', 'PPL103'} for g in actual['groups']))
|
|
with self.assertRaises(ValueError): N.legacy_audit(self.report['cells'], None)
|
|
|
|
def test_training_evaluation_seed_collision_and_missing_control_fail_closed(self):
|
|
registry = copy.deepcopy(self.registry)
|
|
block = next(iter(registry['blocks'].values()))
|
|
block['evaluation_sources'][N.M.CUES[0]]['seed'] = block['training_sources'][0]['seed']
|
|
with self.assertRaisesRegex(ValueError, 'collision'):
|
|
N.G.validate_registry(registry, expected_sha256=N.G.descriptor_sha256(registry))
|
|
report = next(iter(self.full_reports.values()))
|
|
rows = copy.deepcopy(report['acquisition_metric_inputs'])
|
|
del rows[0]['arms']['DAN-only']
|
|
with self.assertRaisesRegex(ValueError, 'six actual controls'):
|
|
N.G.evaluate_acquisition(rows, assay='normal', mode=report['mode'], dan_type=report['dan_type'],
|
|
conditioned_cue=report['conditioned_cue'], registry=self.registry, authority=self.authority)
|
|
|
|
def test_state_mutation_and_absent_probe_never_become_zero_metrics(self):
|
|
probes = dict(self.probes); del probes[next(iter(probes))]
|
|
with self.assertRaises((KeyError, ValueError)):
|
|
N.scientific_reports(probes, self.arms, registry=self.registry, authority=self.authority,
|
|
nodes=self.nodes, populations=self.pops, load_state=state_for)
|
|
def corrupt(record):
|
|
state = state_for(record); state['g'][0] = np.nan; return state
|
|
with self.assertRaisesRegex(ValueError, 'Nonfinite'):
|
|
N.scientific_reports(self.probes, self.arms, registry=self.registry, authority=self.authority,
|
|
nodes=self.nodes, populations=self.pops, load_state=corrupt)
|
|
|
|
def test_actual_parent_receipt_nested_layout_and_exit_failure(self):
|
|
exp, values, final, proc, _ = process_fixture()
|
|
evidence = N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final)
|
|
observed = evidence.process('input-generation', 'sample'); evidence.finish()
|
|
self.assertEqual(observed['process_path'], str(exp / 'receipts/input-generation/process.json'))
|
|
values[str(proc)]['exit_code'] = 1
|
|
# Even a caller report saying pass_all cannot rescue a real exit failure.
|
|
r = MockReader(values)
|
|
entry = next(v for v in values.values() if 'new_outputs' in v)
|
|
for row in entry['new_outputs']:
|
|
if row['path'] == str(proc): row.update(r.inventory[str(proc)])
|
|
with self.assertRaisesRegex(ValueError, 'Actual parent exit'):
|
|
N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final).process('input-generation', 'sample')
|
|
|
|
def test_result_process_must_be_ledger_owned_not_only_metadata(self):
|
|
exp, values, final, proc, entry_path = process_fixture()
|
|
entry = values[str(entry_path)]
|
|
entry['new_outputs'] = [r for r in entry['new_outputs'] if r['path'] != str(proc)]
|
|
with self.assertRaisesRegex(ValueError, 'not owned'):
|
|
N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final).process('input-generation', 'sample')
|
|
bad = dict(final, uncommitted_entry_ids=['unknown-child'])
|
|
with self.assertRaisesRegex(ValueError, 'Incomplete'):
|
|
N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], bad)
|
|
|
|
def test_reader_per_invocation_actual_sha_cache_and_no_boolean_substitution(self):
|
|
path = Path.cwd() / 'mock-artifact.json'; raw = b'{"actual":true}'
|
|
identity = dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest())
|
|
with patch.object(Path, 'is_symlink', return_value=False), patch.object(Path, 'is_junction', return_value=False, create=True), \
|
|
patch.object(Path, 'is_file', return_value=True), patch.object(Path, 'open', side_effect=lambda *a, **k: io.BytesIO(raw)) as opened:
|
|
reader = N.Reader({str(path): identity})
|
|
reader.check(path); reader.check(path)
|
|
self.assertEqual(opened.call_count, 1)
|
|
N.Reader({str(path): identity}).check(path)
|
|
self.assertEqual(opened.call_count, 2)
|
|
with self.assertRaisesRegex(ValueError, 'Actual file checksum'):
|
|
N.Reader({str(path): dict(bytes=len(raw), sha256='0' * 64)}).check(path)
|
|
|
|
def test_real_inventory_process_count_and_no_followup(self):
|
|
tasks = [dict(task_id='train-' + str(i), kind='train') for i in range(30000)]
|
|
tasks += [dict(task_id='probe-' + str(i), kind='probe') for i in range(3920)]
|
|
entries = N.expected_entries(dict(tasks=tasks))
|
|
self.assertEqual(len(entries), 93921)
|
|
self.assertEqual(len(set(entries)), 93921)
|
|
self.assertEqual(entries[0], 'input-generation')
|
|
self.assertTrue(entries[-1].endswith('--brain'))
|
|
|
|
def test_original_train_cs_us_seed_and_model_checked_independently(self):
|
|
reader, exp, task, job, args, values, arrays, case = case_fixture()
|
|
N.audit_case(reader, exp, task, job, **args)
|
|
arrays[str(case / 'us_requests.npz')]['tick'][0] = 901
|
|
with self.assertRaisesRegex(ValueError, 'sorted unique|scheduled US'):
|
|
N.audit_case(reader, exp, task, job, **args)
|
|
arrays[str(case / 'us_requests.npz')]['tick'][0] = 900
|
|
values[str(case / 'spec.json')]['seed'] += 1
|
|
with self.assertRaisesRegex(ValueError, 'model/reset/clock'):
|
|
N.audit_case(reader, exp, task, job, **args)
|
|
values[str(case / 'spec.json')]['seed'] -= 1
|
|
generated = next(v for v in values.values() if 'audit' in v)
|
|
generated['audit']['generator_initial_seed'] %= 2**32
|
|
with self.assertRaisesRegex(ValueError, 'sampler seed'):
|
|
N.audit_case(reader, exp, task, job, **args)
|
|
|
|
def test_no_cs_is_empty_and_dan_only_still_uses_original_fixed_us(self):
|
|
reader, exp, task, job, args, values, arrays, case = case_fixture(no_cs=True)
|
|
N.audit_case(reader, exp, task, job, **args)
|
|
arrays[str(case / 'cs_requests.npz')] = dict(tick=np.asarray([10], np.int32), neuron_idx=np.asarray([3], np.int32))
|
|
with self.assertRaisesRegex(ValueError, 'case CS'):
|
|
N.audit_case(reader, exp, task, job, **args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|