"""CPU arrays and mocked artifact I/O; these are not scientific observations.""" import copy import hashlib import importlib.util import io from pathlib import Path import unittest from unittest.mock import patch import numpy as np spec = importlib.util.spec_from_file_location('_test_normal_metrics', Path(__file__).with_name('normal_metrics.py')) N = importlib.util.module_from_spec(spec) spec.loader.exec_module(N) def fixture(): nodes = np.arange(6, dtype=np.int32) pops = {k: np.asarray(v, np.int32) for k, v in dict(MBON=[0, 1], MBON01=[0], MBON12=[1], legacyDownstream=[0, 1, 2, 3], nonMBONDownstream=[2, 3]).items()} blocks, probes, arms = {}, {}, {} seed = 1 for mode in N.M.MODES: for dan in N.M.DAN_TYPES: for cue in ('A', 'B'): for rep in range(1, 6): identity = f'{mode}__{dan}__reinforce_{cue}__rep{rep:02d}' sources = [] for split in ('training', 'design'): sources.append(dict(stream_id=identity + '/' + split, seed=seed, split=split)); seed += 1 evaluation = {} for c in ('A', 'B'): evaluation[N.ODORS[c]] = dict(stream_id=identity + '/evaluation/' + c, seed=seed, split='evaluation'); seed += 1 blocks[identity] = dict(mode=mode, dan_type=dan, compartment_verbatim=N.G.DAN_COMPARTMENTS[dan], conditioned_cue=N.ODORS[cue], replicate=rep, training_sources=sources, evaluation_sources=evaluation, evaluation_input_sha256={N.ODORS[c]: N.sha([identity, c]) for c in ('A', 'B')}) for arm in N.ARM_NAMES: arms[(identity, arm)] = [dict(result=[identity, arm, e], requested=e < 50 and arm != 'cs_only') for e in range(150 if arm == 'unpaired' else 100)] for point in N.POINTS: for c in ('A', 'B'): probes[(identity, arm, point, c)] = dict(block=identity, arm=arm, point=point, cue=c, active=arm == 'paired' and point > 0 and c == cue and rep <= 3, dan=dan) membership = N.M.validate_populations(nodes, pops) registry = dict(schema=N.G.REGISTRY_SCHEMA, protocol_sha256='1' * 64, population_binding_sha256='2' * 64, nodes_raw_sha256=N.M._hash(nodes), population_membership_sha256=membership['membership_sha256'], blocks=blocks) authority = dict(schema=N.G.AUTHORITY_SCHEMA, registry_sha256=N.G.descriptor_sha256(registry), verification_receipt_sha256='3' * 64, execution_inventory_sha256='4' * 64, population_binding_sha256='2' * 64) return nodes, pops, registry, authority, probes, arms def state_for(record): ids = np.asarray(sorted([1 if record['dan'] == 'PAM15' else 0, 2]) if record['active'] else [], np.int32) state = dict(step=np.full(len(ids), 10, np.int32), neuron_idx=ids, counts=np.bincount(ids, minlength=6).astype(np.int32), v=np.zeros(6, np.float64), g=np.zeros(6, np.float64), ref=np.zeros(6, np.int32)) if record['active']: state['g'][ids[0]] = .1 return state class MockReader: def __init__(self, values): self.values = values self.inventory = {str(p): dict(bytes=len(N.canonical(v)), sha256=N.sha(v)) for p, v in values.items()} def json(self, path): return copy.deepcopy(self.values[str(path)]) def check(self, path): return self.inventory[str(path)] def path(self, path): return Path(path), self.inventory[str(path)] def process_fixture(): exp = Path.cwd() / 'mock-normal-evidence' entry_id = 'input-generation'; regsha = '1' * 64 job_path = exp / 'jobs/input-generation.json' proc_path = exp / 'receipts/input-generation/process.json' start_path = exp / '.append-ledger/started/input-generation.json' entry_path = exp / '.append-ledger/entries/0000000000000001.json' values = {str(job_path): dict(kind='sample', registration_sha256=regsha, input_sha256={})} for label in ('stdout', 'stderr'): values[str(exp / f'logs/input-generation.{label}.log')] = {'log': label} values[str(proc_path)] = dict(entry_id=entry_id, registration_sha256=regsha, job_sha256=N.sha(values[str(job_path)]), complete=True, exit_code=0, timed_out=False, exit_observed_by_parent=True, stdout_sha256=N.sha({'log': 'stdout'}), stderr_sha256=N.sha({'log': 'stderr'})) r = MockReader(values) row = lambda p: dict(path=str(p), relative_path=Path(p).relative_to(exp).as_posix(), **r.inventory[str(p)]) values[str(start_path)] = dict(entry_id=entry_id, registration_sha256=regsha, next_sequence=1, prev_entry_sha256=None, retry_authorized=False, job=row(job_path)) values[str(entry_path)] = dict(entry_id=entry_id, registration_sha256=regsha, sequence=1, prev_entry_sha256=None, start_sha256=N.sha(values[str(start_path)]), consumed_inputs=[row(job_path)], new_outputs=[row(p) for p in list(values) if p != str(start_path)], metadata=dict(job_file=str(job_path), process_receipt_file=str(proc_path), observed_returncode=0)) final = dict(schema='normal-append-ledger-v1-final-audit', registration_sha256=regsha, pass_all=True, entry_ids=[entry_id], expected_entry_ids=[entry_id], expected_order_exact=True, extra_files=[], missing_files=[], changed_files=[], unsafe_files=[], errors=[], uncommitted_entry_ids=[], uncommitted_state_unknown=False, head_sha256=N.sha(values[str(entry_path)])) return exp, values, final, proc_path, entry_path def case_fixture(no_cs=False): exp = Path.cwd() / 'mock-normal-case'; population_file = str(exp / 'population.npz') seed = (1 << 62) + 17 task = dict(task_id='real-recipe/epoch-000', kind='train', mode='physiological', arm='dan_only' if no_cs else 'paired', dan_type='PAM15', recipe=dict(cs=None if no_cs else dict(cue='A', seed=seed, source_stream_id='independent-original'), us=dict(requested=True), D_command=1.0)) case = exp / 'cases' / N.task_key(task) template = dict(kind='fullgraph', steps=1200, dt_ms=.1, apl_delay_steps=18, precision='float64', apl_precision='float32', record_history=False, seed=98765, synaptic_tau=5.0, cs_input_seed=999) spec = {k: v for k, v in template.items() if k != 'cs_input_seed'} spec.update(name=N.task_key(task), steps=1200, duration_ms=120., max_steps=1200, budget_root=str(exp), output_budget_bytes=123, mode=task['mode'], within_trial_weight_updates=False, outer_protocol='normal-acquisition-v1', outer_task_id=task['task_id'], outer_arm=task['arm']) descriptor = dict(mode=task['mode'], cue='A', seed=seed, steps=1200, cs_start_tick=0, cs_stop_tick_exclusive=1000, source_stream_id='independent-original', table_id='physiological--A') csid = None if no_cs else N.sha(descriptor) arrays = {} source = dict(tick=np.asarray([] if no_cs else [10], np.int32), neuron_idx=np.asarray([] if no_cs else [3], np.int32)) dan = np.asarray([4, 5, 6, 7], np.int32) us = dict(tick=np.full(4, 900, np.int32), neuron_idx=dan) combined = {k: np.concatenate((source[k], us[k])) for k in source} for name, value in (('cs_requests.npz', source), ('us_requests.npz', us), ('requests.npz', combined), ('source-mask.npz', dict(source_mask=np.asarray(([1] if not no_cs else []) + [2]*4, np.uint8)))): arrays[str(case / name)] = value values = {str(case / 'spec.json'): spec, population_file: {'artifact': 'actual-static-populations'}} source_pin = None; sampler = dict(inputs={}) if not no_cs: source_path = exp / f'inputs/generated/{csid}.npz'; audit_path = exp / f'inputs/generated/{csid}.json' values[str(source_path)] = {'array-artifact': 'mock-sparse-CS'} arrays[str(source_path)] = source source_pin = dict(path=str(source_path), sha256=N.sha(values[str(source_path)])) values[str(audit_path)] = dict(descriptor=descriptor, requests_sha256=source_pin['sha256'], audit=dict(seed=seed, generator_initial_seed=seed, seed_bits=63, seed_conversion='exact integer; no truncation/modulo/offset', private_generator=True, matrix_draw_calls=1, uniform_matrix_shape=[1200, 1], CS_window=[0, 1000])) sampler['inputs'][csid] = dict(requests=source_pin, audit=dict(path=str(audit_path), sha256=N.sha(values[str(audit_path)]))) values[str(case / 'input-binding.json')] = dict(task_id=task['task_id'], CS_descriptor_id=csid, CS_source=source_pin, DAN_type='PAM15', population_file=dict(path=population_file, sha256=N.sha(values[population_file])), DAN_membership_sha256=hashlib.sha256(dan.tobytes()).hexdigest(), composition=dict(policy='error', cross_source_conflict_count=0)) reader = MockReader(values) reader.arrays = lambda p: {k: v.copy() for k, v in arrays[str(p)].items()} args = dict(template=template, inputs=dict(task_inputs={task['task_id']: csid}, descriptors={} if no_cs else {csid: descriptor}), sampler=sampler, populations=dict(PAM15=dan), population_file=population_file, output_cap=123) return reader, exp, task, dict(case_dir=str(case)), args, values, arrays, case class NormalMetricTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.nodes, cls.pops, cls.registry, cls.authority, cls.probes, cls.arms = fixture() cls.full_reports = {} original = N.G.evaluate_acquisition def capture(*args, **kwargs): result = original(*args, **kwargs) cls.full_reports[(result['mode'], result['dan_type'], result['conditioned_cue'])] = result return result with patch.object(N.G, 'evaluate_acquisition', side_effect=capture): cls.report = N.scientific_reports(cls.probes, cls.arms, registry=cls.registry, authority=cls.authority, nodes=cls.nodes, populations=cls.pops, load_state=state_for) def test_full_fixed_curves_and_eight_same_replicate_gates(self): self.assertEqual(len(self.report['cells']), 8) self.assertEqual(len(self.report['curves']), 1960) self.assertTrue(self.report['cohort']['all_functional_predicates_pass']) self.assertTrue(all(r['successful_same_replicate_conjunctions'] == 3 for r in self.report['cells'])) self.assertTrue(all('acquisition_metric_inputs' not in r for r in self.report['cells'])) self.assertTrue(all(not r['scientific_learning_pass_claimed'] for r in self.report['cells'])) self.assertEqual({r['checkpoint'] for r in self.report['curves']}, set(N.POINTS)) self.assertTrue(all(r['metrics']['rawS'] == 0 for r in self.report['curves'] if r['checkpoint'] == 0)) self.assertTrue(all(not isinstance(v, list) for r in self.report['curves'] for cue in r['metrics']['cues'].values() for group in cue['populations'].values() for v in group.values())) def test_legacy_mapping_exactly_matches_frozen_wrapper(self): expected = N.G.legacy_h_cohort_audit(list(self.full_reports.values()), registry=self.registry, authority=self.authority, declared_mapping=N.G.H_MAPPING) actual = N.legacy_audit(self.report['cells'], N.G.H_MAPPING) self.assertEqual(actual, expected) self.assertEqual(len(actual['groups']), 4) self.assertTrue(all(set(g['control_sources']) == {'PAM15', 'PPL103'} for g in actual['groups'])) with self.assertRaises(ValueError): N.legacy_audit(self.report['cells'], None) def test_training_evaluation_seed_collision_and_missing_control_fail_closed(self): registry = copy.deepcopy(self.registry) block = next(iter(registry['blocks'].values())) block['evaluation_sources'][N.M.CUES[0]]['seed'] = block['training_sources'][0]['seed'] with self.assertRaisesRegex(ValueError, 'collision'): N.G.validate_registry(registry, expected_sha256=N.G.descriptor_sha256(registry)) report = next(iter(self.full_reports.values())) rows = copy.deepcopy(report['acquisition_metric_inputs']) del rows[0]['arms']['DAN-only'] with self.assertRaisesRegex(ValueError, 'six actual controls'): N.G.evaluate_acquisition(rows, assay='normal', mode=report['mode'], dan_type=report['dan_type'], conditioned_cue=report['conditioned_cue'], registry=self.registry, authority=self.authority) def test_state_mutation_and_absent_probe_never_become_zero_metrics(self): probes = dict(self.probes); del probes[next(iter(probes))] with self.assertRaises((KeyError, ValueError)): N.scientific_reports(probes, self.arms, registry=self.registry, authority=self.authority, nodes=self.nodes, populations=self.pops, load_state=state_for) def corrupt(record): state = state_for(record); state['g'][0] = np.nan; return state with self.assertRaisesRegex(ValueError, 'Nonfinite'): N.scientific_reports(self.probes, self.arms, registry=self.registry, authority=self.authority, nodes=self.nodes, populations=self.pops, load_state=corrupt) def test_actual_parent_receipt_nested_layout_and_exit_failure(self): exp, values, final, proc, _ = process_fixture() evidence = N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final) observed = evidence.process('input-generation', 'sample'); evidence.finish() self.assertEqual(observed['process_path'], str(exp / 'receipts/input-generation/process.json')) values[str(proc)]['exit_code'] = 1 # Even a caller report saying pass_all cannot rescue a real exit failure. r = MockReader(values) entry = next(v for v in values.values() if 'new_outputs' in v) for row in entry['new_outputs']: if row['path'] == str(proc): row.update(r.inventory[str(proc)]) with self.assertRaisesRegex(ValueError, 'Actual parent exit'): N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final).process('input-generation', 'sample') def test_result_process_must_be_ledger_owned_not_only_metadata(self): exp, values, final, proc, entry_path = process_fixture() entry = values[str(entry_path)] entry['new_outputs'] = [r for r in entry['new_outputs'] if r['path'] != str(proc)] with self.assertRaisesRegex(ValueError, 'not owned'): N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final).process('input-generation', 'sample') bad = dict(final, uncommitted_entry_ids=['unknown-child']) with self.assertRaisesRegex(ValueError, 'Incomplete'): N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], bad) def test_reader_per_invocation_actual_sha_cache_and_no_boolean_substitution(self): path = Path.cwd() / 'mock-artifact.json'; raw = b'{"actual":true}' identity = dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) with patch.object(Path, 'is_symlink', return_value=False), patch.object(Path, 'is_junction', return_value=False, create=True), \ patch.object(Path, 'is_file', return_value=True), patch.object(Path, 'open', side_effect=lambda *a, **k: io.BytesIO(raw)) as opened: reader = N.Reader({str(path): identity}) reader.check(path); reader.check(path) self.assertEqual(opened.call_count, 1) N.Reader({str(path): identity}).check(path) self.assertEqual(opened.call_count, 2) with self.assertRaisesRegex(ValueError, 'Actual file checksum'): N.Reader({str(path): dict(bytes=len(raw), sha256='0' * 64)}).check(path) def test_real_inventory_process_count_and_no_followup(self): tasks = [dict(task_id='train-' + str(i), kind='train') for i in range(30000)] tasks += [dict(task_id='probe-' + str(i), kind='probe') for i in range(3920)] entries = N.expected_entries(dict(tasks=tasks)) self.assertEqual(len(entries), 93921) self.assertEqual(len(set(entries)), 93921) self.assertEqual(entries[0], 'input-generation') self.assertTrue(entries[-1].endswith('--brain')) def test_original_train_cs_us_seed_and_model_checked_independently(self): reader, exp, task, job, args, values, arrays, case = case_fixture() N.audit_case(reader, exp, task, job, **args) arrays[str(case / 'us_requests.npz')]['tick'][0] = 901 with self.assertRaisesRegex(ValueError, 'sorted unique|scheduled US'): N.audit_case(reader, exp, task, job, **args) arrays[str(case / 'us_requests.npz')]['tick'][0] = 900 values[str(case / 'spec.json')]['seed'] += 1 with self.assertRaisesRegex(ValueError, 'model/reset/clock'): N.audit_case(reader, exp, task, job, **args) values[str(case / 'spec.json')]['seed'] -= 1 generated = next(v for v in values.values() if 'audit' in v) generated['audit']['generator_initial_seed'] %= 2**32 with self.assertRaisesRegex(ValueError, 'sampler seed'): N.audit_case(reader, exp, task, job, **args) def test_no_cs_is_empty_and_dan_only_still_uses_original_fixed_us(self): reader, exp, task, job, args, values, arrays, case = case_fixture(no_cs=True) N.audit_case(reader, exp, task, job, **args) arrays[str(case / 'cs_requests.npz')] = dict(tick=np.asarray([10], np.int32), neuron_idx=np.asarray([3], np.int32)) with self.assertRaisesRegex(ValueError, 'case CS'): N.audit_case(reader, exp, task, job, **args) if __name__ == '__main__': unittest.main()