Digital Drosophila Archive
319 lines
18 KiB
Python
319 lines
18 KiB
Python
"""Synthetic arrays/declared receipts only; no real recordings or science run."""
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import unittest
|
|
|
|
import numpy as np
|
|
|
|
import metrics as m
|
|
|
|
|
|
NODES = np.arange(6, dtype=np.int32)
|
|
POPS = {'MBON': np.array([0, 1, 2], np.int32), 'legacyDownstream': np.arange(6, dtype=np.int32),
|
|
'nonMBONDownstream': np.array([3, 4, 5], np.int32),
|
|
'MBON01': np.array([0], np.int32), 'MBON12': np.array([1], np.int32)}
|
|
|
|
|
|
def state(counts=(1, 1, 0, 1, 0, 0), *, times=None, g=None):
|
|
events = [(10 + 20 * k if times is None else times[i][k], i) for i, count in enumerate(counts) for k in range(count)]
|
|
events.sort()
|
|
return dict(step=np.array([t for t, _ in events], np.int32),
|
|
neuron_idx=np.array([i for _, i in events], np.int32), counts=np.array(counts, np.int32),
|
|
v=np.full(6, -.052, np.float64), g=np.zeros(6, np.float64) if g is None else np.array(g, np.float64),
|
|
ref=np.zeros(6, np.int32))
|
|
|
|
|
|
def effect(*, direction=1, gain=True, g_effect=True, conditioned=m.CUES[0], neutral_g=0.):
|
|
pre = {cue: state() for cue in m.CUES}
|
|
post = {cue: state() for cue in m.CUES}
|
|
counts = [1, 1, 0, 1, 0, 0]
|
|
counts[1 if direction > 0 else 0] += abs(direction)
|
|
if gain:
|
|
counts[3] += abs(direction)
|
|
g = np.zeros(6, np.float64)
|
|
if g_effect:
|
|
g[0] = .002
|
|
post[conditioned] = state(counts, g=g)
|
|
neutral = next(c for c in m.CUES if c != conditioned)
|
|
post[neutral]['g'][0] = neutral_g
|
|
return m.arm_metrics(pre, post, conditioned_cue=conditioned, nodes=NODES, populations=POPS)
|
|
|
|
|
|
def inventory(*, assay='normal', dan='PAM15', paired=None):
|
|
registration = {f'rep{i}': [f'training-source-{i}'] for i in range(5)}
|
|
rows = []
|
|
for i in range(5):
|
|
arms = {}
|
|
for name in m.ARMS:
|
|
metric = effect(direction=0, g_effect=False) if name != 'paired' else (
|
|
effect(direction=1 if dan == 'PAM15' else -1) if paired is None else copy.deepcopy(paired[i]))
|
|
evidence = dict(complete=True, recording_verified=True,
|
|
receipt_sha256=hashlib.sha256(f'{i}:{name}:{assay}'.encode()).hexdigest(), assay=assay,
|
|
epoch_duration_ms=120. if assay == 'normal' else 180., cs_duration_ms=100.,
|
|
us_tick=900 if assay == 'normal' else 1500, completed_epochs=150 if name == 'unpaired' else 100,
|
|
reinforcement_opportunities=0 if name == 'CS-only' else 50, terminal_pairings=50)
|
|
arms[name] = dict(metrics=metric, execution=evidence)
|
|
rows.append(dict(replicate_id=f'rep{i}', replicate_unit='independent_training',
|
|
training_stream_ids=registration[f'rep{i}'], evaluation_stream_ids=[f'evaluation-source-{i}'],
|
|
assay=assay, mode='physiological', dan_type=dan, conditioned_cue=m.CUES[0], arms=arms))
|
|
return rows, registration
|
|
|
|
|
|
def evaluate(rows, registration, *, assay='normal', dan='PAM15'):
|
|
return m.evaluate_acquisition(rows, assay=assay, mode='physiological', dan_type=dan,
|
|
conditioned_cue=m.CUES[0], registered_training_streams=registration)
|
|
|
|
|
|
class StateTests(unittest.TestCase):
|
|
def test_every_key_validated_and_no_input_mutation(self):
|
|
a, b = state(), state((1, 2, 0, 2, 0, 0), g=[.001, -.004, 0, 0, 0, 0])
|
|
before = {key: value.tobytes() for key, value in a.items()}
|
|
r = m.functional_summary(a, b, nodes=NODES, populations=POPS, steps=1200)
|
|
self.assertEqual(r['MBON_final_delta_g_max_abs'], .004)
|
|
self.assertEqual(r['MBON_final_delta_g_units'], 'SI volts')
|
|
self.assertIn('not time peak', r['MBON_final_delta_g_scope'])
|
|
self.assertEqual(r['wholebrain']['spike_count_l1'], 2)
|
|
self.assertEqual(before, {key: value.tobytes() for key, value in a.items()})
|
|
json.dumps(r, allow_nan=False)
|
|
|
|
def test_first_event_minimum_not_last_and_zero_tick_is_defined(self):
|
|
a = state((3, 0, 0, 0, 0, 0), times=[[0, 20, 100], [], [], [], [], []])
|
|
r = m.validate_state(a, nodes=NODES, steps=1200)
|
|
self.assertEqual(r['first_tick'].tolist(), [0, -1, -1, -1, -1, -1])
|
|
|
|
def test_missing_latencies_never_zero_filled(self):
|
|
a, b = state((0, 0, 0, 0, 0, 0)), state((1, 0, 0, 0, 0, 0))
|
|
r = m.functional_summary(a, b, nodes=NODES, populations=POPS, steps=1200)['populations']['MBON']
|
|
self.assertIsNone(r['pre_first_latency_ms'])
|
|
self.assertIsNone(r['first_latency_delta_ms'])
|
|
self.assertFalse(r['defined_first_latency_changed'])
|
|
self.assertEqual(r['first_latency_became_defined_neurons'], 1)
|
|
self.assertGreater(r['spike_count_l1'], 0)
|
|
|
|
def test_nonfinite_input_all_neurons_not_only_scored_targets(self):
|
|
for key in ('v', 'g'):
|
|
for value in (np.nan, np.inf, -np.inf):
|
|
with self.subTest(key=key, value=value):
|
|
a = state(); a[key][5] = value
|
|
with self.assertRaisesRegex(ValueError, 'Nonfinite'):
|
|
m.functional_summary(a, state(), nodes=NODES, populations=POPS, steps=1200)
|
|
|
|
def test_nonfinite_derived_difference_rejected(self):
|
|
a, b = state(), state()
|
|
a['g'][0], b['g'][0] = -1e308, 1e308
|
|
with self.assertRaisesRegex(ValueError, 'Nonfinite derived'):
|
|
m.functional_summary(a, b, nodes=NODES, populations=POPS, steps=1200)
|
|
|
|
def test_bad_state_dtype_shape_counts_order_bounds_rejected(self):
|
|
variants = []
|
|
a = state(); a['counts'] = a['counts'].astype(np.int64); variants.append(a)
|
|
a = state(); a['counts'][0] += 1; variants.append(a)
|
|
a = state(); a['step'][0] = -1; variants.append(a)
|
|
a = state(); a['neuron_idx'][0] = 99; variants.append(a)
|
|
a = state(); a['ref'][0] = 23; variants.append(a)
|
|
a = state(); a['v'] = a['v'][:-1]; variants.append(a)
|
|
a = state(); a['neuron_idx'][1] = 0; variants.append(a)
|
|
a = state(); a['extra'] = np.zeros(1); variants.append(a)
|
|
for a in variants:
|
|
with self.subTest(keys=list(a)), self.assertRaises(ValueError):
|
|
m.validate_state(a, nodes=NODES, steps=1200)
|
|
|
|
def test_empty_wrong_duplicate_and_overlapping_memberships_rejected(self):
|
|
for key, value in [('MBON12', []), ('MBON01', [0, 0]), ('MBON12', [3]),
|
|
('MBON12', [0]), ('nonMBONDownstream', [2, 3]), ('MBON', [0, 1, 99]),
|
|
('nonMBONDownstream', [3, 4]), ('legacyDownstream', [0, 1, 3, 4, 5])]:
|
|
populations = dict(POPS, **{key: np.array(value, np.int32)})
|
|
with self.subTest(key=key, value=value), self.assertRaises(ValueError):
|
|
m.validate_populations(NODES, populations)
|
|
|
|
def test_legacy_downstream_retains_mbon_and_new_setdiff_is_explicit(self):
|
|
result = m.validate_populations(NODES, POPS)
|
|
self.assertEqual(result['indices']['legacyDownstream'], [0, 1, 2, 3, 4, 5])
|
|
self.assertEqual(result['indices']['nonMBONDownstream'], [3, 4, 5])
|
|
self.assertIn('NEW hypothesis', result['population_semantics']['nonMBONDownstream'])
|
|
old_alias = dict(POPS)
|
|
old_alias['trueDownstream'] = old_alias.pop('nonMBONDownstream')
|
|
with self.assertRaisesRegex(ValueError, 'exactly'):
|
|
m.validate_populations(NODES, old_alias)
|
|
|
|
|
|
class ContrastTests(unittest.TestCase):
|
|
def test_fixed_types_not_mask_and_type_mean_not_population_sum(self):
|
|
p = dict(POPS, MBON=np.array([0, 1, 2], np.int32), MBON01=np.array([0], np.int32), MBON12=np.array([1, 2], np.int32))
|
|
pre = {c: state() for c in m.CUES}
|
|
post = {c: state() for c in m.CUES}; post[m.CUES[0]] = state((1, 3, 1, 2, 0, 0))
|
|
r = m.arm_metrics(pre, post, conditioned_cue=m.CUES[0], nodes=NODES, populations=p)
|
|
self.assertEqual(r['rawB']['post'][m.CUES[0]], 1.)
|
|
self.assertEqual(r['rawS'], 1.5)
|
|
self.assertFalse(r['B_is_decoder'])
|
|
|
|
def test_reciprocal_role_keeps_signed_orientation(self):
|
|
first, second = effect(), effect(conditioned=m.CUES[1])
|
|
self.assertEqual(first['rawS'], second['rawS'])
|
|
self.assertEqual(second['conditioned_cue'], m.CUES[1])
|
|
|
|
def test_raw_labels_not_aliases(self):
|
|
with self.assertRaises(ValueError):
|
|
m.arm_metrics({'A': state()}, {'B': state()}, conditioned_cue='A', nodes=NODES, populations=POPS)
|
|
|
|
def test_legacy_and_non_mbon_gains_are_separate_not_an_alias(self):
|
|
result = effect(gain=False)
|
|
self.assertEqual(result['separation']['MBON']['gain'], 1.)
|
|
self.assertEqual(result['separation']['legacyDownstream']['gain'], 1.)
|
|
self.assertEqual(result['separation']['nonMBONDownstream']['gain'], 0.)
|
|
pre = {cue: state((4, 0, 0, 0, 0, 0) if cue == m.CUES[0] else (0, 0, 0, 0, 0, 0)) for cue in m.CUES}
|
|
post = {cue: state((3, 0, 0, 1, 0, 0) if cue == m.CUES[0] else (0, 0, 0, 0, 0, 0)) for cue in m.CUES}
|
|
result = m.arm_metrics(pre, post, conditioned_cue=m.CUES[0], nodes=NODES, populations=POPS)
|
|
self.assertLess(result['separation']['legacyDownstream']['gain'], 0.)
|
|
self.assertGreater(result['separation']['nonMBONDownstream']['gain'], 0.)
|
|
|
|
|
|
class GateTests(unittest.TestCase):
|
|
def test_both_directions_and_suites_pass_only_as_draft_proposal(self):
|
|
for assay in ('normal', 'delayed'):
|
|
for dan in m.DAN_TYPES:
|
|
with self.subTest(assay=assay, dan=dan):
|
|
rows, registration = inventory(assay=assay, dan=dan)
|
|
r = evaluate(rows, registration, assay=assay, dan=dan)
|
|
self.assertTrue(r['proposal_gate_pass'])
|
|
self.assertEqual(r['actual_control_receipts'], 30)
|
|
self.assertEqual(r['successful_same_replicate_conjunctions'], 5)
|
|
self.assertFalse(r['scientific_learning_pass_claimed'])
|
|
self.assertFalse(r['candidate_generated'])
|
|
self.assertTrue(all(v['rawS'] < 0 for v in r['replicates']) if dan == 'PPL103' else
|
|
all(v['rawS'] > 0 for v in r['replicates']))
|
|
json.dumps(r, allow_nan=False)
|
|
|
|
def test_opposite_or_zero_signed_effect_is_failure_not_abs_success(self):
|
|
for direction in (-1, 0):
|
|
rows, registry = inventory(paired=[effect(direction=direction)] * 5)
|
|
r = evaluate(rows, registry)
|
|
self.assertFalse(r['proposal_gate_pass'])
|
|
self.assertEqual(r['successful_same_replicate_conjunctions'], 0)
|
|
|
|
def test_equal_control_is_failure(self):
|
|
rows, registry = inventory()
|
|
for row in rows:
|
|
row['arms']['CS-only']['metrics'] = copy.deepcopy(row['arms']['paired']['metrics'])
|
|
self.assertFalse(evaluate(rows, registry)['proposal_gate_pass'])
|
|
|
|
def test_same_replicate_conjunction_not_majority_of_each_predicate(self):
|
|
effects = [effect(), effect(), effect(gain=False), effect(g_effect=False), effect(direction=-1)]
|
|
rows, registry = inventory(paired=effects)
|
|
r = evaluate(rows, registry)
|
|
self.assertEqual(r['successful_same_replicate_conjunctions'], 2)
|
|
self.assertFalse(r['proposal_gate_pass'])
|
|
|
|
def test_exact_three_of_five_pass(self):
|
|
rows, registry = inventory(paired=[effect()] * 3 + [effect(direction=0)] * 2)
|
|
self.assertTrue(evaluate(rows, registry)['proposal_gate_pass'])
|
|
|
|
def test_positive_legacy_gain_cannot_rescue_new_non_mbon_gate(self):
|
|
rows, registry = inventory(paired=[effect(gain=False)] * 5)
|
|
result = evaluate(rows, registry)
|
|
self.assertFalse(result['proposal_gate_pass'])
|
|
for row in result['replicates']:
|
|
self.assertTrue(row['predicates']['legacyDownstream_AB_euclidean_gain_positive'])
|
|
self.assertFalse(row['predicates']['nonMBONDownstream_AB_euclidean_gain_positive'])
|
|
self.assertEqual(row['population_separation_gains']['legacyDownstream'], 1.)
|
|
self.assertEqual(row['population_separation_gains']['nonMBONDownstream'], 0.)
|
|
self.assertEqual(row['legacy_audit']['H_like_downstream_population'], 'legacyDownstream')
|
|
self.assertFalse(row['new_hypothesis_audit']['inherited_from_old_H'])
|
|
|
|
def test_population_metadata_or_overlapping_raw_vectors_cannot_be_forged(self):
|
|
for mutation in ('semantics', 'setdiff', 'subset_final'):
|
|
rows, registry = inventory()
|
|
summary = rows[0]['arms']['paired']['metrics']['cues'][m.CUES[0]]
|
|
if mutation == 'semantics':
|
|
summary['membership']['population_semantics']['nonMBONDownstream'] = 'old H population'
|
|
elif mutation == 'setdiff':
|
|
ids = summary['membership']['indices']
|
|
ids['nonMBONDownstream'] = [2, 3, 4]
|
|
summary['membership']['membership_sha256'] = m._json_hash(ids)
|
|
summary['populations']['nonMBONDownstream']['neuron_idx'] = [2, 3, 4]
|
|
else:
|
|
# Self-consistent terminal deltas can still contradict the full parent population.
|
|
child = summary['populations']['nonMBONDownstream']
|
|
child['pre_final_v'][0] += .001
|
|
child['post_final_v'][0] += .001
|
|
with self.subTest(mutation=mutation), self.assertRaises(ValueError):
|
|
evaluate(rows, registry)
|
|
|
|
def test_missing_control_incomplete_receipt_and_wrong_delayed_schedule_rejected(self):
|
|
for mutation in ('missing', 'incomplete', 'timing', 'epochs', 'duplicate_receipt', 'best_checkpoint'):
|
|
rows, registry = inventory()
|
|
if mutation == 'missing':
|
|
rows[0]['arms'].pop('unpaired')
|
|
else:
|
|
e = rows[0]['arms']['unpaired']['execution']
|
|
if mutation == 'incomplete': e['complete'] = False
|
|
if mutation == 'timing': e['us_tick'] = 1500
|
|
if mutation == 'epochs': e['completed_epochs'] = 100
|
|
if mutation == 'duplicate_receipt': e['receipt_sha256'] = rows[0]['arms']['paired']['execution']['receipt_sha256']
|
|
if mutation == 'best_checkpoint': e['terminal_pairings'] = 20
|
|
with self.subTest(mutation=mutation), self.assertRaises(ValueError): evaluate(rows, registry)
|
|
|
|
def test_evaluation_draws_cannot_be_training_replicates(self):
|
|
for mutation in ('unit', 'duplicate_id', 'training_overlap', 'eval_training_overlap', 'eval_overlap', 'extra_draw'):
|
|
rows, registry = inventory()
|
|
if mutation == 'unit': rows[0]['replicate_unit'] = 'evaluation'
|
|
if mutation == 'duplicate_id': rows[1]['replicate_id'] = rows[0]['replicate_id']
|
|
if mutation == 'training_overlap': registry['rep1'] = registry['rep0']
|
|
if mutation == 'eval_training_overlap': rows[0]['evaluation_stream_ids'] = registry['rep0']
|
|
if mutation == 'eval_overlap': rows[1]['evaluation_stream_ids'] = rows[0]['evaluation_stream_ids']
|
|
if mutation == 'extra_draw': rows.append(copy.deepcopy(rows[0]))
|
|
with self.subTest(mutation=mutation), self.assertRaises(ValueError): evaluate(rows, registry)
|
|
|
|
def test_raw_nan_forged_B_g_function_and_separation_rejected(self):
|
|
for mutation in ('nan', 'rawB', 'g', 'count', 'separation', 'latency'):
|
|
rows, registry = inventory()
|
|
metric = rows[0]['arms']['paired']['metrics']
|
|
group = metric['cues'][m.CUES[0]]['populations']['MBON']
|
|
if mutation == 'nan': metric['rawS'] = np.inf
|
|
if mutation == 'rawB': metric['rawB']['post'][m.CUES[0]] = 100.
|
|
if mutation == 'g': group['final_delta_g_max_abs'] = 100.
|
|
if mutation == 'count': group['spike_count_l1'] = 100
|
|
if mutation == 'separation': metric['separation']['MBON']['gain'] = 100.
|
|
if mutation == 'latency': group['defined_first_latency_changed'] = True
|
|
with self.subTest(mutation=mutation), self.assertRaises(ValueError): evaluate(rows, registry)
|
|
|
|
def test_control_baseline_and_cell_pooling_rejected(self):
|
|
for mutation in ('baseline', 'mode', 'DAN', 'assay', 'cue'):
|
|
rows, registry = inventory()
|
|
if mutation == 'baseline': rows[0]['arms']['frozen']['metrics']['cues'][m.CUES[0]]['pre_state_raw_sha256']['g'] = 'a' * 64
|
|
if mutation == 'mode': rows[0]['mode'] = 'matched_energy'
|
|
if mutation == 'DAN': rows[0]['dan_type'] = 'PAM-positive'
|
|
if mutation == 'assay': rows[0]['assay'] = 'delayed'
|
|
if mutation == 'cue': rows[0]['conditioned_cue'] = m.CUES[1]
|
|
with self.subTest(mutation=mutation), self.assertRaises(ValueError): evaluate(rows, registry)
|
|
|
|
def test_reversal_retention_remain_missing_not_fabricated_pass(self):
|
|
for assay in ('reversal', 'retention'):
|
|
r = m.unavailable_assay(assay)
|
|
self.assertEqual(r['status'], 'NOT-IMPLEMENTED')
|
|
self.assertIsNone(r['proposal_gate_pass'])
|
|
self.assertTrue(r['missing'])
|
|
|
|
def test_legacy_mean_control_audit_separate_and_strict_not_new_signed_gate(self):
|
|
raw = {'appetitive-conditioning': [2.] * 5, 'aversive-conditioning': [1.] * 5,
|
|
'd-zero-control': [0.] * 5, 'random-reward-control': [.5] * 5, 'frozen-brain-control': [0.] * 5}
|
|
r = m.legacy_h_mean_control_audit(raw)
|
|
self.assertTrue(r['predicate_pass'])
|
|
self.assertFalse(r['full_old_H_gate_computed'])
|
|
raw['random-reward-control'] = [1.] * 5
|
|
self.assertFalse(m.legacy_h_mean_control_audit(raw)['predicate_pass'])
|
|
raw.pop('d-zero-control')
|
|
with self.assertRaises(ValueError): m.legacy_h_mean_control_audit(raw)
|
|
|
|
def test_legacy_nan_or_evaluation_count_not_admitted(self):
|
|
raw = {'appetitive-conditioning': [2.] * 5, 'aversive-conditioning': [1.] * 5,
|
|
'd-zero-control': [0.] * 5, 'random-reward-control': [.5] * 5, 'frozen-brain-control': [0.] * 5}
|
|
for values in ([0.] * 25, [0., 0., np.nan, 0., 0.]):
|
|
raw['frozen-brain-control'] = values
|
|
with self.assertRaises(ValueError): m.legacy_h_mean_control_audit(raw)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|