Add post-exit normal acquisition audit with frozen gate regression
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,176 @@
|
||||
"""Synthetic six-neuron CPU fixtures only; NO experiment states or outcomes.
|
||||
|
||||
The fixture is generated by SHA-pinned real frozen G/M through normal_metrics,
|
||||
not by a second hand-written imitation of its predicates or report schema.
|
||||
"""
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
import normal_outcome as O
|
||||
|
||||
|
||||
def load_fixture_module():
|
||||
pins = {
|
||||
'normal_metrics.py': O.ANALYSIS_SOURCE_SHA256,
|
||||
'test_normal_metrics.py': '204586b30edbe57bb294c1d8f8ad309863e5bcb3baad32118955ac3d4adcdefb',
|
||||
}
|
||||
candidates = (Path(__file__).resolve().parent.parent / 'normal-conditioning-v1',
|
||||
Path('/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1'))
|
||||
source = next((p for p in candidates if all((p / n).is_file() and
|
||||
hashlib.sha256((p / n).read_bytes()).hexdigest() == sha for n, sha in pins.items())), None)
|
||||
if source is None:
|
||||
raise AssertionError('No exact pinned frozen synthetic fixture source at either declared location')
|
||||
spec = importlib.util.spec_from_file_location('_outcome_frozen_fixture', source / 'test_normal_metrics.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def synthetic_report(fixture_module, scenario='pass'):
|
||||
F = fixture_module; N = F.N
|
||||
nodes, pops, registry, authority, probes, arms = F.fixture()
|
||||
first = next(iter(registry['blocks']))
|
||||
target = registry['blocks'][first]
|
||||
# Only one cell changes; all intermediate successes remain available but
|
||||
# must not rescue a failing terminal checkpoint.
|
||||
for (block, arm, point, cue), record in probes.items():
|
||||
b = registry['blocks'][block]
|
||||
same = all(b[k] == target[k] for k in ('mode', 'dan_type', 'conditioned_cue'))
|
||||
if scenario == 'terminal_fail' and same and b['replicate'] == 3 and point == 50:
|
||||
record['active'] = False
|
||||
if scenario in ('h_fail', 'h_equal') and same and arm == 'd_zero' and point > 0:
|
||||
record['active'] = N.ODORS[cue] != b['conditioned_cue'] and (
|
||||
scenario == 'h_fail' or b['replicate'] <= 3)
|
||||
scientific = N.scientific_reports(probes, arms, registry=registry, authority=authority,
|
||||
nodes=nodes, populations=pops, load_state=F.state_for)
|
||||
scientific['legacy_H'] = N.legacy_audit(scientific['cells'], N.G.H_MAPPING)
|
||||
return dict(schema='normal-scientific-analysis-v1', status='CONDITIONAL-PENDING-OUTER-INTEGRITY',
|
||||
registration_sha256=registry['protocol_sha256'], normal_plan_sha256='5' * 64,
|
||||
registry=registry, authority=authority, analysis=scientific,
|
||||
analyzed_probe_calls=3920, curve_points=1960, consumed_file_count=123,
|
||||
subsequent_assays_status='NOT-RUN', subsequent_assays_authorized=False,
|
||||
candidate_generated=False, learning_pass_claimed=False, scientific_learning_pass_claimed=False,
|
||||
data_backed_claimed=False, external_full_tree_anatomy_historical_integrity_verified_here=False,
|
||||
retained_raw_evidence='synthetic arrays generated in memory; no actual experiment read')
|
||||
|
||||
|
||||
def binding(report):
|
||||
# This is simulated external evidence, not an actual audit or authorization.
|
||||
return dict(registration_sha256=report['registration_sha256'], terminal_reconciliation_pass=True,
|
||||
historical_integrity_after_pass=True, analysis_sha256=O.analysis_digest(report),
|
||||
actual_parent_exit_code=0, analyzed_source_sha256=O.ANALYSIS_SOURCE_SHA256,
|
||||
analysis_file_sha256='a' * 64)
|
||||
|
||||
|
||||
class NormalOutcomeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
F = load_fixture_module()
|
||||
cls.reports = {name: synthetic_report(F, name) for name in ('pass', 'terminal_fail', 'h_fail', 'h_equal')}
|
||||
|
||||
def outcome(self, report, evidence=None):
|
||||
return O.summarize_normal_analysis(report, binding(report) if evidence is None else evidence)
|
||||
|
||||
def test_pinned_real_frozen_gates_compact_output_pass_and_no_mutation(self):
|
||||
report = self.reports['pass']; before = O.analysis_digest(report)
|
||||
actual = self.outcome(report)
|
||||
self.assertEqual(actual['status'], 'NORMAL-ACQUISITION-PASS', actual)
|
||||
self.assertTrue(actual['normal_acquisition_pass'])
|
||||
self.assertEqual(len(actual['cell_table']), 8)
|
||||
self.assertTrue(all(r['successful_replicates'] == 3 for r in actual['cell_table']))
|
||||
self.assertEqual(len(report['analysis']['curves']), 1960)
|
||||
self.assertEqual(before, O.analysis_digest(report))
|
||||
for key in ('learning_pass_claimed', 'usable_learning_pass', 'candidate_generated',
|
||||
'scientific_learning_pass_claimed', 'subsequent_assays_authorized',
|
||||
'outer_evidence_authenticated_here', 'raw_scientific_arrays_recomputed_here'):
|
||||
self.assertIs(actual[key], False)
|
||||
self.assertEqual(set(actual['subsequent_assays'].values()), {'NOT-RUN'})
|
||||
|
||||
def test_terminal_same_replicate_fail_cannot_use_intermediate_or_other_cell(self):
|
||||
report = self.reports['terminal_fail']; actual = self.outcome(report)
|
||||
self.assertEqual(actual['status'], 'NORMAL-ACQUISITION-FAIL', actual)
|
||||
self.assertFalse(actual['all_eight_cell_conjunctions_pass'])
|
||||
self.assertTrue(actual['both_H_control_sources_all_groups_pass'])
|
||||
self.assertEqual(sorted(r['successful_replicates'] for r in actual['cell_table']), [2] + [3] * 7)
|
||||
|
||||
def test_H_one_control_branch_failure_and_equality_are_strict(self):
|
||||
for name in ('h_fail', 'h_equal'):
|
||||
with self.subTest(name=name):
|
||||
report = self.reports[name]; actual = self.outcome(report)
|
||||
self.assertEqual(actual['status'], 'NORMAL-ACQUISITION-FAIL', actual)
|
||||
self.assertTrue(actual['all_eight_cell_conjunctions_pass'])
|
||||
self.assertFalse(actual['both_H_control_sources_all_groups_pass'])
|
||||
audits = [v for g in report['analysis']['legacy_H']['groups'] for v in g['control_sources'].values()]
|
||||
self.assertEqual(sum(not v['predicate_pass'] for v in audits), 1)
|
||||
if name == 'h_equal':
|
||||
bad = next(v for v in audits if not v['predicate_pass'])
|
||||
self.assertEqual(bad['condition_floor'], bad['control_ceiling'])
|
||||
|
||||
def test_missing_outer_and_failed_analysis_are_incomplete_not_scientific_fail(self):
|
||||
report = self.reports['pass']
|
||||
for key in O.OUTER_KEYS:
|
||||
value = binding(report); del value[key]
|
||||
actual = self.outcome(report, value)
|
||||
self.assertEqual(actual['status'], 'INCOMPLETE', (key, actual))
|
||||
self.assertIsNone(actual['normal_acquisition_pass'])
|
||||
value = binding(report); value['actual_parent_exit_code'] = 9
|
||||
self.assertEqual(self.outcome(report, value)['status'], 'INCOMPLETE')
|
||||
|
||||
def test_actual_source_digest_registration_and_integrity_fail_closed(self):
|
||||
report = self.reports['pass']
|
||||
for key, wrong in (
|
||||
('terminal_reconciliation_pass', False), ('historical_integrity_after_pass', False),
|
||||
('terminal_reconciliation_pass', 1), ('analyzed_source_sha256', '0' * 64),
|
||||
('analysis_sha256', '0' * 64), ('registration_sha256', '0' * 64),
|
||||
('actual_parent_exit_code', False),
|
||||
):
|
||||
value = binding(report); value[key] = wrong
|
||||
actual = self.outcome(report, value)
|
||||
self.assertEqual(actual['status'], 'INTEGRITY-FAIL', (key, actual))
|
||||
self.assertIsNone(actual['normal_acquisition_pass'])
|
||||
|
||||
def test_exact_cell_replicate_curve_inventory_missing_not_scientific_negative(self):
|
||||
for collection in ('cells', 'curves'):
|
||||
report = copy.deepcopy(self.reports['pass']); report['analysis'][collection].pop()
|
||||
self.assertEqual(self.outcome(report)['status'], 'INCOMPLETE')
|
||||
report = copy.deepcopy(self.reports['pass'])
|
||||
report['analysis'][collection][-1] = report['analysis'][collection][0]
|
||||
self.assertEqual(self.outcome(report)['status'], 'INTEGRITY-FAIL')
|
||||
report = copy.deepcopy(self.reports['pass'])
|
||||
report['analysis']['cells'][0]['replicates'][0]['replicate_id'] = report['analysis']['cells'][1]['replicates'][0]['replicate_id']
|
||||
self.assertEqual(self.outcome(report)['status'], 'INTEGRITY-FAIL')
|
||||
report = copy.deepcopy(self.reports['pass']); del report['analysis']['curves'][0]['metrics']
|
||||
self.assertEqual(self.outcome(report)['status'], 'INCOMPLETE')
|
||||
|
||||
def test_rehashed_top_boolean_and_predicate_mutations_cannot_forge_success(self):
|
||||
for alter in (
|
||||
lambda a: a['cohort'].__setitem__('all_functional_predicates_pass', True),
|
||||
lambda a: a['cells'][0].__setitem__('predicate_pass', True),
|
||||
lambda a: a['cells'][0]['replicates'][2].__setitem__('conjunction_pass', True),
|
||||
lambda a: a['cells'][0]['replicates'][2]['predicates'].__setitem__('raw_signed_direction', True),
|
||||
):
|
||||
report = copy.deepcopy(self.reports['terminal_fail']); alter(report['analysis'])
|
||||
self.assertEqual(self.outcome(report)['status'], 'INTEGRITY-FAIL')
|
||||
report = copy.deepcopy(self.reports['h_fail'])
|
||||
report['analysis']['legacy_H']['both_pass'] = True
|
||||
self.assertEqual(self.outcome(report)['status'], 'INTEGRITY-FAIL')
|
||||
report = copy.deepcopy(self.reports['pass'])
|
||||
report['analysis']['legacy_H']['groups'][0]['control_sources'].pop('PAM15')
|
||||
self.assertEqual(self.outcome(report)['status'], 'INTEGRITY-FAIL')
|
||||
|
||||
def test_canonical_scope_unicode_no_newline_and_nonfinite_rejected(self):
|
||||
value = {'z': '\u89c6\u89c9', 'a': [1, 2.5]}
|
||||
raw = json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False).encode('utf-8')
|
||||
self.assertEqual(O.analysis_digest(value), hashlib.sha256(raw).hexdigest())
|
||||
self.assertNotEqual(O.analysis_digest(value), hashlib.sha256(raw + b'\n').hexdigest())
|
||||
with self.assertRaises(ValueError): O.analysis_digest({'nonfinite': float('nan')})
|
||||
report = copy.deepcopy(self.reports['pass']); report['curve_points'] = 1959
|
||||
self.assertEqual(self.outcome(report)['status'], 'INCOMPLETE')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user