Bind synthetic edge cases and GPU comparisons to original CPU oracle

models/male-cns-lif/conditioning-acceleration-v1/export_synthetic.py

@@ -156,9 +156,113 @@ def fixed_cases(template):
                              'Synthetic generation changed an anatomical array: ' + name)
         original.require(not np.any((item['arrays']['delta_before'] == 0)
                                     & np.signbit(item['arrays']['delta_before'])), 'Synthetic delta has negative zero')
+    cases[4]['trace_command_steps'] = [0, 900, 1199]
+    cases[4]['tags'].append('multiple-US-snapshots')
     return cases
 
 
+def supplemental_worker(directory):
+    """Fresh original CPU process for multiple snapshots or rejection evidence."""
+    original.cpu_environment()
+    import numpy as np
+    directory = original.unlinked(directory)
+    sources = original.read(directory / 'reference-sources.json')
+    expected = dict(conditioning_update_source=original.U_SHA,
+                    eligibility_bridge_source=original.BRIDGE_SHA, phase4j_source=original.J_SHA)
+    original.require(set(sources) == set(expected), 'Supplemental source roles differ')
+    for role, sha in expected.items():
+        original.require(sources[role]['sha256'] == sha, 'Supplemental frozen source differs')
+        original.check(sources[role]['path'], sha)
+    if (directory / 'trace-command-steps.json').exists():
+        arrays = original.load_npz(directory / 'inputs.npz')
+        command = original.read(directory / 'command.json')
+        commands = original.read(directory / 'trace-command-steps.json')
+        bridge = original.load_verified_source('_synthetic_original_bridge',
+            sources['eligibility_bridge_source']['path'], original.BRIDGE_SHA)
+        trace = bridge.trace_from_events(arrays['step'], arrays['neuron_idx'], arrays['kc'],
+            command['steps'], commands, phase4j_source=sources['phase4j_source']['path'],
+            backend='torch', n_neurons=command['n_neurons'])
+        sole = original.load_npz(directory / 'reference.npz')
+        original.require(original.array_evidence(trace.final) == original.array_evidence(sole['reference_final_trace'])
+                         and original.array_evidence(trace.last_command_trace)
+                         == original.array_evidence(sole['reference_sampled_trace']),
+                         'Multiple-US last sample or final differs from original single-US U reference')
+        original.save_npz(directory / 'multiple-reference.npz', dict(reference_snapshots=trace.snapshots,
+            reference_command_steps=trace.command_steps, reference_sampled_trace=trace.last_command_trace,
+            reference_final_trace=trace.final))
+        import torch
+        original.require(not torch.cuda.is_initialized(), 'Supplemental CPU oracle initialized CUDA')
+        original.save(directory / 'multiple-reference.json', dict(complete=True, trace_audit=trace.audit,
+            source_pins=sources, numpy_version=np.__version__, torch_version=torch.__version__,
+            final_and_last_sample_match_original_U=True, brain_executed=False, weights_installed=False))
+        return
+    cases = original.read(directory / 'invalid-selection.json')
+    update = original.load_verified_source('_synthetic_rejection_original_U',
+        sources['conditioning_update_source']['path'], original.U_SHA)
+    records = []
+    for item in cases:
+        arrays = original.load_npz(directory / item['npz'])
+        command = original.read(directory / item['command'])
+        try:
+            update.update_epoch(**arrays, **command,
+                bridge_source=sources['eligibility_bridge_source']['path'],
+                phase4j_source=sources['phase4j_source']['path'])
+        except ValueError as error:
+            records.append(dict(case_name=item['case_name'], rejected=True, exception='ValueError', message=str(error)))
+        else:
+            raise ValueError('Original U unexpectedly accepted invalid case: ' + item['case_name'])
+    original.save(directory / 'original-rejections.json', dict(complete=True, cases=records,
+        source_pins=sources, brain_executed=False, weights_installed=False))
+
+
+def export_invalid(output, valid_case, sources):
+    import numpy as np
+    directory = output / 'invalid'
+    directory.mkdir()
+    original.save(directory / 'reference-sources.json', sources)
+    definitions = [('duplicate-events', 'prepare'), ('unsorted-events', 'prepare'),
+                   ('out-of-range-tick', 'prepare'), ('negative-zero-delta', 'run'),
+                   ('non-kc-candidate-source', 'run'), ('unsorted-kc', 'prepare')]
+    records = []
+    for index, (name, stage) in enumerate(definitions):
+        arrays = {key: value.copy() for key, value in valid_case['arrays'].items()}
+        if name == 'duplicate-events':
+            arrays['step'] = np.array([0, 0], np.int32)
+            arrays['neuron_idx'] = np.full(2, arrays['kc'][0], np.int32)
+        elif name == 'unsorted-events':
+            arrays['step'] = np.array([1, 0], np.int32)
+            arrays['neuron_idx'] = np.full(2, arrays['kc'][0], np.int32)
+        elif name == 'out-of-range-tick':
+            arrays['step'] = np.array([STEPS], np.int32)
+            arrays['neuron_idx'] = np.array([arrays['kc'][0]], np.int32)
+        elif name == 'negative-zero-delta':
+            arrays['delta_before'][0] = np.float32(-0.)
+        elif name == 'non-kc-candidate-source':
+            arrays['candidate_sources'][0] = int(np.flatnonzero(
+                ~np.isin(np.arange(N_NEURONS, dtype=np.int32), arrays['kc']))[0])
+        elif name == 'unsorted-kc':
+            arrays['kc'] = arrays['kc'][::-1].copy()
+        npz = f'{index:02d}-{name}.npz'
+        command = f'{index:02d}-{name}.json'
+        original.save_npz(directory / npz, arrays)
+        original.save(directory / command, valid_case['command'])
+        records.append(dict(case_name=name, npz=npz, command=command, stage=stage,
+                            expected_exception='ValueError', npz_sha256=original.digest(directory / npz),
+                            command_sha256=original.digest(directory / command)))
+    original.save(directory / 'invalid-selection.json', records)
+    original.run_cpu([sys.executable, '-B', str(Path(__file__).absolute()), '--supplemental-worker', str(directory)],
+                     directory, 'original-rejection')
+    evidence = original.read(directory / 'original-rejections.json')
+    original.require(evidence['complete'] is True and len(evidence['cases']) == len(records),
+                     'Original rejection coverage incomplete')
+    for record, result in zip(records, evidence['cases']):
+        original.require(record['case_name'] == result['case_name'] and result['rejected'] is True,
+                         'Original rejection identity differs')
+        record.update(npz='invalid/' + record['npz'], command='invalid/' + record['command'],
+                      original_oracle_rejection=result)
+    return records
+
+
 def export(template, output, metadata=None, manifest_path=None):
     original.cpu_environment()
     started = time.perf_counter()
@@ -195,6 +299,11 @@ def export(template, output, metadata=None, manifest_path=None):
         for name in original.OUTPUT_ARRAYS:
             original.require(original.array_evidence(reference['reference_' + name]) == receipt['outputs'][name],
                              'Synthetic reference output/receipt differs: ' + name)
+        if 'trace_command_steps' in item:
+            original.save(directory / 'trace-command-steps.json', item['trace_command_steps'])
+            original.run_cpu([sys.executable, '-B', str(Path(__file__).absolute()),
+                              '--supplemental-worker', str(directory)], directory, 'multiple-reference')
+            reference.update(original.load_npz(directory / 'multiple-reference.npz'))
         original.save_npz(directory / 'fixture.npz', {**item['arrays'], **reference})
         record = dict(fixture_id=fixture_id, kind='synthetic', name=item['name'], tags=item['tags'],
             npz=f'{fixture_id}/fixture.npz', command=f'{fixture_id}/command.json',
@@ -208,9 +317,13 @@ def export(template, output, metadata=None, manifest_path=None):
             reference_timing=timing, reference_subprocess_wall_seconds=seconds,
             input_arrays={name: original.array_evidence(value) for name, value in item['arrays'].items()},
             output_arrays={name: original.array_evidence(value) for name, value in reference.items()})
+        if 'trace_command_steps' in item:
+            record['snapshot_scope'] = 'Original Torch bridge multiple-US snapshots; last sample matches original U single final-US output'
+            record['command_steps'] = item['trace_command_steps']
         original.save(directory / 'fixture.json', record)
         fixtures.append(record)
         print(original.canonical(dict(fixture=fixture_id, kind='synthetic', complete=True)), flush=True)
+    invalid_cases = export_invalid(output, cases[5], sources)
     for path, pin in pins.items():
         original.check(path, pin)
     original.check(Path(__file__).absolute(), own)
@@ -220,6 +333,7 @@ def export(template, output, metadata=None, manifest_path=None):
         exporter_source=dict(path=str(Path(__file__).absolute()), **own),
         original_helper_source=dict(path=str(Path(original.__file__).absolute()), **helper),
         created_utc=datetime.now(timezone.utc).isoformat(), fixtures=fixtures, fixture_count=len(fixtures),
+        invalid_cases=invalid_cases, invalid_case_count=len(invalid_cases),
         selection_sha256=original.digest(output / 'selection.json'),
         template_binding_sha256=original.digest(output / 'template-binding.json'),
         total_export_seconds=time.perf_counter() - started, original_files_modified=False,
@@ -233,10 +347,16 @@ def export(template, output, metadata=None, manifest_path=None):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--template', type=Path, required=True)
+    parser.add_argument('--template', type=Path)
     parser.add_argument('--metadata', type=Path)
     parser.add_argument('--manifest', type=Path)
-    parser.add_argument('--output', type=Path, required=True)
+    parser.add_argument('--output', type=Path)
+    parser.add_argument('--supplemental-worker', type=Path, help=argparse.SUPPRESS)
     args = parser.parse_args()
-    result = export(args.template, args.output, args.metadata, args.manifest)
-    print(original.canonical(dict(complete=True, output=str(args.output), fixtures=result['fixture_count'])), flush=True)
+    if args.supplemental_worker:
+        original.require(not any((args.template, args.output, args.metadata, args.manifest)), 'Do not combine worker/export modes')
+        supplemental_worker(args.supplemental_worker)
+    else:
+        original.require(args.template is not None and args.output is not None, '--template and --output are required')
+        result = export(args.template, args.output, args.metadata, args.manifest)
+        print(original.canonical(dict(complete=True, output=str(args.output), fixtures=result['fixture_count'])), flush=True)