Digital Drosophila Archive
363 lines
22 KiB
Python
363 lines
22 KiB
Python
"""Fixed synthetic full-mapping fixtures, using only the original CPU oracle.
|
|
|
|
Run the original science Python -B with --template pointing to a completed real
|
|
export's fixture.npz and --output a new directory. Anatomical arrays retain the
|
|
real fixture's authenticated identity; events and delta-before are explicitly
|
|
synthetic. No brain, training, installed weights or original files are changed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
|
|
import export_fixtures as original
|
|
|
|
|
|
COMPARTMENTS = ("B2B'2a", "a1", "y2a'1", "y5B'2a")
|
|
STATIC_ARRAYS = ("kc", "candidate_edge_id", "candidate_sources", "candidate_targets",
|
|
"candidate_compartments", "base_candidate")
|
|
STEPS = 1200
|
|
N_NEURONS = 166700
|
|
SEEDS = (717, 20261001)
|
|
|
|
|
|
def template_binding(template, metadata=None, manifest_path=None):
|
|
"""Bind to a completed real export and its original full-graph validation."""
|
|
template = original.unlinked(template)
|
|
directory = template.parent
|
|
root = directory.parent
|
|
metadata = original.unlinked(metadata or directory / 'fixture.json')
|
|
manifest_path = original.unlinked(manifest_path or root / 'manifest.json')
|
|
original.require(template == directory / 'fixture.npz' and metadata == directory / 'fixture.json'
|
|
and manifest_path == root / 'manifest.json', 'Use one intact real-fixture export')
|
|
manifest = original.read(manifest_path)
|
|
original.require(manifest['schema'] == 'conditioning-acceleration-fixtures-v1'
|
|
and manifest['complete'] is True and manifest['registration_sha256'] == original.REG_SHA,
|
|
'Template requires a completed real-fixture manifest')
|
|
required = ('fixture.npz', 'fixture.json', 'command.json', 'reference-sources.json',
|
|
'fresh-cpu-validation.json', 'validation-process.json', 'reference-process.json')
|
|
pins = {str(manifest_path): original.identity(manifest_path)}
|
|
for name in required:
|
|
path = directory / name
|
|
rel = path.relative_to(root).as_posix()
|
|
original.require(rel in manifest['files'], 'Template file absent from manifest: ' + rel)
|
|
pins[str(path)] = original.check(path, manifest['files'][rel])
|
|
record = original.read(metadata)
|
|
original.require(record in manifest['fixtures'] and record['fixture_id'] == directory.name
|
|
and record['npz'] == template.relative_to(root).as_posix()
|
|
and record['npz_sha256'] == pins[str(template)]['sha256']
|
|
and record['exact_stored_delta_and_trace_verified'] is True
|
|
and record['original_validator_verified_full_graph_mapping'] is True
|
|
and record['full_canonical_KC_count'] == 4064
|
|
and record['full_candidate_edge_count'] == 14232,
|
|
'Template record does not bind an authenticated complete original mapping')
|
|
fresh = original.read(directory / 'fresh-cpu-validation.json')
|
|
original.require(fresh['complete'] is True and fresh['full_candidate_mapping_verified'] is True
|
|
and fresh['frozen_U_verify_receipt_recomputed'] is True
|
|
and fresh['epoch_authority_sha256'] == record['authority_sha256'],
|
|
'Original full-graph CPU validation differs')
|
|
for name in ('validation-process.json', 'reference-process.json'):
|
|
process = original.read(directory / name)
|
|
original.require(type(process['exit_code']) is int and process['exit_code'] == 0
|
|
and process['argv'][0] == str(original.ORIGINAL_ENV / 'bin/python')
|
|
and process['CUDA_VISIBLE_DEVICES'] == '', 'Template original CPU process invalid')
|
|
sources = original.read(directory / 'reference-sources.json')
|
|
expected = dict(conditioning_update_source=original.U_SHA,
|
|
eligibility_bridge_source=original.BRIDGE_SHA, phase4j_source=original.J_SHA)
|
|
original.require(set(sources) == set(expected), 'Original source roles differ')
|
|
for role, sha in expected.items():
|
|
original.require(sources[role]['sha256'] == sha, 'Original source identity differs: ' + role)
|
|
pins[sources[role]['path']] = original.check(sources[role]['path'], sha)
|
|
arrays = original.load_npz(template)
|
|
for name in original.INPUT_ARRAYS:
|
|
original.require(original.array_evidence(arrays[name]) == record['input_arrays'][name],
|
|
'Template input array identity differs: ' + name)
|
|
command = original.read(directory / 'command.json')
|
|
original.require(command['steps'] == STEPS and command['n_neurons'] == N_NEURONS
|
|
and command['trace_backend'] == 'torch', 'Unexpected original fixture dimensions')
|
|
return arrays, sources, pins, record
|
|
|
|
|
|
def fixed_cases(template):
|
|
"""Create the declared sixteen cases without inspecting candidate outputs."""
|
|
import numpy as np
|
|
kc = template['kc']
|
|
sources = template['candidate_sources']
|
|
base = template['base_candidate']
|
|
lower = np.float32(-0.25) * base
|
|
original.require(kc.dtype == np.int32 and len(kc) == 4064 and np.all(kc[1:] > kc[:-1])
|
|
and len(base) == len(template['candidate_edge_id']) == 14232,
|
|
'Expected complete canonical anatomical arrays')
|
|
original.require(set(np.unique(template['candidate_compartments'])) == set(COMPARTMENTS),
|
|
'Template must cover all four original compartments')
|
|
active_sources = np.unique(sources)
|
|
chosen = active_sources[:3]
|
|
original.require(len(chosen) == 3, 'Need three distinct candidate KC sources')
|
|
non_kc = int(np.flatnonzero(~np.isin(np.arange(N_NEURONS, dtype=np.int32), kc))[0])
|
|
zero = np.zeros(len(base), np.float32)
|
|
partial = np.float32(-0.125) * base
|
|
partial[::3] = np.float32(0)
|
|
|
|
def events(rows):
|
|
rows = sorted(set((int(t), int(n)) for t, n in rows))
|
|
return (np.asarray([row[0] for row in rows], np.int32),
|
|
np.asarray([row[1] for row in rows], np.int32))
|
|
|
|
def case(name, rows, *, us=900, d=1., frozen=False, compartment=COMPARTMENTS[0], delta=None, tags=()):
|
|
tick, neuron = events(rows)
|
|
arrays = {key: template[key].copy() for key in STATIC_ARRAYS}
|
|
arrays.update(step=tick, neuron_idx=neuron,
|
|
delta_before=(zero if delta is None else delta).copy())
|
|
return dict(name=name, arrays=arrays,
|
|
command=dict(steps=STEPS, n_neurons=N_NEURONS, compartment=compartment,
|
|
us_tick=us, d_command=d, frozen=frozen, trace_backend='torch'),
|
|
tags=list(tags), synthetic_events=True, synthetic_delta_before=True)
|
|
|
|
boundary = [(899, chosen[0]), (900, chosen[1]), (901, chosen[2]),
|
|
(0, non_kc), (900, non_kc), (1199, non_kc)]
|
|
ends = [(0, chosen[0]), (1, chosen[1]), (1199, chosen[2]), (0, non_kc), (1199, non_kc)]
|
|
all_active = [(900, n) for n in active_sources]
|
|
cases = [
|
|
case('empty-us900', [], tags=('empty', 'positive-zero')),
|
|
case('non-kc-only', [(0, non_kc), (900, non_kc), (1199, non_kc)], tags=('ignore-non-KC',)),
|
|
case('no-us-mixed', boundary, us=None, d=0., tags=('no-US', 'sampled-positive-zero', 'nonzero-final')),
|
|
case('us-first-tick', ends, us=0, tags=('US-zero', 'after-current-spike')),
|
|
case('us-final-tick', ends, us=1199, tags=('US-final', 'after-current-spike')),
|
|
case('before-on-after-us900', boundary, tags=('US-boundary', 'mixed-KC-non-KC')),
|
|
case('d-zero-existing-delta', boundary, d=0., delta=partial, tags=('D-zero', 'bitwise-identity')),
|
|
case('frozen-existing-delta', boundary, frozen=True, delta=partial, tags=('frozen', 'bitwise-identity')),
|
|
]
|
|
for index, compartment in enumerate(COMPARTMENTS):
|
|
cases.append(case(f'compartment-{index}', all_active, compartment=compartment, delta=partial,
|
|
tags=('full-source-activation', 'off-compartment-bitwise-identity', compartment)))
|
|
cases.append(case('already-at-floor', all_active, delta=lower, tags=('floor', 'already-saturated')))
|
|
near = np.nextafter(lower, np.full_like(lower, np.inf))
|
|
cases.append(case('one-ulp-above-floor', all_active, delta=near, tags=('floor', 'new-saturation')))
|
|
for seed in SEEDS:
|
|
rng = np.random.default_rng(seed)
|
|
ticks = rng.integers(0, STEPS, size=5000)
|
|
neurons = rng.choice(kc, size=5000)
|
|
extra_ticks = rng.integers(0, STEPS, size=1000)
|
|
extra_neurons = rng.integers(0, N_NEURONS, size=1000)
|
|
rows = list(zip(ticks, neurons)) + list(zip(extra_ticks, extra_neurons))
|
|
delta = -(rng.uniform(0., .249, len(base)).astype(np.float32) * base)
|
|
delta[::7] = np.float32(0)
|
|
delta[1::11] = lower[1::11]
|
|
item = case(f'seeded-sparse-{seed}', rows, delta=delta, tags=('seeded-sparse', 'mixed-delta', 'canonical-zero'))
|
|
item['rng_seed'] = seed
|
|
cases.append(item)
|
|
original.require(len(cases) == 16, 'Fixed synthetic coverage changed')
|
|
for item in cases:
|
|
for name in STATIC_ARRAYS:
|
|
original.require(original.array_evidence(item['arrays'][name]) == original.array_evidence(template[name]),
|
|
'Synthetic generation changed an anatomical array: ' + name)
|
|
original.require(not np.any((item['arrays']['delta_before'] == 0)
|
|
& np.signbit(item['arrays']['delta_before'])), 'Synthetic delta has negative zero')
|
|
cases[4]['trace_command_steps'] = [0, 900, 1199]
|
|
cases[4]['tags'].append('multiple-US-snapshots')
|
|
return cases
|
|
|
|
|
|
def supplemental_worker(directory):
|
|
"""Fresh original CPU process for multiple snapshots or rejection evidence."""
|
|
original.cpu_environment()
|
|
import numpy as np
|
|
directory = original.unlinked(directory)
|
|
sources = original.read(directory / 'reference-sources.json')
|
|
expected = dict(conditioning_update_source=original.U_SHA,
|
|
eligibility_bridge_source=original.BRIDGE_SHA, phase4j_source=original.J_SHA)
|
|
original.require(set(sources) == set(expected), 'Supplemental source roles differ')
|
|
for role, sha in expected.items():
|
|
original.require(sources[role]['sha256'] == sha, 'Supplemental frozen source differs')
|
|
original.check(sources[role]['path'], sha)
|
|
if (directory / 'trace-command-steps.json').exists():
|
|
arrays = original.load_npz(directory / 'inputs.npz')
|
|
command = original.read(directory / 'command.json')
|
|
commands = original.read(directory / 'trace-command-steps.json')
|
|
bridge = original.load_verified_source('_synthetic_original_bridge',
|
|
sources['eligibility_bridge_source']['path'], original.BRIDGE_SHA)
|
|
trace = bridge.trace_from_events(arrays['step'], arrays['neuron_idx'], arrays['kc'],
|
|
command['steps'], commands, phase4j_source=sources['phase4j_source']['path'],
|
|
backend='torch', n_neurons=command['n_neurons'])
|
|
sole = original.load_npz(directory / 'reference.npz')
|
|
original.require(original.array_evidence(trace.final) == original.array_evidence(sole['reference_final_trace'])
|
|
and original.array_evidence(trace.last_command_trace)
|
|
== original.array_evidence(sole['reference_sampled_trace']),
|
|
'Multiple-US last sample or final differs from original single-US U reference')
|
|
original.save_npz(directory / 'multiple-reference.npz', dict(reference_snapshots=trace.snapshots,
|
|
reference_command_steps=trace.command_steps, reference_sampled_trace=trace.last_command_trace,
|
|
reference_final_trace=trace.final))
|
|
import torch
|
|
original.require(not torch.cuda.is_initialized(), 'Supplemental CPU oracle initialized CUDA')
|
|
original.save(directory / 'multiple-reference.json', dict(complete=True, trace_audit=trace.audit,
|
|
source_pins=sources, numpy_version=np.__version__, torch_version=torch.__version__,
|
|
final_and_last_sample_match_original_U=True, brain_executed=False, weights_installed=False))
|
|
return
|
|
cases = original.read(directory / 'invalid-selection.json')
|
|
update = original.load_verified_source('_synthetic_rejection_original_U',
|
|
sources['conditioning_update_source']['path'], original.U_SHA)
|
|
records = []
|
|
for item in cases:
|
|
arrays = original.load_npz(directory / item['npz'])
|
|
command = original.read(directory / item['command'])
|
|
try:
|
|
update.update_epoch(**arrays, **command,
|
|
bridge_source=sources['eligibility_bridge_source']['path'],
|
|
phase4j_source=sources['phase4j_source']['path'])
|
|
except ValueError as error:
|
|
records.append(dict(case_name=item['case_name'], rejected=True, exception='ValueError', message=str(error)))
|
|
else:
|
|
raise ValueError('Original U unexpectedly accepted invalid case: ' + item['case_name'])
|
|
original.save(directory / 'original-rejections.json', dict(complete=True, cases=records,
|
|
source_pins=sources, brain_executed=False, weights_installed=False))
|
|
|
|
|
|
def export_invalid(output, valid_case, sources):
|
|
import numpy as np
|
|
directory = output / 'invalid'
|
|
directory.mkdir()
|
|
original.save(directory / 'reference-sources.json', sources)
|
|
definitions = [('duplicate-events', 'prepare'), ('unsorted-events', 'prepare'),
|
|
('out-of-range-tick', 'prepare'), ('negative-zero-delta', 'run'),
|
|
('non-kc-candidate-source', 'run'), ('unsorted-kc', 'prepare')]
|
|
records = []
|
|
for index, (name, stage) in enumerate(definitions):
|
|
arrays = {key: value.copy() for key, value in valid_case['arrays'].items()}
|
|
if name == 'duplicate-events':
|
|
arrays['step'] = np.array([0, 0], np.int32)
|
|
arrays['neuron_idx'] = np.full(2, arrays['kc'][0], np.int32)
|
|
elif name == 'unsorted-events':
|
|
arrays['step'] = np.array([1, 0], np.int32)
|
|
arrays['neuron_idx'] = np.full(2, arrays['kc'][0], np.int32)
|
|
elif name == 'out-of-range-tick':
|
|
arrays['step'] = np.array([STEPS], np.int32)
|
|
arrays['neuron_idx'] = np.array([arrays['kc'][0]], np.int32)
|
|
elif name == 'negative-zero-delta':
|
|
arrays['delta_before'][0] = np.float32(-0.)
|
|
elif name == 'non-kc-candidate-source':
|
|
arrays['candidate_sources'][0] = int(np.flatnonzero(
|
|
~np.isin(np.arange(N_NEURONS, dtype=np.int32), arrays['kc']))[0])
|
|
elif name == 'unsorted-kc':
|
|
arrays['kc'] = arrays['kc'][::-1].copy()
|
|
npz = f'{index:02d}-{name}.npz'
|
|
command = f'{index:02d}-{name}.json'
|
|
original.save_npz(directory / npz, arrays)
|
|
original.save(directory / command, valid_case['command'])
|
|
records.append(dict(case_name=name, npz=npz, command=command, stage=stage,
|
|
expected_exception='ValueError', npz_sha256=original.digest(directory / npz),
|
|
command_sha256=original.digest(directory / command)))
|
|
original.save(directory / 'invalid-selection.json', records)
|
|
original.run_cpu([sys.executable, '-B', str(Path(__file__).absolute()), '--supplemental-worker', str(directory)],
|
|
directory, 'original-rejection')
|
|
evidence = original.read(directory / 'original-rejections.json')
|
|
original.require(evidence['complete'] is True and len(evidence['cases']) == len(records),
|
|
'Original rejection coverage incomplete')
|
|
for record, result in zip(records, evidence['cases']):
|
|
original.require(record['case_name'] == result['case_name'] and result['rejected'] is True,
|
|
'Original rejection identity differs')
|
|
record.update(npz='invalid/' + record['npz'], command='invalid/' + record['command'],
|
|
original_oracle_rejection=result)
|
|
return records
|
|
|
|
|
|
def export(template, output, metadata=None, manifest_path=None):
|
|
original.cpu_environment()
|
|
started = time.perf_counter()
|
|
own = original.identity(Path(__file__).absolute())
|
|
helper = original.identity(Path(original.__file__).absolute())
|
|
arrays, sources, pins, template_record = template_binding(template, metadata, manifest_path)
|
|
cases = fixed_cases(arrays)
|
|
protected = {original.ROOT, Path(template).parent.parent, Path(__file__).absolute().parent,
|
|
*(Path(info['path']).parent for info in sources.values())}
|
|
output = original.validate_destination(output, protected)
|
|
output.mkdir()
|
|
original.save(output / 'template-binding.json', dict(template=str(template), template_record=template_record,
|
|
input_pins=pins, scope='complete anatomical identity inherited from original full-graph CPU validation',
|
|
original_events_reused=False, original_delta_before_reused=False))
|
|
original.save(output / 'selection.json', dict(schema='conditioning-fixed-synthetic-selection-v1',
|
|
names=[item['name'] for item in cases], count=16, seeds=list(SEEDS),
|
|
chosen_before_candidate_execution=True, original_full_mapping_preserved=True))
|
|
fixtures = []
|
|
for index, item in enumerate(cases):
|
|
fixture_id = f'{index:02d}-{item["name"]}'
|
|
directory = output / fixture_id
|
|
directory.mkdir()
|
|
original.save_npz(directory / 'inputs.npz', item['arrays'])
|
|
original.save(directory / 'command.json', item['command'])
|
|
original.save(directory / 'reference-sources.json', sources)
|
|
seconds = original.run_cpu([sys.executable, '-B', str(Path(original.__file__).absolute()),
|
|
'--reference-worker', str(directory)], directory, 'reference')
|
|
reference = original.load_npz(directory / 'reference.npz')
|
|
receipt = original.read(directory / 'reference-update-receipt.json')
|
|
timing = original.read(directory / 'reference-timing.json')
|
|
original.require(timing['numpy_version'] == template_record['reference_timing']['numpy_version']
|
|
and timing['torch_version'] == template_record['reference_timing']['torch_version'],
|
|
'Synthetic oracle differs from the real fixture original environment')
|
|
for name in original.OUTPUT_ARRAYS:
|
|
original.require(original.array_evidence(reference['reference_' + name]) == receipt['outputs'][name],
|
|
'Synthetic reference output/receipt differs: ' + name)
|
|
if 'trace_command_steps' in item:
|
|
original.save(directory / 'trace-command-steps.json', item['trace_command_steps'])
|
|
original.run_cpu([sys.executable, '-B', str(Path(__file__).absolute()),
|
|
'--supplemental-worker', str(directory)], directory, 'multiple-reference')
|
|
reference.update(original.load_npz(directory / 'multiple-reference.npz'))
|
|
original.save_npz(directory / 'fixture.npz', {**item['arrays'], **reference})
|
|
record = dict(fixture_id=fixture_id, kind='synthetic', name=item['name'], tags=item['tags'],
|
|
npz=f'{fixture_id}/fixture.npz', command=f'{fixture_id}/command.json',
|
|
npz_sha256=original.digest(directory / 'fixture.npz'),
|
|
command_sha256=original.digest(directory / 'command.json'),
|
|
original_update_payload_sha256=receipt['payload_sha256'],
|
|
exact_stored_delta_and_trace_verified=False, historical_scientific_output_exists=False,
|
|
full_canonical_KC_count=4064, full_candidate_edge_count=14232,
|
|
original_validator_verified_full_graph_mapping=True,
|
|
mapping_identity_scope='inherited unchanged from template; events and delta-before are synthetic',
|
|
reference_timing=timing, reference_subprocess_wall_seconds=seconds,
|
|
input_arrays={name: original.array_evidence(value) for name, value in item['arrays'].items()},
|
|
output_arrays={name: original.array_evidence(value) for name, value in reference.items()})
|
|
if 'trace_command_steps' in item:
|
|
record['snapshot_scope'] = 'Original Torch bridge multiple-US snapshots; last sample matches original U single final-US output'
|
|
record['command_steps'] = item['trace_command_steps']
|
|
original.save(directory / 'fixture.json', record)
|
|
fixtures.append(record)
|
|
print(original.canonical(dict(fixture=fixture_id, kind='synthetic', complete=True)), flush=True)
|
|
invalid_cases = export_invalid(output, cases[5], sources)
|
|
for path, pin in pins.items():
|
|
original.check(path, pin)
|
|
original.check(Path(__file__).absolute(), own)
|
|
original.check(Path(original.__file__).absolute(), helper)
|
|
manifest = dict(schema='conditioning-acceleration-fixtures-v1', kind='synthetic', complete=True,
|
|
registration_sha256=original.REG_SHA, source_code_commit=original.SOURCE_COMMIT,
|
|
exporter_source=dict(path=str(Path(__file__).absolute()), **own),
|
|
original_helper_source=dict(path=str(Path(original.__file__).absolute()), **helper),
|
|
created_utc=datetime.now(timezone.utc).isoformat(), fixtures=fixtures, fixture_count=len(fixtures),
|
|
invalid_cases=invalid_cases, invalid_case_count=len(invalid_cases),
|
|
selection_sha256=original.digest(output / 'selection.json'),
|
|
template_binding_sha256=original.digest(output / 'template-binding.json'),
|
|
total_export_seconds=time.perf_counter() - started, original_files_modified=False,
|
|
brain_executed=False, training_executed=False, weights_installed=False, learning_pass_claimed=False,
|
|
authority_limit='synthetic arithmetic edge cases with original mapping; not historical scientific trials',
|
|
files={str(path.relative_to(output)): original.identity(path)
|
|
for path in sorted(output.rglob('*')) if path.is_file()})
|
|
original.save(output / 'manifest.json', manifest)
|
|
return manifest
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--template', type=Path)
|
|
parser.add_argument('--metadata', type=Path)
|
|
parser.add_argument('--manifest', type=Path)
|
|
parser.add_argument('--output', type=Path)
|
|
parser.add_argument('--supplemental-worker', type=Path, help=argparse.SUPPRESS)
|
|
args = parser.parse_args()
|
|
if args.supplemental_worker:
|
|
original.require(not any((args.template, args.output, args.metadata, args.manifest)), 'Do not combine worker/export modes')
|
|
supplemental_worker(args.supplemental_worker)
|
|
else:
|
|
original.require(args.template is not None and args.output is not None, '--template and --output are required')
|
|
result = export(args.template, args.output, args.metadata, args.manifest)
|
|
print(original.canonical(dict(complete=True, output=str(args.output), fixtures=result['fixture_count'])), flush=True)
|