Audit conditional PN recruitment and graded APL source timing
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,23 @@
|
||||
# Phase 4AI — PN recruitment and graded-APL source audit
|
||||
|
||||
Read-only, post-outcome conditional accounting of 80 already sealed Phase 4AG-r2 records. This is not a new whole-brain experiment, a learning trial, an independent brain oracle, or an APL-removal counterfactual.
|
||||
|
||||
## Frozen contracts
|
||||
|
||||
- Reuse sealed Phase 4AH `replay_core.py` without modification; reconstruct actual ALPN targets (686 in the full graph).
|
||||
- Keep the full frozen fast-APL effective-edge zeroing, delay, refractory, reset, threshold, ordering and float64 LIF coefficients. PN graded inhibition is exactly zero because the existing overlay applied only to KC.
|
||||
- Separate accepted timed incoming-edge antecedents, non-feedback voltage lineage, and the legacy nonnegative graded-APL input map. The latter is not signed PN-to-APL LIF current.
|
||||
- APL source groups: trial design ALPN, other ALPN, KC. Reject unaccounted nonzero input support. Compare summed mathematical float64 traces with stored float32 using prior analytic per-tick bounds, including grouping roundoff. Accumulated source shares remain descriptive; their robust majority is not proven.
|
||||
- Strict timing: direct PN arrival at s+18 must precede the KC threshold tick; APL write at s follows the threshold and can affect only later ticks. Never-fired cells remain explicitly censored.
|
||||
- Primary odor statistics retain fixed 686 coordinates; secondary statistics retain the fixed complement of six-odor design union. Technical repeats are excluded from odor comparisons.
|
||||
|
||||
## Execution
|
||||
|
||||
Use the existing CPU environment, Python `-B`, and four-thread OMP/MKL/OpenBLAS/Arrow limits. No Torch import, GPU or parameter changes are needed.
|
||||
|
||||
1. `prepare.py`: synthetic tests, source/input hashes, prior integrity, fixed inventory, metadata schema and preregistration.
|
||||
2. `run_audit.py`: exact PN conditional replay, event-stream provenance, APL split, twelve repeat checks.
|
||||
3. `finalize.py`: independently re-read saved evidence, validate timed edge events, aggregate all fixed records, check prior integrity, seal model and experiment.
|
||||
4. `publication.py`: verify both seals, then exclusively publish small evidence to a new registry. Bulk arrays stay on the server; Git is not their off-host backup.
|
||||
|
||||
Any failure is preserved. Do not edit a registered source or reuse a populated experiment directory. A new revision requires an explicit separate directory and record. Audit PASS never implies learning PASS, identified unique causality, or permission to tune frozen models.
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Non-feedback float64 attribution of the frozen, recorded graded APL trace.
|
||||
|
||||
No files, Torch, GPU, neuronal state, or counterfactual simulation are used.
|
||||
The reference is mathematical source accounting, NOT bitwise CUDA replay.
|
||||
Full bundle output is generated even for graph fixtures; the caller must not
|
||||
interpret inhibition for absent fixture KCs as inhibition actually applied.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import numpy as np
|
||||
|
||||
|
||||
N = 166700
|
||||
GROUPS = ("design_ALPN", "non_design_ALPN", "KC")
|
||||
DECAY32 = np.float32(math.exp(-.1 / 20.))
|
||||
GAIN32 = np.float32(.01)
|
||||
|
||||
|
||||
def _require(ok, message):
|
||||
if not ok:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _ids(value, name):
|
||||
value = np.asarray(value)
|
||||
_require(value.dtype == np.int32 and value.ndim == 1, name + " must be an int32 vector")
|
||||
_require(np.all((value >= 0) & (value < N)) and np.all(value[1:] > value[:-1]),
|
||||
name + " must contain sorted unique global IDs")
|
||||
return value
|
||||
|
||||
|
||||
def _positive_array(value, shape, name):
|
||||
value = np.asarray(value)
|
||||
_require(value.dtype == np.float32 and value.shape == shape, name + " has the wrong shape/dtype")
|
||||
_require(np.isfinite(value).all() and np.all(value >= 0), name + " must be finite and nonnegative")
|
||||
return value
|
||||
|
||||
|
||||
def _up(value):
|
||||
return np.nextafter(np.asarray(value, np.float64), np.inf)
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return _up(np.asarray(a, np.float64) + np.asarray(b, np.float64))
|
||||
|
||||
|
||||
def _mul(a, b):
|
||||
return _up(np.asarray(a, np.float64) * np.asarray(b, np.float64))
|
||||
|
||||
|
||||
def _gamma(count, unit_roundoff):
|
||||
_require(count * unit_roundoff < 1, "Undefined gamma bound")
|
||||
numerator = _mul(count, unit_roundoff)
|
||||
denominator = np.nextafter(1. - numerator, -np.inf)
|
||||
return _up(numerator / denominator)
|
||||
|
||||
|
||||
def _sum_upper(values):
|
||||
"""Outward-rounded sum on axis zero; inputs are nonnegative."""
|
||||
result = np.zeros(values.shape[1:], np.float64)
|
||||
for value in values:
|
||||
result = _add(result, value)
|
||||
return result
|
||||
|
||||
|
||||
def _sum_groups(values):
|
||||
# Explicit separate additions; never rely on a backend reduction over G.
|
||||
return np.add(np.add(values[0], values[1]), values[2])
|
||||
|
||||
|
||||
def _positive_dot_upper(state, matrix):
|
||||
result = np.zeros(matrix.shape[1], np.float64)
|
||||
for index in range(len(state)):
|
||||
result = _add(result, _mul(state[index], matrix[index]))
|
||||
return result
|
||||
|
||||
|
||||
def _normal(value, tiny):
|
||||
return bool(np.isfinite(value).all() and np.all((value == 0) | (value >= tiny)))
|
||||
|
||||
|
||||
def _metric():
|
||||
return dict(elements=0, sum_abs_error=0., max_abs_error=0., max_error_bound=0.,
|
||||
max_bound_utilization=0., bound_violations=0, zero_reference_nonzero_actual=0)
|
||||
|
||||
|
||||
def _measure(metric, actual, reference, bound):
|
||||
_require(np.isfinite(reference).all() and np.isfinite(bound).all(), "Nonfinite reference/bound")
|
||||
error = np.abs(actual.astype(np.float64) - reference)
|
||||
upper = np.where(error == 0, 0., _up(error))
|
||||
impossible_zero = (reference == 0) & (actual != 0)
|
||||
bad = (upper > bound) | impossible_zero
|
||||
usage = np.divide(upper, bound, out=np.zeros_like(upper), where=bound > 0)
|
||||
metric['elements'] += error.size
|
||||
metric['sum_abs_error'] += float(error.sum(dtype=np.float64))
|
||||
metric['max_abs_error'] = max(metric['max_abs_error'], float(error.max(initial=0)))
|
||||
metric['max_error_bound'] = max(metric['max_error_bound'], float(bound.max(initial=0)))
|
||||
metric['max_bound_utilization'] = max(metric['max_bound_utilization'], float(usage.max(initial=0)))
|
||||
metric['bound_violations'] += int(bad.sum())
|
||||
metric['zero_reference_nonzero_actual'] += int(impossible_zero.sum())
|
||||
|
||||
|
||||
def decompose_apl_sources(I32, O32, pn_globals, kc_globals, design_pn_globals,
|
||||
event_step, event_node, drive32, state32, h32, *,
|
||||
decay32, gain32, standard_fp32, chunk_size=512, callback=None,
|
||||
first_kc_tick=None):
|
||||
"""Attribute APL input/output to three exact anatomical source groups.
|
||||
|
||||
I32 is the full (2,166700) normalized float32 input matrix; O32 is (2,K),
|
||||
with columns in sorted kc_globals order. Traces are drive/state float32
|
||||
(T,2), inhibition float32 (T,K), including ALL bundle KC output columns.
|
||||
Actual event vectors are int32 and sorted/unique by tick/global neuron.
|
||||
Design IDs must be a subset of pn_globals; fixtures' design definition is
|
||||
supplied explicitly by the caller, never inferred from response outcome.
|
||||
|
||||
standard_fp32 must be True only after historical metadata verifies TF32 is
|
||||
disabled and matmul precision is 'highest'. This function cannot inspect a
|
||||
GPU environment. d/q must have the exact frozen float32 constant bits.
|
||||
|
||||
Return summary (pure JSON values), tick_rows (T*3*2 JSON dictionaries),
|
||||
first_kc_tick (int32[K], -1 for never fired), and per_kc cumulative arrays.
|
||||
Optional first_kc_tick is checked against actual events, never overrides
|
||||
them. No T*3*K history is retained; source cumulative storage is O(3*K).
|
||||
callback(tick, dict) optionally receives copies of drive/state group arrays
|
||||
(3,2), plus their totals, never a full inhibition history.
|
||||
|
||||
Early means strictly tick < observed first KC spike; same-tick APL output
|
||||
follows threshold and cannot cause that spike. Never-fired KC output and
|
||||
output strictly before the first spike of ANY KC are reported separately.
|
||||
"""
|
||||
_require(standard_fp32 is True, "Historical standard FP32 arithmetic must be verified by caller")
|
||||
_require(isinstance(decay32, np.float32) and isinstance(gain32, np.float32) and
|
||||
decay32.view(np.uint32) == DECAY32.view(np.uint32) and
|
||||
gain32.view(np.uint32) == GAIN32.view(np.uint32), "Frozen float32 decay/gain bits changed")
|
||||
_require(isinstance(chunk_size, (int, np.integer)) and not isinstance(chunk_size, (bool, np.bool_)) and
|
||||
chunk_size > 0, "chunk_size must be a positive integer")
|
||||
_require(callback is None or callable(callback), "callback must be callable")
|
||||
pn = _ids(pn_globals, 'pn_globals')
|
||||
kc = _ids(kc_globals, 'kc_globals')
|
||||
design = _ids(design_pn_globals, 'design_pn_globals')
|
||||
_require(len(kc) > 0 and not np.intersect1d(pn, kc).size, "PN/KC identities overlap or KC set is empty")
|
||||
_require(np.isin(design, pn).all(), "Design PN is outside the anatomical PN population")
|
||||
k = len(kc)
|
||||
I = _positive_array(I32, (2, N), 'I32').astype(np.float64)
|
||||
O = _positive_array(O32, (2, k), 'O32').astype(np.float64)
|
||||
state32 = np.asarray(state32)
|
||||
_require(state32.ndim == 2 and len(state32) > 0, "Empty/invalid state trace")
|
||||
steps = len(state32)
|
||||
state32 = _positive_array(state32, (steps, 2), 'state32')
|
||||
drive32 = _positive_array(drive32, (steps, 2), 'drive32')
|
||||
h32 = np.asarray(h32)
|
||||
_require(h32.dtype == np.float32 and h32.shape == (steps, k), "h32 has wrong shape/dtype")
|
||||
# Validate without allocating a dense T*K temporary boolean array.
|
||||
for start in range(0, k, chunk_size):
|
||||
value = h32[:, start:start + chunk_size]
|
||||
_require(np.isfinite(value).all() and np.all(value >= 0), "h32 must be finite and nonnegative")
|
||||
labels = np.full(N, -1, np.int8)
|
||||
labels[pn], labels[design], labels[kc] = 1, 0, 2
|
||||
support = np.any(I != 0, axis=0)
|
||||
unknown = np.flatnonzero(support & (labels < 0))
|
||||
_require(not len(unknown), "Nonzero APL input outside design/non-design PN/KC: " + str(unknown[:10].tolist()))
|
||||
ticks, nodes = np.asarray(event_step), np.asarray(event_node)
|
||||
_require(ticks.dtype == nodes.dtype == np.int32 and ticks.ndim == nodes.ndim == 1 and
|
||||
ticks.shape == nodes.shape, "Actual events must be matching int32 vectors")
|
||||
_require(np.all((ticks >= 0) & (ticks < steps)) and np.all((nodes >= 0) & (nodes < N)), "Event out of range")
|
||||
_require(np.all((ticks[1:] > ticks[:-1]) | ((ticks[1:] == ticks[:-1]) & (nodes[1:] > nodes[:-1]))),
|
||||
"Actual events must be sorted and unique")
|
||||
ptr = np.searchsorted(ticks, np.arange(steps + 1))
|
||||
lookup = np.full(N, -1, np.int32)
|
||||
lookup[kc] = np.arange(k, dtype=np.int32)
|
||||
first = np.full(k, steps, np.int32)
|
||||
is_kc = lookup[nodes] >= 0
|
||||
np.minimum.at(first, lookup[nodes[is_kc]], ticks[is_kc])
|
||||
ever = first < steps
|
||||
first_any = int(first.min()) if ever.any() else None
|
||||
first[~ever] = -1
|
||||
if first_kc_tick is not None:
|
||||
supplied = np.asarray(first_kc_tick)
|
||||
_require(supplied.dtype == np.int32 and supplied.shape == (k,) and np.array_equal(supplied, first),
|
||||
"Supplied first_kc_tick disagrees with observed actual KC events")
|
||||
d, q = float(decay32), float(gain32)
|
||||
u32, u64 = 2.**-24, 2.**-53
|
||||
gd32, gd64 = _gamma(N - 1, u32), _gamma(N - 1, u64)
|
||||
gs32, gs64 = _gamma(2, u32), _gamma(2, u64)
|
||||
gh32, gh64, gg64 = _gamma(5, u32), _gamma(5, u64), _gamma(2, u64)
|
||||
group_state = np.zeros((3, 2), np.float64)
|
||||
e64 = np.zeros_like(group_state)
|
||||
e32 = np.zeros(2, np.float64)
|
||||
previous32 = np.zeros(2, np.float64)
|
||||
statistics = {name: _metric() for name in ('drive', 'state', 'inhibition')}
|
||||
tiny32, tiny64 = np.finfo(np.float32).tiny, np.finfo(np.float64).tiny
|
||||
normal = _normal(I, tiny32) and _normal(O, tiny32)
|
||||
sums = {name: dict(actual_spikes=0, input_supported_spikes=0, first_actual_spike_tick=None,
|
||||
first_nonzero_drive_tick=None, cumulative_drive_by_apl=np.zeros(2),
|
||||
cumulative_state_by_apl=np.zeros(2), h_sum_mV=0., h_before_first_KC_sum_mV=0.,
|
||||
h_never_fired_KC_sum_mV=0., h_before_first_any_KC_sum_mV=0.) for name in GROUPS}
|
||||
rows = []
|
||||
per_kc = dict(neuron_idx=kc.copy(), first_spike_tick=first.copy(), ever_fired=ever.copy())
|
||||
for name in GROUPS:
|
||||
for suffix in ('pre_first_h_sum_mV', 'never_fired_h_sum_mV', 'h_sum_mV'):
|
||||
per_kc[name + '_' + suffix] = np.zeros(k, np.float64)
|
||||
O_total, O_never = O.sum(axis=1), O[:, ~ever].sum(axis=1)
|
||||
for tick in range(steps):
|
||||
fired = nodes[ptr[tick]:ptr[tick + 1]]
|
||||
group_drive = np.zeros_like(group_state)
|
||||
spike_counts = np.zeros(3, np.int64)
|
||||
for group, name in enumerate(GROUPS):
|
||||
selected = fired[labels[fired] == group]
|
||||
spike_counts[group] = len(selected)
|
||||
group_drive[group] = I[:, selected].sum(axis=1, dtype=np.float64)
|
||||
acc = sums[name]
|
||||
acc['actual_spikes'] += len(selected)
|
||||
acc['input_supported_spikes'] += int(support[selected].sum())
|
||||
if len(selected) and acc['first_actual_spike_tick'] is None:
|
||||
acc['first_actual_spike_tick'] = tick
|
||||
if np.any(group_drive[group] > 0) and acc['first_nonzero_drive_tick'] is None:
|
||||
acc['first_nonzero_drive_tick'] = tick
|
||||
# Each source-group positive reduction independently bounds its exact
|
||||
# real sum. Outward sum across groups bounds the common exact drive.
|
||||
Dupper = _up(group_drive / np.nextafter(1. - gd64, -np.inf))
|
||||
total_Dupper = _sum_upper(Dupper)
|
||||
drive_ref = _sum_groups(group_drive)
|
||||
drive_bound = _add(_mul(gd32, total_Dupper),
|
||||
_add(_sum_upper(_mul(gd64, Dupper)), _mul(gg64, _sum_upper(group_drive))))
|
||||
e32 = _add(_mul(d, e32), _add(_mul(gs32, _add(_mul(d, previous32), total_Dupper)),
|
||||
_mul(1. + u32, _mul(gd32, total_Dupper))))
|
||||
e64 = _add(_mul(d, e64), _add(_mul(gs64, _add(_mul(d, group_state), Dupper)),
|
||||
_mul(1. + u64, _mul(gd64, Dupper))))
|
||||
normal &= _normal(d * previous32, tiny32) and _normal(d * group_state, tiny64)
|
||||
group_state = np.add(np.multiply(d, group_state), group_drive)
|
||||
state_ref = _sum_groups(group_state)
|
||||
state_bound = _add(e32, _add(_sum_upper(e64), _mul(gg64, _sum_upper(group_state))))
|
||||
observed_state = state32[tick].astype(np.float64)
|
||||
normal &= _normal(drive32[tick], tiny32) and _normal(observed_state, tiny32)
|
||||
_measure(statistics['drive'], drive32[tick], drive_ref, drive_bound)
|
||||
_measure(statistics['state'], state32[tick], state_ref, state_bound)
|
||||
for start in range(0, k, chunk_size):
|
||||
stop = min(k, start + chunk_size)
|
||||
matrix = O[:, start:stop]
|
||||
h_group = np.empty((3, stop - start), np.float64)
|
||||
h_error = np.empty_like(h_group)
|
||||
for group in range(3):
|
||||
products = group_state[group, :, None] * matrix
|
||||
dot = np.add(products[0], products[1])
|
||||
h_group[group] = np.multiply(q, dot)
|
||||
normal &= _normal(products, tiny64) and _normal(h_group[group], tiny64)
|
||||
h_error[group] = _add(_mul(q, _positive_dot_upper(e64[group], matrix)),
|
||||
_mul(gh64, _mul(q, _positive_dot_upper(group_state[group], matrix))))
|
||||
name = GROUPS[group]
|
||||
per_kc[name + '_h_sum_mV'][start:stop] += h_group[group]
|
||||
early_mask = ever[start:stop] & (tick < first[start:stop])
|
||||
per_kc[name + '_pre_first_h_sum_mV'][start:stop] += h_group[group] * early_mask
|
||||
per_kc[name + '_never_fired_h_sum_mV'][start:stop] += h_group[group] * ~ever[start:stop]
|
||||
h_ref = _sum_groups(h_group)
|
||||
h32_error = _add(_mul(q, _positive_dot_upper(e32, matrix)),
|
||||
_mul(gh32, _mul(q, _positive_dot_upper(observed_state, matrix))))
|
||||
bound = _add(h32_error, _add(_sum_upper(h_error), _mul(gg64, _sum_upper(h_group))))
|
||||
actual_products = observed_state[:, None] * matrix
|
||||
actual_dot = np.add(actual_products[0], actual_products[1])
|
||||
normal &= _normal(actual_products, tiny32) and _normal(q * actual_dot, tiny32)
|
||||
normal &= _normal(h32[tick, start:stop], tiny32)
|
||||
_measure(statistics['inhibition'], h32[tick, start:stop], h_ref, bound)
|
||||
# Aggregate output per source and per APL, never T*G*K. This is linear
|
||||
# f64 measurement; grouping/reduction is not claimed bitwise identical.
|
||||
early = ever & (tick < first)
|
||||
O_early = O[:, early].sum(axis=1)
|
||||
before_any = first_any is not None and tick < first_any
|
||||
for group, name in enumerate(GROUPS):
|
||||
acc = sums[name]
|
||||
acc['cumulative_drive_by_apl'] += group_drive[group]
|
||||
acc['cumulative_state_by_apl'] += group_state[group]
|
||||
for apl in range(2):
|
||||
amplitude = q * group_state[group, apl]
|
||||
h_sum = float(amplitude * O_total[apl])
|
||||
h_early = float(amplitude * O_early[apl])
|
||||
h_never = float(amplitude * O_never[apl])
|
||||
h_any = h_sum if before_any else 0.
|
||||
row = dict(tick=tick, group=name, apl_index=apl, source_spikes=int(spike_counts[group]),
|
||||
drive64=float(group_drive[group, apl]), state64=float(group_state[group, apl]),
|
||||
h_sum_mV=h_sum, h_before_first_KC_sum_mV=h_early,
|
||||
h_never_fired_KC_sum_mV=h_never, h_before_first_any_KC_sum_mV=h_any)
|
||||
rows.append(row)
|
||||
for key in ('h_sum_mV', 'h_before_first_KC_sum_mV', 'h_never_fired_KC_sum_mV', 'h_before_first_any_KC_sum_mV'):
|
||||
acc[key] += row[key]
|
||||
if callback is not None:
|
||||
callback(tick, dict(drive_by_group=group_drive.copy(), state_by_group=group_state.copy(),
|
||||
drive_total=drive_ref.copy(), state_total=state_ref.copy()))
|
||||
previous32 = observed_state
|
||||
total_h = sum(value['h_sum_mV'] for value in sums.values())
|
||||
total_drive = sum(float(value['cumulative_drive_by_apl'].sum()) for value in sums.values())
|
||||
for acc in sums.values():
|
||||
acc['generated_h_fraction'] = None if total_h == 0 else acc['h_sum_mV'] / total_h
|
||||
acc['drive_fraction'] = None if total_drive == 0 else float(acc['cumulative_drive_by_apl'].sum()) / total_drive
|
||||
for key in ('cumulative_drive_by_apl', 'cumulative_state_by_apl'):
|
||||
acc[key] = acc[key].tolist()
|
||||
for metric in statistics.values():
|
||||
metric['mean_abs_error'] = metric.pop('sum_abs_error') / metric['elements']
|
||||
finite_metrics = all(np.isfinite([row[key] for row in rows]).all() for key in
|
||||
('drive64', 'state64', 'h_sum_mV', 'h_before_first_KC_sum_mV', 'h_never_fired_KC_sum_mV', 'h_before_first_any_KC_sum_mV'))
|
||||
_require(finite_metrics, "Nonfinite aggregate accounting")
|
||||
_require(all(np.isfinite(value).all() for value in per_kc.values()), "Nonfinite per-KC accounting")
|
||||
passed = bool(normal and all(metric['bound_violations'] == 0 for metric in statistics.values()))
|
||||
summary = dict(pass_all=passed, steps=steps, neurons=N, apl=2, bundle_KC=k,
|
||||
actual_events=len(nodes), groups=sums, group_order=list(GROUPS),
|
||||
input_support_partition_exact=True, unknown_nonzero_input_columns=0,
|
||||
source_population_counts={name: int(np.count_nonzero(labels == index)) for index, name in enumerate(GROUPS)},
|
||||
input_supported_population_counts={name: int(np.count_nonzero(support & (labels == index))) for index, name in enumerate(GROUPS)},
|
||||
first_any_KC_spike_tick=first_any, ever_fired_KC=int(ever.sum()), never_fired_KC=int((~ever).sum()),
|
||||
decay_f32_bits=f'{int(decay32.view(np.uint32)):08x}', gain_f32_bits=f'{int(gain32.view(np.uint32)):08x}',
|
||||
comparison=statistics, theoretical_bound=dict(applicable=bool(normal),
|
||||
normal_or_zero_arithmetic=bool(normal), historical_standard_fp32_verified_by_caller=True,
|
||||
gamma_definition='gamma_n=n*u/(1-n*u); u32=2^-24,u64=2^-53; outward rounded',
|
||||
drive_terms_bound=N-1, state_operation_bound=2, output_operation_bound=5, group_sum_operation_bound=2,
|
||||
state_recurrence='E_p<=d*E_p_prev+gamma_2,p*(d*abs(s_p_prev)+D_upper)+(1+u_p)*gamma_(N-1),p*D_upper',
|
||||
grouped_f64_bound='Sum source-group error bounds, plus gamma_2,64*upward_sum(group_values) for per-tick compared drive/state/h totals only',
|
||||
output_bound='q*upward_dot(E_p,O)+gamma_5,p*q*upward_dot(s_p,O); f32 error plus all f64 group/output/sum errors',
|
||||
tolerance_fitted_to_observed_error=False,
|
||||
scope='Per-tick summed-source drive/state/h trace comparisons only; not cumulative source fractions'),
|
||||
early_definition='strict tick<each observed first KC spike; never-fired columns separate; same-tick end APL excluded',
|
||||
value_units=dict(drive64='legacy normalized-contact spike drive', state64='legacy normalized-drive accumulator',
|
||||
h='legacy mV per state-update step; sums span output KC and recorded ticks, not instantaneous conductance'),
|
||||
generated_full_bundle_trace=True, fixture_absent_KC_output_is_not_claimed_applied=True,
|
||||
row_source_spikes_repeated_for_each_APL=True, nonfeedback=True, CUDA_float32_bitwise_replay=False,
|
||||
cumulative_and_fraction_metrics_descriptive_float64=True,
|
||||
cumulative_source_fraction_error_bounds_verified=False,
|
||||
source_majority_robustness_proven=False,
|
||||
independent_brain_oracle=False, lesion_or_counterfactual=False, parameters_changed=False)
|
||||
return dict(summary=summary, tick_rows=rows, first_kc_tick=first, per_kc=per_kc)
|
||||
@@ -0,0 +1,17 @@
|
||||
58f2f78b94aa783cc30b4b5e79ab003cc83f8a09b66c195d963bca6ee691e44c README.md
|
||||
61f4c35b464315d19bbffce534cfc037a0111725da0eb07ee53a41db81767fcb apl_sources.py
|
||||
a166da3d964b49ca28a013d3a3cf33df1607c76d4a6a53a8ca8e2d955aa92cb0 context.py
|
||||
40e0f8a9e075db42ac63163de8362d808b9110f371e49f6d4088267eecf2eb4c finalize.py
|
||||
0663c1f723f614ebcdb666b46eaa49288aec26125d6aceb18cd237330a705968 pn_observer.py
|
||||
9fbef01968c54637c9063bb1e2d7f45fdc97b886b3c25815a6146fc5f219164c prepare.py
|
||||
7a79c499098b74163e7bb16adabcbc54e7069dafa204d0b99c7f4c03d79f8016 publication.py
|
||||
a5a7861e8ced4739cd1a1f4314b8af7cff71168499f1ae6ed94ac71b481bcdd8 run_audit.py
|
||||
669e91a862efea5f243a054c883bc1a183e96b0bcf02fac5f6972c4a8272a4cb structure_metrics.py
|
||||
a7ac0f38877e0244a4bcc574264c72617ec487544a19dbc705e0c1e26c4be78d test_apl_sources.py
|
||||
c2e66adb18d25264d21bfce16462ef698a3ba09d178908bef1068881d46ac1e7 test_harness.py
|
||||
fff3cb9da17d9d638e53a609a265d6531ed82824d10e06fcb3863edfeab7a267 test_pn_observer.py
|
||||
a3516aefa6a753f8fca6c5a4c2109ae4d65ce0ab0278e4795d537148f04495ff test_publication.py
|
||||
de1298a4a92facad97339e9a4cdb60ea2b7c30adc1a6f05274bd3be034239717 test_structure_metrics.py
|
||||
d687ceb171c7dad77dfe2eb53524091a7ee02ddc04d9bbafb3b4a59b0d1eb66c test_validate_events.py
|
||||
32a33988c5c9619b2ff3553c2d508924a404ade17b526541a2ff113a61df68e4 validate_events.py
|
||||
a53443ac124e9bebc813867a5015d77276c2bd3edf89cd606a57d54756fe0dd7 manifest.json
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Phase 4AI: read-only PN/APL accounting of sealed Phase 4AG-r2 records."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path('/data/digital-drosophila')
|
||||
MODEL = Path(__file__).resolve().parent
|
||||
EXP = ROOT / 'experiments/phase4ai-pn-recruitment-audit'
|
||||
AH_MODEL = ROOT / 'models/male-cns-lif/kc-drive-audit-v1'
|
||||
AH = ROOT / 'experiments/phase4ah-kc-drive-audit'
|
||||
AG_MODEL = ROOT / 'models/male-cns-lif/author-apl-lab-v2'
|
||||
AG = ROOT / 'experiments/phase4ag-author-apl-r2'
|
||||
META = ROOT / 'datasets/male-cns-v1.0/processed/neuron-only/neuron-index/unified-metadata.parquet'
|
||||
COEFF = ROOT / 'experiments/phase4af-author-fullgraph/oracle/no_input/coefficients.json'
|
||||
GROUPS = ('KC', 'APL', 'ALPN', 'MBON', 'anatomical_DAN', 'dopamine_annotation_only',
|
||||
'ALLN', 'ALIN', 'ALON', 'olfactory_sensory', 'other', 'unclassified')
|
||||
|
||||
|
||||
def load_module(name, path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as f:
|
||||
for b in iter(lambda: f.read(8 << 20), b''):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(Path(path).read_text(encoding='utf-8'))
|
||||
|
||||
|
||||
def save(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open('x', encoding='utf-8') as f:
|
||||
json.dump(value, f, indent=2, ensure_ascii=False, allow_nan=False)
|
||||
f.write('\n')
|
||||
|
||||
|
||||
def sources():
|
||||
return {p.name: digest(p) for p in sorted(MODEL.iterdir()) if p.is_file()
|
||||
and p.name not in ('manifest.json', 'checksums.sha256')}
|
||||
|
||||
|
||||
def integrity():
|
||||
prior = load_module('sealed_ah_context_ai', AH_MODEL / 'context.py')
|
||||
result = prior.integrity()
|
||||
sys.path.insert(0, str(ROOT / 'models/male-cns-lif/terminal-scope-diagnostic-v1'))
|
||||
from diagnostic import manifest_check
|
||||
for path in (AH_MODEL, AH):
|
||||
result[str(path.relative_to(ROOT))] = manifest_check(path, digest)
|
||||
assert all(row['pass'] for row in result.values())
|
||||
return result
|
||||
|
||||
|
||||
def check():
|
||||
record = read(EXP / 'protocol/preregistered.json')
|
||||
assert record['sources'] == sources(), 'Registered source changed'
|
||||
for row in record['inputs']:
|
||||
assert digest(ROOT / row['path']) == row['sha256'], row['path']
|
||||
for row in read(EXP / 'protocol/prepared-hashes.json'):
|
||||
assert digest(EXP / row['path']) == row['sha256'], row['path']
|
||||
return record
|
||||
|
||||
|
||||
def seal(root):
|
||||
rows = [dict(path=str(p.relative_to(root)), bytes=p.stat().st_size, sha256=digest(p))
|
||||
for p in sorted(root.rglob('*')) if p.is_file() and '__pycache__' not in p.parts]
|
||||
save(root / 'manifest.json', dict(root=str(root), files=rows))
|
||||
with (root / 'checksums.sha256').open('x') as f:
|
||||
for row in rows:
|
||||
f.write(row['sha256'] + ' ' + row['path'] + '\n')
|
||||
f.write(digest(root / 'manifest.json') + ' manifest.json\n')
|
||||
|
||||
|
||||
def bit_equal(a, b):
|
||||
return a.dtype == b.dtype and a.shape == b.shape and a.tobytes() == b.tobytes()
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Independently re-read evidence, aggregate descriptive outcomes, and seal."""
|
||||
from collections import Counter
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pyarrow.parquet as pq
|
||||
from context import (ROOT, MODEL, EXP, AG, AG_MODEL, GROUPS, read, save, digest, check,
|
||||
integrity, seal, bit_equal)
|
||||
from run_audit import graph_view, first_ticks, temporal_relation
|
||||
from structure_metrics import aggregate_structure
|
||||
from validate_events import validate_batch
|
||||
|
||||
|
||||
def expected_inventory():
|
||||
result = [(n, 0, 'fixture') for n in read(AG_MODEL / 'config.json')['fixtures']]
|
||||
for row in read(AG / 'protocol/calibration-cases.json'):
|
||||
result.append((row['name'], 0, 'calibration'))
|
||||
if row['seed'] == 51101:
|
||||
result.append((row['name'], 1, 'calibration'))
|
||||
return result
|
||||
|
||||
|
||||
def verify_trial(item):
|
||||
out = EXP / 'trials' / (item['case'] + '-r' + str(item['repeat']))
|
||||
source = ROOT / item['source']
|
||||
result = read(out / 'summary.json')
|
||||
hashes = read(out / 'output-hashes.json')
|
||||
assert len({r['path'] for r in hashes}) == len(hashes)
|
||||
assert {r['path'] for r in hashes} == {p.name for p in out.iterdir() if p.is_file() and p.name != 'output-hashes.json'}
|
||||
for row in hashes:
|
||||
assert digest(out / row['path']) == row['sha256'] and (out / row['path']).stat().st_size == row['bytes']
|
||||
assert (result['case'], result['repeat'], result['kind']) == (item['case'], item['repeat'], item['kind'])
|
||||
assert result['pass_all'] and all(result['exact'].values()) and result['APL_source_bound_pass']
|
||||
for name, sha in result['input_sha256'].items():
|
||||
assert digest(source / name) == sha
|
||||
spec = read(AG / 'cases' / item['case'] / 'spec.json')
|
||||
assert result['steps'] == spec['steps'] and result['duration_ms'] == spec['duration_ms']
|
||||
for key in ('mode', 'seed', 'odor_id'):
|
||||
assert result[key] == spec.get(key)
|
||||
graph = 'fixtures' if item['kind'] == 'fixture' else 'full'
|
||||
_, incoming, _, codes, targets, positions, _ = graph_view(graph)
|
||||
pn_table = pd.read_parquet(out / 'per-pn.parquet')
|
||||
kc_table = pd.read_parquet(out / 'per-kc-apl-sources.parquet')
|
||||
apl = read(out / 'apl-source-summary.json')
|
||||
assert apl['pass_all'] and apl['theoretical_bound']['applicable']
|
||||
assert all(v['bound_violations'] == 0 for v in apl['comparison'].values())
|
||||
assert not apl['source_majority_robustness_proven']
|
||||
with np.load(source / 'state.npz') as expected, np.load(out / 'replay-final.npz') as actual:
|
||||
assert bit_equal(actual['target_globals'], targets)
|
||||
for key in ('v', 'g', 'ref', 'counts'):
|
||||
assert bit_equal(actual[key], expected[key][positions])
|
||||
selected = np.isin(expected['neuron_idx'], targets)
|
||||
for key in ('step', 'neuron_idx'):
|
||||
assert bit_equal(actual[key], expected[key][selected])
|
||||
assert bit_equal(pn_table.neuron_idx.to_numpy(), targets)
|
||||
assert bit_equal(pn_table.spike_count.to_numpy(), actual['counts'])
|
||||
assert bit_equal(pn_table.first_spike_tick.to_numpy(), actual['first_spike'])
|
||||
assert bit_equal(actual['first_spike'], first_ticks(expected['step'], expected['neuron_idx'], targets))
|
||||
kc = np.load(AG / 'apl-bundle/kc_globals.npy')
|
||||
assert bit_equal(kc_table.neuron_idx.to_numpy(), kc)
|
||||
assert bit_equal(kc_table.first_spike_tick.to_numpy(), first_ticks(expected['step'], expected['neuron_idx'], kc))
|
||||
source_keys = expected['step'].astype(np.int64) * 166700 + expected['neuron_idx']
|
||||
assert np.all(source_keys[1:] > source_keys[:-1])
|
||||
source_time_keys = np.sort(expected['neuron_idx'].astype(np.int64) * spec['steps'] + expected['step'])
|
||||
final_first = actual['first_spike'].copy()
|
||||
assert np.array_equal(pn_table.first_spike_observed, pn_table.first_spike_tick >= 0)
|
||||
for name in pn_table.select_dtypes(include='number'):
|
||||
values = pn_table[name].to_numpy()
|
||||
assert not np.isinf(values).any()
|
||||
if np.isnan(values).any():
|
||||
assert name.startswith('first_') and np.array_equal(np.isnan(values), ~pn_table.first_spike_observed.to_numpy())
|
||||
assert np.isfinite(kc_table.select_dtypes(include='number').to_numpy()).all()
|
||||
design = pn_table.is_design_target.to_numpy()
|
||||
with np.load(AG / 'cases' / item['case'] / 'requests.npz') as req:
|
||||
table = pd.read_csv(source / 'requests.csv')
|
||||
assert np.array_equal(table.neuron_idx, req['neuron_idx']) and np.array_equal(table.tick, req['tick'])
|
||||
requested = np.bincount(req['neuron_idx'], minlength=166700)[targets]
|
||||
accepted = np.bincount(req['neuron_idx'][table.accepted.to_numpy().astype(bool)], minlength=166700)[targets]
|
||||
assert np.array_equal(pn_table.requested, requested) and np.array_equal(pn_table.accepted_requests, accepted)
|
||||
if item['kind'] == 'calibration':
|
||||
with np.load(AG / 'cases' / item['case'] / 'pn-drive.npz') as pdz:
|
||||
assert np.array_equal(design, np.isin(targets, pdz['targets']))
|
||||
totals = np.zeros(len(targets), np.int64)
|
||||
edge_totals = np.zeros(len(incoming.edge_id), np.int64)
|
||||
count = 0
|
||||
last_record_key = -1
|
||||
for batch in pq.ParquetFile(out / 'accepted-prefirst-edge-events.parquet').iter_batches(batch_size=65536):
|
||||
columns = {name: batch.column(name).to_numpy(zero_copy_only=False) for name in batch.schema.names}
|
||||
checked = validate_batch(columns, edge_ids=incoming.edge_id, srcglobals=incoming.src_global,
|
||||
dstglobals=targets[incoming.dst_local], weights=incoming.weights, target_globals=targets,
|
||||
target_first=final_first, source_event_keys=source_keys)
|
||||
totals += checked['per_target_count']
|
||||
count += checked['count']
|
||||
pos = np.searchsorted(incoming.edge_id, columns['edge_id'])
|
||||
keys = columns['arrival_tick'].astype(np.int64) * 25582938 + columns['edge_id']
|
||||
assert np.all(keys[1:] > keys[:-1]) and (not len(keys) or int(keys[0]) > last_record_key)
|
||||
if len(keys):
|
||||
last_record_key = int(keys[-1])
|
||||
np.add.at(edge_totals, pos, 1)
|
||||
assert np.array_equal(columns['incoming_edge_row'], pos)
|
||||
assert np.array_equal(columns['source_group'], np.asarray(GROUPS)[codes[pos]])
|
||||
assert np.array_equal(columns['target_is_design'], design[incoming.dst_local[pos]])
|
||||
assert count == result['observer']['first_arrival_record_count']
|
||||
expected_totals = sum((pn_table[g + '_accepted_nonzero_before_first_edges'].to_numpy() for g in GROUPS), start=np.zeros(len(targets), np.int64))
|
||||
assert np.array_equal(totals, expected_totals)
|
||||
edge_summary = pd.read_parquet(out / 'prefirst-edge-summary.parquet')
|
||||
assert int(edge_summary.accepted_event_count.sum()) == count
|
||||
assert len(edge_summary) == result['observer']['edge_summary_count']
|
||||
selected_edges = np.flatnonzero(edge_totals)
|
||||
assert np.array_equal(edge_summary.edge_id.to_numpy(), incoming.edge_id[selected_edges])
|
||||
assert np.array_equal(edge_summary.accepted_event_count.to_numpy(), edge_totals[selected_edges])
|
||||
# Independent completeness: initially ref=0 and no reset before the first spike.
|
||||
# Every nonzero source event arriving strictly before that boundary must be present.
|
||||
end = np.where(final_first[incoming.dst_local] >= 0, final_first[incoming.dst_local], spec['steps'])
|
||||
source_start = incoming.src_global.astype(np.int64) * spec['steps']
|
||||
source_end = source_start + np.maximum(end - 18, 0)
|
||||
independent_counts = np.searchsorted(source_time_keys, source_end) - np.searchsorted(source_time_keys, source_start)
|
||||
independent_counts[incoming.weights == 0] = 0
|
||||
assert np.array_equal(edge_totals, independent_counts), 'Independent timed antecedent coverage mismatch'
|
||||
# Validate aggregation receipts against per-neuron evidence instead of trusting JSON alone.
|
||||
for stratum, mask in (('all_PN', np.ones(len(targets), bool)), ('design_PN', design),
|
||||
('non_design_PN', ~design), ('never_active_PN', final_first < 0)):
|
||||
agg = result['observer']['aggregates'][stratum]
|
||||
assert agg['neurons'] == int(mask.sum()) and agg['spikes'] == int(pn_table.spike_count.to_numpy()[mask].sum())
|
||||
for group, metrics in agg['source_groups'].items():
|
||||
for metric, value in metrics.items():
|
||||
summed = pn_table[group + '_' + metric].to_numpy()[mask].sum(dtype=np.longdouble)
|
||||
if isinstance(value, int):
|
||||
assert int(summed) == value
|
||||
else:
|
||||
# Serialization to per-neuron f64 and regrouping changes only final reporting arithmetic.
|
||||
assert abs(float(summed) - value) <= 64 * np.finfo(np.float64).eps * max(1., abs(value))
|
||||
tick_table = pd.read_parquet(out / 'apl-per-tick.parquet')
|
||||
assert len(tick_table) == spec['steps'] * 6
|
||||
assert not tick_table[['tick', 'group', 'apl_index']].duplicated().any()
|
||||
assert set(tick_table.group.unique()) == {'design_ALPN', 'non_design_ALPN', 'KC'}
|
||||
for group in ('design_ALPN', 'non_design_ALPN', 'KC'):
|
||||
metric = group + '_h_sum_mV'
|
||||
observed = float(kc_table[metric].sum())
|
||||
expected_sum = apl['groups'][group]['h_sum_mV']
|
||||
assert abs(observed - expected_sum) <= 128 * spec['steps'] * np.finfo(np.float64).eps * max(1., abs(expected_sum))
|
||||
return result, pn_table, kc_table, apl
|
||||
|
||||
|
||||
def main():
|
||||
check()
|
||||
assert not (EXP / 'audit/failure.json').exists(), 'Cannot promote a preserved failure'
|
||||
inventory = read(EXP / 'protocol/inventory.json')
|
||||
identities = [(r['case'], r['repeat'], r['kind']) for r in inventory]
|
||||
assert identities == expected_inventory() and len(set(identities)) == 80
|
||||
completed = read(EXP / 'audit/completed.json')
|
||||
assert completed['pass_all']
|
||||
ids = [(r['case'], r['repeat']) for r in completed['trials']]
|
||||
assert len(ids) == len(set(ids)) == 80 and set(ids) == {(r[0], r[1]) for r in identities}
|
||||
assert all(r['pass_all'] for r in completed['trials'])
|
||||
repeats = completed['repeat_checks']
|
||||
assert len(repeats) == len({r['case'] for r in repeats}) == 12
|
||||
assert {r['case'] for r in repeats} == {r['case'] for r in inventory if r['repeat'] == 1}
|
||||
assert all(r['pass_all'] and all(r['checks'].values()) for r in repeats)
|
||||
reports, metrics, by_mode = [], [], {}
|
||||
cal_rows = read(AG / 'protocol/calibration-cases.json')
|
||||
odors = list(dict.fromkeys(r['odor_id'] for r in cal_rows))
|
||||
assert len(odors) == 6
|
||||
for item in inventory:
|
||||
report, pn, kc, apl = verify_trial(item)
|
||||
reports.append(report)
|
||||
print('VERIFIED', item['case'], item['repeat'], flush=True)
|
||||
if item['kind'] != 'calibration' or item['repeat'] != 0:
|
||||
continue
|
||||
metrics.append(dict(mode=report['mode'], odor_index=odors.index(report['odor_id']), seed=report['seed'],
|
||||
counts=pn.spike_count.to_numpy(), design_mask=pn.is_design_target.to_numpy()))
|
||||
by_mode.setdefault(report['mode'], []).append((report, pn, kc, apl))
|
||||
# Independently repeat the stored repeat gate without executing a trajectory.
|
||||
for item in inventory:
|
||||
if item['repeat'] != 1:
|
||||
continue
|
||||
a = EXP / 'trials' / (item['case'] + '-r0')
|
||||
b = EXP / 'trials' / (item['case'] + '-r1')
|
||||
for name in ('per-pn.parquet', 'prefirst-edge-summary.parquet', 'apl-per-tick.parquet', 'per-kc-apl-sources.parquet'):
|
||||
assert pd.read_parquet(a / name).equals(pd.read_parquet(b / name))
|
||||
assert digest(a / 'accepted-prefirst-edge-events.parquet') == digest(b / 'accepted-prefirst-edge-events.parquet')
|
||||
assert read(a / 'apl-source-summary.json') == read(b / 'apl-source-summary.json')
|
||||
with np.load(a / 'replay-final.npz') as aa, np.load(b / 'replay-final.npz') as bb:
|
||||
assert set(aa.files) == set(bb.files) and all(bit_equal(aa[k], bb[k]) for k in aa.files)
|
||||
structure = aggregate_structure(metrics)
|
||||
structure['odor_index_to_id'] = odors
|
||||
structure['ALPN_column_neuron_idx'] = np.load(AG / 'apl-bundle/pn_globals.npy').tolist()
|
||||
save(EXP / 'audit/PN-odor-structure.json', structure)
|
||||
aggregate = {}
|
||||
for mode, trials in by_mode.items():
|
||||
assert len(trials) == 30
|
||||
sources = {}
|
||||
for group in ('design_ALPN', 'non_design_ALPN', 'KC'):
|
||||
sources[group] = {metric: sum(a['groups'][group][metric] for _, _, _, a in trials) for metric in (
|
||||
'h_sum_mV', 'h_before_first_KC_sum_mV', 'h_never_fired_KC_sum_mV', 'h_before_first_any_KC_sum_mV')}
|
||||
sources[group]['drive_sum'] = sum(sum(a['groups'][group]['cumulative_drive_by_apl']) for _, _, _, a in trials)
|
||||
for group in sources:
|
||||
for key in ('h_sum_mV', 'h_before_first_KC_sum_mV', 'h_before_first_any_KC_sum_mV', 'drive_sum'):
|
||||
total = sum(s[key] for s in sources.values())
|
||||
sources[group][key + '_fraction'] = None if total == 0 else sources[group][key] / total
|
||||
source_inputs = {group: {metric: sum(r['observer']['aggregates']['non_design_PN']['source_groups'][group][metric]
|
||||
for r, _, _, _ in trials) for metric in trials[0][0]['observer']['aggregates']['non_design_PN']['source_groups'][group]}
|
||||
for group in GROUPS}
|
||||
allpn = pd.concat([p for _, p, _, _ in trials], ignore_index=True)
|
||||
allkc = pd.concat([k for _, _, k, _ in trials], ignore_index=True)
|
||||
activated = allkc[allkc.first_spike_tick >= 0]
|
||||
first_metrics = [r for r, _, _, _ in trials]
|
||||
aggregate[mode] = dict(trials=30,
|
||||
first_non_design_vs_first_KC=dict(Counter(r['first_non_design_vs_first_KC'] for r in first_metrics)),
|
||||
first_non_design_PN_tick_range=[min(r['first_non_design_PN_tick'] for r in first_metrics), max(r['first_non_design_PN_tick'] for r in first_metrics)],
|
||||
first_any_KC_tick_range=[min(r['first_any_KC_tick'] for r in first_metrics), max(r['first_any_KC_tick'] for r in first_metrics)],
|
||||
each_active_KC_relation=dict(activated.first_non_design_relation.value_counts()),
|
||||
each_active_KC_count=len(activated),
|
||||
each_active_KC_has_strict_pre_first_nondesign_PN_edge=int((activated.non_design_ALPN_strict_pre_first_edge_events > 0).sum()),
|
||||
PN_neuron_trials=len(allpn), PN_active_neuron_trials=int(allpn.first_spike_observed.sum()),
|
||||
PN_spikes=int(allpn.spike_count.sum()), design_PN_spikes=int(allpn.loc[allpn.is_design_target, 'spike_count'].sum()),
|
||||
nondesign_PN_spikes=int(allpn.loc[~allpn.is_design_target, 'spike_count'].sum()),
|
||||
non_design_PN_input_sources=source_inputs, APL_sources=sources,
|
||||
common_non_design_dimensions=structure['design']['common_non_design_count'],
|
||||
counts_are_neuron_trials_not_distinct_neurons=True)
|
||||
aggregate[mode]['each_active_KC_relation'] = {k: int(v) for k, v in aggregate[mode]['each_active_KC_relation'].items()}
|
||||
after = integrity()
|
||||
save(EXP / 'regression/old-integrity-after.json', after)
|
||||
assert read(AG / 'regression/frozen-five.json')['pass_all']
|
||||
result = dict(status='PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT', pass_all=True,
|
||||
exact_PN_trials=80, fixture_dense_pre_post_trials=8, CAL_event_final_trials=72, repeat_accounting_pass=12,
|
||||
modes=aggregate, max_PN_voltage_ledger_residual_SI=max(r['observer']['voltage_decomposition']['maximum_reconstruction_residual_SI'] for r in reports),
|
||||
CPU_audit_wall_seconds=sum(r['CPU_audit_wall_seconds'] for r in reports),
|
||||
old_integrity_pass=all(r['pass'] for r in after.values()),
|
||||
prior_frozen_five_pass=True, frozen_five_reexecuted_this_audit=False,
|
||||
prior_frozen_five_sha256=digest(AG / 'regression/frozen-five.json'),
|
||||
new_brain_simulation=False, GPU_used=False, training=False, heldout_access=False,
|
||||
candidate_generated=False, learning_achieved=False, prior_CAL_failure_unchanged=True,
|
||||
source_fraction_robustness_proven=False, counterfactual_causality_tested=False,
|
||||
limitations=['Logged upstream spikes are conditioning inputs, not an independent whole-brain oracle',
|
||||
'Real timed edges are accepted antecedents, not unique causes or necessity proofs',
|
||||
'APL uses the unchanged nonnegative engineering input map, not signed PN-to-APL LIF drive',
|
||||
'APL summed traces pass prior per-tick error bounds; accumulated shares remain descriptive',
|
||||
'Shared PN activity does not imply absent odor information; no learning gate applied',
|
||||
'No independently saved per-tick PN states for CAL; fixtures have full histories',
|
||||
'Frozen five simulation executions were not repeated; previous sealed PASS checked by hash'])
|
||||
save(EXP / 'audit/final-status.json', result)
|
||||
with (EXP / 'REPORT.md').open('x') as f:
|
||||
f.write('# Phase 4AI — conditional PN recruitment and APL source audit\n\n'
|
||||
'Status: `PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT`.\n\n'
|
||||
'80 existing records; PN events/counts/final states exact; eight fixture histories exact; twelve repeats exact. '
|
||||
'No new brain experiment, tuning, training, GPU, or learning candidate.\n\n'
|
||||
'See audit/final-status.json and audit/PN-odor-structure.json. Accepted edges are timed antecedents, '
|
||||
'not unique causes. Graded-APL cumulative source shares are descriptive, not causal or error-bounded majority claims. '
|
||||
'Prior CAL failure remains unchanged. Frozen integrity passed; five reference simulations were not rerun.\n')
|
||||
check()
|
||||
assert sum(p.stat().st_size for p in EXP.rglob('*') if p.is_file()) < 2 * 1024**3
|
||||
seal(MODEL)
|
||||
seal(EXP)
|
||||
print(result['status'], flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"root": "/data/digital-drosophila/models/male-cns-lif/pn-recruitment-audit-v1",
|
||||
"files": [
|
||||
{
|
||||
"path": "README.md",
|
||||
"bytes": 2504,
|
||||
"sha256": "58f2f78b94aa783cc30b4b5e79ab003cc83f8a09b66c195d963bca6ee691e44c"
|
||||
},
|
||||
{
|
||||
"path": "apl_sources.py",
|
||||
"bytes": 19165,
|
||||
"sha256": "61f4c35b464315d19bbffce534cfc037a0111725da0eb07ee53a41db81767fcb"
|
||||
},
|
||||
{
|
||||
"path": "context.py",
|
||||
"bytes": 3182,
|
||||
"sha256": "a166da3d964b49ca28a013d3a3cf33df1607c76d4a6a53a8ca8e2d955aa92cb0"
|
||||
},
|
||||
{
|
||||
"path": "finalize.py",
|
||||
"bytes": 17774,
|
||||
"sha256": "40e0f8a9e075db42ac63163de8362d808b9110f371e49f6d4088267eecf2eb4c"
|
||||
},
|
||||
{
|
||||
"path": "pn_observer.py",
|
||||
"bytes": 23294,
|
||||
"sha256": "0663c1f723f614ebcdb666b46eaa49288aec26125d6aceb18cd237330a705968"
|
||||
},
|
||||
{
|
||||
"path": "prepare.py",
|
||||
"bytes": 7062,
|
||||
"sha256": "9fbef01968c54637c9063bb1e2d7f45fdc97b886b3c25815a6146fc5f219164c"
|
||||
},
|
||||
{
|
||||
"path": "publication.py",
|
||||
"bytes": 14239,
|
||||
"sha256": "7a79c499098b74163e7bb16adabcbc54e7069dafa204d0b99c7f4c03d79f8016"
|
||||
},
|
||||
{
|
||||
"path": "run_audit.py",
|
||||
"bytes": 16829,
|
||||
"sha256": "a5a7861e8ced4739cd1a1f4314b8af7cff71168499f1ae6ed94ac71b481bcdd8"
|
||||
},
|
||||
{
|
||||
"path": "structure_metrics.py",
|
||||
"bytes": 10327,
|
||||
"sha256": "669e91a862efea5f243a054c883bc1a183e96b0bcf02fac5f6972c4a8272a4cb"
|
||||
},
|
||||
{
|
||||
"path": "test_apl_sources.py",
|
||||
"bytes": 12733,
|
||||
"sha256": "a7ac0f38877e0244a4bcc574264c72617ec487544a19dbc705e0c1e26c4be78d"
|
||||
},
|
||||
{
|
||||
"path": "test_harness.py",
|
||||
"bytes": 1452,
|
||||
"sha256": "c2e66adb18d25264d21bfce16462ef698a3ba09d178908bef1068881d46ac1e7"
|
||||
},
|
||||
{
|
||||
"path": "test_pn_observer.py",
|
||||
"bytes": 8472,
|
||||
"sha256": "fff3cb9da17d9d638e53a609a265d6531ed82824d10e06fcb3863edfeab7a267"
|
||||
},
|
||||
{
|
||||
"path": "test_publication.py",
|
||||
"bytes": 10514,
|
||||
"sha256": "a3516aefa6a753f8fca6c5a4c2109ae4d65ce0ab0278e4795d537148f04495ff"
|
||||
},
|
||||
{
|
||||
"path": "test_structure_metrics.py",
|
||||
"bytes": 10003,
|
||||
"sha256": "de1298a4a92facad97339e9a4cdb60ea2b7c30adc1a6f05274bd3be034239717"
|
||||
},
|
||||
{
|
||||
"path": "test_validate_events.py",
|
||||
"bytes": 6426,
|
||||
"sha256": "d687ceb171c7dad77dfe2eb53524091a7ee02ddc04d9bbafb3b4a59b0d1eb66c"
|
||||
},
|
||||
{
|
||||
"path": "validate_events.py",
|
||||
"bytes": 6884,
|
||||
"sha256": "32a33988c5c9619b2ff3553c2d508924a404ade17b526541a2ff113a61df68e4"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
"""CPU-only observer for the unchanged Phase 4AH replay_core callback.
|
||||
|
||||
All input events are fixed observations, not interventions. This module writes
|
||||
no files and never changes a callback array. Incoming records are antecedents,
|
||||
not a selected cause, shortest path, necessity test, or whole-brain simulation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
DELAY = 18
|
||||
COEFF_KEYS = ("gd", "c0", "cg", "cv", "threshold", "reset", "seed_volt")
|
||||
FLOAT_KEYS = ("old_v", "old_g", "v_integrated", "g_decayed", "v_threshold",
|
||||
"v_after_reset", "g_after_reset", "v_pre_apl", "g_pre_apl", "delta_si", "v", "g")
|
||||
INT_KEYS = ("ref_before", "ref_after_reset", "ref_pre_apl", "ref")
|
||||
BOOL_KEYS = ("available", "request", "request_accepted", "request_rejected", "fired")
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def bit_equal(a, b):
|
||||
return a.dtype == b.dtype and a.shape == b.shape and np.array_equal(
|
||||
np.ascontiguousarray(a).view(np.uint8), np.ascontiguousarray(b).view(np.uint8))
|
||||
|
||||
|
||||
class PNObserver:
|
||||
def __init__(self, target_globals, design_target_globals, incoming, edge_groups,
|
||||
group_names, coeff, steps, *, max_first_event_rows, event_sink=None):
|
||||
targets = np.asarray(target_globals)
|
||||
require(targets.dtype == np.int32 and targets.ndim == 1 and len(targets) > 0 and
|
||||
np.all((targets >= 0) & (targets < 166700)) and np.all(np.diff(targets) > 0),
|
||||
"Targets must be sorted unique global int32 IDs")
|
||||
design = np.asarray(design_target_globals)
|
||||
require(design.dtype == np.int32 and design.ndim == 1 and len(np.unique(design)) == len(design) and
|
||||
np.isin(design, targets).all(), "Design target IDs must be unique int32 subset of observed PNs")
|
||||
require(type(steps) is int and steps > 0, "steps must be a positive integer")
|
||||
require(type(max_first_event_rows) is int and max_first_event_rows > 0, "Explicit positive row budget required")
|
||||
require(event_sink is None or callable(event_sink), "event_sink must be callable or None")
|
||||
names = tuple(group_names)
|
||||
require(names and all(isinstance(name, str) and name for name in names) and len(set(names)) == len(names),
|
||||
"Source group names must be nonempty and unique")
|
||||
codes = np.asarray(edge_groups)
|
||||
e = len(incoming.edge_id)
|
||||
require(codes.ndim == 1 and codes.shape == (e,) and codes.dtype.kind in "iu" and
|
||||
np.all((codes >= 0) & (codes < len(names))), "Invalid source-group code per incoming edge")
|
||||
require(incoming.target_count == len(targets) and incoming.weights.dtype == np.float64 and
|
||||
np.isfinite(incoming.weights).all(), "Incoming index/weights mismatch")
|
||||
require(all(name in coeff and np.isfinite(coeff[name]) for name in COEFF_KEYS), "Missing/nonfinite coefficient")
|
||||
self.targets = targets.copy()
|
||||
self.design = np.isin(targets, design)
|
||||
self.incoming, self.groups, self.names = incoming, codes.astype(np.int64, copy=True), names
|
||||
self.coeff = {name: np.float64(coeff[name]) for name in COEFF_KEYS}
|
||||
self.steps, self.n = steps, len(targets)
|
||||
self.max_rows, self.sink = max_first_event_rows, event_sink
|
||||
self.callback_count, self.record_count = 0, 0
|
||||
self.first_arrival_records = []
|
||||
self.first = np.full(self.n, -1, np.int32)
|
||||
self.first_request = np.full(self.n, -1, np.int32)
|
||||
self.first_accepted_request = np.full(self.n, -1, np.int32)
|
||||
self.first_positive_arrival = np.full(self.n, -1, np.int32)
|
||||
self.counts = np.zeros(self.n, np.int32)
|
||||
self.requested = np.zeros(self.n, np.int64)
|
||||
self.request_accepted = np.zeros(self.n, np.int64)
|
||||
self.request_rejected = np.zeros(self.n, np.int64)
|
||||
self.first_request_coincident = np.zeros(self.n, bool)
|
||||
self.first_state = {name: np.full(self.n, np.nan, np.float64) for name in
|
||||
("old_v", "old_g", "v_integrated", "v_threshold", "threshold_margin", "ref_before")}
|
||||
shape = (len(names), self.n)
|
||||
self.stats = {name: np.zeros(shape, np.int64) for name in
|
||||
("due_edges", "accepted_edges", "rejected_edges", "accepted_zero_edges", "rejected_zero_edges",
|
||||
"accepted_positive_edges", "accepted_negative_edges", "rejected_positive_edges", "rejected_negative_edges",
|
||||
"accepted_nonzero_before_first_edges", "accepted_positive_before_first_edges", "accepted_negative_before_first_edges")}
|
||||
self.sums = {name: np.zeros(shape, np.longdouble) for name in
|
||||
("accepted_positive_V", "accepted_negative_abs_V", "rejected_positive_V", "rejected_negative_abs_V",
|
||||
"accepted_positive_before_first_V", "accepted_negative_before_first_abs_V")}
|
||||
self.edge_prefirst_count = np.zeros(e, np.int64)
|
||||
self.edge_first_arrival = np.full(e, -1, np.int32)
|
||||
self.edge_last_arrival = np.full(e, -1, np.int32)
|
||||
# Non-feedback signed-source decomposition. Residual corrections are
|
||||
# explicit lanes, never silently assigned to an anatomical population.
|
||||
self.lane_names = [name + "/positive_weight" for name in names] + [name + "/negative_weight" for name in names]
|
||||
self.lane_names += ["initial_and_affine_baseline", "request_or_reset", "roundoff_residual"]
|
||||
self.baseline_lane, self.request_lane, self.round_lane = 2 * len(names), 2 * len(names) + 1, 2 * len(names) + 2
|
||||
self.lv = np.zeros((len(self.lane_names), self.n), np.longdouble)
|
||||
self.lg = np.zeros_like(self.lv)
|
||||
self.first_integrated_lanes = np.full_like(self.lv, np.nan)
|
||||
self.first_threshold_lanes = np.full_like(self.lv, np.nan)
|
||||
self.first_ledger_residual = np.full(self.n, np.nan, np.longdouble)
|
||||
self.integration_roundoff_v = np.zeros(self.n, np.longdouble)
|
||||
self.integration_roundoff_g = np.zeros(self.n, np.longdouble)
|
||||
self.arrival_roundoff_g = np.zeros(self.n, np.longdouble)
|
||||
self.max_ledger_residual = np.longdouble(0)
|
||||
self.previous = None
|
||||
self.finished = False
|
||||
|
||||
def _validate_stage(self, tick, s):
|
||||
require(not self.finished and type(tick) in (int, np.int32, np.int64) and
|
||||
int(tick) == self.callback_count and tick < self.steps, "Missing, duplicate or out-of-order callback tick")
|
||||
for names, dtype in ((FLOAT_KEYS, np.float64), (INT_KEYS, np.int32), (BOOL_KEYS, bool)):
|
||||
for name in names:
|
||||
require(name in s and isinstance(s[name], np.ndarray) and s[name].shape == (self.n,) and
|
||||
s[name].dtype == dtype, "Missing/wrong callback array: " + name)
|
||||
require(np.isfinite(s[name]).all(), "Nonfinite callback array: " + name)
|
||||
require("h32" in s and s["h32"].dtype == np.float32 and s["h32"].shape == (self.n,) and
|
||||
np.all(s["h32"] == 0) and np.all(s["delta_si"] == 0), "PN observer requires zero graded APL h/delta")
|
||||
require(bit_equal(s["g"], s["g_pre_apl"]) and bit_equal(s["v"], s["v_pre_apl"]) and
|
||||
bit_equal(s["ref"], s["ref_pre_apl"]), "Nonzero or unexpected post-APL PN state change")
|
||||
require(np.all((s["ref_before"] >= 0) & (s["ref_before"] <= 22)) and
|
||||
np.all((s["ref"] >= 0) & (s["ref"] <= 22)), "Refractory range mismatch")
|
||||
require(np.array_equal(s["available"], s["ref_before"] <= 1), "Availability does not match pre-step refractory")
|
||||
require(np.array_equal(s["request_accepted"], s["request"] & s["available"]) and
|
||||
np.array_equal(s["request_rejected"], s["request"] & ~s["available"]), "Request accounting mismatch")
|
||||
require(not np.any(s["request"] & ~self.design), "Observed request outside frozen odor design targets")
|
||||
require(np.array_equal(s["fired"], s["available"] & (s["v_threshold"] > self.coeff["threshold"])),
|
||||
"Callback firing does not match strict threshold")
|
||||
if self.previous is not None:
|
||||
require(all(bit_equal(s[key], self.previous[old]) for key, old in
|
||||
(("old_v", "v"), ("old_g", "g"), ("ref_before", "ref"))), "Discontinuous callback state")
|
||||
due = s.get("due_edge_indices")
|
||||
require(isinstance(due, np.ndarray) and due.dtype == np.int64 and due.ndim == 1 and
|
||||
np.all((due >= 0) & (due < len(self.incoming.edge_id))) and np.all(np.diff(due) > 0),
|
||||
"Incoming queue must retain unique canonical edge rows")
|
||||
require("due_edge_id" in s and bit_equal(s["due_edge_id"], self.incoming.edge_id[due]), "Incoming edge-ID mismatch")
|
||||
accepted = s.get("arrival_accepted")
|
||||
require(isinstance(accepted, np.ndarray) and accepted.dtype == bool and accepted.shape == due.shape and
|
||||
np.array_equal(accepted, s["ref_after_reset"][self.incoming.dst_local[due]] == 0), "Arrival guard mismatch")
|
||||
require(tick >= DELAY or not len(due), "Incoming event before the fixed 18-step delay")
|
||||
return due, accepted
|
||||
|
||||
def __call__(self, tick, s):
|
||||
due, accepted = self._validate_stage(tick, s)
|
||||
tick = int(tick)
|
||||
dst, weights, groups = self.incoming.dst_local[due], self.incoming.weights[due], self.groups[due]
|
||||
before = self.first < 0
|
||||
first_now = before & s["fired"]
|
||||
# Arrivals happen AFTER this tick's threshold. Same-tick arrivals are
|
||||
# never antecedents to a first spike, even if malformed data said accepted.
|
||||
strictly_before = before[dst] & ~s["fired"][dst]
|
||||
record_mask = accepted & (weights != 0) & strictly_before
|
||||
new_records = int(np.count_nonzero(record_mask))
|
||||
require(self.record_count + new_records <= self.max_rows,
|
||||
"First-arrival record resource limit exceeded; no truncation or event selection permitted")
|
||||
if self.previous is None:
|
||||
self.lv[self.baseline_lane] = s["old_v"].astype(np.longdouble)
|
||||
self.lg[self.baseline_lane] = s["old_g"].astype(np.longdouble)
|
||||
available = s["available"]
|
||||
# Use old lane-g for V, as in the author update; do NOT use decayed g.
|
||||
self.lv[:, available] = (np.longdouble(self.coeff["cv"]) * self.lv[:, available] +
|
||||
np.longdouble(self.coeff["cg"]) * self.lg[:, available])
|
||||
self.lv[self.baseline_lane, available] += np.longdouble(self.coeff["c0"])
|
||||
self.lg[:, available] *= np.longdouble(self.coeff["gd"])
|
||||
rv = s["v_integrated"].astype(np.longdouble) - self.lv.sum(axis=0, dtype=np.longdouble)
|
||||
rg = s["g_decayed"].astype(np.longdouble) - self.lg.sum(axis=0, dtype=np.longdouble)
|
||||
self.integration_roundoff_v += rv
|
||||
self.integration_roundoff_g += rg
|
||||
self.lv[self.round_lane] += rv
|
||||
self.lg[self.round_lane] += rg
|
||||
self.first_integrated_lanes[:, first_now] = self.lv[:, first_now]
|
||||
req = s["request_accepted"]
|
||||
self.lv[:, req] = 0
|
||||
self.lv[self.request_lane, req] = np.longdouble(self.coeff["seed_volt"])
|
||||
self.first_threshold_lanes[:, first_now] = self.lv[:, first_now]
|
||||
vres = s["v_threshold"].astype(np.longdouble) - self.lv.sum(axis=0, dtype=np.longdouble)
|
||||
self.first_ledger_residual[first_now] = vres[first_now]
|
||||
self.max_ledger_residual = max(self.max_ledger_residual, np.max(np.abs(vres), initial=np.longdouble(0)))
|
||||
for name in self.first_state:
|
||||
values = (self.coeff["threshold"] - s["v_threshold"]) if name == "threshold_margin" else s[name]
|
||||
self.first_state[name][first_now] = values[first_now]
|
||||
self.first_request_coincident[first_now] = req[first_now]
|
||||
self.first[first_now] = tick
|
||||
for array, selected in ((self.first_request, s["request"]), (self.first_accepted_request, req)):
|
||||
array[(array < 0) & selected] = tick
|
||||
self.requested += s["request"].astype(np.int64)
|
||||
self.request_accepted += req.astype(np.int64)
|
||||
self.request_rejected += s["request_rejected"].astype(np.int64)
|
||||
self.counts += s["fired"].astype(np.int32)
|
||||
fired = s["fired"]
|
||||
self.lv[:, fired] = 0
|
||||
self.lv[self.request_lane, fired] = np.longdouble(self.coeff["reset"])
|
||||
self.lg[:, fired] = 0
|
||||
positive, negative, zero = weights > 0, weights < 0, weights == 0
|
||||
picks = {"due_edges": np.ones(len(due), bool), "accepted_edges": accepted,
|
||||
"rejected_edges": ~accepted, "accepted_zero_edges": accepted & zero,
|
||||
"rejected_zero_edges": ~accepted & zero,
|
||||
"accepted_positive_edges": accepted & positive, "accepted_negative_edges": accepted & negative,
|
||||
"rejected_positive_edges": ~accepted & positive, "rejected_negative_edges": ~accepted & negative,
|
||||
"accepted_nonzero_before_first_edges": record_mask,
|
||||
"accepted_positive_before_first_edges": record_mask & positive,
|
||||
"accepted_negative_before_first_edges": record_mask & negative}
|
||||
for name, mask in picks.items():
|
||||
np.add.at(self.stats[name], (groups[mask], dst[mask]), 1)
|
||||
for name, mask, sign in (
|
||||
("accepted_positive_V", accepted & positive, 1), ("accepted_negative_abs_V", accepted & negative, -1),
|
||||
("rejected_positive_V", ~accepted & positive, 1), ("rejected_negative_abs_V", ~accepted & negative, -1),
|
||||
("accepted_positive_before_first_V", record_mask & positive, 1),
|
||||
("accepted_negative_before_first_abs_V", record_mask & negative, -1)):
|
||||
np.add.at(self.sums[name], (groups[mask], dst[mask]), sign * weights[mask].astype(np.longdouble))
|
||||
for offset, mask in ((0, accepted & positive), (len(self.names), accepted & negative)):
|
||||
np.add.at(self.lg, (groups[mask] + offset, dst[mask]), weights[mask].astype(np.longdouble))
|
||||
ag = s["g_pre_apl"].astype(np.longdouble) - self.lg.sum(axis=0, dtype=np.longdouble)
|
||||
self.arrival_roundoff_g += ag
|
||||
self.lg[self.round_lane] += ag
|
||||
gres = s["g"].astype(np.longdouble) - self.lg.sum(axis=0, dtype=np.longdouble)
|
||||
self.max_ledger_residual = max(self.max_ledger_residual, np.max(np.abs(gres), initial=np.longdouble(0)))
|
||||
require(np.isfinite(self.lv).all() and np.isfinite(self.lg).all(), "Nonfinite non-feedback lineage ledger")
|
||||
positive_targets = np.unique(dst[accepted & positive])
|
||||
self.first_positive_arrival[positive_targets[self.first_positive_arrival[positive_targets] < 0]] = tick
|
||||
selected_rows = due[record_mask]
|
||||
np.add.at(self.edge_prefirst_count, selected_rows, 1)
|
||||
self.edge_first_arrival[selected_rows[self.edge_first_arrival[selected_rows] < 0]] = tick
|
||||
self.edge_last_arrival[selected_rows] = tick
|
||||
rows = []
|
||||
for edge in selected_rows:
|
||||
target = int(self.incoming.dst_local[edge])
|
||||
weight = float(self.incoming.weights[edge])
|
||||
rows.append({"edge_id": int(self.incoming.edge_id[edge]), "incoming_edge_row": int(edge),
|
||||
"source_neuron_idx": int(self.incoming.src_global[edge]), "target_neuron_idx": int(self.targets[target]),
|
||||
"source_group": self.names[int(self.groups[edge])], "source_spike_tick": tick - DELAY,
|
||||
"arrival_tick": tick, "target_is_design": bool(self.design[target]), "accepted": True,
|
||||
"weight_SI": weight, "weight_sign": 1 if weight > 0 else -1,
|
||||
"ref_before": int(s["ref_before"][target]), "ref_at_arrival": int(s["ref_after_reset"][target]),
|
||||
"available_at_integration": bool(s["available"][target]),
|
||||
"old_v_SI": float(s["old_v"][target]), "old_g_SI": float(s["old_g"][target]),
|
||||
"v_integrated_SI": float(s["v_integrated"][target]), "v_prethreshold_SI": float(s["v_threshold"][target]),
|
||||
"threshold_margin_SI": float(self.coeff["threshold"] - s["v_threshold"][target]),
|
||||
"g_before_arrivals_SI": float(s["g_after_reset"][target]),
|
||||
"record_scope": "accepted nonzero input strictly before any observed first spike; join final first_spike_tick"})
|
||||
if rows:
|
||||
if self.sink is None:
|
||||
self.first_arrival_records.extend(rows)
|
||||
else:
|
||||
self.sink(rows)
|
||||
self.record_count += len(rows)
|
||||
self.previous = {name: s[name].copy() for name in ("v", "g", "ref")}
|
||||
self.callback_count += 1
|
||||
|
||||
def edge_summary(self):
|
||||
"""All edges with stored pre-first records, including never-active PNs."""
|
||||
selected = np.flatnonzero(self.edge_prefirst_count)
|
||||
dst = self.incoming.dst_local[selected]
|
||||
return {"edge_id": self.incoming.edge_id[selected].copy(),
|
||||
"source_neuron_idx": self.incoming.src_global[selected].copy(), "target_neuron_idx": self.targets[dst].copy(),
|
||||
"source_group": np.asarray(self.names, dtype=str)[self.groups[selected]],
|
||||
"weight_SI": self.incoming.weights[selected].copy(),
|
||||
"accepted_event_count": self.edge_prefirst_count[selected].copy(),
|
||||
"nominal_weight_times_event_count_SI": (self.incoming.weights[selected].astype(np.longdouble) *
|
||||
self.edge_prefirst_count[selected]).astype(np.float64),
|
||||
"first_arrival_tick": self.edge_first_arrival[selected].copy(),
|
||||
"last_arrival_tick": self.edge_last_arrival[selected].copy(),
|
||||
"target_first_spike_tick": self.first[dst].copy(), "target_is_design": self.design[dst].copy()}
|
||||
|
||||
def finish(self, result):
|
||||
"""Return (per-PN arrays, JSON report); caller verifies original events/state.
|
||||
|
||||
first_* floating fields are NaN when first_spike_observed is false.
|
||||
They must not be imputed to zero or interpreted as observed voltages.
|
||||
"""
|
||||
require(not self.finished and self.callback_count == self.steps and self.previous is not None, "Observer did not cover every tick")
|
||||
require(bit_equal(result["target_globals"], self.targets) and bit_equal(result["first_spike"], self.first) and
|
||||
bit_equal(result["counts"], self.counts), "Replay result identity/count/first spike differs from observer")
|
||||
require(all(bit_equal(result[name], self.previous[name]) for name in ("v", "g", "ref")), "Replay final state differs from last callback")
|
||||
require(int(self.edge_prefirst_count.sum()) == self.record_count ==
|
||||
int(self.stats["accepted_nonzero_before_first_edges"].sum()), "Pre-first event coverage mismatch")
|
||||
require(np.array_equal(self.stats["due_edges"], self.stats["accepted_edges"] + self.stats["rejected_edges"]),
|
||||
"Due/accepted/rejected coverage mismatch")
|
||||
data = {"neuron_idx": self.targets.copy(), "is_design_target": self.design.copy(),
|
||||
"spike_count": self.counts.copy(), "first_spike_tick": self.first.copy(), "first_spike_observed": self.first >= 0,
|
||||
"first_request_tick": self.first_request.copy(), "first_accepted_request_tick": self.first_accepted_request.copy(),
|
||||
"first_accepted_positive_arrival_tick": self.first_positive_arrival.copy(),
|
||||
"requested": self.requested.copy(), "accepted_requests": self.request_accepted.copy(),
|
||||
"rejected_requests": self.request_rejected.copy(), "first_spike_coincident_with_accepted_request": self.first_request_coincident.copy(),
|
||||
"first_voltage_ledger_residual_SI": self.first_ledger_residual.astype(np.float64),
|
||||
"integration_roundoff_v_SI": self.integration_roundoff_v.astype(np.float64),
|
||||
"integration_roundoff_g_SI": self.integration_roundoff_g.astype(np.float64),
|
||||
"arrival_roundoff_g_SI": self.arrival_roundoff_g.astype(np.float64)}
|
||||
data.update({"first_" + name + ("_SI" if name != "ref_before" else ""): value.copy()
|
||||
for name, value in self.first_state.items()})
|
||||
for gi, name in enumerate(self.names):
|
||||
for key, value in {**self.stats, **self.sums}.items():
|
||||
data[name + "_" + key] = value[gi].copy() if key in self.stats else value[gi].astype(np.float64)
|
||||
for index, name in enumerate(self.lane_names):
|
||||
data["first_integrated_V_from_" + name] = self.first_integrated_lanes[index].astype(np.float64)
|
||||
data["first_prethreshold_V_from_" + name] = self.first_threshold_lanes[index].astype(np.float64)
|
||||
aggregates = {}
|
||||
for name, mask in (("all_PN", np.ones(self.n, bool)), ("design_PN", self.design),
|
||||
("non_design_PN", ~self.design), ("never_active_PN", self.first < 0)):
|
||||
aggregates[name] = {"neurons": int(mask.sum()), "active": int(np.count_nonzero(mask & (self.first >= 0))),
|
||||
"spikes": int(self.counts[mask].sum()), "source_groups": {
|
||||
group: {**{key: int(value[gi, mask].sum()) for key, value in self.stats.items()},
|
||||
**{key: float(value[gi, mask].sum(dtype=np.longdouble)) for key, value in self.sums.items()}}
|
||||
for gi, group in enumerate(self.names)}}
|
||||
report = {"pass_all": True, "scope": "Observer coverage/internal consistency only; caller must verify recorded PN events and final state exactly",
|
||||
"callback_count": self.callback_count, "steps": self.steps, "PN_count": self.n,
|
||||
"source_group_names": list(self.names), "first_arrival_record_count": self.record_count,
|
||||
"first_arrival_record_limit": self.max_rows, "records_truncated": False,
|
||||
"edge_summary_count": int(np.count_nonzero(self.edge_prefirst_count)), "aggregates": aggregates,
|
||||
"time_contract": "source_spike_tick=arrival_tick-18; arrivals follow threshold; first-spike tick arrivals excluded",
|
||||
"record_contract": "Every accepted nonzero edge event strictly before first spike for every PN; never-active PNs retain full-window records with final first=-1",
|
||||
"sum_contract": "Observer nominal signed weight sums, not the sequentially rounded runtime g; weight-times-count edge summaries are descriptive",
|
||||
"voltage_decomposition": {"nonfeedback": True, "precision": str(np.dtype(np.longdouble)),
|
||||
"lanes": list(self.lane_names), "maximum_reconstruction_residual_SI": float(self.max_ledger_residual),
|
||||
"roundoff_lane_explicit": True, "fitted_tolerance": False,
|
||||
"reset_clears_prior_lanes": True, "request_override_has_separate_lane": True,
|
||||
"first_integrated_is_before_request_override": True,
|
||||
"never_spiked_first_fields_are_NaN": True,
|
||||
"threshold_crossing_contributions_are_not_necessity_or_unique_cause": True},
|
||||
"graded_APL_to_PN": "Required h32=0 and delta_si=0 every tick; fast APL weights supplied already frozen-masked by caller",
|
||||
"new_brain_run": False, "parameter_changes": False, "shortest_path_claim": False,
|
||||
"unique_cause_claim": False, "whole_brain_intervention_claim": False}
|
||||
self.finished = True
|
||||
return data, report
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Register fixed conditional audit before reading PN contribution results."""
|
||||
from datetime import datetime, timezone
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from context import (ROOT, MODEL, EXP, AH, AH_MODEL, AG, AG_MODEL, META, COEFF,
|
||||
GROUPS, read, save, digest, sources, integrity, load_module)
|
||||
|
||||
|
||||
def main():
|
||||
assert not EXP.exists() or not any(EXP.iterdir()), 'Populated audit directory must not be reused'
|
||||
assert read(AH / 'audit/final-status.json')['status'] == 'PASS-CONDITIONAL-KC-DRIVE-AUDIT'
|
||||
assert read(AG / 'audit/final-status.json')['numerical_gate_pass']
|
||||
for sub in ('protocol', 'regression', 'trials', 'audit', 'logs'):
|
||||
(EXP / sub).mkdir(parents=True, exist_ok=False)
|
||||
test_command = [sys.executable, '-B', '-m', 'unittest', '-v',
|
||||
'test_pn_observer', 'test_apl_sources', 'test_structure_metrics', 'test_harness',
|
||||
'test_validate_events', 'test_publication']
|
||||
tested = subprocess.run(test_command, cwd=MODEL, capture_output=True, text=True)
|
||||
with (EXP / 'regression/unit-tests.log').open('x') as f:
|
||||
f.write(tested.stdout + tested.stderr)
|
||||
save(EXP / 'regression/unit-tests.json', dict(command=test_command, exit_code=tested.returncode,
|
||||
pass_all=tested.returncode == 0, sha256=digest(EXP / 'regression/unit-tests.log')))
|
||||
assert tested.returncode == 0
|
||||
paths = [AH / 'manifest.json', AH / 'checksums.sha256', AH_MODEL / 'manifest.json',
|
||||
AH_MODEL / 'checksums.sha256', AH_MODEL / 'replay_core.py', AH_MODEL / 'populations.py',
|
||||
AG / 'manifest.json', AG / 'checksums.sha256', AG_MODEL / 'manifest.json',
|
||||
AG_MODEL / 'checksums.sha256', META, COEFF,
|
||||
ROOT / 'datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-weights.int64.npy']
|
||||
save(EXP / 'protocol/preregistered.json', dict(
|
||||
timestamp_utc=datetime.now(timezone.utc).isoformat(), sources=sources(),
|
||||
inputs=[dict(path=str(p.relative_to(ROOT)), sha256=digest(p)) for p in paths],
|
||||
parent_commit=subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], text=True).strip(),
|
||||
scope='Post-outcome conditional PN/APL source audit; no new whole-brain simulation or learning',
|
||||
previous_KC_outcome_known=True, before_PN_contribution_accounting=True,
|
||||
new_GPU_use=False, max_threads=4, output_budget_bytes=2 * 1024**3,
|
||||
per_trial_first_arrival_row_limit=1000000, truncation_allowed=False,
|
||||
exact_gate='All PN events/counts/final v,g,ref; fixture pre/post histories; 12 deterministic accounting repeats',
|
||||
APL_gate='Source float64 mathematical accounting within prior standard-float32 analytic roundoff bounds; not CUDA bitwise replay',
|
||||
hypotheses={
|
||||
'temporal_precedence': 'For every trial and every KC first spike, non-design PN spike must be strictly earlier; equals/after/absent retained',
|
||||
'direct_PN_KC_arrival': 'PN source tick+18 strictly less than KC first spike; same tick arrival is after threshold',
|
||||
'APL_source_precedence': 'Group h write at tick s affects only KC threshold ticks greater than s',
|
||||
'APL_share': 'Cumulative group shares are descriptive float64; no accumulated source-share error bound or robust-majority claim',
|
||||
'odor_structure': 'Fixed 686-dimensional counts; secondary fixed outside-six-odor-design-union subset; no activity-fraction or learning gate'},
|
||||
attribution_limits=['Accepted edge antecedents are not proof of single-neuron necessity or intervention causality',
|
||||
'Non-design means not in static design targets; does not mean noise or biologically incorrect',
|
||||
'Source-V lanes are non-feedback accounting with initial/constant/request/reset/roundoff lanes',
|
||||
'APL input matrix is legacy nonnegative raw-contact-normalized engineering mapping, not signed LIF current',
|
||||
'No CAL per-tick target state was independently saved; final states and events only',
|
||||
'Fixtures generated h for 4064 KC but applied only to fixture KC'],
|
||||
python=sys.version, numpy=np.__version__, pandas=pd.__version__, machine=platform.machine(),
|
||||
longdouble_mantissa_bits=int(np.finfo(np.longdouble).nmant)))
|
||||
save(EXP / 'regression/old-integrity-before.json', integrity())
|
||||
population = load_module('frozen_population_ai', AH_MODEL / 'populations.py')
|
||||
assert digest(META) == population.METADATA_SHA256
|
||||
meta = pd.read_parquet(META).sort_values('neuron_idx').reset_index(drop=True)
|
||||
labels, report = population.classify(meta)
|
||||
assert report['pass_all']
|
||||
for name, count in (('ALLN', 420), ('ALIN', 24), ('ALON', 14), ('olfactory', 2639)):
|
||||
selected = (meta.annotation_class == name).fillna(False).to_numpy(dtype=bool)
|
||||
assert int(selected.sum()) == count and np.all(labels[selected] == 'other')
|
||||
labels[selected] = 'olfactory_sensory' if name == 'olfactory' else name
|
||||
assert set(np.unique(labels)).issubset(GROUPS)
|
||||
for label, fname, count in (('KC', 'kc_globals.npy', 4064), ('APL', 'apl_globals.npy', 2), ('ALPN', 'pn_globals.npy', 686)):
|
||||
observed = np.flatnonzero(labels == label).astype(np.int32)
|
||||
assert len(observed) == count and np.array_equal(observed, np.load(AG / 'apl-bundle' / fname))
|
||||
report.update(refined_exact_class_counts={label: int((labels == label).sum()) for label in GROUPS},
|
||||
refinement='AH anatomical classifier, then exact ALLN/ALIN/ALON/olfactory classes split only from other; no name-prefix guessing')
|
||||
save(EXP / 'protocol/population-audit.json', report)
|
||||
save(EXP / 'protocol/metadata-schema.json', {c: dict(dtype=str(meta[c].dtype), nulls=int(meta[c].isna().sum())) for c in meta})
|
||||
meta[['neuron_idx', 'body_id', 'annotation_class', 'annotation_type', 'neurotransmitter']].assign(
|
||||
audit_source_group=labels).to_parquet(EXP / 'protocol/neuron-source-groups.parquet', index=False)
|
||||
inventory = [dict(case=name, repeat=0, kind='fixture') for name in read(AG_MODEL / 'config.json')['fixtures']]
|
||||
for row in read(AG / 'protocol/calibration-cases.json'):
|
||||
inventory.append(dict(case=row['name'], repeat=0, kind='calibration'))
|
||||
if row['seed'] == 51101:
|
||||
inventory.append(dict(case=row['name'], repeat=1, kind='calibration'))
|
||||
assert len(inventory) == len({(r['case'], r['repeat']) for r in inventory}) == 80
|
||||
for row in inventory:
|
||||
source = AG / 'gpu' / (row['case'] + '-r' + str(row['repeat']))
|
||||
assert read(source / 'summary.json')['complete']
|
||||
row['source'] = str(source.relative_to(ROOT))
|
||||
save(EXP / 'protocol/inventory.json', inventory)
|
||||
save(EXP / 'protocol/prepared-hashes.json', [dict(path=str(p.relative_to(EXP)), sha256=digest(p))
|
||||
for p in sorted((EXP / 'protocol').iterdir()) if p.is_file() and p.name != 'prepared-hashes.json'])
|
||||
print('REGISTERED fixed 80 existing records; no new brain simulation', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Publish small evidence only after independent verification of both seals.
|
||||
|
||||
No prior verification helper is imported or executed. Publishing verifies
|
||||
bytes and declared completion; it does not rerun or certify a scientific gate.
|
||||
Every destination is exclusive. An interrupted publication is retained, not
|
||||
deleted, overwritten, or silently resumed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
|
||||
|
||||
ROOT = Path('/data/digital-drosophila')
|
||||
MODEL_RELATIVE = Path('models/male-cns-lif/pn-recruitment-audit-v1')
|
||||
EXPERIMENT_RELATIVE = Path('experiments/phase4ai-pn-recruitment-audit')
|
||||
REGISTRY_RELATIVE = Path('registry/phase4ai-pn-recruitment-audit')
|
||||
FINAL_STATUS = 'PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT'
|
||||
MAX_EVIDENCE_BYTES = 4 * 1024 ** 2
|
||||
EVIDENCE_SUFFIXES = frozenset(('.json', '.csv', '.md', '.log', '.sha256'))
|
||||
CONTROLS = frozenset(('manifest.json', 'checksums.sha256'))
|
||||
SHA256 = re.compile(r'^[0-9a-f]{64}$')
|
||||
|
||||
|
||||
def _require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _unique_object(pairs):
|
||||
result = {}
|
||||
for name, value in pairs:
|
||||
_require(name not in result, 'Duplicate JSON object key: ' + name)
|
||||
result[name] = value
|
||||
return result
|
||||
|
||||
|
||||
def _bad_constant(value):
|
||||
raise ValueError('Nonfinite JSON constant: ' + value)
|
||||
|
||||
|
||||
def _json_bytes(data):
|
||||
return json.loads(data.decode('utf-8'), object_pairs_hook=_unique_object,
|
||||
parse_constant=_bad_constant)
|
||||
|
||||
|
||||
def _digest(path):
|
||||
checksum = hashlib.sha256()
|
||||
with Path(path).open('rb') as handle:
|
||||
for block in iter(lambda: handle.read(8 << 20), b''):
|
||||
checksum.update(block)
|
||||
return checksum.hexdigest()
|
||||
|
||||
|
||||
def _write_json(path, value):
|
||||
with Path(path).open('x', encoding='utf-8') as handle:
|
||||
json.dump(value, handle, indent=2, ensure_ascii=False, allow_nan=False)
|
||||
handle.write('\n')
|
||||
|
||||
|
||||
def _relative_name(value):
|
||||
_require(isinstance(value, str) and bool(value) and '\\' not in value and
|
||||
'\x00' not in value and '\n' not in value and '\r' not in value,
|
||||
'Invalid manifest/checksum relative path')
|
||||
path = PurePosixPath(value)
|
||||
_require(not path.is_absolute() and str(path) == value and
|
||||
all(part not in ('', '.', '..') and ':' not in part for part in path.parts),
|
||||
'Noncanonical or escaping relative path: ' + value)
|
||||
return path
|
||||
|
||||
|
||||
def _regular_file(base, relative):
|
||||
rel = _relative_name(relative)
|
||||
target = base
|
||||
for part in rel.parts:
|
||||
target = target / part
|
||||
_require(not target.is_symlink(), 'Symlink not allowed in sealed path: ' + str(target))
|
||||
_require(target.exists() and stat.S_ISREG(target.stat().st_mode),
|
||||
'Missing/nonregular sealed file: ' + str(target))
|
||||
_require(target.resolve().is_relative_to(base), 'Sealed file escapes its root')
|
||||
return target
|
||||
|
||||
|
||||
def verify_seal(directory):
|
||||
"""Independently verify every listed byte/hash and the checksum inventory.
|
||||
|
||||
Unlisted files fail, apart from __pycache__ files explicitly excluded by
|
||||
the producer's seal contract. Such caches are counted, never published.
|
||||
The checksum file must describe exactly the manifest payload plus the
|
||||
manifest itself. Symlinks, duplicate paths and path traversal fail.
|
||||
"""
|
||||
supplied = Path(directory)
|
||||
_require(supplied.is_absolute() and supplied.is_dir() and not supplied.is_symlink(),
|
||||
'A sealed root must be an absolute real directory')
|
||||
base = supplied.resolve(strict=True)
|
||||
manifest_path = _regular_file(base, 'manifest.json')
|
||||
checksums_path = _regular_file(base, 'checksums.sha256')
|
||||
manifest_bytes = manifest_path.read_bytes()
|
||||
manifest_sha = hashlib.sha256(manifest_bytes).hexdigest()
|
||||
manifest = _json_bytes(manifest_bytes)
|
||||
_require(isinstance(manifest, dict) and set(manifest) == {'root', 'files'}, 'Unexpected seal manifest schema')
|
||||
_require(isinstance(manifest['root'], str) and Path(manifest['root']).is_absolute() and
|
||||
Path(manifest['root']).resolve(strict=True) == base, 'Manifest names another root')
|
||||
_require(isinstance(manifest['files'], list) and bool(manifest['files']), 'Empty/invalid manifest payload')
|
||||
rows, expected = [], {}
|
||||
for row in manifest['files']:
|
||||
_require(isinstance(row, dict) and set(row) == {'path', 'bytes', 'sha256'}, 'Invalid manifest file row')
|
||||
relative = row['path']
|
||||
parsed = _relative_name(relative)
|
||||
_require(relative not in CONTROLS and '__pycache__' not in parsed.parts,
|
||||
'Control/cache file incorrectly listed as payload: ' + relative)
|
||||
_require(relative not in expected, 'Duplicate manifest path: ' + relative)
|
||||
_require(type(row['bytes']) is int and row['bytes'] >= 0 and isinstance(row['sha256'], str) and
|
||||
SHA256.fullmatch(row['sha256']), 'Invalid byte count or SHA-256: ' + relative)
|
||||
path = _regular_file(base, relative)
|
||||
_require(path.stat().st_size == row['bytes'], 'Payload byte count mismatch: ' + relative)
|
||||
_require(_digest(path) == row['sha256'], 'Payload SHA-256 mismatch: ' + relative)
|
||||
_require(path.stat().st_size == row['bytes'], 'Payload changed during verification: ' + relative)
|
||||
expected[relative] = row['sha256']
|
||||
rows.append(dict(row))
|
||||
observed, excluded_caches = set(), 0
|
||||
for path in base.rglob('*'):
|
||||
_require(not path.is_symlink(), 'Symlink found in sealed tree: ' + str(path))
|
||||
if path.is_dir():
|
||||
continue
|
||||
_require(stat.S_ISREG(path.stat().st_mode), 'Nonregular file in sealed tree: ' + str(path))
|
||||
relative = path.relative_to(base).as_posix()
|
||||
if '__pycache__' in path.relative_to(base).parts:
|
||||
excluded_caches += 1
|
||||
else:
|
||||
observed.add(relative)
|
||||
_require(observed == set(expected) | CONTROLS,
|
||||
'Sealed tree has unlisted/missing files: ' + repr(sorted(observed ^ (set(expected) | CONTROLS))))
|
||||
checksums_bytes = checksums_path.read_bytes()
|
||||
checksum_rows = {}
|
||||
for line in checksums_bytes.decode('utf-8').splitlines():
|
||||
_require(len(line) >= 67 and line[64:66] == ' ' and SHA256.fullmatch(line[:64]),
|
||||
'Malformed checksums.sha256 line')
|
||||
relative = line[66:]
|
||||
_relative_name(relative)
|
||||
_require(relative not in checksum_rows, 'Duplicate checksum path: ' + relative)
|
||||
checksum_rows[relative] = line[:64]
|
||||
_require(checksum_rows == {**expected, 'manifest.json': manifest_sha},
|
||||
'Checksum inventory or manifest self-checksum mismatch')
|
||||
return dict(pass_all=True, root=str(base), payload_file_count=len(rows),
|
||||
payload_bytes=sum(row['bytes'] for row in rows), manifest_sha256=manifest_sha,
|
||||
checksums_sha256=hashlib.sha256(checksums_bytes).hexdigest(),
|
||||
excluded_pycache_files=excluded_caches, files=sorted(rows, key=lambda row: row['path']),
|
||||
verification_scope='All sealed payload bytes/SHA-256, exact checksum inventory, and manifest self-SHA; no scientific gate execution')
|
||||
|
||||
|
||||
def _copy_verified(source, destination, byte_count, checksum):
|
||||
_require(source.stat().st_size == byte_count, 'Source size changed before evidence copy')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with source.open('rb') as src, destination.open('xb') as dst:
|
||||
shutil.copyfileobj(src, dst, length=1 << 20)
|
||||
_require(destination.stat().st_size == byte_count and _digest(destination) == checksum,
|
||||
'Copied evidence failed its sealed byte/hash check')
|
||||
_require(source.stat().st_size == byte_count and _digest(source) == checksum,
|
||||
'Sealed source changed during evidence copy')
|
||||
|
||||
|
||||
def _control_rows(root, verification):
|
||||
return [dict(path='manifest.json', bytes=(root / 'manifest.json').stat().st_size,
|
||||
sha256=verification['manifest_sha256']),
|
||||
dict(path='checksums.sha256', bytes=(root / 'checksums.sha256').stat().st_size,
|
||||
sha256=verification['checksums_sha256'])]
|
||||
|
||||
|
||||
def publish(root=ROOT):
|
||||
"""Create the one new registry exclusively; call only after finalize seals.
|
||||
|
||||
``root`` exists for isolated synthetic tests. Production uses the fixed
|
||||
default project root and fixed Phase 4AI relative directories above.
|
||||
"""
|
||||
root = Path(root)
|
||||
_require(root.is_absolute() and root.is_dir() and not root.is_symlink(), 'Invalid project root')
|
||||
root = root.resolve(strict=True)
|
||||
model, experiment, registry = (root / part for part in
|
||||
(MODEL_RELATIVE, EXPERIMENT_RELATIVE, REGISTRY_RELATIVE))
|
||||
_require(not registry.exists() and not registry.is_symlink(), 'Registry already exists; no overwrite/resume')
|
||||
for path in (model, experiment, registry.parent):
|
||||
current = root
|
||||
for part in path.relative_to(root).parts:
|
||||
current = current / part
|
||||
_require(not current.is_symlink(), 'Symlink in publication root path: ' + str(current))
|
||||
verified = {'model': verify_seal(model), 'experiment': verify_seal(experiment)}
|
||||
experiment_rows = {row['path']: row for row in verified['experiment']['files']}
|
||||
_require('audit/final-status.json' in experiment_rows,
|
||||
'Final status must exist inside the sealed experiment payload')
|
||||
status = _json_bytes((experiment / 'audit/final-status.json').read_bytes())
|
||||
_require(isinstance(status, dict) and status.get('status') == FINAL_STATUS and
|
||||
status.get('pass_all') is True, 'Finalize completion status is missing or unexpected')
|
||||
for flag in ('candidate_generated', 'learning_achieved', 'new_brain_simulation', 'GPU_used'):
|
||||
_require(status.get(flag) is False, 'Unexpected finalized scope flag: ' + flag)
|
||||
plans, omitted = [], []
|
||||
for row in verified['experiment']['files'] + _control_rows(experiment, verified['experiment']):
|
||||
allowed = PurePosixPath(row['path']).suffix.lower() in EVIDENCE_SUFFIXES
|
||||
if allowed and row['bytes'] <= MAX_EVIDENCE_BYTES:
|
||||
plans.append((experiment, row, 'evidence/' + row['path']))
|
||||
else:
|
||||
omitted.append(dict(path=row['path'], bytes=row['bytes'],
|
||||
reason='exceeds_4_MiB' if row['bytes'] > MAX_EVIDENCE_BYTES else 'bulk_or_non_evidence_extension'))
|
||||
for row in _control_rows(model, verified['model']):
|
||||
_require(row['bytes'] <= MAX_EVIDENCE_BYTES, 'Model seal control exceeds small-evidence limit')
|
||||
plans.append((model, row, 'model/' + row['path']))
|
||||
for control in ('manifest.json', 'checksums.sha256'):
|
||||
_require(any(target == 'evidence/' + control for _, _, target in plans),
|
||||
'Experiment seal control exceeds small-evidence limit')
|
||||
_require(any(target == 'evidence/audit/final-status.json' for _, _, target in plans),
|
||||
'Final status exceeds small-evidence limit')
|
||||
registry.parent.mkdir(parents=True, exist_ok=True)
|
||||
registry.mkdir(exist_ok=False)
|
||||
try:
|
||||
records = []
|
||||
for source_root, row, relative in plans:
|
||||
source = _regular_file(source_root.resolve(), row['path'])
|
||||
target = registry / relative
|
||||
_copy_verified(source, target, row['bytes'], row['sha256'])
|
||||
records.append(dict(source=source.relative_to(root).as_posix(), copy=relative,
|
||||
bytes=row['bytes'], sha256=row['sha256']))
|
||||
# Revalidate after copying, including payloads deliberately not copied.
|
||||
after = {'model': verify_seal(model), 'experiment': verify_seal(experiment)}
|
||||
_require(after == verified, 'A sealed tree changed during publication')
|
||||
proof = {name: {key: value for key, value in report.items() if key != 'files'}
|
||||
for name, report in verified.items()}
|
||||
_write_json(registry / 'seal-verification.json', proof)
|
||||
_write_json(registry / 'evidence-index.json', records)
|
||||
receipt_files = []
|
||||
for path in sorted(registry.rglob('*')):
|
||||
if path.is_file():
|
||||
receipt_files.append(dict(path=path.relative_to(registry).as_posix(),
|
||||
bytes=path.stat().st_size, sha256=_digest(path)))
|
||||
receipt = dict(publication_complete=True, registry=registry.relative_to(root).as_posix(),
|
||||
project_root=str(root), declared_audit_status=status['status'],
|
||||
model_manifest_sha256=verified['model']['manifest_sha256'],
|
||||
experiment_manifest_sha256=verified['experiment']['manifest_sha256'],
|
||||
copied_evidence_files=len(records), copied_evidence_bytes=sum(row['bytes'] for row in records),
|
||||
files=receipt_files, experiment_payload_not_copied=omitted,
|
||||
model_payload_not_duplicated=True, model_payload_remains_at=MODEL_RELATIVE.as_posix(),
|
||||
bulk_remains_at=EXPERIMENT_RELATIVE.as_posix(),
|
||||
complete_bulk_backup=False, scientific_gate_executed_by_publisher=False,
|
||||
scientific_gate_certified_by_publication=False, candidate_generated=False,
|
||||
learning_achieved=False, existing_files_modified=False,
|
||||
receipt_self_hash='Not self-referential; returned by publish() and printed by main()',
|
||||
limits=dict(max_selected_evidence_file_bytes=MAX_EVIDENCE_BYTES,
|
||||
selected_suffixes=sorted(EVIDENCE_SUFFIXES)))
|
||||
receipt_path = registry / 'publication-receipt.json'
|
||||
_write_json(receipt_path, receipt)
|
||||
return dict(publication_complete=True, registry=str(registry), copied_evidence_files=len(records),
|
||||
receipt_sha256=_digest(receipt_path), scientific_gate_executed_by_publisher=False)
|
||||
except Exception as error:
|
||||
failure = registry / 'publication-failure.json'
|
||||
if not failure.exists():
|
||||
_write_json(failure, dict(publication_complete=False, type=type(error).__name__,
|
||||
message=str(error), partial_outputs_preserved=True,
|
||||
existing_files_modified=False))
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
print(json.dumps(publish(), ensure_ascii=False, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Read existing records and replay PN targets only; no GPU or counterfactual."""
|
||||
from functools import lru_cache
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
from context import (ROOT, EXP, AH_MODEL, AG, COEFF, GROUPS, read, save, digest,
|
||||
check, bit_equal, load_module)
|
||||
from pn_observer import PNObserver
|
||||
from apl_sources import decompose_apl_sources
|
||||
|
||||
|
||||
def first_ticks(event_tick, event_node, targets):
|
||||
first = np.full(166700, np.iinfo(np.int32).max, np.int32)
|
||||
np.minimum.at(first, event_node, event_tick)
|
||||
first[first == np.iinfo(np.int32).max] = -1
|
||||
return first[targets]
|
||||
|
||||
|
||||
def temporal_relation(source_tick, target_tick):
|
||||
if source_tick < 0 or target_tick < 0:
|
||||
return 'unobserved_source' if source_tick < 0 and target_tick >= 0 else (
|
||||
'unobserved_target' if source_tick >= 0 else 'both_unobserved')
|
||||
return 'before' if source_tick < target_tick else ('same_tick' if source_tick == target_tick else 'after')
|
||||
|
||||
|
||||
def min_observed(values):
|
||||
values = np.asarray(values)
|
||||
values = values[values >= 0]
|
||||
return int(values.min()) if len(values) else -1
|
||||
|
||||
|
||||
class EventSink:
|
||||
"""Streaming accepted antecedents, never silently truncate."""
|
||||
def __init__(self, path):
|
||||
ints64 = ('edge_id', 'incoming_edge_row')
|
||||
ints32 = ('source_neuron_idx', 'target_neuron_idx', 'source_spike_tick', 'arrival_tick',
|
||||
'weight_sign', 'ref_before', 'ref_at_arrival')
|
||||
bools = ('target_is_design', 'accepted', 'available_at_integration')
|
||||
floats = ('weight_SI', 'old_v_SI', 'old_g_SI', 'v_integrated_SI', 'v_prethreshold_SI',
|
||||
'threshold_margin_SI', 'g_before_arrivals_SI')
|
||||
self.schema = pa.schema([(k, t) for names, t in ((ints64, pa.int64()), (ints32, pa.int32()),
|
||||
(bools, pa.bool_()), (floats, pa.float64()), (('source_group', 'record_scope'), pa.string())) for k in names])
|
||||
assert not path.exists()
|
||||
self.writer = pq.ParquetWriter(path, self.schema, compression='zstd')
|
||||
self.rows = 0
|
||||
|
||||
def __call__(self, rows):
|
||||
self.writer.write_table(pa.Table.from_pylist(rows, schema=self.schema))
|
||||
self.rows += len(rows)
|
||||
|
||||
def close(self):
|
||||
self.writer.close()
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def graph_view(graph_name):
|
||||
core = load_module('sealed_ah_replay_ai_' + graph_name, AH_MODEL / 'replay_core.py')
|
||||
folder = AG / 'graphs' / graph_name
|
||||
nodes = np.load(folder / 'nodes.npy', mmap_mode='r')
|
||||
pn = np.load(AG / 'apl-bundle/pn_globals.npy')
|
||||
apl = np.load(AG / 'apl-bundle/apl_globals.npy')
|
||||
targets, positions, _ = np.intersect1d(nodes, pn, assume_unique=True, return_indices=True)
|
||||
assert len(targets) > 0, 'Empty PN fixture subset: stop and report nonapplicable; never fabricate targets'
|
||||
lookup = np.full(len(nodes), -1, np.int32)
|
||||
lookup[positions] = np.arange(len(targets), dtype=np.int32)
|
||||
dst = np.load(folder / 'dst.npy', mmap_mode='r')
|
||||
selected = np.flatnonzero(lookup[dst] >= 0)
|
||||
src = nodes[np.load(folder / 'src.npy', mmap_mode='r')[selected]]
|
||||
ids = np.load(folder / 'edge_id.npy', mmap_mode='r')[selected]
|
||||
weights = np.load(folder / 'weights.npy', mmap_mode='r')[selected].copy()
|
||||
weights[np.isin(src, apl)] = 0.0
|
||||
raw = np.load(ROOT / 'datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-weights.int64.npy', mmap_mode='r')[ids]
|
||||
assert raw.dtype == np.int64 and np.all(raw > 0)
|
||||
incoming = core.IncomingIndex(src, lookup[dst[selected]], ids, weights, len(targets))
|
||||
groups = pd.read_parquet(EXP / 'protocol/neuron-source-groups.parquet').audit_source_group.to_numpy()
|
||||
codes = np.array([GROUPS.index(x) for x in groups], np.int32)[src]
|
||||
audit = dict(graph=graph_name, PN_targets=len(targets), incoming_edges=len(ids), contacts=int(raw.sum()),
|
||||
zeroed_fast_APL_edges=int(np.isin(src, apl).sum()), graded_APL_to_PN=False,
|
||||
groups={g: dict(edges=int((codes == i).sum()), contacts=int(raw[codes == i].sum())) for i, g in enumerate(GROUPS)},
|
||||
source_graph_unchanged=True, effective_fast_APL_zero_mask_preserved=True)
|
||||
return core, incoming, raw, codes, targets, positions, audit
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def pn_kc_edges(graph_name):
|
||||
folder = AG / 'graphs' / graph_name
|
||||
nodes = np.load(folder / 'nodes.npy', mmap_mode='r')
|
||||
pn = np.load(AG / 'apl-bundle/pn_globals.npy')
|
||||
kc = np.load(AG / 'apl-bundle/kc_globals.npy')
|
||||
src = nodes[np.load(folder / 'src.npy', mmap_mode='r')]
|
||||
dst = nodes[np.load(folder / 'dst.npy', mmap_mode='r')]
|
||||
picked = np.flatnonzero(np.isin(src, pn) & np.isin(dst, kc))
|
||||
return src[picked], np.searchsorted(kc, dst[picked]), np.load(folder / 'weights.npy', mmap_mode='r')[picked]
|
||||
|
||||
|
||||
def pn_kc_before_first(graph_name, state, kc_first, design):
|
||||
"""All PN edge arrivals strictly before first KC spike; no refractory yet."""
|
||||
src, dst, weights = pn_kc_edges(graph_name)
|
||||
rows = {'neuron_idx': np.load(AG / 'apl-bundle/kc_globals.npy').copy(), 'first_spike_tick': kc_first.copy()}
|
||||
for name in ('design_ALPN', 'non_design_ALPN'):
|
||||
rows[name + '_strict_pre_first_edge_events'] = np.zeros(len(kc_first), np.int64)
|
||||
rows[name + '_strict_pre_first_positive_V'] = np.zeros(len(kc_first), np.float64)
|
||||
rows[name + '_strict_pre_first_negative_abs_V'] = np.zeros(len(kc_first), np.float64)
|
||||
# Fixed graph edges only. Before the first spike, initially zero ref has never been reset.
|
||||
for source in np.unique(src):
|
||||
edges = np.flatnonzero((src == source) & (kc_first[dst] >= 0))
|
||||
spikes = state['step'][state['neuron_idx'] == source]
|
||||
count = np.searchsorted(spikes, kc_first[dst[edges]] - 18, side='left')
|
||||
label = 'design_ALPN' if source in design else 'non_design_ALPN'
|
||||
np.add.at(rows[label + '_strict_pre_first_edge_events'], dst[edges], count)
|
||||
np.add.at(rows[label + '_strict_pre_first_positive_V'], dst[edges], count * np.maximum(weights[edges], 0))
|
||||
np.add.at(rows[label + '_strict_pre_first_negative_abs_V'], dst[edges], count * np.maximum(-weights[edges], 0))
|
||||
return rows
|
||||
|
||||
|
||||
def audit_trial(row, population):
|
||||
start = time.perf_counter()
|
||||
out = EXP / 'trials' / (row['case'] + '-r' + str(row['repeat']))
|
||||
out.mkdir(exist_ok=False)
|
||||
source, case = ROOT / row['source'], AG / 'cases' / row['case']
|
||||
spec = read(case / 'spec.json')
|
||||
graph_name = 'fixtures' if row['kind'] == 'fixture' else 'full'
|
||||
core, incoming, raw, codes, targets, positions, graph_audit = graph_view(graph_name)
|
||||
with np.load(source / 'state.npz', allow_pickle=False) as data:
|
||||
state = {k: data[k].copy() for k in data.files}
|
||||
assert set(state) == {'step', 'neuron_idx', 'counts', 'v', 'g', 'ref'}
|
||||
with np.load(case / 'requests.npz', allow_pickle=False) as data:
|
||||
req = {k: data[k].copy() for k in data.files}
|
||||
rt = pd.read_csv(source / 'requests.csv')
|
||||
assert np.array_equal(req['tick'], rt.tick) and np.array_equal(req['neuron_idx'], rt.neuron_idx)
|
||||
assert (rt.attempted == 1).all() and rt.accepted.isin([0, 1]).all()
|
||||
assert np.array_equal(rt.rejected, 1 - rt.accepted)
|
||||
if row['kind'] == 'calibration':
|
||||
with np.load(case / 'pn-drive.npz', allow_pickle=False) as data:
|
||||
design = np.sort(data['targets'].copy())
|
||||
assert design.dtype == np.int32 and data['rates_hz'].dtype == np.float32
|
||||
assert len(design) == spec['target_count'] and (data['rates_hz'] > 0).all()
|
||||
else:
|
||||
design = np.intersect1d(req['neuron_idx'], targets)
|
||||
assert np.isin(design, targets).all() and len(np.unique(design)) == len(design)
|
||||
expected_mask = np.isin(state['neuron_idx'], targets)
|
||||
expected = {k: state[k][expected_mask] for k in ('step', 'neuron_idx')}
|
||||
initial_v = np.full(len(targets), spec['model']['resting_potential_mv'], np.float64) * np.float64(.001)
|
||||
initial_g = np.zeros(len(targets), np.float64)
|
||||
if (case / 'arrays.npz').exists():
|
||||
with np.load(case / 'arrays.npz', allow_pickle=False) as data:
|
||||
initial_v = data['initial_v_mv'][positions] * np.float64(.001)
|
||||
initial_g = data['initial_g_mv'][positions] * np.float64(.001)
|
||||
histories = {}
|
||||
if row['kind'] == 'fixture':
|
||||
for key, dtype in (('v', 'float64'), ('g', 'float64'), ('ref', 'int32')):
|
||||
histories[key] = np.load(source / f'{key}-history.{dtype}.npy', mmap_mode='r')[:, positions]
|
||||
histories['pre_' + key] = np.load(source / f'preAPL-{key}.{dtype}.npy', mmap_mode='r')[:, positions]
|
||||
sink = EventSink(out / 'accepted-prefirst-edge-events.parquet')
|
||||
observer = PNObserver(targets, design, incoming, codes, GROUPS, read(COEFF), spec['steps'],
|
||||
max_first_event_rows=1000000, event_sink=sink)
|
||||
request_lookup = np.full(166700, -1, np.int32)
|
||||
request_lookup[targets] = np.arange(len(targets), dtype=np.int32)
|
||||
def callback(tick, stage):
|
||||
for key in ('v', 'g', 'ref'):
|
||||
if histories:
|
||||
assert bit_equal(stage[key], histories[key][tick]), 'Fixture post state mismatch'
|
||||
assert bit_equal(stage[key + '_pre_apl'], histories['pre_' + key][tick]), 'Fixture pre state mismatch'
|
||||
indices = np.flatnonzero((req['tick'] == tick) & np.isin(req['neuron_idx'], targets))
|
||||
if len(indices):
|
||||
local = request_lookup[req['neuron_idx'][indices]]
|
||||
assert np.array_equal(stage['request_accepted'][local], rt.accepted.to_numpy()[indices].astype(bool))
|
||||
observer(tick, stage)
|
||||
try:
|
||||
result = core.replay(initial_v, initial_g, read(COEFF), spec['steps'], targets, req,
|
||||
dict(step=state['step'], neuron_idx=state['neuron_idx']), incoming,
|
||||
np.zeros((spec['steps'], len(targets)), np.float32), callback)
|
||||
finally:
|
||||
sink.close()
|
||||
exact = {k: bit_equal(result[k], state[k][positions]) for k in ('v', 'g', 'ref', 'counts')}
|
||||
exact.update({k: bit_equal(result[k], expected[k]) for k in ('step', 'neuron_idx')})
|
||||
assert all(exact.values()), 'PN replay diverged: ' + str(exact)
|
||||
data, observer_report = observer.finish(result)
|
||||
assert sink.rows == observer_report['first_arrival_record_count']
|
||||
table = pd.DataFrame(data)
|
||||
table.insert(1, 'body_id', population.iloc[targets].body_id.to_numpy())
|
||||
table.insert(2, 'annotation_type', population.iloc[targets].annotation_type.to_numpy())
|
||||
table.to_parquet(out / 'per-pn.parquet', index=False)
|
||||
edges = pd.DataFrame(observer.edge_summary())
|
||||
if len(edges):
|
||||
edge_positions = np.searchsorted(incoming.edge_id, edges.edge_id.to_numpy())
|
||||
assert np.array_equal(incoming.edge_id[edge_positions], edges.edge_id.to_numpy())
|
||||
edges['raw_contact_count'] = raw[edge_positions]
|
||||
for prefix in ('source', 'target'):
|
||||
global_ids = edges[prefix + '_neuron_idx'].to_numpy()
|
||||
edges[prefix + '_body_id'] = population.iloc[global_ids].body_id.to_numpy()
|
||||
edges[prefix + '_type'] = population.iloc[global_ids].annotation_type.to_numpy()
|
||||
edges.to_parquet(out / 'prefirst-edge-summary.parquet', index=False)
|
||||
with (out / 'replay-final.npz').open('xb') as f:
|
||||
np.savez_compressed(f, **result)
|
||||
kc = np.load(AG / 'apl-bundle/kc_globals.npy')
|
||||
pn = np.load(AG / 'apl-bundle/pn_globals.npy')
|
||||
first_kc = first_ticks(state['step'], state['neuron_idx'], kc)
|
||||
first_pn = first_ticks(state['step'], state['neuron_idx'], pn)
|
||||
first_nondesign = min_observed(first_pn[~np.isin(pn, design)])
|
||||
first_anykc = min_observed(first_kc)
|
||||
component = read(source / 'apl-component.json')
|
||||
env = component['environment']
|
||||
assert env['cuda_matmul_allow_tf32'] is False and env['float32_matmul_precision'] == 'highest'
|
||||
d32 = np.array([int(component['decay_f32_bits'], 16)], np.uint32).view(np.float32)[0]
|
||||
q32 = np.array([int(component['gain_f32_bits'], 16)], np.uint32).view(np.float32)[0]
|
||||
apl = decompose_apl_sources(np.load(AG / 'apl-bundle/input_matrix.npy', mmap_mode='r'),
|
||||
np.load(AG / 'apl-bundle/output_matrix.npy', mmap_mode='r'), pn, kc, design,
|
||||
state['step'], state['neuron_idx'], np.load(source / 'apl-drive.float32.npy', mmap_mode='r'),
|
||||
np.load(source / 'apl-state.float32.npy', mmap_mode='r'),
|
||||
np.load(source / 'apl-inhibition.float32.npy', mmap_mode='r'),
|
||||
decay32=d32, gain32=q32, standard_fp32=True, first_kc_tick=first_kc)
|
||||
assert apl['summary']['pass_all']
|
||||
save(out / 'apl-source-summary.json', apl['summary'])
|
||||
pd.DataFrame(apl['tick_rows']).to_parquet(out / 'apl-per-tick.parquet', index=False)
|
||||
per_kc = pd.DataFrame(apl['per_kc'])
|
||||
per_kc.insert(1, 'body_id', population.iloc[kc].body_id.to_numpy())
|
||||
per_kc['KC_present_in_source_graph'] = np.isin(kc, np.load(spec['graph_dir'] + '/nodes.npy'))
|
||||
per_kc['first_non_design_PN_tick'] = first_nondesign
|
||||
per_kc['first_non_design_relation'] = [temporal_relation(first_nondesign, int(t)) for t in first_kc]
|
||||
for key, value in pn_kc_before_first(graph_name, state, first_kc, design).items():
|
||||
if key not in ('neuron_idx', 'first_spike_tick'):
|
||||
per_kc[key] = value
|
||||
per_kc.to_parquet(out / 'per-kc-apl-sources.parquet', index=False)
|
||||
report = dict(case=row['case'], repeat=row['repeat'], kind=row['kind'], pass_all=True, exact=exact,
|
||||
every_tick_fixture_pre_post_exact=bool(histories), CAL_per_tick_independent_state_available=False,
|
||||
steps=spec['steps'], duration_ms=spec['duration_ms'], mode=spec.get('mode'), seed=spec.get('seed'),
|
||||
odor_id=spec.get('odor_id'), graph=graph_audit, observer=observer_report,
|
||||
first_non_design_PN_tick=first_nondesign, first_any_KC_tick=first_anykc,
|
||||
first_non_design_vs_first_KC=temporal_relation(first_nondesign, first_anykc),
|
||||
fixture_design_definition='Unique requested PN IDs for numerical fixture only; not odor design' if histories else None,
|
||||
APL_source_bound_pass=apl['summary']['pass_all'], candidate_generated=False, new_brain_simulation=False,
|
||||
CPU_audit_wall_seconds=time.perf_counter() - start,
|
||||
input_sha256={f: digest(source / f) for f in ('state.npz', 'requests.csv', 'summary.json',
|
||||
'apl-component.json', 'apl-drive.float32.npy', 'apl-state.float32.npy', 'apl-inhibition.float32.npy')})
|
||||
save(out / 'summary.json', report)
|
||||
save(out / 'output-hashes.json', [dict(path=p.name, sha256=digest(p), bytes=p.stat().st_size)
|
||||
for p in sorted(out.iterdir()) if p.is_file()])
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
check()
|
||||
population = pd.read_parquet(EXP / 'protocol/neuron-source-groups.parquet')
|
||||
inventory = read(EXP / 'protocol/inventory.json')
|
||||
receipts = []
|
||||
for row in inventory:
|
||||
try:
|
||||
report = audit_trial(row, population)
|
||||
receipts.append(dict(case=row['case'], repeat=row['repeat'], pass_all=report['pass_all']))
|
||||
assert sum(p.stat().st_size for p in EXP.rglob('*') if p.is_file()) < 2 * 1024**3
|
||||
print(row['case'], row['repeat'], 'EXACT PN / APL BOUND PASS',
|
||||
round(report['CPU_audit_wall_seconds'], 3), 'CPU sec', flush=True)
|
||||
except Exception as error:
|
||||
save(EXP / 'audit/failure.json', dict(case=row, error=type(error).__name__, message=str(error),
|
||||
traceback=traceback.format_exc(), completed=receipts, new_brain_simulation=False))
|
||||
raise
|
||||
repeats = []
|
||||
for row in inventory:
|
||||
if row['repeat'] != 1:
|
||||
continue
|
||||
a = EXP / 'trials' / (row['case'] + '-r0')
|
||||
b = EXP / 'trials' / (row['case'] + '-r1')
|
||||
checks = {}
|
||||
for f in ('per-pn.parquet', 'prefirst-edge-summary.parquet', 'apl-per-tick.parquet', 'per-kc-apl-sources.parquet'):
|
||||
checks[f] = pd.read_parquet(a / f).equals(pd.read_parquet(b / f))
|
||||
checks['antecedent_events'] = digest(a / 'accepted-prefirst-edge-events.parquet') == digest(b / 'accepted-prefirst-edge-events.parquet')
|
||||
checks['apl-summary'] = read(a / 'apl-source-summary.json') == read(b / 'apl-source-summary.json')
|
||||
with np.load(a / 'replay-final.npz') as aa, np.load(b / 'replay-final.npz') as bb:
|
||||
checks['replay'] = set(aa.files) == set(bb.files) and all(bit_equal(aa[k], bb[k]) for k in aa.files)
|
||||
assert all(checks.values()), 'Repeated accounting differs'
|
||||
repeats.append(dict(case=row['case'], pass_all=True, checks=checks))
|
||||
assert len(receipts) == 80 and len(repeats) == 12
|
||||
save(EXP / 'audit/completed.json', dict(pass_all=True, trials=receipts, repeat_checks=repeats,
|
||||
meaning='Conditional PN exactness and bounded APL attribution only; not a learning or causal gate'))
|
||||
print('PASS 80 conditional audits and 12 repeats', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Fixed descriptive ALPN-count comparisons; pure NumPy, no files or tuning.
|
||||
|
||||
Column positions refer to the caller's fixed canonical 686-ALPN order. This
|
||||
module cannot assign body IDs or infer anatomical identities. Repeated runs
|
||||
are integrity evidence, never additional independent seeds in these metrics.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
MODES = ('physiological', 'matched_energy')
|
||||
ODORS = tuple(range(6))
|
||||
SEEDS = tuple(range(51101, 51106))
|
||||
N_ALPN = 686
|
||||
FIELDS = frozenset(('mode', 'odor_index', 'seed', 'counts', 'design_mask'))
|
||||
METRICS = ('cosine_distance', 'pearson_correlation', 'jaccard', 'euclidean_distance')
|
||||
|
||||
|
||||
def _require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _integer(value, name):
|
||||
_require(isinstance(value, (int, np.integer)) and not isinstance(value, (bool, np.bool_)),
|
||||
name + ' must be an integer, not a Boolean')
|
||||
return int(value)
|
||||
|
||||
|
||||
def _array(value, dtype, name):
|
||||
_require(isinstance(value, np.ndarray) and not np.ma.isMaskedArray(value),
|
||||
name + ' must be an unmasked NumPy array')
|
||||
_require(value.dtype == np.dtype(dtype) and value.shape == (N_ALPN,),
|
||||
name + ' must have dtype ' + np.dtype(dtype).name + ' and shape (686,)')
|
||||
return np.array(value, copy=True, order='C')
|
||||
|
||||
|
||||
def _pair_metrics(first, second):
|
||||
# Cast BEFORE subtraction/norm so int32 extrema do not overflow.
|
||||
x, y = first.astype(np.float64), second.astype(np.float64)
|
||||
active_x, active_y = first > 0, second > 0
|
||||
intersection = int(np.count_nonzero(active_x & active_y))
|
||||
union = int(np.count_nonzero(active_x | active_y))
|
||||
result = dict(active_a=int(active_x.sum()), active_b=int(active_y.sum()),
|
||||
active_intersection=intersection, active_union=union)
|
||||
if not len(first):
|
||||
result.update({name: None for name in METRICS})
|
||||
return result
|
||||
norm_x, norm_y = float(np.linalg.norm(x)), float(np.linalg.norm(y))
|
||||
cosine = None if norm_x == 0 or norm_y == 0 else float(np.clip(
|
||||
np.dot(x, y) / (norm_x * norm_y), -1., 1.))
|
||||
# A constant or one-channel vector has undefined Pearson correlation;
|
||||
# equality does not turn it into evidence of shared structure.
|
||||
correlation = None if bool(np.all(x == x[0]) or np.all(y == y[0])) else float(
|
||||
np.clip(np.corrcoef(x, y)[0, 1], -1., 1.))
|
||||
result.update(cosine_distance=None if cosine is None else float(1. - cosine),
|
||||
pearson_correlation=correlation,
|
||||
jaccard=None if union == 0 else float(intersection / union),
|
||||
euclidean_distance=float(np.linalg.norm(x - y)))
|
||||
_require(all(value is None or np.isfinite(value) for value in result.values()),
|
||||
'A structure metric became nonfinite')
|
||||
return result
|
||||
|
||||
|
||||
def _summary(rows):
|
||||
result = {}
|
||||
for name in METRICS:
|
||||
values = np.asarray([row[name] for row in rows if row[name] is not None], np.float64)
|
||||
result[name] = dict(mean=None if not len(values) else float(values.mean()),
|
||||
variance_ddof0=None if not len(values) else float(values.var(ddof=0)),
|
||||
valid_comparisons=int(len(values)), undefined_comparisons=int(len(rows) - len(values)))
|
||||
return result
|
||||
|
||||
|
||||
def _view(mode, records, positions):
|
||||
arrays = {(odor, seed): records[(mode, odor, seed)]['counts'][positions]
|
||||
for odor in ODORS for seed in SEEDS}
|
||||
within, between, pairs = [], [], []
|
||||
for odor in ODORS:
|
||||
for first_seed, second_seed in itertools.combinations(SEEDS, 2):
|
||||
within.append(dict(odor_index=odor, seed_a=first_seed, seed_b=second_seed,
|
||||
**_pair_metrics(arrays[(odor, first_seed)], arrays[(odor, second_seed)])))
|
||||
for first_odor, second_odor in itertools.combinations(ODORS, 2):
|
||||
pair_between = []
|
||||
for seed in SEEDS:
|
||||
row = dict(odor_index_a=first_odor, odor_index_b=second_odor, seed=seed,
|
||||
**_pair_metrics(arrays[(first_odor, seed)], arrays[(second_odor, seed)]))
|
||||
pair_between.append(row)
|
||||
between.append(row)
|
||||
# Original pair-specific comparison: 10 within-seed pairs for each
|
||||
# member odor, not a baseline pooled across all six odors.
|
||||
pair_within = [row for row in within if row['odor_index'] in (first_odor, second_odor)]
|
||||
within_summary, between_summary = _summary(pair_within), _summary(pair_between)
|
||||
differences = {name: None if (within_summary[name]['mean'] is None or
|
||||
between_summary[name]['mean'] is None) else
|
||||
float(between_summary[name]['mean'] - within_summary[name]['mean'])
|
||||
for name in METRICS}
|
||||
pairs.append(dict(odor_index_a=first_odor, odor_index_b=second_odor,
|
||||
within_comparison_count=len(pair_within), between_comparison_count=len(pair_between),
|
||||
within=within_summary, between=between_summary,
|
||||
mean_between_minus_within=differences,
|
||||
within_comparisons=pair_within, between_comparisons=pair_between))
|
||||
_require(len(within) == 60 and len(between) == 75 and len(pairs) == 15 and
|
||||
all(row['within_comparison_count'] == 20 and row['between_comparison_count'] == 5
|
||||
for row in pairs), 'Internal fixed comparison inventory changed')
|
||||
return dict(dimensions=int(len(positions)), applicable=bool(len(positions)),
|
||||
column_positions=[int(value) for value in positions],
|
||||
within_comparison_count=len(within), between_comparison_count=len(between), pair_count=len(pairs),
|
||||
within=_summary(within), between=_summary(between),
|
||||
within_comparisons=within, between_comparisons=between, pairs=pairs)
|
||||
|
||||
|
||||
def aggregate_structure(records):
|
||||
"""Describe precisely 60 fixed CAL records, without selecting results.
|
||||
|
||||
Each record has exactly mode, odor_index, seed, counts:int32[686] and
|
||||
design_mask:bool[686]. There are two named modes, six odors (0..5), five
|
||||
seeds (51101..51105), and no repeat records. Every odor's design mask must
|
||||
be identical across seeds AND modes. No caller-owned array is modified.
|
||||
|
||||
Undefined cosine (zero norm), Pearson (constant vector), or Jaccard (empty
|
||||
active union) is null with explicit valid/undefined counts. An empty
|
||||
common-nondesign population is nonapplicable, with every metric null.
|
||||
"""
|
||||
_require(isinstance(records, list) and len(records) == 60,
|
||||
'Exactly 60 base CAL records are required; technical repeats must be excluded')
|
||||
indexed, masks = {}, {}
|
||||
for position, record in enumerate(records):
|
||||
_require(isinstance(record, dict) and set(record) == FIELDS,
|
||||
'Record schema must contain exactly ' + ', '.join(sorted(FIELDS)))
|
||||
mode = record['mode']
|
||||
_require(isinstance(mode, str) and mode in MODES, 'Unexpected CAL mode')
|
||||
odor = _integer(record['odor_index'], 'odor_index')
|
||||
seed = _integer(record['seed'], 'seed')
|
||||
_require(odor in ODORS and seed in SEEDS, 'Odor index or seed outside the fixed CAL inventory')
|
||||
key = mode, odor, seed
|
||||
_require(key not in indexed, 'Duplicate CAL identity: ' + str(key))
|
||||
counts = _array(record['counts'], np.int32, 'counts')
|
||||
design = _array(record['design_mask'], np.bool_, 'design_mask')
|
||||
_require(bool(np.all(counts >= 0)), 'Spike counts must be nonnegative')
|
||||
if odor in masks:
|
||||
_require(np.array_equal(design, masks[odor]),
|
||||
'An odor design mask changed across seed or mode: ' + str(odor))
|
||||
else:
|
||||
masks[odor] = design.copy()
|
||||
indexed[key] = dict(counts=counts, design_mask=design)
|
||||
expected = set(itertools.product(MODES, ODORS, SEEDS))
|
||||
_require(set(indexed) == expected, 'Incomplete fixed CAL identity set')
|
||||
union = np.logical_or.reduce([masks[odor] for odor in ODORS])
|
||||
primary = np.arange(N_ALPN, dtype=np.int32)
|
||||
common = np.flatnonzero(~union).astype(np.int32)
|
||||
return dict(schema='phase4ai-alpn-structure-v1', record_count=60, ALPN_dimensions=N_ALPN,
|
||||
design=dict(identical_across_seeds_and_modes=True,
|
||||
design_neuron_count_by_odor={str(odor): int(masks[odor].sum()) for odor in ODORS},
|
||||
design_union_count=int(union.sum()), common_non_design_count=int(len(common)),
|
||||
common_non_design_column_positions=[int(value) for value in common],
|
||||
common_non_design_definition='Fixed complement of the union of all six CAL odor design masks; same columns in all modes/seeds'),
|
||||
modes={mode: dict(trials=30, fixed_686=_view(mode, indexed, primary),
|
||||
common_non_design=_view(mode, indexed, common)) for mode in MODES},
|
||||
protocol=dict(odor_indices=list(ODORS), seeds=list(SEEDS), variance_ddof=0,
|
||||
within='All 10 distinct seed pairs within each of six odors: 60 comparisons per mode',
|
||||
between='Same-seed pairs for each of 15 odor pairs: 75 comparisons per mode',
|
||||
pair_baseline='Only each odor pair: 10+10 within comparisons and five same-seed between comparisons',
|
||||
counts='Original nonnegative int32 spike counts; no normalization, fitting or resampling',
|
||||
undefined_cosine='null if either vector has zero norm',
|
||||
undefined_Pearson='null if either vector is constant, including dimension one',
|
||||
undefined_Jaccard='null if active union is empty',
|
||||
empty_population='All metrics null, applicable=false; no fabricated zero distance',
|
||||
clipping='Only cosine/correlation numerical roundoff is clipped to their mathematical [-1,1] range',
|
||||
technical_repeats_included=False, population_columns='Caller-supplied fixed canonical ALPN order; positions are not neuron_idx'),
|
||||
decision_gate_applied=False, learning_claimed=False, absence_of_encoding_claimed=False,
|
||||
limitations=['Descriptive CAL structure only, not a held-out or learning result',
|
||||
'Shared activity can coexist with odor-specific information',
|
||||
'Active-set Jaccard can saturate even when spike-count patterns differ',
|
||||
'Valid metric counts vary when a metric is undefined; null is not zero similarity',
|
||||
'No KC sparsity gate is transferred to the ALPN population'])
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Synthetic CPU-only tests. No real records, files, Torch, GPU, or model runs."""
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from apl_sources import N, GROUPS, DECAY32, GAIN32, decompose_apl_sources
|
||||
|
||||
|
||||
def fixture(events=((0, 1), (1, 4), (2, 10), (4, 20)), steps=6):
|
||||
I = np.zeros((2, N), np.float32)
|
||||
I[:, 1] = [1., .3]
|
||||
I[:, 4] = [.5, 2.]
|
||||
I[:, 10] = [1.1, .2]
|
||||
I[:, 20] = [.7, 1.5]
|
||||
O = np.array([[1., .7, .2, 0.], [.3, .5, 1., 2.]], np.float32)
|
||||
pairs = np.asarray(events, np.int32).reshape(-1, 2)
|
||||
tick, node = pairs[:, 0], pairs[:, 1]
|
||||
# This is merely one legal ordinary f32 reduction implementation for
|
||||
# exercising the mathematical bound, not a CUDA-equality oracle.
|
||||
state = np.zeros(2, np.float32)
|
||||
drive_log, state_log, h_log = [], [], []
|
||||
for t in range(steps):
|
||||
drive = I[:, node[tick == t]].sum(axis=1, dtype=np.float32)
|
||||
state = np.add(np.multiply(DECAY32, state), drive)
|
||||
h = np.multiply(np.matmul(state, O), GAIN32)
|
||||
drive_log.append(drive)
|
||||
state_log.append(state.copy())
|
||||
h_log.append(h)
|
||||
return dict(I32=I, O32=O, pn_globals=np.array([1, 4, 6], np.int32),
|
||||
kc_globals=np.array([10, 20, 30, 40], np.int32), design_pn_globals=np.array([1], np.int32),
|
||||
event_step=tick, event_node=node, drive32=np.asarray(drive_log),
|
||||
state32=np.asarray(state_log), h32=np.asarray(h_log),
|
||||
decay32=DECAY32, gain32=GAIN32, standard_fp32=True)
|
||||
|
||||
|
||||
class AttributionTests(unittest.TestCase):
|
||||
def test_three_groups_cover_drive_and_pass_prior_bound(self):
|
||||
f = fixture()
|
||||
calls = []
|
||||
out = decompose_apl_sources(**f, callback=lambda t, s: calls.append(s))
|
||||
summary = out['summary']
|
||||
self.assertTrue(summary['pass_all'])
|
||||
self.assertTrue(summary['input_support_partition_exact'])
|
||||
np.testing.assert_array_equal(out['first_kc_tick'], [2, 4, -1, -1])
|
||||
self.assertEqual(len(out['tick_rows']), 6 * 3 * 2)
|
||||
self.assertEqual([summary['groups'][g]['actual_spikes'] for g in GROUPS], [1, 1, 2])
|
||||
for s in calls:
|
||||
np.testing.assert_array_equal(s['drive_total'], (s['drive_by_group'][0] + s['drive_by_group'][1]) + s['drive_by_group'][2])
|
||||
np.testing.assert_array_equal(s['state_total'], (s['state_by_group'][0] + s['state_by_group'][1]) + s['state_by_group'][2])
|
||||
self.assertFalse(summary['CUDA_float32_bitwise_replay'])
|
||||
self.assertFalse(summary['independent_brain_oracle'])
|
||||
self.assertFalse(summary['theoretical_bound']['tolerance_fitted_to_observed_error'])
|
||||
self.assertFalse(summary['cumulative_source_fraction_error_bounds_verified'])
|
||||
self.assertFalse(summary['source_majority_robustness_proven'])
|
||||
json.dumps(summary, allow_nan=False)
|
||||
json.dumps(out['tick_rows'], allow_nan=False)
|
||||
|
||||
def test_early_excludes_same_tick_and_never_fired_columns(self):
|
||||
f = fixture(events=((0, 1), (2, 10)))
|
||||
out = decompose_apl_sources(**f)
|
||||
rows = [r for r in out['tick_rows'] if r['group'] == 'design_ALPN' and r['apl_index'] == 0]
|
||||
d, q = float(DECAY32), float(GAIN32)
|
||||
self.assertAlmostEqual(rows[0]['h_before_first_KC_sum_mV'], q * 1.)
|
||||
self.assertAlmostEqual(rows[1]['h_before_first_KC_sum_mV'], q * d)
|
||||
self.assertEqual(rows[2]['h_before_first_KC_sum_mV'], 0.)
|
||||
self.assertGreater(rows[2]['h_never_fired_KC_sum_mV'], 0.)
|
||||
self.assertGreater(rows[1]['h_before_first_any_KC_sum_mV'], 0.)
|
||||
self.assertEqual(rows[2]['h_before_first_any_KC_sum_mV'], 0.)
|
||||
|
||||
def test_first_kc_at_zero_has_no_early_output(self):
|
||||
out = decompose_apl_sources(**fixture(events=((0, 1), (0, 10))))
|
||||
self.assertEqual(out['summary']['first_any_KC_spike_tick'], 0)
|
||||
self.assertTrue(all(r['h_before_first_any_KC_sum_mV'] == 0 for r in out['tick_rows']))
|
||||
self.assertTrue(all(r['h_before_first_KC_sum_mV'] == 0 for r in out['tick_rows']))
|
||||
|
||||
def test_no_kc_spikes_keep_never_output_separate(self):
|
||||
out = decompose_apl_sources(**fixture(events=((0, 1),)))
|
||||
self.assertIsNone(out['summary']['first_any_KC_spike_tick'])
|
||||
np.testing.assert_array_equal(out['first_kc_tick'], [-1] * 4)
|
||||
for row in out['tick_rows']:
|
||||
self.assertEqual(row['h_sum_mV'], row['h_never_fired_KC_sum_mV'])
|
||||
self.assertEqual(row['h_before_first_KC_sum_mV'], 0.)
|
||||
self.assertEqual(row['h_before_first_any_KC_sum_mV'], 0.)
|
||||
|
||||
def test_zero_input_exact_zeros_and_undefined_fractions(self):
|
||||
out = decompose_apl_sources(**fixture(events=()))
|
||||
self.assertTrue(out['summary']['pass_all'])
|
||||
for group in out['summary']['groups'].values():
|
||||
self.assertEqual(group['h_sum_mV'], 0.)
|
||||
self.assertIsNone(group['generated_h_fraction'])
|
||||
self.assertIsNone(group['drive_fraction'])
|
||||
|
||||
def test_source_first_spike_is_not_first_nonzero_drive(self):
|
||||
out = decompose_apl_sources(**fixture(events=((0, 6), (2, 4), (3, 100))))
|
||||
group = out['summary']['groups']['non_design_ALPN']
|
||||
self.assertEqual(group['actual_spikes'], 2)
|
||||
self.assertEqual(group['input_supported_spikes'], 1)
|
||||
self.assertEqual(group['first_actual_spike_tick'], 0)
|
||||
self.assertEqual(group['first_nonzero_drive_tick'], 2)
|
||||
self.assertEqual(out['summary']['actual_events'], 3)
|
||||
self.assertTrue(out['summary']['pass_all'])
|
||||
|
||||
def test_empty_design_set_reclassifies_pn_not_drops_them(self):
|
||||
f = fixture()
|
||||
f['design_pn_globals'] = np.empty(0, np.int32)
|
||||
out = decompose_apl_sources(**f)
|
||||
self.assertTrue(out['summary']['pass_all'])
|
||||
self.assertEqual(out['summary']['groups']['design_ALPN']['actual_spikes'], 0)
|
||||
self.assertEqual(out['summary']['groups']['non_design_ALPN']['actual_spikes'], 2)
|
||||
|
||||
def test_chunk_size_does_not_change_source_accounting(self):
|
||||
f = fixture()
|
||||
a = decompose_apl_sources(**f, chunk_size=1)
|
||||
b = decompose_apl_sources(**f, chunk_size=7)
|
||||
self.assertTrue(a['summary']['pass_all'] and b['summary']['pass_all'])
|
||||
self.assertEqual(a['tick_rows'], b['tick_rows'])
|
||||
self.assertEqual(a['summary']['groups'], b['summary']['groups'])
|
||||
for name in ('drive', 'state', 'inhibition'):
|
||||
for key in ('max_abs_error', 'max_error_bound', 'max_bound_utilization', 'bound_violations', 'elements'):
|
||||
self.assertEqual(a['summary']['comparison'][name][key], b['summary']['comparison'][name][key])
|
||||
|
||||
def test_callback_and_core_never_modify_inputs_or_running_state(self):
|
||||
f = fixture()
|
||||
inputs = {k: v.copy() for k, v in f.items() if isinstance(v, np.ndarray)}
|
||||
expected = decompose_apl_sources(**f)
|
||||
def mutate(t, s):
|
||||
for value in s.values():
|
||||
value[:] = 100.
|
||||
actual = decompose_apl_sources(**f, callback=mutate)
|
||||
self.assertEqual(actual['summary'], expected['summary'])
|
||||
self.assertEqual(actual['tick_rows'], expected['tick_rows'])
|
||||
for name, before in inputs.items():
|
||||
np.testing.assert_array_equal(np.ascontiguousarray(f[name]).view(np.uint8),
|
||||
np.ascontiguousarray(before).view(np.uint8))
|
||||
|
||||
def test_per_kc_strict_pre_first_and_never_sums(self):
|
||||
f = fixture(events=((0, 1), (2, 10)))
|
||||
out = decompose_apl_sources(**f, first_kc_tick=np.array([2, -1, -1, -1], np.int32))
|
||||
per = out['per_kc']
|
||||
d, q = float(DECAY32), float(GAIN32)
|
||||
first_two = q * (1. + d) * (float(f['I32'][0, 1]) * float(f['O32'][0, 0]) +
|
||||
float(f['I32'][1, 1]) * float(f['O32'][1, 0]))
|
||||
self.assertAlmostEqual(per['design_ALPN_pre_first_h_sum_mV'][0], first_two)
|
||||
np.testing.assert_array_equal(per['design_ALPN_pre_first_h_sum_mV'][1:], [0., 0., 0.])
|
||||
self.assertEqual(per['design_ALPN_never_fired_h_sum_mV'][0], 0.)
|
||||
np.testing.assert_array_equal(per['design_ALPN_never_fired_h_sum_mV'][1:], per['design_ALPN_h_sum_mV'][1:])
|
||||
self.assertEqual(per['KC_pre_first_h_sum_mV'][0], 0.)
|
||||
self.assertTrue(out['summary']['fixture_absent_KC_output_is_not_claimed_applied'])
|
||||
|
||||
def test_supplied_first_times_must_match_actual_logs(self):
|
||||
for first in (np.array([3, 4, -1, -1], np.int32), np.array([2, 4, -1, -1], np.int64)):
|
||||
with self.subTest(first=first), self.assertRaises(ValueError):
|
||||
decompose_apl_sources(**fixture(), first_kc_tick=first)
|
||||
|
||||
def test_corrupted_positive_trace_violates_unfitted_bound(self):
|
||||
for field in ('drive32', 'state32', 'h32'):
|
||||
with self.subTest(field=field):
|
||||
f = fixture()
|
||||
f[field][1, 0] += np.float32(100.)
|
||||
out = decompose_apl_sources(**f)
|
||||
self.assertFalse(out['summary']['pass_all'])
|
||||
target = {'drive32': 'drive', 'state32': 'state', 'h32': 'inhibition'}[field]
|
||||
self.assertGreater(out['summary']['comparison'][target]['bound_violations'], 0)
|
||||
|
||||
def test_zero_reference_cannot_accept_nonzero_actual(self):
|
||||
f = fixture(events=())
|
||||
f['h32'][0, 0] = np.finfo(np.float32).tiny
|
||||
out = decompose_apl_sources(**f)
|
||||
self.assertFalse(out['summary']['pass_all'])
|
||||
self.assertEqual(out['summary']['comparison']['inhibition']['zero_reference_nonzero_actual'], 1)
|
||||
|
||||
def test_subnormal_disables_theoretical_pass(self):
|
||||
f = fixture(events=())
|
||||
f['I32'][0, 1] = np.float32(np.finfo(np.float32).tiny / 2)
|
||||
out = decompose_apl_sources(**f)
|
||||
self.assertFalse(out['summary']['pass_all'])
|
||||
self.assertFalse(out['summary']['theoretical_bound']['applicable'])
|
||||
|
||||
|
||||
class ValidationTests(unittest.TestCase):
|
||||
def test_nonzero_unknown_source_is_rejected(self):
|
||||
f = fixture()
|
||||
f['I32'][0, 100] = 1.
|
||||
with self.assertRaisesRegex(ValueError, 'outside design/non-design'):
|
||||
decompose_apl_sources(**f)
|
||||
|
||||
def test_bad_identity_and_nonpartition_are_rejected(self):
|
||||
for key, value in [('design_pn_globals', np.array([10], np.int32)),
|
||||
('design_pn_globals', np.array([1, 1], np.int32)),
|
||||
('pn_globals', np.array([1, 4, 10], np.int32)),
|
||||
('kc_globals', np.array([10, 20, 40, 30], np.int32)),
|
||||
('pn_globals', np.array([1, 4, 6], np.int64))]:
|
||||
with self.subTest(key=key):
|
||||
f = fixture()
|
||||
f[key] = value
|
||||
with self.assertRaises(ValueError):
|
||||
decompose_apl_sources(**f)
|
||||
|
||||
def test_invalid_events_dtype_order_duplicate_range(self):
|
||||
changes = [('event_step', np.array([0, 1, 2, 4], np.int64)),
|
||||
('event_step', np.array([1, 0, 2, 4], np.int32)),
|
||||
('event_step', np.array([0, 1, 2, 6], np.int32)),
|
||||
('event_node', np.array([1, 4, 10, N], np.int32))]
|
||||
for key, value in changes:
|
||||
with self.subTest(key=key):
|
||||
f = fixture()
|
||||
f[key] = value
|
||||
with self.assertRaises(ValueError):
|
||||
decompose_apl_sources(**f)
|
||||
f = fixture(events=((0, 1), (0, 1)))
|
||||
with self.assertRaises(ValueError):
|
||||
decompose_apl_sources(**f)
|
||||
|
||||
def test_invalid_trace_matrix_precision_constants_or_environment(self):
|
||||
variants = [dict(standard_fp32=False), dict(standard_fp32=1), dict(chunk_size=0),
|
||||
dict(decay32=float(DECAY32)), dict(gain32=np.nextafter(GAIN32, np.float32(np.inf))),
|
||||
dict(callback=3)]
|
||||
for change in variants:
|
||||
with self.subTest(change=change), self.assertRaises(ValueError):
|
||||
decompose_apl_sources(**(fixture() | change))
|
||||
for name in ('I32', 'O32', 'drive32', 'state32', 'h32'):
|
||||
for bad in ('dtype', 'negative', 'nonfinite'):
|
||||
with self.subTest(name=name, bad=bad):
|
||||
f = fixture()
|
||||
if bad == 'dtype':
|
||||
f[name] = f[name].astype(np.float64)
|
||||
else:
|
||||
f[name].flat[0] = -1. if bad == 'negative' else np.nan
|
||||
with self.assertRaises(ValueError):
|
||||
decompose_apl_sources(**f)
|
||||
|
||||
def test_no_dense_source_history_or_scientific_runtime(self):
|
||||
out = decompose_apl_sources(**fixture())
|
||||
self.assertEqual(set(out), {'summary', 'tick_rows', 'first_kc_tick', 'per_kc'})
|
||||
self.assertNotIn('torch', sys.modules)
|
||||
self.assertNotIn('brian2', sys.modules)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,36 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import pyarrow.parquet as pq
|
||||
from run_audit import first_ticks, temporal_relation, min_observed, EventSink
|
||||
|
||||
|
||||
class HarnessTests(unittest.TestCase):
|
||||
def test_first_tick_uses_minimum_not_order(self):
|
||||
self.assertEqual(first_ticks(np.array([4, 2, 3], np.int32), np.array([7, 7, 8], np.int32),
|
||||
np.array([7, 8, 9], np.int32)).tolist(), [2, 3, -1])
|
||||
|
||||
def test_strict_temporal_boundary(self):
|
||||
self.assertEqual(temporal_relation(17, 18), 'before')
|
||||
self.assertEqual(temporal_relation(18, 18), 'same_tick')
|
||||
self.assertEqual(temporal_relation(19, 18), 'after')
|
||||
self.assertEqual(temporal_relation(-1, 18), 'unobserved_source')
|
||||
self.assertEqual(temporal_relation(18, -1), 'unobserved_target')
|
||||
self.assertEqual(temporal_relation(-1, -1), 'both_unobserved')
|
||||
|
||||
def test_min_observed(self):
|
||||
self.assertEqual(min_observed([-1, 0, 3]), 0)
|
||||
self.assertEqual(min_observed([-1, -1]), -1)
|
||||
|
||||
def test_empty_event_sink_has_schema(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / 'events.parquet'
|
||||
sink = EventSink(path)
|
||||
sink.close()
|
||||
self.assertEqual(pq.read_table(path).num_rows, 0)
|
||||
self.assertIn('source_spike_tick', pq.read_schema(path).names)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Synthetic CPU-only tests, including the untouched sibling AH callback."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from pn_observer import PNObserver
|
||||
|
||||
|
||||
path = Path(__file__).resolve().parent.parent / "kc-drive-audit-v1/replay_core.py"
|
||||
spec = importlib.util.spec_from_file_location("frozen_ah_observer_test_core", path)
|
||||
core = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(core)
|
||||
|
||||
|
||||
def event(rows=()):
|
||||
values = np.array(rows, np.int32).reshape(-1, 2)
|
||||
return {"step": values[:, 0], "neuron_idx": values[:, 1]}
|
||||
|
||||
|
||||
def run(*, edges=(), events=(), requests=(), n=1, design=(), steps=25, coeff=None,
|
||||
initial_v=None, initial_g=None, sink=None, cap=1000, h=None, hook=None):
|
||||
targets = np.arange(100, 100 + n, dtype=np.int32)
|
||||
constants = dict(gd=1., c0=0., cg=1., cv=1., threshold=1., reset=0., seed_volt=2.)
|
||||
if coeff: constants.update(coeff)
|
||||
arrays = np.asarray(edges).reshape(-1, 4)
|
||||
incoming = core.IncomingIndex(arrays[:, 0].astype(np.int32), arrays[:, 1].astype(np.int32),
|
||||
np.arange(len(arrays), dtype=np.int64) + 81, arrays[:, 2].astype(np.float64), n)
|
||||
groups = arrays[:, 3].astype(np.int16)
|
||||
observer = PNObserver(targets, np.array(design, np.int32), incoming, groups, ("ALPN", "other"), constants,
|
||||
steps, max_first_event_rows=cap, event_sink=sink)
|
||||
def callback(tick, state):
|
||||
if hook: hook(tick, state, observer)
|
||||
observer(tick, state)
|
||||
result = core.replay(np.zeros(n, np.float64) if initial_v is None else np.array(initial_v, np.float64),
|
||||
np.zeros(n, np.float64) if initial_g is None else np.array(initial_g, np.float64), constants, steps, targets,
|
||||
event(requests), event(events), incoming, np.zeros((steps, n), np.float32) if h is None else h, callback)
|
||||
data, report = observer.finish(result)
|
||||
return observer, result, data, report
|
||||
|
||||
|
||||
class PNObserverTests(unittest.TestCase):
|
||||
def test_18_step_arrival_then_19_step_first_spike(self):
|
||||
observer, result, data, report = run(edges=[(10, 0, 2., 0)], events=[(0, 10)])
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [19])
|
||||
self.assertEqual(len(observer.first_arrival_records), 1)
|
||||
row = observer.first_arrival_records[0]
|
||||
self.assertEqual((row["source_spike_tick"], row["arrival_tick"]), (0, 18))
|
||||
self.assertEqual(row["v_prethreshold_SI"], 0.)
|
||||
self.assertEqual(row["threshold_margin_SI"], 1.)
|
||||
self.assertEqual(data["first_integrated_V_from_ALPN/positive_weight"].tolist(), [2.])
|
||||
self.assertEqual(observer.edge_summary()["target_first_spike_tick"].tolist(), [19])
|
||||
self.assertTrue(report["pass_all"])
|
||||
|
||||
def test_same_tick_first_spike_arrival_is_rejected_not_antecedent(self):
|
||||
observer, _, data, _ = run(edges=[(10, 0, 2., 0)], events=[(0, 10)], requests=[(18, 100)], design=[100])
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [18])
|
||||
self.assertEqual(observer.first_arrival_records, [])
|
||||
self.assertEqual(data["ALPN_rejected_positive_edges"].tolist(), [1])
|
||||
self.assertEqual(data["ALPN_accepted_positive_before_first_edges"].tolist(), [0])
|
||||
self.assertEqual(data["first_prethreshold_V_from_request_or_reset"].tolist(), [2.])
|
||||
|
||||
def test_all_positive_and_inhibitory_events_are_retained(self):
|
||||
observer, _, data, _ = run(edges=[(10, 0, 2., 0), (11, 0, -.5, 1), (12, 0, 0., 1)],
|
||||
events=[(0, 10), (0, 11), (0, 12)])
|
||||
self.assertEqual([r["weight_sign"] for r in observer.first_arrival_records], [1, -1])
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [19])
|
||||
self.assertEqual(data["other_accepted_zero_edges"].tolist(), [1])
|
||||
self.assertEqual(data["other_accepted_negative_before_first_abs_V"].tolist(), [.5])
|
||||
self.assertEqual(data["first_integrated_V_from_other/negative_weight"].tolist(), [-.5])
|
||||
|
||||
def test_never_active_targets_retain_all_prefirst_records(self):
|
||||
observer, _, data, report = run(edges=[(10, 0, .25, 0)], events=[(0, 10), (1, 10)],
|
||||
coeff={"cg": 0.})
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [-1])
|
||||
self.assertTrue(np.isnan(data["first_v_threshold_SI"][0]))
|
||||
self.assertEqual(observer.edge_summary()["accepted_event_count"].tolist(), [2])
|
||||
self.assertEqual(observer.edge_summary()["target_first_spike_tick"].tolist(), [-1])
|
||||
self.assertEqual(report["first_arrival_record_count"], 2)
|
||||
|
||||
def test_multiple_targets_independent_first_times_and_design(self):
|
||||
observer, _, data, _ = run(n=2, edges=[(10, 0, 2., 0), (10, 1, .25, 0)],
|
||||
events=[(0, 10)], requests=[(23, 101)], design=[101])
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [19, 23])
|
||||
self.assertEqual([r["target_is_design"] for r in observer.first_arrival_records], [False, True])
|
||||
self.assertEqual(data["first_spike_coincident_with_accepted_request"].tolist(), [False, True])
|
||||
|
||||
def test_old_g_not_decayed_g_used_for_first_voltage(self):
|
||||
_, _, data, _ = run(edges=[(10, 0, 4., 0)], events=[(0, 10)],
|
||||
coeff={"gd": .1, "cg": .5, "cv": 0.})
|
||||
self.assertEqual(data["first_integrated_V_from_ALPN/positive_weight"].tolist(), [2.])
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [19])
|
||||
|
||||
def test_initial_state_is_not_assigned_to_source_class(self):
|
||||
_, _, data, _ = run(initial_g=[2.])
|
||||
self.assertEqual(data["first_spike_tick"].tolist(), [0])
|
||||
self.assertEqual(data["first_integrated_V_from_initial_and_affine_baseline"].tolist(), [2.])
|
||||
self.assertEqual(data["first_integrated_V_from_ALPN/positive_weight"].tolist(), [0.])
|
||||
|
||||
def test_streaming_sink_and_row_budget_fail_closed(self):
|
||||
received = []
|
||||
observer, _, _, report = run(edges=[(10, 0, .2, 0)], events=[(0, 10)],
|
||||
sink=lambda rows: received.extend(rows))
|
||||
self.assertEqual(observer.first_arrival_records, [])
|
||||
self.assertEqual(len(received), report["first_arrival_record_count"])
|
||||
with self.assertRaisesRegex(ValueError, "resource limit"):
|
||||
run(edges=[(10, 0, .2, 0), (11, 0, .2, 1)], events=[(0, 10), (0, 11)], cap=1)
|
||||
|
||||
def test_nonzero_PN_APL_and_non_design_request_are_rejected(self):
|
||||
h = np.zeros((25, 1), np.float32); h[0, 0] = 1
|
||||
with self.assertRaisesRegex(ValueError, "zero graded APL"):
|
||||
run(h=h)
|
||||
with self.assertRaisesRegex(ValueError, "outside frozen odor design"):
|
||||
run(requests=[(0, 100)])
|
||||
|
||||
def test_callback_arrays_not_modified(self):
|
||||
snapshots = []
|
||||
def hook(tick, state, observer):
|
||||
if tick == 0:
|
||||
before = {key: value.copy() for key, value in state.items() if isinstance(value, np.ndarray)}
|
||||
observer(tick, state)
|
||||
for key, value in before.items():
|
||||
self.assertEqual(state[key].tobytes(), value.tobytes())
|
||||
snapshots.append(True)
|
||||
# Already consumed this tick; stop this synthetic test directly.
|
||||
raise RuntimeError("checked immutability")
|
||||
with self.assertRaisesRegex(RuntimeError, "checked immutability"):
|
||||
run(hook=hook)
|
||||
self.assertEqual(snapshots, [True])
|
||||
|
||||
def test_bad_queue_guard_and_tick_fail_closed(self):
|
||||
def wrong_guard(tick, state, observer):
|
||||
if tick == 18: state["arrival_accepted"][:] = False
|
||||
with self.assertRaisesRegex(ValueError, "Arrival guard"):
|
||||
run(edges=[(10, 0, 2., 0)], events=[(0, 10)], hook=wrong_guard)
|
||||
def wrong_tick(tick, state, observer):
|
||||
observer(tick + 1, state)
|
||||
with self.assertRaisesRegex(ValueError, "out-of-order"):
|
||||
run(hook=wrong_tick)
|
||||
|
||||
def test_sequential_arrival_roundoff_has_own_lane(self):
|
||||
_, _, data, report = run(edges=[(10, 0, 1e16, 0), (11, 0, 1., 0), (12, 0, -1e16, 1)],
|
||||
events=[(0, 10), (0, 11), (0, 12)], coeff={"cg": 0.})
|
||||
if np.finfo(np.longdouble).nmant > np.finfo(np.float64).nmant:
|
||||
self.assertEqual(data["arrival_roundoff_g_SI"].tolist(), [-1.])
|
||||
self.assertTrue(report["voltage_decomposition"]["roundoff_lane_explicit"])
|
||||
self.assertFalse(report["unique_cause_claim"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Only temporary synthetic seals are used; production publication is never run."""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import publication
|
||||
|
||||
|
||||
def _sha(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _write(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value), encoding='utf-8')
|
||||
|
||||
|
||||
def _seal(root):
|
||||
rows = [dict(path=path.relative_to(root).as_posix(), bytes=path.stat().st_size, sha256=_sha(path))
|
||||
for path in sorted(root.rglob('*')) if path.is_file() and
|
||||
path.name not in ('manifest.json', 'checksums.sha256') and '__pycache__' not in path.parts]
|
||||
_write(root / 'manifest.json', dict(root=str(root), files=rows))
|
||||
lines = [row['sha256'] + ' ' + row['path'] for row in rows]
|
||||
lines.append(_sha(root / 'manifest.json') + ' manifest.json')
|
||||
(root / 'checksums.sha256').write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
class PublicationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory(prefix='phase4ai-publication-unit-')
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = Path(self.temporary.name).resolve()
|
||||
self.model = self.root / publication.MODEL_RELATIVE
|
||||
self.experiment = self.root / publication.EXPERIMENT_RELATIVE
|
||||
self.registry = self.root / publication.REGISTRY_RELATIVE
|
||||
self.model.mkdir(parents=True)
|
||||
self.experiment.mkdir(parents=True)
|
||||
(self.model / 'module.py').write_text('# synthetic\n', encoding='utf-8')
|
||||
status = dict(status=publication.FINAL_STATUS, pass_all=True, candidate_generated=False,
|
||||
learning_achieved=False, new_brain_simulation=False, GPU_used=False)
|
||||
_write(self.experiment / 'audit/final-status.json', status)
|
||||
_write(self.experiment / 'trials/example/summary.json', dict(synthetic=True))
|
||||
_write(self.experiment / 'trials/example/apl-summary.json', dict(synthetic=True))
|
||||
(self.experiment / 'REPORT.md').write_text('Synthetic evidence.\n', encoding='utf-8')
|
||||
(self.experiment / 'unit-tests.log').write_text('Synthetic tests passed.\n', encoding='utf-8')
|
||||
(self.experiment / 'counts.npy').write_bytes(b'not-a-real-array')
|
||||
_seal(self.model)
|
||||
_seal(self.experiment)
|
||||
|
||||
def test_all_payload_and_checksum_self_hash_are_verified(self):
|
||||
result = publication.verify_seal(self.experiment)
|
||||
self.assertTrue(result['pass_all'])
|
||||
self.assertEqual(result['payload_file_count'], 6)
|
||||
self.assertEqual(result['manifest_sha256'], _sha(self.experiment / 'manifest.json'))
|
||||
|
||||
def test_publication_only_copies_small_evidence_and_leaves_sources_unchanged(self):
|
||||
before = {str(path): _sha(path) for base in (self.model, self.experiment)
|
||||
for path in base.rglob('*') if path.is_file()}
|
||||
outcome = publication.publish(self.root)
|
||||
self.assertTrue(outcome['publication_complete'])
|
||||
self.assertFalse(outcome['scientific_gate_executed_by_publisher'])
|
||||
self.assertTrue((self.registry / 'evidence/trials/example/apl-summary.json').is_file())
|
||||
self.assertTrue((self.registry / 'evidence/unit-tests.log').is_file())
|
||||
self.assertFalse((self.registry / 'evidence/counts.npy').exists())
|
||||
self.assertFalse((self.registry / 'model/module.py').exists())
|
||||
self.assertTrue((self.registry / 'model/manifest.json').is_file())
|
||||
receipt = json.loads((self.registry / 'publication-receipt.json').read_text())
|
||||
self.assertFalse(receipt['complete_bulk_backup'])
|
||||
self.assertFalse(receipt['scientific_gate_certified_by_publication'])
|
||||
for row in receipt['files']:
|
||||
path = self.registry / row['path']
|
||||
self.assertEqual(path.stat().st_size, row['bytes'])
|
||||
self.assertEqual(_sha(path), row['sha256'])
|
||||
for name, checksum in before.items():
|
||||
self.assertEqual(_sha(Path(name)), checksum)
|
||||
|
||||
def test_existing_registry_is_never_overwritten_or_resumed(self):
|
||||
self.registry.mkdir(parents=True)
|
||||
sentinel = self.registry / 'keep.txt'
|
||||
sentinel.write_text('keep', encoding='utf-8')
|
||||
with self.assertRaisesRegex(ValueError, 'Registry already exists'):
|
||||
publication.publish(self.root)
|
||||
self.assertEqual(sentinel.read_text(), 'keep')
|
||||
self.assertEqual(list(self.registry.iterdir()), [sentinel])
|
||||
|
||||
def test_bulk_payload_corruption_blocks_before_registry_creation(self):
|
||||
(self.experiment / 'counts.npy').write_bytes(b'corrupt')
|
||||
with self.assertRaisesRegex(ValueError, 'Payload byte count mismatch'):
|
||||
publication.publish(self.root)
|
||||
self.assertFalse(self.registry.exists())
|
||||
|
||||
def test_same_size_payload_hash_corruption_is_detected(self):
|
||||
path = self.experiment / 'counts.npy'
|
||||
path.write_bytes(b'x' * path.stat().st_size)
|
||||
with self.assertRaisesRegex(ValueError, 'Payload SHA-256 mismatch'):
|
||||
publication.verify_seal(self.experiment)
|
||||
|
||||
def test_missing_payload_or_unlisted_file_is_detected(self):
|
||||
(self.experiment / 'extra.txt').write_text('unlisted', encoding='utf-8')
|
||||
with self.assertRaisesRegex(ValueError, 'unlisted/missing'):
|
||||
publication.verify_seal(self.experiment)
|
||||
|
||||
def test_wrong_manifest_self_checksum_is_detected(self):
|
||||
path = self.experiment / 'checksums.sha256'
|
||||
lines = path.read_text().splitlines()
|
||||
lines[-1] = '0' * 64 + ' manifest.json'
|
||||
path.write_text('\n'.join(lines) + '\n')
|
||||
with self.assertRaisesRegex(ValueError, 'manifest self-checksum'):
|
||||
publication.verify_seal(self.experiment)
|
||||
|
||||
def test_duplicate_checksum_path_is_detected(self):
|
||||
path = self.experiment / 'checksums.sha256'
|
||||
text = path.read_text()
|
||||
path.write_text(text + text.splitlines()[0] + '\n')
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate checksum'):
|
||||
publication.verify_seal(self.experiment)
|
||||
|
||||
def test_manifest_root_mismatch_is_detected(self):
|
||||
path = self.experiment / 'manifest.json'
|
||||
value = json.loads(path.read_text())
|
||||
value['root'] = str(self.model)
|
||||
_write(path, value)
|
||||
with self.assertRaisesRegex(ValueError, 'another root'):
|
||||
publication.verify_seal(self.experiment)
|
||||
|
||||
def test_manifest_duplicate_path_and_traversal_are_rejected(self):
|
||||
manifest = self.experiment / 'manifest.json'
|
||||
original = json.loads(manifest.read_text())
|
||||
for bad_path in ('../escape.txt', '/absolute', 'a/../escape', 'a\\escape', 'C:/escape'):
|
||||
with self.subTest(path=bad_path):
|
||||
value = json.loads(json.dumps(original))
|
||||
value['files'][0]['path'] = bad_path
|
||||
_write(manifest, value)
|
||||
with self.assertRaises(ValueError):
|
||||
publication.verify_seal(self.experiment)
|
||||
original['files'].append(original['files'][0].copy())
|
||||
_write(manifest, original)
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate manifest path'):
|
||||
publication.verify_seal(self.experiment)
|
||||
|
||||
def test_explicitly_excluded_pycache_is_not_published(self):
|
||||
cache = self.experiment / '__pycache__/unused.pyc'
|
||||
cache.parent.mkdir()
|
||||
cache.write_bytes(b'cache')
|
||||
self.assertEqual(publication.verify_seal(self.experiment)['excluded_pycache_files'], 1)
|
||||
publication.publish(self.root)
|
||||
self.assertFalse((self.registry / 'evidence/__pycache__').exists())
|
||||
|
||||
def test_file_size_limit_skips_large_json_without_omitting_verification(self):
|
||||
huge = self.experiment / 'huge.json'
|
||||
huge.write_bytes(b' ' * (publication.MAX_EVIDENCE_BYTES + 1))
|
||||
_seal(self.experiment)
|
||||
publication.publish(self.root)
|
||||
self.assertFalse((self.registry / 'evidence/huge.json').exists())
|
||||
receipt = json.loads((self.registry / 'publication-receipt.json').read_text())
|
||||
omitted = {row['path']: row for row in receipt['experiment_payload_not_copied']}
|
||||
self.assertEqual(omitted['huge.json']['reason'], 'exceeds_4_MiB')
|
||||
|
||||
def test_unexpected_final_status_or_scope_flag_blocks_publication(self):
|
||||
path = self.experiment / 'audit/final-status.json'
|
||||
original = json.loads(path.read_text())
|
||||
for field, value in [('status', 'OTHER-PASS'), ('pass_all', False), ('learning_achieved', True),
|
||||
('GPU_used', True), ('new_brain_simulation', True), ('candidate_generated', True)]:
|
||||
with self.subTest(field=field):
|
||||
bad = dict(original)
|
||||
bad[field] = value
|
||||
_write(path, bad)
|
||||
_seal(self.experiment)
|
||||
with self.assertRaises(ValueError):
|
||||
publication.publish(self.root)
|
||||
self.assertFalse(self.registry.exists())
|
||||
|
||||
def test_copy_failure_is_retained_without_success_receipt(self):
|
||||
with mock.patch.object(publication, '_copy_verified', side_effect=OSError('synthetic interruption')):
|
||||
with self.assertRaisesRegex(OSError, 'synthetic interruption'):
|
||||
publication.publish(self.root)
|
||||
self.assertTrue(self.registry.is_dir())
|
||||
self.assertTrue((self.registry / 'publication-failure.json').is_file())
|
||||
self.assertFalse((self.registry / 'publication-receipt.json').exists())
|
||||
with self.assertRaisesRegex(ValueError, 'Registry already exists'):
|
||||
publication.publish(self.root)
|
||||
|
||||
def test_source_change_during_copy_prevents_success(self):
|
||||
original = publication._copy_verified
|
||||
changed = False
|
||||
|
||||
def copy_then_modify(source, target, byte_count, checksum):
|
||||
nonlocal changed
|
||||
original(source, target, byte_count, checksum)
|
||||
if not changed:
|
||||
changed = True
|
||||
(self.experiment / 'counts.npy').write_bytes(b'deliberately-changed-after-initial-verification')
|
||||
|
||||
with mock.patch.object(publication, '_copy_verified', side_effect=copy_then_modify):
|
||||
with self.assertRaisesRegex(ValueError, 'Payload byte count mismatch'):
|
||||
publication.publish(self.root)
|
||||
self.assertFalse((self.registry / 'publication-receipt.json').exists())
|
||||
self.assertTrue((self.registry / 'publication-failure.json').exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Synthetic tests only; no saved scientific results are accessed."""
|
||||
import copy
|
||||
import json
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from structure_metrics import aggregate_structure, MODES, SEEDS, METRICS
|
||||
|
||||
|
||||
def records():
|
||||
rows = []
|
||||
for mode in MODES:
|
||||
for odor in range(6):
|
||||
design = np.zeros(686, bool)
|
||||
design[odor] = True
|
||||
for seed in SEEDS:
|
||||
count = np.zeros(686, np.int32)
|
||||
count[odor] = 10
|
||||
count[10] = seed - SEEDS[0]
|
||||
count[11] = odor
|
||||
rows.append(dict(mode=mode, odor_index=odor, seed=seed,
|
||||
counts=count, design_mask=design.copy()))
|
||||
return rows
|
||||
|
||||
|
||||
class StructureMetricsTests(unittest.TestCase):
|
||||
def test_fixed_complete_inventory_and_pair_specific_baseline(self):
|
||||
result = aggregate_structure(records())
|
||||
self.assertEqual(result['record_count'], 60)
|
||||
self.assertEqual(result['design']['design_union_count'], 6)
|
||||
self.assertEqual(result['design']['common_non_design_count'], 680)
|
||||
self.assertEqual(result['design']['common_non_design_column_positions'], list(range(6, 686)))
|
||||
for mode in MODES:
|
||||
self.assertEqual(result['modes'][mode]['trials'], 30)
|
||||
for view_name in ('fixed_686', 'common_non_design'):
|
||||
view = result['modes'][mode][view_name]
|
||||
self.assertEqual(len(view['within_comparisons']), 60)
|
||||
self.assertEqual(len(view['between_comparisons']), 75)
|
||||
self.assertEqual(len(view['pairs']), 15)
|
||||
for pair in view['pairs']:
|
||||
self.assertEqual(len(pair['within_comparisons']), 20)
|
||||
self.assertEqual(len(pair['between_comparisons']), 5)
|
||||
self.assertEqual({row['odor_index'] for row in pair['within_comparisons']},
|
||||
{pair['odor_index_a'], pair['odor_index_b']})
|
||||
self.assertEqual([row['seed'] for row in pair['between_comparisons']], list(SEEDS))
|
||||
json.dumps(result, allow_nan=False)
|
||||
|
||||
def test_primary_retains_design_columns_and_common_view_is_fixed(self):
|
||||
result = aggregate_structure(records())['modes'][MODES[0]]
|
||||
pair = result['fixed_686']['pairs'][0]
|
||||
common_pair = result['common_non_design']['pairs'][0]
|
||||
# Odors zero and one differ by two ten-count design axes plus one
|
||||
# shared-coordinate count; same-seed variability cancels between.
|
||||
self.assertEqual(pair['between']['euclidean_distance']['mean'], np.sqrt(201.))
|
||||
self.assertEqual(common_pair['between']['euclidean_distance']['mean'], 1.)
|
||||
self.assertEqual(pair['within']['euclidean_distance']['mean'], 2.)
|
||||
self.assertEqual(common_pair['within']['euclidean_distance']['mean'], 2.)
|
||||
|
||||
def test_input_order_does_not_change_result(self):
|
||||
rows = records()
|
||||
expected = aggregate_structure(rows)
|
||||
actual = aggregate_structure(list(reversed(rows)))
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_inputs_unchanged_and_readonly_arrays_accepted(self):
|
||||
rows = records()
|
||||
before = copy.deepcopy(rows)
|
||||
for row in rows:
|
||||
row['counts'].setflags(write=False)
|
||||
row['design_mask'].setflags(write=False)
|
||||
aggregate_structure(rows)
|
||||
for old, new in zip(before, rows):
|
||||
np.testing.assert_array_equal(old['counts'], new['counts'])
|
||||
np.testing.assert_array_equal(old['design_mask'], new['design_mask'])
|
||||
self.assertFalse(new['counts'].flags.writeable)
|
||||
|
||||
def test_zero_vectors_do_not_become_similarity_evidence(self):
|
||||
rows = records()
|
||||
for row in rows:
|
||||
row['counts'].fill(0)
|
||||
result = aggregate_structure(rows)
|
||||
for mode in MODES:
|
||||
for name in ('fixed_686', 'common_non_design'):
|
||||
view = result['modes'][mode][name]
|
||||
for metric in ('cosine_distance', 'pearson_correlation', 'jaccard'):
|
||||
self.assertIsNone(view['within'][metric]['mean'])
|
||||
self.assertEqual(view['within'][metric]['valid_comparisons'], 0)
|
||||
self.assertEqual(view['between'][metric]['undefined_comparisons'], 75)
|
||||
self.assertEqual(view['between']['euclidean_distance']['mean'], 0.)
|
||||
json.dumps(result, allow_nan=False)
|
||||
|
||||
def test_constant_nonzero_vectors_have_undefined_pearson(self):
|
||||
rows = records()
|
||||
for row in rows:
|
||||
row['counts'].fill(1)
|
||||
view = aggregate_structure(rows)['modes'][MODES[0]]['fixed_686']
|
||||
self.assertIsNone(view['within']['pearson_correlation']['mean'])
|
||||
self.assertEqual(view['within']['jaccard']['mean'], 1.)
|
||||
self.assertAlmostEqual(view['within']['cosine_distance']['mean'], 0., places=14)
|
||||
|
||||
def test_empty_common_population_is_nonapplicable(self):
|
||||
rows = records()
|
||||
for row in rows:
|
||||
row['design_mask'].fill(True)
|
||||
result = aggregate_structure(rows)
|
||||
self.assertEqual(result['design']['common_non_design_count'], 0)
|
||||
for mode in MODES:
|
||||
view = result['modes'][mode]['common_non_design']
|
||||
self.assertFalse(view['applicable'])
|
||||
self.assertEqual(view['dimensions'], 0)
|
||||
for metric in METRICS:
|
||||
self.assertIsNone(view['between'][metric]['mean'])
|
||||
self.assertEqual(view['within'][metric]['undefined_comparisons'], 60)
|
||||
|
||||
def test_int32_extrema_are_promoted_before_subtraction_and_norm(self):
|
||||
rows = records()
|
||||
maximum = np.iinfo(np.int32).max
|
||||
for row in rows:
|
||||
row['counts'].fill(0)
|
||||
row['counts'][row['odor_index']] = maximum
|
||||
result = aggregate_structure(rows)['modes'][MODES[0]]['fixed_686']
|
||||
distance = result['pairs'][0]['between']['euclidean_distance']['mean']
|
||||
self.assertAlmostEqual(distance / float(maximum), np.sqrt(2.))
|
||||
self.assertEqual(result['between']['cosine_distance']['mean'], 1.)
|
||||
json.dumps(result, allow_nan=False)
|
||||
|
||||
def test_missing_record_or_extra_repeat_rejected(self):
|
||||
rows = records()
|
||||
for bad in (rows[:-1], rows + [copy.deepcopy(rows[0])]):
|
||||
with self.assertRaisesRegex(ValueError, 'Exactly 60'):
|
||||
aggregate_structure(bad)
|
||||
|
||||
def test_duplicate_identity_rejected_even_with_sixty_rows(self):
|
||||
rows = records()
|
||||
rows[-1] = copy.deepcopy(rows[0])
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate CAL identity'):
|
||||
aggregate_structure(rows)
|
||||
|
||||
def test_mask_change_across_seed_rejected(self):
|
||||
rows = records()
|
||||
rows[1]['design_mask'][100] = True
|
||||
with self.assertRaisesRegex(ValueError, 'mask changed across seed or mode'):
|
||||
aggregate_structure(rows)
|
||||
|
||||
def test_mask_change_across_mode_rejected(self):
|
||||
rows = records()
|
||||
for row in rows:
|
||||
if row['mode'] == MODES[1] and row['odor_index'] == 0:
|
||||
row['design_mask'][100] = True
|
||||
with self.assertRaisesRegex(ValueError, 'mask changed across seed or mode'):
|
||||
aggregate_structure(rows)
|
||||
|
||||
def test_negative_counts_and_incorrect_dtype_or_shape_rejected(self):
|
||||
changes = [('negative', lambda row: row['counts'].__setitem__(0, -1)),
|
||||
('counts64', lambda row: row.update(counts=row['counts'].astype(np.int64))),
|
||||
('counts_float', lambda row: row.update(counts=row['counts'].astype(np.float64))),
|
||||
('shape', lambda row: row.update(counts=np.zeros(685, np.int32))),
|
||||
('mask_dtype', lambda row: row.update(design_mask=row['design_mask'].astype(np.uint8))),
|
||||
('mask_shape', lambda row: row.update(design_mask=np.zeros(685, bool)))]
|
||||
for label, change in changes:
|
||||
with self.subTest(label=label):
|
||||
rows = records()
|
||||
change(rows[0])
|
||||
with self.assertRaises(ValueError):
|
||||
aggregate_structure(rows)
|
||||
|
||||
def test_strict_record_schema_and_id_ranges(self):
|
||||
changes = [lambda row: row.update(extra=1), lambda row: row.pop('counts'),
|
||||
lambda row: row.update(mode='new_mode'), lambda row: row.update(odor_index=6),
|
||||
lambda row: row.update(odor_index=True), lambda row: row.update(seed=51106),
|
||||
lambda row: row.update(seed=float(SEEDS[0]))]
|
||||
for change in changes:
|
||||
rows = records()
|
||||
change(rows[0])
|
||||
with self.assertRaises(ValueError):
|
||||
aggregate_structure(rows)
|
||||
|
||||
def test_partial_undefined_metrics_report_counts_without_imputation(self):
|
||||
rows = records()
|
||||
# One zero-count record per mode. Four of the 60 within comparisons
|
||||
# and five of the 75 between comparisons then have undefined cosine.
|
||||
for row in rows:
|
||||
if row['odor_index'] == 0 and row['seed'] == SEEDS[0]:
|
||||
row['counts'].fill(0)
|
||||
view = aggregate_structure(rows)['modes'][MODES[0]]['fixed_686']
|
||||
self.assertEqual(view['within']['cosine_distance']['undefined_comparisons'], 4)
|
||||
self.assertEqual(view['between']['cosine_distance']['undefined_comparisons'], 5)
|
||||
self.assertEqual(view['within']['cosine_distance']['valid_comparisons'], 56)
|
||||
self.assertEqual(view['between']['cosine_distance']['valid_comparisons'], 70)
|
||||
|
||||
def test_one_common_channel_pearson_is_undefined(self):
|
||||
rows = records()
|
||||
for row in rows:
|
||||
row['design_mask'][:685] = True
|
||||
row['counts'][685] = row['odor_index'] + 1
|
||||
view = aggregate_structure(rows)['modes'][MODES[0]]['common_non_design']
|
||||
self.assertEqual(view['dimensions'], 1)
|
||||
self.assertIsNone(view['between']['pearson_correlation']['mean'])
|
||||
self.assertEqual(view['between']['cosine_distance']['mean'], 0.)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Pure synthetic checks for stateless pre-first event validation."""
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from validate_events import COLUMN_DTYPES, N_GLOBAL, validate_batch
|
||||
|
||||
|
||||
def fixture():
|
||||
reference = dict(edge_ids=np.array([7, 20, 99], np.int64),
|
||||
srcglobals=np.array([3, 4, 3], np.int32), dstglobals=np.array([10, 10, 20], np.int32),
|
||||
weights=np.array([.001, -.002, .003], np.float64),
|
||||
target_globals=np.array([10, 20, 30], np.int32), target_first=np.array([30, -1, 0], np.int32),
|
||||
source_event_keys=np.array([3, 4, 4 * N_GLOBAL + 3], np.int64))
|
||||
# Arbitrary order, repeated edge at a later tick, negative and positive.
|
||||
values = dict(edge_id=[99, 20, 7, 99], source_neuron_idx=[3, 4, 3, 3],
|
||||
target_neuron_idx=[20, 10, 10, 20], source_spike_tick=[4, 0, 0, 0],
|
||||
arrival_tick=[22, 18, 18, 18], weight_sign=[1, -1, 1, 1],
|
||||
ref_at_arrival=[0] * 4, weight_SI=[.003, -.002, .001, .003], accepted=[True] * 4)
|
||||
columns = {name: np.array(values[name], dtype) for name, dtype in COLUMN_DTYPES.items()}
|
||||
return columns, reference
|
||||
|
||||
|
||||
class EventValidationTests(unittest.TestCase):
|
||||
def test_unordered_rows_real_edges_signed_weights_and_never_first(self):
|
||||
columns, ref = fixture()
|
||||
original = {k: v.copy() for k, v in columns.items()}
|
||||
result = validate_batch(columns, **ref)
|
||||
self.assertEqual(result['count'], 4)
|
||||
np.testing.assert_array_equal(result['per_target_count'], [2, 2, 0])
|
||||
self.assertEqual(result['per_target_count'].dtype, np.int64)
|
||||
self.assertFalse(result['unique_cause_claim'])
|
||||
self.assertFalse(result['cross_batch_completeness_checked'])
|
||||
for key in columns:
|
||||
np.testing.assert_array_equal(columns[key], original[key])
|
||||
|
||||
def test_empty_valid_but_missing_wrong_dtype_or_shape_still_rejected(self):
|
||||
columns, ref = fixture()
|
||||
empty = {name: value[:0] for name, value in columns.items()}
|
||||
result = validate_batch(empty, **ref)
|
||||
self.assertEqual(result['count'], 0)
|
||||
np.testing.assert_array_equal(result['per_target_count'], [0, 0, 0])
|
||||
bad = empty.copy()
|
||||
bad['arrival_tick'] = np.empty(0, np.int64)
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(bad, **ref)
|
||||
bad = empty.copy()
|
||||
del bad['accepted']
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(bad, **ref)
|
||||
bad = empty.copy()
|
||||
bad['weight_sign'] = np.empty((0, 1), np.int32)
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(bad, **ref)
|
||||
|
||||
def test_every_edge_field_and_weight_bit_is_checked(self):
|
||||
for field, value in [('edge_id', 8), ('source_neuron_idx', 4), ('target_neuron_idx', 10),
|
||||
('weight_sign', -1), ('weight_SI', np.nextafter(.003, np.inf))]:
|
||||
with self.subTest(field=field):
|
||||
columns, ref = fixture()
|
||||
columns[field][0] = value
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(columns, **ref)
|
||||
columns, ref = fixture()
|
||||
ref['weights'][2] = 0.
|
||||
columns['weight_SI'][[0, 3]] = -0.
|
||||
with self.assertRaisesRegex(ValueError, 'bits mismatch'):
|
||||
validate_batch(columns, **ref)
|
||||
columns['weight_SI'][[0, 3]] = 0.
|
||||
with self.assertRaisesRegex(ValueError, 'zero-weight'):
|
||||
validate_batch(columns, **ref)
|
||||
|
||||
def test_guard_strict_prefirst_and_source_delay(self):
|
||||
changes = [('accepted', False), ('ref_at_arrival', 1), ('source_spike_tick', -1),
|
||||
('source_spike_tick', 3), ('arrival_tick', 21)]
|
||||
for field, value in changes:
|
||||
with self.subTest(field=field):
|
||||
columns, ref = fixture()
|
||||
columns[field][0] = value
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(columns, **ref)
|
||||
for first in (22, 21):
|
||||
columns, ref = fixture()
|
||||
ref['target_first'][1] = first
|
||||
with self.assertRaisesRegex(ValueError, 'strictly before'):
|
||||
validate_batch(columns, **ref)
|
||||
|
||||
def test_source_event_membership_and_target_population(self):
|
||||
columns, ref = fixture()
|
||||
ref['source_event_keys'] = ref['source_event_keys'][:2]
|
||||
with self.assertRaisesRegex(ValueError, 'source spike'):
|
||||
validate_batch(columns, **ref)
|
||||
columns, ref = fixture()
|
||||
ref['target_globals'] = np.array([10, 30], np.int32)
|
||||
ref['target_first'] = np.array([30, 0], np.int32)
|
||||
with self.assertRaisesRegex(ValueError, 'target global'):
|
||||
validate_batch(columns, **ref)
|
||||
|
||||
def test_duplicate_rows_rejected_without_sort_assumption(self):
|
||||
columns, ref = fixture()
|
||||
for value in columns.values():
|
||||
value[0] = value[3]
|
||||
with self.assertRaisesRegex(ValueError, 'Duplicate edge event'):
|
||||
validate_batch(columns, **ref)
|
||||
|
||||
def test_invalid_reference_schema_order_duplicate_range_nonfinite(self):
|
||||
variants = [dict(edge_ids=np.array([7, 20, 99], np.int32)),
|
||||
dict(edge_ids=np.array([7, 99, 20], np.int64)),
|
||||
dict(edge_ids=np.array([7, 20, 20], np.int64)),
|
||||
dict(srcglobals=np.array([3, 4, N_GLOBAL], np.int32)),
|
||||
dict(weights=np.array([.001, -.002, np.inf])),
|
||||
dict(target_first=np.array([30, -2, 0], np.int32)),
|
||||
dict(source_event_keys=np.array([3, 3, 4], np.int64)),
|
||||
dict(source_event_keys=np.array([4, 3], np.int64)),
|
||||
dict(source_event_keys=np.array([-1, 3], np.int64))]
|
||||
for change in variants:
|
||||
with self.subTest(change=change):
|
||||
columns, ref = fixture()
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(columns, **(ref | change))
|
||||
|
||||
def test_optional_design_and_local_row_fields_are_not_assumed(self):
|
||||
columns, ref = fixture()
|
||||
columns['incoming_edge_row'] = np.array([1000, 2, 1, 1000], np.int64)
|
||||
columns['target_is_design'] = np.array([True, False, True, False])
|
||||
self.assertEqual(validate_batch(columns, **ref)['count'], 4)
|
||||
bad = columns.copy()
|
||||
bad['accepted'] = columns['accepted'].astype(np.uint8)
|
||||
with self.assertRaises(ValueError):
|
||||
validate_batch(bad, **ref)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Stateless independent checks for accepted pre-first anatomical event rows.
|
||||
|
||||
Pure NumPy: no files or brain execution. Passing establishes row consistency
|
||||
with the supplied graph and recorded source events, not necessity/unique cause.
|
||||
The caller owns provenance, design labels, trial bounds, cross-batch duplicate
|
||||
detection, expected totals, and completeness of the full record collection.
|
||||
"""
|
||||
from collections.abc import Mapping
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
N_GLOBAL = 166700
|
||||
DELAY_STEPS = 18
|
||||
COLUMN_DTYPES = {
|
||||
'edge_id': np.int64,
|
||||
'source_neuron_idx': np.int32,
|
||||
'target_neuron_idx': np.int32,
|
||||
'source_spike_tick': np.int32,
|
||||
'arrival_tick': np.int32,
|
||||
'weight_sign': np.int32,
|
||||
'ref_at_arrival': np.int32,
|
||||
'weight_SI': np.float64,
|
||||
'accepted': np.bool_,
|
||||
}
|
||||
|
||||
|
||||
def _require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _vector(value, dtype, name, length=None):
|
||||
_require(isinstance(value, np.ndarray), name + ' must be a NumPy array')
|
||||
_require(value.dtype == np.dtype(dtype) and value.ndim == 1, name + ' has the wrong dtype/shape')
|
||||
_require(length is None or len(value) == length, name + ' has the wrong length')
|
||||
return value
|
||||
|
||||
|
||||
def _globals(value, name, length=None, sorted_unique=False):
|
||||
value = _vector(value, np.int32, name, length)
|
||||
_require(np.all((value >= 0) & (value < N_GLOBAL)), name + ' contains an out-of-range global ID')
|
||||
if sorted_unique:
|
||||
_require(np.all(value[1:] > value[:-1]), name + ' must be sorted and unique')
|
||||
return value
|
||||
|
||||
|
||||
def _lookup(sorted_reference, wanted, name):
|
||||
positions = np.searchsorted(sorted_reference, wanted)
|
||||
_require(np.all(positions < len(sorted_reference)), name + ' is absent from the reference')
|
||||
_require(np.array_equal(sorted_reference[positions], wanted), name + ' is absent from the reference')
|
||||
return positions
|
||||
|
||||
|
||||
def validate_batch(columns, *, edge_ids, srcglobals, dstglobals, weights,
|
||||
target_globals, target_first, source_event_keys):
|
||||
"""Validate arbitrary-order rows; return count and target-indexed int64 counts.
|
||||
|
||||
Required columns and exact dtypes are COLUMN_DTYPES. Additional columns are
|
||||
intentionally not interpreted: in particular incoming_edge_row has a
|
||||
caller-specific coordinate system, and design/source labels are separate.
|
||||
References: sorted unique nonnegative edge_ids:int64[E]; source/destination
|
||||
globals:int32[E]; finite effective SI weights:float64[E]; sorted unique
|
||||
target_globals:int32[K]; target_first:int32[K] (-1 means never fired);
|
||||
source_event_keys:int64 sorted unique, encoding tick*166700+global ID.
|
||||
|
||||
Every row must reference a real nonzero edge, its exact weight bits/sign,
|
||||
accepted=True/ref_at_arrival=0, a recorded source event exactly 18 ticks
|
||||
earlier, and arrival strictly before the target's first spike, if any.
|
||||
Duplicate (arrival_tick,edge_id) rows within this batch are rejected without
|
||||
assuming any input row ordering. Cross-batch duplicates require caller state.
|
||||
Empty batches still require valid complete schema/references and return zero.
|
||||
No supplied array is modified or sorted in place.
|
||||
"""
|
||||
_require(isinstance(columns, Mapping), 'columns must be a mapping')
|
||||
_require(set(COLUMN_DTYPES).issubset(columns), 'Missing required event columns')
|
||||
ids = _vector(edge_ids, np.int64, 'edge_ids')
|
||||
e = len(ids)
|
||||
_require(np.all(ids >= 0) and np.all(ids[1:] > ids[:-1]), 'edge_ids must be sorted unique nonnegative IDs')
|
||||
sources = _globals(srcglobals, 'srcglobals', e)
|
||||
destinations = _globals(dstglobals, 'dstglobals', e)
|
||||
weight = _vector(weights, np.float64, 'weights', e)
|
||||
_require(np.isfinite(weight).all(), 'Nonfinite reference weight')
|
||||
targets = _globals(target_globals, 'target_globals', sorted_unique=True)
|
||||
first = _vector(target_first, np.int32, 'target_first', len(targets))
|
||||
_require(np.all(first >= -1), 'target_first must be -1 or a nonnegative tick')
|
||||
keys = _vector(source_event_keys, np.int64, 'source_event_keys')
|
||||
largest_key = int(np.iinfo(np.int32).max) * N_GLOBAL + N_GLOBAL - 1
|
||||
_require(np.all((keys >= 0) & (keys <= largest_key)) and np.all(keys[1:] > keys[:-1]),
|
||||
'source_event_keys must be sorted unique valid int32-tick/global encodings')
|
||||
count = len(_vector(columns['edge_id'], np.int64, 'columns.edge_id'))
|
||||
data = {name: _vector(columns[name], dtype, 'columns.' + name, count)
|
||||
for name, dtype in COLUMN_DTYPES.items()}
|
||||
_require(np.isfinite(data['weight_SI']).all(), 'Nonfinite recorded weight')
|
||||
_require(np.all(data['accepted']) and np.all(data['ref_at_arrival'] == 0),
|
||||
'Recorded arrival was not accepted at ref=0')
|
||||
row_edge = _lookup(ids, data['edge_id'], 'Recorded edge ID')
|
||||
_require(np.array_equal(data['source_neuron_idx'], sources[row_edge]), 'Recorded edge source mismatch')
|
||||
_require(np.array_equal(data['target_neuron_idx'], destinations[row_edge]), 'Recorded edge target mismatch')
|
||||
actual_weight = weight[row_edge]
|
||||
_require(np.array_equal(np.ascontiguousarray(data['weight_SI']).view(np.uint64),
|
||||
np.ascontiguousarray(actual_weight).view(np.uint64)), 'Recorded weight bits mismatch')
|
||||
_require(np.all(actual_weight != 0), 'Pre-first nonzero-event table contains a zero-weight edge')
|
||||
_require(np.array_equal(data['weight_sign'], np.sign(actual_weight).astype(np.int32)), 'Recorded weight sign mismatch')
|
||||
local_targets = _lookup(targets, data['target_neuron_idx'], 'Recorded target global')
|
||||
arrival = data['arrival_tick'].astype(np.int64)
|
||||
source_tick = data['source_spike_tick'].astype(np.int64)
|
||||
_require(np.all(source_tick >= 0) and np.array_equal(source_tick, arrival - DELAY_STEPS),
|
||||
'Source/arrival delay is not exactly 18 nonnegative ticks')
|
||||
wanted_keys = source_tick * np.int64(N_GLOBAL) + data['source_neuron_idx'].astype(np.int64)
|
||||
_lookup(keys, wanted_keys, 'Recorded source spike')
|
||||
target_first_rows = first[local_targets].astype(np.int64)
|
||||
_require(np.all((target_first_rows == -1) | (arrival < target_first_rows)),
|
||||
'Arrival is not strictly before the observed first target spike')
|
||||
if count > 1:
|
||||
order = np.lexsort((data['edge_id'], arrival))
|
||||
sorted_arrival, sorted_edge = arrival[order], data['edge_id'][order]
|
||||
_require(not np.any((sorted_arrival[1:] == sorted_arrival[:-1]) &
|
||||
(sorted_edge[1:] == sorted_edge[:-1])), 'Duplicate edge event within batch')
|
||||
per_target = np.bincount(local_targets, minlength=len(targets)).astype(np.int64, copy=False)
|
||||
_require(int(per_target.sum()) == count, 'Target counts do not cover the batch')
|
||||
return dict(pass_all=True, count=count, per_target_count=per_target,
|
||||
unique_cause_claim=False, cross_batch_completeness_checked=False)
|
||||
Reference in New Issue
Block a user