Digital Drosophila Archive
514 lines
33 KiB
Python
514 lines
33 KiB
Python
"""Conditional audit of the earliest non-design ALPN cohort in 60 sealed CALs.
|
|
|
|
CPU only. No new network trajectory, learning, parameter change, heldout input,
|
|
or shortest-path attribution. Full ALLN inputs and compact outgoing witnesses
|
|
are retained separately so a potentially large temporal join is not expanded.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
import traceback
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
N = 166700
|
|
E = 25582938
|
|
DELAY = 18
|
|
BUDGET = 2 * 1024**3
|
|
MODES = ('physiological', 'matched_energy')
|
|
SEEDS = tuple(range(51101, 51106))
|
|
|
|
|
|
def require(value, message):
|
|
if not value:
|
|
raise ValueError(message)
|
|
|
|
|
|
def bit_equal(a, b):
|
|
return a.dtype == b.dtype and a.shape == b.shape and a.tobytes() == b.tobytes()
|
|
|
|
|
|
def first_ticks(ticks, nodes, targets, n=N):
|
|
first = np.full(n, np.iinfo(np.int32).max, np.int32)
|
|
np.minimum.at(first, nodes, ticks)
|
|
first[first == np.iinfo(np.int32).max] = -1
|
|
return first[targets]
|
|
|
|
|
|
def select_cohort(pn, first_pn, design, first_kc, steps):
|
|
"""Static-design complement, ALL ties, never choose by antecedent outcome."""
|
|
require(pn.dtype == np.int32 and first_pn.dtype == np.int32 and pn.shape == first_pn.shape,
|
|
'PN identity/first-tick schema mismatch')
|
|
require(np.all(np.diff(pn) > 0) and np.isin(design, pn).all(), 'Invalid PN/design IDs')
|
|
require(np.all((first_pn >= -1) & (first_pn < steps)), 'PN first tick outside trial')
|
|
require(first_kc == -1 or 0 <= first_kc < steps, 'KC first tick outside trial')
|
|
non_design = ~np.isin(pn, design)
|
|
observed = first_pn[non_design & (first_pn >= 0)]
|
|
earliest = int(observed.min()) if len(observed) else -1
|
|
cohort = pn[non_design & (first_pn == earliest)] if earliest >= 0 else np.empty(0, np.int32)
|
|
cutoff = int(first_kc) if first_kc >= 0 else int(steps)
|
|
return dict(cohort=cohort, earliest_tick=earliest, pre_kc_stop_exclusive=cutoff,
|
|
pre_kc_window_censored=first_kc < 0,
|
|
matched_stop_exclusive=min(cutoff, earliest) if earliest >= 0 else cutoff,
|
|
cohort_before_first_KC=bool(earliest >= 0 and first_kc >= 0 and earliest < first_kc),
|
|
cohort_relation_to_first_KC=('no_non_design_spike' if earliest < 0 else
|
|
'KC_unobserved' if first_kc < 0 else 'before' if earliest < first_kc else
|
|
'same_tick' if earliest == first_kc else 'after'))
|
|
|
|
|
|
def predecessor_counts(incoming_ticks, incoming_weights, source_tick, previous_source_tick):
|
|
"""Strict temporal antecedents. Old-reset history is reported separately."""
|
|
ticks, weights = np.asarray(incoming_ticks), np.asarray(incoming_weights)
|
|
require(ticks.ndim == weights.ndim == 1 and ticks.shape == weights.shape,
|
|
'Predecessor arrays differ')
|
|
require(np.isfinite(weights).all() and np.all(np.diff(ticks) >= 0), 'Bad predecessor sequence')
|
|
require(-1 <= previous_source_tick < source_tick, 'Bad previous/source spike order')
|
|
prior = ticks < source_tick
|
|
surviving = prior & (ticks > previous_source_tick)
|
|
return dict(all_prior_design_PN_events=int(prior.sum()),
|
|
since_last_reset_design_PN_events=int(surviving.sum()),
|
|
since_last_reset_nonzero_design_PN_events=int(np.count_nonzero(surviving & (weights != 0))),
|
|
since_last_reset_positive_events=int(np.count_nonzero(surviving & (weights > 0))),
|
|
since_last_reset_negative_events=int(np.count_nonzero(surviving & (weights < 0))),
|
|
since_last_reset_zero_events=int(np.count_nonzero(surviving & (weights == 0))),
|
|
since_last_reset_positive_weight_SI=float(np.maximum(weights[surviving], 0).sum(dtype=np.longdouble)),
|
|
since_last_reset_negative_abs_weight_SI=float(np.maximum(-weights[surviving], 0).sum(dtype=np.longdouble)),
|
|
prior_reset_excluded_events=int(np.count_nonzero(prior & ~surviving)))
|
|
|
|
|
|
def source_labels(global_ids, role, design_mask):
|
|
result = role[global_ids].astype(object).copy()
|
|
alpn = result == 'ALPN'
|
|
result[alpn & design_mask[global_ids]] = 'design_ALPN'
|
|
result[alpn & ~design_mask[global_ids]] = 'non_design_ALPN'
|
|
return result
|
|
|
|
|
|
def file_budget(root):
|
|
size = sum(p.stat().st_size for p in root.rglob('*') if p.is_file())
|
|
require(size <= BUDGET, 'Origin audit exceeds 2 GiB; no truncation/resampling allowed')
|
|
return size
|
|
|
|
|
|
class Sink:
|
|
def __init__(self, path, schema, budget_root):
|
|
import pyarrow.parquet as pq
|
|
require(not path.exists(), 'Output exists: ' + str(path))
|
|
self.writer = pq.ParquetWriter(path, schema, compression='zstd')
|
|
self.schema, self.rows, self.root = schema, 0, budget_root
|
|
|
|
def write(self, columns):
|
|
import pyarrow as pa
|
|
table = pa.Table.from_pydict(columns, schema=self.schema)
|
|
if table.num_rows:
|
|
self.writer.write_table(table)
|
|
self.rows += table.num_rows
|
|
file_budget(self.root)
|
|
|
|
def close(self):
|
|
self.writer.close()
|
|
file_budget(self.root)
|
|
|
|
|
|
def event_schema():
|
|
import pyarrow as pa
|
|
return pa.schema([
|
|
('edge_id', pa.int64()), ('source_neuron_idx', pa.int32()), ('target_neuron_idx', pa.int32()),
|
|
('source_body_id', pa.uint64()), ('target_body_id', pa.uint64()),
|
|
('source_spike_tick', pa.int32()), ('arrival_tick', pa.int32()),
|
|
('source_event_key', pa.int64()), ('weight_SI', pa.float64()), ('weight_sign', pa.int32()),
|
|
('source_group', pa.string()), ('accepted', pa.bool_()), ('ref_at_arrival', pa.int32()),
|
|
('target_previous_spike_tick', pa.int32()), ('before_first_any_KC', pa.bool_())])
|
|
|
|
|
|
class ALLNObserver:
|
|
"""Save every accepted incoming event, including frozen-zero edges."""
|
|
def __init__(self, targets, incoming, role, design_mask, body, steps, cutoff, sink):
|
|
self.targets, self.incoming = targets, incoming
|
|
self.role, self.design, self.body = role, design_mask, body
|
|
self.steps, self.cutoff, self.sink = steps, cutoff, sink
|
|
self.last = np.full(len(targets), -1, np.int32)
|
|
self.callbacks = self.due = self.accepted = self.rejected = 0
|
|
self.groups = defaultdict(lambda: dict(due=0, accepted=0, rejected=0, positive=0, negative=0, zero=0))
|
|
self.design_ticks = defaultdict(list)
|
|
self.design_weights = defaultdict(list)
|
|
|
|
def __call__(self, tick, stage):
|
|
require(tick == self.callbacks and tick < self.steps, 'ALLN callback coverage mismatch')
|
|
require(np.all(stage['h32'] == 0) and np.all(stage['delta_si'] == 0), 'ALLN received graded APL')
|
|
require(not np.any(stage['request']), 'CAL request targeted ALLN rather than PN')
|
|
require(bit_equal(stage['g'], stage['g_pre_apl']), 'Unexpected ALLN post-APL change')
|
|
due, accepted = stage['due_edge_indices'], stage['arrival_accepted']
|
|
require(np.array_equal(accepted, stage['ref_after_reset'][self.incoming.dst_local[due]] == 0),
|
|
'ALLN arrival acceptance differs from post-reset ref guard')
|
|
require(tick >= DELAY or not len(due), 'Premature ALLN event')
|
|
self.last[stage['fired']] = tick
|
|
labels = source_labels(self.incoming.src_global[due], self.role, self.design)
|
|
weight = self.incoming.weights[due]
|
|
for label in np.unique(labels):
|
|
mask = labels == label
|
|
item = self.groups[str(label)]
|
|
item['due'] += int(mask.sum())
|
|
item['accepted'] += int(np.count_nonzero(mask & accepted))
|
|
item['rejected'] += int(np.count_nonzero(mask & ~accepted))
|
|
for name, select in (('positive', weight > 0), ('negative', weight < 0), ('zero', weight == 0)):
|
|
item[name] += int(np.count_nonzero(mask & accepted & select))
|
|
selected = due[accepted]
|
|
src = self.incoming.src_global[selected]
|
|
local = self.incoming.dst_local[selected]
|
|
dst = self.targets[local]
|
|
w = self.incoming.weights[selected]
|
|
selected_labels = labels[accepted]
|
|
count = len(selected)
|
|
self.sink.write(dict(edge_id=self.incoming.edge_id[selected], source_neuron_idx=src,
|
|
target_neuron_idx=dst, source_body_id=self.body[src], target_body_id=self.body[dst],
|
|
source_spike_tick=np.full(count, tick - DELAY, np.int32),
|
|
arrival_tick=np.full(count, tick, np.int32),
|
|
source_event_key=np.int64(tick - DELAY) * N + src.astype(np.int64),
|
|
weight_SI=w, weight_sign=np.sign(w).astype(np.int32), source_group=selected_labels,
|
|
accepted=np.ones(count, bool), ref_at_arrival=stage['ref_after_reset'][local],
|
|
target_previous_spike_tick=self.last[local], before_first_any_KC=np.full(count, tick < self.cutoff, bool)))
|
|
# At most all accepted design-PN events are retained in RAM. Full event
|
|
# rows are already streamed; this compact index never expands path pairs.
|
|
for target in np.unique(dst[selected_labels == 'design_ALPN']):
|
|
mask = (dst == target) & (selected_labels == 'design_ALPN')
|
|
self.design_ticks[int(target)].extend([tick] * int(mask.sum()))
|
|
self.design_weights[int(target)].extend(w[mask].tolist())
|
|
self.callbacks += 1
|
|
self.due += len(due)
|
|
self.accepted += count
|
|
self.rejected += int(np.count_nonzero(~accepted))
|
|
|
|
def report(self):
|
|
require(self.callbacks == self.steps and self.due == self.accepted + self.rejected,
|
|
'Incomplete ALLN replay observer')
|
|
require(self.accepted == self.sink.rows, 'ALLN event sink row mismatch')
|
|
return dict(callbacks=self.callbacks, ALLN_targets=len(self.targets), due_events=self.due,
|
|
accepted_events=self.accepted, rejected_events=self.rejected,
|
|
source_groups=dict(self.groups), graded_APL_h_zero=True,
|
|
accepted_zero_weight_rows_preserved=True)
|
|
|
|
|
|
def _window_add(stats, frame, mask, labels):
|
|
for label in np.unique(labels[mask]):
|
|
chosen = mask & (labels == label)
|
|
weights = frame.weight_SI.to_numpy()[chosen]
|
|
item = stats.setdefault(str(label), dict(events=0, positive_events=0, negative_events=0,
|
|
positive_weight_SI=0.0, negative_abs_weight_SI=0.0))
|
|
item['events'] += len(weights)
|
|
item['positive_events'] += int(np.count_nonzero(weights > 0))
|
|
item['negative_events'] += int(np.count_nonzero(weights < 0))
|
|
item['positive_weight_SI'] += float(np.maximum(weights, 0).sum(dtype=np.longdouble))
|
|
item['negative_abs_weight_SI'] += float(np.maximum(-weights, 0).sum(dtype=np.longdouble))
|
|
|
|
|
|
def _read_npz(path):
|
|
with np.load(path, allow_pickle=False) as file:
|
|
return {key: file[key].copy() for key in file.files}
|
|
|
|
|
|
def main():
|
|
# Lazy import makes all selection/join helpers testable without a server.
|
|
import pyarrow as pa
|
|
import pyarrow.parquet as pq
|
|
from context import (ROOT, EXP, AG, AG_MODEL, AI, AI_MODEL, AH_MODEL, COEFF,
|
|
load_module, read, save, digest, check)
|
|
check()
|
|
require(read(AI / 'audit/final-status.json')['status'] == 'PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT',
|
|
'AI is not successfully sealed')
|
|
require(read(AI / 'audit/final-status.json')['pass_all'] is True, 'AI seal failed')
|
|
outroot = EXP / 'origin-audit'
|
|
outroot.mkdir(parents=True, exist_ok=False)
|
|
receipts = []
|
|
try:
|
|
sealed_roots = (AI, AI_MODEL, AG, AG_MODEL, AH_MODEL)
|
|
sealed_inventory = {root: {r['path']: r for r in read(root / 'manifest.json')['files']}
|
|
for root in sealed_roots}
|
|
verified_hashes = {}
|
|
def input_hash(path):
|
|
path = Path(path)
|
|
if path in verified_hashes:
|
|
return verified_hashes[path]
|
|
actual = digest(path)
|
|
for root in sealed_roots:
|
|
if path.is_relative_to(root) and path.name not in ('manifest.json', 'checksums.sha256'):
|
|
key = path.relative_to(root).as_posix()
|
|
require(key in sealed_inventory[root], 'Input absent from frozen seal: ' + str(path))
|
|
frozen = sealed_inventory[root][key]
|
|
require(actual == frozen['sha256'] and path.stat().st_size == frozen['bytes'],
|
|
'Input differs from sealed digest/bytes: ' + str(path))
|
|
break
|
|
verified_hashes[path] = actual
|
|
return actual
|
|
# Verify entry-point metadata and code before importing the frozen code.
|
|
for path in (AI / 'protocol/inventory.json', AI / 'protocol/neuron-source-groups.parquet',
|
|
AG / 'protocol/calibration-cases.json', AH_MODEL / 'replay_core.py',
|
|
AI_MODEL / 'validate_events.py', AG_MODEL / 'calibration.py'):
|
|
input_hash(path)
|
|
inventory = [row for row in read(AI / 'protocol/inventory.json')
|
|
if row['kind'] == 'calibration' and row['repeat'] == 0]
|
|
require(len(inventory) == 60 and len({r['case'] for r in inventory}) == 60,
|
|
'Expected exactly the fixed 60 distinct CAL r0 inputs')
|
|
reference_cases = read(AG / 'protocol/calibration-cases.json')
|
|
require({r['case'] for r in inventory} == {r['name'] for r in reference_cases},
|
|
'AI CAL inventory differs from AG CAL inventory')
|
|
specifications = {r['case']: read(AG / 'cases' / r['case'] / 'spec.json') for r in inventory}
|
|
cells = {(s['mode'], s['odor_id'], s['seed']) for s in specifications.values()}
|
|
odors = sorted({s['odor_id'] for s in specifications.values()})
|
|
require(len(odors) == 6 and cells == {(m, o, seed) for m in MODES for o in odors for seed in SEEDS},
|
|
'CAL identity grid incomplete or includes repeats/heldout')
|
|
meta_path = AI / 'protocol/neuron-source-groups.parquet'
|
|
meta = pd.read_parquet(meta_path)
|
|
require(len(meta) == N and np.array_equal(meta.neuron_idx.to_numpy(), np.arange(N)),
|
|
'Metadata global identity mismatch')
|
|
require(meta.audit_source_group.notna().all() and meta.body_id.dtype == np.uint64,
|
|
'Invalid frozen role/body metadata')
|
|
role, body = meta.audit_source_group.to_numpy(), meta.body_id.to_numpy()
|
|
alln = np.flatnonzero(role == 'ALLN').astype(np.int32)
|
|
pn = np.load(AG / 'apl-bundle/pn_globals.npy')
|
|
kc = np.load(AG / 'apl-bundle/kc_globals.npy')
|
|
apl = np.load(AG / 'apl-bundle/apl_globals.npy')
|
|
require(len(alln) == 420 and len(pn) == 686 and len(kc) == 4064 and len(apl) == 2,
|
|
'Frozen role population sizes differ')
|
|
require(np.array_equal(pn, np.flatnonzero(role == 'ALPN').astype(np.int32)), 'ALPN role IDs differ')
|
|
require((meta.iloc[alln].annotation_class == 'ALLN').all(), 'ALLN role is not exact annotation class')
|
|
graph = AG / 'graphs/full'
|
|
arrays = {k: np.load(graph / (k + '.npy'), mmap_mode='r')
|
|
for k in ('nodes', 'src', 'dst', 'edge_id', 'weights')}
|
|
require(bit_equal(arrays['nodes'], np.arange(N, dtype=np.int32)), 'Full graph global node order differs')
|
|
require(len(arrays['edge_id']) == E, 'Full graph edge count differs')
|
|
for start in range(0, E, 1000000):
|
|
stop = min(E, start + 1000000)
|
|
require(np.array_equal(arrays['edge_id'][start:stop], np.arange(start, stop, dtype=np.int64)),
|
|
'Full graph edge IDs not canonical identity')
|
|
core = load_module('frozen_ah_alln_replay_aj', AH_MODEL / 'replay_core.py')
|
|
validate = load_module('sealed_ai_event_validation_aj', AI_MODEL / 'validate_events.py')
|
|
cal = load_module('sealed_ag_state_validation_aj', AG_MODEL / 'calibration.py')
|
|
lookup = np.full(N, -1, np.int32)
|
|
lookup[alln] = np.arange(len(alln), dtype=np.int32)
|
|
selected = np.flatnonzero(lookup[arrays['dst']] >= 0)
|
|
src = arrays['src'][selected]
|
|
weights = arrays['weights'][selected].copy()
|
|
weights[np.isin(src, apl)] = 0.0
|
|
incoming = core.IncomingIndex(src, lookup[arrays['dst'][selected]], arrays['edge_id'][selected], weights, len(alln))
|
|
# Only incoming PN graph views are required to independently validate AI rows.
|
|
pnedges = np.flatnonzero(np.isin(arrays['dst'], pn))
|
|
pnweights = arrays['weights'][pnedges].copy()
|
|
pnweights[np.isin(arrays['src'][pnedges], apl)] = 0.0
|
|
shared_inputs = [meta_path, COEFF, AH_MODEL / 'replay_core.py', AI_MODEL / 'validate_events.py',
|
|
AG_MODEL / 'calibration.py', AI / 'manifest.json', AI / 'checksums.sha256',
|
|
AG / 'manifest.json', AG / 'checksums.sha256']
|
|
shared_inputs += [graph / (key + '.npy') for key in arrays]
|
|
shared_inputs += [AG / 'apl-bundle' / (name + '_globals.npy') for name in ('pn', 'kc', 'apl')]
|
|
save(outroot / 'shared-inputs.json', [dict(path=str(p.relative_to(ROOT)), bytes=p.stat().st_size,
|
|
sha256=input_hash(p)) for p in shared_inputs])
|
|
save(outroot / 'inventory.json', [dict(**r, mode=specifications[r['case']]['mode'],
|
|
odor_id=specifications[r['case']]['odor_id'], seed=specifications[r['case']]['seed']) for r in inventory])
|
|
for row in inventory:
|
|
name = row['case']
|
|
require(Path(name).name == name and '/' not in name and '\\' not in name, 'Invalid case path')
|
|
case = AG / 'cases' / name
|
|
source = AG / 'gpu' / (name + '-r0')
|
|
require(ROOT / row['source'] == source, 'Inventory source differs from fixed CAL r0 source')
|
|
previous = AI / 'trials' / (name + '-r0')
|
|
dest = outroot / name
|
|
dest.mkdir(exist_ok=False)
|
|
spec = specifications[name]
|
|
for path in (source / 'state.npz', source / 'summary.json', case / 'spec.json', case / 'requests.npz',
|
|
case / 'pn-drive.npz', previous / 'accepted-prefirst-edge-events.parquet',
|
|
previous / 'per-pn.parquet', previous / 'summary.json'):
|
|
input_hash(path)
|
|
require(spec['steps'] == 1000 and spec['duration_ms'] == 100.0, 'CAL duration changed')
|
|
require(not (case / 'arrays.npz').exists(), 'Unexpected CAL initial-state override')
|
|
source_summary = read(source / 'summary.json')
|
|
require(source_summary['complete'] is True and source_summary['completed_ticks'] == spec['steps'],
|
|
'Incomplete source simulation')
|
|
ai_summary = read(previous / 'summary.json')
|
|
require(ai_summary['pass_all'] is True and all(ai_summary['exact'].values()), 'AI exactness failed')
|
|
state = _read_npz(source / 'state.npz')
|
|
event_keys = cal.validate_state(state, N, spec['steps'])
|
|
requests = _read_npz(case / 'requests.npz')
|
|
drive = _read_npz(case / 'pn-drive.npz')
|
|
design = np.sort(drive['targets'])
|
|
require(design.dtype == np.int32 and len(np.unique(design)) == len(design) and
|
|
np.isin(design, pn).all(), 'Design identities invalid')
|
|
require(len(design) == spec['target_count'] and drive['rates_hz'].dtype == np.float32 and
|
|
drive['rates_hz'].shape == design.shape and np.all(drive['rates_hz'] > 0),
|
|
'Static PN design count/rate schema differs')
|
|
require(np.isin(requests['neuron_idx'], design).all(), 'Request outside static odor design')
|
|
design_mask = np.zeros(N, bool)
|
|
design_mask[design] = True
|
|
first_pn = first_ticks(state['step'], state['neuron_idx'], pn)
|
|
observed_kc = first_ticks(state['step'], state['neuron_idx'], kc)
|
|
first_kc = int(observed_kc[observed_kc >= 0].min()) if np.any(observed_kc >= 0) else -1
|
|
selection = select_cohort(pn, first_pn, design, first_kc, spec['steps'])
|
|
cohort = selection.pop('cohort')
|
|
cohort_mask = np.zeros(N, bool)
|
|
cohort_mask[cohort] = True
|
|
never_mask = np.zeros(N, bool)
|
|
never_mask[pn[first_pn < 0]] = True
|
|
saved_pn = pd.read_parquet(previous / 'per-pn.parquet')
|
|
require(np.array_equal(saved_pn.neuron_idx, pn) and np.array_equal(saved_pn.first_spike_tick, first_pn),
|
|
'Sealed AI PN first-spike identity mismatch')
|
|
cutoff = selection['pre_kc_stop_exclusive']
|
|
window_stats = {key: {} for key in ('cohort_pre_KC_and_pre_first', 'never_active_pre_KC',
|
|
'never_active_non_design_pre_KC', 'never_active_matched_cohort_window',
|
|
'never_active_non_design_matched_cohort_window')}
|
|
outgoing_chunks = []
|
|
cohort_chunks = []
|
|
rows_seen, previous_event_key = 0, -1
|
|
counts_per_pn = np.zeros(len(pn), np.int64)
|
|
for batch in pq.ParquetFile(previous / 'accepted-prefirst-edge-events.parquet').iter_batches(batch_size=65536):
|
|
frame = batch.to_pandas()
|
|
columns = {k: frame[k].to_numpy() for k in validate.COLUMN_DTYPES}
|
|
audit = validate.validate_batch(columns, edge_ids=arrays['edge_id'][pnedges],
|
|
srcglobals=arrays['src'][pnedges], dstglobals=arrays['dst'][pnedges], weights=pnweights,
|
|
target_globals=pn, target_first=first_pn, source_event_keys=event_keys)
|
|
key = frame.arrival_tick.to_numpy().astype(np.int64) * E + frame.edge_id.to_numpy()
|
|
require(not len(key) or (key[0] > previous_event_key and np.all(np.diff(key) > 0)),
|
|
'AI event order/uniqueness violated across batches')
|
|
if len(key):
|
|
previous_event_key = int(key[-1])
|
|
require(np.all(frame.arrival_tick.to_numpy() < spec['steps']), 'AI event beyond trial')
|
|
labels = source_labels(frame.source_neuron_idx.to_numpy(), role, design_mask)
|
|
require(np.array_equal(frame.source_group, role[frame.source_neuron_idx.to_numpy()]), 'AI source role differs')
|
|
require(np.array_equal(frame.target_is_design, design_mask[frame.target_neuron_idx.to_numpy()]),
|
|
'AI target design differs')
|
|
targets = frame.target_neuron_idx.to_numpy()
|
|
pre = frame.arrival_tick.to_numpy() < cutoff
|
|
matched = frame.arrival_tick.to_numpy() < selection['matched_stop_exclusive']
|
|
cohort_sel = cohort_mask[targets] & pre
|
|
masks = dict(cohort_pre_KC_and_pre_first=cohort_sel,
|
|
never_active_pre_KC=never_mask[targets] & pre,
|
|
never_active_non_design_pre_KC=never_mask[targets] & ~design_mask[targets] & pre,
|
|
never_active_matched_cohort_window=never_mask[targets] & matched,
|
|
never_active_non_design_matched_cohort_window=never_mask[targets] & ~design_mask[targets] & matched)
|
|
for key, mask in masks.items():
|
|
_window_add(window_stats[key], frame, mask, labels)
|
|
# The fixed earliest cohort normally makes these small, but the
|
|
# on-disk ceiling remains hard and no row may be sampled away.
|
|
if np.any(cohort_sel):
|
|
selected_frame = frame.loc[cohort_sel].copy()
|
|
selected_frame['audit_source_group'] = labels[cohort_sel]
|
|
cohort_chunks.append(selected_frame)
|
|
out_sel = cohort_sel & (role[frame.source_neuron_idx.to_numpy()] == 'ALLN')
|
|
if np.any(out_sel):
|
|
outgoing_chunks.append(frame.loc[out_sel].copy())
|
|
counts_per_pn += audit['per_target_count']
|
|
rows_seen += audit['count']
|
|
require(rows_seen == ai_summary['observer']['first_arrival_record_count'], 'AI antecedent rows incomplete')
|
|
require(np.array_equal(counts_per_pn, saved_pn[[g + '_accepted_nonzero_before_first_edges'
|
|
for g in ai_summary['observer']['source_group_names']]].sum(axis=1).to_numpy()),
|
|
'AI per-PN antecedent completeness differs')
|
|
saved_event_schema = pq.ParquetFile(previous / 'accepted-prefirst-edge-events.parquet').schema_arrow
|
|
cohort_table = (pd.concat(cohort_chunks, ignore_index=True) if cohort_chunks else
|
|
pa.Table.from_batches([], schema=saved_event_schema).to_pandas().assign(audit_source_group=pd.Series(dtype=str)))
|
|
cohort_table.to_parquet(dest / 'cohort-all-source-antecedents.parquet', index=False, compression='zstd')
|
|
output = (pd.concat(outgoing_chunks, ignore_index=True) if outgoing_chunks else
|
|
pa.Table.from_batches([], schema=saved_event_schema).to_pandas())
|
|
sink = Sink(dest / 'ALLN-all-accepted-incoming.parquet', event_schema(), outroot)
|
|
observer = ALLNObserver(alln, incoming, role, design_mask, body, spec['steps'], cutoff, sink)
|
|
try:
|
|
result = core.replay(np.full(len(alln), spec['model']['resting_potential_mv'], np.float64) * np.float64(.001),
|
|
np.zeros(len(alln), np.float64), read(COEFF), spec['steps'], alln, requests,
|
|
dict(step=state['step'], neuron_idx=state['neuron_idx']), incoming,
|
|
np.zeros((spec['steps'], len(alln)), np.float32), observer)
|
|
finally:
|
|
sink.close()
|
|
observed = np.isin(state['neuron_idx'], alln)
|
|
exact = {k: bit_equal(result[k], state[k][alln]) for k in ('v', 'g', 'ref', 'counts')}
|
|
exact.update({k: bit_equal(result[k], state[k][observed]) for k in ('step', 'neuron_idx')})
|
|
require(all(exact.values()), 'Conditional ALLN replay differs from full AG events/final states: ' + str(exact))
|
|
with (dest / 'ALLN-replay-final.npz').open('xb') as stream:
|
|
np.savez_compressed(stream, **result)
|
|
join_rows = []
|
|
alln_source_spikes = {int(node): state['step'][state['neuron_idx'] == node]
|
|
for node in np.unique(output.source_neuron_idx.to_numpy())}
|
|
for record in output.to_dict('records'):
|
|
target = int(record['source_neuron_idx']) # Intermediate ALLN.
|
|
tick = int(record['source_spike_tick'])
|
|
spikes = alln_source_spikes[target]
|
|
pos = int(np.searchsorted(spikes, tick))
|
|
require(pos < len(spikes) and spikes[pos] == tick, 'ALLN outgoing source event absent')
|
|
last = int(spikes[pos - 1]) if pos else -1
|
|
predecessors = predecessor_counts(np.asarray(observer.design_ticks[target], np.int32),
|
|
np.asarray(observer.design_weights[target], np.float64), tick, last)
|
|
join_rows.append(dict(**record, ALLN_previous_spike_tick=last,
|
|
ALLN_source_event_key=tick * N + target, source_body_id=int(body[target]),
|
|
target_body_id=int(body[int(record['target_neuron_idx'])]), **predecessors,
|
|
witness_status=('timed_design_PN_antecedent_since_last_reset' if
|
|
predecessors['since_last_reset_nonzero_design_PN_events'] else 'no_direct_design_PN_antecedent_since_last_reset')))
|
|
# Add declared columns even if the fixed cohort has no ALLN antecedent.
|
|
join_extra = dict(ALLN_previous_spike_tick='int32', ALLN_source_event_key='int64',
|
|
source_body_id='uint64', target_body_id='uint64', all_prior_design_PN_events='int64',
|
|
since_last_reset_design_PN_events='int64', since_last_reset_positive_events='int64',
|
|
since_last_reset_nonzero_design_PN_events='int64',
|
|
since_last_reset_negative_events='int64', since_last_reset_zero_events='int64',
|
|
since_last_reset_positive_weight_SI='float64', since_last_reset_negative_abs_weight_SI='float64',
|
|
prior_reset_excluded_events='int64', witness_status='object')
|
|
witnesses = pd.DataFrame(join_rows) if join_rows else output.copy()
|
|
for key, dtype in join_extra.items():
|
|
witnesses[key] = witnesses[key].astype(dtype) if key in witnesses else pd.Series(dtype=dtype)
|
|
witnesses.to_parquet(dest / 'ALLN-to-cohort-PN-witness-join.parquet', index=False, compression='zstd')
|
|
per_pn = meta.iloc[pn].copy()
|
|
per_pn['first_spike_tick'] = first_pn
|
|
per_pn['design_target'] = design_mask[pn]
|
|
per_pn['earliest_non_design_cohort'] = cohort_mask[pn]
|
|
per_pn['never_active'] = first_pn < 0
|
|
per_pn.to_parquet(dest / 'PN-population-and-cohort.parquet', index=False, compression='zstd')
|
|
per_alln = meta.iloc[alln].copy()
|
|
per_alln['first_spike_tick'] = result['first_spike']
|
|
per_alln['spike_count'] = result['counts']
|
|
per_alln.to_parquet(dest / 'ALLN-population.parquet', index=False, compression='zstd')
|
|
witness_rows = int((witnesses.since_last_reset_nonzero_design_PN_events > 0).sum())
|
|
inputs = [source / 'state.npz', source / 'summary.json', case / 'spec.json', case / 'requests.npz',
|
|
case / 'pn-drive.npz', previous / 'accepted-prefirst-edge-events.parquet',
|
|
previous / 'per-pn.parquet', previous / 'summary.json']
|
|
report = dict(case=name, repeat=0, mode=spec['mode'], odor_id=spec['odor_id'], seed=spec['seed'],
|
|
pass_all=True, exact=exact, ALLN_replay=observer.report(), first_any_KC_tick=first_kc,
|
|
**selection, cohort_neurons=int(len(cohort)), cohort_neuron_idx=cohort.tolist(),
|
|
cohort_body_id=[int(x) for x in body[cohort]], never_active_PN=int(never_mask.sum()),
|
|
never_active_non_design_PN=int(np.count_nonzero(never_mask & ~design_mask)),
|
|
window_source_accounting=window_stats, AI_antecedent_rows_checked=rows_seen,
|
|
cohort_ALLN_outgoing_events=len(output), outgoing_events_with_since_reset_design_antecedent=witness_rows,
|
|
outgoing_events_without_since_reset_design_antecedent=len(output) - witness_rows,
|
|
compact_join_pair_count=int(witnesses.since_last_reset_nonzero_design_PN_events.sum()),
|
|
no_direct_antecedent_is_not_unexplained_wholebrain=True,
|
|
join_contract='same trial; incoming.target_neuron_idx=outgoing.source_neuron_idx; incoming.source_group=design_ALPN; '
|
|
'incoming.weight_SI != 0; '
|
|
'outgoing.ALLN_previous_spike_tick < incoming.arrival_tick < outgoing.source_spike_tick; '
|
|
'outgoing.arrival_tick < cohort first spike and first-any-KC cutoff; both legs exactly 18 ticks; '
|
|
'all incoming rows accepted at ref=0; signs retained; zero legs not causal transmission',
|
|
accounting_scope='Accepted nominal SI weight sums, not membrane contribution, necessity or unique cause; '
|
|
'prior-reset antecedents retained only as history; since-reset nonzero weights can still cancel/round away',
|
|
window_scope='Cohort first-spike antecedents are censored at their first spike; never-active pre-KC windows are longer. '
|
|
'Use matched_cohort_window for duration-matched comparisons. If KC never fires, full trial is marked censored.',
|
|
new_brain_simulation=False, GPU_used=False, candidate_generated=False,
|
|
CAL_independent_ALLN_per_tick_history_available=False,
|
|
input_sha256={str(p.relative_to(ROOT)): input_hash(p) for p in inputs})
|
|
save(dest / 'summary.json', report)
|
|
save(dest / 'output-hashes.json', [dict(path=p.name, bytes=p.stat().st_size, sha256=digest(p))
|
|
for p in sorted(dest.iterdir()) if p.is_file()])
|
|
receipts.append(dict(case=name, repeat=0, mode=spec['mode'], odor_id=spec['odor_id'],
|
|
seed=spec['seed'], ALLN_targets=420, exact=exact, pass_all=True,
|
|
summary_sha256=digest(dest / 'summary.json')))
|
|
file_budget(outroot)
|
|
print(name, '420 ALLN conditional exact; earliest cohort', len(cohort), 'PASS', flush=True)
|
|
require(len(receipts) == 60, 'Incomplete origin audit inventory')
|
|
save(outroot / 'summary.json', dict(pass_all=True, status='PASS-CONDITIONAL-EARLIEST-AL-ORIGIN-AUDIT',
|
|
trials=receipts, fixed_CAL_r0_count=60, ALLN_per_trial=420, new_brain_simulation=False,
|
|
GPU_used=False, no_cartesian_witness_expansion=True, truncated=False,
|
|
unique_cause_proven=False, necessity_proven=False, output_budget_bytes=BUDGET,
|
|
meaning='Exact conditional ALLN reconstruction and timed anatomical antecedent audit, not a biological causal gate'))
|
|
file_budget(outroot)
|
|
except Exception as error:
|
|
save(outroot / 'failure.json', dict(pass_all=False, error=type(error).__name__, message=str(error),
|
|
traceback=traceback.format_exc(), completed=receipts, new_brain_simulation=False))
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|