Digital Drosophila Archive
107 lines
9.6 KiB
Python
107 lines
9.6 KiB
Python
"""Descriptive paired effects only; no condition selection or parameter fitting."""
|
||
from pathlib import Path
|
||
import json
|
||
import pandas as pd
|
||
import numpy as np
|
||
import yaml
|
||
from phase4r import save
|
||
|
||
cfg=yaml.safe_load(Path(__file__).with_name('phase4y.yaml').read_text());exp=Path(cfg['experiment_root'])
|
||
assert not (exp/'checksums.sha256').exists(),'Sealed output is immutable'
|
||
w=pd.read_parquet(exp/'comparisons/watched-KC.parquet');t=pd.read_parquet(exp/'comparisons/trials.parquet')
|
||
p=pd.read_parquet(exp/'comparisons/populations.parquet');x=pd.read_parquet(Path(cfg['prior_root'])/'replay/results.parquet')
|
||
inputrows=[]
|
||
for _,r in t.iterrows():
|
||
obs=pd.read_parquet(exp/'observations'/f'{r.odor_index}-{r.seed}-{r.condition}.parquet').iloc[-1]
|
||
inputrows.append({'odor_index':int(r.odor_index),'seed':int(r.seed),'condition':r.condition,
|
||
'requested':int(obs.PN_requested_cumulative),'accepted_new_spikes':int(obs.PN_accepted_cumulative),
|
||
'preexisting_spikes':int(obs.PN_preexisting_cumulative),'refractory_blocked':int(obs.PN_blocked_cumulative)})
|
||
inp=pd.DataFrame(inputrows);assert (inp.groupby(['odor_index','seed']).requested.nunique()==1).all()
|
||
assert (inp.requested==inp.accepted_new_spikes+inp.preexisting_spikes+inp.refractory_blocked).all()
|
||
inp.to_parquet(exp/'comparisons/input-request-vs-acceptance.parquet',index=False)
|
||
save(exp/'audit/input-matching.json',{'scheduled_requests_matched_all_pairs':True,'partition_exact':True,
|
||
'pairs_with_changed_accepted_count':int((inp.groupby(['odor_index','seed']).accepted_new_spikes.nunique()>1).sum()),
|
||
'interpretation':'identical exogenous requests, not guaranteed identical accepted PN spikes; acceptance depends on endogenous refractory/spike state through the unchanged force interface'})
|
||
# Reporting-only schema correction. Preserve original tables and the registered run source.
|
||
# These exact superclass values are used by the frozen Phase4C propagation summaries.
|
||
meta_path=Path('/data/digital-drosophila/datasets/male-cns-v1.0/processed/neuron-only/neuron-index/unified-metadata.parquet')
|
||
sup=pd.read_parquet(meta_path,columns=['neuron_idx','annotation_superclass']).sort_values('neuron_idx')
|
||
definitions={'descending':['descending_neuron','descending_neuron_tbc'],'motor':['vnc_motor','cb_motor']}
|
||
corrected=[]
|
||
for _,r in t.iterrows():
|
||
name=f'{r.odor_index}-{r.seed}-{r.condition}'
|
||
with np.load(exp/'states'/f'{name}.npz') as z:
|
||
for pop,values in definitions.items():
|
||
idx=np.flatnonzero(sup.annotation_superclass.isin(values).to_numpy());assert len(idx)>0
|
||
c=z['counts'][idx];l=z['late_counts'][idx]
|
||
corrected.append({'odor_index':int(r.odor_index),'seed':int(r.seed),'condition':r.condition,'population':pop,
|
||
'neurons':len(idx),'total_spikes':int(c.astype(np.int64).sum()),'active':int(np.count_nonzero(c)),
|
||
'late_spikes':int(l.astype(np.int64).sum()),'late_active':int(np.count_nonzero(l))})
|
||
p=pd.concat([p[~p.population.isin(definitions)],pd.DataFrame(corrected)],ignore_index=True)
|
||
p.to_parquet(exp/'comparisons/populations-corrected.parquet',index=False)
|
||
save(exp/'audit/population-schema-correction.json',{'scope':'report aggregation only, no dynamics or registered conditions changed',
|
||
'issue':'initial literal superclass descending/motor matches zero rows; those zero-population rows are invalid readouts, not evidence of silence',
|
||
'definitions_from':'frozen phase4c.py summary definitions, verified against actual metadata',
|
||
'corrected_values':definitions,'corrected_counts':{k:int(sup.annotation_superclass.isin(v).sum()) for k,v in definitions.items()},
|
||
'source':str(meta_path),'original_tables_preserved':True,'recomputed_from':'saved complete per-neuron total and late counts','simulation_rerun':False})
|
||
(exp/'READ-ME-FIRST.md').write_text(
|
||
'# Phase 4Y evidence routing\n\n'
|
||
'Use audit/MECHANISM.md and comparisons/populations-corrected.parquet for population readouts.\n\n'
|
||
'The original comparisons/populations.parquet descending/motor rows used invalid literal superclass names and matched zero neurons. '
|
||
'Those rows are preserved for audit but must not be interpreted as no activity. Corrected rows use the frozen Phase4C definitions '
|
||
'(descending_neuron/descending_neuron_tbc; vnc_motor/cb_motor) and were recomputed from saved complete counts, without rerunning dynamics. '
|
||
'See audit/population-schema-correction.json. Other original population rows are unchanged.\n\n'
|
||
'All three hypotheses are reported. None is promoted to a learned or biological baseline.\n',encoding='utf-8')
|
||
effects=[];isolated=[]
|
||
for (oi,seed,node),g in w.groupby(['odor_index','seed','neuron_idx']):
|
||
baseline=int(g.loc[g.condition.eq('frozen_reference'),'late_spikes'].iloc[0])
|
||
for _,r in g.iterrows():
|
||
effects.append({'odor_index':int(oi),'seed':int(seed),'neuron_idx':int(node),'condition':r.condition,
|
||
'baseline_late_spikes':baseline,'condition_late_spikes':int(r.late_spikes),'paired_change':int(r.late_spikes)-baseline})
|
||
if seed==cfg['old_replay_seed']:
|
||
name=f'{oi}-{seed}-offset';z=x[(x.trial==name)&(x.neuron_idx==node)]
|
||
ib=int(z.loc[z['mode'].eq('base'),'late_spikes'].iloc[0]);io=int(z.loc[z['mode'].eq('omit_DPM_annotation'),'late_spikes'].iloc[0])
|
||
off=int(g.loc[g.condition.eq('DPM_fast_current_off'),'late_spikes'].iloc[0]);assert ib==baseline
|
||
isolated.append({'odor_index':int(oi),'neuron_idx':int(node),'baseline_late_spikes':baseline,
|
||
'isolated_DPM_off_late_spikes':io,'whole_brain_DPM_off_late_spikes':off,
|
||
'whole_minus_isolated':off-io,'interpretation':'different feedback assumptions; not a numerical regression failure'})
|
||
pd.DataFrame(effects).to_parquet(exp/'comparisons/watched-paired-effects.parquet',index=False)
|
||
pd.DataFrame(isolated).to_parquet(exp/'comparisons/isolated-vs-whole-brain.parquet',index=False)
|
||
new=[];loss=[]
|
||
metadata=pd.read_parquet(Path(cfg['prior_root'])/'roles/neuron-evidence-axes.parquet').set_index('neuron_idx')
|
||
for (oi,seed),group in t.groupby(['odor_index','seed']):
|
||
def late_set(condition):
|
||
e=pd.read_parquet(exp/'events'/f'{oi}-{seed}-{condition}.parquet');return set(map(int,e.loc[e.step>=5000,'neuron_idx']))
|
||
ref=late_set('frozen_reference')
|
||
for condition in ('DPM_fast_current_off','DPM_fast_current_reversed'):
|
||
s=late_set(condition)
|
||
for label,nodes in [('new',s-ref),('lost',ref-s)]:
|
||
for node in sorted(nodes):
|
||
row={'odor_index':int(oi),'seed':int(seed),'condition':condition,'change':label,'neuron_idx':node,
|
||
'body_id':int(metadata.loc[node,'body_id']),'annotation_type':metadata.loc[node,'annotation_type']}
|
||
(new if label=='new' else loss).append(row)
|
||
pd.DataFrame(new+loss,columns=['odor_index','seed','condition','change','neuron_idx','body_id','annotation_type']).to_parquet(exp/'comparisons/late-KC-membership-changes.parquet',index=False)
|
||
ep=pd.DataFrame(effects);summary=[]
|
||
for (node,condition),g in ep.groupby(['neuron_idx','condition']):
|
||
summary.append({'neuron_idx':int(node),'condition':condition,'n':len(g),'late_min':int(g.condition_late_spikes.min()),
|
||
'late_max':int(g.condition_late_spikes.max()),'change_min':int(g.paired_change.min()),'change_max':int(g.paired_change.max()),
|
||
'decreases':int((g.paired_change<0).sum()),'increases':int((g.paired_change>0).sum()),'unchanged':int((g.paired_change==0).sum())})
|
||
save(exp/'audit/descriptive-effects.json',{'scope':'post-run descriptive summaries; no promotion or new gate',
|
||
'watched':summary,'new_late_KC_memberships':len(new),'lost_late_KC_memberships':len(loss)})
|
||
lines=['# Phase 4Y — paired whole-brain effects','',
|
||
'All branches are diagnostic hypotheses; none is selected as a better biological model.','',
|
||
'| KC | condition | late spike range | paired change range | decreases / unchanged / increases |','|---:|---|---:|---:|---:|']
|
||
for r in summary:lines.append(f"| {r['neuron_idx']} | {r['condition']} | {r['late_min']}–{r['late_max']} | {r['change_min']}–{r['change_max']} | {r['decreases']} / {r['unchanged']} / {r['increases']} |")
|
||
lines+=['','| KC | isolated DPM-off range | whole-brain DPM-off range (same old seed) |','|---:|---:|---:|']
|
||
for node,g in pd.DataFrame(isolated).groupby('neuron_idx'):
|
||
lines.append(f'| {node} | {g.isolated_DPM_off_late_spikes.min()}–{g.isolated_DPM_off_late_spikes.max()} | {g.whole_brain_DPM_off_late_spikes.min()}–{g.whole_brain_DPM_off_late_spikes.max()} |')
|
||
lines+=['','| population | condition | mean late spikes | mean late active neurons |','|---|---|---:|---:|']
|
||
for (pop,condition),g in p.groupby(['population','condition']):
|
||
lines.append(f'| {pop} | {condition} | {g.late_spikes.mean():.2f} | {g.late_active.mean():.2f} |')
|
||
lines+=['','Whole-brain feedback is allowed to respond; isolated replay holds the other recorded arrivals fixed. A difference between them is the target of the experiment, not grounds for choosing one result.',
|
||
'Input matching is at the scheduled request level. Accepted new PN spikes can differ through feedback-dependent refractory state; see input-request-vs-acceptance.parquet. Do not claim identical delivered PN spike trains.',
|
||
'Descending/motor rows here use populations-corrected.parquet, reconstructed from saved full counts and verified actual superclass names. The original zero-population rows are retained but invalid; see audit/population-schema-correction.json.',
|
||
'No learning, no reward events, no parameter fitting, no promotion; smaller late activity is not itself success. Biological signs and long-term stability remain unproven.']
|
||
(exp/'audit/MECHANISM.md').write_text('\n'.join(lines)+'\n',encoding='utf-8')
|
||
print('\n'.join(lines[:20]))
|