Archive MaleCNS research sources, provenance, bibliography and immutable data inventory
This commit is contained in:
Digital Drosophila Archive
+10
@@ -0,0 +1,10 @@
|
||||
Phase 3B: superclass != null; 166700 neurons including null type.
|
||||
Raw type strings preserved; 11710 paper types are reference only.
|
||||
Original annotation and NT fields use annotation_ and nt_ prefixes, with original dtypes and nulls.
|
||||
NT coverage uses nonempty consensus_nt, without inferred fallback.
|
||||
Mapping is body_id ascending; reverse lookup uses searchsorted with exact equality check.
|
||||
CSR rows are presynaptic sources; columns are postsynaptic targets.
|
||||
Fragment comparison excludes each edge once if either endpoint is a fragment.
|
||||
Nine category totals separate annotation fragments from other unannotated segments.
|
||||
Re-run using processed/.venv/bin/python processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py; completed runs verify and do not overwrite.
|
||||
Historical audit in parent audit/ remains unchanged.
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff audit/phase3b-166700/README.md
|
||||
beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583 audit/phase3b-166700/edge-category-audit.json
|
||||
bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702 audit/phase3b-166700/excluded-fragment-annotations.parquet
|
||||
c061452f341d190946f80ca96b4589ab1e2a8dca6a67c57ad5d14f6ce34b1b64 audit/phase3b-166700/manifest.json
|
||||
b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234 audit/phase3b-166700/missing-neurotransmitter-ids.parquet
|
||||
4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e audit/phase3b-166700/missing-swc-ids.parquet
|
||||
b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b audit/phase3b-166700/missing-type-ids.parquet
|
||||
3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97 audit/phase3b-166700/phase3b-neuron-only.py
|
||||
f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6 audit/phase3b-166700/schema.json
|
||||
f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd audit/phase3b-166700/statistics.json
|
||||
497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a audit/phase3b-166700/type-field-audit.json
|
||||
598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1 audit/phase3b-166700/validation.json
|
||||
a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f connectivity/canonical-edges.parquet
|
||||
5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b connectivity/degrees.parquet
|
||||
ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150 gpu/csr-indices.int32.npy
|
||||
fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4 gpu/csr-indptr.int64.npy
|
||||
3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8 gpu/csr-metadata.json
|
||||
c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b gpu/csr-weights.int64.npy
|
||||
5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242 neuron-index/body-id-neuron-idx.feather
|
||||
2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009 neuron-index/body-id-neuron-idx.parquet
|
||||
c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206 neuron-index/body-id-to-neuron-idx.npz
|
||||
ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6 neuron-index/neuron-idx-to-body-id.uint64.npy
|
||||
ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1 neuron-index/unified-metadata.feather
|
||||
922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63 neuron-index/unified-metadata.parquet
|
||||
4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8 phase3b-repeat.log
|
||||
b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e phase3b-run.log
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"fragment->fragment": {
|
||||
"edges": 14135,
|
||||
"weight": 49566
|
||||
},
|
||||
"fragment->neuron": {
|
||||
"edges": 249969,
|
||||
"weight": 661930
|
||||
},
|
||||
"fragment->other": {
|
||||
"edges": 766887,
|
||||
"weight": 1131611
|
||||
},
|
||||
"neuron->fragment": {
|
||||
"edges": 181344,
|
||||
"weight": 476820
|
||||
},
|
||||
"neuron->neuron": {
|
||||
"edges": 25582938,
|
||||
"weight": 124177617
|
||||
},
|
||||
"neuron->other": {
|
||||
"edges": 112452244,
|
||||
"weight": 170414577
|
||||
},
|
||||
"other->fragment": {
|
||||
"edges": 78570,
|
||||
"weight": 107066
|
||||
},
|
||||
"other->neuron": {
|
||||
"edges": 4552783,
|
||||
"weight": 5614376
|
||||
},
|
||||
"other->other": {
|
||||
"edges": 7977814,
|
||||
"weight": 9199680
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"canonical_rule": "superclass != null",
|
||||
"disk_reload_check": "PASS: all original annotation/NT fields exact and CSR dtype/range/count checks passed",
|
||||
"inputs": [
|
||||
{
|
||||
"bytes": 14483314,
|
||||
"path": "raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather",
|
||||
"sha256": "2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2"
|
||||
},
|
||||
{
|
||||
"bytes": 43282834,
|
||||
"path": "raw/annotations/body-neurotransmitters-male-cns-v1.0.feather",
|
||||
"sha256": "95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621"
|
||||
},
|
||||
{
|
||||
"bytes": 1051241946,
|
||||
"path": "raw/connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather",
|
||||
"sha256": "e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"bytes": 823,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/README.md",
|
||||
"sha256": "cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff"
|
||||
},
|
||||
{
|
||||
"bytes": 643,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/edge-category-audit.json",
|
||||
"sha256": "beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583"
|
||||
},
|
||||
{
|
||||
"bytes": 261095,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/excluded-fragment-annotations.parquet",
|
||||
"sha256": "bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702"
|
||||
},
|
||||
{
|
||||
"bytes": 1350,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/missing-neurotransmitter-ids.parquet",
|
||||
"sha256": "b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234"
|
||||
},
|
||||
{
|
||||
"bytes": 364,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/missing-swc-ids.parquet",
|
||||
"sha256": "4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e"
|
||||
},
|
||||
{
|
||||
"bytes": 8105,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/missing-type-ids.parquet",
|
||||
"sha256": "b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b"
|
||||
},
|
||||
{
|
||||
"bytes": 13055,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py",
|
||||
"sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
|
||||
},
|
||||
{
|
||||
"bytes": 3383,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/schema.json",
|
||||
"sha256": "f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6"
|
||||
},
|
||||
{
|
||||
"bytes": 753,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/statistics.json",
|
||||
"sha256": "f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd"
|
||||
},
|
||||
{
|
||||
"bytes": 649,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/type-field-audit.json",
|
||||
"sha256": "497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a"
|
||||
},
|
||||
{
|
||||
"bytes": 371,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/validation.json",
|
||||
"sha256": "598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1"
|
||||
},
|
||||
{
|
||||
"bytes": 137157039,
|
||||
"path": "processed/neuron-only/connectivity/canonical-edges.parquet",
|
||||
"sha256": "a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f"
|
||||
},
|
||||
{
|
||||
"bytes": 1665145,
|
||||
"path": "processed/neuron-only/connectivity/degrees.parquet",
|
||||
"sha256": "5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b"
|
||||
},
|
||||
{
|
||||
"bytes": 102331880,
|
||||
"path": "processed/neuron-only/gpu/csr-indices.int32.npy",
|
||||
"sha256": "ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150"
|
||||
},
|
||||
{
|
||||
"bytes": 1333736,
|
||||
"path": "processed/neuron-only/gpu/csr-indptr.int64.npy",
|
||||
"sha256": "fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4"
|
||||
},
|
||||
{
|
||||
"bytes": 361,
|
||||
"path": "processed/neuron-only/gpu/csr-metadata.json",
|
||||
"sha256": "3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8"
|
||||
},
|
||||
{
|
||||
"bytes": 204663632,
|
||||
"path": "processed/neuron-only/gpu/csr-weights.int64.npy",
|
||||
"sha256": "c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b"
|
||||
},
|
||||
{
|
||||
"bytes": 610146,
|
||||
"path": "processed/neuron-only/neuron-index/body-id-neuron-idx.feather",
|
||||
"sha256": "5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242"
|
||||
},
|
||||
{
|
||||
"bytes": 1197954,
|
||||
"path": "processed/neuron-only/neuron-index/body-id-neuron-idx.parquet",
|
||||
"sha256": "2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009"
|
||||
},
|
||||
{
|
||||
"bytes": 2000920,
|
||||
"path": "processed/neuron-only/neuron-index/body-id-to-neuron-idx.npz",
|
||||
"sha256": "c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206"
|
||||
},
|
||||
{
|
||||
"bytes": 1333728,
|
||||
"path": "processed/neuron-only/neuron-index/neuron-idx-to-body-id.uint64.npy",
|
||||
"sha256": "ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6"
|
||||
},
|
||||
{
|
||||
"bytes": 16822034,
|
||||
"path": "processed/neuron-only/neuron-index/unified-metadata.feather",
|
||||
"sha256": "ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1"
|
||||
},
|
||||
{
|
||||
"bytes": 9283041,
|
||||
"path": "processed/neuron-only/neuron-index/unified-metadata.parquet",
|
||||
"sha256": "922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63"
|
||||
},
|
||||
{
|
||||
"bytes": 929,
|
||||
"path": "processed/neuron-only/phase3b-run.log",
|
||||
"sha256": "b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e"
|
||||
},
|
||||
{
|
||||
"bytes": 106,
|
||||
"path": "processed/neuron-only/phase3b-repeat.log",
|
||||
"sha256": "4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8"
|
||||
}
|
||||
],
|
||||
"phase": "3B",
|
||||
"repeat_run_check": "PASS: full inputs/output checksums verified without overwrite",
|
||||
"script_sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic neuron-only preprocessing. Existing completed outputs are verified, never replaced."""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
import pyarrow.compute as pc
|
||||
import pyarrow.feather as ft
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
|
||||
def sha(p):
|
||||
h = hashlib.sha256()
|
||||
with p.open('rb') as f:
|
||||
for b in iter(lambda: f.read(8388608), b''):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def js(p, obj):
|
||||
p.write_text(json.dumps(obj, indent=2, sort_keys=True)+'\n')
|
||||
|
||||
|
||||
def member(ids, q):
|
||||
pos = np.searchsorted(ids, q)
|
||||
return pos, (pos < len(ids)) & (ids[np.minimum(pos, len(ids)-1)] == q)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--root', default='/data/digital-drosophila/datasets/male-cns-v1.0')
|
||||
args = ap.parse_args()
|
||||
root = Path(args.root)
|
||||
raw = root/'raw'
|
||||
out = root/'processed/neuron-only'
|
||||
audit_name = 'phase3b-166700'
|
||||
final_audit = out/'audit'/audit_name
|
||||
def checks():
|
||||
subprocess.run(['md5sum','--quiet','-c','_manifests/checksums.md5'],cwd=raw,check=True)
|
||||
subprocess.run(['md5sum','--quiet','-c','../_manifests/skeletons/checksums.md5'],cwd=raw/'skeletons',check=True)
|
||||
subprocess.run(['sha256sum','--quiet','-c','audit/checksums.sha256'],cwd=root/'processed',check=True)
|
||||
subprocess.run(['sha256sum','--quiet','-c','checksums.sha256'],cwd=out/'audit',check=True)
|
||||
if (final_audit/'manifest.json').exists():
|
||||
checks()
|
||||
manifest=json.loads((final_audit/'manifest.json').read_text())
|
||||
for item in manifest['inputs']+manifest['outputs']:
|
||||
assert sha(root/item['path']) == item['sha256'], item['path']
|
||||
assert sha(Path(__file__)) == manifest['script_sha256']
|
||||
subprocess.run(['sha256sum','--quiet','-c',str(final_audit/'checksums.sha256')],cwd=out,check=True)
|
||||
print('REPEAT_RUN_OK: all inputs, existing Phase3, historical audit and Phase3B outputs unchanged; no overwrite.',flush=True)
|
||||
return
|
||||
for d in ['neuron-index','connectivity','gpu']:
|
||||
if (out/d).exists():
|
||||
raise RuntimeError(f'Refusing existing destination {out/d}')
|
||||
assert not final_audit.exists()
|
||||
print('Verifying raw and preserved Phase3/historical audit',flush=True)
|
||||
checks()
|
||||
stage=out/f'.staging-{int(time.time())}'
|
||||
for d in ['neuron-index','connectivity','gpu',f'audit/{audit_name}']:
|
||||
(stage/d).mkdir(parents=True)
|
||||
audit=stage/'audit'/audit_name
|
||||
paths=[raw/'annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather',raw/'annotations/body-neurotransmitters-male-cns-v1.0.feather',raw/'connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather']
|
||||
inputs=[dict(path=str(p.relative_to(root)),sha256=sha(p),bytes=p.stat().st_size) for p in paths]
|
||||
a,nt,c=[ft.read_table(p) for p in paths]
|
||||
schemas={k:str(t.schema) for k,t in zip(['annotation','neurotransmitter','connectome'],[a,nt,c])}
|
||||
assert a.schema.field('superclass').type == pa.string()
|
||||
assert set(c.column_names)=={'body_pre','body_post','weight'}
|
||||
def ids(t,k):
|
||||
assert t[k].null_count==0 and pa.types.is_integer(t[k].type)
|
||||
arr=t[k].to_numpy();assert np.all(arr>=0)
|
||||
return arr.astype(np.uint64)
|
||||
aid=ids(a,'bodyId');nid=ids(nt,'body')
|
||||
assert len(np.unique(aid))==len(aid) and len(np.unique(nid))==len(nid)
|
||||
mask=pc.is_valid(a['superclass'])
|
||||
neurons=a.filter(mask);fragments=a.filter(pc.invert(mask))
|
||||
assert neurons.num_rows==166700 and fragments.num_rows==44877
|
||||
neurons=neurons.sort_by([('bodyId','ascending')]);canon=ids(neurons,'bodyId')
|
||||
frag=np.sort(ids(fragments,'bodyId'));allids=np.sort(aid);n=len(canon)
|
||||
pq.write_table(fragments,audit/'excluded-fragment-annotations.parquet',compression='zstd')
|
||||
swc=np.array(sorted(int(p.stem) for p in (raw/'skeletons').glob('*.swc')),dtype=np.uint64)
|
||||
has_swc=member(swc,canon)[1]
|
||||
ni=pc.index_in(pa.array(canon),value_set=nt['body'].cast(pa.uint64()))
|
||||
aligned=nt.take(ni)
|
||||
md={'neuron_idx':pa.array(np.arange(n,dtype=np.int32)),'body_id':pa.array(canon),
|
||||
'neuron_type':neurons['type'],'neuron_class':neurons['class'],'side':neurons['somaSide'],
|
||||
'neurotransmitter':aligned['consensus_nt'],'has_swc':pa.array(has_swc),
|
||||
'swc_path':pa.array([str(raw/'skeletons'/f'{int(b)}.swc') if ok else None for b,ok in zip(canon,has_swc)])}
|
||||
md.update({'annotation_'+k:neurons[k] for k in neurons.column_names})
|
||||
md.update({'nt_'+k:aligned[k] for k in aligned.column_names})
|
||||
metadata=pa.table(md)
|
||||
mapping=metadata.select(['neuron_idx','body_id'])
|
||||
for name,t in [('unified-metadata',metadata),('body-id-neuron-idx',mapping)]:
|
||||
pq.write_table(t,stage/'neuron-index'/f'{name}.parquet',compression='zstd')
|
||||
ft.write_feather(t,stage/'neuron-index'/f'{name}.feather',compression='zstd')
|
||||
np.save(stage/'neuron-index/neuron-idx-to-body-id.uint64.npy',canon)
|
||||
np.savez(stage/'neuron-index/body-id-to-neuron-idx.npz',body_id=canon,neuron_idx=np.arange(n,dtype=np.int32))
|
||||
type_stats={}
|
||||
for k in neurons.column_names:
|
||||
if 'type' in k.lower():
|
||||
v=neurons[k].to_pylist();good=[x for x in v if x is not None and x!='']
|
||||
type_stats[k]=dict(null=sum(x is None for x in v),empty=sum(x=='' for x in v),unique_nonempty=len(set(good)),present=len(good))
|
||||
ntv=aligned['consensus_nt'].to_pylist();hasnt=np.array([x is not None and x!='' for x in ntv])
|
||||
for name,present in [('swc',has_swc),('neurotransmitter',hasnt),('type',np.array([x is not None and x!='' for x in neurons['type'].to_pylist()]))]:
|
||||
pq.write_table(pa.table({'body_id':canon[~present]}),audit/f'missing-{name}-ids.parquet',compression='zstd')
|
||||
print('Filtering original connectome and auditing fragment edges',flush=True)
|
||||
src_chunks=[];dst_chunks=[];w_chunks=[]
|
||||
categories={f'{s}->{t}':{'edges':0,'weight':0} for s in ['neuron','fragment','other'] for t in ['neuron','fragment','other']}
|
||||
for batch in c.to_batches(max_chunksize=1000000):
|
||||
pre=ids(pa.Table.from_batches([batch]),'body_pre');post=ids(pa.Table.from_batches([batch]),'body_post')
|
||||
w=batch.column(batch.schema.get_field_index('weight')).to_numpy()
|
||||
assert w.dtype==np.int64 and np.all(w>0)
|
||||
si,sm=member(canon,pre);ti,tm=member(canon,post)
|
||||
sf=member(frag,pre)[1];tf=member(frag,post)[1]
|
||||
for sk,sv in [('neuron',sm),('fragment',sf),('other',~(sm|sf))]:
|
||||
for tk,tv in [('neuron',tm),('fragment',tf),('other',~(tm|tf))]:
|
||||
keep=sv&tv;z=categories[f'{sk}->{tk}'];z['edges']+=int(keep.sum());z['weight']+=int(w[keep].sum())
|
||||
keep=sm&tm
|
||||
src_chunks.append(si[keep].astype(np.int32));dst_chunks.append(ti[keep].astype(np.int32));w_chunks.append(w[keep])
|
||||
s=np.concatenate(src_chunks);t=np.concatenate(dst_chunks);w=np.concatenate(w_chunks)
|
||||
order=np.lexsort((t,s));s=s[order];t=t[order];w=w[order]
|
||||
duplicates=int(np.sum((s[1:]==s[:-1])&(t[1:]==t[:-1])))
|
||||
indeg=np.bincount(t,minlength=n);outdeg=np.bincount(s,minlength=n)
|
||||
ptr=np.r_[np.int64(0),np.cumsum(outdeg,dtype=np.int64)]
|
||||
for filename,x in [('csr-indptr.int64',ptr),('csr-indices.int32',t),('csr-weights.int64',w)]:
|
||||
np.save(stage/'gpu'/f'{filename}.npy',x)
|
||||
edges=pa.table({'source_neuron_idx':s,'target_neuron_idx':t,'body_pre':canon[s],'body_post':canon[t],'weight':w})
|
||||
pq.write_table(edges,stage/'connectivity/canonical-edges.parquet',compression='zstd')
|
||||
pq.write_table(pa.table({'body_id':canon,'neuron_idx':np.arange(n,dtype=np.int32),'indegree':indeg,'outdegree':outdeg}),stage/'connectivity/degrees.parquet',compression='zstd')
|
||||
old_edges=sum(categories[k]['edges'] for k in ['neuron->neuron','neuron->fragment','fragment->neuron','fragment->fragment'])
|
||||
old_weight=sum(categories[k]['weight'] for k in ['neuron->neuron','neuron->fragment','fragment->neuron','fragment->fragment'])
|
||||
old=json.loads((root/'processed/connectivity/connectivity-stats.json').read_text())
|
||||
assert old_edges==old['canonical_edge_rows'] and old_weight==old['canonical_weight_sum']
|
||||
print('Independent full raw edge lookup validation',flush=True)
|
||||
keys=s.astype(np.int64)*n+t;seen=np.zeros(len(keys),dtype=bool)
|
||||
for b in c.to_batches(max_chunksize=1000000):
|
||||
bt=pa.Table.from_batches([b]);pre=ids(bt,'body_pre');post=ids(bt,'body_post');bw=bt['weight'].to_numpy()
|
||||
si,sm=member(canon,pre);ti,tm=member(canon,post);keep=sm&tm
|
||||
loc=np.searchsorted(keys,si[keep]*n+ti[keep])
|
||||
assert np.all(loc<len(keys)) and np.array_equal(keys[loc],si[keep]*n+ti[keep])
|
||||
assert np.array_equal(w[loc],bw[keep]);seen[loc]=True
|
||||
assert duplicates==0 and seen.all()
|
||||
assert ptr[-1]==len(w) and ptr.dtype==np.int64 and t.dtype==np.int32
|
||||
assert np.array_equal(np.repeat(np.arange(n,dtype=np.int32),np.diff(ptr)),s)
|
||||
assert np.array_equal(canon[np.searchsorted(canon,aid[mask.to_numpy()])],aid[mask.to_numpy()])
|
||||
assert sum(z['edges'] for z in categories.values())==c.num_rows
|
||||
stats=dict(neurons=n,fragments=len(frag),with_swc=int(has_swc.sum()),without_swc=int((~has_swc).sum()),
|
||||
with_neurotransmitter=int(hasnt.sum()),without_neurotransmitter=int((~hasnt).sum()),neurotransmitter_coverage=float(hasnt.mean()),
|
||||
with_type=type_stats['type']['present'],without_type=n-type_stats['type']['present'],raw_unique_type_strings=type_stats['type']['unique_nonempty'],
|
||||
nnz=len(w),synapse_weight=int(w.sum()),weight_min=int(w.min()),weight_max=int(w.max()),
|
||||
connected_neurons=int(((indeg+outdeg)>0).sum()),isolated_neurons=int(((indeg+outdeg)==0).sum()),
|
||||
indegree_zero=int((indeg==0).sum()),outdegree_zero=int((outdeg==0).sum()),self_loops=int((s==t).sum()),duplicate_pairs=duplicates,
|
||||
excluded_fragment_edges_from_phase3=old_edges-len(w),excluded_fragment_weight_from_phase3=old_weight-int(w.sum()),
|
||||
all_raw_fragment_related_edges=sum(z['edges'] for k,z in categories.items() if 'fragment' in k),
|
||||
all_raw_fragment_related_weight=sum(z['weight'] for k,z in categories.items() if 'fragment' in k),
|
||||
csr_bytes=sum(p.stat().st_size for p in (stage/'gpu').glob('*.npy')))
|
||||
js(audit/'statistics.json',stats);js(audit/'edge-category-audit.json',categories);js(audit/'type-field-audit.json',type_stats)
|
||||
schemas.update(metadata=str(metadata.schema),mapping=str(mapping.schema),edges=str(edges.schema))
|
||||
js(audit/'schema.json',schemas)
|
||||
js(stage/'gpu/csr-metadata.json',dict(shape=[n,n],nnz=len(w),orientation='row=presynaptic source; column=postsynaptic target; incoming drive is A.T @ source_activity',indptr='int64',indices='int32',weights='int64 original positive synapse counts',self_loops='retained',threshold='none',signed_weight_conversion=False))
|
||||
js(audit/'validation.json',dict(all_raw_edges_checked_against_csr=True,weights_exact=True,old_phase3_counts_reconciled=True,raw_and_prior_artifacts_verified=True,canonical_rule='superclass is not null',type_filter=False,repeat_behavior='verify input/output SHA256 and return without writes',software=dict(numpy=np.__version__,pyarrow=pa.__version__)))
|
||||
shutil.copyfile(__file__,audit/'phase3b-neuron-only.py')
|
||||
(audit/'README.md').write_text('Phase 3B: superclass != null; 166700 neurons including null type.\nRaw type strings preserved; 11710 paper types are reference only.\nOriginal annotation and NT fields use annotation_ and nt_ prefixes, with original dtypes and nulls.\nNT coverage uses nonempty consensus_nt, without inferred fallback.\nMapping is body_id ascending; reverse lookup uses searchsorted with exact equality check.\nCSR rows are presynaptic sources; columns are postsynaptic targets.\nFragment comparison excludes each edge once if either endpoint is a fragment.\nNine category totals separate annotation fragments from other unannotated segments.\nRe-run using processed/.venv/bin/python processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py; completed runs verify and do not overwrite.\nHistorical audit in parent audit/ remains unchanged.\n')
|
||||
checks()
|
||||
assert all(sha(root/i['path'])==i['sha256'] for i in inputs)
|
||||
inventory=[dict(path=str((out/p.relative_to(stage)).relative_to(root)),bytes=p.stat().st_size,sha256=sha(p)) for p in sorted(stage.rglob('*')) if p.is_file()]
|
||||
js(audit/'manifest.json',dict(inputs=inputs,outputs=inventory,script_sha256=sha(Path(__file__)),phase='3B',canonical_rule='superclass != null'))
|
||||
(audit/'checksums.sha256').write_text(''.join(f'{sha(p)} {p.relative_to(stage).as_posix()}\n' for p in sorted(stage.rglob('*')) if p.is_file()))
|
||||
for d in ['neuron-index','connectivity','gpu']:
|
||||
os.rename(stage/d,out/d)
|
||||
os.rename(audit,final_audit)
|
||||
(stage/'audit').rmdir();stage.rmdir()
|
||||
print(json.dumps(stats,indent=2),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"annotation": "assignedOlHex1: double\nassignedOlHex2: double\nbodyId: int64\nflywireType: string\ngroup: double\ninstance: string\nsomaSide: string\nstatusLabel: dictionary<values=string, indices=int8, ordered=1>\nsuperclass: string\ntype: string\nvfbId: string\nhemibrainType: string\nitoleeHl: string\nsupertype: string\nbirthtime: string\nmancBodyid: double\nmancGroup: double\nmancType: string\nsubclass: string\nsynonyms: string\nclass: string\nrootSide: string\nsomaNeuromere: string\ntrumanHl: string\ndimorphism: string\nmatchingNotes: string\nentryNerve: string\nmancSerial: double\nmcnsSerial: double\nserialMotif: string\nfruDsx: string\nexitNerve: string\nreceptorType: string\nsomaLocation: list<item: int64>\n child 0, item: int64\ntosomaLocation: list<item: int64>\n child 0, item: int64\nstatus: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 4583",
|
||||
"connectome": "body_pre: int64\nbody_post: int64\nweight: int64\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 604",
|
||||
"edges": "source_neuron_idx: int32\ntarget_neuron_idx: int32\nbody_pre: uint64\nbody_post: uint64\nweight: int64",
|
||||
"mapping": "neuron_idx: int32\nbody_id: uint64",
|
||||
"metadata": "neuron_idx: int32\nbody_id: uint64\nneuron_type: string\nneuron_class: string\nside: string\nneurotransmitter: string\nhas_swc: bool\nswc_path: string\nannotation_assignedOlHex1: double\nannotation_assignedOlHex2: double\nannotation_bodyId: int64\nannotation_flywireType: string\nannotation_group: double\nannotation_instance: string\nannotation_somaSide: string\nannotation_statusLabel: dictionary<values=string, indices=int8, ordered=1>\nannotation_superclass: string\nannotation_type: string\nannotation_vfbId: string\nannotation_hemibrainType: string\nannotation_itoleeHl: string\nannotation_supertype: string\nannotation_birthtime: string\nannotation_mancBodyid: double\nannotation_mancGroup: double\nannotation_mancType: string\nannotation_subclass: string\nannotation_synonyms: string\nannotation_class: string\nannotation_rootSide: string\nannotation_somaNeuromere: string\nannotation_trumanHl: string\nannotation_dimorphism: string\nannotation_matchingNotes: string\nannotation_entryNerve: string\nannotation_mancSerial: double\nannotation_mcnsSerial: double\nannotation_serialMotif: string\nannotation_fruDsx: string\nannotation_exitNerve: string\nannotation_receptorType: string\nannotation_somaLocation: list<item: int64>\n child 0, item: int64\nannotation_tosomaLocation: list<item: int64>\n child 0, item: int64\nannotation_status: string\nnt_body: int64\nnt_cell_type: string\nnt_total_nt_predictions: int32\nnt_predicted_nt_confidence: double\nnt_predicted_nt: string\nnt_ground_truth: string\nnt_celltype_total_nt_predictions: int32\nnt_celltype_predicted_nt: string\nnt_celltype_predicted_nt_confidence: double\nnt_consensus_nt: string",
|
||||
"neurotransmitter": "body: int64\ncell_type: string\ntotal_nt_predictions: int32\npredicted_nt_confidence: double\npredicted_nt: string\nground_truth: string\ncelltype_total_nt_predictions: int32\ncelltype_predicted_nt: string\ncelltype_predicted_nt_confidence: double\nconsensus_nt: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 1606"
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"all_raw_fragment_related_edges": 1290905,
|
||||
"all_raw_fragment_related_weight": 2426993,
|
||||
"connected_neurons": 166483,
|
||||
"csr_bytes": 308329248,
|
||||
"duplicate_pairs": 0,
|
||||
"excluded_fragment_edges_from_phase3": 445448,
|
||||
"excluded_fragment_weight_from_phase3": 1188316,
|
||||
"fragments": 44877,
|
||||
"indegree_zero": 370,
|
||||
"isolated_neurons": 217,
|
||||
"neurons": 166700,
|
||||
"neurotransmitter_coverage": 0.9989322135572886,
|
||||
"nnz": 25582938,
|
||||
"outdegree_zero": 1220,
|
||||
"raw_unique_type_strings": 11751,
|
||||
"self_loops": 101,
|
||||
"synapse_weight": 124177617,
|
||||
"weight_max": 2591,
|
||||
"weight_min": 1,
|
||||
"with_neurotransmitter": 166522,
|
||||
"with_swc": 166700,
|
||||
"with_type": 164506,
|
||||
"without_neurotransmitter": 178,
|
||||
"without_swc": 0,
|
||||
"without_type": 2194
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"flywireType": {
|
||||
"empty": 0,
|
||||
"null": 23546,
|
||||
"present": 143154,
|
||||
"unique_nonempty": 8199
|
||||
},
|
||||
"hemibrainType": {
|
||||
"empty": 0,
|
||||
"null": 133781,
|
||||
"present": 32919,
|
||||
"unique_nonempty": 4495
|
||||
},
|
||||
"mancType": {
|
||||
"empty": 0,
|
||||
"null": 143956,
|
||||
"present": 22744,
|
||||
"unique_nonempty": 3893
|
||||
},
|
||||
"receptorType": {
|
||||
"empty": 0,
|
||||
"null": 165948,
|
||||
"present": 752,
|
||||
"unique_nonempty": 3
|
||||
},
|
||||
"supertype": {
|
||||
"empty": 0,
|
||||
"null": 132604,
|
||||
"present": 34096,
|
||||
"unique_nonempty": 1925
|
||||
},
|
||||
"type": {
|
||||
"empty": 0,
|
||||
"null": 2194,
|
||||
"present": 164506,
|
||||
"unique_nonempty": 11751
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"all_raw_edges_checked_against_csr": true,
|
||||
"canonical_rule": "superclass is not null",
|
||||
"old_phase3_counts_reconciled": true,
|
||||
"raw_and_prior_artifacts_verified": true,
|
||||
"repeat_behavior": "verify input/output SHA256 and return without writes",
|
||||
"software": {
|
||||
"numpy": "2.2.6",
|
||||
"pyarrow": "25.0.1"
|
||||
},
|
||||
"type_filter": false,
|
||||
"weights_exact": true
|
||||
}
|
||||
Reference in New Issue
Block a user