Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

2026-09-29 03:38:32 +08:00
commit 3981628ff8
682 changed files with 161475 additions and 0 deletions
@@ -0,0 +1,9 @@
+# Phase 3B stopped at count gate
+
+Rule: non-null, non-blank superclass; original values are not rewritten.
+Actual neurons: 166700; expected: 166691; difference: +9. Fragments: 44877.
+Primary type: 11751 unique non-blank values; expected: 11691; difference: +60; null rows: 2194.
+No neuron index, unified metadata, connectivity or CSR was generated.
+All type-like columns are reported separately. No type fallback or subclass exclusions were applied.
+The nine differing body IDs cannot be identified without a paper-matched reference ID list.
+To reproduce, run the archived script with the dataset processed/.venv/bin/python; choose a fresh OUT directory because the script refuses existing audit contents.
@@ -0,0 +1,40 @@
+assignedOlHex1: double
+assignedOlHex2: double
+bodyId: int64
+flywireType: string
+group: double
+instance: string
+somaSide: string
+statusLabel: dictionary<values=string, indices=int8, ordered=1>
+superclass: string
+type: string
+vfbId: string
+hemibrainType: string
+itoleeHl: string
+supertype: string
+birthtime: string
+mancBodyid: double
+mancGroup: double
+mancType: string
+subclass: string
+synonyms: string
+class: string
+rootSide: string
+somaNeuromere: string
+trumanHl: string
+dimorphism: string
+matchingNotes: string
+entryNerve: string
+mancSerial: double
+mcnsSerial: double
+serialMotif: string
+fruDsx: string
+exitNerve: string
+receptorType: string
+somaLocation: list<item: int64>
+  child 0, item: int64
+tosomaLocation: list<item: int64>
+  child 0, item: int64
+status: string
+-- schema metadata --
+pandas: '{"index_columns": [{"kind": "range", "name": null, "start": 0, "' + 4583
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Read-only source audit; stop Phase 3B if its neuron-count gate fails."""
+import collections
+import hashlib
+import json
+import platform
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.feather as feather
+import pyarrow.parquet as pq
+
+ROOT = Path('/data/digital-drosophila/datasets/male-cns-v1.0')
+OUT = ROOT / 'processed/neuron-only/audit'
+
+
+def digest(path):
+    h = hashlib.sha256()
+    with path.open('rb') as f:
+        for block in iter(lambda: f.read(8 * 1024 * 1024), b''):
+            h.update(block)
+    return h.hexdigest()
+
+
+def main():
+    OUT.mkdir(parents=True, exist_ok=True)
+    if any(OUT.iterdir()):
+        raise RuntimeError('Refusing to overwrite an existing audit; use a fresh audit directory.')
+    source = ROOT / 'raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather'
+    before = digest(source)
+    table = feather.read_table(source)
+    fields = [n for n in table.column_names if n.casefold() == 'superclass']
+    if len(fields) != 1:
+        raise RuntimeError(f'Ambiguous superclass field: {fields}')
+    name = fields[0]
+    if not pa.types.is_string(table.schema.field(name).type):
+        raise RuntimeError(f'Uninspected superclass dtype: {table.schema.field(name).type}')
+    values = table[name].to_pylist()
+    counts = collections.Counter(values)
+    valid = [v is not None and bool(v.strip()) for v in values]
+    neurons = table.filter(pa.array(valid))
+    types = {}
+    for field in table.schema:
+        if 'type' not in field.name.casefold():
+            continue
+        vals = neurons[field.name].to_pylist()
+        good = [v for v in vals if v is not None and (not isinstance(v, str) or bool(v.strip()))]
+        freq = collections.Counter(good)
+        types[field.name] = dict(dtype=str(field.type), nulls=sum(v is None for v in vals),
+                                blank_strings=sum(isinstance(v, str) and not v.strip() for v in vals),
+                                populated_rows=len(good), unique_valid_values=len(freq))
+        (OUT / f'{field.name}-value-counts.json').write_text(json.dumps(freq, indent=2, sort_keys=True)+'\n')
+    # This is a classification audit, not a new neuron index or metadata product.
+    pq.write_table(pa.table({'body_id': table['bodyId'].cast(pa.uint64()),
+                            'superclass': table[name], 'passes_superclass_rule': valid}),
+                   OUT / 'annotation-classification-audit.parquet', compression='zstd')
+    report = dict(status='BLOCKED_NEURON_COUNT_MISMATCH', annotation_rows=table.num_rows,
+                  superclass_field=name, superclass_dtype=str(table.schema.field(name).type),
+                  actual_neurons=neurons.num_rows, expected_neurons=166691,
+                  neuron_difference=neurons.num_rows-166691, fragments=len(values)-neurons.num_rows,
+                  null_count=counts[None], empty_string_count=counts[''],
+                  whitespace_only_count=sum(isinstance(v,str) and v!='' and not v.strip() for v in values),
+                  nonnull_values_with_surrounding_whitespace=sum(isinstance(v,str) and v!=v.strip() for v in values),
+                  superclass_value_counts={str(k): v for k,v in counts.items()},
+                  type_fields=types, expected_type_count=11691,
+                  actual_primary_type_count=types['type']['unique_valid_values'],
+                  type_difference=types['type']['unique_valid_values']-11691,
+                  expected_counts_source='User-specified formal paper reference counts',
+                  reason='166700 != 166691: stopped before generating mappings, unified metadata or CSR.',
+                  specific_nine_ids='Undetermined: a paper-matched reference ID set is required; count alone cannot identify the nine IDs.',
+                  software=dict(python=platform.python_version(),pyarrow=pa.__version__))
+    (OUT/'difference-audit.json').write_text(json.dumps(report,indent=2,sort_keys=True)+'\n')
+    (OUT/'annotation-schema.txt').write_text(str(table.schema)+'\n')
+    (OUT/'README.md').write_text(
+        '# Phase 3B stopped at count gate\n\n'
+        'Rule: non-null, non-blank superclass; original values are not rewritten.\n'
+        'Actual neurons: 166700; expected: 166691; difference: +9. Fragments: 44877.\n'
+        'Primary type: 11751 unique non-blank values; expected: 11691; difference: +60; null rows: 2194.\n'
+        'No neuron index, unified metadata, connectivity or CSR was generated.\n'
+        'All type-like columns are reported separately. No type fallback or subclass exclusions were applied.\n'
+        'The nine differing body IDs cannot be identified without a paper-matched reference ID list.\n'
+        'To reproduce, run the archived script with the dataset processed/.venv/bin/python; '
+        'choose a fresh OUT directory because the script refuses existing audit contents.\n')
+    (OUT/'audit-malecns-phase3b.py').write_bytes(Path(__file__).read_bytes())
+    assert digest(source)==before
+    inventory=[dict(path=p.name,bytes=p.stat().st_size,sha256=digest(p)) for p in sorted(OUT.iterdir()) if p.is_file()]
+    manifest=dict(status=report['status'], source=str(source),source_sha256=before,
+                  source_unchanged_after_audit=True,outputs=inventory,
+                  downstream_generation_performed=False)
+    (OUT/'manifest.json').write_text(json.dumps(manifest,indent=2,sort_keys=True)+'\n')
+    (OUT/'checksums.sha256').write_text(''.join(f'{digest(p)}  {p.name}\n' for p in sorted(OUT.iterdir()) if p.is_file()))
+    print(json.dumps(report,indent=2))
+
+
+if __name__=='__main__':
+    main()
@@ -0,0 +1,12 @@
+deb42f7727a38276921d5bf93043906f8edf26501fc65dc3ecf948cacc668d7e  README.md
+2c7ba16d2a532be4e7ded3d651758db55b31b4ab0958926194fa85aeccbc49d8  annotation-classification-audit.parquet
+4385079022f1ee050c95bc602367da8c0f5f36c74749babc434e230c370ddb6e  annotation-schema.txt
+8fd5159cc0fc5be179901423fd0b36ec17af06818a7891f019bbf927a0c025b1  audit-malecns-phase3b.py
+5faf964f64ae433bfbce1ad63551c3caf3503d1f0a53e9c2bbdb63df5e33064f  difference-audit.json
+f063b464479d68802335cd0049df88a989c95fc54bf6b840308b7addabae5e5a  flywireType-value-counts.json
+28e261d43676610a6c53e49dce1db9bb974d1211712fd70da619e571b4bfa61b  hemibrainType-value-counts.json
+1233ef5ab606b89156266c8bd9f239b4dd7fbce02601f59290e04802c951dcb0  mancType-value-counts.json
+c6bd94818c1a231e8b353d6b10da5f0b99b2c125c27dac3e31a5023effa92ee2  manifest.json
+d868b3cc489659d1d376f5ea37b1408eb2ce6345e878fa4de0765f04b4e1d00c  receptorType-value-counts.json
+5f5460baafeb51e303c71d16bae2bd049cac28bbde89bd028a100e1b06944c09  supertype-value-counts.json
+1cb76d53dcfdd6f0624714119e22a7fa3f65bc57723f6d39797df1ca336775e0  type-value-counts.json
@@ -0,0 +1,98 @@
+{
+  "actual_neurons": 166700,
+  "actual_primary_type_count": 11751,
+  "annotation_rows": 211577,
+  "empty_string_count": 0,
+  "expected_counts_source": "User-specified formal paper reference counts",
+  "expected_neurons": 166691,
+  "expected_type_count": 11691,
+  "fragments": 44877,
+  "neuron_difference": 9,
+  "nonnull_values_with_surrounding_whitespace": 0,
+  "null_count": 44877,
+  "reason": "166700 != 166691: stopped before generating mappings, unified metadata or CSR.",
+  "software": {
+    "pyarrow": "25.0.1",
+    "python": "3.10.12"
+  },
+  "specific_nine_ids": "Undetermined: a paper-matched reference ID set is required; count alone cannot identify the nine IDs.",
+  "status": "BLOCKED_NEURON_COUNT_MISMATCH",
+  "superclass_dtype": "string",
+  "superclass_field": "superclass",
+  "superclass_value_counts": {
+    "ENS": 50,
+    "None": 44877,
+    "ascending_neuron": 1846,
+    "cb_efferent": 4,
+    "cb_endocrine": 72,
+    "cb_intrinsic": 32164,
+    "cb_motor": 107,
+    "cb_sensory": 4868,
+    "cb_sensory_tbc": 14,
+    "descending_neuron": 1314,
+    "descending_neuron_tbc": 2,
+    "efferent_ascending": 8,
+    "efferent_descending": 4,
+    "ol_intrinsic": 89403,
+    "ol_sensory": 6098,
+    "sensory_ascending": 537,
+    "sensory_ascending_tbc": 2,
+    "sensory_descending": 12,
+    "visual_centrifugal": 563,
+    "visual_projection": 9201,
+    "visual_projection_tbc": 2,
+    "vnc_efferent": 94,
+    "vnc_endocrine": 22,
+    "vnc_intrinsic": 13161,
+    "vnc_motor": 708,
+    "vnc_sensory": 6370,
+    "vnc_sensory_tbc": 36,
+    "vnc_tbc": 38
+  },
+  "type_difference": 60,
+  "type_fields": {
+    "flywireType": {
+      "blank_strings": 0,
+      "dtype": "string",
+      "nulls": 23546,
+      "populated_rows": 143154,
+      "unique_valid_values": 8199
+    },
+    "hemibrainType": {
+      "blank_strings": 0,
+      "dtype": "string",
+      "nulls": 133781,
+      "populated_rows": 32919,
+      "unique_valid_values": 4495
+    },
+    "mancType": {
+      "blank_strings": 0,
+      "dtype": "string",
+      "nulls": 143956,
+      "populated_rows": 22744,
+      "unique_valid_values": 3893
+    },
+    "receptorType": {
+      "blank_strings": 0,
+      "dtype": "string",
+      "nulls": 165948,
+      "populated_rows": 752,
+      "unique_valid_values": 3
+    },
+    "supertype": {
+      "blank_strings": 0,
+      "dtype": "string",
+      "nulls": 132604,
+      "populated_rows": 34096,
+      "unique_valid_values": 1925
+    },
+    "type": {
+      "blank_strings": 0,
+      "dtype": "string",
+      "nulls": 2194,
+      "populated_rows": 164506,
+      "unique_valid_values": 11751
+    }
+  },
+  "whitespace_only_count": 0
+}
@@ -0,0 +1,64 @@
+{
+  "downstream_generation_performed": false,
+  "outputs": [
+    {
+      "bytes": 706,
+      "path": "README.md",
+      "sha256": "deb42f7727a38276921d5bf93043906f8edf26501fc65dc3ecf948cacc668d7e"
+    },
+    {
+      "bytes": 601375,
+      "path": "annotation-classification-audit.parquet",
+      "sha256": "2c7ba16d2a532be4e7ded3d651758db55b31b4ab0958926194fa85aeccbc49d8"
+    },
+    {
+      "bytes": 874,
+      "path": "annotation-schema.txt",
+      "sha256": "4385079022f1ee050c95bc602367da8c0f5f36c74749babc434e230c370ddb6e"
+    },
+    {
+      "bytes": 5611,
+      "path": "audit-malecns-phase3b.py",
+      "sha256": "8fd5159cc0fc5be179901423fd0b36ec17af06818a7891f019bbf927a0c025b1"
+    },
+    {
+      "bytes": 2628,
+      "path": "difference-audit.json",
+      "sha256": "5faf964f64ae433bfbce1ad63551c3caf3503d1f0a53e9c2bbdb63df5e33064f"
+    },
+    {
+      "bytes": 130369,
+      "path": "flywireType-value-counts.json",
+      "sha256": "f063b464479d68802335cd0049df88a989c95fc54bf6b840308b7addabae5e5a"
+    },
+    {
+      "bytes": 70934,
+      "path": "hemibrainType-value-counts.json",
+      "sha256": "28e261d43676610a6c53e49dce1db9bb974d1211712fd70da619e571b4bfa61b"
+    },
+    {
+      "bytes": 66152,
+      "path": "mancType-value-counts.json",
+      "sha256": "1233ef5ab606b89156266c8bd9f239b4dd7fbce02601f59290e04802c951dcb0"
+    },
+    {
+      "bytes": 78,
+      "path": "receptorType-value-counts.json",
+      "sha256": "d868b3cc489659d1d376f5ea37b1408eb2ce6345e878fa4de0765f04b4e1d00c"
+    },
+    {
+      "bytes": 27895,
+      "path": "supertype-value-counts.json",
+      "sha256": "5f5460baafeb51e303c71d16bae2bd049cac28bbde89bd028a100e1b06944c09"
+    },
+    {
+      "bytes": 188083,
+      "path": "type-value-counts.json",
+      "sha256": "1cb76d53dcfdd6f0624714119e22a7fa3f65bc57723f6d39797df1ca336775e0"
+    }
+  ],
+  "source": "/data/digital-drosophila/datasets/male-cns-v1.0/raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather",
+  "source_sha256": "2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2",
+  "source_unchanged_after_audit": true,
+  "status": "BLOCKED_NEURON_COUNT_MISMATCH"
+}
@@ -0,0 +1,10 @@
+Phase 3B: superclass != null; 166700 neurons including null type.
+Raw type strings preserved; 11710 paper types are reference only.
+Original annotation and NT fields use annotation_ and nt_ prefixes, with original dtypes and nulls.
+NT coverage uses nonempty consensus_nt, without inferred fallback.
+Mapping is body_id ascending; reverse lookup uses searchsorted with exact equality check.
+CSR rows are presynaptic sources; columns are postsynaptic targets.
+Fragment comparison excludes each edge once if either endpoint is a fragment.
+Nine category totals separate annotation fragments from other unannotated segments.
+Re-run using processed/.venv/bin/python processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py; completed runs verify and do not overwrite.
+Historical audit in parent audit/ remains unchanged.
@@ -0,0 +1,26 @@
+cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff  audit/phase3b-166700/README.md
+beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583  audit/phase3b-166700/edge-category-audit.json
+bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702  audit/phase3b-166700/excluded-fragment-annotations.parquet
+c061452f341d190946f80ca96b4589ab1e2a8dca6a67c57ad5d14f6ce34b1b64  audit/phase3b-166700/manifest.json
+b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234  audit/phase3b-166700/missing-neurotransmitter-ids.parquet
+4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e  audit/phase3b-166700/missing-swc-ids.parquet
+b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b  audit/phase3b-166700/missing-type-ids.parquet
+3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97  audit/phase3b-166700/phase3b-neuron-only.py
+f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6  audit/phase3b-166700/schema.json
+f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd  audit/phase3b-166700/statistics.json
+497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a  audit/phase3b-166700/type-field-audit.json
+598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1  audit/phase3b-166700/validation.json
+a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f  connectivity/canonical-edges.parquet
+5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b  connectivity/degrees.parquet
+ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150  gpu/csr-indices.int32.npy
+fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4  gpu/csr-indptr.int64.npy
+3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8  gpu/csr-metadata.json
+c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b  gpu/csr-weights.int64.npy
+5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242  neuron-index/body-id-neuron-idx.feather
+2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009  neuron-index/body-id-neuron-idx.parquet
+c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206  neuron-index/body-id-to-neuron-idx.npz
+ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6  neuron-index/neuron-idx-to-body-id.uint64.npy
+ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1  neuron-index/unified-metadata.feather
+922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63  neuron-index/unified-metadata.parquet
+4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8  phase3b-repeat.log
+b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e  phase3b-run.log
@@ -0,0 +1,38 @@
+{
+  "fragment->fragment": {
+    "edges": 14135,
+    "weight": 49566
+  },
+  "fragment->neuron": {
+    "edges": 249969,
+    "weight": 661930
+  },
+  "fragment->other": {
+    "edges": 766887,
+    "weight": 1131611
+  },
+  "neuron->fragment": {
+    "edges": 181344,
+    "weight": 476820
+  },
+  "neuron->neuron": {
+    "edges": 25582938,
+    "weight": 124177617
+  },
+  "neuron->other": {
+    "edges": 112452244,
+    "weight": 170414577
+  },
+  "other->fragment": {
+    "edges": 78570,
+    "weight": 107066
+  },
+  "other->neuron": {
+    "edges": 4552783,
+    "weight": 5614376
+  },
+  "other->other": {
+    "edges": 7977814,
+    "weight": 9199680
+  }
+}
@@ -0,0 +1,151 @@
+{
+  "canonical_rule": "superclass != null",
+  "disk_reload_check": "PASS: all original annotation/NT fields exact and CSR dtype/range/count checks passed",
+  "inputs": [
+    {
+      "bytes": 14483314,
+      "path": "raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather",
+      "sha256": "2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2"
+    },
+    {
+      "bytes": 43282834,
+      "path": "raw/annotations/body-neurotransmitters-male-cns-v1.0.feather",
+      "sha256": "95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621"
+    },
+    {
+      "bytes": 1051241946,
+      "path": "raw/connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather",
+      "sha256": "e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1"
+    }
+  ],
+  "outputs": [
+    {
+      "bytes": 823,
+      "path": "processed/neuron-only/audit/phase3b-166700/README.md",
+      "sha256": "cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff"
+    },
+    {
+      "bytes": 643,
+      "path": "processed/neuron-only/audit/phase3b-166700/edge-category-audit.json",
+      "sha256": "beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583"
+    },
+    {
+      "bytes": 261095,
+      "path": "processed/neuron-only/audit/phase3b-166700/excluded-fragment-annotations.parquet",
+      "sha256": "bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702"
+    },
+    {
+      "bytes": 1350,
+      "path": "processed/neuron-only/audit/phase3b-166700/missing-neurotransmitter-ids.parquet",
+      "sha256": "b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234"
+    },
+    {
+      "bytes": 364,
+      "path": "processed/neuron-only/audit/phase3b-166700/missing-swc-ids.parquet",
+      "sha256": "4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e"
+    },
+    {
+      "bytes": 8105,
+      "path": "processed/neuron-only/audit/phase3b-166700/missing-type-ids.parquet",
+      "sha256": "b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b"
+    },
+    {
+      "bytes": 13055,
+      "path": "processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py",
+      "sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
+    },
+    {
+      "bytes": 3383,
+      "path": "processed/neuron-only/audit/phase3b-166700/schema.json",
+      "sha256": "f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6"
+    },
+    {
+      "bytes": 753,
+      "path": "processed/neuron-only/audit/phase3b-166700/statistics.json",
+      "sha256": "f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd"
+    },
+    {
+      "bytes": 649,
+      "path": "processed/neuron-only/audit/phase3b-166700/type-field-audit.json",
+      "sha256": "497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a"
+    },
+    {
+      "bytes": 371,
+      "path": "processed/neuron-only/audit/phase3b-166700/validation.json",
+      "sha256": "598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1"
+    },
+    {
+      "bytes": 137157039,
+      "path": "processed/neuron-only/connectivity/canonical-edges.parquet",
+      "sha256": "a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f"
+    },
+    {
+      "bytes": 1665145,
+      "path": "processed/neuron-only/connectivity/degrees.parquet",
+      "sha256": "5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b"
+    },
+    {
+      "bytes": 102331880,
+      "path": "processed/neuron-only/gpu/csr-indices.int32.npy",
+      "sha256": "ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150"
+    },
+    {
+      "bytes": 1333736,
+      "path": "processed/neuron-only/gpu/csr-indptr.int64.npy",
+      "sha256": "fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4"
+    },
+    {
+      "bytes": 361,
+      "path": "processed/neuron-only/gpu/csr-metadata.json",
+      "sha256": "3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8"
+    },
+    {
+      "bytes": 204663632,
+      "path": "processed/neuron-only/gpu/csr-weights.int64.npy",
+      "sha256": "c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b"
+    },
+    {
+      "bytes": 610146,
+      "path": "processed/neuron-only/neuron-index/body-id-neuron-idx.feather",
+      "sha256": "5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242"
+    },
+    {
+      "bytes": 1197954,
+      "path": "processed/neuron-only/neuron-index/body-id-neuron-idx.parquet",
+      "sha256": "2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009"
+    },
+    {
+      "bytes": 2000920,
+      "path": "processed/neuron-only/neuron-index/body-id-to-neuron-idx.npz",
+      "sha256": "c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206"
+    },
+    {
+      "bytes": 1333728,
+      "path": "processed/neuron-only/neuron-index/neuron-idx-to-body-id.uint64.npy",
+      "sha256": "ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6"
+    },
+    {
+      "bytes": 16822034,
+      "path": "processed/neuron-only/neuron-index/unified-metadata.feather",
+      "sha256": "ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1"
+    },
+    {
+      "bytes": 9283041,
+      "path": "processed/neuron-only/neuron-index/unified-metadata.parquet",
+      "sha256": "922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63"
+    },
+    {
+      "bytes": 929,
+      "path": "processed/neuron-only/phase3b-run.log",
+      "sha256": "b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e"
+    },
+    {
+      "bytes": 106,
+      "path": "processed/neuron-only/phase3b-repeat.log",
+      "sha256": "4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8"
+    }
+  ],
+  "phase": "3B",
+  "repeat_run_check": "PASS: full inputs/output checksums verified without overwrite",
+  "script_sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
+}
@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""Deterministic neuron-only preprocessing. Existing completed outputs are verified, never replaced."""
+import argparse
+import collections
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import time
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.feather as ft
+import pyarrow.parquet as pq
+
+
+def sha(p):
+    h = hashlib.sha256()
+    with p.open('rb') as f:
+        for b in iter(lambda: f.read(8388608), b''):
+            h.update(b)
+    return h.hexdigest()
+
+
+def js(p, obj):
+    p.write_text(json.dumps(obj, indent=2, sort_keys=True)+'\n')
+
+
+def member(ids, q):
+    pos = np.searchsorted(ids, q)
+    return pos, (pos < len(ids)) & (ids[np.minimum(pos, len(ids)-1)] == q)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--root', default='/data/digital-drosophila/datasets/male-cns-v1.0')
+    args = ap.parse_args()
+    root = Path(args.root)
+    raw = root/'raw'
+    out = root/'processed/neuron-only'
+    audit_name = 'phase3b-166700'
+    final_audit = out/'audit'/audit_name
+    def checks():
+        subprocess.run(['md5sum','--quiet','-c','_manifests/checksums.md5'],cwd=raw,check=True)
+        subprocess.run(['md5sum','--quiet','-c','../_manifests/skeletons/checksums.md5'],cwd=raw/'skeletons',check=True)
+        subprocess.run(['sha256sum','--quiet','-c','audit/checksums.sha256'],cwd=root/'processed',check=True)
+        subprocess.run(['sha256sum','--quiet','-c','checksums.sha256'],cwd=out/'audit',check=True)
+    if (final_audit/'manifest.json').exists():
+        checks()
+        manifest=json.loads((final_audit/'manifest.json').read_text())
+        for item in manifest['inputs']+manifest['outputs']:
+            assert sha(root/item['path']) == item['sha256'], item['path']
+        assert sha(Path(__file__)) == manifest['script_sha256']
+        subprocess.run(['sha256sum','--quiet','-c',str(final_audit/'checksums.sha256')],cwd=out,check=True)
+        print('REPEAT_RUN_OK: all inputs, existing Phase3, historical audit and Phase3B outputs unchanged; no overwrite.',flush=True)
+        return
+    for d in ['neuron-index','connectivity','gpu']:
+        if (out/d).exists():
+            raise RuntimeError(f'Refusing existing destination {out/d}')
+    assert not final_audit.exists()
+    print('Verifying raw and preserved Phase3/historical audit',flush=True)
+    checks()
+    stage=out/f'.staging-{int(time.time())}'
+    for d in ['neuron-index','connectivity','gpu',f'audit/{audit_name}']:
+        (stage/d).mkdir(parents=True)
+    audit=stage/'audit'/audit_name
+    paths=[raw/'annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather',raw/'annotations/body-neurotransmitters-male-cns-v1.0.feather',raw/'connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather']
+    inputs=[dict(path=str(p.relative_to(root)),sha256=sha(p),bytes=p.stat().st_size) for p in paths]
+    a,nt,c=[ft.read_table(p) for p in paths]
+    schemas={k:str(t.schema) for k,t in zip(['annotation','neurotransmitter','connectome'],[a,nt,c])}
+    assert a.schema.field('superclass').type == pa.string()
+    assert set(c.column_names)=={'body_pre','body_post','weight'}
+    def ids(t,k):
+        assert t[k].null_count==0 and pa.types.is_integer(t[k].type)
+        arr=t[k].to_numpy();assert np.all(arr>=0)
+        return arr.astype(np.uint64)
+    aid=ids(a,'bodyId');nid=ids(nt,'body')
+    assert len(np.unique(aid))==len(aid) and len(np.unique(nid))==len(nid)
+    mask=pc.is_valid(a['superclass'])
+    neurons=a.filter(mask);fragments=a.filter(pc.invert(mask))
+    assert neurons.num_rows==166700 and fragments.num_rows==44877
+    neurons=neurons.sort_by([('bodyId','ascending')]);canon=ids(neurons,'bodyId')
+    frag=np.sort(ids(fragments,'bodyId'));allids=np.sort(aid);n=len(canon)
+    pq.write_table(fragments,audit/'excluded-fragment-annotations.parquet',compression='zstd')
+    swc=np.array(sorted(int(p.stem) for p in (raw/'skeletons').glob('*.swc')),dtype=np.uint64)
+    has_swc=member(swc,canon)[1]
+    ni=pc.index_in(pa.array(canon),value_set=nt['body'].cast(pa.uint64()))
+    aligned=nt.take(ni)
+    md={'neuron_idx':pa.array(np.arange(n,dtype=np.int32)),'body_id':pa.array(canon),
+        'neuron_type':neurons['type'],'neuron_class':neurons['class'],'side':neurons['somaSide'],
+        'neurotransmitter':aligned['consensus_nt'],'has_swc':pa.array(has_swc),
+        'swc_path':pa.array([str(raw/'skeletons'/f'{int(b)}.swc') if ok else None for b,ok in zip(canon,has_swc)])}
+    md.update({'annotation_'+k:neurons[k] for k in neurons.column_names})
+    md.update({'nt_'+k:aligned[k] for k in aligned.column_names})
+    metadata=pa.table(md)
+    mapping=metadata.select(['neuron_idx','body_id'])
+    for name,t in [('unified-metadata',metadata),('body-id-neuron-idx',mapping)]:
+        pq.write_table(t,stage/'neuron-index'/f'{name}.parquet',compression='zstd')
+        ft.write_feather(t,stage/'neuron-index'/f'{name}.feather',compression='zstd')
+    np.save(stage/'neuron-index/neuron-idx-to-body-id.uint64.npy',canon)
+    np.savez(stage/'neuron-index/body-id-to-neuron-idx.npz',body_id=canon,neuron_idx=np.arange(n,dtype=np.int32))
+    type_stats={}
+    for k in neurons.column_names:
+        if 'type' in k.lower():
+            v=neurons[k].to_pylist();good=[x for x in v if x is not None and x!='']
+            type_stats[k]=dict(null=sum(x is None for x in v),empty=sum(x=='' for x in v),unique_nonempty=len(set(good)),present=len(good))
+    ntv=aligned['consensus_nt'].to_pylist();hasnt=np.array([x is not None and x!='' for x in ntv])
+    for name,present in [('swc',has_swc),('neurotransmitter',hasnt),('type',np.array([x is not None and x!='' for x in neurons['type'].to_pylist()]))]:
+        pq.write_table(pa.table({'body_id':canon[~present]}),audit/f'missing-{name}-ids.parquet',compression='zstd')
+    print('Filtering original connectome and auditing fragment edges',flush=True)
+    src_chunks=[];dst_chunks=[];w_chunks=[]
+    categories={f'{s}->{t}':{'edges':0,'weight':0} for s in ['neuron','fragment','other'] for t in ['neuron','fragment','other']}
+    for batch in c.to_batches(max_chunksize=1000000):
+        pre=ids(pa.Table.from_batches([batch]),'body_pre');post=ids(pa.Table.from_batches([batch]),'body_post')
+        w=batch.column(batch.schema.get_field_index('weight')).to_numpy()
+        assert w.dtype==np.int64 and np.all(w>0)
+        si,sm=member(canon,pre);ti,tm=member(canon,post)
+        sf=member(frag,pre)[1];tf=member(frag,post)[1]
+        for sk,sv in [('neuron',sm),('fragment',sf),('other',~(sm|sf))]:
+            for tk,tv in [('neuron',tm),('fragment',tf),('other',~(tm|tf))]:
+                keep=sv&tv;z=categories[f'{sk}->{tk}'];z['edges']+=int(keep.sum());z['weight']+=int(w[keep].sum())
+        keep=sm&tm
+        src_chunks.append(si[keep].astype(np.int32));dst_chunks.append(ti[keep].astype(np.int32));w_chunks.append(w[keep])
+    s=np.concatenate(src_chunks);t=np.concatenate(dst_chunks);w=np.concatenate(w_chunks)
+    order=np.lexsort((t,s));s=s[order];t=t[order];w=w[order]
+    duplicates=int(np.sum((s[1:]==s[:-1])&(t[1:]==t[:-1])))
+    indeg=np.bincount(t,minlength=n);outdeg=np.bincount(s,minlength=n)
+    ptr=np.r_[np.int64(0),np.cumsum(outdeg,dtype=np.int64)]
+    for filename,x in [('csr-indptr.int64',ptr),('csr-indices.int32',t),('csr-weights.int64',w)]:
+        np.save(stage/'gpu'/f'{filename}.npy',x)
+    edges=pa.table({'source_neuron_idx':s,'target_neuron_idx':t,'body_pre':canon[s],'body_post':canon[t],'weight':w})
+    pq.write_table(edges,stage/'connectivity/canonical-edges.parquet',compression='zstd')
+    pq.write_table(pa.table({'body_id':canon,'neuron_idx':np.arange(n,dtype=np.int32),'indegree':indeg,'outdegree':outdeg}),stage/'connectivity/degrees.parquet',compression='zstd')
+    old_edges=sum(categories[k]['edges'] for k in ['neuron->neuron','neuron->fragment','fragment->neuron','fragment->fragment'])
+    old_weight=sum(categories[k]['weight'] for k in ['neuron->neuron','neuron->fragment','fragment->neuron','fragment->fragment'])
+    old=json.loads((root/'processed/connectivity/connectivity-stats.json').read_text())
+    assert old_edges==old['canonical_edge_rows'] and old_weight==old['canonical_weight_sum']
+    print('Independent full raw edge lookup validation',flush=True)
+    keys=s.astype(np.int64)*n+t;seen=np.zeros(len(keys),dtype=bool)
+    for b in c.to_batches(max_chunksize=1000000):
+        bt=pa.Table.from_batches([b]);pre=ids(bt,'body_pre');post=ids(bt,'body_post');bw=bt['weight'].to_numpy()
+        si,sm=member(canon,pre);ti,tm=member(canon,post);keep=sm&tm
+        loc=np.searchsorted(keys,si[keep]*n+ti[keep])
+        assert np.all(loc<len(keys)) and np.array_equal(keys[loc],si[keep]*n+ti[keep])
+        assert np.array_equal(w[loc],bw[keep]);seen[loc]=True
+    assert duplicates==0 and seen.all()
+    assert ptr[-1]==len(w) and ptr.dtype==np.int64 and t.dtype==np.int32
+    assert np.array_equal(np.repeat(np.arange(n,dtype=np.int32),np.diff(ptr)),s)
+    assert np.array_equal(canon[np.searchsorted(canon,aid[mask.to_numpy()])],aid[mask.to_numpy()])
+    assert sum(z['edges'] for z in categories.values())==c.num_rows
+    stats=dict(neurons=n,fragments=len(frag),with_swc=int(has_swc.sum()),without_swc=int((~has_swc).sum()),
+               with_neurotransmitter=int(hasnt.sum()),without_neurotransmitter=int((~hasnt).sum()),neurotransmitter_coverage=float(hasnt.mean()),
+               with_type=type_stats['type']['present'],without_type=n-type_stats['type']['present'],raw_unique_type_strings=type_stats['type']['unique_nonempty'],
+               nnz=len(w),synapse_weight=int(w.sum()),weight_min=int(w.min()),weight_max=int(w.max()),
+               connected_neurons=int(((indeg+outdeg)>0).sum()),isolated_neurons=int(((indeg+outdeg)==0).sum()),
+               indegree_zero=int((indeg==0).sum()),outdegree_zero=int((outdeg==0).sum()),self_loops=int((s==t).sum()),duplicate_pairs=duplicates,
+               excluded_fragment_edges_from_phase3=old_edges-len(w),excluded_fragment_weight_from_phase3=old_weight-int(w.sum()),
+               all_raw_fragment_related_edges=sum(z['edges'] for k,z in categories.items() if 'fragment' in k),
+               all_raw_fragment_related_weight=sum(z['weight'] for k,z in categories.items() if 'fragment' in k),
+               csr_bytes=sum(p.stat().st_size for p in (stage/'gpu').glob('*.npy')))
+    js(audit/'statistics.json',stats);js(audit/'edge-category-audit.json',categories);js(audit/'type-field-audit.json',type_stats)
+    schemas.update(metadata=str(metadata.schema),mapping=str(mapping.schema),edges=str(edges.schema))
+    js(audit/'schema.json',schemas)
+    js(stage/'gpu/csr-metadata.json',dict(shape=[n,n],nnz=len(w),orientation='row=presynaptic source; column=postsynaptic target; incoming drive is A.T @ source_activity',indptr='int64',indices='int32',weights='int64 original positive synapse counts',self_loops='retained',threshold='none',signed_weight_conversion=False))
+    js(audit/'validation.json',dict(all_raw_edges_checked_against_csr=True,weights_exact=True,old_phase3_counts_reconciled=True,raw_and_prior_artifacts_verified=True,canonical_rule='superclass is not null',type_filter=False,repeat_behavior='verify input/output SHA256 and return without writes',software=dict(numpy=np.__version__,pyarrow=pa.__version__)))
+    shutil.copyfile(__file__,audit/'phase3b-neuron-only.py')
+    (audit/'README.md').write_text('Phase 3B: superclass != null; 166700 neurons including null type.\nRaw type strings preserved; 11710 paper types are reference only.\nOriginal annotation and NT fields use annotation_ and nt_ prefixes, with original dtypes and nulls.\nNT coverage uses nonempty consensus_nt, without inferred fallback.\nMapping is body_id ascending; reverse lookup uses searchsorted with exact equality check.\nCSR rows are presynaptic sources; columns are postsynaptic targets.\nFragment comparison excludes each edge once if either endpoint is a fragment.\nNine category totals separate annotation fragments from other unannotated segments.\nRe-run using processed/.venv/bin/python processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py; completed runs verify and do not overwrite.\nHistorical audit in parent audit/ remains unchanged.\n')
+    checks()
+    assert all(sha(root/i['path'])==i['sha256'] for i in inputs)
+    inventory=[dict(path=str((out/p.relative_to(stage)).relative_to(root)),bytes=p.stat().st_size,sha256=sha(p)) for p in sorted(stage.rglob('*')) if p.is_file()]
+    js(audit/'manifest.json',dict(inputs=inputs,outputs=inventory,script_sha256=sha(Path(__file__)),phase='3B',canonical_rule='superclass != null'))
+    (audit/'checksums.sha256').write_text(''.join(f'{sha(p)}  {p.relative_to(stage).as_posix()}\n' for p in sorted(stage.rglob('*')) if p.is_file()))
+    for d in ['neuron-index','connectivity','gpu']:
+        os.rename(stage/d,out/d)
+    os.rename(audit,final_audit)
+    (stage/'audit').rmdir();stage.rmdir()
+    print(json.dumps(stats,indent=2),flush=True)
+
+
+if __name__=='__main__':
+    main()
@@ -0,0 +1,8 @@
+{
+  "annotation": "assignedOlHex1: double\nassignedOlHex2: double\nbodyId: int64\nflywireType: string\ngroup: double\ninstance: string\nsomaSide: string\nstatusLabel: dictionary<values=string, indices=int8, ordered=1>\nsuperclass: string\ntype: string\nvfbId: string\nhemibrainType: string\nitoleeHl: string\nsupertype: string\nbirthtime: string\nmancBodyid: double\nmancGroup: double\nmancType: string\nsubclass: string\nsynonyms: string\nclass: string\nrootSide: string\nsomaNeuromere: string\ntrumanHl: string\ndimorphism: string\nmatchingNotes: string\nentryNerve: string\nmancSerial: double\nmcnsSerial: double\nserialMotif: string\nfruDsx: string\nexitNerve: string\nreceptorType: string\nsomaLocation: list<item: int64>\n  child 0, item: int64\ntosomaLocation: list<item: int64>\n  child 0, item: int64\nstatus: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 4583",
+  "connectome": "body_pre: int64\nbody_post: int64\nweight: int64\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 604",
+  "edges": "source_neuron_idx: int32\ntarget_neuron_idx: int32\nbody_pre: uint64\nbody_post: uint64\nweight: int64",
+  "mapping": "neuron_idx: int32\nbody_id: uint64",
+  "metadata": "neuron_idx: int32\nbody_id: uint64\nneuron_type: string\nneuron_class: string\nside: string\nneurotransmitter: string\nhas_swc: bool\nswc_path: string\nannotation_assignedOlHex1: double\nannotation_assignedOlHex2: double\nannotation_bodyId: int64\nannotation_flywireType: string\nannotation_group: double\nannotation_instance: string\nannotation_somaSide: string\nannotation_statusLabel: dictionary<values=string, indices=int8, ordered=1>\nannotation_superclass: string\nannotation_type: string\nannotation_vfbId: string\nannotation_hemibrainType: string\nannotation_itoleeHl: string\nannotation_supertype: string\nannotation_birthtime: string\nannotation_mancBodyid: double\nannotation_mancGroup: double\nannotation_mancType: string\nannotation_subclass: string\nannotation_synonyms: string\nannotation_class: string\nannotation_rootSide: string\nannotation_somaNeuromere: string\nannotation_trumanHl: string\nannotation_dimorphism: string\nannotation_matchingNotes: string\nannotation_entryNerve: string\nannotation_mancSerial: double\nannotation_mcnsSerial: double\nannotation_serialMotif: string\nannotation_fruDsx: string\nannotation_exitNerve: string\nannotation_receptorType: string\nannotation_somaLocation: list<item: int64>\n  child 0, item: int64\nannotation_tosomaLocation: list<item: int64>\n  child 0, item: int64\nannotation_status: string\nnt_body: int64\nnt_cell_type: string\nnt_total_nt_predictions: int32\nnt_predicted_nt_confidence: double\nnt_predicted_nt: string\nnt_ground_truth: string\nnt_celltype_total_nt_predictions: int32\nnt_celltype_predicted_nt: string\nnt_celltype_predicted_nt_confidence: double\nnt_consensus_nt: string",
+  "neurotransmitter": "body: int64\ncell_type: string\ntotal_nt_predictions: int32\npredicted_nt_confidence: double\npredicted_nt: string\nground_truth: string\ncelltype_total_nt_predictions: int32\ncelltype_predicted_nt: string\ncelltype_predicted_nt_confidence: double\nconsensus_nt: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 1606"
+}
@@ -0,0 +1,27 @@
+{
+  "all_raw_fragment_related_edges": 1290905,
+  "all_raw_fragment_related_weight": 2426993,
+  "connected_neurons": 166483,
+  "csr_bytes": 308329248,
+  "duplicate_pairs": 0,
+  "excluded_fragment_edges_from_phase3": 445448,
+  "excluded_fragment_weight_from_phase3": 1188316,
+  "fragments": 44877,
+  "indegree_zero": 370,
+  "isolated_neurons": 217,
+  "neurons": 166700,
+  "neurotransmitter_coverage": 0.9989322135572886,
+  "nnz": 25582938,
+  "outdegree_zero": 1220,
+  "raw_unique_type_strings": 11751,
+  "self_loops": 101,
+  "synapse_weight": 124177617,
+  "weight_max": 2591,
+  "weight_min": 1,
+  "with_neurotransmitter": 166522,
+  "with_swc": 166700,
+  "with_type": 164506,
+  "without_neurotransmitter": 178,
+  "without_swc": 0,
+  "without_type": 2194
+}
@@ -0,0 +1,38 @@
+{
+  "flywireType": {
+    "empty": 0,
+    "null": 23546,
+    "present": 143154,
+    "unique_nonempty": 8199
+  },
+  "hemibrainType": {
+    "empty": 0,
+    "null": 133781,
+    "present": 32919,
+    "unique_nonempty": 4495
+  },
+  "mancType": {
+    "empty": 0,
+    "null": 143956,
+    "present": 22744,
+    "unique_nonempty": 3893
+  },
+  "receptorType": {
+    "empty": 0,
+    "null": 165948,
+    "present": 752,
+    "unique_nonempty": 3
+  },
+  "supertype": {
+    "empty": 0,
+    "null": 132604,
+    "present": 34096,
+    "unique_nonempty": 1925
+  },
+  "type": {
+    "empty": 0,
+    "null": 2194,
+    "present": 164506,
+    "unique_nonempty": 11751
+  }
+}
@@ -0,0 +1,13 @@
+{
+  "all_raw_edges_checked_against_csr": true,
+  "canonical_rule": "superclass is not null",
+  "old_phase3_counts_reconciled": true,
+  "raw_and_prior_artifacts_verified": true,
+  "repeat_behavior": "verify input/output SHA256 and return without writes",
+  "software": {
+    "numpy": "2.2.6",
+    "pyarrow": "25.0.1"
+  },
+  "type_filter": false,
+  "weights_exact": true
+}
@@ -0,0 +1,5 @@
+{
+  "putative_IR52b": 226,
+  "putative_ppk23": 269,
+  "putative_ppk25": 257
+}
@@ -0,0 +1,14 @@
+{
+  "indices": "int32",
+  "indptr": "int64",
+  "nnz": 25582938,
+  "orientation": "row=presynaptic source; column=postsynaptic target; incoming drive is A.T @ source_activity",
+  "self_loops": "retained",
+  "shape": [
+    166700,
+    166700
+  ],
+  "signed_weight_conversion": false,
+  "threshold": "none",
+  "weights": "int64 original positive synapse counts"
+}
@@ -0,0 +1,3 @@
+50a7718770c57220f160ba4f431ab89e  annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather
+3d842b12fe5c49eefade528d7dd24a1f  annotations/body-neurotransmitters-male-cns-v1.0.feather
+f30e9dcca25cfd021bf1e7b3d975599e  connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather
@@ -0,0 +1,3 @@
+2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2  annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather
+95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621  annotations/body-neurotransmitters-male-cns-v1.0.feather
+e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1  connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
+readonly BASE_URL="https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome"
+readonly LOG_DIR="${RAW_ROOT}/_logs"
+readonly MANIFEST_DIR="${RAW_ROOT}/_manifests"
+
+mkdir -p "${RAW_ROOT}/annotations" "${RAW_ROOT}/connectivity" "$LOG_DIR" "$MANIFEST_DIR"
+
+download_one() {
+    local subdir="$1"
+    local filename="$2"
+    local expected_md5="$3"
+    local destination="${RAW_ROOT}/${subdir}/${filename}"
+    local attempt
+
+    if [[ -f "$destination" ]] && printf '%s  %s\n' "$expected_md5" "$destination" | md5sum --check --status; then
+        printf '[OK] already verified: %s/%s\n' "$subdir" "$filename"
+        return 0
+    fi
+
+    for attempt in 1 2 3; do
+        if [[ -f "$destination" && ! -f "${destination}.aria2" ]]; then
+            mv -- "$destination" "${destination}.corrupt-$(date +%Y%m%dT%H%M%S)"
+        fi
+
+        printf '[DOWNLOAD] attempt %d: %s/%s\n' "$attempt" "$subdir" "$filename"
+        aria2c \
+            --continue=true \
+            --max-tries=0 \
+            --retry-wait=10 \
+            --connect-timeout=30 \
+            --timeout=60 \
+            --max-connection-per-server=8 \
+            --split=8 \
+            --min-split-size=16M \
+            --file-allocation=falloc \
+            --auto-file-renaming=false \
+            --allow-overwrite=false \
+            --check-integrity=true \
+            --console-log-level=notice \
+            --summary-interval=60 \
+            --dir="${RAW_ROOT}/${subdir}" \
+            --out="$filename" \
+            "${BASE_URL}/${filename}" \
+            >"${LOG_DIR}/${filename}.log" 2>&1
+
+        if printf '%s  %s\n' "$expected_md5" "$destination" | md5sum --check --status; then
+            printf '[VERIFIED] %s/%s\n' "$subdir" "$filename"
+            return 0
+        fi
+
+        printf '[CHECKSUM FAILED] %s/%s\n' "$subdir" "$filename" >&2
+        [[ -f "$destination" ]] && mv -- "$destination" "${destination}.corrupt-$(date +%Y%m%dT%H%M%S)"
+        [[ -f "${destination}.aria2" ]] && mv -- "${destination}.aria2" "${destination}.aria2.corrupt-$(date +%Y%m%dT%H%M%S)"
+    done
+
+    printf '[FAILED] checksum mismatch after 3 complete downloads: %s/%s\n' "$subdir" "$filename" >&2
+    return 1
+}
+
+jobs=(
+    "annotations|body-annotations-male-cns-v1.0-minconf-0.5.feather|50a7718770c57220f160ba4f431ab89e"
+    "annotations|body-neurotransmitters-male-cns-v1.0.feather|3d842b12fe5c49eefade528d7dd24a1f"
+    "connectivity|connectome-weights-male-cns-v1.0-minconf-0.5.feather|f30e9dcca25cfd021bf1e7b3d975599e"
+)
+
+pids=()
+for job in "${jobs[@]}"; do
+    IFS='|' read -r subdir filename expected_md5 <<<"$job"
+    download_one "$subdir" "$filename" "$expected_md5" &
+    pids+=("$!")
+done
+
+status=0
+for pid in "${pids[@]}"; do
+    wait "$pid" || status=1
+done
+
+if (( status != 0 )); then
+    printf '[FAILED] one or more files could not be downloaded and verified\n' >&2
+    exit 1
+fi
+
+cd "$RAW_ROOT"
+md5sum --check "${MANIFEST_DIR}/checksums.md5"
+sha256sum \
+    annotations/*.feather \
+    connectivity/*.feather \
+    >"${MANIFEST_DIR}/checksums.sha256"
+
+find annotations connectivity -maxdepth 1 -type f -name '*.feather' -printf '%P\t%s bytes\n' \
+    | sort >"${MANIFEST_DIR}/files.tsv"
+
+printf '[COMPLETE] all MaleCNS v1.0 phase-1 files passed official MD5 verification\n'
@@ -0,0 +1,3 @@
+body-annotations-male-cns-v1.0-minconf-0.5.feather	14483314 bytes
+body-neurotransmitters-male-cns-v1.0.feather	43282834 bytes
+connectome-weights-male-cns-v1.0-minconf-0.5.feather	1051241946 bytes
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
+readonly SKELETON_DIR="${RAW_ROOT}/skeletons"
+readonly MANIFEST_DIR="${RAW_ROOT}/_manifests/skeletons"
+readonly LOG_DIR="${RAW_ROOT}/_logs/skeletons"
+readonly CONCURRENCY="${MALECNS_SWC_CONCURRENCY:-32}"
+
+mkdir -p "$SKELETON_DIR" "$MANIFEST_DIR" "$LOG_DIR"
+
+for required in remote-files.tsv checksums.md5 download.aria2 remote-summary.json; do
+    if [[ ! -s "${MANIFEST_DIR}/${required}" ]]; then
+        printf 'missing manifest: %s\n' "${MANIFEST_DIR}/${required}" >&2
+        exit 2
+    fi
+done
+
+expected_count="$(jq -r '.swc_file_count' "${MANIFEST_DIR}/remote-summary.json")"
+expected_bytes="$(jq -r '.total_bytes' "${MANIFEST_DIR}/remote-summary.json")"
+printf '[REMOTE] files=%s bytes=%s\n' "$expected_count" "$expected_bytes"
+printf '[DOWNLOAD] concurrency=%s destination=%s\n' "$CONCURRENCY" "$SKELETON_DIR"
+
+aria2c \
+    --input-file="${MANIFEST_DIR}/download.aria2" \
+    --continue=true \
+    --always-resume=true \
+    --max-resume-failure-tries=20 \
+    --max-tries=0 \
+    --retry-wait=5 \
+    --connect-timeout=30 \
+    --timeout=60 \
+    --max-concurrent-downloads="$CONCURRENCY" \
+    --max-connection-per-server=1 \
+    --split=1 \
+    --file-allocation=none \
+    --auto-file-renaming=false \
+    --allow-overwrite=true \
+    --check-integrity=true \
+    --disk-cache=64M \
+    --console-log-level=notice \
+    --summary-interval=60 \
+    >"${LOG_DIR}/aria2.log" 2>&1
+
+cd "$SKELETON_DIR"
+
+actual_count="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | wc -l)"
+actual_bytes="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%s\n' | awk '{sum += $1} END {print sum + 0}')"
+empty_count="$(find . -maxdepth 1 -type f -name '*.swc' -size 0 -printf '%f\n' | wc -l)"
+partial_count="$(find . -maxdepth 1 -type f -name '*.aria2' -printf '%f\n' | wc -l)"
+
+find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | sort >"${MANIFEST_DIR}/local-files.txt"
+cut -f1 "${MANIFEST_DIR}/remote-files.tsv" | sort >"${MANIFEST_DIR}/expected-files.txt"
+comm -23 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/missing-files.txt"
+comm -13 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/unexpected-files.txt"
+missing_count="$(wc -l <"${MANIFEST_DIR}/missing-files.txt")"
+unexpected_count="$(wc -l <"${MANIFEST_DIR}/unexpected-files.txt")"
+
+printf '[LOCAL] files=%s bytes=%s empty=%s partial=%s missing=%s unexpected=%s\n' \
+    "$actual_count" "$actual_bytes" "$empty_count" "$partial_count" "$missing_count" "$unexpected_count"
+
+if [[ "$actual_count" != "$expected_count" || "$actual_bytes" != "$expected_bytes" || \
+      "$empty_count" != 0 || "$partial_count" != 0 || "$missing_count" != 0 || "$unexpected_count" != 0 ]]; then
+    printf '[FAILED] local inventory does not match the remote manifest\n' >&2
+    exit 1
+fi
+
+if ! md5sum --check --quiet "${MANIFEST_DIR}/checksums.md5" >"${LOG_DIR}/md5-verification.log" 2>&1; then
+    printf '[FAILED] one or more SWC files failed the official GCS MD5 check\n' >&2
+    exit 1
+fi
+
+printf '[COMPLETE] all SWC files match count, size, names, and official MD5 checksums\n'
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Enumerate the public MaleCNS v1.0 SWC objects and build download manifests."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+
+BUCKET = "flyem-male-cns"
+PREFIX = "v1.0/segmentation/skeletons-malecns/skeletons-swc/"
+API_URL = f"https://storage.googleapis.com/storage/v1/b/{BUCKET}/o"
+OBJECT_BASE_URL = f"https://storage.googleapis.com/{BUCKET}/"
+
+
+def fetch_json(url: str, attempts: int = 10) -> dict:
+    for attempt in range(1, attempts + 1):
+        try:
+            request = urllib.request.Request(url, headers={"User-Agent": "malecns-swc-enumerator/1.0"})
+            with urllib.request.urlopen(request, timeout=60) as response:
+                return json.load(response)
+        except Exception as exc:  # noqa: BLE001 - retries cover transient HTTP/TLS failures
+            if attempt == attempts:
+                raise
+            delay = min(60, 2 ** (attempt - 1))
+            print(f"listing attempt {attempt} failed: {exc}; retrying in {delay}s", file=sys.stderr)
+            time.sleep(delay)
+    raise RuntimeError("unreachable")
+
+
+def atomic_write(path: str, content: str) -> None:
+    temporary = f"{path}.tmp.{os.getpid()}"
+    with open(temporary, "w", encoding="utf-8", newline="\n") as handle:
+        handle.write(content)
+        handle.flush()
+        os.fsync(handle.fileno())
+    os.replace(temporary, path)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("output_dir")
+    parser.add_argument("--download-dir", required=True)
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    objects: list[tuple[str, int, str, str]] = []
+    page_token: str | None = None
+    page_count = 0
+
+    while True:
+        query = {
+            "prefix": PREFIX,
+            "maxResults": "1000",
+            "fields": "items(name,size,md5Hash),nextPageToken",
+        }
+        if page_token:
+            query["pageToken"] = page_token
+        payload = fetch_json(f"{API_URL}?{urllib.parse.urlencode(query)}")
+        page_count += 1
+
+        for item in payload.get("items", []):
+            name = item["name"]
+            if not name.endswith(".swc"):
+                continue
+            relative_name = name[len(PREFIX) :]
+            if not relative_name or "/" in relative_name:
+                raise RuntimeError(f"unexpected nested SWC object: {name}")
+            md5_hex = base64.b64decode(item["md5Hash"]).hex()
+            url = OBJECT_BASE_URL + urllib.parse.quote(name, safe="/")
+            objects.append((relative_name, int(item["size"]), md5_hex, url))
+
+        page_token = payload.get("nextPageToken")
+        if page_count % 25 == 0:
+            print(f"listed {page_count} pages, {len(objects)} SWC files", file=sys.stderr)
+        if not page_token:
+            break
+
+    objects.sort(key=lambda row: row[0])
+    names = [row[0] for row in objects]
+    if len(names) != len(set(names)):
+        raise RuntimeError("duplicate SWC basenames in remote listing")
+
+    total_bytes = sum(row[1] for row in objects)
+    listing_digest = hashlib.sha256(
+        "".join(f"{name}\t{size}\t{md5_hex}\n" for name, size, md5_hex, _ in objects).encode()
+    ).hexdigest()
+
+    files_tsv = "".join(
+        f"{name}\t{size}\t{md5_hex}\t{url}\n" for name, size, md5_hex, url in objects
+    )
+    checksums_md5 = "".join(f"{md5_hex}  {name}\n" for name, _, md5_hex, _ in objects)
+    aria2_input = "".join(
+        f"{url}\n  dir={args.download_dir}\n  out={name}\n  checksum=md5={md5_hex}\n"
+        for name, _, md5_hex, url in objects
+    )
+    summary = {
+        "bucket": BUCKET,
+        "prefix": PREFIX,
+        "swc_file_count": len(objects),
+        "total_bytes": total_bytes,
+        "listing_pages": page_count,
+        "listing_sha256": listing_digest,
+        "generated_at_unix": int(time.time()),
+    }
+
+    atomic_write(os.path.join(args.output_dir, "remote-files.tsv"), files_tsv)
+    atomic_write(os.path.join(args.output_dir, "checksums.md5"), checksums_md5)
+    atomic_write(os.path.join(args.output_dir, "download.aria2"), aria2_input)
+    atomic_write(os.path.join(args.output_dir, "remote-summary.json"), json.dumps(summary, indent=2) + "\n")
+    print(json.dumps(summary, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,9 @@
+{
+  "bucket": "flyem-male-cns",
+  "prefix": "v1.0/segmentation/skeletons-malecns/skeletons-swc/",
+  "swc_file_count": 211573,
+  "total_bytes": 9299975999,
+  "listing_pages": 212,
+  "listing_sha256": "8d81e5cf21daac97518319ed79baad52431968be6b4057f419112bc05f848942",
+  "generated_at_unix": 1789756376
+}