Archive MaleCNS research sources, provenance, bibliography and immutable data inventory
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,514 @@
|
||||
{
|
||||
"dataset": "MaleCNS v1.0",
|
||||
"formal_paper_doi": "10.1016/j.cell.2026.08.015",
|
||||
"download_base_from_original_script": "https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome",
|
||||
"download_source_evidence": "datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
|
||||
"license_status": "UNKNOWN / redistribution not asserted by this repository",
|
||||
"canonical_directory": "datasets/male-cns-v1.0/processed/neuron-only/",
|
||||
"raw_and_processed_unchanged": true,
|
||||
"schema_reads_are_not_new_scientific_validation": true,
|
||||
"key_files": [
|
||||
{
|
||||
"bytes": 706,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214152204680,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/README.md",
|
||||
"sha256": "deb42f7727a38276921d5bf93043906f8edf26501fc65dc3ecf948cacc668d7e"
|
||||
},
|
||||
{
|
||||
"bytes": 601375,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214116203640,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/annotation-classification-audit.parquet",
|
||||
"sha256": "2c7ba16d2a532be4e7ded3d651758db55b31b4ab0958926194fa85aeccbc49d8"
|
||||
},
|
||||
{
|
||||
"bytes": 874,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214152204680,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/annotation-schema.txt",
|
||||
"sha256": "4385079022f1ee050c95bc602367da8c0f5f36c74749babc434e230c370ddb6e"
|
||||
},
|
||||
{
|
||||
"bytes": 5611,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214152204680,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/audit-malecns-phase3b.py",
|
||||
"sha256": "8fd5159cc0fc5be179901423fd0b36ec17af06818a7891f019bbf927a0c025b1"
|
||||
},
|
||||
{
|
||||
"bytes": 1096,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214172205258,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/checksums.sha256",
|
||||
"sha256": "21460a3f3ab4bac73f03b88610e6af300175927affd09b9e2f2198a399e5bd57"
|
||||
},
|
||||
{
|
||||
"bytes": 2628,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214152204680,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/difference-audit.json",
|
||||
"sha256": "5faf964f64ae433bfbce1ad63551c3caf3503d1f0a53e9c2bbdb63df5e33064f"
|
||||
},
|
||||
{
|
||||
"bytes": 130369,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804213884196929,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/flywireType-value-counts.json",
|
||||
"sha256": "f063b464479d68802335cd0049df88a989c95fc54bf6b840308b7addabae5e5a"
|
||||
},
|
||||
{
|
||||
"bytes": 70934,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214004200402,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/hemibrainType-value-counts.json",
|
||||
"sha256": "28e261d43676610a6c53e49dce1db9bb974d1211712fd70da619e571b4bfa61b"
|
||||
},
|
||||
{
|
||||
"bytes": 66152,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214064202137,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/mancType-value-counts.json",
|
||||
"sha256": "1233ef5ab606b89156266c8bd9f239b4dd7fbce02601f59290e04802c951dcb0"
|
||||
},
|
||||
{
|
||||
"bytes": 2137,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214172205258,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/manifest.json",
|
||||
"sha256": "c6bd94818c1a231e8b353d6b10da5f0b99b2c125c27dac3e31a5023effa92ee2"
|
||||
},
|
||||
{
|
||||
"bytes": 823,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933750190697,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/README.md",
|
||||
"sha256": "cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff"
|
||||
},
|
||||
{
|
||||
"bytes": 2671,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789805079968064160,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/checksums.sha256",
|
||||
"sha256": "9c5626c48b34796f1c0bc82168c7a66dac7cb147bb8f9b1d1f201b98a91d75fd"
|
||||
},
|
||||
{
|
||||
"bytes": 643,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933746190643,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/edge-category-audit.json",
|
||||
"sha256": "beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583"
|
||||
},
|
||||
{
|
||||
"bytes": 261095,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804866681264559,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/excluded-fragment-annotations.parquet",
|
||||
"sha256": "bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702"
|
||||
},
|
||||
{
|
||||
"bytes": 5852,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789805079580059416,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/manifest.json",
|
||||
"sha256": "c061452f341d190946f80ca96b4589ab1e2a8dca6a67c57ad5d14f6ce34b1b64"
|
||||
},
|
||||
{
|
||||
"bytes": 1350,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804869033297851,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/missing-neurotransmitter-ids.parquet",
|
||||
"sha256": "b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234"
|
||||
},
|
||||
{
|
||||
"bytes": 364,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804869033297851,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/missing-swc-ids.parquet",
|
||||
"sha256": "4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e"
|
||||
},
|
||||
{
|
||||
"bytes": 8105,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804869037297908,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/missing-type-ids.parquet",
|
||||
"sha256": "b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b"
|
||||
},
|
||||
{
|
||||
"bytes": 13055,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933750190697,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py",
|
||||
"sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
|
||||
},
|
||||
{
|
||||
"bytes": 3383,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933750190697,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/schema.json",
|
||||
"sha256": "f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6"
|
||||
},
|
||||
{
|
||||
"bytes": 753,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933746190643,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/statistics.json",
|
||||
"sha256": "f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd"
|
||||
},
|
||||
{
|
||||
"bytes": 649,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933746190643,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/type-field-audit.json",
|
||||
"sha256": "497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a"
|
||||
},
|
||||
{
|
||||
"bytes": 371,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933750190697,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/validation.json",
|
||||
"sha256": "598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1"
|
||||
},
|
||||
{
|
||||
"bytes": 78,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214084202714,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/receptorType-value-counts.json",
|
||||
"sha256": "d868b3cc489659d1d376f5ea37b1408eb2ce6345e878fa4de0765f04b4e1d00c"
|
||||
},
|
||||
{
|
||||
"bytes": 27895,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804214032201211,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/supertype-value-counts.json",
|
||||
"sha256": "5f5460baafeb51e303c71d16bae2bd049cac28bbde89bd028a100e1b06944c09"
|
||||
},
|
||||
{
|
||||
"bytes": 188083,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804213964199244,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/audit/type-value-counts.json",
|
||||
"sha256": "1cb76d53dcfdd6f0624714119e22a7fa3f65bc57723f6d39797df1ca336775e0"
|
||||
},
|
||||
{
|
||||
"bytes": 137157039,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804909837865863,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/connectivity/canonical-edges.parquet",
|
||||
"sha256": "a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f"
|
||||
},
|
||||
{
|
||||
"bytes": 1665145,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804909869866302,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/connectivity/degrees.parquet",
|
||||
"sha256": "5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b"
|
||||
},
|
||||
{
|
||||
"bytes": 102331880,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804907097828272,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-indices.int32.npy",
|
||||
"sha256": "ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150"
|
||||
},
|
||||
{
|
||||
"bytes": 1333736,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804906973826570,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-indptr.int64.npy",
|
||||
"sha256": "fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4"
|
||||
},
|
||||
{
|
||||
"bytes": 361,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804933750190697,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-metadata.json",
|
||||
"sha256": "3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8"
|
||||
},
|
||||
{
|
||||
"bytes": 204663632,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804907357831842,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-weights.int64.npy",
|
||||
"sha256": "c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b"
|
||||
},
|
||||
{
|
||||
"bytes": 610146,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804868837295079,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/neuron-index/body-id-neuron-idx.feather",
|
||||
"sha256": "5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242"
|
||||
},
|
||||
{
|
||||
"bytes": 1197954,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804868833295023,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/neuron-index/body-id-neuron-idx.parquet",
|
||||
"sha256": "2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009"
|
||||
},
|
||||
{
|
||||
"bytes": 2000920,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804868845295192,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/neuron-index/body-id-to-neuron-idx.npz",
|
||||
"sha256": "c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206"
|
||||
},
|
||||
{
|
||||
"bytes": 1333728,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804868841295136,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/neuron-index/neuron-idx-to-body-id.uint64.npy",
|
||||
"sha256": "ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6"
|
||||
},
|
||||
{
|
||||
"bytes": 16822034,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804868801294570,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/neuron-index/unified-metadata.feather",
|
||||
"sha256": "ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1"
|
||||
},
|
||||
{
|
||||
"bytes": 9283041,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804868737293664,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/neuron-index/unified-metadata.parquet",
|
||||
"sha256": "922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63"
|
||||
},
|
||||
{
|
||||
"bytes": 106,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789805032299473042,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/phase3b-repeat.log",
|
||||
"sha256": "4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8"
|
||||
},
|
||||
{
|
||||
"bytes": 929,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789804957042501720,
|
||||
"path": "datasets/male-cns-v1.0/processed/neuron-only/phase3b-run.log",
|
||||
"sha256": "b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e"
|
||||
},
|
||||
{
|
||||
"bytes": 14483314,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789742734313688746,
|
||||
"path": "datasets/male-cns-v1.0/raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather",
|
||||
"sha256": "2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2"
|
||||
},
|
||||
{
|
||||
"bytes": 43282834,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789742693529813795,
|
||||
"path": "datasets/male-cns-v1.0/raw/annotations/body-neurotransmitters-male-cns-v1.0.feather",
|
||||
"sha256": "95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621"
|
||||
},
|
||||
{
|
||||
"bytes": 1051241946,
|
||||
"kind": "file",
|
||||
"mode": "0o664",
|
||||
"mtime_ns": 1789742899081279944,
|
||||
"path": "datasets/male-cns-v1.0/raw/connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather",
|
||||
"sha256": "e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1"
|
||||
}
|
||||
],
|
||||
"copied_evidence": [
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/README.md",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/README.md",
|
||||
"sha256": "deb42f7727a38276921d5bf93043906f8edf26501fc65dc3ecf948cacc668d7e"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/annotation-schema.txt",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/annotation-schema.txt",
|
||||
"sha256": "4385079022f1ee050c95bc602367da8c0f5f36c74749babc434e230c370ddb6e"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/audit-malecns-phase3b.py",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/audit-malecns-phase3b.py",
|
||||
"sha256": "8fd5159cc0fc5be179901423fd0b36ec17af06818a7891f019bbf927a0c025b1"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/checksums.sha256",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/checksums.sha256",
|
||||
"sha256": "21460a3f3ab4bac73f03b88610e6af300175927affd09b9e2f2198a399e5bd57"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/difference-audit.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/difference-audit.json",
|
||||
"sha256": "5faf964f64ae433bfbce1ad63551c3caf3503d1f0a53e9c2bbdb63df5e33064f"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/flywireType-value-counts.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/flywireType-value-counts.json",
|
||||
"sha256": "f063b464479d68802335cd0049df88a989c95fc54bf6b840308b7addabae5e5a"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/hemibrainType-value-counts.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/hemibrainType-value-counts.json",
|
||||
"sha256": "28e261d43676610a6c53e49dce1db9bb974d1211712fd70da619e571b4bfa61b"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/mancType-value-counts.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/mancType-value-counts.json",
|
||||
"sha256": "1233ef5ab606b89156266c8bd9f239b4dd7fbce02601f59290e04802c951dcb0"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/manifest.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/manifest.json",
|
||||
"sha256": "c6bd94818c1a231e8b353d6b10da5f0b99b2c125c27dac3e31a5023effa92ee2"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/README.md",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/README.md",
|
||||
"sha256": "cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/checksums.sha256",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/checksums.sha256",
|
||||
"sha256": "9c5626c48b34796f1c0bc82168c7a66dac7cb147bb8f9b1d1f201b98a91d75fd"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/edge-category-audit.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/edge-category-audit.json",
|
||||
"sha256": "beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/manifest.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/manifest.json",
|
||||
"sha256": "c061452f341d190946f80ca96b4589ab1e2a8dca6a67c57ad5d14f6ce34b1b64"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py",
|
||||
"sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/schema.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/schema.json",
|
||||
"sha256": "f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/statistics.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/statistics.json",
|
||||
"sha256": "f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/type-field-audit.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/type-field-audit.json",
|
||||
"sha256": "497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/validation.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/phase3b-166700/validation.json",
|
||||
"sha256": "598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/receptorType-value-counts.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/receptorType-value-counts.json",
|
||||
"sha256": "d868b3cc489659d1d376f5ea37b1408eb2ce6345e878fa4de0765f04b4e1d00c"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/supertype-value-counts.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/supertype-value-counts.json",
|
||||
"sha256": "5f5460baafeb51e303c71d16bae2bd049cac28bbde89bd028a100e1b06944c09"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/audit/type-value-counts.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/audit/type-value-counts.json",
|
||||
"sha256": "1cb76d53dcfdd6f0624714119e22a7fa3f65bc57723f6d39797df1ca336775e0"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-metadata.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-metadata.json",
|
||||
"sha256": "3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/checksums.md5",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/checksums.md5",
|
||||
"sha256": "6c01d768863727b474024e97661e7f2fcb19f28b06efc4cc722f92c86b1d34b3"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/checksums.sha256",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/checksums.sha256",
|
||||
"sha256": "81a144a4539069e486e1e6c7f8c066804b403f1b9cf255193d495ef4721fca8c"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
|
||||
"sha256": "0b9bd867b06b365b775e0f52efcef8390254e54e5d5459a9f0dd56acad5d362a"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/files.tsv",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/files.tsv",
|
||||
"sha256": "f89d30c4c64f0d51197426756bf038eee39afa54a9cf09d449009f092d230c4e"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/skeletons/download-malecns-v1.0-swc.sh",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/skeletons/download-malecns-v1.0-swc.sh",
|
||||
"sha256": "b105cc5db220d2e5ace5aa365419db933ac82eabb1108dc18ad3f7058ae4c647"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/skeletons/enumerate-malecns-v1.0-swc.py",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/skeletons/enumerate-malecns-v1.0-swc.py",
|
||||
"sha256": "30672f42b0541dc57cbbb122c60a37b2c5b5f37693ea037942f74cbc81289240"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/skeletons/missing-files.txt",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/skeletons/missing-files.txt",
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/skeletons/remote-summary.json",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/skeletons/remote-summary.json",
|
||||
"sha256": "c334e0ad5685be2b2b11a67d3a230cbb654a1beef1f7574b319a84675743281e"
|
||||
},
|
||||
{
|
||||
"source": "datasets/male-cns-v1.0/raw/_manifests/skeletons/unexpected-files.txt",
|
||||
"copy": "evidence/datasets/male-cns-v1.0/raw/_manifests/skeletons/unexpected-files.txt",
|
||||
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
}
|
||||
],
|
||||
"full_inventory": "registry/snapshots/bootstrap-20260929/files.jsonl.gz"
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
# Phase 3B stopped at count gate
|
||||
|
||||
Rule: non-null, non-blank superclass; original values are not rewritten.
|
||||
Actual neurons: 166700; expected: 166691; difference: +9. Fragments: 44877.
|
||||
Primary type: 11751 unique non-blank values; expected: 11691; difference: +60; null rows: 2194.
|
||||
No neuron index, unified metadata, connectivity or CSR was generated.
|
||||
All type-like columns are reported separately. No type fallback or subclass exclusions were applied.
|
||||
The nine differing body IDs cannot be identified without a paper-matched reference ID list.
|
||||
To reproduce, run the archived script with the dataset processed/.venv/bin/python; choose a fresh OUT directory because the script refuses existing audit contents.
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
assignedOlHex1: double
|
||||
assignedOlHex2: double
|
||||
bodyId: int64
|
||||
flywireType: string
|
||||
group: double
|
||||
instance: string
|
||||
somaSide: string
|
||||
statusLabel: dictionary<values=string, indices=int8, ordered=1>
|
||||
superclass: string
|
||||
type: string
|
||||
vfbId: string
|
||||
hemibrainType: string
|
||||
itoleeHl: string
|
||||
supertype: string
|
||||
birthtime: string
|
||||
mancBodyid: double
|
||||
mancGroup: double
|
||||
mancType: string
|
||||
subclass: string
|
||||
synonyms: string
|
||||
class: string
|
||||
rootSide: string
|
||||
somaNeuromere: string
|
||||
trumanHl: string
|
||||
dimorphism: string
|
||||
matchingNotes: string
|
||||
entryNerve: string
|
||||
mancSerial: double
|
||||
mcnsSerial: double
|
||||
serialMotif: string
|
||||
fruDsx: string
|
||||
exitNerve: string
|
||||
receptorType: string
|
||||
somaLocation: list<item: int64>
|
||||
child 0, item: int64
|
||||
tosomaLocation: list<item: int64>
|
||||
child 0, item: int64
|
||||
status: string
|
||||
-- schema metadata --
|
||||
pandas: '{"index_columns": [{"kind": "range", "name": null, "start": 0, "' + 4583
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only source audit; stop Phase 3B if its neuron-count gate fails."""
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.feather as feather
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
ROOT = Path('/data/digital-drosophila/datasets/male-cns-v1.0')
|
||||
OUT = ROOT / 'processed/neuron-only/audit'
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with path.open('rb') as f:
|
||||
for block in iter(lambda: f.read(8 * 1024 * 1024), b''):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
if any(OUT.iterdir()):
|
||||
raise RuntimeError('Refusing to overwrite an existing audit; use a fresh audit directory.')
|
||||
source = ROOT / 'raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather'
|
||||
before = digest(source)
|
||||
table = feather.read_table(source)
|
||||
fields = [n for n in table.column_names if n.casefold() == 'superclass']
|
||||
if len(fields) != 1:
|
||||
raise RuntimeError(f'Ambiguous superclass field: {fields}')
|
||||
name = fields[0]
|
||||
if not pa.types.is_string(table.schema.field(name).type):
|
||||
raise RuntimeError(f'Uninspected superclass dtype: {table.schema.field(name).type}')
|
||||
values = table[name].to_pylist()
|
||||
counts = collections.Counter(values)
|
||||
valid = [v is not None and bool(v.strip()) for v in values]
|
||||
neurons = table.filter(pa.array(valid))
|
||||
types = {}
|
||||
for field in table.schema:
|
||||
if 'type' not in field.name.casefold():
|
||||
continue
|
||||
vals = neurons[field.name].to_pylist()
|
||||
good = [v for v in vals if v is not None and (not isinstance(v, str) or bool(v.strip()))]
|
||||
freq = collections.Counter(good)
|
||||
types[field.name] = dict(dtype=str(field.type), nulls=sum(v is None for v in vals),
|
||||
blank_strings=sum(isinstance(v, str) and not v.strip() for v in vals),
|
||||
populated_rows=len(good), unique_valid_values=len(freq))
|
||||
(OUT / f'{field.name}-value-counts.json').write_text(json.dumps(freq, indent=2, sort_keys=True)+'\n')
|
||||
# This is a classification audit, not a new neuron index or metadata product.
|
||||
pq.write_table(pa.table({'body_id': table['bodyId'].cast(pa.uint64()),
|
||||
'superclass': table[name], 'passes_superclass_rule': valid}),
|
||||
OUT / 'annotation-classification-audit.parquet', compression='zstd')
|
||||
report = dict(status='BLOCKED_NEURON_COUNT_MISMATCH', annotation_rows=table.num_rows,
|
||||
superclass_field=name, superclass_dtype=str(table.schema.field(name).type),
|
||||
actual_neurons=neurons.num_rows, expected_neurons=166691,
|
||||
neuron_difference=neurons.num_rows-166691, fragments=len(values)-neurons.num_rows,
|
||||
null_count=counts[None], empty_string_count=counts[''],
|
||||
whitespace_only_count=sum(isinstance(v,str) and v!='' and not v.strip() for v in values),
|
||||
nonnull_values_with_surrounding_whitespace=sum(isinstance(v,str) and v!=v.strip() for v in values),
|
||||
superclass_value_counts={str(k): v for k,v in counts.items()},
|
||||
type_fields=types, expected_type_count=11691,
|
||||
actual_primary_type_count=types['type']['unique_valid_values'],
|
||||
type_difference=types['type']['unique_valid_values']-11691,
|
||||
expected_counts_source='User-specified formal paper reference counts',
|
||||
reason='166700 != 166691: stopped before generating mappings, unified metadata or CSR.',
|
||||
specific_nine_ids='Undetermined: a paper-matched reference ID set is required; count alone cannot identify the nine IDs.',
|
||||
software=dict(python=platform.python_version(),pyarrow=pa.__version__))
|
||||
(OUT/'difference-audit.json').write_text(json.dumps(report,indent=2,sort_keys=True)+'\n')
|
||||
(OUT/'annotation-schema.txt').write_text(str(table.schema)+'\n')
|
||||
(OUT/'README.md').write_text(
|
||||
'# Phase 3B stopped at count gate\n\n'
|
||||
'Rule: non-null, non-blank superclass; original values are not rewritten.\n'
|
||||
'Actual neurons: 166700; expected: 166691; difference: +9. Fragments: 44877.\n'
|
||||
'Primary type: 11751 unique non-blank values; expected: 11691; difference: +60; null rows: 2194.\n'
|
||||
'No neuron index, unified metadata, connectivity or CSR was generated.\n'
|
||||
'All type-like columns are reported separately. No type fallback or subclass exclusions were applied.\n'
|
||||
'The nine differing body IDs cannot be identified without a paper-matched reference ID list.\n'
|
||||
'To reproduce, run the archived script with the dataset processed/.venv/bin/python; '
|
||||
'choose a fresh OUT directory because the script refuses existing audit contents.\n')
|
||||
(OUT/'audit-malecns-phase3b.py').write_bytes(Path(__file__).read_bytes())
|
||||
assert digest(source)==before
|
||||
inventory=[dict(path=p.name,bytes=p.stat().st_size,sha256=digest(p)) for p in sorted(OUT.iterdir()) if p.is_file()]
|
||||
manifest=dict(status=report['status'], source=str(source),source_sha256=before,
|
||||
source_unchanged_after_audit=True,outputs=inventory,
|
||||
downstream_generation_performed=False)
|
||||
(OUT/'manifest.json').write_text(json.dumps(manifest,indent=2,sort_keys=True)+'\n')
|
||||
(OUT/'checksums.sha256').write_text(''.join(f'{digest(p)} {p.name}\n' for p in sorted(OUT.iterdir()) if p.is_file()))
|
||||
print(json.dumps(report,indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
deb42f7727a38276921d5bf93043906f8edf26501fc65dc3ecf948cacc668d7e README.md
|
||||
2c7ba16d2a532be4e7ded3d651758db55b31b4ab0958926194fa85aeccbc49d8 annotation-classification-audit.parquet
|
||||
4385079022f1ee050c95bc602367da8c0f5f36c74749babc434e230c370ddb6e annotation-schema.txt
|
||||
8fd5159cc0fc5be179901423fd0b36ec17af06818a7891f019bbf927a0c025b1 audit-malecns-phase3b.py
|
||||
5faf964f64ae433bfbce1ad63551c3caf3503d1f0a53e9c2bbdb63df5e33064f difference-audit.json
|
||||
f063b464479d68802335cd0049df88a989c95fc54bf6b840308b7addabae5e5a flywireType-value-counts.json
|
||||
28e261d43676610a6c53e49dce1db9bb974d1211712fd70da619e571b4bfa61b hemibrainType-value-counts.json
|
||||
1233ef5ab606b89156266c8bd9f239b4dd7fbce02601f59290e04802c951dcb0 mancType-value-counts.json
|
||||
c6bd94818c1a231e8b353d6b10da5f0b99b2c125c27dac3e31a5023effa92ee2 manifest.json
|
||||
d868b3cc489659d1d376f5ea37b1408eb2ce6345e878fa4de0765f04b4e1d00c receptorType-value-counts.json
|
||||
5f5460baafeb51e303c71d16bae2bd049cac28bbde89bd028a100e1b06944c09 supertype-value-counts.json
|
||||
1cb76d53dcfdd6f0624714119e22a7fa3f65bc57723f6d39797df1ca336775e0 type-value-counts.json
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"actual_neurons": 166700,
|
||||
"actual_primary_type_count": 11751,
|
||||
"annotation_rows": 211577,
|
||||
"empty_string_count": 0,
|
||||
"expected_counts_source": "User-specified formal paper reference counts",
|
||||
"expected_neurons": 166691,
|
||||
"expected_type_count": 11691,
|
||||
"fragments": 44877,
|
||||
"neuron_difference": 9,
|
||||
"nonnull_values_with_surrounding_whitespace": 0,
|
||||
"null_count": 44877,
|
||||
"reason": "166700 != 166691: stopped before generating mappings, unified metadata or CSR.",
|
||||
"software": {
|
||||
"pyarrow": "25.0.1",
|
||||
"python": "3.10.12"
|
||||
},
|
||||
"specific_nine_ids": "Undetermined: a paper-matched reference ID set is required; count alone cannot identify the nine IDs.",
|
||||
"status": "BLOCKED_NEURON_COUNT_MISMATCH",
|
||||
"superclass_dtype": "string",
|
||||
"superclass_field": "superclass",
|
||||
"superclass_value_counts": {
|
||||
"ENS": 50,
|
||||
"None": 44877,
|
||||
"ascending_neuron": 1846,
|
||||
"cb_efferent": 4,
|
||||
"cb_endocrine": 72,
|
||||
"cb_intrinsic": 32164,
|
||||
"cb_motor": 107,
|
||||
"cb_sensory": 4868,
|
||||
"cb_sensory_tbc": 14,
|
||||
"descending_neuron": 1314,
|
||||
"descending_neuron_tbc": 2,
|
||||
"efferent_ascending": 8,
|
||||
"efferent_descending": 4,
|
||||
"ol_intrinsic": 89403,
|
||||
"ol_sensory": 6098,
|
||||
"sensory_ascending": 537,
|
||||
"sensory_ascending_tbc": 2,
|
||||
"sensory_descending": 12,
|
||||
"visual_centrifugal": 563,
|
||||
"visual_projection": 9201,
|
||||
"visual_projection_tbc": 2,
|
||||
"vnc_efferent": 94,
|
||||
"vnc_endocrine": 22,
|
||||
"vnc_intrinsic": 13161,
|
||||
"vnc_motor": 708,
|
||||
"vnc_sensory": 6370,
|
||||
"vnc_sensory_tbc": 36,
|
||||
"vnc_tbc": 38
|
||||
},
|
||||
"type_difference": 60,
|
||||
"type_fields": {
|
||||
"flywireType": {
|
||||
"blank_strings": 0,
|
||||
"dtype": "string",
|
||||
"nulls": 23546,
|
||||
"populated_rows": 143154,
|
||||
"unique_valid_values": 8199
|
||||
},
|
||||
"hemibrainType": {
|
||||
"blank_strings": 0,
|
||||
"dtype": "string",
|
||||
"nulls": 133781,
|
||||
"populated_rows": 32919,
|
||||
"unique_valid_values": 4495
|
||||
},
|
||||
"mancType": {
|
||||
"blank_strings": 0,
|
||||
"dtype": "string",
|
||||
"nulls": 143956,
|
||||
"populated_rows": 22744,
|
||||
"unique_valid_values": 3893
|
||||
},
|
||||
"receptorType": {
|
||||
"blank_strings": 0,
|
||||
"dtype": "string",
|
||||
"nulls": 165948,
|
||||
"populated_rows": 752,
|
||||
"unique_valid_values": 3
|
||||
},
|
||||
"supertype": {
|
||||
"blank_strings": 0,
|
||||
"dtype": "string",
|
||||
"nulls": 132604,
|
||||
"populated_rows": 34096,
|
||||
"unique_valid_values": 1925
|
||||
},
|
||||
"type": {
|
||||
"blank_strings": 0,
|
||||
"dtype": "string",
|
||||
"nulls": 2194,
|
||||
"populated_rows": 164506,
|
||||
"unique_valid_values": 11751
|
||||
}
|
||||
},
|
||||
"whitespace_only_count": 0
|
||||
}
|
||||
+8201
File diff suppressed because it is too large
Load Diff
+4497
File diff suppressed because it is too large
Load Diff
+3895
File diff suppressed because it is too large
Load Diff
+64
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"downstream_generation_performed": false,
|
||||
"outputs": [
|
||||
{
|
||||
"bytes": 706,
|
||||
"path": "README.md",
|
||||
"sha256": "deb42f7727a38276921d5bf93043906f8edf26501fc65dc3ecf948cacc668d7e"
|
||||
},
|
||||
{
|
||||
"bytes": 601375,
|
||||
"path": "annotation-classification-audit.parquet",
|
||||
"sha256": "2c7ba16d2a532be4e7ded3d651758db55b31b4ab0958926194fa85aeccbc49d8"
|
||||
},
|
||||
{
|
||||
"bytes": 874,
|
||||
"path": "annotation-schema.txt",
|
||||
"sha256": "4385079022f1ee050c95bc602367da8c0f5f36c74749babc434e230c370ddb6e"
|
||||
},
|
||||
{
|
||||
"bytes": 5611,
|
||||
"path": "audit-malecns-phase3b.py",
|
||||
"sha256": "8fd5159cc0fc5be179901423fd0b36ec17af06818a7891f019bbf927a0c025b1"
|
||||
},
|
||||
{
|
||||
"bytes": 2628,
|
||||
"path": "difference-audit.json",
|
||||
"sha256": "5faf964f64ae433bfbce1ad63551c3caf3503d1f0a53e9c2bbdb63df5e33064f"
|
||||
},
|
||||
{
|
||||
"bytes": 130369,
|
||||
"path": "flywireType-value-counts.json",
|
||||
"sha256": "f063b464479d68802335cd0049df88a989c95fc54bf6b840308b7addabae5e5a"
|
||||
},
|
||||
{
|
||||
"bytes": 70934,
|
||||
"path": "hemibrainType-value-counts.json",
|
||||
"sha256": "28e261d43676610a6c53e49dce1db9bb974d1211712fd70da619e571b4bfa61b"
|
||||
},
|
||||
{
|
||||
"bytes": 66152,
|
||||
"path": "mancType-value-counts.json",
|
||||
"sha256": "1233ef5ab606b89156266c8bd9f239b4dd7fbce02601f59290e04802c951dcb0"
|
||||
},
|
||||
{
|
||||
"bytes": 78,
|
||||
"path": "receptorType-value-counts.json",
|
||||
"sha256": "d868b3cc489659d1d376f5ea37b1408eb2ce6345e878fa4de0765f04b4e1d00c"
|
||||
},
|
||||
{
|
||||
"bytes": 27895,
|
||||
"path": "supertype-value-counts.json",
|
||||
"sha256": "5f5460baafeb51e303c71d16bae2bd049cac28bbde89bd028a100e1b06944c09"
|
||||
},
|
||||
{
|
||||
"bytes": 188083,
|
||||
"path": "type-value-counts.json",
|
||||
"sha256": "1cb76d53dcfdd6f0624714119e22a7fa3f65bc57723f6d39797df1ca336775e0"
|
||||
}
|
||||
],
|
||||
"source": "/data/digital-drosophila/datasets/male-cns-v1.0/raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather",
|
||||
"source_sha256": "2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2",
|
||||
"source_unchanged_after_audit": true,
|
||||
"status": "BLOCKED_NEURON_COUNT_MISMATCH"
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
Phase 3B: superclass != null; 166700 neurons including null type.
|
||||
Raw type strings preserved; 11710 paper types are reference only.
|
||||
Original annotation and NT fields use annotation_ and nt_ prefixes, with original dtypes and nulls.
|
||||
NT coverage uses nonempty consensus_nt, without inferred fallback.
|
||||
Mapping is body_id ascending; reverse lookup uses searchsorted with exact equality check.
|
||||
CSR rows are presynaptic sources; columns are postsynaptic targets.
|
||||
Fragment comparison excludes each edge once if either endpoint is a fragment.
|
||||
Nine category totals separate annotation fragments from other unannotated segments.
|
||||
Re-run using processed/.venv/bin/python processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py; completed runs verify and do not overwrite.
|
||||
Historical audit in parent audit/ remains unchanged.
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff audit/phase3b-166700/README.md
|
||||
beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583 audit/phase3b-166700/edge-category-audit.json
|
||||
bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702 audit/phase3b-166700/excluded-fragment-annotations.parquet
|
||||
c061452f341d190946f80ca96b4589ab1e2a8dca6a67c57ad5d14f6ce34b1b64 audit/phase3b-166700/manifest.json
|
||||
b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234 audit/phase3b-166700/missing-neurotransmitter-ids.parquet
|
||||
4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e audit/phase3b-166700/missing-swc-ids.parquet
|
||||
b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b audit/phase3b-166700/missing-type-ids.parquet
|
||||
3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97 audit/phase3b-166700/phase3b-neuron-only.py
|
||||
f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6 audit/phase3b-166700/schema.json
|
||||
f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd audit/phase3b-166700/statistics.json
|
||||
497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a audit/phase3b-166700/type-field-audit.json
|
||||
598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1 audit/phase3b-166700/validation.json
|
||||
a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f connectivity/canonical-edges.parquet
|
||||
5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b connectivity/degrees.parquet
|
||||
ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150 gpu/csr-indices.int32.npy
|
||||
fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4 gpu/csr-indptr.int64.npy
|
||||
3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8 gpu/csr-metadata.json
|
||||
c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b gpu/csr-weights.int64.npy
|
||||
5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242 neuron-index/body-id-neuron-idx.feather
|
||||
2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009 neuron-index/body-id-neuron-idx.parquet
|
||||
c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206 neuron-index/body-id-to-neuron-idx.npz
|
||||
ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6 neuron-index/neuron-idx-to-body-id.uint64.npy
|
||||
ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1 neuron-index/unified-metadata.feather
|
||||
922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63 neuron-index/unified-metadata.parquet
|
||||
4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8 phase3b-repeat.log
|
||||
b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e phase3b-run.log
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"fragment->fragment": {
|
||||
"edges": 14135,
|
||||
"weight": 49566
|
||||
},
|
||||
"fragment->neuron": {
|
||||
"edges": 249969,
|
||||
"weight": 661930
|
||||
},
|
||||
"fragment->other": {
|
||||
"edges": 766887,
|
||||
"weight": 1131611
|
||||
},
|
||||
"neuron->fragment": {
|
||||
"edges": 181344,
|
||||
"weight": 476820
|
||||
},
|
||||
"neuron->neuron": {
|
||||
"edges": 25582938,
|
||||
"weight": 124177617
|
||||
},
|
||||
"neuron->other": {
|
||||
"edges": 112452244,
|
||||
"weight": 170414577
|
||||
},
|
||||
"other->fragment": {
|
||||
"edges": 78570,
|
||||
"weight": 107066
|
||||
},
|
||||
"other->neuron": {
|
||||
"edges": 4552783,
|
||||
"weight": 5614376
|
||||
},
|
||||
"other->other": {
|
||||
"edges": 7977814,
|
||||
"weight": 9199680
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"canonical_rule": "superclass != null",
|
||||
"disk_reload_check": "PASS: all original annotation/NT fields exact and CSR dtype/range/count checks passed",
|
||||
"inputs": [
|
||||
{
|
||||
"bytes": 14483314,
|
||||
"path": "raw/annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather",
|
||||
"sha256": "2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2"
|
||||
},
|
||||
{
|
||||
"bytes": 43282834,
|
||||
"path": "raw/annotations/body-neurotransmitters-male-cns-v1.0.feather",
|
||||
"sha256": "95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621"
|
||||
},
|
||||
{
|
||||
"bytes": 1051241946,
|
||||
"path": "raw/connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather",
|
||||
"sha256": "e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"bytes": 823,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/README.md",
|
||||
"sha256": "cc46e2d24128f8216aee41c244b2eff7dd6bc4752f5efe30a7bc6d3014b79dff"
|
||||
},
|
||||
{
|
||||
"bytes": 643,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/edge-category-audit.json",
|
||||
"sha256": "beab39e25d7892902eedae102cb44aa236ef1621462aea2267f5ccdcbda65583"
|
||||
},
|
||||
{
|
||||
"bytes": 261095,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/excluded-fragment-annotations.parquet",
|
||||
"sha256": "bb25b318407313b54b00075c6c0b539fa0c02644a862ace569fc6aa94c27a702"
|
||||
},
|
||||
{
|
||||
"bytes": 1350,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/missing-neurotransmitter-ids.parquet",
|
||||
"sha256": "b253203f9cdc3da46da046382e7797f5bc825b18ac48a7ff4f58e2e8f6959234"
|
||||
},
|
||||
{
|
||||
"bytes": 364,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/missing-swc-ids.parquet",
|
||||
"sha256": "4241d5aadb9c618bf9e95a3b018b85ec09079f5570eab6e63e4555bf965e3a4e"
|
||||
},
|
||||
{
|
||||
"bytes": 8105,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/missing-type-ids.parquet",
|
||||
"sha256": "b607b7278a4153e5c408a96ee2e2c60dcec1a3163c6b75f22c47b24b5033e48b"
|
||||
},
|
||||
{
|
||||
"bytes": 13055,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py",
|
||||
"sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
|
||||
},
|
||||
{
|
||||
"bytes": 3383,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/schema.json",
|
||||
"sha256": "f78d13caee791b442f1be1cbe463a5cc13847499368b7d4fb351624e604332b6"
|
||||
},
|
||||
{
|
||||
"bytes": 753,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/statistics.json",
|
||||
"sha256": "f2a5056d90712700b4c7776454ac6f44064f98d8d1604da0a10c78b4c2dbcebd"
|
||||
},
|
||||
{
|
||||
"bytes": 649,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/type-field-audit.json",
|
||||
"sha256": "497ac09d9135c5369a11b7d39f0a8d9a9fafee4d6f8a357af48caff389ddac3a"
|
||||
},
|
||||
{
|
||||
"bytes": 371,
|
||||
"path": "processed/neuron-only/audit/phase3b-166700/validation.json",
|
||||
"sha256": "598980575c97249d38da39f562e7a762c7820200b8fbb2ce3ce1c2eba80f29e1"
|
||||
},
|
||||
{
|
||||
"bytes": 137157039,
|
||||
"path": "processed/neuron-only/connectivity/canonical-edges.parquet",
|
||||
"sha256": "a7146c0dfb60a2d2761b40f521fabcc3d47c4ae6ffdf45bf017b6b6ae164939f"
|
||||
},
|
||||
{
|
||||
"bytes": 1665145,
|
||||
"path": "processed/neuron-only/connectivity/degrees.parquet",
|
||||
"sha256": "5e803105ca3a9285b0a6ac2a6bff291aa3921a1776a90c6b76e99720e920943b"
|
||||
},
|
||||
{
|
||||
"bytes": 102331880,
|
||||
"path": "processed/neuron-only/gpu/csr-indices.int32.npy",
|
||||
"sha256": "ce20f7433f78442c717ad2a7b5394cf4a3b109d990b4be0788040b7b0f4ed150"
|
||||
},
|
||||
{
|
||||
"bytes": 1333736,
|
||||
"path": "processed/neuron-only/gpu/csr-indptr.int64.npy",
|
||||
"sha256": "fe43420f11e6242a11f8199ae7f097cab4b8a6db181d36f2a46b9e46670acbb4"
|
||||
},
|
||||
{
|
||||
"bytes": 361,
|
||||
"path": "processed/neuron-only/gpu/csr-metadata.json",
|
||||
"sha256": "3786a0c79fc25bdc4d11f67d4de1f9f6a88da750431e96508bf72bbfa42bb6f8"
|
||||
},
|
||||
{
|
||||
"bytes": 204663632,
|
||||
"path": "processed/neuron-only/gpu/csr-weights.int64.npy",
|
||||
"sha256": "c7ae101451dd19c47fd976bcfacbc52d67193a68ffa04c5b4d2e911cfd52d99b"
|
||||
},
|
||||
{
|
||||
"bytes": 610146,
|
||||
"path": "processed/neuron-only/neuron-index/body-id-neuron-idx.feather",
|
||||
"sha256": "5c493a1eff7317c748d85260a1098be43b2971483b03d59ea689e4b1c0f06242"
|
||||
},
|
||||
{
|
||||
"bytes": 1197954,
|
||||
"path": "processed/neuron-only/neuron-index/body-id-neuron-idx.parquet",
|
||||
"sha256": "2b615c00d290e34fbe7792a5050eee4ce086446cd54ec89c96d7b1bc4a8e6009"
|
||||
},
|
||||
{
|
||||
"bytes": 2000920,
|
||||
"path": "processed/neuron-only/neuron-index/body-id-to-neuron-idx.npz",
|
||||
"sha256": "c778a59a87756c8d79f558d57f694ae809de63b4b7f7b599692b463203500206"
|
||||
},
|
||||
{
|
||||
"bytes": 1333728,
|
||||
"path": "processed/neuron-only/neuron-index/neuron-idx-to-body-id.uint64.npy",
|
||||
"sha256": "ddfa789213c8c5066400c026b65c112c267781c2e3ab380a39dd437866e704d6"
|
||||
},
|
||||
{
|
||||
"bytes": 16822034,
|
||||
"path": "processed/neuron-only/neuron-index/unified-metadata.feather",
|
||||
"sha256": "ce5c910e2fbdcdf682bc64deef64185d7f56bf1ccf4e9e6dfe19f4533e58d6c1"
|
||||
},
|
||||
{
|
||||
"bytes": 9283041,
|
||||
"path": "processed/neuron-only/neuron-index/unified-metadata.parquet",
|
||||
"sha256": "922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63"
|
||||
},
|
||||
{
|
||||
"bytes": 929,
|
||||
"path": "processed/neuron-only/phase3b-run.log",
|
||||
"sha256": "b30750bae90a63721412511d31a751964d968569446f769d7adfa540b4a9b08e"
|
||||
},
|
||||
{
|
||||
"bytes": 106,
|
||||
"path": "processed/neuron-only/phase3b-repeat.log",
|
||||
"sha256": "4d2500d1fd261027bfbac40f3b6560e9a4e4a20aeb3051d3775d9825a7719ce8"
|
||||
}
|
||||
],
|
||||
"phase": "3B",
|
||||
"repeat_run_check": "PASS: full inputs/output checksums verified without overwrite",
|
||||
"script_sha256": "3ad98a814328e882b0c463725bb644d6a8e7f6081078dd2dfdccebc883ef4d97"
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic neuron-only preprocessing. Existing completed outputs are verified, never replaced."""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
import pyarrow.compute as pc
|
||||
import pyarrow.feather as ft
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
|
||||
def sha(p):
|
||||
h = hashlib.sha256()
|
||||
with p.open('rb') as f:
|
||||
for b in iter(lambda: f.read(8388608), b''):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def js(p, obj):
|
||||
p.write_text(json.dumps(obj, indent=2, sort_keys=True)+'\n')
|
||||
|
||||
|
||||
def member(ids, q):
|
||||
pos = np.searchsorted(ids, q)
|
||||
return pos, (pos < len(ids)) & (ids[np.minimum(pos, len(ids)-1)] == q)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--root', default='/data/digital-drosophila/datasets/male-cns-v1.0')
|
||||
args = ap.parse_args()
|
||||
root = Path(args.root)
|
||||
raw = root/'raw'
|
||||
out = root/'processed/neuron-only'
|
||||
audit_name = 'phase3b-166700'
|
||||
final_audit = out/'audit'/audit_name
|
||||
def checks():
|
||||
subprocess.run(['md5sum','--quiet','-c','_manifests/checksums.md5'],cwd=raw,check=True)
|
||||
subprocess.run(['md5sum','--quiet','-c','../_manifests/skeletons/checksums.md5'],cwd=raw/'skeletons',check=True)
|
||||
subprocess.run(['sha256sum','--quiet','-c','audit/checksums.sha256'],cwd=root/'processed',check=True)
|
||||
subprocess.run(['sha256sum','--quiet','-c','checksums.sha256'],cwd=out/'audit',check=True)
|
||||
if (final_audit/'manifest.json').exists():
|
||||
checks()
|
||||
manifest=json.loads((final_audit/'manifest.json').read_text())
|
||||
for item in manifest['inputs']+manifest['outputs']:
|
||||
assert sha(root/item['path']) == item['sha256'], item['path']
|
||||
assert sha(Path(__file__)) == manifest['script_sha256']
|
||||
subprocess.run(['sha256sum','--quiet','-c',str(final_audit/'checksums.sha256')],cwd=out,check=True)
|
||||
print('REPEAT_RUN_OK: all inputs, existing Phase3, historical audit and Phase3B outputs unchanged; no overwrite.',flush=True)
|
||||
return
|
||||
for d in ['neuron-index','connectivity','gpu']:
|
||||
if (out/d).exists():
|
||||
raise RuntimeError(f'Refusing existing destination {out/d}')
|
||||
assert not final_audit.exists()
|
||||
print('Verifying raw and preserved Phase3/historical audit',flush=True)
|
||||
checks()
|
||||
stage=out/f'.staging-{int(time.time())}'
|
||||
for d in ['neuron-index','connectivity','gpu',f'audit/{audit_name}']:
|
||||
(stage/d).mkdir(parents=True)
|
||||
audit=stage/'audit'/audit_name
|
||||
paths=[raw/'annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather',raw/'annotations/body-neurotransmitters-male-cns-v1.0.feather',raw/'connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather']
|
||||
inputs=[dict(path=str(p.relative_to(root)),sha256=sha(p),bytes=p.stat().st_size) for p in paths]
|
||||
a,nt,c=[ft.read_table(p) for p in paths]
|
||||
schemas={k:str(t.schema) for k,t in zip(['annotation','neurotransmitter','connectome'],[a,nt,c])}
|
||||
assert a.schema.field('superclass').type == pa.string()
|
||||
assert set(c.column_names)=={'body_pre','body_post','weight'}
|
||||
def ids(t,k):
|
||||
assert t[k].null_count==0 and pa.types.is_integer(t[k].type)
|
||||
arr=t[k].to_numpy();assert np.all(arr>=0)
|
||||
return arr.astype(np.uint64)
|
||||
aid=ids(a,'bodyId');nid=ids(nt,'body')
|
||||
assert len(np.unique(aid))==len(aid) and len(np.unique(nid))==len(nid)
|
||||
mask=pc.is_valid(a['superclass'])
|
||||
neurons=a.filter(mask);fragments=a.filter(pc.invert(mask))
|
||||
assert neurons.num_rows==166700 and fragments.num_rows==44877
|
||||
neurons=neurons.sort_by([('bodyId','ascending')]);canon=ids(neurons,'bodyId')
|
||||
frag=np.sort(ids(fragments,'bodyId'));allids=np.sort(aid);n=len(canon)
|
||||
pq.write_table(fragments,audit/'excluded-fragment-annotations.parquet',compression='zstd')
|
||||
swc=np.array(sorted(int(p.stem) for p in (raw/'skeletons').glob('*.swc')),dtype=np.uint64)
|
||||
has_swc=member(swc,canon)[1]
|
||||
ni=pc.index_in(pa.array(canon),value_set=nt['body'].cast(pa.uint64()))
|
||||
aligned=nt.take(ni)
|
||||
md={'neuron_idx':pa.array(np.arange(n,dtype=np.int32)),'body_id':pa.array(canon),
|
||||
'neuron_type':neurons['type'],'neuron_class':neurons['class'],'side':neurons['somaSide'],
|
||||
'neurotransmitter':aligned['consensus_nt'],'has_swc':pa.array(has_swc),
|
||||
'swc_path':pa.array([str(raw/'skeletons'/f'{int(b)}.swc') if ok else None for b,ok in zip(canon,has_swc)])}
|
||||
md.update({'annotation_'+k:neurons[k] for k in neurons.column_names})
|
||||
md.update({'nt_'+k:aligned[k] for k in aligned.column_names})
|
||||
metadata=pa.table(md)
|
||||
mapping=metadata.select(['neuron_idx','body_id'])
|
||||
for name,t in [('unified-metadata',metadata),('body-id-neuron-idx',mapping)]:
|
||||
pq.write_table(t,stage/'neuron-index'/f'{name}.parquet',compression='zstd')
|
||||
ft.write_feather(t,stage/'neuron-index'/f'{name}.feather',compression='zstd')
|
||||
np.save(stage/'neuron-index/neuron-idx-to-body-id.uint64.npy',canon)
|
||||
np.savez(stage/'neuron-index/body-id-to-neuron-idx.npz',body_id=canon,neuron_idx=np.arange(n,dtype=np.int32))
|
||||
type_stats={}
|
||||
for k in neurons.column_names:
|
||||
if 'type' in k.lower():
|
||||
v=neurons[k].to_pylist();good=[x for x in v if x is not None and x!='']
|
||||
type_stats[k]=dict(null=sum(x is None for x in v),empty=sum(x=='' for x in v),unique_nonempty=len(set(good)),present=len(good))
|
||||
ntv=aligned['consensus_nt'].to_pylist();hasnt=np.array([x is not None and x!='' for x in ntv])
|
||||
for name,present in [('swc',has_swc),('neurotransmitter',hasnt),('type',np.array([x is not None and x!='' for x in neurons['type'].to_pylist()]))]:
|
||||
pq.write_table(pa.table({'body_id':canon[~present]}),audit/f'missing-{name}-ids.parquet',compression='zstd')
|
||||
print('Filtering original connectome and auditing fragment edges',flush=True)
|
||||
src_chunks=[];dst_chunks=[];w_chunks=[]
|
||||
categories={f'{s}->{t}':{'edges':0,'weight':0} for s in ['neuron','fragment','other'] for t in ['neuron','fragment','other']}
|
||||
for batch in c.to_batches(max_chunksize=1000000):
|
||||
pre=ids(pa.Table.from_batches([batch]),'body_pre');post=ids(pa.Table.from_batches([batch]),'body_post')
|
||||
w=batch.column(batch.schema.get_field_index('weight')).to_numpy()
|
||||
assert w.dtype==np.int64 and np.all(w>0)
|
||||
si,sm=member(canon,pre);ti,tm=member(canon,post)
|
||||
sf=member(frag,pre)[1];tf=member(frag,post)[1]
|
||||
for sk,sv in [('neuron',sm),('fragment',sf),('other',~(sm|sf))]:
|
||||
for tk,tv in [('neuron',tm),('fragment',tf),('other',~(tm|tf))]:
|
||||
keep=sv&tv;z=categories[f'{sk}->{tk}'];z['edges']+=int(keep.sum());z['weight']+=int(w[keep].sum())
|
||||
keep=sm&tm
|
||||
src_chunks.append(si[keep].astype(np.int32));dst_chunks.append(ti[keep].astype(np.int32));w_chunks.append(w[keep])
|
||||
s=np.concatenate(src_chunks);t=np.concatenate(dst_chunks);w=np.concatenate(w_chunks)
|
||||
order=np.lexsort((t,s));s=s[order];t=t[order];w=w[order]
|
||||
duplicates=int(np.sum((s[1:]==s[:-1])&(t[1:]==t[:-1])))
|
||||
indeg=np.bincount(t,minlength=n);outdeg=np.bincount(s,minlength=n)
|
||||
ptr=np.r_[np.int64(0),np.cumsum(outdeg,dtype=np.int64)]
|
||||
for filename,x in [('csr-indptr.int64',ptr),('csr-indices.int32',t),('csr-weights.int64',w)]:
|
||||
np.save(stage/'gpu'/f'{filename}.npy',x)
|
||||
edges=pa.table({'source_neuron_idx':s,'target_neuron_idx':t,'body_pre':canon[s],'body_post':canon[t],'weight':w})
|
||||
pq.write_table(edges,stage/'connectivity/canonical-edges.parquet',compression='zstd')
|
||||
pq.write_table(pa.table({'body_id':canon,'neuron_idx':np.arange(n,dtype=np.int32),'indegree':indeg,'outdegree':outdeg}),stage/'connectivity/degrees.parquet',compression='zstd')
|
||||
old_edges=sum(categories[k]['edges'] for k in ['neuron->neuron','neuron->fragment','fragment->neuron','fragment->fragment'])
|
||||
old_weight=sum(categories[k]['weight'] for k in ['neuron->neuron','neuron->fragment','fragment->neuron','fragment->fragment'])
|
||||
old=json.loads((root/'processed/connectivity/connectivity-stats.json').read_text())
|
||||
assert old_edges==old['canonical_edge_rows'] and old_weight==old['canonical_weight_sum']
|
||||
print('Independent full raw edge lookup validation',flush=True)
|
||||
keys=s.astype(np.int64)*n+t;seen=np.zeros(len(keys),dtype=bool)
|
||||
for b in c.to_batches(max_chunksize=1000000):
|
||||
bt=pa.Table.from_batches([b]);pre=ids(bt,'body_pre');post=ids(bt,'body_post');bw=bt['weight'].to_numpy()
|
||||
si,sm=member(canon,pre);ti,tm=member(canon,post);keep=sm&tm
|
||||
loc=np.searchsorted(keys,si[keep]*n+ti[keep])
|
||||
assert np.all(loc<len(keys)) and np.array_equal(keys[loc],si[keep]*n+ti[keep])
|
||||
assert np.array_equal(w[loc],bw[keep]);seen[loc]=True
|
||||
assert duplicates==0 and seen.all()
|
||||
assert ptr[-1]==len(w) and ptr.dtype==np.int64 and t.dtype==np.int32
|
||||
assert np.array_equal(np.repeat(np.arange(n,dtype=np.int32),np.diff(ptr)),s)
|
||||
assert np.array_equal(canon[np.searchsorted(canon,aid[mask.to_numpy()])],aid[mask.to_numpy()])
|
||||
assert sum(z['edges'] for z in categories.values())==c.num_rows
|
||||
stats=dict(neurons=n,fragments=len(frag),with_swc=int(has_swc.sum()),without_swc=int((~has_swc).sum()),
|
||||
with_neurotransmitter=int(hasnt.sum()),without_neurotransmitter=int((~hasnt).sum()),neurotransmitter_coverage=float(hasnt.mean()),
|
||||
with_type=type_stats['type']['present'],without_type=n-type_stats['type']['present'],raw_unique_type_strings=type_stats['type']['unique_nonempty'],
|
||||
nnz=len(w),synapse_weight=int(w.sum()),weight_min=int(w.min()),weight_max=int(w.max()),
|
||||
connected_neurons=int(((indeg+outdeg)>0).sum()),isolated_neurons=int(((indeg+outdeg)==0).sum()),
|
||||
indegree_zero=int((indeg==0).sum()),outdegree_zero=int((outdeg==0).sum()),self_loops=int((s==t).sum()),duplicate_pairs=duplicates,
|
||||
excluded_fragment_edges_from_phase3=old_edges-len(w),excluded_fragment_weight_from_phase3=old_weight-int(w.sum()),
|
||||
all_raw_fragment_related_edges=sum(z['edges'] for k,z in categories.items() if 'fragment' in k),
|
||||
all_raw_fragment_related_weight=sum(z['weight'] for k,z in categories.items() if 'fragment' in k),
|
||||
csr_bytes=sum(p.stat().st_size for p in (stage/'gpu').glob('*.npy')))
|
||||
js(audit/'statistics.json',stats);js(audit/'edge-category-audit.json',categories);js(audit/'type-field-audit.json',type_stats)
|
||||
schemas.update(metadata=str(metadata.schema),mapping=str(mapping.schema),edges=str(edges.schema))
|
||||
js(audit/'schema.json',schemas)
|
||||
js(stage/'gpu/csr-metadata.json',dict(shape=[n,n],nnz=len(w),orientation='row=presynaptic source; column=postsynaptic target; incoming drive is A.T @ source_activity',indptr='int64',indices='int32',weights='int64 original positive synapse counts',self_loops='retained',threshold='none',signed_weight_conversion=False))
|
||||
js(audit/'validation.json',dict(all_raw_edges_checked_against_csr=True,weights_exact=True,old_phase3_counts_reconciled=True,raw_and_prior_artifacts_verified=True,canonical_rule='superclass is not null',type_filter=False,repeat_behavior='verify input/output SHA256 and return without writes',software=dict(numpy=np.__version__,pyarrow=pa.__version__)))
|
||||
shutil.copyfile(__file__,audit/'phase3b-neuron-only.py')
|
||||
(audit/'README.md').write_text('Phase 3B: superclass != null; 166700 neurons including null type.\nRaw type strings preserved; 11710 paper types are reference only.\nOriginal annotation and NT fields use annotation_ and nt_ prefixes, with original dtypes and nulls.\nNT coverage uses nonempty consensus_nt, without inferred fallback.\nMapping is body_id ascending; reverse lookup uses searchsorted with exact equality check.\nCSR rows are presynaptic sources; columns are postsynaptic targets.\nFragment comparison excludes each edge once if either endpoint is a fragment.\nNine category totals separate annotation fragments from other unannotated segments.\nRe-run using processed/.venv/bin/python processed/neuron-only/audit/phase3b-166700/phase3b-neuron-only.py; completed runs verify and do not overwrite.\nHistorical audit in parent audit/ remains unchanged.\n')
|
||||
checks()
|
||||
assert all(sha(root/i['path'])==i['sha256'] for i in inputs)
|
||||
inventory=[dict(path=str((out/p.relative_to(stage)).relative_to(root)),bytes=p.stat().st_size,sha256=sha(p)) for p in sorted(stage.rglob('*')) if p.is_file()]
|
||||
js(audit/'manifest.json',dict(inputs=inputs,outputs=inventory,script_sha256=sha(Path(__file__)),phase='3B',canonical_rule='superclass != null'))
|
||||
(audit/'checksums.sha256').write_text(''.join(f'{sha(p)} {p.relative_to(stage).as_posix()}\n' for p in sorted(stage.rglob('*')) if p.is_file()))
|
||||
for d in ['neuron-index','connectivity','gpu']:
|
||||
os.rename(stage/d,out/d)
|
||||
os.rename(audit,final_audit)
|
||||
(stage/'audit').rmdir();stage.rmdir()
|
||||
print(json.dumps(stats,indent=2),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"annotation": "assignedOlHex1: double\nassignedOlHex2: double\nbodyId: int64\nflywireType: string\ngroup: double\ninstance: string\nsomaSide: string\nstatusLabel: dictionary<values=string, indices=int8, ordered=1>\nsuperclass: string\ntype: string\nvfbId: string\nhemibrainType: string\nitoleeHl: string\nsupertype: string\nbirthtime: string\nmancBodyid: double\nmancGroup: double\nmancType: string\nsubclass: string\nsynonyms: string\nclass: string\nrootSide: string\nsomaNeuromere: string\ntrumanHl: string\ndimorphism: string\nmatchingNotes: string\nentryNerve: string\nmancSerial: double\nmcnsSerial: double\nserialMotif: string\nfruDsx: string\nexitNerve: string\nreceptorType: string\nsomaLocation: list<item: int64>\n child 0, item: int64\ntosomaLocation: list<item: int64>\n child 0, item: int64\nstatus: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 4583",
|
||||
"connectome": "body_pre: int64\nbody_post: int64\nweight: int64\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 604",
|
||||
"edges": "source_neuron_idx: int32\ntarget_neuron_idx: int32\nbody_pre: uint64\nbody_post: uint64\nweight: int64",
|
||||
"mapping": "neuron_idx: int32\nbody_id: uint64",
|
||||
"metadata": "neuron_idx: int32\nbody_id: uint64\nneuron_type: string\nneuron_class: string\nside: string\nneurotransmitter: string\nhas_swc: bool\nswc_path: string\nannotation_assignedOlHex1: double\nannotation_assignedOlHex2: double\nannotation_bodyId: int64\nannotation_flywireType: string\nannotation_group: double\nannotation_instance: string\nannotation_somaSide: string\nannotation_statusLabel: dictionary<values=string, indices=int8, ordered=1>\nannotation_superclass: string\nannotation_type: string\nannotation_vfbId: string\nannotation_hemibrainType: string\nannotation_itoleeHl: string\nannotation_supertype: string\nannotation_birthtime: string\nannotation_mancBodyid: double\nannotation_mancGroup: double\nannotation_mancType: string\nannotation_subclass: string\nannotation_synonyms: string\nannotation_class: string\nannotation_rootSide: string\nannotation_somaNeuromere: string\nannotation_trumanHl: string\nannotation_dimorphism: string\nannotation_matchingNotes: string\nannotation_entryNerve: string\nannotation_mancSerial: double\nannotation_mcnsSerial: double\nannotation_serialMotif: string\nannotation_fruDsx: string\nannotation_exitNerve: string\nannotation_receptorType: string\nannotation_somaLocation: list<item: int64>\n child 0, item: int64\nannotation_tosomaLocation: list<item: int64>\n child 0, item: int64\nannotation_status: string\nnt_body: int64\nnt_cell_type: string\nnt_total_nt_predictions: int32\nnt_predicted_nt_confidence: double\nnt_predicted_nt: string\nnt_ground_truth: string\nnt_celltype_total_nt_predictions: int32\nnt_celltype_predicted_nt: string\nnt_celltype_predicted_nt_confidence: double\nnt_consensus_nt: string",
|
||||
"neurotransmitter": "body: int64\ncell_type: string\ntotal_nt_predictions: int32\npredicted_nt_confidence: double\npredicted_nt: string\nground_truth: string\ncelltype_total_nt_predictions: int32\ncelltype_predicted_nt: string\ncelltype_predicted_nt_confidence: double\nconsensus_nt: string\n-- schema metadata --\npandas: '{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"' + 1606"
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"all_raw_fragment_related_edges": 1290905,
|
||||
"all_raw_fragment_related_weight": 2426993,
|
||||
"connected_neurons": 166483,
|
||||
"csr_bytes": 308329248,
|
||||
"duplicate_pairs": 0,
|
||||
"excluded_fragment_edges_from_phase3": 445448,
|
||||
"excluded_fragment_weight_from_phase3": 1188316,
|
||||
"fragments": 44877,
|
||||
"indegree_zero": 370,
|
||||
"isolated_neurons": 217,
|
||||
"neurons": 166700,
|
||||
"neurotransmitter_coverage": 0.9989322135572886,
|
||||
"nnz": 25582938,
|
||||
"outdegree_zero": 1220,
|
||||
"raw_unique_type_strings": 11751,
|
||||
"self_loops": 101,
|
||||
"synapse_weight": 124177617,
|
||||
"weight_max": 2591,
|
||||
"weight_min": 1,
|
||||
"with_neurotransmitter": 166522,
|
||||
"with_swc": 166700,
|
||||
"with_type": 164506,
|
||||
"without_neurotransmitter": 178,
|
||||
"without_swc": 0,
|
||||
"without_type": 2194
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"flywireType": {
|
||||
"empty": 0,
|
||||
"null": 23546,
|
||||
"present": 143154,
|
||||
"unique_nonempty": 8199
|
||||
},
|
||||
"hemibrainType": {
|
||||
"empty": 0,
|
||||
"null": 133781,
|
||||
"present": 32919,
|
||||
"unique_nonempty": 4495
|
||||
},
|
||||
"mancType": {
|
||||
"empty": 0,
|
||||
"null": 143956,
|
||||
"present": 22744,
|
||||
"unique_nonempty": 3893
|
||||
},
|
||||
"receptorType": {
|
||||
"empty": 0,
|
||||
"null": 165948,
|
||||
"present": 752,
|
||||
"unique_nonempty": 3
|
||||
},
|
||||
"supertype": {
|
||||
"empty": 0,
|
||||
"null": 132604,
|
||||
"present": 34096,
|
||||
"unique_nonempty": 1925
|
||||
},
|
||||
"type": {
|
||||
"empty": 0,
|
||||
"null": 2194,
|
||||
"present": 164506,
|
||||
"unique_nonempty": 11751
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"all_raw_edges_checked_against_csr": true,
|
||||
"canonical_rule": "superclass is not null",
|
||||
"old_phase3_counts_reconciled": true,
|
||||
"raw_and_prior_artifacts_verified": true,
|
||||
"repeat_behavior": "verify input/output SHA256 and return without writes",
|
||||
"software": {
|
||||
"numpy": "2.2.6",
|
||||
"pyarrow": "25.0.1"
|
||||
},
|
||||
"type_filter": false,
|
||||
"weights_exact": true
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"putative_IR52b": 226,
|
||||
"putative_ppk23": 269,
|
||||
"putative_ppk25": 257
|
||||
}
|
||||
+1927
File diff suppressed because it is too large
Load Diff
+11753
File diff suppressed because it is too large
Load Diff
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"indices": "int32",
|
||||
"indptr": "int64",
|
||||
"nnz": 25582938,
|
||||
"orientation": "row=presynaptic source; column=postsynaptic target; incoming drive is A.T @ source_activity",
|
||||
"self_loops": "retained",
|
||||
"shape": [
|
||||
166700,
|
||||
166700
|
||||
],
|
||||
"signed_weight_conversion": false,
|
||||
"threshold": "none",
|
||||
"weights": "int64 original positive synapse counts"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
50a7718770c57220f160ba4f431ab89e annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather
|
||||
3d842b12fe5c49eefade528d7dd24a1f annotations/body-neurotransmitters-male-cns-v1.0.feather
|
||||
f30e9dcca25cfd021bf1e7b3d975599e connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather
|
||||
@@ -0,0 +1,3 @@
|
||||
2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2 annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather
|
||||
95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621 annotations/body-neurotransmitters-male-cns-v1.0.feather
|
||||
e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1 connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
|
||||
readonly BASE_URL="https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome"
|
||||
readonly LOG_DIR="${RAW_ROOT}/_logs"
|
||||
readonly MANIFEST_DIR="${RAW_ROOT}/_manifests"
|
||||
|
||||
mkdir -p "${RAW_ROOT}/annotations" "${RAW_ROOT}/connectivity" "$LOG_DIR" "$MANIFEST_DIR"
|
||||
|
||||
download_one() {
|
||||
local subdir="$1"
|
||||
local filename="$2"
|
||||
local expected_md5="$3"
|
||||
local destination="${RAW_ROOT}/${subdir}/${filename}"
|
||||
local attempt
|
||||
|
||||
if [[ -f "$destination" ]] && printf '%s %s\n' "$expected_md5" "$destination" | md5sum --check --status; then
|
||||
printf '[OK] already verified: %s/%s\n' "$subdir" "$filename"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for attempt in 1 2 3; do
|
||||
if [[ -f "$destination" && ! -f "${destination}.aria2" ]]; then
|
||||
mv -- "$destination" "${destination}.corrupt-$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
|
||||
printf '[DOWNLOAD] attempt %d: %s/%s\n' "$attempt" "$subdir" "$filename"
|
||||
aria2c \
|
||||
--continue=true \
|
||||
--max-tries=0 \
|
||||
--retry-wait=10 \
|
||||
--connect-timeout=30 \
|
||||
--timeout=60 \
|
||||
--max-connection-per-server=8 \
|
||||
--split=8 \
|
||||
--min-split-size=16M \
|
||||
--file-allocation=falloc \
|
||||
--auto-file-renaming=false \
|
||||
--allow-overwrite=false \
|
||||
--check-integrity=true \
|
||||
--console-log-level=notice \
|
||||
--summary-interval=60 \
|
||||
--dir="${RAW_ROOT}/${subdir}" \
|
||||
--out="$filename" \
|
||||
"${BASE_URL}/${filename}" \
|
||||
>"${LOG_DIR}/${filename}.log" 2>&1
|
||||
|
||||
if printf '%s %s\n' "$expected_md5" "$destination" | md5sum --check --status; then
|
||||
printf '[VERIFIED] %s/%s\n' "$subdir" "$filename"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '[CHECKSUM FAILED] %s/%s\n' "$subdir" "$filename" >&2
|
||||
[[ -f "$destination" ]] && mv -- "$destination" "${destination}.corrupt-$(date +%Y%m%dT%H%M%S)"
|
||||
[[ -f "${destination}.aria2" ]] && mv -- "${destination}.aria2" "${destination}.aria2.corrupt-$(date +%Y%m%dT%H%M%S)"
|
||||
done
|
||||
|
||||
printf '[FAILED] checksum mismatch after 3 complete downloads: %s/%s\n' "$subdir" "$filename" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
jobs=(
|
||||
"annotations|body-annotations-male-cns-v1.0-minconf-0.5.feather|50a7718770c57220f160ba4f431ab89e"
|
||||
"annotations|body-neurotransmitters-male-cns-v1.0.feather|3d842b12fe5c49eefade528d7dd24a1f"
|
||||
"connectivity|connectome-weights-male-cns-v1.0-minconf-0.5.feather|f30e9dcca25cfd021bf1e7b3d975599e"
|
||||
)
|
||||
|
||||
pids=()
|
||||
for job in "${jobs[@]}"; do
|
||||
IFS='|' read -r subdir filename expected_md5 <<<"$job"
|
||||
download_one "$subdir" "$filename" "$expected_md5" &
|
||||
pids+=("$!")
|
||||
done
|
||||
|
||||
status=0
|
||||
for pid in "${pids[@]}"; do
|
||||
wait "$pid" || status=1
|
||||
done
|
||||
|
||||
if (( status != 0 )); then
|
||||
printf '[FAILED] one or more files could not be downloaded and verified\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$RAW_ROOT"
|
||||
md5sum --check "${MANIFEST_DIR}/checksums.md5"
|
||||
sha256sum \
|
||||
annotations/*.feather \
|
||||
connectivity/*.feather \
|
||||
>"${MANIFEST_DIR}/checksums.sha256"
|
||||
|
||||
find annotations connectivity -maxdepth 1 -type f -name '*.feather' -printf '%P\t%s bytes\n' \
|
||||
| sort >"${MANIFEST_DIR}/files.tsv"
|
||||
|
||||
printf '[COMPLETE] all MaleCNS v1.0 phase-1 files passed official MD5 verification\n'
|
||||
@@ -0,0 +1,3 @@
|
||||
body-annotations-male-cns-v1.0-minconf-0.5.feather 14483314 bytes
|
||||
body-neurotransmitters-male-cns-v1.0.feather 43282834 bytes
|
||||
connectome-weights-male-cns-v1.0-minconf-0.5.feather 1051241946 bytes
|
||||
|
+73
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
|
||||
readonly SKELETON_DIR="${RAW_ROOT}/skeletons"
|
||||
readonly MANIFEST_DIR="${RAW_ROOT}/_manifests/skeletons"
|
||||
readonly LOG_DIR="${RAW_ROOT}/_logs/skeletons"
|
||||
readonly CONCURRENCY="${MALECNS_SWC_CONCURRENCY:-32}"
|
||||
|
||||
mkdir -p "$SKELETON_DIR" "$MANIFEST_DIR" "$LOG_DIR"
|
||||
|
||||
for required in remote-files.tsv checksums.md5 download.aria2 remote-summary.json; do
|
||||
if [[ ! -s "${MANIFEST_DIR}/${required}" ]]; then
|
||||
printf 'missing manifest: %s\n' "${MANIFEST_DIR}/${required}" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
expected_count="$(jq -r '.swc_file_count' "${MANIFEST_DIR}/remote-summary.json")"
|
||||
expected_bytes="$(jq -r '.total_bytes' "${MANIFEST_DIR}/remote-summary.json")"
|
||||
printf '[REMOTE] files=%s bytes=%s\n' "$expected_count" "$expected_bytes"
|
||||
printf '[DOWNLOAD] concurrency=%s destination=%s\n' "$CONCURRENCY" "$SKELETON_DIR"
|
||||
|
||||
aria2c \
|
||||
--input-file="${MANIFEST_DIR}/download.aria2" \
|
||||
--continue=true \
|
||||
--always-resume=true \
|
||||
--max-resume-failure-tries=20 \
|
||||
--max-tries=0 \
|
||||
--retry-wait=5 \
|
||||
--connect-timeout=30 \
|
||||
--timeout=60 \
|
||||
--max-concurrent-downloads="$CONCURRENCY" \
|
||||
--max-connection-per-server=1 \
|
||||
--split=1 \
|
||||
--file-allocation=none \
|
||||
--auto-file-renaming=false \
|
||||
--allow-overwrite=true \
|
||||
--check-integrity=true \
|
||||
--disk-cache=64M \
|
||||
--console-log-level=notice \
|
||||
--summary-interval=60 \
|
||||
>"${LOG_DIR}/aria2.log" 2>&1
|
||||
|
||||
cd "$SKELETON_DIR"
|
||||
|
||||
actual_count="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | wc -l)"
|
||||
actual_bytes="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%s\n' | awk '{sum += $1} END {print sum + 0}')"
|
||||
empty_count="$(find . -maxdepth 1 -type f -name '*.swc' -size 0 -printf '%f\n' | wc -l)"
|
||||
partial_count="$(find . -maxdepth 1 -type f -name '*.aria2' -printf '%f\n' | wc -l)"
|
||||
|
||||
find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | sort >"${MANIFEST_DIR}/local-files.txt"
|
||||
cut -f1 "${MANIFEST_DIR}/remote-files.tsv" | sort >"${MANIFEST_DIR}/expected-files.txt"
|
||||
comm -23 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/missing-files.txt"
|
||||
comm -13 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/unexpected-files.txt"
|
||||
missing_count="$(wc -l <"${MANIFEST_DIR}/missing-files.txt")"
|
||||
unexpected_count="$(wc -l <"${MANIFEST_DIR}/unexpected-files.txt")"
|
||||
|
||||
printf '[LOCAL] files=%s bytes=%s empty=%s partial=%s missing=%s unexpected=%s\n' \
|
||||
"$actual_count" "$actual_bytes" "$empty_count" "$partial_count" "$missing_count" "$unexpected_count"
|
||||
|
||||
if [[ "$actual_count" != "$expected_count" || "$actual_bytes" != "$expected_bytes" || \
|
||||
"$empty_count" != 0 || "$partial_count" != 0 || "$missing_count" != 0 || "$unexpected_count" != 0 ]]; then
|
||||
printf '[FAILED] local inventory does not match the remote manifest\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! md5sum --check --quiet "${MANIFEST_DIR}/checksums.md5" >"${LOG_DIR}/md5-verification.log" 2>&1; then
|
||||
printf '[FAILED] one or more SWC files failed the official GCS MD5 check\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '[COMPLETE] all SWC files match count, size, names, and official MD5 checksums\n'
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Enumerate the public MaleCNS v1.0 SWC objects and build download manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
BUCKET = "flyem-male-cns"
|
||||
PREFIX = "v1.0/segmentation/skeletons-malecns/skeletons-swc/"
|
||||
API_URL = f"https://storage.googleapis.com/storage/v1/b/{BUCKET}/o"
|
||||
OBJECT_BASE_URL = f"https://storage.googleapis.com/{BUCKET}/"
|
||||
|
||||
|
||||
def fetch_json(url: str, attempts: int = 10) -> dict:
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "malecns-swc-enumerator/1.0"})
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return json.load(response)
|
||||
except Exception as exc: # noqa: BLE001 - retries cover transient HTTP/TLS failures
|
||||
if attempt == attempts:
|
||||
raise
|
||||
delay = min(60, 2 ** (attempt - 1))
|
||||
print(f"listing attempt {attempt} failed: {exc}; retrying in {delay}s", file=sys.stderr)
|
||||
time.sleep(delay)
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
|
||||
def atomic_write(path: str, content: str) -> None:
|
||||
temporary = f"{path}.tmp.{os.getpid()}"
|
||||
with open(temporary, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(content)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("output_dir")
|
||||
parser.add_argument("--download-dir", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
objects: list[tuple[str, int, str, str]] = []
|
||||
page_token: str | None = None
|
||||
page_count = 0
|
||||
|
||||
while True:
|
||||
query = {
|
||||
"prefix": PREFIX,
|
||||
"maxResults": "1000",
|
||||
"fields": "items(name,size,md5Hash),nextPageToken",
|
||||
}
|
||||
if page_token:
|
||||
query["pageToken"] = page_token
|
||||
payload = fetch_json(f"{API_URL}?{urllib.parse.urlencode(query)}")
|
||||
page_count += 1
|
||||
|
||||
for item in payload.get("items", []):
|
||||
name = item["name"]
|
||||
if not name.endswith(".swc"):
|
||||
continue
|
||||
relative_name = name[len(PREFIX) :]
|
||||
if not relative_name or "/" in relative_name:
|
||||
raise RuntimeError(f"unexpected nested SWC object: {name}")
|
||||
md5_hex = base64.b64decode(item["md5Hash"]).hex()
|
||||
url = OBJECT_BASE_URL + urllib.parse.quote(name, safe="/")
|
||||
objects.append((relative_name, int(item["size"]), md5_hex, url))
|
||||
|
||||
page_token = payload.get("nextPageToken")
|
||||
if page_count % 25 == 0:
|
||||
print(f"listed {page_count} pages, {len(objects)} SWC files", file=sys.stderr)
|
||||
if not page_token:
|
||||
break
|
||||
|
||||
objects.sort(key=lambda row: row[0])
|
||||
names = [row[0] for row in objects]
|
||||
if len(names) != len(set(names)):
|
||||
raise RuntimeError("duplicate SWC basenames in remote listing")
|
||||
|
||||
total_bytes = sum(row[1] for row in objects)
|
||||
listing_digest = hashlib.sha256(
|
||||
"".join(f"{name}\t{size}\t{md5_hex}\n" for name, size, md5_hex, _ in objects).encode()
|
||||
).hexdigest()
|
||||
|
||||
files_tsv = "".join(
|
||||
f"{name}\t{size}\t{md5_hex}\t{url}\n" for name, size, md5_hex, url in objects
|
||||
)
|
||||
checksums_md5 = "".join(f"{md5_hex} {name}\n" for name, _, md5_hex, _ in objects)
|
||||
aria2_input = "".join(
|
||||
f"{url}\n dir={args.download_dir}\n out={name}\n checksum=md5={md5_hex}\n"
|
||||
for name, _, md5_hex, url in objects
|
||||
)
|
||||
summary = {
|
||||
"bucket": BUCKET,
|
||||
"prefix": PREFIX,
|
||||
"swc_file_count": len(objects),
|
||||
"total_bytes": total_bytes,
|
||||
"listing_pages": page_count,
|
||||
"listing_sha256": listing_digest,
|
||||
"generated_at_unix": int(time.time()),
|
||||
}
|
||||
|
||||
atomic_write(os.path.join(args.output_dir, "remote-files.tsv"), files_tsv)
|
||||
atomic_write(os.path.join(args.output_dir, "checksums.md5"), checksums_md5)
|
||||
atomic_write(os.path.join(args.output_dir, "download.aria2"), aria2_input)
|
||||
atomic_write(os.path.join(args.output_dir, "remote-summary.json"), json.dumps(summary, indent=2) + "\n")
|
||||
print(json.dumps(summary, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"bucket": "flyem-male-cns",
|
||||
"prefix": "v1.0/segmentation/skeletons-malecns/skeletons-swc/",
|
||||
"swc_file_count": 211573,
|
||||
"total_bytes": 9299975999,
|
||||
"listing_pages": 212,
|
||||
"listing_sha256": "8d81e5cf21daac97518319ed79baad52431968be6b4057f419112bc05f848942",
|
||||
"generated_at_unix": 1789756376
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user