Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

2026-09-29 03:38:32 +08:00
commit 3981628ff8
682 changed files with 161475 additions and 0 deletions
@@ -0,0 +1,3 @@
+50a7718770c57220f160ba4f431ab89e  annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather
+3d842b12fe5c49eefade528d7dd24a1f  annotations/body-neurotransmitters-male-cns-v1.0.feather
+f30e9dcca25cfd021bf1e7b3d975599e  connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather
@@ -0,0 +1,3 @@
+2177e246113e4cfbf1e7772ec37c6da1955ff22e8063d0b1f833101f99a9a3b2  annotations/body-annotations-male-cns-v1.0-minconf-0.5.feather
+95c9289220663abeb3409f3ad9e5a7f8a53f8093f5139d15502cd08da8879621  annotations/body-neurotransmitters-male-cns-v1.0.feather
+e35da783d1c686b2b58b3b87cd6a403ae43bfcfba8bff28e08ef752c1a56afc1  connectivity/connectome-weights-male-cns-v1.0-minconf-0.5.feather
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
+readonly BASE_URL="https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome"
+readonly LOG_DIR="${RAW_ROOT}/_logs"
+readonly MANIFEST_DIR="${RAW_ROOT}/_manifests"
+
+mkdir -p "${RAW_ROOT}/annotations" "${RAW_ROOT}/connectivity" "$LOG_DIR" "$MANIFEST_DIR"
+
+download_one() {
+    local subdir="$1"
+    local filename="$2"
+    local expected_md5="$3"
+    local destination="${RAW_ROOT}/${subdir}/${filename}"
+    local attempt
+
+    if [[ -f "$destination" ]] && printf '%s  %s\n' "$expected_md5" "$destination" | md5sum --check --status; then
+        printf '[OK] already verified: %s/%s\n' "$subdir" "$filename"
+        return 0
+    fi
+
+    for attempt in 1 2 3; do
+        if [[ -f "$destination" && ! -f "${destination}.aria2" ]]; then
+            mv -- "$destination" "${destination}.corrupt-$(date +%Y%m%dT%H%M%S)"
+        fi
+
+        printf '[DOWNLOAD] attempt %d: %s/%s\n' "$attempt" "$subdir" "$filename"
+        aria2c \
+            --continue=true \
+            --max-tries=0 \
+            --retry-wait=10 \
+            --connect-timeout=30 \
+            --timeout=60 \
+            --max-connection-per-server=8 \
+            --split=8 \
+            --min-split-size=16M \
+            --file-allocation=falloc \
+            --auto-file-renaming=false \
+            --allow-overwrite=false \
+            --check-integrity=true \
+            --console-log-level=notice \
+            --summary-interval=60 \
+            --dir="${RAW_ROOT}/${subdir}" \
+            --out="$filename" \
+            "${BASE_URL}/${filename}" \
+            >"${LOG_DIR}/${filename}.log" 2>&1
+
+        if printf '%s  %s\n' "$expected_md5" "$destination" | md5sum --check --status; then
+            printf '[VERIFIED] %s/%s\n' "$subdir" "$filename"
+            return 0
+        fi
+
+        printf '[CHECKSUM FAILED] %s/%s\n' "$subdir" "$filename" >&2
+        [[ -f "$destination" ]] && mv -- "$destination" "${destination}.corrupt-$(date +%Y%m%dT%H%M%S)"
+        [[ -f "${destination}.aria2" ]] && mv -- "${destination}.aria2" "${destination}.aria2.corrupt-$(date +%Y%m%dT%H%M%S)"
+    done
+
+    printf '[FAILED] checksum mismatch after 3 complete downloads: %s/%s\n' "$subdir" "$filename" >&2
+    return 1
+}
+
+jobs=(
+    "annotations|body-annotations-male-cns-v1.0-minconf-0.5.feather|50a7718770c57220f160ba4f431ab89e"
+    "annotations|body-neurotransmitters-male-cns-v1.0.feather|3d842b12fe5c49eefade528d7dd24a1f"
+    "connectivity|connectome-weights-male-cns-v1.0-minconf-0.5.feather|f30e9dcca25cfd021bf1e7b3d975599e"
+)
+
+pids=()
+for job in "${jobs[@]}"; do
+    IFS='|' read -r subdir filename expected_md5 <<<"$job"
+    download_one "$subdir" "$filename" "$expected_md5" &
+    pids+=("$!")
+done
+
+status=0
+for pid in "${pids[@]}"; do
+    wait "$pid" || status=1
+done
+
+if (( status != 0 )); then
+    printf '[FAILED] one or more files could not be downloaded and verified\n' >&2
+    exit 1
+fi
+
+cd "$RAW_ROOT"
+md5sum --check "${MANIFEST_DIR}/checksums.md5"
+sha256sum \
+    annotations/*.feather \
+    connectivity/*.feather \
+    >"${MANIFEST_DIR}/checksums.sha256"
+
+find annotations connectivity -maxdepth 1 -type f -name '*.feather' -printf '%P\t%s bytes\n' \
+    | sort >"${MANIFEST_DIR}/files.tsv"
+
+printf '[COMPLETE] all MaleCNS v1.0 phase-1 files passed official MD5 verification\n'
@@ -0,0 +1,3 @@
+body-annotations-male-cns-v1.0-minconf-0.5.feather	14483314 bytes
+body-neurotransmitters-male-cns-v1.0.feather	43282834 bytes
+connectome-weights-male-cns-v1.0-minconf-0.5.feather	1051241946 bytes
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
+readonly SKELETON_DIR="${RAW_ROOT}/skeletons"
+readonly MANIFEST_DIR="${RAW_ROOT}/_manifests/skeletons"
+readonly LOG_DIR="${RAW_ROOT}/_logs/skeletons"
+readonly CONCURRENCY="${MALECNS_SWC_CONCURRENCY:-32}"
+
+mkdir -p "$SKELETON_DIR" "$MANIFEST_DIR" "$LOG_DIR"
+
+for required in remote-files.tsv checksums.md5 download.aria2 remote-summary.json; do
+    if [[ ! -s "${MANIFEST_DIR}/${required}" ]]; then
+        printf 'missing manifest: %s\n' "${MANIFEST_DIR}/${required}" >&2
+        exit 2
+    fi
+done
+
+expected_count="$(jq -r '.swc_file_count' "${MANIFEST_DIR}/remote-summary.json")"
+expected_bytes="$(jq -r '.total_bytes' "${MANIFEST_DIR}/remote-summary.json")"
+printf '[REMOTE] files=%s bytes=%s\n' "$expected_count" "$expected_bytes"
+printf '[DOWNLOAD] concurrency=%s destination=%s\n' "$CONCURRENCY" "$SKELETON_DIR"
+
+aria2c \
+    --input-file="${MANIFEST_DIR}/download.aria2" \
+    --continue=true \
+    --always-resume=true \
+    --max-resume-failure-tries=20 \
+    --max-tries=0 \
+    --retry-wait=5 \
+    --connect-timeout=30 \
+    --timeout=60 \
+    --max-concurrent-downloads="$CONCURRENCY" \
+    --max-connection-per-server=1 \
+    --split=1 \
+    --file-allocation=none \
+    --auto-file-renaming=false \
+    --allow-overwrite=true \
+    --check-integrity=true \
+    --disk-cache=64M \
+    --console-log-level=notice \
+    --summary-interval=60 \
+    >"${LOG_DIR}/aria2.log" 2>&1
+
+cd "$SKELETON_DIR"
+
+actual_count="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | wc -l)"
+actual_bytes="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%s\n' | awk '{sum += $1} END {print sum + 0}')"
+empty_count="$(find . -maxdepth 1 -type f -name '*.swc' -size 0 -printf '%f\n' | wc -l)"
+partial_count="$(find . -maxdepth 1 -type f -name '*.aria2' -printf '%f\n' | wc -l)"
+
+find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | sort >"${MANIFEST_DIR}/local-files.txt"
+cut -f1 "${MANIFEST_DIR}/remote-files.tsv" | sort >"${MANIFEST_DIR}/expected-files.txt"
+comm -23 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/missing-files.txt"
+comm -13 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/unexpected-files.txt"
+missing_count="$(wc -l <"${MANIFEST_DIR}/missing-files.txt")"
+unexpected_count="$(wc -l <"${MANIFEST_DIR}/unexpected-files.txt")"
+
+printf '[LOCAL] files=%s bytes=%s empty=%s partial=%s missing=%s unexpected=%s\n' \
+    "$actual_count" "$actual_bytes" "$empty_count" "$partial_count" "$missing_count" "$unexpected_count"
+
+if [[ "$actual_count" != "$expected_count" || "$actual_bytes" != "$expected_bytes" || \
+      "$empty_count" != 0 || "$partial_count" != 0 || "$missing_count" != 0 || "$unexpected_count" != 0 ]]; then
+    printf '[FAILED] local inventory does not match the remote manifest\n' >&2
+    exit 1
+fi
+
+if ! md5sum --check --quiet "${MANIFEST_DIR}/checksums.md5" >"${LOG_DIR}/md5-verification.log" 2>&1; then
+    printf '[FAILED] one or more SWC files failed the official GCS MD5 check\n' >&2
+    exit 1
+fi
+
+printf '[COMPLETE] all SWC files match count, size, names, and official MD5 checksums\n'
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Enumerate the public MaleCNS v1.0 SWC objects and build download manifests."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+
+BUCKET = "flyem-male-cns"
+PREFIX = "v1.0/segmentation/skeletons-malecns/skeletons-swc/"
+API_URL = f"https://storage.googleapis.com/storage/v1/b/{BUCKET}/o"
+OBJECT_BASE_URL = f"https://storage.googleapis.com/{BUCKET}/"
+
+
+def fetch_json(url: str, attempts: int = 10) -> dict:
+    for attempt in range(1, attempts + 1):
+        try:
+            request = urllib.request.Request(url, headers={"User-Agent": "malecns-swc-enumerator/1.0"})
+            with urllib.request.urlopen(request, timeout=60) as response:
+                return json.load(response)
+        except Exception as exc:  # noqa: BLE001 - retries cover transient HTTP/TLS failures
+            if attempt == attempts:
+                raise
+            delay = min(60, 2 ** (attempt - 1))
+            print(f"listing attempt {attempt} failed: {exc}; retrying in {delay}s", file=sys.stderr)
+            time.sleep(delay)
+    raise RuntimeError("unreachable")
+
+
+def atomic_write(path: str, content: str) -> None:
+    temporary = f"{path}.tmp.{os.getpid()}"
+    with open(temporary, "w", encoding="utf-8", newline="\n") as handle:
+        handle.write(content)
+        handle.flush()
+        os.fsync(handle.fileno())
+    os.replace(temporary, path)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("output_dir")
+    parser.add_argument("--download-dir", required=True)
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    objects: list[tuple[str, int, str, str]] = []
+    page_token: str | None = None
+    page_count = 0
+
+    while True:
+        query = {
+            "prefix": PREFIX,
+            "maxResults": "1000",
+            "fields": "items(name,size,md5Hash),nextPageToken",
+        }
+        if page_token:
+            query["pageToken"] = page_token
+        payload = fetch_json(f"{API_URL}?{urllib.parse.urlencode(query)}")
+        page_count += 1
+
+        for item in payload.get("items", []):
+            name = item["name"]
+            if not name.endswith(".swc"):
+                continue
+            relative_name = name[len(PREFIX) :]
+            if not relative_name or "/" in relative_name:
+                raise RuntimeError(f"unexpected nested SWC object: {name}")
+            md5_hex = base64.b64decode(item["md5Hash"]).hex()
+            url = OBJECT_BASE_URL + urllib.parse.quote(name, safe="/")
+            objects.append((relative_name, int(item["size"]), md5_hex, url))
+
+        page_token = payload.get("nextPageToken")
+        if page_count % 25 == 0:
+            print(f"listed {page_count} pages, {len(objects)} SWC files", file=sys.stderr)
+        if not page_token:
+            break
+
+    objects.sort(key=lambda row: row[0])
+    names = [row[0] for row in objects]
+    if len(names) != len(set(names)):
+        raise RuntimeError("duplicate SWC basenames in remote listing")
+
+    total_bytes = sum(row[1] for row in objects)
+    listing_digest = hashlib.sha256(
+        "".join(f"{name}\t{size}\t{md5_hex}\n" for name, size, md5_hex, _ in objects).encode()
+    ).hexdigest()
+
+    files_tsv = "".join(
+        f"{name}\t{size}\t{md5_hex}\t{url}\n" for name, size, md5_hex, url in objects
+    )
+    checksums_md5 = "".join(f"{md5_hex}  {name}\n" for name, _, md5_hex, _ in objects)
+    aria2_input = "".join(
+        f"{url}\n  dir={args.download_dir}\n  out={name}\n  checksum=md5={md5_hex}\n"
+        for name, _, md5_hex, url in objects
+    )
+    summary = {
+        "bucket": BUCKET,
+        "prefix": PREFIX,
+        "swc_file_count": len(objects),
+        "total_bytes": total_bytes,
+        "listing_pages": page_count,
+        "listing_sha256": listing_digest,
+        "generated_at_unix": int(time.time()),
+    }
+
+    atomic_write(os.path.join(args.output_dir, "remote-files.tsv"), files_tsv)
+    atomic_write(os.path.join(args.output_dir, "checksums.md5"), checksums_md5)
+    atomic_write(os.path.join(args.output_dir, "download.aria2"), aria2_input)
+    atomic_write(os.path.join(args.output_dir, "remote-summary.json"), json.dumps(summary, indent=2) + "\n")
+    print(json.dumps(summary, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,9 @@
+{
+  "bucket": "flyem-male-cns",
+  "prefix": "v1.0/segmentation/skeletons-malecns/skeletons-swc/",
+  "swc_file_count": 211573,
+  "total_bytes": 9299975999,
+  "listing_pages": 212,
+  "listing_sha256": "8d81e5cf21daac97518319ed79baad52431968be6b4057f419112bc05f848942",
+  "generated_at_unix": 1789756376
+}