Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

2026-09-29 03:38:32 +08:00
commit 3981628ff8
682 changed files with 161475 additions and 0 deletions
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly RAW_ROOT="${1:-/data/digital-drosophila/datasets/male-cns-v1.0/raw}"
+readonly SKELETON_DIR="${RAW_ROOT}/skeletons"
+readonly MANIFEST_DIR="${RAW_ROOT}/_manifests/skeletons"
+readonly LOG_DIR="${RAW_ROOT}/_logs/skeletons"
+readonly CONCURRENCY="${MALECNS_SWC_CONCURRENCY:-32}"
+
+mkdir -p "$SKELETON_DIR" "$MANIFEST_DIR" "$LOG_DIR"
+
+for required in remote-files.tsv checksums.md5 download.aria2 remote-summary.json; do
+    if [[ ! -s "${MANIFEST_DIR}/${required}" ]]; then
+        printf 'missing manifest: %s\n' "${MANIFEST_DIR}/${required}" >&2
+        exit 2
+    fi
+done
+
+expected_count="$(jq -r '.swc_file_count' "${MANIFEST_DIR}/remote-summary.json")"
+expected_bytes="$(jq -r '.total_bytes' "${MANIFEST_DIR}/remote-summary.json")"
+printf '[REMOTE] files=%s bytes=%s\n' "$expected_count" "$expected_bytes"
+printf '[DOWNLOAD] concurrency=%s destination=%s\n' "$CONCURRENCY" "$SKELETON_DIR"
+
+aria2c \
+    --input-file="${MANIFEST_DIR}/download.aria2" \
+    --continue=true \
+    --always-resume=true \
+    --max-resume-failure-tries=20 \
+    --max-tries=0 \
+    --retry-wait=5 \
+    --connect-timeout=30 \
+    --timeout=60 \
+    --max-concurrent-downloads="$CONCURRENCY" \
+    --max-connection-per-server=1 \
+    --split=1 \
+    --file-allocation=none \
+    --auto-file-renaming=false \
+    --allow-overwrite=true \
+    --check-integrity=true \
+    --disk-cache=64M \
+    --console-log-level=notice \
+    --summary-interval=60 \
+    >"${LOG_DIR}/aria2.log" 2>&1
+
+cd "$SKELETON_DIR"
+
+actual_count="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | wc -l)"
+actual_bytes="$(find . -maxdepth 1 -type f -name '*.swc' -printf '%s\n' | awk '{sum += $1} END {print sum + 0}')"
+empty_count="$(find . -maxdepth 1 -type f -name '*.swc' -size 0 -printf '%f\n' | wc -l)"
+partial_count="$(find . -maxdepth 1 -type f -name '*.aria2' -printf '%f\n' | wc -l)"
+
+find . -maxdepth 1 -type f -name '*.swc' -printf '%f\n' | sort >"${MANIFEST_DIR}/local-files.txt"
+cut -f1 "${MANIFEST_DIR}/remote-files.tsv" | sort >"${MANIFEST_DIR}/expected-files.txt"
+comm -23 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/missing-files.txt"
+comm -13 "${MANIFEST_DIR}/expected-files.txt" "${MANIFEST_DIR}/local-files.txt" >"${MANIFEST_DIR}/unexpected-files.txt"
+missing_count="$(wc -l <"${MANIFEST_DIR}/missing-files.txt")"
+unexpected_count="$(wc -l <"${MANIFEST_DIR}/unexpected-files.txt")"
+
+printf '[LOCAL] files=%s bytes=%s empty=%s partial=%s missing=%s unexpected=%s\n' \
+    "$actual_count" "$actual_bytes" "$empty_count" "$partial_count" "$missing_count" "$unexpected_count"
+
+if [[ "$actual_count" != "$expected_count" || "$actual_bytes" != "$expected_bytes" || \
+      "$empty_count" != 0 || "$partial_count" != 0 || "$missing_count" != 0 || "$unexpected_count" != 0 ]]; then
+    printf '[FAILED] local inventory does not match the remote manifest\n' >&2
+    exit 1
+fi
+
+if ! md5sum --check --quiet "${MANIFEST_DIR}/checksums.md5" >"${LOG_DIR}/md5-verification.log" 2>&1; then
+    printf '[FAILED] one or more SWC files failed the official GCS MD5 check\n' >&2
+    exit 1
+fi
+
+printf '[COMPLETE] all SWC files match count, size, names, and official MD5 checksums\n'
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Enumerate the public MaleCNS v1.0 SWC objects and build download manifests."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+
+BUCKET = "flyem-male-cns"
+PREFIX = "v1.0/segmentation/skeletons-malecns/skeletons-swc/"
+API_URL = f"https://storage.googleapis.com/storage/v1/b/{BUCKET}/o"
+OBJECT_BASE_URL = f"https://storage.googleapis.com/{BUCKET}/"
+
+
+def fetch_json(url: str, attempts: int = 10) -> dict:
+    for attempt in range(1, attempts + 1):
+        try:
+            request = urllib.request.Request(url, headers={"User-Agent": "malecns-swc-enumerator/1.0"})
+            with urllib.request.urlopen(request, timeout=60) as response:
+                return json.load(response)
+        except Exception as exc:  # noqa: BLE001 - retries cover transient HTTP/TLS failures
+            if attempt == attempts:
+                raise
+            delay = min(60, 2 ** (attempt - 1))
+            print(f"listing attempt {attempt} failed: {exc}; retrying in {delay}s", file=sys.stderr)
+            time.sleep(delay)
+    raise RuntimeError("unreachable")
+
+
+def atomic_write(path: str, content: str) -> None:
+    temporary = f"{path}.tmp.{os.getpid()}"
+    with open(temporary, "w", encoding="utf-8", newline="\n") as handle:
+        handle.write(content)
+        handle.flush()
+        os.fsync(handle.fileno())
+    os.replace(temporary, path)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("output_dir")
+    parser.add_argument("--download-dir", required=True)
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    objects: list[tuple[str, int, str, str]] = []
+    page_token: str | None = None
+    page_count = 0
+
+    while True:
+        query = {
+            "prefix": PREFIX,
+            "maxResults": "1000",
+            "fields": "items(name,size,md5Hash),nextPageToken",
+        }
+        if page_token:
+            query["pageToken"] = page_token
+        payload = fetch_json(f"{API_URL}?{urllib.parse.urlencode(query)}")
+        page_count += 1
+
+        for item in payload.get("items", []):
+            name = item["name"]
+            if not name.endswith(".swc"):
+                continue
+            relative_name = name[len(PREFIX) :]
+            if not relative_name or "/" in relative_name:
+                raise RuntimeError(f"unexpected nested SWC object: {name}")
+            md5_hex = base64.b64decode(item["md5Hash"]).hex()
+            url = OBJECT_BASE_URL + urllib.parse.quote(name, safe="/")
+            objects.append((relative_name, int(item["size"]), md5_hex, url))
+
+        page_token = payload.get("nextPageToken")
+        if page_count % 25 == 0:
+            print(f"listed {page_count} pages, {len(objects)} SWC files", file=sys.stderr)
+        if not page_token:
+            break
+
+    objects.sort(key=lambda row: row[0])
+    names = [row[0] for row in objects]
+    if len(names) != len(set(names)):
+        raise RuntimeError("duplicate SWC basenames in remote listing")
+
+    total_bytes = sum(row[1] for row in objects)
+    listing_digest = hashlib.sha256(
+        "".join(f"{name}\t{size}\t{md5_hex}\n" for name, size, md5_hex, _ in objects).encode()
+    ).hexdigest()
+
+    files_tsv = "".join(
+        f"{name}\t{size}\t{md5_hex}\t{url}\n" for name, size, md5_hex, url in objects
+    )
+    checksums_md5 = "".join(f"{md5_hex}  {name}\n" for name, _, md5_hex, _ in objects)
+    aria2_input = "".join(
+        f"{url}\n  dir={args.download_dir}\n  out={name}\n  checksum=md5={md5_hex}\n"
+        for name, _, md5_hex, url in objects
+    )
+    summary = {
+        "bucket": BUCKET,
+        "prefix": PREFIX,
+        "swc_file_count": len(objects),
+        "total_bytes": total_bytes,
+        "listing_pages": page_count,
+        "listing_sha256": listing_digest,
+        "generated_at_unix": int(time.time()),
+    }
+
+    atomic_write(os.path.join(args.output_dir, "remote-files.tsv"), files_tsv)
+    atomic_write(os.path.join(args.output_dir, "checksums.md5"), checksums_md5)
+    atomic_write(os.path.join(args.output_dir, "download.aria2"), aria2_input)
+    atomic_write(os.path.join(args.output_dir, "remote-summary.json"), json.dumps(summary, indent=2) + "\n")
+    print(json.dumps(summary, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,9 @@
+{
+  "bucket": "flyem-male-cns",
+  "prefix": "v1.0/segmentation/skeletons-malecns/skeletons-swc/",
+  "swc_file_count": 211573,
+  "total_bytes": 9299975999,
+  "listing_pages": 212,
+  "listing_sha256": "8d81e5cf21daac97518319ed79baad52431968be6b4057f419112bc05f848942",
+  "generated_at_unix": 1789756376
+}