Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

2026-09-29 03:38:32 +08:00
commit 3981628ff8
682 changed files with 161475 additions and 0 deletions
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Observe actual file schemas and copy small provenance records without rewriting data."""
+import gzip
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+root = Path.cwd()
+out = root / "registry/data-20260929"
+out.mkdir(exist_ok=False)
+with gzip.open(root/"registry/snapshots/bootstrap-20260929/files.jsonl.gz", "rt") as f:
+    rows = [json.loads(x) for x in f]
+data = [r for r in rows if r["path"].startswith("datasets/")]
+canonical = "datasets/male-cns-v1.0/processed/neuron-only/"
+files, schemas = [], []
+for r in data:
+    p = root / r["path"]
+    if r["path"].startswith(canonical) or ("/raw/" in r["path"] and p.suffix==".feather"):
+        files.append(r)
+        if p.suffix in (".feather", ".parquet", ".npy"):
+            result = {"path":r["path"], "sha256":r["sha256"]}
+            try:
+                if p.suffix==".feather":
+                    with pa.memory_map(str(p), "r") as source:
+                        schema = pa.ipc.open_file(source).schema
+                    result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in schema]
+                elif p.suffix==".parquet":
+                    pf = pq.ParquetFile(p)
+                    result["rows"] = pf.metadata.num_rows
+                    result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in pf.schema_arrow]
+                else:
+                    a = np.load(p, mmap_mode="r", allow_pickle=False)
+                    result.update(dtype=str(a.dtype), shape=list(a.shape))
+                    del a
+                result["status"] = "OBSERVED"
+            except Exception as e:
+                result.update(status="UNKNOWN", error=str(e))
+            schemas.append(result)
+copies = []
+for r in data:
+    p = root / r["path"]
+    # Existing small authoritative records; no code execution or regeneration.
+    if r["bytes"] > 2*1024*1024 or p.suffix.lower() not in {".md",".json",".txt",".sha256",".md5",".tsv",".sh",".py"}:
+        continue
+    if not (r["path"].startswith(canonical) or "/raw/_manifests/" in r["path"]):
+        continue
+    target = out / "evidence" / r["path"]
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with p.open("rb") as source, target.open("xb") as dest:
+        shutil.copyfileobj(source, dest)
+    if hashlib.sha256(target.read_bytes()).hexdigest() != r["sha256"]:
+        raise RuntimeError("Copied provenance mismatch: "+r["path"])
+    copies.append({"source":r["path"], "copy":str(target.relative_to(out)), "sha256":r["sha256"]})
+sources = {
+    "dataset":"MaleCNS v1.0", "formal_paper_doi":"10.1016/j.cell.2026.08.015",
+    "download_base_from_original_script":"https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome",
+    "download_source_evidence":"datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
+    "license_status":"UNKNOWN / redistribution not asserted by this repository",
+    "canonical_directory":canonical, "raw_and_processed_unchanged":True,
+    "schema_reads_are_not_new_scientific_validation":True,
+    "key_files":files,"copied_evidence":copies,
+    "full_inventory":"registry/snapshots/bootstrap-20260929/files.jsonl.gz"
+}
+for name, obj in (("dataset.json",sources),("observed-schemas.json",schemas)):
+    with (out/name).open("x",encoding="utf-8") as f:
+        json.dump(obj,f,indent=2,ensure_ascii=False)
+        f.write("\n")
+print(json.dumps({"key_files":len(files),"schema_files":len(schemas),"schema_unknown":sum(x["status"]!="OBSERVED" for x in schemas),"evidence_copies":len(copies)}))