Digital Drosophila Archive
73 lines
3.6 KiB
Python
73 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Observe actual file schemas and copy small provenance records without rewriting data."""
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import numpy as np
|
|
import pyarrow as pa
|
|
import pyarrow.parquet as pq
|
|
|
|
root = Path.cwd()
|
|
out = root / "registry/data-20260929"
|
|
out.mkdir(exist_ok=False)
|
|
with gzip.open(root/"registry/snapshots/bootstrap-20260929/files.jsonl.gz", "rt") as f:
|
|
rows = [json.loads(x) for x in f]
|
|
data = [r for r in rows if r["path"].startswith("datasets/")]
|
|
canonical = "datasets/male-cns-v1.0/processed/neuron-only/"
|
|
files, schemas = [], []
|
|
for r in data:
|
|
p = root / r["path"]
|
|
if r["path"].startswith(canonical) or ("/raw/" in r["path"] and p.suffix==".feather"):
|
|
files.append(r)
|
|
if p.suffix in (".feather", ".parquet", ".npy"):
|
|
result = {"path":r["path"], "sha256":r["sha256"]}
|
|
try:
|
|
if p.suffix==".feather":
|
|
with pa.memory_map(str(p), "r") as source:
|
|
schema = pa.ipc.open_file(source).schema
|
|
result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in schema]
|
|
elif p.suffix==".parquet":
|
|
pf = pq.ParquetFile(p)
|
|
result["rows"] = pf.metadata.num_rows
|
|
result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in pf.schema_arrow]
|
|
else:
|
|
a = np.load(p, mmap_mode="r", allow_pickle=False)
|
|
result.update(dtype=str(a.dtype), shape=list(a.shape))
|
|
del a
|
|
result["status"] = "OBSERVED"
|
|
except Exception as e:
|
|
result.update(status="UNKNOWN", error=str(e))
|
|
schemas.append(result)
|
|
copies = []
|
|
for r in data:
|
|
p = root / r["path"]
|
|
# Existing small authoritative records; no code execution or regeneration.
|
|
if r["bytes"] > 2*1024*1024 or p.suffix.lower() not in {".md",".json",".txt",".sha256",".md5",".tsv",".sh",".py"}:
|
|
continue
|
|
if not (r["path"].startswith(canonical) or "/raw/_manifests/" in r["path"]):
|
|
continue
|
|
target = out / "evidence" / r["path"]
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with p.open("rb") as source, target.open("xb") as dest:
|
|
shutil.copyfileobj(source, dest)
|
|
if hashlib.sha256(target.read_bytes()).hexdigest() != r["sha256"]:
|
|
raise RuntimeError("Copied provenance mismatch: "+r["path"])
|
|
copies.append({"source":r["path"], "copy":str(target.relative_to(out)), "sha256":r["sha256"]})
|
|
sources = {
|
|
"dataset":"MaleCNS v1.0", "formal_paper_doi":"10.1016/j.cell.2026.08.015",
|
|
"download_base_from_original_script":"https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome",
|
|
"download_source_evidence":"datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
|
|
"license_status":"UNKNOWN / redistribution not asserted by this repository",
|
|
"canonical_directory":canonical, "raw_and_processed_unchanged":True,
|
|
"schema_reads_are_not_new_scientific_validation":True,
|
|
"key_files":files,"copied_evidence":copies,
|
|
"full_inventory":"registry/snapshots/bootstrap-20260929/files.jsonl.gz"
|
|
}
|
|
for name, obj in (("dataset.json",sources),("observed-schemas.json",schemas)):
|
|
with (out/name).open("x",encoding="utf-8") as f:
|
|
json.dump(obj,f,indent=2,ensure_ascii=False)
|
|
f.write("\n")
|
|
print(json.dumps({"key_files":len(files),"schema_files":len(schemas),"schema_unknown":sum(x["status"]!="OBSERVED" for x in schemas),"evidence_copies":len(copies)}))
|