#!/usr/bin/env python3
"""Observe actual file schemas and copy small provenance records without rewriting data."""
import gzip
import hashlib
import json
from pathlib import Path
import shutil
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq

root = Path.cwd()
out = root / "registry/data-20260929"
out.mkdir(exist_ok=False)
with gzip.open(root/"registry/snapshots/bootstrap-20260929/files.jsonl.gz", "rt") as f:
    rows = [json.loads(x) for x in f]
data = [r for r in rows if r["path"].startswith("datasets/")]
canonical = "datasets/male-cns-v1.0/processed/neuron-only/"
files, schemas = [], []
for r in data:
    p = root / r["path"]
    if r["path"].startswith(canonical) or ("/raw/" in r["path"] and p.suffix==".feather"):
        files.append(r)
        if p.suffix in (".feather", ".parquet", ".npy"):
            result = {"path":r["path"], "sha256":r["sha256"]}
            try:
                if p.suffix==".feather":
                    with pa.memory_map(str(p), "r") as source:
                        schema = pa.ipc.open_file(source).schema
                    result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in schema]
                elif p.suffix==".parquet":
                    pf = pq.ParquetFile(p)
                    result["rows"] = pf.metadata.num_rows
                    result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in pf.schema_arrow]
                else:
                    a = np.load(p, mmap_mode="r", allow_pickle=False)
                    result.update(dtype=str(a.dtype), shape=list(a.shape))
                    del a
                result["status"] = "OBSERVED"
            except Exception as e:
                result.update(status="UNKNOWN", error=str(e))
            schemas.append(result)
copies = []
for r in data:
    p = root / r["path"]
    # Existing small authoritative records; no code execution or regeneration.
    if r["bytes"] > 2*1024*1024 or p.suffix.lower() not in {".md",".json",".txt",".sha256",".md5",".tsv",".sh",".py"}:
        continue
    if not (r["path"].startswith(canonical) or "/raw/_manifests/" in r["path"]):
        continue
    target = out / "evidence" / r["path"]
    target.parent.mkdir(parents=True, exist_ok=True)
    with p.open("rb") as source, target.open("xb") as dest:
        shutil.copyfileobj(source, dest)
    if hashlib.sha256(target.read_bytes()).hexdigest() != r["sha256"]:
        raise RuntimeError("Copied provenance mismatch: "+r["path"])
    copies.append({"source":r["path"], "copy":str(target.relative_to(out)), "sha256":r["sha256"]})
sources = {
    "dataset":"MaleCNS v1.0", "formal_paper_doi":"10.1016/j.cell.2026.08.015",
    "download_base_from_original_script":"https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome",
    "download_source_evidence":"datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
    "license_status":"UNKNOWN / redistribution not asserted by this repository",
    "canonical_directory":canonical, "raw_and_processed_unchanged":True,
    "schema_reads_are_not_new_scientific_validation":True,
    "key_files":files,"copied_evidence":copies,
    "full_inventory":"registry/snapshots/bootstrap-20260929/files.jsonl.gz"
}
for name, obj in (("dataset.json",sources),("observed-schemas.json",schemas)):
    with (out/name).open("x",encoding="utf-8") as f:
        json.dump(obj,f,indent=2,ensure_ascii=False)
        f.write("\n")
print(json.dumps({"key_files":len(files),"schema_files":len(schemas),"schema_unknown":sum(x["status"]!="OBSERVED" for x in schemas),"evidence_copies":len(copies)}))