Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

tools/build_bibliography.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Fetch DOI metadata from Crossref into new files; no paper full-text copying."""
+import argparse
+import concurrent.futures as cf
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import re
+import time
+import urllib.request
+import urllib.parse
+
+CORE = {
+    "10.1016/j.cell.2026.08.015": "Formal MaleCNS Cell 2026; distinguish release/schema from publication counts",
+    "10.1016/j.cell.2026.08.014": "Official MaleCNS visual pathways",
+    "10.1038/s41586-024-07763-9": "Shiu-like LIF reference; not a strict replication claim",
+    "10.1038/s41586-024-07939-3": "FlyVis independent visual model",
+    "10.1038/srep21841": "DoOR physiology / odor-receptor-glomerular reference",
+    "10.1016/j.cell.2024.03.016": "EM neurotransmitter prediction and limitations",
+    "10.7554/eLife.03868": "DPM physiology; not a universal edge sign assignment",
+    "10.7554/eLife.56954": "Localized APL inhibition",
+    "10.7554/eLife.74172": "APL / PN-bouton and KC-dendrite normalization",
+    "10.1073/pnas.0914912107": "MB to antennal-lobe feedback physiology",
+    "10.1016/j.cell.2006.12.034": "Excitatory antennal-lobe local circuits",
+    "10.1016/j.neuron.2010.08.025": "Excitatory local interneuron connectivity",
+    "10.1016/j.cell.2019.05.040": "Timing-dependent dopamine plasticity",
+    "10.1038/s41467-023-38487-5": "KC subthreshold response and decision timing",
+    "10.1038/s41586-025-09029-4": "FlyBody / Google DeepMind; separate physical body/controller research",
+    "10.7554/eLife.76712": "KC acetylcholine release plasticity",
+    "10.1523/ENEURO.0275-23.2023": "Compartment-specific KC release and bouton heterogeneity",
+    "10.1016/j.cell.2015.11.019": "Compartment and timing dependence of transmission",
+    "10.7554/eLife.80445": "Postsynaptic cholinergic memory mechanisms",
+    "10.1523/JNEUROSCI.1420-19.2019": "Pre/post synaptic odor suppression",
+}
+
+def normalize(s):
+    # Only known extraction artifacts; do not invent or fuzzy-match DOI.
+    return s.rstrip(".,;:)]}").removesuffix("/full").lower()
+
+def json_new(path, obj):
+    with path.open("x", encoding="utf-8", newline="\n") as f:
+        json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True)
+        f.write("\n")
+
+def escape(s):
+    return str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%").replace("_", "\\_")
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--catalog", type=Path, required=True)
+    ap.add_argument("--output", type=Path, required=True)
+    args = ap.parse_args()
+    out = args.output
+    out.mkdir(parents=True, exist_ok=True)
+    for filename in ("papers.json", "papers.bib", "papers.md", "crossref"):
+        if (out / filename).exists():
+            raise RuntimeError("Refusing to overwrite " + filename)
+    (out / "crossref").mkdir()
+    items = {normalize(d): {"doi":d, "use":u, "source_files":[], "extracted_forms":[]} for d,u in CORE.items()}
+    for r in json.loads(args.catalog.read_text(encoding="utf-8")):
+        for d in r["doi_candidates"]:
+            key = normalize(d)
+            row = items.setdefault(key, {"doi":key, "use":"Historical citation; consult source file for context", "source_files":[], "extracted_forms":[]})
+            row["source_files"].append(r["source_path"])
+            row["extracted_forms"].append(d)
+    def fetch(pair):
+        key, row = pair
+        row["source_files"] = sorted(set(row["source_files"]))
+        row["extracted_forms"] = sorted(set(row["extracted_forms"]))
+        url = "https://api.crossref.org/works/" + urllib.parse.quote(row["doi"], safe="")
+        row.update(metadata_url=url, checked_utc=datetime.now(timezone.utc).isoformat(), fulltext_archived=False)
+        last = None
+        for attempt in range(3):
+            try:
+                request = urllib.request.Request(url, headers={"User-Agent":"DigitalDrosophilaBibliography/1.0 (research archive)", "Accept":"application/json"})
+                with urllib.request.urlopen(request, timeout=45) as r:
+                    raw = r.read()
+                result = json.loads(raw)
+                m = result["message"]
+                if normalize(m["DOI"]) != key:
+                    raise ValueError("Returned DOI differs")
+                token = hashlib.sha256(key.encode()).hexdigest()[:16]
+                with (out / "crossref" / (token + ".json")).open("xb") as f:
+                    f.write(raw)
+                row.update(status="VERIFIED-METADATA", title="; ".join(m.get("title", [])), authors=m.get("author", []),
+                           journal="; ".join(m.get("container-title", [])), published=m.get("published", {}),
+                           publisher=m.get("publisher"), volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"),
+                           doi_url=m.get("URL", "https://doi.org/" + row["doi"]),
+                           publisher_fulltext_links=m.get("link", []), license_metadata=m.get("license", []),
+                           metadata_file="crossref/"+token+".json", metadata_sha256=hashlib.sha256(raw).hexdigest())
+                print("metadata OK", row["doi"], flush=True)
+                return row
+            except Exception as e:
+                last = type(e).__name__ + ": " + str(e)
+                if attempt < 2:
+                    time.sleep(2 * (attempt+1))
+        row.update(status="UNRESOLVED", error=last, doi_url="https://doi.org/"+row["doi"])
+        print("metadata UNRESOLVED", row["doi"], flush=True)
+        return row
+    with cf.ThreadPoolExecutor(max_workers=4) as ex:
+        rows = list(ex.map(fetch, sorted(items.items())))
+    json_new(out / "papers.json", {"schema_version":1, "metadata_provider":"Crossref / publisher-deposited", "papers":rows})
+    with (out / "papers.bib").open("x", encoding="utf-8") as bib, (out / "papers.md").open("x", encoding="utf-8") as md:
+        md.write("# 论文索引\n\n出版元数据由 Crossref 获取；未打包论文全文。许可证据见 JSON，不能默认再分发。\n\n")
+        for i,r in enumerate(rows,1):
+            if r["status"] != "VERIFIED-METADATA":
+                md.write(f"- UNRESOLVED [{r['doi']}]({r['doi_url']})；保留原位引用，未补造标题。\n")
+                continue
+            year = r["published"].get("date-parts", [[None]])[0][0]
+            authors = " and ".join(a.get("family",a.get("name","")) + (", " + a["given"] if a.get("given") else "") for a in r["authors"])
+            bib.write(f"@article{{malecns_ref_{i:03d},\n")
+            for k,v in {"title":r["title"], "author":authors, "year":year, "journal":r["journal"], "doi":r["doi"], "url":r["doi_url"], "volume":r["volume"], "pages":r["pages"]}.items():
+                if v is not None:
+                    bib.write(f"  {k} = {{{escape(v)}}},\n")
+            bib.write("}\n\n")
+            md.write(f"- **{year}** [{r['title']}]({r['doi_url']}) — {r['journal']}。用途：{r['use']}。\n")
+    print(json.dumps({"papers":len(rows), "verified":sum(r['status']=='VERIFIED-METADATA' for r in rows), "unresolved":sum(r['status']=='UNRESOLVED' for r in rows)}))
+
+if __name__ == "__main__":
+    main()

tools/build_data_registry.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Observe actual file schemas and copy small provenance records without rewriting data."""
+import gzip
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+root = Path.cwd()
+out = root / "registry/data-20260929"
+out.mkdir(exist_ok=False)
+with gzip.open(root/"registry/snapshots/bootstrap-20260929/files.jsonl.gz", "rt") as f:
+    rows = [json.loads(x) for x in f]
+data = [r for r in rows if r["path"].startswith("datasets/")]
+canonical = "datasets/male-cns-v1.0/processed/neuron-only/"
+files, schemas = [], []
+for r in data:
+    p = root / r["path"]
+    if r["path"].startswith(canonical) or ("/raw/" in r["path"] and p.suffix==".feather"):
+        files.append(r)
+        if p.suffix in (".feather", ".parquet", ".npy"):
+            result = {"path":r["path"], "sha256":r["sha256"]}
+            try:
+                if p.suffix==".feather":
+                    with pa.memory_map(str(p), "r") as source:
+                        schema = pa.ipc.open_file(source).schema
+                    result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in schema]
+                elif p.suffix==".parquet":
+                    pf = pq.ParquetFile(p)
+                    result["rows"] = pf.metadata.num_rows
+                    result["fields"] = [{"name":x.name,"type":str(x.type),"nullable":x.nullable} for x in pf.schema_arrow]
+                else:
+                    a = np.load(p, mmap_mode="r", allow_pickle=False)
+                    result.update(dtype=str(a.dtype), shape=list(a.shape))
+                    del a
+                result["status"] = "OBSERVED"
+            except Exception as e:
+                result.update(status="UNKNOWN", error=str(e))
+            schemas.append(result)
+copies = []
+for r in data:
+    p = root / r["path"]
+    # Existing small authoritative records; no code execution or regeneration.
+    if r["bytes"] > 2*1024*1024 or p.suffix.lower() not in {".md",".json",".txt",".sha256",".md5",".tsv",".sh",".py"}:
+        continue
+    if not (r["path"].startswith(canonical) or "/raw/_manifests/" in r["path"]):
+        continue
+    target = out / "evidence" / r["path"]
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with p.open("rb") as source, target.open("xb") as dest:
+        shutil.copyfileobj(source, dest)
+    if hashlib.sha256(target.read_bytes()).hexdigest() != r["sha256"]:
+        raise RuntimeError("Copied provenance mismatch: "+r["path"])
+    copies.append({"source":r["path"], "copy":str(target.relative_to(out)), "sha256":r["sha256"]})
+sources = {
+    "dataset":"MaleCNS v1.0", "formal_paper_doi":"10.1016/j.cell.2026.08.015",
+    "download_base_from_original_script":"https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome",
+    "download_source_evidence":"datasets/male-cns-v1.0/raw/_manifests/download-malecns-v1.0-phase1.sh",
+    "license_status":"UNKNOWN / redistribution not asserted by this repository",
+    "canonical_directory":canonical, "raw_and_processed_unchanged":True,
+    "schema_reads_are_not_new_scientific_validation":True,
+    "key_files":files,"copied_evidence":copies,
+    "full_inventory":"registry/snapshots/bootstrap-20260929/files.jsonl.gz"
+}
+for name, obj in (("dataset.json",sources),("observed-schemas.json",schemas)):
+    with (out/name).open("x",encoding="utf-8") as f:
+        json.dump(obj,f,indent=2,ensure_ascii=False)
+        f.write("\n")
+print(json.dumps({"key_files":len(files),"schema_files":len(schemas),"schema_unknown":sum(x["status"]!="OBSERVED" for x in schemas),"evidence_copies":len(copies)}))

tools/finalize_repository_audit.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Validate repository navigation and original evidence, create a NEW receipt."""
+from datetime import datetime, timezone
+import gzip
+import hashlib
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+root = Path.cwd().resolve()
+def sha(p):
+    h = hashlib.sha256()
+    with p.open("rb") as f:
+        for b in iter(lambda:f.read(4*1024*1024), b""):
+            h.update(b)
+    return h.hexdigest()
+
+errors = []
+cat = root / "registry/catalog-20260929"
+copies = json.loads((cat / "evidence-index.json").read_text())
+for r in copies:
+    if sha(root / r["source_path"]) != r["sha256"] or sha(cat / r["copy_path"]) != r["sha256"]:
+        errors.append("evidence checksum: " + r["source_path"])
+
+links = 0
+# New navigation only; old historical reports keep their original links unchanged.
+pages = [root/"README.md", root/"NOTICE.md", *sorted((root/"docs").rglob("*.md")), root/"references/README.md", root/"references/evidence-map.md"]
+for p in pages:
+    for link in re.findall(r"\]\(([^)]+)\)", p.read_text(encoding="utf-8")):
+        if re.match(r"[a-z]+://|#", link):
+            continue
+        links += 1
+        if not (p.parent / link.split("#",1)[0]).exists():
+            errors.append("missing link: " + str(p.relative_to(root)) + " -> " + link)
+
+papers = json.loads((root/"references/papers.json").read_text())["papers"]
+for p in papers:
+    if p["status"] != "VERIFIED-METADATA" or sha(root/"references"/p["metadata_file"]) != p["metadata_sha256"]:
+        errors.append("bibliography metadata: " + p["doi"])
+snapshot = root/"registry/snapshots/bootstrap-20260929/files.jsonl.gz"
+with gzip.open(snapshot, "rt", encoding="utf-8") as f:
+    originals = {r["path"]:r for r in map(json.loads, f)}
+tracked = subprocess.check_output(["git", "ls-files", "-z"]).decode().split("\0")
+models = 0
+for path in filter(None, tracked):
+    if path.startswith("models/"):
+        models += 1
+        old = originals.get(path)
+        blob = subprocess.check_output(["git", "show", ":"+path])
+        if old is None or hashlib.sha256(blob).hexdigest() != old.get("sha256"):
+            errors.append("staged original model differs: " + path)
+verification = json.loads((root/"registry/bootstrap-verification.json").read_text())
+if not verification["pass"]:
+    errors.append("full scientific snapshot verification failed")
+guard = subprocess.run([sys.executable, "-B", "tools/repo_guard.py"], capture_output=True, text=True)
+if guard.returncode:
+    errors.append("staged content guard failed")
+tests = subprocess.run([sys.executable, "-B", "tools/test_repository_audit.py"], capture_output=True, text=True)
+if tests.returncode:
+    errors.append("tool unit tests failed")
+receipt = {"pass":not errors, "errors":errors, "checked_utc":datetime.now(timezone.utc).isoformat(),
+           "evidence_copies_verified":len(copies), "original_model_blobs_verified":models,
+           "navigation_local_links_verified":links, "papers_metadata_verified":len(papers),
+           "scientific_files_verified":verification["checked"], "staged_guard":json.loads(guard.stdout),
+           "tool_tests_exit_code":tests.returncode,
+           "scientific_regressions_reexecuted":False,
+           "data_deleted_moved_or_rewritten":False,
+           "permissions_change":{"scope":"directory ownership only, nonrecursive",
+                                 "paths":["/data/digital-drosophila","/data/digital-drosophila/docs","/data/digital-drosophila/tools"],
+                                 "before":"root:root", "after":"qishizhe:qishizhe"},
+           "large_data_pushed":False, "offsite_full_data_backup_created":False}
+out = root/"registry/repository-validation.json"
+with out.open("x", encoding="utf-8") as f:
+    json.dump(receipt, f, ensure_ascii=False, indent=2)
+    f.write("\n")
+print(json.dumps(receipt, ensure_ascii=False, indent=2))
+sys.exit(int(bool(errors)))

tools/repo_guard.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Check staged Git blobs, not working-tree copies. Does not alter the index."""
+import json
+import re
+import subprocess
+import sys
+
+def git(*args):
+    return subprocess.check_output(["git", *args])
+
+def main():
+    errors, total, count = [], 0, 0
+    for row in git("ls-files", "--stage", "-z").split(b"\0"):
+        if not row:
+            continue
+        meta, rawpath = row.split(b"\t", 1)
+        mode, oid, stage = meta.decode().split()
+        path = rawpath.decode("utf-8")
+        count += 1
+        if mode == "160000" or path.startswith(("datasets/", "experiments/")) or any(p in path.split("/") for p in (".venv", ".git", "__pycache__")):
+            errors.append({"path":path, "rule":"forbidden data/environment/submodule"})
+            continue
+        size = int(git("cat-file", "-s", oid))
+        total += size
+        if size > 25 * 1024 * 1024:
+            errors.append({"path":path, "rule":"blob exceeds 25 MiB policy"})
+            continue
+        if re.search(r"(^|/)(?:\.env(?:\..*)?|id_rsa|id_ed25519)$|\.(?:pem|key|p12)$", path, re.I):
+            errors.append({"path":path, "rule":"credential filename"})
+        if path.endswith(".gz"):
+            continue  # generated hash-only inventory, validated separately
+        data = git("cat-file", "blob", oid).decode("utf-8", errors="replace")
+        rules = {
+            "private key": r"-----BEGIN (?:OPENSSH |RSA |EC )?PRIVATE KEY-----",
+            "access token": r"(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}|AKIA[A-Z0-9]{16})",
+            "credential URL": r"https?://[^\s/]+:[^\s/@]+@",
+            "literal password": r"(?i)(?:password|passwd|api_key|secret_key)\s*[:=]\s*['\"][^'\"\n]{6,}['\"]",
+        }
+        for rule, pattern in rules.items():
+            if re.search(pattern, data):
+                errors.append({"path":path, "rule":rule})
+    print(json.dumps({"pass":not errors, "files":count, "logical_bytes":total, "issues":errors}, indent=2))
+    return int(bool(errors))
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/repository_audit.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""Read-only project inventory; writes only to a NEW explicit output directory.
+
+No model imports, simulations, data migration, deletion, or dependency installation.
+Python >=3.10, standard library only. Snapshot excludes reproducible environment/cache
+directories and nested Git databases, never follows symlinks. Verify checks additions too.
+"""
+import argparse
+import concurrent.futures as cf
+import gzip
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+ROOTS = ("datasets", "experiments", "models", "backend", "frontend", "game", "logs", "recordings")
+EXCLUDE = {".git", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", ".cache", "node_modules"}
+TEXT_EXT = {".py", ".md", ".json", ".yaml", ".yml", ".toml", ".txt", ".sh", ".csv", ".tsv"}
+
+def dump(path, obj):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("x", encoding="utf-8", newline="\n") as f:
+        json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True)
+        f.write("\n")
+
+def sha(path):
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for block in iter(lambda: f.read(4 * 1024 * 1024), b""):
+            h.update(block)
+    return h.hexdigest()
+
+def entries(root):
+    for name in ROOTS:
+        base = root / name
+        if not base.exists():
+            continue
+        for folder, dirs, files in os.walk(base, followlinks=False):
+            dirs[:] = sorted(d for d in dirs if d not in EXCLUDE)
+            for d in dirs[:]:
+                p = Path(folder) / d
+                if p.is_symlink():
+                    yield p
+                    dirs.remove(d)
+            for f in sorted(files):
+                yield Path(folder) / f
+
+def record(root, path):
+    st = path.lstat()
+    row = {"path": path.relative_to(root).as_posix(), "bytes": st.st_size,
+           "mtime_ns": st.st_mtime_ns, "mode": oct(st.st_mode & 0o7777)}
+    if path.is_symlink():
+        row.update(kind="symlink", target=os.readlink(path))
+    elif path.is_file():
+        row.update(kind="file", sha256=sha(path))
+        after = path.stat()
+        if (st.st_size, st.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
+            raise RuntimeError("File changed during hashing: " + row["path"])
+    else:
+        raise RuntimeError("Unexpected special file: " + str(path))
+    return row
+
+def snapshot(args):
+    root, out = args.root.resolve(), args.output.resolve()
+    if out.exists():
+        raise RuntimeError("Refusing existing output directory")
+    out.mkdir(parents=True)
+    paths = sorted(entries(root))
+    counts = {}
+    with gzip.open(out / "files.jsonl.gz", "xt", encoding="utf-8") as f, cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
+        for n, row in enumerate(ex.map(lambda p: record(root, p), paths), 1):
+            f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
+            key = row["path"].split("/")[0]
+            c = counts.setdefault(key, {"files": 0, "logical_bytes": 0})
+            c["files"] += 1
+            c["logical_bytes"] += row["bytes"]
+            if n % 25000 == 0:
+                print(f"hashed {n}/{len(paths)}", flush=True)
+    summary = {"schema_version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
+               "root": str(root), "roots": ROOTS, "excluded_directory_names": sorted(EXCLUDE),
+               "files": len(paths), "groups": counts, "symlinks_followed": False,
+               "files_jsonl_gz_sha256": sha(out / "files.jsonl.gz"),
+               "not_a_data_backup": True, "historical_authenticity_claim": False}
+    dump(out / "summary.json", summary)
+    print(json.dumps(summary, ensure_ascii=False), flush=True)
+
+def verify(args):
+    root = args.root.resolve()
+    summary = json.loads((args.snapshot / "summary.json").read_text())
+    manifest = args.snapshot / "files.jsonl.gz"
+    if sha(manifest) != summary["files_jsonl_gz_sha256"]:
+        raise RuntimeError("Snapshot manifest checksum mismatch")
+    with gzip.open(manifest, "rt", encoding="utf-8") as f:
+        rows = [json.loads(line) for line in f]
+    def check(row):
+        p = root / row["path"]
+        if p.is_absolute() and not p.absolute().is_relative_to(root):
+            return {"path": row["path"], "problem": "unsafe path"}
+        if ".." in Path(row["path"]).parts:
+            return {"path": row["path"], "problem": "unsafe path"}
+        try:
+            now = record(root, p)
+            keys = ("kind", "target", "mode") if row["kind"] == "symlink" else ("kind", "bytes", "sha256", "mode")
+            different = [k for k in keys if row.get(k) != now.get(k)]
+            return {"path": row["path"], "changed_fields": different} if different else None
+        except (OSError, RuntimeError) as e:
+            return {"path": row["path"], "problem": str(e)}
+    with cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
+        mismatches = [r for r in ex.map(check, rows) if r]
+    before = {r["path"] for r in rows}
+    after = {p.relative_to(root).as_posix() for p in entries(root)}
+    result = {"checked": len(rows), "mismatches": mismatches, "added": sorted(after-before),
+              "missing": sorted(before-after), "checked_utc": datetime.now(timezone.utc).isoformat(),
+              "snapshot_sha256": sha(manifest), "excluded_directory_names": sorted(EXCLUDE),
+              "pass": not mismatches and before == after}
+    if args.output:
+        dump(args.output, result)
+    print(json.dumps(result, ensure_ascii=False), flush=True)
+    return 0 if result["pass"] else 1
+
+def run(root, *cmd):
+    p = subprocess.run(cmd, cwd=root, text=True, capture_output=True, check=False)
+    return {"exit_code": p.returncode, "stdout": p.stdout.strip(), "stderr": p.stderr.strip()}
+
+def catalog(args):
+    root, out = args.root.resolve(), args.output.resolve()
+    if out.exists():
+        raise RuntimeError("Refusing existing output directory")
+    out.mkdir(parents=True)
+    stages, evidence, refs, resources = [], [], [], []
+    for d in sorted((root / "experiments").iterdir()):
+        if not d.is_dir():
+            continue
+        selected = []
+        # Small audit/config/result evidence only. Retain originals byte-for-byte.
+        for p in sorted(d.rglob("*")):
+            rel = p.relative_to(d)
+            if "external-resources" in rel.parts or any(x in EXCLUDE for x in rel.parts):
+                continue
+            if not p.is_file() or p.is_symlink() or p.stat().st_size > 512 * 1024:
+                continue
+            suffix = p.suffix.lower()
+            include = suffix in {".md", ".yaml", ".yml"} or (
+                suffix == ".json" and (len(rel.parts) == 1 or rel.parts[0] in {
+                    "audit", "regression", "literature-audit", "numerical-contract", "configs",
+                    "config", "revalidation", "phase4c-v2", "manifests"} or "summary" in p.name or "gate" in p.name))
+            if include:
+                dest = out / "evidence" / d.name / rel
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                with p.open("rb") as src, dest.open("xb") as dst:
+                    shutil.copyfileobj(src, dst)
+                item = {"source_path": p.relative_to(root).as_posix(), "copy_path": dest.relative_to(out).as_posix(),
+                        "sha256": sha(p), "bytes": p.stat().st_size}
+                evidence.append(item)
+                selected.append(item["copy_path"])
+        stages.append({"stage_directory": d.name, "evidence": selected,
+                       "status_policy": "Use original evidence; completion is not a scientific PASS"})
+    # Index citations in project-authored sources, configs, provenance and literature notes.
+    for p in entries(root):
+        rel = p.relative_to(root)
+        if p.is_symlink() or p.suffix.lower() not in TEXT_EXT or p.stat().st_size > 2 * 1024 * 1024:
+            continue
+        if "external-resources" in rel.parts or "skeletons" in rel.parts:
+            continue
+        body = p.read_text(encoding="utf-8", errors="replace")
+        urls = sorted(set(re.findall(r"https?://[^\s<>\"'`]+", body)))
+        dois = sorted(set(x.rstrip(".,;)]}") for x in re.findall(r"10\.\d{4,9}/[-._;()/:A-Za-z0-9]+", body)))
+        # Never export URLs containing user credentials or common secret query parameters.
+        urls = [u.rstrip(".,;)]}") for u in urls if not re.search(r"https?://[^/]*@|[?&](?:token|key|secret|password|signature)=", u, re.I)]
+        if urls or dois:
+            refs.append({"source_path": rel.as_posix(), "sha256": sha(p), "urls": urls, "doi_candidates": dois,
+                         "review_status": "extracted-not-automatically-endorsed"})
+    known = [
+        ("experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer", "CC-BY-4.0", "LICENSE"),
+        ("experiments/phase4f-retinotopic-vision/external-resources/visualpathways", "GPL-3.0 license text; grant/version options require upstream review", "LICENSE"),
+        ("experiments/phase4f-retinotopic-vision/external-resources/flyvis", "MIT declared by pyproject classifier; standalone LICENSE absent in frozen checkout", "pyproject.toml"),
+        ("experiments/phase4l-olfactory-grounding/external-resources/DoOR.data", "CC BY-SA 4.0 declared in DESCRIPTION", "DESCRIPTION"),
+    ]
+    for value, license_id, license_file in known:
+        p = root / value
+        if not p.exists():
+            resources.append({"path": value, "status": "MISSING"})
+            continue
+        lp = p / license_file
+        resources.append({"path": value, "commit": run(p, "git", "rev-parse", "HEAD"),
+                          "origin": run(p, "git", "remote", "get-url", "origin"),
+                          "worktree_status": run(p, "git", "--no-optional-locks", "status", "--short"),
+                          "license_observation": license_id, "license_evidence": str(lp.relative_to(root)),
+                          "license_evidence_sha256": sha(lp) if lp.is_file() else None,
+                          "vendored_in_parent_git": False})
+    dump(out / "stages.json", stages)
+    dump(out / "evidence-index.json", evidence)
+    dump(out / "source-citations.json", refs)
+    dump(out / "external-resources.lock.json", resources)
+    env = {"captured_utc": datetime.now(timezone.utc).isoformat(),
+           "python": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "--version"),
+           "packages": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "-c",
+                           "import importlib.metadata as m,json;print(json.dumps(sorted((x.metadata['Name'],x.version) for x in m.distributions())))"),
+           "gpu": run(root, "nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"),
+           "os": run(root, "uname", "-a"), "git": run(root, "git", "--version"),
+           "nvcc": run(root, "sh", "-c", "command -v nvcc >/dev/null && nvcc --version || true"),
+           "historical_environment_guaranteed": False}
+    dump(out / "environment.json", env)
+    print(json.dumps({"stages":len(stages), "evidence_files":len(evidence), "citation_source_files":len(refs), "external_resources":len(resources)}))
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--root", type=Path, default=Path("."))
+    subs = ap.add_subparsers(dest="cmd", required=True)
+    s = subs.add_parser("snapshot"); s.add_argument("--output", type=Path, required=True); s.add_argument("--workers", type=int, default=8)
+    v = subs.add_parser("verify"); v.add_argument("--snapshot", type=Path, required=True); v.add_argument("--output", type=Path); v.add_argument("--workers", type=int, default=8)
+    c = subs.add_parser("catalog"); c.add_argument("--output", type=Path, required=True)
+    args = ap.parse_args()
+    return {"snapshot":snapshot, "verify":verify, "catalog":catalog}[args.cmd](args)
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/seal_repository.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Seal currently staged file bytes into a new SHA-256 file; no scientific writes."""
+import hashlib
+from pathlib import Path
+import subprocess
+
+root = Path.cwd()
+target = root/"registry/repository-files.sha256"
+if target.exists():
+    raise RuntimeError("Refusing to overwrite repository seal")
+lines = []
+for name in sorted(filter(None, subprocess.check_output(["git","ls-files","-z"]).decode().split("\0"))):
+    blob = subprocess.check_output(["git","show",":"+name])
+    actual = (root/name).read_bytes()
+    if blob != actual:
+        raise RuntimeError("Staged/worktree bytes differ: "+name)
+    lines.append(hashlib.sha256(blob).hexdigest()+"  "+name+"\n")
+with target.open("x",encoding="utf-8",newline="\n") as f:
+    f.writelines(lines)
+print("sealed files:",len(lines))

tools/test_repository_audit.py

@@ -0,0 +1,36 @@
+"""Tests use fresh temporary directories, never scientific project data."""
+import gzip
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+TOOL = Path(__file__).with_name("repository_audit.py")
+
+class AuditTests(unittest.TestCase):
+    def test_snapshot_detects_change_and_addition(self):
+        with tempfile.TemporaryDirectory(prefix="drosophila-audit-test-") as tmp:
+            root = Path(tmp)
+            (root / "datasets").mkdir()
+            (root / "datasets/a.bin").write_bytes(b"raw\0data")
+            (root / "datasets/.venv").mkdir()
+            (root / "datasets/.venv/excluded").write_bytes(b"environment")
+            snap = root / "snapshot"
+            cmd = [sys.executable, "-B", str(TOOL), "--root", str(root)]
+            subprocess.run(cmd+["snapshot", "--output", str(snap)], check=True, capture_output=True)
+            with gzip.open(snap / "files.jsonl.gz", "rt") as f:
+                self.assertEqual(len([json.loads(x) for x in f]), 1)
+            self.assertEqual(subprocess.run(cmd+["verify", "--snapshot", str(snap)], capture_output=True).returncode, 0)
+            self.assertNotEqual(subprocess.run(cmd+["snapshot", "--output", str(snap)], capture_output=True).returncode, 0)
+            (root / "datasets/a.bin").write_bytes(b"changed!")
+            (root / "datasets/new.bin").write_bytes(b"new")
+            check = subprocess.run(cmd+["verify", "--snapshot", str(snap)], capture_output=True)
+            result = json.loads(check.stdout)
+            self.assertEqual(check.returncode, 1)
+            self.assertEqual(result["added"], ["datasets/new.bin"])
+            self.assertEqual(result["mismatches"][0]["path"], "datasets/a.bin")
+
+if __name__ == "__main__":
+    unittest.main()

tools/verify_repository_files.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+"""Verify committed repository payload on any clone, without data or model imports."""
+import hashlib
+from pathlib import Path
+import sys
+
+root = Path(__file__).resolve().parents[1]
+manifest = root / "registry/repository-files.sha256"
+errors, count = [], 0
+for line in manifest.read_text(encoding="utf-8").splitlines():
+    digest, name = line.split("  ", 1)
+    path = Path(name)
+    if path.is_absolute() or ".." in path.parts:
+        errors.append("unsafe path: " + name)
+        continue
+    target = root / path
+    count += 1
+    if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest() != digest:
+        errors.append(name)
+print(f"verified={count} mismatches={len(errors)}")
+for error in errors:
+    print(error)
+sys.exit(int(bool(errors)))