digital-drosophila/tools/repository_audit.py

#!/usr/bin/env python3
"""Read-only project inventory; writes only to a NEW explicit output directory.

No model imports, simulations, data migration, deletion, or dependency installation.
Python >=3.10, standard library only. Snapshot excludes reproducible environment/cache
directories and nested Git databases, never follows symlinks. Verify checks additions too.
"""
import argparse
import concurrent.futures as cf
import gzip
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
from datetime import datetime, timezone

ROOTS = ("datasets", "experiments", "models", "backend", "frontend", "game", "logs", "recordings")
EXCLUDE = {".git", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", ".cache", "node_modules"}
TEXT_EXT = {".py", ".md", ".json", ".yaml", ".yml", ".toml", ".txt", ".sh", ".csv", ".tsv"}

def dump(path, obj):
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("x", encoding="utf-8", newline="\n") as f:
        json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True)
        f.write("\n")

def sha(path):
    h = hashlib.sha256()
    with path.open("rb") as f:
        for block in iter(lambda: f.read(4 * 1024 * 1024), b""):
            h.update(block)
    return h.hexdigest()

def entries(root):
    for name in ROOTS:
        base = root / name
        if not base.exists():
            continue
        for folder, dirs, files in os.walk(base, followlinks=False):
            dirs[:] = sorted(d for d in dirs if d not in EXCLUDE)
            for d in dirs[:]:
                p = Path(folder) / d
                if p.is_symlink():
                    yield p
                    dirs.remove(d)
            for f in sorted(files):
                yield Path(folder) / f

def record(root, path):
    st = path.lstat()
    row = {"path": path.relative_to(root).as_posix(), "bytes": st.st_size,
           "mtime_ns": st.st_mtime_ns, "mode": oct(st.st_mode & 0o7777)}
    if path.is_symlink():
        row.update(kind="symlink", target=os.readlink(path))
    elif path.is_file():
        row.update(kind="file", sha256=sha(path))
        after = path.stat()
        if (st.st_size, st.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
            raise RuntimeError("File changed during hashing: " + row["path"])
    else:
        raise RuntimeError("Unexpected special file: " + str(path))
    return row

def snapshot(args):
    root, out = args.root.resolve(), args.output.resolve()
    if out.exists():
        raise RuntimeError("Refusing existing output directory")
    out.mkdir(parents=True)
    paths = sorted(entries(root))
    counts = {}
    with gzip.open(out / "files.jsonl.gz", "xt", encoding="utf-8") as f, cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
        for n, row in enumerate(ex.map(lambda p: record(root, p), paths), 1):
            f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
            key = row["path"].split("/")[0]
            c = counts.setdefault(key, {"files": 0, "logical_bytes": 0})
            c["files"] += 1
            c["logical_bytes"] += row["bytes"]
            if n % 25000 == 0:
                print(f"hashed {n}/{len(paths)}", flush=True)
    summary = {"schema_version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
               "root": str(root), "roots": ROOTS, "excluded_directory_names": sorted(EXCLUDE),
               "files": len(paths), "groups": counts, "symlinks_followed": False,
               "files_jsonl_gz_sha256": sha(out / "files.jsonl.gz"),
               "not_a_data_backup": True, "historical_authenticity_claim": False}
    dump(out / "summary.json", summary)
    print(json.dumps(summary, ensure_ascii=False), flush=True)

def verify(args):
    root = args.root.resolve()
    summary = json.loads((args.snapshot / "summary.json").read_text())
    manifest = args.snapshot / "files.jsonl.gz"
    if sha(manifest) != summary["files_jsonl_gz_sha256"]:
        raise RuntimeError("Snapshot manifest checksum mismatch")
    with gzip.open(manifest, "rt", encoding="utf-8") as f:
        rows = [json.loads(line) for line in f]
    def check(row):
        p = root / row["path"]
        if p.is_absolute() and not p.absolute().is_relative_to(root):
            return {"path": row["path"], "problem": "unsafe path"}
        if ".." in Path(row["path"]).parts:
            return {"path": row["path"], "problem": "unsafe path"}
        try:
            now = record(root, p)
            keys = ("kind", "target", "mode") if row["kind"] == "symlink" else ("kind", "bytes", "sha256", "mode")
            different = [k for k in keys if row.get(k) != now.get(k)]
            return {"path": row["path"], "changed_fields": different} if different else None
        except (OSError, RuntimeError) as e:
            return {"path": row["path"], "problem": str(e)}
    with cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
        mismatches = [r for r in ex.map(check, rows) if r]
    before = {r["path"] for r in rows}
    after = {p.relative_to(root).as_posix() for p in entries(root)}
    result = {"checked": len(rows), "mismatches": mismatches, "added": sorted(after-before),
              "missing": sorted(before-after), "checked_utc": datetime.now(timezone.utc).isoformat(),
              "snapshot_sha256": sha(manifest), "excluded_directory_names": sorted(EXCLUDE),
              "pass": not mismatches and before == after}
    if args.output:
        dump(args.output, result)
    print(json.dumps(result, ensure_ascii=False), flush=True)
    return 0 if result["pass"] else 1

def run(root, *cmd):
    p = subprocess.run(cmd, cwd=root, text=True, capture_output=True, check=False)
    return {"exit_code": p.returncode, "stdout": p.stdout.strip(), "stderr": p.stderr.strip()}

def catalog(args):
    root, out = args.root.resolve(), args.output.resolve()
    if out.exists():
        raise RuntimeError("Refusing existing output directory")
    out.mkdir(parents=True)
    stages, evidence, refs, resources = [], [], [], []
    for d in sorted((root / "experiments").iterdir()):
        if not d.is_dir():
            continue
        selected = []
        # Small audit/config/result evidence only. Retain originals byte-for-byte.
        for p in sorted(d.rglob("*")):
            rel = p.relative_to(d)
            if "external-resources" in rel.parts or any(x in EXCLUDE for x in rel.parts):
                continue
            if not p.is_file() or p.is_symlink() or p.stat().st_size > 512 * 1024:
                continue
            suffix = p.suffix.lower()
            include = suffix in {".md", ".yaml", ".yml"} or (
                suffix == ".json" and (len(rel.parts) == 1 or rel.parts[0] in {
                    "audit", "regression", "literature-audit", "numerical-contract", "configs",
                    "config", "revalidation", "phase4c-v2", "manifests"} or "summary" in p.name or "gate" in p.name))
            if include:
                dest = out / "evidence" / d.name / rel
                dest.parent.mkdir(parents=True, exist_ok=True)
                with p.open("rb") as src, dest.open("xb") as dst:
                    shutil.copyfileobj(src, dst)
                item = {"source_path": p.relative_to(root).as_posix(), "copy_path": dest.relative_to(out).as_posix(),
                        "sha256": sha(p), "bytes": p.stat().st_size}
                evidence.append(item)
                selected.append(item["copy_path"])
        stages.append({"stage_directory": d.name, "evidence": selected,
                       "status_policy": "Use original evidence; completion is not a scientific PASS"})
    # Index citations in project-authored sources, configs, provenance and literature notes.
    for p in entries(root):
        rel = p.relative_to(root)
        if p.is_symlink() or p.suffix.lower() not in TEXT_EXT or p.stat().st_size > 2 * 1024 * 1024:
            continue
        if "external-resources" in rel.parts or "skeletons" in rel.parts:
            continue
        body = p.read_text(encoding="utf-8", errors="replace")
        urls = sorted(set(re.findall(r"https?://[^\s<>\"'`]+", body)))
        dois = sorted(set(x.rstrip(".,;)]}") for x in re.findall(r"10\.\d{4,9}/[-._;()/:A-Za-z0-9]+", body)))
        # Never export URLs containing user credentials or common secret query parameters.
        urls = [u.rstrip(".,;)]}") for u in urls if not re.search(r"https?://[^/]*@|[?&](?:token|key|secret|password|signature)=", u, re.I)]
        if urls or dois:
            refs.append({"source_path": rel.as_posix(), "sha256": sha(p), "urls": urls, "doi_candidates": dois,
                         "review_status": "extracted-not-automatically-endorsed"})
    known = [
        ("experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer", "CC-BY-4.0", "LICENSE"),
        ("experiments/phase4f-retinotopic-vision/external-resources/visualpathways", "GPL-3.0 license text; grant/version options require upstream review", "LICENSE"),
        ("experiments/phase4f-retinotopic-vision/external-resources/flyvis", "MIT declared by pyproject classifier; standalone LICENSE absent in frozen checkout", "pyproject.toml"),
        ("experiments/phase4l-olfactory-grounding/external-resources/DoOR.data", "CC BY-SA 4.0 declared in DESCRIPTION", "DESCRIPTION"),
    ]
    for value, license_id, license_file in known:
        p = root / value
        if not p.exists():
            resources.append({"path": value, "status": "MISSING"})
            continue
        lp = p / license_file
        resources.append({"path": value, "commit": run(p, "git", "rev-parse", "HEAD"),
                          "origin": run(p, "git", "remote", "get-url", "origin"),
                          "worktree_status": run(p, "git", "--no-optional-locks", "status", "--short"),
                          "license_observation": license_id, "license_evidence": str(lp.relative_to(root)),
                          "license_evidence_sha256": sha(lp) if lp.is_file() else None,
                          "vendored_in_parent_git": False})
    dump(out / "stages.json", stages)
    dump(out / "evidence-index.json", evidence)
    dump(out / "source-citations.json", refs)
    dump(out / "external-resources.lock.json", resources)
    env = {"captured_utc": datetime.now(timezone.utc).isoformat(),
           "python": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "--version"),
           "packages": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "-c",
                           "import importlib.metadata as m,json;print(json.dumps(sorted((x.metadata['Name'],x.version) for x in m.distributions())))"),
           "gpu": run(root, "nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"),
           "os": run(root, "uname", "-a"), "git": run(root, "git", "--version"),
           "nvcc": run(root, "sh", "-c", "command -v nvcc >/dev/null && nvcc --version || true"),
           "historical_environment_guaranteed": False}
    dump(out / "environment.json", env)
    print(json.dumps({"stages":len(stages), "evidence_files":len(evidence), "citation_source_files":len(refs), "external_resources":len(resources)}))

def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--root", type=Path, default=Path("."))
    subs = ap.add_subparsers(dest="cmd", required=True)
    s = subs.add_parser("snapshot"); s.add_argument("--output", type=Path, required=True); s.add_argument("--workers", type=int, default=8)
    v = subs.add_parser("verify"); v.add_argument("--snapshot", type=Path, required=True); v.add_argument("--output", type=Path); v.add_argument("--workers", type=int, default=8)
    c = subs.add_parser("catalog"); c.add_argument("--output", type=Path, required=True)
    args = ap.parse_args()
    return {"snapshot":snapshot, "verify":verify, "catalog":catalog}[args.cmd](args)

if __name__ == "__main__":
    sys.exit(main())