#!/usr/bin/env python3 """Read-only project inventory; writes only to a NEW explicit output directory. No model imports, simulations, data migration, deletion, or dependency installation. Python >=3.10, standard library only. Snapshot excludes reproducible environment/cache directories and nested Git databases, never follows symlinks. Verify checks additions too. """ import argparse import concurrent.futures as cf import gzip import hashlib import json import os from pathlib import Path import re import shutil import subprocess import sys from datetime import datetime, timezone ROOTS = ("datasets", "experiments", "models", "backend", "frontend", "game", "logs", "recordings") EXCLUDE = {".git", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", ".cache", "node_modules"} TEXT_EXT = {".py", ".md", ".json", ".yaml", ".yml", ".toml", ".txt", ".sh", ".csv", ".tsv"} def dump(path, obj): path.parent.mkdir(parents=True, exist_ok=True) with path.open("x", encoding="utf-8", newline="\n") as f: json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True) f.write("\n") def sha(path): h = hashlib.sha256() with path.open("rb") as f: for block in iter(lambda: f.read(4 * 1024 * 1024), b""): h.update(block) return h.hexdigest() def entries(root): for name in ROOTS: base = root / name if not base.exists(): continue for folder, dirs, files in os.walk(base, followlinks=False): dirs[:] = sorted(d for d in dirs if d not in EXCLUDE) for d in dirs[:]: p = Path(folder) / d if p.is_symlink(): yield p dirs.remove(d) for f in sorted(files): yield Path(folder) / f def record(root, path): st = path.lstat() row = {"path": path.relative_to(root).as_posix(), "bytes": st.st_size, "mtime_ns": st.st_mtime_ns, "mode": oct(st.st_mode & 0o7777)} if path.is_symlink(): row.update(kind="symlink", target=os.readlink(path)) elif path.is_file(): row.update(kind="file", sha256=sha(path)) after = path.stat() if (st.st_size, st.st_mtime_ns) != (after.st_size, after.st_mtime_ns): raise RuntimeError("File changed during hashing: " + row["path"]) else: raise RuntimeError("Unexpected special file: " + str(path)) return row def snapshot(args): root, out = args.root.resolve(), args.output.resolve() if out.exists(): raise RuntimeError("Refusing existing output directory") out.mkdir(parents=True) paths = sorted(entries(root)) counts = {} with gzip.open(out / "files.jsonl.gz", "xt", encoding="utf-8") as f, cf.ThreadPoolExecutor(max_workers=args.workers) as ex: for n, row in enumerate(ex.map(lambda p: record(root, p), paths), 1): f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") key = row["path"].split("/")[0] c = counts.setdefault(key, {"files": 0, "logical_bytes": 0}) c["files"] += 1 c["logical_bytes"] += row["bytes"] if n % 25000 == 0: print(f"hashed {n}/{len(paths)}", flush=True) summary = {"schema_version": 1, "created_utc": datetime.now(timezone.utc).isoformat(), "root": str(root), "roots": ROOTS, "excluded_directory_names": sorted(EXCLUDE), "files": len(paths), "groups": counts, "symlinks_followed": False, "files_jsonl_gz_sha256": sha(out / "files.jsonl.gz"), "not_a_data_backup": True, "historical_authenticity_claim": False} dump(out / "summary.json", summary) print(json.dumps(summary, ensure_ascii=False), flush=True) def verify(args): root = args.root.resolve() summary = json.loads((args.snapshot / "summary.json").read_text()) manifest = args.snapshot / "files.jsonl.gz" if sha(manifest) != summary["files_jsonl_gz_sha256"]: raise RuntimeError("Snapshot manifest checksum mismatch") with gzip.open(manifest, "rt", encoding="utf-8") as f: rows = [json.loads(line) for line in f] def check(row): p = root / row["path"] if p.is_absolute() and not p.absolute().is_relative_to(root): return {"path": row["path"], "problem": "unsafe path"} if ".." in Path(row["path"]).parts: return {"path": row["path"], "problem": "unsafe path"} try: now = record(root, p) keys = ("kind", "target", "mode") if row["kind"] == "symlink" else ("kind", "bytes", "sha256", "mode") different = [k for k in keys if row.get(k) != now.get(k)] return {"path": row["path"], "changed_fields": different} if different else None except (OSError, RuntimeError) as e: return {"path": row["path"], "problem": str(e)} with cf.ThreadPoolExecutor(max_workers=args.workers) as ex: mismatches = [r for r in ex.map(check, rows) if r] before = {r["path"] for r in rows} after = {p.relative_to(root).as_posix() for p in entries(root)} result = {"checked": len(rows), "mismatches": mismatches, "added": sorted(after-before), "missing": sorted(before-after), "checked_utc": datetime.now(timezone.utc).isoformat(), "snapshot_sha256": sha(manifest), "excluded_directory_names": sorted(EXCLUDE), "pass": not mismatches and before == after} if args.output: dump(args.output, result) print(json.dumps(result, ensure_ascii=False), flush=True) return 0 if result["pass"] else 1 def run(root, *cmd): p = subprocess.run(cmd, cwd=root, text=True, capture_output=True, check=False) return {"exit_code": p.returncode, "stdout": p.stdout.strip(), "stderr": p.stderr.strip()} def catalog(args): root, out = args.root.resolve(), args.output.resolve() if out.exists(): raise RuntimeError("Refusing existing output directory") out.mkdir(parents=True) stages, evidence, refs, resources = [], [], [], [] for d in sorted((root / "experiments").iterdir()): if not d.is_dir(): continue selected = [] # Small audit/config/result evidence only. Retain originals byte-for-byte. for p in sorted(d.rglob("*")): rel = p.relative_to(d) if "external-resources" in rel.parts or any(x in EXCLUDE for x in rel.parts): continue if not p.is_file() or p.is_symlink() or p.stat().st_size > 512 * 1024: continue suffix = p.suffix.lower() include = suffix in {".md", ".yaml", ".yml"} or ( suffix == ".json" and (len(rel.parts) == 1 or rel.parts[0] in { "audit", "regression", "literature-audit", "numerical-contract", "configs", "config", "revalidation", "phase4c-v2", "manifests"} or "summary" in p.name or "gate" in p.name)) if include: dest = out / "evidence" / d.name / rel dest.parent.mkdir(parents=True, exist_ok=True) with p.open("rb") as src, dest.open("xb") as dst: shutil.copyfileobj(src, dst) item = {"source_path": p.relative_to(root).as_posix(), "copy_path": dest.relative_to(out).as_posix(), "sha256": sha(p), "bytes": p.stat().st_size} evidence.append(item) selected.append(item["copy_path"]) stages.append({"stage_directory": d.name, "evidence": selected, "status_policy": "Use original evidence; completion is not a scientific PASS"}) # Index citations in project-authored sources, configs, provenance and literature notes. for p in entries(root): rel = p.relative_to(root) if p.is_symlink() or p.suffix.lower() not in TEXT_EXT or p.stat().st_size > 2 * 1024 * 1024: continue if "external-resources" in rel.parts or "skeletons" in rel.parts: continue body = p.read_text(encoding="utf-8", errors="replace") urls = sorted(set(re.findall(r"https?://[^\s<>\"'`]+", body))) dois = sorted(set(x.rstrip(".,;)]}") for x in re.findall(r"10\.\d{4,9}/[-._;()/:A-Za-z0-9]+", body))) # Never export URLs containing user credentials or common secret query parameters. urls = [u.rstrip(".,;)]}") for u in urls if not re.search(r"https?://[^/]*@|[?&](?:token|key|secret|password|signature)=", u, re.I)] if urls or dois: refs.append({"source_path": rel.as_posix(), "sha256": sha(p), "urls": urls, "doi_candidates": dois, "review_status": "extracted-not-automatically-endorsed"}) known = [ ("experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer", "CC-BY-4.0", "LICENSE"), ("experiments/phase4f-retinotopic-vision/external-resources/visualpathways", "GPL-3.0 license text; grant/version options require upstream review", "LICENSE"), ("experiments/phase4f-retinotopic-vision/external-resources/flyvis", "MIT declared by pyproject classifier; standalone LICENSE absent in frozen checkout", "pyproject.toml"), ("experiments/phase4l-olfactory-grounding/external-resources/DoOR.data", "CC BY-SA 4.0 declared in DESCRIPTION", "DESCRIPTION"), ] for value, license_id, license_file in known: p = root / value if not p.exists(): resources.append({"path": value, "status": "MISSING"}) continue lp = p / license_file resources.append({"path": value, "commit": run(p, "git", "rev-parse", "HEAD"), "origin": run(p, "git", "remote", "get-url", "origin"), "worktree_status": run(p, "git", "--no-optional-locks", "status", "--short"), "license_observation": license_id, "license_evidence": str(lp.relative_to(root)), "license_evidence_sha256": sha(lp) if lp.is_file() else None, "vendored_in_parent_git": False}) dump(out / "stages.json", stages) dump(out / "evidence-index.json", evidence) dump(out / "source-citations.json", refs) dump(out / "external-resources.lock.json", resources) env = {"captured_utc": datetime.now(timezone.utc).isoformat(), "python": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "--version"), "packages": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "-c", "import importlib.metadata as m,json;print(json.dumps(sorted((x.metadata['Name'],x.version) for x in m.distributions())))"), "gpu": run(root, "nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"), "os": run(root, "uname", "-a"), "git": run(root, "git", "--version"), "nvcc": run(root, "sh", "-c", "command -v nvcc >/dev/null && nvcc --version || true"), "historical_environment_guaranteed": False} dump(out / "environment.json", env) print(json.dumps({"stages":len(stages), "evidence_files":len(evidence), "citation_source_files":len(refs), "external_resources":len(resources)})) def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--root", type=Path, default=Path(".")) subs = ap.add_subparsers(dest="cmd", required=True) s = subs.add_parser("snapshot"); s.add_argument("--output", type=Path, required=True); s.add_argument("--workers", type=int, default=8) v = subs.add_parser("verify"); v.add_argument("--snapshot", type=Path, required=True); v.add_argument("--output", type=Path); v.add_argument("--workers", type=int, default=8) c = subs.add_parser("catalog"); c.add_argument("--output", type=Path, required=True) args = ap.parse_args() return {"snapshot":snapshot, "verify":verify, "catalog":catalog}[args.cmd](args) if __name__ == "__main__": sys.exit(main())