Archive MaleCNS research sources, provenance, bibliography and immutable data inventory
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only project inventory; writes only to a NEW explicit output directory.
|
||||
|
||||
No model imports, simulations, data migration, deletion, or dependency installation.
|
||||
Python >=3.10, standard library only. Snapshot excludes reproducible environment/cache
|
||||
directories and nested Git databases, never follows symlinks. Verify checks additions too.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures as cf
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
ROOTS = ("datasets", "experiments", "models", "backend", "frontend", "game", "logs", "recordings")
|
||||
EXCLUDE = {".git", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", ".cache", "node_modules"}
|
||||
TEXT_EXT = {".py", ".md", ".json", ".yaml", ".yml", ".toml", ".txt", ".sh", ".csv", ".tsv"}
|
||||
|
||||
def dump(path, obj):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("x", encoding="utf-8", newline="\n") as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
|
||||
def sha(path):
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for block in iter(lambda: f.read(4 * 1024 * 1024), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
def entries(root):
|
||||
for name in ROOTS:
|
||||
base = root / name
|
||||
if not base.exists():
|
||||
continue
|
||||
for folder, dirs, files in os.walk(base, followlinks=False):
|
||||
dirs[:] = sorted(d for d in dirs if d not in EXCLUDE)
|
||||
for d in dirs[:]:
|
||||
p = Path(folder) / d
|
||||
if p.is_symlink():
|
||||
yield p
|
||||
dirs.remove(d)
|
||||
for f in sorted(files):
|
||||
yield Path(folder) / f
|
||||
|
||||
def record(root, path):
|
||||
st = path.lstat()
|
||||
row = {"path": path.relative_to(root).as_posix(), "bytes": st.st_size,
|
||||
"mtime_ns": st.st_mtime_ns, "mode": oct(st.st_mode & 0o7777)}
|
||||
if path.is_symlink():
|
||||
row.update(kind="symlink", target=os.readlink(path))
|
||||
elif path.is_file():
|
||||
row.update(kind="file", sha256=sha(path))
|
||||
after = path.stat()
|
||||
if (st.st_size, st.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
|
||||
raise RuntimeError("File changed during hashing: " + row["path"])
|
||||
else:
|
||||
raise RuntimeError("Unexpected special file: " + str(path))
|
||||
return row
|
||||
|
||||
def snapshot(args):
|
||||
root, out = args.root.resolve(), args.output.resolve()
|
||||
if out.exists():
|
||||
raise RuntimeError("Refusing existing output directory")
|
||||
out.mkdir(parents=True)
|
||||
paths = sorted(entries(root))
|
||||
counts = {}
|
||||
with gzip.open(out / "files.jsonl.gz", "xt", encoding="utf-8") as f, cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
for n, row in enumerate(ex.map(lambda p: record(root, p), paths), 1):
|
||||
f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
key = row["path"].split("/")[0]
|
||||
c = counts.setdefault(key, {"files": 0, "logical_bytes": 0})
|
||||
c["files"] += 1
|
||||
c["logical_bytes"] += row["bytes"]
|
||||
if n % 25000 == 0:
|
||||
print(f"hashed {n}/{len(paths)}", flush=True)
|
||||
summary = {"schema_version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"root": str(root), "roots": ROOTS, "excluded_directory_names": sorted(EXCLUDE),
|
||||
"files": len(paths), "groups": counts, "symlinks_followed": False,
|
||||
"files_jsonl_gz_sha256": sha(out / "files.jsonl.gz"),
|
||||
"not_a_data_backup": True, "historical_authenticity_claim": False}
|
||||
dump(out / "summary.json", summary)
|
||||
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
||||
|
||||
def verify(args):
|
||||
root = args.root.resolve()
|
||||
summary = json.loads((args.snapshot / "summary.json").read_text())
|
||||
manifest = args.snapshot / "files.jsonl.gz"
|
||||
if sha(manifest) != summary["files_jsonl_gz_sha256"]:
|
||||
raise RuntimeError("Snapshot manifest checksum mismatch")
|
||||
with gzip.open(manifest, "rt", encoding="utf-8") as f:
|
||||
rows = [json.loads(line) for line in f]
|
||||
def check(row):
|
||||
p = root / row["path"]
|
||||
if p.is_absolute() and not p.absolute().is_relative_to(root):
|
||||
return {"path": row["path"], "problem": "unsafe path"}
|
||||
if ".." in Path(row["path"]).parts:
|
||||
return {"path": row["path"], "problem": "unsafe path"}
|
||||
try:
|
||||
now = record(root, p)
|
||||
keys = ("kind", "target", "mode") if row["kind"] == "symlink" else ("kind", "bytes", "sha256", "mode")
|
||||
different = [k for k in keys if row.get(k) != now.get(k)]
|
||||
return {"path": row["path"], "changed_fields": different} if different else None
|
||||
except (OSError, RuntimeError) as e:
|
||||
return {"path": row["path"], "problem": str(e)}
|
||||
with cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
mismatches = [r for r in ex.map(check, rows) if r]
|
||||
before = {r["path"] for r in rows}
|
||||
after = {p.relative_to(root).as_posix() for p in entries(root)}
|
||||
result = {"checked": len(rows), "mismatches": mismatches, "added": sorted(after-before),
|
||||
"missing": sorted(before-after), "checked_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"snapshot_sha256": sha(manifest), "excluded_directory_names": sorted(EXCLUDE),
|
||||
"pass": not mismatches and before == after}
|
||||
if args.output:
|
||||
dump(args.output, result)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
return 0 if result["pass"] else 1
|
||||
|
||||
def run(root, *cmd):
|
||||
p = subprocess.run(cmd, cwd=root, text=True, capture_output=True, check=False)
|
||||
return {"exit_code": p.returncode, "stdout": p.stdout.strip(), "stderr": p.stderr.strip()}
|
||||
|
||||
def catalog(args):
|
||||
root, out = args.root.resolve(), args.output.resolve()
|
||||
if out.exists():
|
||||
raise RuntimeError("Refusing existing output directory")
|
||||
out.mkdir(parents=True)
|
||||
stages, evidence, refs, resources = [], [], [], []
|
||||
for d in sorted((root / "experiments").iterdir()):
|
||||
if not d.is_dir():
|
||||
continue
|
||||
selected = []
|
||||
# Small audit/config/result evidence only. Retain originals byte-for-byte.
|
||||
for p in sorted(d.rglob("*")):
|
||||
rel = p.relative_to(d)
|
||||
if "external-resources" in rel.parts or any(x in EXCLUDE for x in rel.parts):
|
||||
continue
|
||||
if not p.is_file() or p.is_symlink() or p.stat().st_size > 512 * 1024:
|
||||
continue
|
||||
suffix = p.suffix.lower()
|
||||
include = suffix in {".md", ".yaml", ".yml"} or (
|
||||
suffix == ".json" and (len(rel.parts) == 1 or rel.parts[0] in {
|
||||
"audit", "regression", "literature-audit", "numerical-contract", "configs",
|
||||
"config", "revalidation", "phase4c-v2", "manifests"} or "summary" in p.name or "gate" in p.name))
|
||||
if include:
|
||||
dest = out / "evidence" / d.name / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with p.open("rb") as src, dest.open("xb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
item = {"source_path": p.relative_to(root).as_posix(), "copy_path": dest.relative_to(out).as_posix(),
|
||||
"sha256": sha(p), "bytes": p.stat().st_size}
|
||||
evidence.append(item)
|
||||
selected.append(item["copy_path"])
|
||||
stages.append({"stage_directory": d.name, "evidence": selected,
|
||||
"status_policy": "Use original evidence; completion is not a scientific PASS"})
|
||||
# Index citations in project-authored sources, configs, provenance and literature notes.
|
||||
for p in entries(root):
|
||||
rel = p.relative_to(root)
|
||||
if p.is_symlink() or p.suffix.lower() not in TEXT_EXT or p.stat().st_size > 2 * 1024 * 1024:
|
||||
continue
|
||||
if "external-resources" in rel.parts or "skeletons" in rel.parts:
|
||||
continue
|
||||
body = p.read_text(encoding="utf-8", errors="replace")
|
||||
urls = sorted(set(re.findall(r"https?://[^\s<>\"'`]+", body)))
|
||||
dois = sorted(set(x.rstrip(".,;)]}") for x in re.findall(r"10\.\d{4,9}/[-._;()/:A-Za-z0-9]+", body)))
|
||||
# Never export URLs containing user credentials or common secret query parameters.
|
||||
urls = [u.rstrip(".,;)]}") for u in urls if not re.search(r"https?://[^/]*@|[?&](?:token|key|secret|password|signature)=", u, re.I)]
|
||||
if urls or dois:
|
||||
refs.append({"source_path": rel.as_posix(), "sha256": sha(p), "urls": urls, "doi_candidates": dois,
|
||||
"review_status": "extracted-not-automatically-endorsed"})
|
||||
known = [
|
||||
("experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer", "CC-BY-4.0", "LICENSE"),
|
||||
("experiments/phase4f-retinotopic-vision/external-resources/visualpathways", "GPL-3.0 license text; grant/version options require upstream review", "LICENSE"),
|
||||
("experiments/phase4f-retinotopic-vision/external-resources/flyvis", "MIT declared by pyproject classifier; standalone LICENSE absent in frozen checkout", "pyproject.toml"),
|
||||
("experiments/phase4l-olfactory-grounding/external-resources/DoOR.data", "CC BY-SA 4.0 declared in DESCRIPTION", "DESCRIPTION"),
|
||||
]
|
||||
for value, license_id, license_file in known:
|
||||
p = root / value
|
||||
if not p.exists():
|
||||
resources.append({"path": value, "status": "MISSING"})
|
||||
continue
|
||||
lp = p / license_file
|
||||
resources.append({"path": value, "commit": run(p, "git", "rev-parse", "HEAD"),
|
||||
"origin": run(p, "git", "remote", "get-url", "origin"),
|
||||
"worktree_status": run(p, "git", "--no-optional-locks", "status", "--short"),
|
||||
"license_observation": license_id, "license_evidence": str(lp.relative_to(root)),
|
||||
"license_evidence_sha256": sha(lp) if lp.is_file() else None,
|
||||
"vendored_in_parent_git": False})
|
||||
dump(out / "stages.json", stages)
|
||||
dump(out / "evidence-index.json", evidence)
|
||||
dump(out / "source-citations.json", refs)
|
||||
dump(out / "external-resources.lock.json", resources)
|
||||
env = {"captured_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"python": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "--version"),
|
||||
"packages": run(root, str(root / "models/male-cns-lif/.venv/bin/python"), "-c",
|
||||
"import importlib.metadata as m,json;print(json.dumps(sorted((x.metadata['Name'],x.version) for x in m.distributions())))"),
|
||||
"gpu": run(root, "nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"),
|
||||
"os": run(root, "uname", "-a"), "git": run(root, "git", "--version"),
|
||||
"nvcc": run(root, "sh", "-c", "command -v nvcc >/dev/null && nvcc --version || true"),
|
||||
"historical_environment_guaranteed": False}
|
||||
dump(out / "environment.json", env)
|
||||
print(json.dumps({"stages":len(stages), "evidence_files":len(evidence), "citation_source_files":len(refs), "external_resources":len(resources)}))
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--root", type=Path, default=Path("."))
|
||||
subs = ap.add_subparsers(dest="cmd", required=True)
|
||||
s = subs.add_parser("snapshot"); s.add_argument("--output", type=Path, required=True); s.add_argument("--workers", type=int, default=8)
|
||||
v = subs.add_parser("verify"); v.add_argument("--snapshot", type=Path, required=True); v.add_argument("--output", type=Path); v.add_argument("--workers", type=int, default=8)
|
||||
c = subs.add_parser("catalog"); c.add_argument("--output", type=Path, required=True)
|
||||
args = ap.parse_args()
|
||||
return {"snapshot":snapshot, "verify":verify, "catalog":catalog}[args.cmd](args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user