qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish licensed data snapshots with LAN LFS transfer and content audits

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-10-01 03:42:51 +08:00
parent 13cd0e0be7
commit 6e33e05ede
173 changed files with 3614 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/lfs-publication/verify_publication.py
+233
View File
@@ -0,0 +1,233 @@
"""Verify a complete downloaded publication and exclusively write a new receipt.
This verifies the supplied bytes, not how they arrived or their external trust
anchor. No extraction, LFS-cache lookup, source modification, or Git mutation.
"""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import gzip
import hashlib
import json
from pathlib import Path
import subprocess
import publish_archives as archives
def require(condition, message):
if not condition:
raise ValueError(message)
def equal(left, right):
return archives.canonical(left) == archives.canonical(right)
def history_path(value, source_root):
require(isinstance(value, str), "Historical path must be text")
prefix = source_root.rstrip("/") + "/"
if value.startswith(prefix):
value = value[len(prefix):]
return archives.relpath(value)
def load_history(path, source_root):
files, nonfiles, seen = {}, {}, set()
total = 0
with gzip.open(path, "rt", encoding="utf-8") as inp:
for line in inp:
row = archives.json_bytes(line)
require(isinstance(row, dict), "Historical row must be an object")
relative = history_path(row.get("path"), source_root)
require(relative not in seen, "Duplicate historical path: " + relative)
seen.add(relative)
total += 1
if row.get("kind", "file") != "file":
nonfiles[relative] = row.get("kind")
continue
archives.check_digest_row(row)
files[relative] = {"bytes": row["bytes"], "sha256": row["sha256"]}
return files, nonfiles, total
def git_identity():
try:
process = subprocess.run(["git", "rev-parse", "--verify", "HEAD"],
capture_output=True, check=False)
head = process.stdout.decode("ascii").strip()
if process.returncode == 0 and len(head) in (40, 64) and all(x in "0123456789abcdef" for x in head):
return {"cwd": str(Path.cwd()), "head": head,
"scope": "HEAD observed at invocation cwd; not a clean-tree or payload-to-commit proof"}
except (OSError, UnicodeError):
pass
return {"cwd": str(Path.cwd()), "head": None, "scope": "Git HEAD unavailable"}
def verify_publication(payload_root, receipt, historical_inventory=None):
root, receipt = archives.no_links(payload_root), archives.no_links(receipt)
require(root.is_dir(), "Payload root is not a directory")
require(receipt != root and root not in receipt.parents,
"Receipt must be outside the immutable payload tree")
require(not receipt.exists(), "Receipt already exists; refusing overwrite/retry")
require(receipt.parent.is_dir(), "Receipt parent must already exist")
controls = {name: archives.sha_file(root / name) for name in ("plan.json", "pack-summary.json")}
plan = archives.json_read(root / "plan.json")
summary = archives.json_read(root / "pack-summary.json")
archives.validate_plan(plan)
require(summary.get("schema") == "lfs-publication-pack-v1" and summary.get("complete") is True,
"Pack summary is not complete")
require(summary.get("plan_sha256") == controls["plan.json"]["sha256"], "Root plan SHA mismatch")
plan_groups = {g["id"]: g for g in plan["groups"]}
summary_groups = summary.get("groups")
require(isinstance(summary_groups, list), "Missing summary groups")
require(all(isinstance(g, dict) and isinstance(g.get("group"), str) for g in summary_groups),
"Invalid summary group")
listed = [g["group"] for g in summary_groups]
require(listed == list(plan_groups), "Summary group inventory/order differs from plan")
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
"Payload root has extra/missing entries; preserve and investigate")
for p in root.iterdir():
archives.no_links(p)
historical = None
if historical_inventory is not None:
hpath = archives.no_links(historical_inventory)
require(hpath != receipt, "Receipt cannot replace historical inventory")
hpin = archives.sha_file(hpath)
hfiles, hnonfiles, htotal = load_history(hpath, plan["source_root"])
historical = {"path": str(hpath), "sha256": hpin["sha256"], "bytes": hpin["bytes"],
"records": htotal, "regular_file_records": len(hfiles),
"nonfile_records_skipped": len(hnonfiles)}
else:
hfiles, hnonfiles = {}, {}
matched, missing, nonfile_current = 0, [], 0
publication_paths, results, manifest_controls = set(), [], {}
for group_summary in summary_groups:
group_id = group_summary["group"]
group, directory = plan_groups[group_id], root / group_id
require(directory.is_dir(), "Missing group directory: " + group_id)
cat_path = directory / "catalog.json"
cat_pin = archives.sha_file(cat_path)
require(cat_pin["sha256"] == group_summary.get("catalog_sha256"), "Catalog SHA mismatch: " + group_id)
catalog = archives.json_read(cat_path)
require(catalog.get("group_id") == group_id and catalog.get("source_root") == plan["source_root"]
and catalog.get("plan_sha256") == controls["plan.json"]["sha256"],
"Catalog identity/plan differs: " + group_id)
for key, wanted in (("source_paths", group["paths"]), ("license", group["license"]),
("provenance", group["provenance"])):
require(equal(catalog.get(key), wanted), "Catalog plan metadata mismatch: " + group_id + "/" + key)
actual = archives.verify(directory)
expected_summary = {"group": group_id, "files": actual["source_files"],
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"],
"chunks": len(catalog["chunks"]), "catalog_sha256": cat_pin["sha256"]}
require(equal(group_summary, expected_summary), "Group summary differs from actual payload: " + group_id)
members_path = directory / "members.jsonl.gz"
manifest_controls[str(members_path)] = catalog["members"]
manifest_controls[str(cat_path)] = cat_pin
manifest_controls[str(directory / "exclusions.jsonl.gz")] = catalog["exclusions"]
feather, member_lookup, group_matched, group_missing = [], {}, 0, 0
for row in archives.iter_members(members_path):
name = row["path"]
require(name not in publication_paths, "Duplicate member across groups: " + name)
require(any(name == p or name.startswith(p + "/") for p in group["paths"]),
"Member outside group whitelist: " + name)
require(not any(name == p or name.startswith(p + "/") for p in plan.get("exclude_paths", [])),
"Excluded path appears in payload: " + name)
publication_paths.add(name)
member_lookup[name] = row
if name.lower().endswith(".feather"):
feather.append(row)
if historical is not None:
if name in hfiles:
require(equal(hfiles[name], {"bytes": row["bytes"], "sha256": row["sha256"]}),
"Historical member mismatch: " + name)
matched += 1
group_matched += 1
else:
missing.append(name)
group_missing += 1
nonfile_current += int(name in hnonfiles)
require(len(member_lookup) == actual["source_files"], "Member inventory count changed")
provenance = group["provenance"]
registered = provenance.get("registered_file_sha256") if isinstance(provenance, dict) else None
if group_id == "malecns-synaptic-partners":
require(registered is not None, "Synaptic-partners registered file SHA missing")
file_anchor = None
if registered is not None:
require(isinstance(registered, str) and archives.HEX.fullmatch(registered) is not None,
"Invalid registered file SHA")
require(len(feather) == 1, "Registered-file group must contain exactly one feather: " + group_id)
require(feather[0]["sha256"] == registered, "Registered feather SHA mismatch: " + group_id)
file_anchor = feather[0]
license_anchor = None
license_meta = group["license"]
if isinstance(license_meta, dict) and "evidence_file" in license_meta:
relative = archives.relpath(license_meta["evidence_file"])
path = group["paths"][0] + "/" + relative
require(path in member_lookup, "License evidence absent from payload: " + path)
require(member_lookup[path]["sha256"] == license_meta.get("evidence_sha256"),
"License evidence SHA mismatch: " + path)
license_anchor = member_lookup[path]
result = dict(actual, catalog_path=str(cat_path), registered_file_verified=file_anchor,
license_evidence_verified=license_anchor,
historical_matched=group_matched if historical is not None else None,
historical_missing=group_missing if historical is not None else None)
results.append(result)
print(json.dumps({"group_verified": group_id, "source_files": actual["source_files"],
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"]}), flush=True)
files = sum(r["source_files"] for r in results)
source_bytes = sum(r["source_bytes"] for r in results)
archive_bytes = sum(r["archive_bytes"] for r in results)
require(type(summary.get("source_files")) is int and summary["source_files"] == files,
"Root source file count mismatch")
require(type(summary.get("source_bytes")) is int and summary["source_bytes"] == source_bytes,
"Root source byte count mismatch")
for name, pin in controls.items():
require(archives.sha_file(root / name) == pin, "Root control changed during verification: " + name)
for path, pin in manifest_controls.items():
require(archives.sha_file(path) == pin, "Group control changed during verification: " + path)
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
"Payload root inventory changed during verification")
if historical is not None:
require(archives.sha_file(hpath) == hpin, "Historical inventory changed during verification")
historical.update(status="FULL_CURRENT_MEMBER_MATCH" if not missing else "PARTIAL_CURRENT_MEMBER_COVERAGE",
all_current_members_historically_matched=not missing,
current_members_matched=matched, current_members_without_regular_history=len(missing),
current_members_with_nonfile_history=nonfile_current,
missing_sample=missing[:20], missing_sample_truncated=len(missing) > 20,
historical_regular_files_outside_publication=len(set(hfiles) - publication_paths),
scope="Only publication members with regular-file history compared; not a full historical source-tree audit")
else:
historical = {"status": "NOT_REQUESTED", "all_current_members_historically_matched": None}
aggregate = [{"group": r["group"], "catalog_sha256": r["catalog_sha256"],
"archive_sha256": r["archive_sha256"], "source_files": r["source_files"],
"source_bytes": r["source_bytes"], "archive_bytes": r["archive_bytes"]} for r in results]
report = {"schema": "lfs-publication-aggregate-verification-v1", "pass_all": True,
"verified_utc": datetime.now(timezone.utc).isoformat(), "payload_root": str(root),
"receipt": str(receipt), "plan": {"path": str(root / "plan.json"), **controls["plan.json"]},
"pack_summary": {"path": str(root / "pack-summary.json"), **controls["pack-summary.json"]},
"groups": results, "source_files": files, "source_bytes": source_bytes,
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": hashlib.sha256(archives.canonical(aggregate)).hexdigest(),
"historical_inventory": historical, "git": git_identity(),
"fresh_download_proven": False,
"scope": "Given payload bytes verified against plan/catalog/members and optional historical overlap; provenance/plan authenticity and fresh-clone transfer evidence are caller responsibilities"}
archives.write_new(receipt, archives.canonical(report))
return {"pass_all": True, "receipt": str(receipt), "receipt_sha256": archives.sha_file(receipt)["sha256"],
"groups": len(results), "source_files": files, "source_bytes": source_bytes,
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": report["aggregate_inventory_sha256"],
"historical_status": historical["status"], "fresh_download_proven": False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--payload-root", required=True)
parser.add_argument("--receipt", required=True)
parser.add_argument("--historical-inventory")
args = parser.parse_args()
print(json.dumps(verify_publication(args.payload_root, args.receipt, args.historical_inventory),
sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
if __name__ == "__main__":
main()