"""Verify a complete downloaded publication and exclusively write a new receipt. This verifies the supplied bytes, not how they arrived or their external trust anchor. No extraction, LFS-cache lookup, source modification, or Git mutation. """ from __future__ import annotations import argparse from datetime import datetime, timezone import gzip import hashlib import json from pathlib import Path import subprocess import publish_archives as archives def require(condition, message): if not condition: raise ValueError(message) def equal(left, right): return archives.canonical(left) == archives.canonical(right) def history_path(value, source_root): require(isinstance(value, str), "Historical path must be text") prefix = source_root.rstrip("/") + "/" if value.startswith(prefix): value = value[len(prefix):] return archives.relpath(value) def load_history(path, source_root): files, nonfiles, seen = {}, {}, set() total = 0 with gzip.open(path, "rt", encoding="utf-8") as inp: for line in inp: row = archives.json_bytes(line) require(isinstance(row, dict), "Historical row must be an object") relative = history_path(row.get("path"), source_root) require(relative not in seen, "Duplicate historical path: " + relative) seen.add(relative) total += 1 if row.get("kind", "file") != "file": nonfiles[relative] = row.get("kind") continue archives.check_digest_row(row) files[relative] = {"bytes": row["bytes"], "sha256": row["sha256"]} return files, nonfiles, total def git_identity(): try: process = subprocess.run(["git", "rev-parse", "--verify", "HEAD"], capture_output=True, check=False) head = process.stdout.decode("ascii").strip() if process.returncode == 0 and len(head) in (40, 64) and all(x in "0123456789abcdef" for x in head): return {"cwd": str(Path.cwd()), "head": head, "scope": "HEAD observed at invocation cwd; not a clean-tree or payload-to-commit proof"} except (OSError, UnicodeError): pass return {"cwd": str(Path.cwd()), "head": None, "scope": "Git HEAD unavailable"} def verify_publication(payload_root, receipt, historical_inventory=None): root, receipt = archives.no_links(payload_root), archives.no_links(receipt) require(root.is_dir(), "Payload root is not a directory") require(receipt != root and root not in receipt.parents, "Receipt must be outside the immutable payload tree") require(not receipt.exists(), "Receipt already exists; refusing overwrite/retry") require(receipt.parent.is_dir(), "Receipt parent must already exist") controls = {name: archives.sha_file(root / name) for name in ("plan.json", "pack-summary.json")} plan = archives.json_read(root / "plan.json") summary = archives.json_read(root / "pack-summary.json") archives.validate_plan(plan) require(summary.get("schema") == "lfs-publication-pack-v1" and summary.get("complete") is True, "Pack summary is not complete") require(summary.get("plan_sha256") == controls["plan.json"]["sha256"], "Root plan SHA mismatch") plan_groups = {g["id"]: g for g in plan["groups"]} summary_groups = summary.get("groups") require(isinstance(summary_groups, list), "Missing summary groups") require(all(isinstance(g, dict) and isinstance(g.get("group"), str) for g in summary_groups), "Invalid summary group") listed = [g["group"] for g in summary_groups] require(listed == list(plan_groups), "Summary group inventory/order differs from plan") require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"}, "Payload root has extra/missing entries; preserve and investigate") for p in root.iterdir(): archives.no_links(p) historical = None if historical_inventory is not None: hpath = archives.no_links(historical_inventory) require(hpath != receipt, "Receipt cannot replace historical inventory") hpin = archives.sha_file(hpath) hfiles, hnonfiles, htotal = load_history(hpath, plan["source_root"]) historical = {"path": str(hpath), "sha256": hpin["sha256"], "bytes": hpin["bytes"], "records": htotal, "regular_file_records": len(hfiles), "nonfile_records_skipped": len(hnonfiles)} else: hfiles, hnonfiles = {}, {} matched, missing, nonfile_current = 0, [], 0 publication_paths, results, manifest_controls = set(), [], {} for group_summary in summary_groups: group_id = group_summary["group"] group, directory = plan_groups[group_id], root / group_id require(directory.is_dir(), "Missing group directory: " + group_id) cat_path = directory / "catalog.json" cat_pin = archives.sha_file(cat_path) require(cat_pin["sha256"] == group_summary.get("catalog_sha256"), "Catalog SHA mismatch: " + group_id) catalog = archives.json_read(cat_path) require(catalog.get("group_id") == group_id and catalog.get("source_root") == plan["source_root"] and catalog.get("plan_sha256") == controls["plan.json"]["sha256"], "Catalog identity/plan differs: " + group_id) for key, wanted in (("source_paths", group["paths"]), ("license", group["license"]), ("provenance", group["provenance"])): require(equal(catalog.get(key), wanted), "Catalog plan metadata mismatch: " + group_id + "/" + key) actual = archives.verify(directory) expected_summary = {"group": group_id, "files": actual["source_files"], "source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"], "chunks": len(catalog["chunks"]), "catalog_sha256": cat_pin["sha256"]} require(equal(group_summary, expected_summary), "Group summary differs from actual payload: " + group_id) members_path = directory / "members.jsonl.gz" manifest_controls[str(members_path)] = catalog["members"] manifest_controls[str(cat_path)] = cat_pin manifest_controls[str(directory / "exclusions.jsonl.gz")] = catalog["exclusions"] feather, member_lookup, group_matched, group_missing = [], {}, 0, 0 for row in archives.iter_members(members_path): name = row["path"] require(name not in publication_paths, "Duplicate member across groups: " + name) require(any(name == p or name.startswith(p + "/") for p in group["paths"]), "Member outside group whitelist: " + name) require(not any(name == p or name.startswith(p + "/") for p in plan.get("exclude_paths", [])), "Excluded path appears in payload: " + name) publication_paths.add(name) member_lookup[name] = row if name.lower().endswith(".feather"): feather.append(row) if historical is not None: if name in hfiles: require(equal(hfiles[name], {"bytes": row["bytes"], "sha256": row["sha256"]}), "Historical member mismatch: " + name) matched += 1 group_matched += 1 else: missing.append(name) group_missing += 1 nonfile_current += int(name in hnonfiles) require(len(member_lookup) == actual["source_files"], "Member inventory count changed") provenance = group["provenance"] registered = provenance.get("registered_file_sha256") if isinstance(provenance, dict) else None if group_id == "malecns-synaptic-partners": require(registered is not None, "Synaptic-partners registered file SHA missing") file_anchor = None if registered is not None: require(isinstance(registered, str) and archives.HEX.fullmatch(registered) is not None, "Invalid registered file SHA") require(len(feather) == 1, "Registered-file group must contain exactly one feather: " + group_id) require(feather[0]["sha256"] == registered, "Registered feather SHA mismatch: " + group_id) file_anchor = feather[0] license_anchor = None license_meta = group["license"] if isinstance(license_meta, dict) and "evidence_file" in license_meta: relative = archives.relpath(license_meta["evidence_file"]) path = group["paths"][0] + "/" + relative require(path in member_lookup, "License evidence absent from payload: " + path) require(member_lookup[path]["sha256"] == license_meta.get("evidence_sha256"), "License evidence SHA mismatch: " + path) license_anchor = member_lookup[path] result = dict(actual, catalog_path=str(cat_path), registered_file_verified=file_anchor, license_evidence_verified=license_anchor, historical_matched=group_matched if historical is not None else None, historical_missing=group_missing if historical is not None else None) results.append(result) print(json.dumps({"group_verified": group_id, "source_files": actual["source_files"], "source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"]}), flush=True) files = sum(r["source_files"] for r in results) source_bytes = sum(r["source_bytes"] for r in results) archive_bytes = sum(r["archive_bytes"] for r in results) require(type(summary.get("source_files")) is int and summary["source_files"] == files, "Root source file count mismatch") require(type(summary.get("source_bytes")) is int and summary["source_bytes"] == source_bytes, "Root source byte count mismatch") for name, pin in controls.items(): require(archives.sha_file(root / name) == pin, "Root control changed during verification: " + name) for path, pin in manifest_controls.items(): require(archives.sha_file(path) == pin, "Group control changed during verification: " + path) require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"}, "Payload root inventory changed during verification") if historical is not None: require(archives.sha_file(hpath) == hpin, "Historical inventory changed during verification") historical.update(status="FULL_CURRENT_MEMBER_MATCH" if not missing else "PARTIAL_CURRENT_MEMBER_COVERAGE", all_current_members_historically_matched=not missing, current_members_matched=matched, current_members_without_regular_history=len(missing), current_members_with_nonfile_history=nonfile_current, missing_sample=missing[:20], missing_sample_truncated=len(missing) > 20, historical_regular_files_outside_publication=len(set(hfiles) - publication_paths), scope="Only publication members with regular-file history compared; not a full historical source-tree audit") else: historical = {"status": "NOT_REQUESTED", "all_current_members_historically_matched": None} aggregate = [{"group": r["group"], "catalog_sha256": r["catalog_sha256"], "archive_sha256": r["archive_sha256"], "source_files": r["source_files"], "source_bytes": r["source_bytes"], "archive_bytes": r["archive_bytes"]} for r in results] report = {"schema": "lfs-publication-aggregate-verification-v1", "pass_all": True, "verified_utc": datetime.now(timezone.utc).isoformat(), "payload_root": str(root), "receipt": str(receipt), "plan": {"path": str(root / "plan.json"), **controls["plan.json"]}, "pack_summary": {"path": str(root / "pack-summary.json"), **controls["pack-summary.json"]}, "groups": results, "source_files": files, "source_bytes": source_bytes, "archive_bytes": archive_bytes, "aggregate_inventory_sha256": hashlib.sha256(archives.canonical(aggregate)).hexdigest(), "historical_inventory": historical, "git": git_identity(), "fresh_download_proven": False, "scope": "Given payload bytes verified against plan/catalog/members and optional historical overlap; provenance/plan authenticity and fresh-clone transfer evidence are caller responsibilities"} archives.write_new(receipt, archives.canonical(report)) return {"pass_all": True, "receipt": str(receipt), "receipt_sha256": archives.sha_file(receipt)["sha256"], "groups": len(results), "source_files": files, "source_bytes": source_bytes, "archive_bytes": archive_bytes, "aggregate_inventory_sha256": report["aggregate_inventory_sha256"], "historical_status": historical["status"], "fresh_download_proven": False} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--payload-root", required=True) parser.add_argument("--receipt", required=True) parser.add_argument("--historical-inventory") args = parser.parse_args() print(json.dumps(verify_publication(args.payload_root, args.receipt, args.historical_inventory), sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True) if __name__ == "__main__": main()