Digital Drosophila Archive
234 lines
13 KiB
Python
234 lines
13 KiB
Python
"""Verify a complete downloaded publication and exclusively write a new receipt.
|
|
|
|
This verifies the supplied bytes, not how they arrived or their external trust
|
|
anchor. No extraction, LFS-cache lookup, source modification, or Git mutation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
import publish_archives as archives
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def equal(left, right):
|
|
return archives.canonical(left) == archives.canonical(right)
|
|
|
|
|
|
def history_path(value, source_root):
|
|
require(isinstance(value, str), "Historical path must be text")
|
|
prefix = source_root.rstrip("/") + "/"
|
|
if value.startswith(prefix):
|
|
value = value[len(prefix):]
|
|
return archives.relpath(value)
|
|
|
|
|
|
def load_history(path, source_root):
|
|
files, nonfiles, seen = {}, {}, set()
|
|
total = 0
|
|
with gzip.open(path, "rt", encoding="utf-8") as inp:
|
|
for line in inp:
|
|
row = archives.json_bytes(line)
|
|
require(isinstance(row, dict), "Historical row must be an object")
|
|
relative = history_path(row.get("path"), source_root)
|
|
require(relative not in seen, "Duplicate historical path: " + relative)
|
|
seen.add(relative)
|
|
total += 1
|
|
if row.get("kind", "file") != "file":
|
|
nonfiles[relative] = row.get("kind")
|
|
continue
|
|
archives.check_digest_row(row)
|
|
files[relative] = {"bytes": row["bytes"], "sha256": row["sha256"]}
|
|
return files, nonfiles, total
|
|
|
|
|
|
def git_identity():
|
|
try:
|
|
process = subprocess.run(["git", "rev-parse", "--verify", "HEAD"],
|
|
capture_output=True, check=False)
|
|
head = process.stdout.decode("ascii").strip()
|
|
if process.returncode == 0 and len(head) in (40, 64) and all(x in "0123456789abcdef" for x in head):
|
|
return {"cwd": str(Path.cwd()), "head": head,
|
|
"scope": "HEAD observed at invocation cwd; not a clean-tree or payload-to-commit proof"}
|
|
except (OSError, UnicodeError):
|
|
pass
|
|
return {"cwd": str(Path.cwd()), "head": None, "scope": "Git HEAD unavailable"}
|
|
|
|
|
|
def verify_publication(payload_root, receipt, historical_inventory=None):
|
|
root, receipt = archives.no_links(payload_root), archives.no_links(receipt)
|
|
require(root.is_dir(), "Payload root is not a directory")
|
|
require(receipt != root and root not in receipt.parents,
|
|
"Receipt must be outside the immutable payload tree")
|
|
require(not receipt.exists(), "Receipt already exists; refusing overwrite/retry")
|
|
require(receipt.parent.is_dir(), "Receipt parent must already exist")
|
|
controls = {name: archives.sha_file(root / name) for name in ("plan.json", "pack-summary.json")}
|
|
plan = archives.json_read(root / "plan.json")
|
|
summary = archives.json_read(root / "pack-summary.json")
|
|
archives.validate_plan(plan)
|
|
require(summary.get("schema") == "lfs-publication-pack-v1" and summary.get("complete") is True,
|
|
"Pack summary is not complete")
|
|
require(summary.get("plan_sha256") == controls["plan.json"]["sha256"], "Root plan SHA mismatch")
|
|
plan_groups = {g["id"]: g for g in plan["groups"]}
|
|
summary_groups = summary.get("groups")
|
|
require(isinstance(summary_groups, list), "Missing summary groups")
|
|
require(all(isinstance(g, dict) and isinstance(g.get("group"), str) for g in summary_groups),
|
|
"Invalid summary group")
|
|
listed = [g["group"] for g in summary_groups]
|
|
require(listed == list(plan_groups), "Summary group inventory/order differs from plan")
|
|
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
|
|
"Payload root has extra/missing entries; preserve and investigate")
|
|
for p in root.iterdir():
|
|
archives.no_links(p)
|
|
historical = None
|
|
if historical_inventory is not None:
|
|
hpath = archives.no_links(historical_inventory)
|
|
require(hpath != receipt, "Receipt cannot replace historical inventory")
|
|
hpin = archives.sha_file(hpath)
|
|
hfiles, hnonfiles, htotal = load_history(hpath, plan["source_root"])
|
|
historical = {"path": str(hpath), "sha256": hpin["sha256"], "bytes": hpin["bytes"],
|
|
"records": htotal, "regular_file_records": len(hfiles),
|
|
"nonfile_records_skipped": len(hnonfiles)}
|
|
else:
|
|
hfiles, hnonfiles = {}, {}
|
|
matched, missing, nonfile_current = 0, [], 0
|
|
publication_paths, results, manifest_controls = set(), [], {}
|
|
for group_summary in summary_groups:
|
|
group_id = group_summary["group"]
|
|
group, directory = plan_groups[group_id], root / group_id
|
|
require(directory.is_dir(), "Missing group directory: " + group_id)
|
|
cat_path = directory / "catalog.json"
|
|
cat_pin = archives.sha_file(cat_path)
|
|
require(cat_pin["sha256"] == group_summary.get("catalog_sha256"), "Catalog SHA mismatch: " + group_id)
|
|
catalog = archives.json_read(cat_path)
|
|
require(catalog.get("group_id") == group_id and catalog.get("source_root") == plan["source_root"]
|
|
and catalog.get("plan_sha256") == controls["plan.json"]["sha256"],
|
|
"Catalog identity/plan differs: " + group_id)
|
|
for key, wanted in (("source_paths", group["paths"]), ("license", group["license"]),
|
|
("provenance", group["provenance"])):
|
|
require(equal(catalog.get(key), wanted), "Catalog plan metadata mismatch: " + group_id + "/" + key)
|
|
actual = archives.verify(directory)
|
|
expected_summary = {"group": group_id, "files": actual["source_files"],
|
|
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"],
|
|
"chunks": len(catalog["chunks"]), "catalog_sha256": cat_pin["sha256"]}
|
|
require(equal(group_summary, expected_summary), "Group summary differs from actual payload: " + group_id)
|
|
members_path = directory / "members.jsonl.gz"
|
|
manifest_controls[str(members_path)] = catalog["members"]
|
|
manifest_controls[str(cat_path)] = cat_pin
|
|
manifest_controls[str(directory / "exclusions.jsonl.gz")] = catalog["exclusions"]
|
|
feather, member_lookup, group_matched, group_missing = [], {}, 0, 0
|
|
for row in archives.iter_members(members_path):
|
|
name = row["path"]
|
|
require(name not in publication_paths, "Duplicate member across groups: " + name)
|
|
require(any(name == p or name.startswith(p + "/") for p in group["paths"]),
|
|
"Member outside group whitelist: " + name)
|
|
require(not any(name == p or name.startswith(p + "/") for p in plan.get("exclude_paths", [])),
|
|
"Excluded path appears in payload: " + name)
|
|
publication_paths.add(name)
|
|
member_lookup[name] = row
|
|
if name.lower().endswith(".feather"):
|
|
feather.append(row)
|
|
if historical is not None:
|
|
if name in hfiles:
|
|
require(equal(hfiles[name], {"bytes": row["bytes"], "sha256": row["sha256"]}),
|
|
"Historical member mismatch: " + name)
|
|
matched += 1
|
|
group_matched += 1
|
|
else:
|
|
missing.append(name)
|
|
group_missing += 1
|
|
nonfile_current += int(name in hnonfiles)
|
|
require(len(member_lookup) == actual["source_files"], "Member inventory count changed")
|
|
provenance = group["provenance"]
|
|
registered = provenance.get("registered_file_sha256") if isinstance(provenance, dict) else None
|
|
if group_id == "malecns-synaptic-partners":
|
|
require(registered is not None, "Synaptic-partners registered file SHA missing")
|
|
file_anchor = None
|
|
if registered is not None:
|
|
require(isinstance(registered, str) and archives.HEX.fullmatch(registered) is not None,
|
|
"Invalid registered file SHA")
|
|
require(len(feather) == 1, "Registered-file group must contain exactly one feather: " + group_id)
|
|
require(feather[0]["sha256"] == registered, "Registered feather SHA mismatch: " + group_id)
|
|
file_anchor = feather[0]
|
|
license_anchor = None
|
|
license_meta = group["license"]
|
|
if isinstance(license_meta, dict) and "evidence_file" in license_meta:
|
|
relative = archives.relpath(license_meta["evidence_file"])
|
|
path = group["paths"][0] + "/" + relative
|
|
require(path in member_lookup, "License evidence absent from payload: " + path)
|
|
require(member_lookup[path]["sha256"] == license_meta.get("evidence_sha256"),
|
|
"License evidence SHA mismatch: " + path)
|
|
license_anchor = member_lookup[path]
|
|
result = dict(actual, catalog_path=str(cat_path), registered_file_verified=file_anchor,
|
|
license_evidence_verified=license_anchor,
|
|
historical_matched=group_matched if historical is not None else None,
|
|
historical_missing=group_missing if historical is not None else None)
|
|
results.append(result)
|
|
print(json.dumps({"group_verified": group_id, "source_files": actual["source_files"],
|
|
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"]}), flush=True)
|
|
files = sum(r["source_files"] for r in results)
|
|
source_bytes = sum(r["source_bytes"] for r in results)
|
|
archive_bytes = sum(r["archive_bytes"] for r in results)
|
|
require(type(summary.get("source_files")) is int and summary["source_files"] == files,
|
|
"Root source file count mismatch")
|
|
require(type(summary.get("source_bytes")) is int and summary["source_bytes"] == source_bytes,
|
|
"Root source byte count mismatch")
|
|
for name, pin in controls.items():
|
|
require(archives.sha_file(root / name) == pin, "Root control changed during verification: " + name)
|
|
for path, pin in manifest_controls.items():
|
|
require(archives.sha_file(path) == pin, "Group control changed during verification: " + path)
|
|
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
|
|
"Payload root inventory changed during verification")
|
|
if historical is not None:
|
|
require(archives.sha_file(hpath) == hpin, "Historical inventory changed during verification")
|
|
historical.update(status="FULL_CURRENT_MEMBER_MATCH" if not missing else "PARTIAL_CURRENT_MEMBER_COVERAGE",
|
|
all_current_members_historically_matched=not missing,
|
|
current_members_matched=matched, current_members_without_regular_history=len(missing),
|
|
current_members_with_nonfile_history=nonfile_current,
|
|
missing_sample=missing[:20], missing_sample_truncated=len(missing) > 20,
|
|
historical_regular_files_outside_publication=len(set(hfiles) - publication_paths),
|
|
scope="Only publication members with regular-file history compared; not a full historical source-tree audit")
|
|
else:
|
|
historical = {"status": "NOT_REQUESTED", "all_current_members_historically_matched": None}
|
|
aggregate = [{"group": r["group"], "catalog_sha256": r["catalog_sha256"],
|
|
"archive_sha256": r["archive_sha256"], "source_files": r["source_files"],
|
|
"source_bytes": r["source_bytes"], "archive_bytes": r["archive_bytes"]} for r in results]
|
|
report = {"schema": "lfs-publication-aggregate-verification-v1", "pass_all": True,
|
|
"verified_utc": datetime.now(timezone.utc).isoformat(), "payload_root": str(root),
|
|
"receipt": str(receipt), "plan": {"path": str(root / "plan.json"), **controls["plan.json"]},
|
|
"pack_summary": {"path": str(root / "pack-summary.json"), **controls["pack-summary.json"]},
|
|
"groups": results, "source_files": files, "source_bytes": source_bytes,
|
|
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": hashlib.sha256(archives.canonical(aggregate)).hexdigest(),
|
|
"historical_inventory": historical, "git": git_identity(),
|
|
"fresh_download_proven": False,
|
|
"scope": "Given payload bytes verified against plan/catalog/members and optional historical overlap; provenance/plan authenticity and fresh-clone transfer evidence are caller responsibilities"}
|
|
archives.write_new(receipt, archives.canonical(report))
|
|
return {"pass_all": True, "receipt": str(receipt), "receipt_sha256": archives.sha_file(receipt)["sha256"],
|
|
"groups": len(results), "source_files": files, "source_bytes": source_bytes,
|
|
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": report["aggregate_inventory_sha256"],
|
|
"historical_status": historical["status"], "fresh_download_proven": False}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--payload-root", required=True)
|
|
parser.add_argument("--receipt", required=True)
|
|
parser.add_argument("--historical-inventory")
|
|
args = parser.parse_args()
|
|
print(json.dumps(verify_publication(args.payload_root, args.receipt, args.historical_inventory),
|
|
sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|