digital-drosophila/tools/lfs-publication/download_verify.py

"""Fresh-cache transport verification for one independent publication checkout.

Run only from its top level. This is not a scientific audit or authorization to
publish. Failed attempts retain their cache/partial files and are never retried.
Git/LFS output is suppressed; receipts contain no environment or auth headers.
The caller supplies the trusted commit. Tool source SHA pins are observations,
not signatures; executable versions are not executable-byte identity proofs.
"""
from __future__ import annotations

import argparse
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import re
import stat
import subprocess
import sys

import publish_archives as archives
import verify_publication as content

REMOTE = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
PAYLOAD = "artifacts/public-resources-20261001"
PROTECTED = Path("/data/digital-drosophila")
TRANSFER_OVERRIDES = (r"^(lfs\.(url|pushurl|standalonetransferagent|customtransfer\..*|"
                      r"remote\.(autodetect|searchall))|remote\.(lfsdefault|origin\.lfsurl))$")


def require(condition, message):
    if not condition:
        raise ValueError(message)


def within(path, root):
    return path == root or root in path.parents


def command(args, *, text=False, bytes_output=False):
    """Never return or print command diagnostics, which may contain credentials."""
    result = subprocess.run(args, stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
                            stdout=subprocess.PIPE if text or bytes_output else subprocess.DEVNULL,
                            check=False)
    if text or bytes_output:
        require(result.returncode == 0, "Read-only Git/tool identity command failed")
        if bytes_output:
            return result.stdout
        return result.stdout.decode("utf-8", errors="strict").strip()
    return result.returncode


def identity(expected, checkout):
    require(command(["git", "rev-parse", "--verify", "HEAD"], text=True) == expected,
            "HEAD differs from expected commit")
    require(archives.no_links(command(["git", "rev-parse", "--show-toplevel"], text=True)) == checkout,
            "Run from the independent checkout top level")
    require(archives.no_links(command(["git", "rev-parse", "--absolute-git-dir"], text=True)) == checkout / ".git",
            "External/linked Git directory refused")
    require(command(["git", "remote", "get-url", "--all", "origin"], text=True) == REMOTE,
            "Origin must exactly match the approved remote")


def transfer_policy(checkout, expected_commit, allow_lan_adapter=False):
    require(not (checkout / ".lfsconfig").exists() and not (checkout / ".lfsconfig").is_symlink(),
            "Repository LFS endpoint overrides refused")
    if not allow_lan_adapter:
        # Exit 1 means no matches; potential credential values are never read.
        require(command(["git", "config", "--get-regexp", TRANSFER_OVERRIDES]) == 1,
                "LFS endpoint/custom-transfer override or unreadable configuration refused")
        return None
    relative = "tools/lfs-publication/lan_lfs_transfer.py"
    adapter = archives.no_links(checkout / relative)
    require(not any(c.isspace() for c in str(adapter)), "LAN adapter path must not require argument quoting")
    prefix = "lfs.customtransfer.malecns-lan-https."
    allowed = {"lfs.standalonetransferagent": "malecns-lan-https",
               prefix + "path": "/usr/bin/python3", prefix + "args": "-B " + str(adapter),
               prefix + "concurrent": "true", prefix + "direction": "both"}
    keys = command(["git", "config", "--name-only", "--get-regexp", TRANSFER_OVERRIDES], text=True).splitlines()
    require(len(keys) == len(allowed) and set(keys) == set(allowed),
            "Only the exact approved LAN adapter configuration is permitted")
    for key, value in allowed.items():
        require(command(["git", "config", "--local", "--get-all", key], text=True) == value
                and command(["git", "config", "--get-all", key], text=True) == value,
                "LAN adapter configuration must be exact and exclusively repository-local")
    pin = archives.sha_file(adapter)
    blob = command(["git", "show", expected_commit + ":" + relative], bytes_output=True)
    require(len(blob) == pin["bytes"] and hashlib.sha256(blob).hexdigest() == pin["sha256"],
            "LAN adapter must exactly match the expected HEAD blob")
    return {"path": str(adapter), **pin}


def new_output(path, checkout, cache, payload):
    p = archives.no_links(path)
    require(not within(p, PROTECTED.absolute()), "Output under original project refused")
    require(within(p, checkout.parent), "Output must stay in the independent workspace")
    require(not within(p, payload) and not within(p, checkout / ".git")
            and not within(p, checkout / "artifacts") and not within(p, cache),
            "Receipt must be outside payload, artifacts, Git and cache trees")
    require(not p.exists() and p.parent.is_dir(), "Receipt must be new with an existing parent")
    return p


def cache_inventory(cache):
    count = total = 0
    objects = cache / "objects"
    if not objects.exists():
        return {"objects": 0, "bytes": 0}
    archives.no_links(objects)
    require(objects.is_dir(), "LFS objects must be a directory")
    for p in objects.rglob("*"):
        archives.no_links(p)
        s = p.lstat()
        if stat.S_ISDIR(s.st_mode):
            continue
        require(stat.S_ISREG(s.st_mode), "Nonregular cache object refused")
        parts = p.relative_to(objects).parts
        require(len(parts) == 3 and archives.HEX.fullmatch(parts[2]) is not None
                and parts[0] == parts[2][:2] and parts[1] == parts[2][2:4],
                "Unexpected LFS object layout")
        count += 1
        total += s.st_size
    return {"objects": count, "bytes": total}


def source_pins(adapter=None):
    result = {name: {"path": str(Path(path).absolute()), **archives.sha_file(path)}
            for name, path in (("download_verify.py", __file__),
                               ("verify_publication.py", content.__file__),
                               ("publish_archives.py", archives.__file__))}
    if adapter is not None:
        result["lan_lfs_transfer.py"] = {"path": adapter["path"], **archives.sha_file(adapter["path"])}
    return result


def run(expected_commit, cache, receipt, content_receipt, historical_inventory, *, allow_lan_adapter=False):
    require(isinstance(expected_commit, str) and re.fullmatch(r"[0-9a-f]{40}", expected_commit),
            "Expected commit must be a complete lowercase SHA-1")
    checkout = archives.no_links(Path.cwd())
    require(not within(checkout, PROTECTED.absolute()), "Original project checkout refused")
    gitdir = archives.no_links(checkout / ".git")
    require(gitdir.is_dir(), "Only an independent clone with its own .git directory is accepted")
    identity(expected_commit, checkout)
    adapter = transfer_policy(checkout, expected_commit, allow_lan_adapter)
    require(not command(["git", "status", "--porcelain", "--untracked-files=no"], text=True),
            "Tracked checkout changes refused")
    cache = archives.no_links(cache)
    require(not within(cache, PROTECTED.absolute()), "Cache under original project refused")
    require(within(cache, gitdir) or (within(cache, checkout.parent) and not within(cache, checkout)),
            "Cache must be inside this .git or outside checkout in the independent workspace")
    require(not cache.exists() and cache.parent.is_dir(), "Cache must never have existed for this attempt")
    payload = archives.no_links(checkout / PAYLOAD)
    receipt = new_output(receipt, checkout, cache, payload)
    content_receipt = new_output(content_receipt, checkout, cache, payload)
    history = archives.no_links(historical_inventory)
    require(len({receipt, content_receipt, history, cache}) == 4, "Inputs/outputs must be distinct")
    history_pin = archives.sha_file(history)
    pins = source_pins(adapter)
    versions = {"git": command(["git", "--version"], text=True),
                "git_lfs": command(["git", "lfs", "version"], text=True),
                "zstd": command(["zstd", "--version"], text=True),
                "python": sys.version.split()[0]}
    started = datetime.now(timezone.utc).isoformat()
    cache.mkdir(exist_ok=False)
    initial = cache_inventory(cache)
    require(initial == {"objects": 0, "bytes": 0}, "New cache is not empty")
    print(json.dumps({"stage": "fresh-cache-created", "objects": 0, "bytes": 0}), flush=True)
    git = ["git", "-c", "lfs.storage=" + str(cache), "-c", "lfs.concurrenttransfers=2"]
    pull = command(git + ["lfs", "pull", "--include=artifacts/**", "--exclude=", "origin"])
    require(pull == 0, "LFS pull failed; cache retained, no automatic retry")
    print(json.dumps({"stage": "pull-exited", "exit_code": pull}), flush=True)
    fsck = command(git + ["lfs", "fsck", "--dry-run"])
    require(fsck == 0, "LFS fsck failed; cache retained, no automatic retry")
    print(json.dumps({"stage": "fsck-exited", "exit_code": fsck}), flush=True)
    objects = cache_inventory(cache)
    require(objects["objects"] > 0 and objects["bytes"] > 0, "Fresh LFS cache contains no objects")
    verified = content.verify_publication(payload, content_receipt, history)
    require(verified.get("pass_all") is True
            and archives.HEX.fullmatch(str(verified.get("aggregate_inventory_sha256", ""))) is not None,
            "Content verification did not report success")
    content_pin = archives.sha_file(content_receipt)
    require(content_pin["sha256"] == verified.get("receipt_sha256"), "Content receipt SHA mismatch")
    require(archives.sha_file(history) == history_pin and source_pins(adapter) == pins,
            "History or verification tool changed during attempt")
    identity(expected_commit, checkout)
    require(transfer_policy(checkout, expected_commit, allow_lan_adapter) == adapter,
            "LAN adapter identity/configuration changed during attempt")
    require(cache_inventory(cache) == objects, "Cache object inventory changed during content verification")
    report = {"schema": "lfs-publication-download-verification-v1", "pass_all": True,
              "started_utc": started, "verified_utc": datetime.now(timezone.utc).isoformat(),
              "checkout": str(checkout), "expected_commit": expected_commit,
              "head": expected_commit, "origin": REMOTE,
              "explicit_pull_remote": "origin", "lfs_endpoint_overrides_refused": True,
              "lan_adapter": {"enabled": adapter is not None, "source": adapter,
                              "configuration_scope": "repository-local only" if adapter else None,
                              "temporary_download_directory": str(gitdir / "lfs/lan-transfer-tmp") if adapter else None},
              "cache": {"path": str(cache), "initially_absent": True,
                        "before": initial, "after": objects},
              "pull_exit_code": pull, "fsck_dry_run_exit_code": fsck,
              "include": "artifacts/**", "exclude": "", "concurrent_transfers": 2,
              "content_receipt": {"path": str(content_receipt), **content_pin},
              "aggregate_inventory_sha256": verified["aggregate_inventory_sha256"],
              "historical_inventory": {"path": str(history), **history_pin,
                                       "status": verified.get("historical_status")},
              "tool_sources": pins, "tool_versions": versions, "fresh_download_proven": True,
              "learning_pass_claimed": False, "original_project_write_requested": False,
              "scope": "Fresh isolated LFS storage, actual pull/fsck exits and complete publication member verification; smoke covered by pull/fsck. Not a scientific gate, all-history audit, or executable-byte identity proof."}
    archives.write_new(receipt, archives.canonical(report))
    return report


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--expected-commit", required=True)
    parser.add_argument("--cache", required=True)
    parser.add_argument("--receipt", required=True)
    parser.add_argument("--content-receipt", required=True)
    parser.add_argument("--historical-inventory", required=True)
    parser.add_argument("--allow-lan-adapter", action="store_true")
    args = parser.parse_args()
    try:
        run(args.expected_commit, args.cache, args.receipt, args.content_receipt, args.historical_inventory,
            allow_lan_adapter=args.allow_lan_adapter)
    except (Exception, KeyboardInterrupt) as exc:
        print(json.dumps({"status": "FAILED-NOT-RETRIED", "error_type": type(exc).__name__}), flush=True)
        return 1
    print(json.dumps({"status": "PASS", "receipt": str(Path(args.receipt).absolute())}), flush=True)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())