Publish licensed data snapshots with LAN LFS transfer and content audits
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,232 @@
|
||||
"""Fresh-cache transport verification for one independent publication checkout.
|
||||
|
||||
Run only from its top level. This is not a scientific audit or authorization to
|
||||
publish. Failed attempts retain their cache/partial files and are never retried.
|
||||
Git/LFS output is suppressed; receipts contain no environment or auth headers.
|
||||
The caller supplies the trusted commit. Tool source SHA pins are observations,
|
||||
not signatures; executable versions are not executable-byte identity proofs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import publish_archives as archives
|
||||
import verify_publication as content
|
||||
|
||||
REMOTE = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
|
||||
PAYLOAD = "artifacts/public-resources-20261001"
|
||||
PROTECTED = Path("/data/digital-drosophila")
|
||||
TRANSFER_OVERRIDES = (r"^(lfs\.(url|pushurl|standalonetransferagent|customtransfer\..*|"
|
||||
r"remote\.(autodetect|searchall))|remote\.(lfsdefault|origin\.lfsurl))$")
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def within(path, root):
|
||||
return path == root or root in path.parents
|
||||
|
||||
|
||||
def command(args, *, text=False, bytes_output=False):
|
||||
"""Never return or print command diagnostics, which may contain credentials."""
|
||||
result = subprocess.run(args, stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE if text or bytes_output else subprocess.DEVNULL,
|
||||
check=False)
|
||||
if text or bytes_output:
|
||||
require(result.returncode == 0, "Read-only Git/tool identity command failed")
|
||||
if bytes_output:
|
||||
return result.stdout
|
||||
return result.stdout.decode("utf-8", errors="strict").strip()
|
||||
return result.returncode
|
||||
|
||||
|
||||
def identity(expected, checkout):
|
||||
require(command(["git", "rev-parse", "--verify", "HEAD"], text=True) == expected,
|
||||
"HEAD differs from expected commit")
|
||||
require(archives.no_links(command(["git", "rev-parse", "--show-toplevel"], text=True)) == checkout,
|
||||
"Run from the independent checkout top level")
|
||||
require(archives.no_links(command(["git", "rev-parse", "--absolute-git-dir"], text=True)) == checkout / ".git",
|
||||
"External/linked Git directory refused")
|
||||
require(command(["git", "remote", "get-url", "--all", "origin"], text=True) == REMOTE,
|
||||
"Origin must exactly match the approved remote")
|
||||
|
||||
|
||||
def transfer_policy(checkout, expected_commit, allow_lan_adapter=False):
|
||||
require(not (checkout / ".lfsconfig").exists() and not (checkout / ".lfsconfig").is_symlink(),
|
||||
"Repository LFS endpoint overrides refused")
|
||||
if not allow_lan_adapter:
|
||||
# Exit 1 means no matches; potential credential values are never read.
|
||||
require(command(["git", "config", "--get-regexp", TRANSFER_OVERRIDES]) == 1,
|
||||
"LFS endpoint/custom-transfer override or unreadable configuration refused")
|
||||
return None
|
||||
relative = "tools/lfs-publication/lan_lfs_transfer.py"
|
||||
adapter = archives.no_links(checkout / relative)
|
||||
require(not any(c.isspace() for c in str(adapter)), "LAN adapter path must not require argument quoting")
|
||||
prefix = "lfs.customtransfer.malecns-lan-https."
|
||||
allowed = {"lfs.standalonetransferagent": "malecns-lan-https",
|
||||
prefix + "path": "/usr/bin/python3", prefix + "args": "-B " + str(adapter),
|
||||
prefix + "concurrent": "true", prefix + "direction": "both"}
|
||||
keys = command(["git", "config", "--name-only", "--get-regexp", TRANSFER_OVERRIDES], text=True).splitlines()
|
||||
require(len(keys) == len(allowed) and set(keys) == set(allowed),
|
||||
"Only the exact approved LAN adapter configuration is permitted")
|
||||
for key, value in allowed.items():
|
||||
require(command(["git", "config", "--local", "--get-all", key], text=True) == value
|
||||
and command(["git", "config", "--get-all", key], text=True) == value,
|
||||
"LAN adapter configuration must be exact and exclusively repository-local")
|
||||
pin = archives.sha_file(adapter)
|
||||
blob = command(["git", "show", expected_commit + ":" + relative], bytes_output=True)
|
||||
require(len(blob) == pin["bytes"] and hashlib.sha256(blob).hexdigest() == pin["sha256"],
|
||||
"LAN adapter must exactly match the expected HEAD blob")
|
||||
return {"path": str(adapter), **pin}
|
||||
|
||||
|
||||
def new_output(path, checkout, cache, payload):
|
||||
p = archives.no_links(path)
|
||||
require(not within(p, PROTECTED.absolute()), "Output under original project refused")
|
||||
require(within(p, checkout.parent), "Output must stay in the independent workspace")
|
||||
require(not within(p, payload) and not within(p, checkout / ".git")
|
||||
and not within(p, checkout / "artifacts") and not within(p, cache),
|
||||
"Receipt must be outside payload, artifacts, Git and cache trees")
|
||||
require(not p.exists() and p.parent.is_dir(), "Receipt must be new with an existing parent")
|
||||
return p
|
||||
|
||||
|
||||
def cache_inventory(cache):
|
||||
count = total = 0
|
||||
objects = cache / "objects"
|
||||
if not objects.exists():
|
||||
return {"objects": 0, "bytes": 0}
|
||||
archives.no_links(objects)
|
||||
require(objects.is_dir(), "LFS objects must be a directory")
|
||||
for p in objects.rglob("*"):
|
||||
archives.no_links(p)
|
||||
s = p.lstat()
|
||||
if stat.S_ISDIR(s.st_mode):
|
||||
continue
|
||||
require(stat.S_ISREG(s.st_mode), "Nonregular cache object refused")
|
||||
parts = p.relative_to(objects).parts
|
||||
require(len(parts) == 3 and archives.HEX.fullmatch(parts[2]) is not None
|
||||
and parts[0] == parts[2][:2] and parts[1] == parts[2][2:4],
|
||||
"Unexpected LFS object layout")
|
||||
count += 1
|
||||
total += s.st_size
|
||||
return {"objects": count, "bytes": total}
|
||||
|
||||
|
||||
def source_pins(adapter=None):
|
||||
result = {name: {"path": str(Path(path).absolute()), **archives.sha_file(path)}
|
||||
for name, path in (("download_verify.py", __file__),
|
||||
("verify_publication.py", content.__file__),
|
||||
("publish_archives.py", archives.__file__))}
|
||||
if adapter is not None:
|
||||
result["lan_lfs_transfer.py"] = {"path": adapter["path"], **archives.sha_file(adapter["path"])}
|
||||
return result
|
||||
|
||||
|
||||
def run(expected_commit, cache, receipt, content_receipt, historical_inventory, *, allow_lan_adapter=False):
|
||||
require(isinstance(expected_commit, str) and re.fullmatch(r"[0-9a-f]{40}", expected_commit),
|
||||
"Expected commit must be a complete lowercase SHA-1")
|
||||
checkout = archives.no_links(Path.cwd())
|
||||
require(not within(checkout, PROTECTED.absolute()), "Original project checkout refused")
|
||||
gitdir = archives.no_links(checkout / ".git")
|
||||
require(gitdir.is_dir(), "Only an independent clone with its own .git directory is accepted")
|
||||
identity(expected_commit, checkout)
|
||||
adapter = transfer_policy(checkout, expected_commit, allow_lan_adapter)
|
||||
require(not command(["git", "status", "--porcelain", "--untracked-files=no"], text=True),
|
||||
"Tracked checkout changes refused")
|
||||
cache = archives.no_links(cache)
|
||||
require(not within(cache, PROTECTED.absolute()), "Cache under original project refused")
|
||||
require(within(cache, gitdir) or (within(cache, checkout.parent) and not within(cache, checkout)),
|
||||
"Cache must be inside this .git or outside checkout in the independent workspace")
|
||||
require(not cache.exists() and cache.parent.is_dir(), "Cache must never have existed for this attempt")
|
||||
payload = archives.no_links(checkout / PAYLOAD)
|
||||
receipt = new_output(receipt, checkout, cache, payload)
|
||||
content_receipt = new_output(content_receipt, checkout, cache, payload)
|
||||
history = archives.no_links(historical_inventory)
|
||||
require(len({receipt, content_receipt, history, cache}) == 4, "Inputs/outputs must be distinct")
|
||||
history_pin = archives.sha_file(history)
|
||||
pins = source_pins(adapter)
|
||||
versions = {"git": command(["git", "--version"], text=True),
|
||||
"git_lfs": command(["git", "lfs", "version"], text=True),
|
||||
"zstd": command(["zstd", "--version"], text=True),
|
||||
"python": sys.version.split()[0]}
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
cache.mkdir(exist_ok=False)
|
||||
initial = cache_inventory(cache)
|
||||
require(initial == {"objects": 0, "bytes": 0}, "New cache is not empty")
|
||||
print(json.dumps({"stage": "fresh-cache-created", "objects": 0, "bytes": 0}), flush=True)
|
||||
git = ["git", "-c", "lfs.storage=" + str(cache), "-c", "lfs.concurrenttransfers=2"]
|
||||
pull = command(git + ["lfs", "pull", "--include=artifacts/**", "--exclude=", "origin"])
|
||||
require(pull == 0, "LFS pull failed; cache retained, no automatic retry")
|
||||
print(json.dumps({"stage": "pull-exited", "exit_code": pull}), flush=True)
|
||||
fsck = command(git + ["lfs", "fsck", "--dry-run"])
|
||||
require(fsck == 0, "LFS fsck failed; cache retained, no automatic retry")
|
||||
print(json.dumps({"stage": "fsck-exited", "exit_code": fsck}), flush=True)
|
||||
objects = cache_inventory(cache)
|
||||
require(objects["objects"] > 0 and objects["bytes"] > 0, "Fresh LFS cache contains no objects")
|
||||
verified = content.verify_publication(payload, content_receipt, history)
|
||||
require(verified.get("pass_all") is True
|
||||
and archives.HEX.fullmatch(str(verified.get("aggregate_inventory_sha256", ""))) is not None,
|
||||
"Content verification did not report success")
|
||||
content_pin = archives.sha_file(content_receipt)
|
||||
require(content_pin["sha256"] == verified.get("receipt_sha256"), "Content receipt SHA mismatch")
|
||||
require(archives.sha_file(history) == history_pin and source_pins(adapter) == pins,
|
||||
"History or verification tool changed during attempt")
|
||||
identity(expected_commit, checkout)
|
||||
require(transfer_policy(checkout, expected_commit, allow_lan_adapter) == adapter,
|
||||
"LAN adapter identity/configuration changed during attempt")
|
||||
require(cache_inventory(cache) == objects, "Cache object inventory changed during content verification")
|
||||
report = {"schema": "lfs-publication-download-verification-v1", "pass_all": True,
|
||||
"started_utc": started, "verified_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"checkout": str(checkout), "expected_commit": expected_commit,
|
||||
"head": expected_commit, "origin": REMOTE,
|
||||
"explicit_pull_remote": "origin", "lfs_endpoint_overrides_refused": True,
|
||||
"lan_adapter": {"enabled": adapter is not None, "source": adapter,
|
||||
"configuration_scope": "repository-local only" if adapter else None,
|
||||
"temporary_download_directory": str(gitdir / "lfs/lan-transfer-tmp") if adapter else None},
|
||||
"cache": {"path": str(cache), "initially_absent": True,
|
||||
"before": initial, "after": objects},
|
||||
"pull_exit_code": pull, "fsck_dry_run_exit_code": fsck,
|
||||
"include": "artifacts/**", "exclude": "", "concurrent_transfers": 2,
|
||||
"content_receipt": {"path": str(content_receipt), **content_pin},
|
||||
"aggregate_inventory_sha256": verified["aggregate_inventory_sha256"],
|
||||
"historical_inventory": {"path": str(history), **history_pin,
|
||||
"status": verified.get("historical_status")},
|
||||
"tool_sources": pins, "tool_versions": versions, "fresh_download_proven": True,
|
||||
"learning_pass_claimed": False, "original_project_write_requested": False,
|
||||
"scope": "Fresh isolated LFS storage, actual pull/fsck exits and complete publication member verification; smoke covered by pull/fsck. Not a scientific gate, all-history audit, or executable-byte identity proof."}
|
||||
archives.write_new(receipt, archives.canonical(report))
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--expected-commit", required=True)
|
||||
parser.add_argument("--cache", required=True)
|
||||
parser.add_argument("--receipt", required=True)
|
||||
parser.add_argument("--content-receipt", required=True)
|
||||
parser.add_argument("--historical-inventory", required=True)
|
||||
parser.add_argument("--allow-lan-adapter", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
run(args.expected_commit, args.cache, args.receipt, args.content_receipt, args.historical_inventory,
|
||||
allow_lan_adapter=args.allow_lan_adapter)
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
print(json.dumps({"status": "FAILED-NOT-RETRIED", "error_type": type(exc).__name__}), flush=True)
|
||||
return 1
|
||||
print(json.dumps({"status": "PASS", "receipt": str(Path(args.receipt).absolute())}), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user