qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish licensed data snapshots with LAN LFS transfer and content audits

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-10-01 03:42:51 +08:00
parent 13cd0e0be7
commit 6e33e05ede
173 changed files with 3614 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/lfs-publication/download_verify.py
+232
View File
@@ -0,0 +1,232 @@
"""Fresh-cache transport verification for one independent publication checkout.
Run only from its top level. This is not a scientific audit or authorization to
publish. Failed attempts retain their cache/partial files and are never retried.
Git/LFS output is suppressed; receipts contain no environment or auth headers.
The caller supplies the trusted commit. Tool source SHA pins are observations,
not signatures; executable versions are not executable-byte identity proofs.
"""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import re
import stat
import subprocess
import sys
import publish_archives as archives
import verify_publication as content
REMOTE = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
PAYLOAD = "artifacts/public-resources-20261001"
PROTECTED = Path("/data/digital-drosophila")
TRANSFER_OVERRIDES = (r"^(lfs\.(url|pushurl|standalonetransferagent|customtransfer\..*|"
r"remote\.(autodetect|searchall))|remote\.(lfsdefault|origin\.lfsurl))$")
def require(condition, message):
if not condition:
raise ValueError(message)
def within(path, root):
return path == root or root in path.parents
def command(args, *, text=False, bytes_output=False):
"""Never return or print command diagnostics, which may contain credentials."""
result = subprocess.run(args, stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
stdout=subprocess.PIPE if text or bytes_output else subprocess.DEVNULL,
check=False)
if text or bytes_output:
require(result.returncode == 0, "Read-only Git/tool identity command failed")
if bytes_output:
return result.stdout
return result.stdout.decode("utf-8", errors="strict").strip()
return result.returncode
def identity(expected, checkout):
require(command(["git", "rev-parse", "--verify", "HEAD"], text=True) == expected,
"HEAD differs from expected commit")
require(archives.no_links(command(["git", "rev-parse", "--show-toplevel"], text=True)) == checkout,
"Run from the independent checkout top level")
require(archives.no_links(command(["git", "rev-parse", "--absolute-git-dir"], text=True)) == checkout / ".git",
"External/linked Git directory refused")
require(command(["git", "remote", "get-url", "--all", "origin"], text=True) == REMOTE,
"Origin must exactly match the approved remote")
def transfer_policy(checkout, expected_commit, allow_lan_adapter=False):
require(not (checkout / ".lfsconfig").exists() and not (checkout / ".lfsconfig").is_symlink(),
"Repository LFS endpoint overrides refused")
if not allow_lan_adapter:
# Exit 1 means no matches; potential credential values are never read.
require(command(["git", "config", "--get-regexp", TRANSFER_OVERRIDES]) == 1,
"LFS endpoint/custom-transfer override or unreadable configuration refused")
return None
relative = "tools/lfs-publication/lan_lfs_transfer.py"
adapter = archives.no_links(checkout / relative)
require(not any(c.isspace() for c in str(adapter)), "LAN adapter path must not require argument quoting")
prefix = "lfs.customtransfer.malecns-lan-https."
allowed = {"lfs.standalonetransferagent": "malecns-lan-https",
prefix + "path": "/usr/bin/python3", prefix + "args": "-B " + str(adapter),
prefix + "concurrent": "true", prefix + "direction": "both"}
keys = command(["git", "config", "--name-only", "--get-regexp", TRANSFER_OVERRIDES], text=True).splitlines()
require(len(keys) == len(allowed) and set(keys) == set(allowed),
"Only the exact approved LAN adapter configuration is permitted")
for key, value in allowed.items():
require(command(["git", "config", "--local", "--get-all", key], text=True) == value
and command(["git", "config", "--get-all", key], text=True) == value,
"LAN adapter configuration must be exact and exclusively repository-local")
pin = archives.sha_file(adapter)
blob = command(["git", "show", expected_commit + ":" + relative], bytes_output=True)
require(len(blob) == pin["bytes"] and hashlib.sha256(blob).hexdigest() == pin["sha256"],
"LAN adapter must exactly match the expected HEAD blob")
return {"path": str(adapter), **pin}
def new_output(path, checkout, cache, payload):
p = archives.no_links(path)
require(not within(p, PROTECTED.absolute()), "Output under original project refused")
require(within(p, checkout.parent), "Output must stay in the independent workspace")
require(not within(p, payload) and not within(p, checkout / ".git")
and not within(p, checkout / "artifacts") and not within(p, cache),
"Receipt must be outside payload, artifacts, Git and cache trees")
require(not p.exists() and p.parent.is_dir(), "Receipt must be new with an existing parent")
return p
def cache_inventory(cache):
count = total = 0
objects = cache / "objects"
if not objects.exists():
return {"objects": 0, "bytes": 0}
archives.no_links(objects)
require(objects.is_dir(), "LFS objects must be a directory")
for p in objects.rglob("*"):
archives.no_links(p)
s = p.lstat()
if stat.S_ISDIR(s.st_mode):
continue
require(stat.S_ISREG(s.st_mode), "Nonregular cache object refused")
parts = p.relative_to(objects).parts
require(len(parts) == 3 and archives.HEX.fullmatch(parts[2]) is not None
and parts[0] == parts[2][:2] and parts[1] == parts[2][2:4],
"Unexpected LFS object layout")
count += 1
total += s.st_size
return {"objects": count, "bytes": total}
def source_pins(adapter=None):
result = {name: {"path": str(Path(path).absolute()), **archives.sha_file(path)}
for name, path in (("download_verify.py", __file__),
("verify_publication.py", content.__file__),
("publish_archives.py", archives.__file__))}
if adapter is not None:
result["lan_lfs_transfer.py"] = {"path": adapter["path"], **archives.sha_file(adapter["path"])}
return result
def run(expected_commit, cache, receipt, content_receipt, historical_inventory, *, allow_lan_adapter=False):
require(isinstance(expected_commit, str) and re.fullmatch(r"[0-9a-f]{40}", expected_commit),
"Expected commit must be a complete lowercase SHA-1")
checkout = archives.no_links(Path.cwd())
require(not within(checkout, PROTECTED.absolute()), "Original project checkout refused")
gitdir = archives.no_links(checkout / ".git")
require(gitdir.is_dir(), "Only an independent clone with its own .git directory is accepted")
identity(expected_commit, checkout)
adapter = transfer_policy(checkout, expected_commit, allow_lan_adapter)
require(not command(["git", "status", "--porcelain", "--untracked-files=no"], text=True),
"Tracked checkout changes refused")
cache = archives.no_links(cache)
require(not within(cache, PROTECTED.absolute()), "Cache under original project refused")
require(within(cache, gitdir) or (within(cache, checkout.parent) and not within(cache, checkout)),
"Cache must be inside this .git or outside checkout in the independent workspace")
require(not cache.exists() and cache.parent.is_dir(), "Cache must never have existed for this attempt")
payload = archives.no_links(checkout / PAYLOAD)
receipt = new_output(receipt, checkout, cache, payload)
content_receipt = new_output(content_receipt, checkout, cache, payload)
history = archives.no_links(historical_inventory)
require(len({receipt, content_receipt, history, cache}) == 4, "Inputs/outputs must be distinct")
history_pin = archives.sha_file(history)
pins = source_pins(adapter)
versions = {"git": command(["git", "--version"], text=True),
"git_lfs": command(["git", "lfs", "version"], text=True),
"zstd": command(["zstd", "--version"], text=True),
"python": sys.version.split()[0]}
started = datetime.now(timezone.utc).isoformat()
cache.mkdir(exist_ok=False)
initial = cache_inventory(cache)
require(initial == {"objects": 0, "bytes": 0}, "New cache is not empty")
print(json.dumps({"stage": "fresh-cache-created", "objects": 0, "bytes": 0}), flush=True)
git = ["git", "-c", "lfs.storage=" + str(cache), "-c", "lfs.concurrenttransfers=2"]
pull = command(git + ["lfs", "pull", "--include=artifacts/**", "--exclude=", "origin"])
require(pull == 0, "LFS pull failed; cache retained, no automatic retry")
print(json.dumps({"stage": "pull-exited", "exit_code": pull}), flush=True)
fsck = command(git + ["lfs", "fsck", "--dry-run"])
require(fsck == 0, "LFS fsck failed; cache retained, no automatic retry")
print(json.dumps({"stage": "fsck-exited", "exit_code": fsck}), flush=True)
objects = cache_inventory(cache)
require(objects["objects"] > 0 and objects["bytes"] > 0, "Fresh LFS cache contains no objects")
verified = content.verify_publication(payload, content_receipt, history)
require(verified.get("pass_all") is True
and archives.HEX.fullmatch(str(verified.get("aggregate_inventory_sha256", ""))) is not None,
"Content verification did not report success")
content_pin = archives.sha_file(content_receipt)
require(content_pin["sha256"] == verified.get("receipt_sha256"), "Content receipt SHA mismatch")
require(archives.sha_file(history) == history_pin and source_pins(adapter) == pins,
"History or verification tool changed during attempt")
identity(expected_commit, checkout)
require(transfer_policy(checkout, expected_commit, allow_lan_adapter) == adapter,
"LAN adapter identity/configuration changed during attempt")
require(cache_inventory(cache) == objects, "Cache object inventory changed during content verification")
report = {"schema": "lfs-publication-download-verification-v1", "pass_all": True,
"started_utc": started, "verified_utc": datetime.now(timezone.utc).isoformat(),
"checkout": str(checkout), "expected_commit": expected_commit,
"head": expected_commit, "origin": REMOTE,
"explicit_pull_remote": "origin", "lfs_endpoint_overrides_refused": True,
"lan_adapter": {"enabled": adapter is not None, "source": adapter,
"configuration_scope": "repository-local only" if adapter else None,
"temporary_download_directory": str(gitdir / "lfs/lan-transfer-tmp") if adapter else None},
"cache": {"path": str(cache), "initially_absent": True,
"before": initial, "after": objects},
"pull_exit_code": pull, "fsck_dry_run_exit_code": fsck,
"include": "artifacts/**", "exclude": "", "concurrent_transfers": 2,
"content_receipt": {"path": str(content_receipt), **content_pin},
"aggregate_inventory_sha256": verified["aggregate_inventory_sha256"],
"historical_inventory": {"path": str(history), **history_pin,
"status": verified.get("historical_status")},
"tool_sources": pins, "tool_versions": versions, "fresh_download_proven": True,
"learning_pass_claimed": False, "original_project_write_requested": False,
"scope": "Fresh isolated LFS storage, actual pull/fsck exits and complete publication member verification; smoke covered by pull/fsck. Not a scientific gate, all-history audit, or executable-byte identity proof."}
archives.write_new(receipt, archives.canonical(report))
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--expected-commit", required=True)
parser.add_argument("--cache", required=True)
parser.add_argument("--receipt", required=True)
parser.add_argument("--content-receipt", required=True)
parser.add_argument("--historical-inventory", required=True)
parser.add_argument("--allow-lan-adapter", action="store_true")
args = parser.parse_args()
try:
run(args.expected_commit, args.cache, args.receipt, args.content_receipt, args.historical_inventory,
allow_lan_adapter=args.allow_lan_adapter)
except (Exception, KeyboardInterrupt) as exc:
print(json.dumps({"status": "FAILED-NOT-RETRIED", "error_type": type(exc).__name__}), flush=True)
return 1
print(json.dumps({"status": "PASS", "receipt": str(Path(args.receipt).absolute())}), flush=True)
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/lfs-publication/lan_lfs_transfer.py
+336
View File
@@ -0,0 +1,336 @@
"""Repository-local, LAN-pinned Git LFS standalone transfer agent.
Protocol: https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md
Only fixed-origin SSH authentication and HTTPS basic transfers are supported.
No config/network changes, retries, redirects, credential files, or cache-based
download substitution. Credentials are passed to curl exclusively on stdin.
"""
from __future__ import annotations
import hashlib
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import sys
import tempfile
from urllib.parse import urlsplit
ORIGIN = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
HOST = "git.lain.asia"
IP = "192.168.10.31"
PREFIX = "/qishizhe/digital-drosophila.git/info/lfs"
SOURCE = Path("/data/digital-drosophila")
OID = re.compile(r"^[0-9a-f]{64}$")
HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
MAX_JSON = 2 * 1024 * 1024
BLOCK = 1024 * 1024
class TransferError(Exception):
"""Only fixed, non-secret messages may be passed to this class."""
def require(ok, code):
if not ok:
raise TransferError(code)
def decode_json(data):
def unique(pairs):
result = {}
for key, value in pairs:
require(key not in result, "duplicate-json-key")
result[key] = value
return result
require(len(data) <= MAX_JSON, "json-response-too-large")
try:
return json.loads(data, object_pairs_hook=unique)
except (UnicodeError, ValueError):
raise TransferError("invalid-json") from None
def safe_path(value):
require(isinstance(value, (str, Path)), "invalid-file-path")
p = Path(value)
require(p.is_absolute() and ".." not in p.parts, "absolute-safe-path-required")
for component in (p, *p.parents):
if component.exists() or component.is_symlink():
s = component.lstat()
require(not stat.S_ISLNK(s.st_mode) and not
(getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)),
"symlink-or-reparse-refused")
return p
def safe_url(value):
require(isinstance(value, str) and len(value) <= 16384 and
not any(ord(c) <= 32 or ord(c) == 127 for c in value), "invalid-action-url")
try:
p = urlsplit(value)
valid = (p.scheme == "https" and p.netloc in (HOST, HOST + ":443") and
p.hostname == HOST and p.port in (None, 443) and
p.username is None and p.password is None and not p.fragment)
except ValueError:
raise TransferError("url-authority-refused") from None
require(valid, "url-authority-refused")
require(p.path == PREFIX or p.path.startswith(PREFIX + "/"), "url-repository-prefix-refused")
require("%" not in p.path and "\\" not in p.path and
all(x not in ("", ".", "..") for x in p.path.split("/")[1:]), "url-path-refused")
return value
def safe_headers(value, *, allow_upload_chunked=False):
require(isinstance(value, dict) and len(value) <= 32, "invalid-action-headers")
result, seen = {}, set()
denied = {"host", "connection", "content-length", "transfer-encoding", "te",
"trailer", "upgrade", "proxy-authorization", "proxy-connection", "range"}
for key, item in value.items():
upload_chunked = (allow_upload_chunked is True and isinstance(key, str) and
key.lower() == "transfer-encoding" and item == "chunked")
require(isinstance(key, str) and HEADER.fullmatch(key) is not None and
key.lower() not in seen and (key.lower() not in denied or upload_chunked),
"header-name-refused")
require(isinstance(item, str) and len(item) <= 16384 and
not any(ord(c) < 32 or ord(c) == 127 for c in item), "header-value-refused")
seen.add(key.lower())
result[key] = item
return result
def quote(value):
require(isinstance(value, str) and not any(ord(c) < 32 or ord(c) == 127 for c in value),
"curl-config-value-refused")
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
def fingerprint(s):
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
def verify_file(path, oid, size):
path = safe_path(path)
before = fingerprint(path.lstat())
require(stat.S_ISREG(before[2]) and before[3] == size, "object-file-type-or-size-mismatch")
digest, count = hashlib.sha256(), 0
with path.open("rb") as handle:
opened = fingerprint(os.fstat(handle.fileno()))
# Windows path stat and fstat can expose different ctime definitions.
require(opened[:-1] == before[:-1] if os.name == "nt" else opened == before,
"object-changed-at-open")
while data := handle.read(BLOCK):
digest.update(data)
count += len(data)
require(fingerprint(os.fstat(handle.fileno())) == opened, "object-changed-during-read")
require(fingerprint(path.lstat()) == before and count == size and digest.hexdigest() == oid,
"object-sha-or-stability-mismatch")
return before
def git_output(cwd, *args):
try:
result = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True,
timeout=15, check=False)
require(result.returncode == 0, "git-inspection-failed")
return result.stdout.decode("utf-8").strip()
except (OSError, UnicodeError, subprocess.TimeoutExpired):
raise TransferError("git-inspection-failed") from None
class Agent:
def __init__(self, message, emit, cwd=None):
require(message.get("event") == "init" and message.get("operation") in ("upload", "download"),
"invalid-init")
require(message.get("remote") in ("origin", ORIGIN), "remote-refused")
concurrency = message.get("concurrenttransfers")
require(type(concurrency) is int and 1 <= concurrency <= 2 and
type(message.get("concurrent")) is bool, "concurrency-must-be-at-most-two")
cwd = safe_path(Path.cwd() if cwd is None else cwd)
require(cwd != SOURCE and SOURCE not in cwd.parents, "scientific-source-cwd-refused")
self.root = safe_path(git_output(cwd, "rev-parse", "--show-toplevel"))
require(self.root != SOURCE and SOURCE not in self.root.parents, "scientific-source-repository-refused")
require(git_output(cwd, "config", "--get", "remote.origin.url") == ORIGIN, "origin-refused")
self.git_dir = safe_path(git_output(cwd, "rev-parse", "--absolute-git-dir"))
require(self.git_dir == self.root / ".git" and self.git_dir.is_dir(), "ordinary-isolated-clone-required")
self.operation, self.emit = message["operation"], emit
def authenticate(self):
command = ["ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=15", "-o", "ConnectionAttempts=1",
"-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=2",
"-p", "2222", "git@192.168.10.31", "git-lfs-authenticate",
"qishizhe/digital-drosophila.git", self.operation]
try:
result = subprocess.run(command, capture_output=True, timeout=45, check=False)
require(result.returncode == 0, "ssh-authentication-failed")
auth = decode_json(result.stdout)
except (OSError, subprocess.TimeoutExpired):
raise TransferError("ssh-authentication-failed") from None
require(isinstance(auth, dict), "invalid-ssh-authentication-response")
href = safe_url(auth.get("href"))
parsed = urlsplit(href)
require(parsed.path == PREFIX and not parsed.query, "authentication-base-url-refused")
return href, safe_headers(auth.get("header", {}))
def curl(self, method, url, headers, *, body=None, upload=None, download=None, size=None):
safe_url(url)
headers = safe_headers(headers, allow_upload_chunked=(self.operation == "upload" and
method == "PUT" and upload is not None and body is None and download is None))
require(method in ("GET", "POST", "PUT"), "unsupported-http-method")
present = {name.lower() for name in headers}
if "accept" not in present:
headers["Accept"] = "application/vnd.git-lfs+json"
if method == "PUT" and "content-type" not in present:
headers["Content-Type"] = "application/octet-stream"
if body is not None and "content-type" not in present:
headers["Content-Type"] = "application/vnd.git-lfs+json"
lines = ["url = " + quote(url), "request = " + quote(method),
'noproxy = "*"', 'resolve = "' + HOST + ':443:' + IP + '"',
'proto = "=https"', 'proto-redir = "=https"', "max-redirs = 0",
"connect-timeout = 15", "max-time = 1800", "fail", "silent", "show-error",
'write-out = "\\n%{http_code}"']
for name, value in headers.items():
lines.append("header = " + quote(name + ": " + value))
if body is not None:
encoded = json.dumps(body, separators=(",", ":"), allow_nan=False)
lines.append("data-binary = " + quote(encoded))
if upload is not None:
lines.append("upload-file = " + quote(str(safe_path(upload))))
if download is not None:
lines.append("output = " + quote(str(safe_path(download))))
require(type(size) is int and size >= 0, "invalid-download-limit")
# A zero-byte object still needs a full HTTP fetch, not a cache hit.
lines.append("max-filesize = " + str(max(size, 1)))
else:
lines.append("max-filesize = " + str(MAX_JSON))
config = ("\n".join(lines) + "\n").encode("utf-8")
try:
result = subprocess.run(["curl", "-q", "--config", "-"], input=config,
capture_output=True, timeout=1830, check=False)
except (OSError, subprocess.TimeoutExpired):
raise TransferError("curl-execution-failed") from None
# Never print stderr, response data, URLs, configuration, or headers.
require(result.returncode == 0, "curl-request-failed")
require(len(result.stdout) <= MAX_JSON + 32, "http-response-too-large")
content, separator, status = result.stdout.rpartition(b"\n")
require(separator and len(status) == 3 and status.isdigit() and 200 <= int(status) < 300,
"non-success-http-status")
return content
def batch(self, oid, size):
href, headers = self.authenticate()
headers = {k: v for k, v in headers.items() if k.lower() not in ("content-type", "accept")}
headers.update({"Content-Type": "application/vnd.git-lfs+json", "Accept": "application/vnd.git-lfs+json"})
response = decode_json(self.curl("POST", href + "/objects/batch", headers,
body={"operation": self.operation, "transfers": ["basic"],
"objects": [{"oid": oid, "size": size}]}))
require(isinstance(response, dict) and response.get("transfer", "basic") == "basic",
"non-basic-batch-response")
objects = response.get("objects")
require(isinstance(objects, list) and len(objects) == 1 and isinstance(objects[0], dict),
"invalid-batch-object-count")
item = objects[0]
require(item.get("oid") == oid and type(item.get("size")) is int and item["size"] == size,
"batch-object-identity-mismatch")
require("error" not in item, "batch-object-rejected")
actions = item.get("actions", {})
require(isinstance(actions, dict), "invalid-batch-actions")
allowed = {"upload", "verify"} if self.operation == "upload" else {"download"}
require(set(actions) <= allowed, "unexpected-batch-action")
for name, action in actions.items():
require(isinstance(action, dict), "invalid-batch-action")
safe_url(action.get("href"))
safe_headers(action.get("header", {}), allow_upload_chunked=(self.operation == "upload" and name == "upload"))
return actions
def progress(self, oid, size, previous=0):
self.emit({"event": "progress", "oid": oid, "bytesSoFar": size, "bytesSinceLast": size - previous})
def transfer(self, message):
require(message.get("event") == self.operation and message.get("action") is None,
"standalone-operation-required")
oid, size = message.get("oid"), message.get("size")
require(isinstance(oid, str) and OID.fullmatch(oid) is not None and
type(size) is int and 0 <= size <= 2**63 - 1, "invalid-object-identity")
source, before = None, None
if self.operation == "upload":
source = safe_path(message.get("path"))
require(source == self.git_dir / "lfs" / "objects" / oid[:2] / oid[2:4] / oid,
"upload-outside-current-lfs-object-store")
before = verify_file(source, oid, size)
actions = self.batch(oid, size)
self.progress(oid, 0)
if self.operation == "upload":
if "upload" in actions:
action = actions["upload"]
self.curl("PUT", action["href"], action.get("header", {}), upload=source)
require(fingerprint(source.lstat()) == before, "upload-source-changed")
if "verify" in actions:
action = actions["verify"]
headers = {k: v for k, v in action.get("header", {}).items() if k.lower() != "content-type"}
headers["Content-Type"] = "application/vnd.git-lfs+json"
self.curl("POST", action["href"], headers, body={"oid": oid, "size": size})
else:
require(not actions, "verify-without-upload-refused")
require(fingerprint(source.lstat()) == before, "upload-source-changed")
self.progress(oid, size)
return {"event": "complete", "oid": oid}
require("download" in actions, "download-action-missing")
directory = safe_path(self.git_dir / "lfs" / "lan-transfer-tmp")
directory.mkdir(parents=True, exist_ok=True)
safe_path(directory)
fd, name = tempfile.mkstemp(prefix=oid + ".", suffix=".partial", dir=directory)
os.close(fd)
target = Path(name)
action = actions["download"]
# On failure the uniquely created partial remains for investigation;
# it is never used as a fallback/cache input on a later transfer.
self.curl("GET", action["href"], action.get("header", {}), download=target, size=size)
verify_file(target, oid, size)
self.progress(oid, size)
return {"event": "complete", "oid": oid, "path": str(target)}
def emit(value):
sys.stdout.write(json.dumps(value, separators=(",", ":"), allow_nan=False) + "\n")
sys.stdout.flush()
def serve(inp=None, send=None, factory=Agent):
inp, send = sys.stdin if inp is None else inp, emit if send is None else send
agent = None
while True:
line = inp.readline(MAX_JSON + 1)
if not line:
return 0
try:
message = decode_json(line)
require(isinstance(message, dict), "protocol-object-required")
if agent is None:
agent = factory(message, send)
send({})
elif message.get("event") == "terminate":
return 0
else:
oid = message.get("oid")
safe_oid = oid if isinstance(oid, str) and OID.fullmatch(oid) else ""
try:
send(agent.transfer(message))
except Exception as exc:
code = str(exc) if isinstance(exc, TransferError) else "transfer-internal-failure"
send({"event": "complete", "oid": safe_oid,
"error": {"code": 2, "message": code}})
except Exception as exc:
code = str(exc) if isinstance(exc, TransferError) else "agent-initialization-or-protocol-failure"
if agent is None:
send({"error": {"code": 32, "message": code}})
else:
# No attacker-supplied raw line or exception text is printed.
sys.stderr.write("lan-lfs: invalid protocol; stopping\n")
return 1
if __name__ == "__main__":
raise SystemExit(serve())
tools/lfs-publication/run_lan_smoke.py
+52
View File
@@ -0,0 +1,52 @@
"""One real archive chunk upload and fresh download; never modifies source data."""
import argparse
import json
from pathlib import Path
import time
from datetime import datetime, timezone
import lan_lfs_transfer as lan
import publish_archives as archives
def run(output):
root = Path.cwd()
receipt = archives.no_links(output)
if receipt.exists() or root / 'registry/publication-20261001' not in receipt.parents:
raise ValueError('New publication receipt required')
catalog = archives.json_read(root / 'artifacts/public-resources-20261001/malecns-synaptic-partners/catalog.json')
chunk = catalog['chunks'][0]
oid, size = chunk['sha256'], chunk['bytes']
source = root / '.git/lfs/objects' / oid[:2] / oid[2:4] / oid
records = []
for operation in ('upload', 'download'):
agent = lan.Agent(dict(event='init', operation=operation, remote='origin',
concurrent=True, concurrenttransfers=2), lambda value: None)
original = agent.curl
methods = []
def observe(method, url, headers, **kwargs):
result = original(method, url, headers, **kwargs)
methods.append(method)
return result
agent.curl = observe
start = time.perf_counter()
result = agent.transfer(dict(event=operation, action=None, oid=oid, size=size,
**({'path': str(source)} if operation == 'upload' else {})))
record = dict(operation=operation, seconds=time.perf_counter()-start, methods=methods,
sha256=oid, bytes=size, complete=result['event'] == 'complete')
if operation == 'download':
record['new_download_file'] = result['path']
records.append(record)
report = dict(schema='lfs-lan-roundtrip-smoke-v1', pass_all=True,
observed_utc=datetime.now(timezone.utc).isoformat(),
ip=lan.IP, hostname=lan.HOST, port=443, certificate_verification=True,
proxy_used=False, transfers=records,
adapter=archives.sha_file(lan.__file__))
archives.write_new(receipt, archives.canonical(report))
print(json.dumps(report), flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--receipt', required=True)
run(parser.parse_args().receipt)
tools/lfs-publication/test_download_verify.py
+245
View File
@@ -0,0 +1,245 @@
"""Synthetic temporary fixtures and mocked Git/content operations; no transfer."""
import contextlib
import io
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import download_verify as D
class DownloadTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.base = Path(self.temp.name)
self.checkout = self.base / "verification"
(self.checkout / ".git").mkdir(parents=True)
(self.checkout / "registry").mkdir()
(self.checkout / D.PAYLOAD).mkdir(parents=True)
self.cache = self.checkout / ".git" / "brand-new-lfs"
self.receipt = self.checkout / "registry" / "transport.json"
self.content_receipt = self.checkout / "registry" / "content.json"
self.history = self.base / "history.gz"
self.history.write_bytes(b"synthetic history placeholder")
self.commit = "a" * 40
self.calls = []
self.pull_code = self.fsck_code = 0
self.remote = D.REMOTE
self.head = self.commit
self.dirty = ""
self.content_failure = False
self.config_exit = 1
self.adapter_config = {}
self.local_adapter_config = {}
self.adapter_blob = b"# Synthetic LAN adapter fixture; never executed.\n"
self.addCleanup(patch.stopall)
patch.object(D.Path, "cwd", return_value=self.checkout).start()
patch.object(D, "command", side_effect=self.command).start()
self.verifier = patch.object(D.content, "verify_publication", side_effect=self.verify).start()
self.capture = contextlib.redirect_stdout(io.StringIO())
self.capture.__enter__()
self.addCleanup(self.capture.__exit__, None, None, None)
def command(self, args, *, text=False, bytes_output=False):
self.calls.append(args)
if "rev-parse" in args:
if "HEAD" in args:
return self.head
return str(self.checkout / ".git") if "--absolute-git-dir" in args else str(self.checkout)
if "remote" in args:
return self.remote
if "status" in args:
return self.dirty
if "--get-regexp" in args:
if "--name-only" in args:
return "\n".join(self.adapter_config)
return self.config_exit
if "--get-all" in args:
values = self.local_adapter_config if "--local" in args else self.adapter_config
return values.get(args[-1], "")
if "show" in args:
return self.adapter_blob
if "pull" in args:
oid = "f" * 64
obj = self.cache / "objects" / oid[:2] / oid[2:4] / oid
obj.parent.mkdir(parents=True)
obj.write_bytes(b"synthetic object")
return self.pull_code
if "fsck" in args:
return self.fsck_code
return "synthetic-version"
def verify(self, root, receipt, history):
self.assertEqual(root, self.checkout / D.PAYLOAD)
self.assertEqual(history, self.history)
if self.content_failure:
raise ValueError("synthetic failure")
D.archives.write_new(receipt, b'{"synthetic":true}\n')
return {"pass_all": True, "receipt_sha256": D.archives.sha_file(receipt)["sha256"],
"aggregate_inventory_sha256": "b" * 64, "historical_status": "FULL_CURRENT_MEMBER_MATCH"}
def run_attempt(self):
return D.run(self.commit, self.cache, self.receipt, self.content_receipt, self.history)
def prepare_adapter(self):
adapter = self.checkout / "tools/lfs-publication/lan_lfs_transfer.py"
adapter.parent.mkdir(parents=True)
adapter.write_bytes(self.adapter_blob)
prefix = "lfs.customtransfer.malecns-lan-https."
self.adapter_config = {"lfs.standalonetransferagent": "malecns-lan-https",
prefix + "path": "/usr/bin/python3", prefix + "args": "-B " + str(adapter),
prefix + "concurrent": "true", prefix + "direction": "both"}
self.local_adapter_config = self.adapter_config.copy()
self.config_exit = 0
return adapter
def assert_not_started(self):
self.assertFalse(self.cache.exists())
self.assertFalse(self.receipt.exists())
self.assertFalse(any("pull" in args for args in self.calls))
def test_success_binds_new_cache_exits_content_and_source_sha(self):
result = self.run_attempt()
self.assertEqual(json.loads(self.receipt.read_bytes()), result)
self.assertTrue(result["cache"]["initially_absent"])
self.assertEqual(result["cache"]["before"], {"objects": 0, "bytes": 0})
self.assertEqual(result["cache"]["after"], {"objects": 1, "bytes": 16})
self.assertEqual(result["content_receipt"]["sha256"], D.archives.sha_file(self.content_receipt)["sha256"])
self.assertEqual(len(result["tool_sources"]), 3)
self.assertFalse(result["learning_pass_claimed"])
pull = next(args for args in self.calls if "pull" in args)
fsck = next(args for args in self.calls if "fsck" in args)
self.assertIn("--include=artifacts/**", pull)
self.assertIn("--exclude=", pull)
self.assertEqual(pull[-1], "origin")
self.assertIn("--dry-run", fsck)
for args in (pull, fsck):
self.assertIn("lfs.storage=" + str(self.cache), args)
self.assertIn("lfs.concurrenttransfers=2", args)
def test_existing_cache_receipt_and_content_receipt_are_never_reused(self):
for path in (self.cache, self.receipt, self.content_receipt):
with self.subTest(path=path):
path.write_bytes(b"keep")
with self.assertRaises(ValueError):
self.run_attempt()
self.assertEqual(path.read_bytes(), b"keep")
self.assertFalse(any("pull" in args for args in self.calls))
path.unlink()
def test_wrong_head_or_remote_or_dirty_tree_refused(self):
for attr, value in (("head", "c" * 40), ("remote", "https://wrong.invalid/repo.git"), ("dirty", " M file")):
original = getattr(self, attr)
setattr(self, attr, value)
with self.subTest(attr=attr), self.assertRaises(ValueError):
self.run_attempt()
self.assert_not_started()
setattr(self, attr, original)
def test_unsafe_paths_refused_before_transfer(self):
original = self.cache
for unsafe in (self.checkout / "payload-cache", self.base / ".." / "escape",
D.PROTECTED / "should-never-create"):
self.cache = unsafe
with self.subTest(path=unsafe), self.assertRaises(ValueError):
self.run_attempt()
self.assertFalse(any("pull" in args for args in self.calls))
self.cache = original
self.receipt = self.checkout / D.PAYLOAD / "receipt.json"
with self.assertRaises(ValueError):
self.run_attempt()
self.assert_not_started()
def test_pull_failure_retains_partial_and_never_fscks_or_retries(self):
self.pull_code = 9
with self.assertRaises(ValueError):
self.run_attempt()
self.assertTrue(self.cache.exists())
self.assertFalse(self.receipt.exists())
self.assertFalse(self.content_receipt.exists())
self.assertFalse(any("fsck" in args for args in self.calls))
with self.assertRaises(ValueError):
self.run_attempt()
self.assertEqual(sum("pull" in args for args in self.calls), 1)
def test_endpoint_and_custom_transfer_overrides_fail_closed(self):
for code in (0, 2):
self.config_exit = code
with self.subTest(code=code), self.assertRaises(ValueError):
self.run_attempt()
self.assert_not_started()
self.config_exit = 1
(self.checkout / ".lfsconfig").write_text("[lfs]\nurl = https://not-approved.invalid\n")
with self.assertRaises(ValueError):
self.run_attempt()
self.assert_not_started()
def test_optional_lan_adapter_is_exact_local_and_head_bound(self):
adapter = self.prepare_adapter()
with self.assertRaises(ValueError):
self.run_attempt()
self.assert_not_started()
report = D.run(self.commit, self.cache, self.receipt, self.content_receipt,
self.history, allow_lan_adapter=True)
self.assertTrue(report["lan_adapter"]["enabled"])
self.assertEqual(report["tool_sources"]["lan_lfs_transfer.py"]["sha256"],
D.archives.sha_file(adapter)["sha256"])
def test_lan_wrong_blob_global_override_or_other_endpoint_rejected(self):
self.prepare_adapter()
self.adapter_blob = b"not the actual file"
with self.assertRaises(ValueError):
D.transfer_policy(self.checkout, self.commit, True)
self.adapter_blob = (self.checkout / "tools/lfs-publication/lan_lfs_transfer.py").read_bytes()
original = self.local_adapter_config["lfs.standalonetransferagent"]
self.local_adapter_config["lfs.standalonetransferagent"] = ""
with self.assertRaises(ValueError):
D.transfer_policy(self.checkout, self.commit, True)
self.local_adapter_config["lfs.standalonetransferagent"] = original
self.adapter_config["lfs.url"] = "https://wrong.invalid"
with self.assertRaises(ValueError):
D.transfer_policy(self.checkout, self.commit, True)
self.assert_not_started()
def test_fsck_failure_does_not_call_content_or_publish_receipt(self):
self.fsck_code = 8
with self.assertRaises(ValueError):
self.run_attempt()
self.verifier.assert_not_called()
self.assertTrue(self.cache.exists())
self.assertFalse(self.receipt.exists())
def test_content_failure_leaves_no_transport_pass(self):
self.content_failure = True
with self.assertRaises(ValueError):
self.run_attempt()
self.assertFalse(self.receipt.exists())
self.assertEqual(sum("pull" in args for args in self.calls), 1)
def test_head_changed_during_verification_preserves_content_but_no_transport_pass(self):
def changed(*args):
result = self.verify(*args)
self.head = "d" * 40
return result
self.verifier.side_effect = changed
with self.assertRaises(ValueError):
self.run_attempt()
self.assertTrue(self.content_receipt.exists())
self.assertFalse(self.receipt.exists())
def test_cli_failure_never_prints_exception_diagnostics(self):
with patch.object(D, "run", side_effect=RuntimeError("secret should not escape")), \
patch.object(D.sys, "argv", ["download_verify", "--expected-commit", self.commit,
"--cache", str(self.cache), "--receipt", str(self.receipt),
"--content-receipt", str(self.content_receipt), "--historical-inventory", str(self.history)]), \
contextlib.redirect_stdout(io.StringIO()) as output:
self.assertEqual(D.main(), 1)
self.assertNotIn("secret should not escape", output.getvalue())
self.assertIn("FAILED-NOT-RETRIED", output.getvalue())
if __name__ == "__main__":
unittest.main()
tools/lfs-publication/test_lan_lfs_transfer.py
+286
View File
@@ -0,0 +1,286 @@
"""Pure temporary/mocked protocol tests: never SSH, curl, GPU, or real Git."""
import contextlib
import hashlib
import io
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
import lan_lfs_transfer as M
BASE = "https://" + M.HOST + M.PREFIX
TOKEN = "synthetic-fixture-" + "credential-value"
class AdapterTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name) / "clone"
self.git = self.root / ".git"
self.git.mkdir(parents=True)
self.data = b"synthetic content\x00\xff\n"
self.oid = hashlib.sha256(self.data).hexdigest()
self.size = len(self.data)
self.events = []
self.agent = object.__new__(M.Agent)
self.agent.root, self.agent.git_dir = self.root, self.git
self.agent.operation, self.agent.emit = "upload", self.events.append
self.source = self.git / "lfs" / "objects" / self.oid[:2] / self.oid[2:4] / self.oid
self.source.parent.mkdir(parents=True)
self.source.write_bytes(self.data)
def request(self, operation="upload", **extra):
value = {"event": operation, "oid": self.oid, "size": self.size, "action": None}
if operation == "upload":
value["path"] = str(self.source)
value.update(extra)
return value
def init_message(self, **extra):
return dict(event="init", operation="upload", remote="origin", concurrent=True,
concurrenttransfers=2, **extra)
def action(self, suffix="objects/" ):
return {"href": BASE + "/" + suffix + self.oid,
"header": {"Authorization": "Bearer " + TOKEN}}
def test_strict_url_policy(self):
self.assertEqual(M.safe_url(BASE + "/objects/" + self.oid + "?signed=opaque"),
BASE + "/objects/" + self.oid + "?signed=opaque")
invalid = [BASE.replace("https:", "http:"), BASE.replace(M.HOST, M.IP),
BASE.replace(M.HOST, "other.example"), BASE.replace(M.HOST, M.HOST + ":444"),
BASE.replace(M.HOST, "user:opaque@" + M.HOST), BASE + "-other/objects/x",
BASE + "/../other", BASE + "/%2e%2e/other", BASE + "/objects/x#fragment",
BASE + "//objects/x", BASE + "/objects\\x", BASE + "\n"]
for url in invalid:
with self.subTest(url=url), self.assertRaises(M.TransferError):
M.safe_url(url)
def test_headers_reject_injection_and_routing_overrides(self):
for headers in ({"Authorization": "x\r\nHost: bad"}, {"Host": "bad"},
{"Authorization": "a", "authorization": "b"},
{"Range": "bytes=1-"}, {"bad\nkey": "x"}, {"X": "\x00"}):
with self.subTest(headers=headers), self.assertRaises(M.TransferError):
M.safe_headers(headers)
def test_init_checks_origin_and_isolated_git_directory(self):
values = [str(self.root), M.ORIGIN, str(self.git)]
with patch.object(M, "git_output", side_effect=values):
actual = M.Agent(self.init_message(), self.events.append, cwd=self.root)
self.assertEqual(actual.git_dir, self.git)
with patch.object(M, "git_output", side_effect=[str(self.root), "ssh://other/repo"]):
with self.assertRaisesRegex(M.TransferError, "origin-refused"):
M.Agent(self.init_message(), self.events.append, cwd=self.root)
def test_original_source_and_excess_concurrency_refused(self):
with patch.object(M, "SOURCE", self.root):
with self.assertRaisesRegex(M.TransferError, "scientific-source-cwd"):
M.Agent(self.init_message(), self.events.append, cwd=self.root)
message = self.init_message()
message["concurrenttransfers"] = 3
with self.assertRaisesRegex(M.TransferError, "concurrency"):
M.Agent(message, self.events.append, cwd=self.root)
def test_ssh_authentication_is_strict_and_operation_scoped(self):
response = {"href": BASE, "header": {"Authorization": "Bearer " + TOKEN}}
result = subprocess.CompletedProcess([], 0, stdout=json.dumps(response).encode(), stderr=b"")
with patch.object(M.subprocess, "run", return_value=result) as call:
href, headers = self.agent.authenticate()
args = call.call_args.args[0]
self.assertIn("BatchMode=yes", args)
self.assertIn("StrictHostKeyChecking=yes", args)
self.assertEqual(args[-3:], ["git-lfs-authenticate", "qishizhe/digital-drosophila.git", "upload"])
self.assertEqual(href, BASE)
self.assertIn(TOKEN, headers["Authorization"])
self.assertNotIn(TOKEN, json.dumps(args))
def test_curl_secrets_only_in_stdin_and_secure_defaults(self):
result = subprocess.CompletedProcess([], 0, stdout=b"{}\n200", stderr=b"")
with patch.object(M.subprocess, "run", return_value=result) as call:
body = self.agent.curl("PUT", BASE + "/objects/" + self.oid + "?signature=" + TOKEN,
{"Authorization": "Bearer " + TOKEN}, upload=self.source)
self.assertEqual(body, b"{}")
self.assertEqual(call.call_args.args[0], ["curl", "-q", "--config", "-"])
self.assertNotIn(TOKEN, json.dumps(call.call_args.args[0]))
config = call.call_args.kwargs["input"].decode()
for wanted in ('noproxy = "*"', M.HOST + ":443:" + M.IP, 'proto = "=https"',
"max-redirs = 0", "Accept: application/vnd.git-lfs+json",
"Content-Type: application/octet-stream", TOKEN):
self.assertIn(wanted, config)
self.assertNotIn("insecure", config)
self.assertNotIn("location", config)
self.assertNotIn("retry", config)
def test_redirect_or_curl_error_is_sanitized(self):
for rc, output in ((0, b"redirect body\n302"), (22, b"")):
result = subprocess.CompletedProcess([], rc, stdout=output, stderr=TOKEN.encode())
with patch.object(M.subprocess, "run", return_value=result):
with self.assertRaises(M.TransferError) as caught:
self.agent.curl("GET", BASE + "/objects/" + self.oid, {})
self.assertNotIn(TOKEN, str(caught.exception))
def test_post_json_sets_both_gitea_headers(self):
result = subprocess.CompletedProcess([], 0, stdout=b"\n204", stderr=b"")
with patch.object(M.subprocess, "run", return_value=result) as call:
self.agent.curl("POST", BASE + "/verify", {}, body={"oid": self.oid, "size": self.size})
config = call.call_args.kwargs["input"].decode()
self.assertIn("Accept: application/vnd.git-lfs+json", config)
self.assertIn("Content-Type: application/vnd.git-lfs+json", config)
def test_batch_requests_basic_and_checks_exact_identity(self):
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": self.action()}}]}
with patch.object(self.agent, "authenticate", return_value=(BASE, {"Authorization": TOKEN})), \
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()) as call:
self.assertIn("upload", self.agent.batch(self.oid, self.size))
args, kwargs = call.call_args
self.assertEqual(args[1], BASE + "/objects/batch")
self.assertEqual(args[2]["Accept"], "application/vnd.git-lfs+json")
self.assertEqual(kwargs["body"]["transfers"], ["basic"])
response["objects"][0]["size"] += 1
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
with self.assertRaisesRegex(M.TransferError, "identity"):
self.agent.batch(self.oid, self.size)
def test_batch_action_host_rejected(self):
action = self.action()
action["href"] = action["href"].replace(M.HOST, "other.example")
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": action}}]}
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
with self.assertRaisesRegex(M.TransferError, "authority"):
self.agent.batch(self.oid, self.size)
def test_upload_chunked_exact_value_allowed_in_action_and_put(self):
action = self.action()
action["header"]["Transfer-Encoding"] = "chunked"
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": action}}]}
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
self.assertEqual(self.agent.batch(self.oid, self.size)["upload"]["header"]["Transfer-Encoding"], "chunked")
result = subprocess.CompletedProcess([], 0, stdout=b"\n200", stderr=b"")
with patch.object(M.subprocess, "run", return_value=result) as call:
self.agent.curl("PUT", action["href"], action["header"], upload=self.source)
config = call.call_args.kwargs["input"].decode()
self.assertIn('header = "Transfer-Encoding: chunked"', config)
self.assertIn('request = "PUT"', config)
self.assertIn("upload-file = ", config)
def test_chunked_forbidden_outside_upload_or_for_other_values(self):
for value in ("gzip", "CHUNKED", "chunked, gzip", "chunked\r\nX: y", " chunked"):
with self.subTest(value=value), self.assertRaises(M.TransferError):
self.agent.curl("PUT", BASE + "/objects/" + self.oid,
{"Transfer-Encoding": value}, upload=self.source)
for method, kwargs in (("GET", {}), ("POST", {"body": {}}), ("PUT", {})):
with self.subTest(method=method, kwargs=kwargs), self.assertRaises(M.TransferError):
self.agent.curl(method, BASE + "/objects/" + self.oid,
{"Transfer-Encoding": "chunked"}, **kwargs)
with self.assertRaises(M.TransferError):
M.safe_headers({"Transfer-Encoding": "chunked"})
for operation, name in (("upload", "verify"), ("download", "download")):
self.agent.operation = operation
action = self.action()
action["header"]["Transfer-Encoding"] = "chunked"
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {name: action}}]}
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
with self.subTest(operation=operation, action=name), self.assertRaises(M.TransferError):
self.agent.batch(self.oid, self.size)
def test_upload_existing_object_needs_no_put(self):
with patch.object(self.agent, "batch", return_value={}) as batch, patch.object(self.agent, "curl") as curl:
result = self.agent.transfer(self.request())
self.assertEqual(result, {"event": "complete", "oid": self.oid})
batch.assert_called_once()
curl.assert_not_called()
self.assertEqual(self.events[-1]["bytesSoFar"], self.size)
def test_upload_put_then_verification_post(self):
actions = {"upload": self.action(), "verify": self.action("verify/")}
with patch.object(self.agent, "batch", return_value=actions), \
patch.object(self.agent, "curl", return_value=b"") as curl:
self.agent.transfer(self.request())
self.assertEqual([c.args[0] for c in curl.call_args_list], ["PUT", "POST"])
self.assertEqual(curl.call_args_list[1].kwargs["body"], {"oid": self.oid, "size": self.size})
def test_upload_wrong_sha_fails_before_network(self):
self.source.write_bytes(b"x" * self.size)
with patch.object(self.agent, "batch") as batch:
with self.assertRaisesRegex(M.TransferError, "sha"):
self.agent.transfer(self.request())
batch.assert_not_called()
def test_upload_outside_lfs_store_refused(self):
other = self.root / self.oid
other.write_bytes(self.data)
with self.assertRaisesRegex(M.TransferError, "outside-current-lfs"):
self.agent.transfer(self.request(path=str(other)))
def test_upload_source_mutation_detected(self):
def change(*args, **kwargs):
self.source.write_bytes(b"changed")
with patch.object(self.agent, "batch", return_value={"upload": self.action()}), \
patch.object(self.agent, "curl", side_effect=change):
with self.assertRaisesRegex(M.TransferError, "upload-source-changed"):
self.agent.transfer(self.request())
def test_download_is_fresh_unique_and_verified(self):
self.agent.operation = "download"
def fetch(*args, **kwargs):
kwargs["download"].write_bytes(self.data)
return b""
with patch.object(self.agent, "batch", return_value={"download": self.action()}), \
patch.object(self.agent, "curl", side_effect=fetch) as curl:
first = self.agent.transfer(self.request("download"))
second = self.agent.transfer(self.request("download"))
self.assertNotEqual(first["path"], second["path"])
self.assertEqual(Path(first["path"]).read_bytes(), self.data)
self.assertEqual(curl.call_count, 2)
self.assertEqual(self.source.read_bytes(), self.data)
def test_download_hash_failure_retains_partial(self):
self.agent.operation = "download"
def fetch(*args, **kwargs):
kwargs["download"].write_bytes(b"x" * self.size)
with patch.object(self.agent, "batch", return_value={"download": self.action()}), \
patch.object(self.agent, "curl", side_effect=fetch):
with self.assertRaisesRegex(M.TransferError, "sha"):
self.agent.transfer(self.request("download"))
partials = list((self.git / "lfs" / "lan-transfer-tmp").glob("*.partial"))
self.assertEqual(len(partials), 1)
self.assertEqual(partials[0].read_bytes(), b"x" * self.size)
def test_protocol_init_progress_completion_terminate(self):
requests = [self.init_message(), self.request(), {"event": "terminate"}]
incoming = io.StringIO("".join(json.dumps(x) + "\n" for x in requests))
with patch.object(self.agent, "batch", return_value={}):
code = M.serve(incoming, self.events.append, factory=lambda message, send: self.agent)
self.assertEqual(code, 0)
self.assertEqual(self.events[0], {})
self.assertEqual(self.events[-1], {"event": "complete", "oid": self.oid})
def test_transfer_error_continues_and_never_echoes_exception_secret(self):
incoming = io.StringIO("\n".join(json.dumps(x) for x in
[self.init_message(), self.request(), self.request(), {"event": "terminate"}]) + "\n")
with patch.object(self.agent, "transfer", side_effect=[RuntimeError(TOKEN), {"event": "complete", "oid": self.oid}]):
code = M.serve(incoming, self.events.append, factory=lambda message, send: self.agent)
self.assertEqual(code, 0)
self.assertIn("error", self.events[1])
self.assertNotIn(TOKEN, json.dumps(self.events))
self.assertNotIn("error", self.events[2])
def test_bad_initialization_uses_init_error_schema(self):
incoming = io.StringIO(json.dumps(self.init_message()) + "\n")
with patch.object(M, "git_output", side_effect=RuntimeError(TOKEN)):
code = M.serve(incoming, self.events.append)
self.assertEqual(code, 1)
self.assertEqual(self.events[0]["error"]["code"], 32)
self.assertNotIn(TOKEN, json.dumps(self.events))
if __name__ == "__main__":
unittest.main()
tools/lfs-publication/test_verify_publication.py
+184
View File
@@ -0,0 +1,184 @@
"""Aggregate verification tests: temporary files and an identity codec only.
These test metadata/content binding, not zstd itself. The separate archive-tool
suite includes its optional real-zstd integration test. No real project data,
network, Git mutation, GPU, or persistent outputs are used here.
"""
import contextlib
import gzip
import io
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
import publish_archives as P
import verify_publication as V
def identity_codec(*args, **kwargs):
return [sys.executable, "-c",
"import sys,shutil;shutil.copyfileobj(sys.stdin.buffer,sys.stdout.buffer)"]
class AggregateTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.base = Path(self.temp.name)
self.source = self.base / "source"
self.data = self.source / "data"
self.data.mkdir(parents=True)
(self.data / "LICENSE").write_bytes(b"Synthetic license\n")
(self.data / "one.feather").write_bytes(b"fake binary bytes\x00")
self.pins = {
p.relative_to(self.source).as_posix(): dict(
path=p.relative_to(self.source).as_posix(), kind="file", **P.sha_file(p))
for p in sorted(self.data.iterdir())
}
self.plan = {
"schema": "lfs-publication-plan-v1", "source_root": str(self.source),
"groups": [{
"id": "malecns-synaptic-partners", "paths": ["data"],
"license": {"evidence_file": "LICENSE",
"evidence_sha256": self.pins["data/LICENSE"]["sha256"]},
"provenance": {"registered_file_sha256": self.pins["data/one.feather"]["sha256"]},
}],
"exclude_paths": [],
}
self.plan_file = self.base / "plan.json"
self.root, self.work = self.base / "payload", self.base / "work"
self.receipt = self.base / "receipt.json"
self.history_file = self.base / "history.jsonl.gz"
for item in (patch.object(P, "SOURCE_ROOT", str(self.source)),
patch.object(P, "CHUNK_BYTES", 4096),
patch.object(P, "zstd_command", identity_codec)):
item.start()
self.addCleanup(item.stop)
def pack(self):
self.plan_file.write_bytes(P.canonical(self.plan))
with contextlib.redirect_stdout(io.StringIO()):
P.pack(self.plan_file, self.root, self.work)
def verify(self, history=False):
with contextlib.redirect_stdout(io.StringIO()):
return V.verify_publication(self.root, self.receipt,
self.history_file if history else None)
def write_history(self, rows):
with gzip.open(self.history_file, "wt", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
def test_full_without_history_and_exclusive_receipt(self):
self.pack()
result = self.verify()
self.assertTrue(result["pass_all"])
self.assertFalse(result["fresh_download_proven"])
self.assertEqual(result["historical_status"], "NOT_REQUESTED")
self.assertEqual(result["source_files"], 2)
with self.assertRaises(ValueError):
self.verify()
def test_full_history_absolute_path_and_skipped_nonfile(self):
self.pack()
rows = [dict(row) for row in self.pins.values()]
# Use the exact registered source_root prefix on either Windows/Linux.
rows[0]["path"] = str(self.source) + "/" + rows[0]["path"]
rows.append({"path": "outside/link", "kind": "symlink"})
self.write_history(rows)
result = self.verify(history=True)
report = json.loads(self.receipt.read_bytes())
self.assertEqual(result["historical_status"], "FULL_CURRENT_MEMBER_MATCH")
self.assertEqual(report["historical_inventory"]["nonfile_records_skipped"], 1)
def test_partial_history_is_not_full_historical_pass(self):
self.pack()
self.write_history([self.pins["data/LICENSE"],
{"path": "data/one.feather", "kind": "symlink"}])
self.verify(history=True)
history = json.loads(self.receipt.read_bytes())["historical_inventory"]
self.assertEqual(history["status"], "PARTIAL_CURRENT_MEMBER_COVERAGE")
self.assertFalse(history["all_current_members_historically_matched"])
self.assertEqual(history["current_members_without_regular_history"], 1)
self.assertEqual(history["current_members_with_nonfile_history"], 1)
def test_history_mismatch_writes_no_receipt(self):
self.pack()
self.write_history([dict(self.pins["data/LICENSE"], sha256="0" * 64)])
with self.assertRaisesRegex(ValueError, "Historical member mismatch"):
self.verify(history=True)
self.assertFalse(self.receipt.exists())
def test_registered_feather_sha_mismatch(self):
self.plan["groups"][0]["provenance"]["registered_file_sha256"] = "0" * 64
self.pack()
with self.assertRaisesRegex(ValueError, "Registered feather SHA"):
self.verify()
def test_multiple_feather_members_refused(self):
(self.data / "two.feather").write_bytes(b"another fixture")
self.pack()
with self.assertRaisesRegex(ValueError, "exactly one feather"):
self.verify()
def test_license_evidence_sha_mismatch(self):
self.plan["groups"][0]["license"]["evidence_sha256"] = "0" * 64
self.pack()
with self.assertRaisesRegex(ValueError, "License evidence SHA"):
self.verify()
def test_extra_root_entry_refused(self):
self.pack()
(self.root / "unlisted").write_bytes(b"x")
with self.assertRaisesRegex(ValueError, "extra/missing"):
self.verify()
def test_wrong_catalog_pin_refused(self):
self.pack()
summary = P.json_read(self.root / "pack-summary.json")
summary["groups"][0]["catalog_sha256"] = "0" * 64
(self.root / "pack-summary.json").write_bytes(P.canonical(summary))
with self.assertRaisesRegex(ValueError, "Catalog SHA"):
self.verify()
def test_root_file_count_mismatch(self):
self.pack()
summary = P.json_read(self.root / "pack-summary.json")
summary["source_files"] = 3
(self.root / "pack-summary.json").write_bytes(P.canonical(summary))
with self.assertRaisesRegex(ValueError, "Root source file count"):
self.verify()
def test_receipt_inside_payload_refused(self):
self.pack()
self.receipt = self.root / "receipt.json"
with self.assertRaisesRegex(ValueError, "outside"):
self.verify()
def test_plan_raw_sha_mismatch(self):
self.pack()
path = self.root / "plan.json"
path.write_bytes(path.read_bytes() + b"\n")
with self.assertRaisesRegex(ValueError, "plan SHA"):
self.verify()
class GitMetadataTests(unittest.TestCase):
def test_ascii_head_with_non_ascii_stderr(self):
result = subprocess.CompletedProcess([], 0, stdout=b"a" * 40 + b"\n", stderr=bytes([255]))
with patch.object(V.subprocess, "run", return_value=result):
self.assertEqual(V.git_identity()["head"], "a" * 40)
def test_failed_git_with_non_ascii_stderr_is_optional(self):
result = subprocess.CompletedProcess([], 128, stdout=b"", stderr=bytes([255]))
with patch.object(V.subprocess, "run", return_value=result):
self.assertIsNone(V.git_identity()["head"])
if __name__ == "__main__":
unittest.main()
tools/lfs-publication/verify_publication.py
+233
View File
@@ -0,0 +1,233 @@
"""Verify a complete downloaded publication and exclusively write a new receipt.
This verifies the supplied bytes, not how they arrived or their external trust
anchor. No extraction, LFS-cache lookup, source modification, or Git mutation.
"""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import gzip
import hashlib
import json
from pathlib import Path
import subprocess
import publish_archives as archives
def require(condition, message):
if not condition:
raise ValueError(message)
def equal(left, right):
return archives.canonical(left) == archives.canonical(right)
def history_path(value, source_root):
require(isinstance(value, str), "Historical path must be text")
prefix = source_root.rstrip("/") + "/"
if value.startswith(prefix):
value = value[len(prefix):]
return archives.relpath(value)
def load_history(path, source_root):
files, nonfiles, seen = {}, {}, set()
total = 0
with gzip.open(path, "rt", encoding="utf-8") as inp:
for line in inp:
row = archives.json_bytes(line)
require(isinstance(row, dict), "Historical row must be an object")
relative = history_path(row.get("path"), source_root)
require(relative not in seen, "Duplicate historical path: " + relative)
seen.add(relative)
total += 1
if row.get("kind", "file") != "file":
nonfiles[relative] = row.get("kind")
continue
archives.check_digest_row(row)
files[relative] = {"bytes": row["bytes"], "sha256": row["sha256"]}
return files, nonfiles, total
def git_identity():
try:
process = subprocess.run(["git", "rev-parse", "--verify", "HEAD"],
capture_output=True, check=False)
head = process.stdout.decode("ascii").strip()
if process.returncode == 0 and len(head) in (40, 64) and all(x in "0123456789abcdef" for x in head):
return {"cwd": str(Path.cwd()), "head": head,
"scope": "HEAD observed at invocation cwd; not a clean-tree or payload-to-commit proof"}
except (OSError, UnicodeError):
pass
return {"cwd": str(Path.cwd()), "head": None, "scope": "Git HEAD unavailable"}
def verify_publication(payload_root, receipt, historical_inventory=None):
root, receipt = archives.no_links(payload_root), archives.no_links(receipt)
require(root.is_dir(), "Payload root is not a directory")
require(receipt != root and root not in receipt.parents,
"Receipt must be outside the immutable payload tree")
require(not receipt.exists(), "Receipt already exists; refusing overwrite/retry")
require(receipt.parent.is_dir(), "Receipt parent must already exist")
controls = {name: archives.sha_file(root / name) for name in ("plan.json", "pack-summary.json")}
plan = archives.json_read(root / "plan.json")
summary = archives.json_read(root / "pack-summary.json")
archives.validate_plan(plan)
require(summary.get("schema") == "lfs-publication-pack-v1" and summary.get("complete") is True,
"Pack summary is not complete")
require(summary.get("plan_sha256") == controls["plan.json"]["sha256"], "Root plan SHA mismatch")
plan_groups = {g["id"]: g for g in plan["groups"]}
summary_groups = summary.get("groups")
require(isinstance(summary_groups, list), "Missing summary groups")
require(all(isinstance(g, dict) and isinstance(g.get("group"), str) for g in summary_groups),
"Invalid summary group")
listed = [g["group"] for g in summary_groups]
require(listed == list(plan_groups), "Summary group inventory/order differs from plan")
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
"Payload root has extra/missing entries; preserve and investigate")
for p in root.iterdir():
archives.no_links(p)
historical = None
if historical_inventory is not None:
hpath = archives.no_links(historical_inventory)
require(hpath != receipt, "Receipt cannot replace historical inventory")
hpin = archives.sha_file(hpath)
hfiles, hnonfiles, htotal = load_history(hpath, plan["source_root"])
historical = {"path": str(hpath), "sha256": hpin["sha256"], "bytes": hpin["bytes"],
"records": htotal, "regular_file_records": len(hfiles),
"nonfile_records_skipped": len(hnonfiles)}
else:
hfiles, hnonfiles = {}, {}
matched, missing, nonfile_current = 0, [], 0
publication_paths, results, manifest_controls = set(), [], {}
for group_summary in summary_groups:
group_id = group_summary["group"]
group, directory = plan_groups[group_id], root / group_id
require(directory.is_dir(), "Missing group directory: " + group_id)
cat_path = directory / "catalog.json"
cat_pin = archives.sha_file(cat_path)
require(cat_pin["sha256"] == group_summary.get("catalog_sha256"), "Catalog SHA mismatch: " + group_id)
catalog = archives.json_read(cat_path)
require(catalog.get("group_id") == group_id and catalog.get("source_root") == plan["source_root"]
and catalog.get("plan_sha256") == controls["plan.json"]["sha256"],
"Catalog identity/plan differs: " + group_id)
for key, wanted in (("source_paths", group["paths"]), ("license", group["license"]),
("provenance", group["provenance"])):
require(equal(catalog.get(key), wanted), "Catalog plan metadata mismatch: " + group_id + "/" + key)
actual = archives.verify(directory)
expected_summary = {"group": group_id, "files": actual["source_files"],
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"],
"chunks": len(catalog["chunks"]), "catalog_sha256": cat_pin["sha256"]}
require(equal(group_summary, expected_summary), "Group summary differs from actual payload: " + group_id)
members_path = directory / "members.jsonl.gz"
manifest_controls[str(members_path)] = catalog["members"]
manifest_controls[str(cat_path)] = cat_pin
manifest_controls[str(directory / "exclusions.jsonl.gz")] = catalog["exclusions"]
feather, member_lookup, group_matched, group_missing = [], {}, 0, 0
for row in archives.iter_members(members_path):
name = row["path"]
require(name not in publication_paths, "Duplicate member across groups: " + name)
require(any(name == p or name.startswith(p + "/") for p in group["paths"]),
"Member outside group whitelist: " + name)
require(not any(name == p or name.startswith(p + "/") for p in plan.get("exclude_paths", [])),
"Excluded path appears in payload: " + name)
publication_paths.add(name)
member_lookup[name] = row
if name.lower().endswith(".feather"):
feather.append(row)
if historical is not None:
if name in hfiles:
require(equal(hfiles[name], {"bytes": row["bytes"], "sha256": row["sha256"]}),
"Historical member mismatch: " + name)
matched += 1
group_matched += 1
else:
missing.append(name)
group_missing += 1
nonfile_current += int(name in hnonfiles)
require(len(member_lookup) == actual["source_files"], "Member inventory count changed")
provenance = group["provenance"]
registered = provenance.get("registered_file_sha256") if isinstance(provenance, dict) else None
if group_id == "malecns-synaptic-partners":
require(registered is not None, "Synaptic-partners registered file SHA missing")
file_anchor = None
if registered is not None:
require(isinstance(registered, str) and archives.HEX.fullmatch(registered) is not None,
"Invalid registered file SHA")
require(len(feather) == 1, "Registered-file group must contain exactly one feather: " + group_id)
require(feather[0]["sha256"] == registered, "Registered feather SHA mismatch: " + group_id)
file_anchor = feather[0]
license_anchor = None
license_meta = group["license"]
if isinstance(license_meta, dict) and "evidence_file" in license_meta:
relative = archives.relpath(license_meta["evidence_file"])
path = group["paths"][0] + "/" + relative
require(path in member_lookup, "License evidence absent from payload: " + path)
require(member_lookup[path]["sha256"] == license_meta.get("evidence_sha256"),
"License evidence SHA mismatch: " + path)
license_anchor = member_lookup[path]
result = dict(actual, catalog_path=str(cat_path), registered_file_verified=file_anchor,
license_evidence_verified=license_anchor,
historical_matched=group_matched if historical is not None else None,
historical_missing=group_missing if historical is not None else None)
results.append(result)
print(json.dumps({"group_verified": group_id, "source_files": actual["source_files"],
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"]}), flush=True)
files = sum(r["source_files"] for r in results)
source_bytes = sum(r["source_bytes"] for r in results)
archive_bytes = sum(r["archive_bytes"] for r in results)
require(type(summary.get("source_files")) is int and summary["source_files"] == files,
"Root source file count mismatch")
require(type(summary.get("source_bytes")) is int and summary["source_bytes"] == source_bytes,
"Root source byte count mismatch")
for name, pin in controls.items():
require(archives.sha_file(root / name) == pin, "Root control changed during verification: " + name)
for path, pin in manifest_controls.items():
require(archives.sha_file(path) == pin, "Group control changed during verification: " + path)
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
"Payload root inventory changed during verification")
if historical is not None:
require(archives.sha_file(hpath) == hpin, "Historical inventory changed during verification")
historical.update(status="FULL_CURRENT_MEMBER_MATCH" if not missing else "PARTIAL_CURRENT_MEMBER_COVERAGE",
all_current_members_historically_matched=not missing,
current_members_matched=matched, current_members_without_regular_history=len(missing),
current_members_with_nonfile_history=nonfile_current,
missing_sample=missing[:20], missing_sample_truncated=len(missing) > 20,
historical_regular_files_outside_publication=len(set(hfiles) - publication_paths),
scope="Only publication members with regular-file history compared; not a full historical source-tree audit")
else:
historical = {"status": "NOT_REQUESTED", "all_current_members_historically_matched": None}
aggregate = [{"group": r["group"], "catalog_sha256": r["catalog_sha256"],
"archive_sha256": r["archive_sha256"], "source_files": r["source_files"],
"source_bytes": r["source_bytes"], "archive_bytes": r["archive_bytes"]} for r in results]
report = {"schema": "lfs-publication-aggregate-verification-v1", "pass_all": True,
"verified_utc": datetime.now(timezone.utc).isoformat(), "payload_root": str(root),
"receipt": str(receipt), "plan": {"path": str(root / "plan.json"), **controls["plan.json"]},
"pack_summary": {"path": str(root / "pack-summary.json"), **controls["pack-summary.json"]},
"groups": results, "source_files": files, "source_bytes": source_bytes,
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": hashlib.sha256(archives.canonical(aggregate)).hexdigest(),
"historical_inventory": historical, "git": git_identity(),
"fresh_download_proven": False,
"scope": "Given payload bytes verified against plan/catalog/members and optional historical overlap; provenance/plan authenticity and fresh-clone transfer evidence are caller responsibilities"}
archives.write_new(receipt, archives.canonical(report))
return {"pass_all": True, "receipt": str(receipt), "receipt_sha256": archives.sha_file(receipt)["sha256"],
"groups": len(results), "source_files": files, "source_bytes": source_bytes,
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": report["aggregate_inventory_sha256"],
"historical_status": historical["status"], "fresh_download_proven": False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--payload-root", required=True)
parser.add_argument("--receipt", required=True)
parser.add_argument("--historical-inventory")
args = parser.parse_args()
print(json.dumps(verify_publication(args.payload_root, args.receipt, args.historical_inventory),
sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
if __name__ == "__main__":
main()