Digital Drosophila Archive
535 lines
24 KiB
Python
535 lines
24 KiB
Python
"""Whitelist-only LFS archives; content verification, not a redistribution grant.
|
|
|
|
No source writes, cache-based trust, extraction, or Git operations. Failed packs
|
|
remain incomplete and cannot be retried into the same output/work directories.
|
|
File stability checks are observations, not an atomic filesystem snapshot.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import threading
|
|
|
|
SOURCE_ROOT = "/data/digital-drosophila"
|
|
CHUNK_BYTES = 64 * 1024 * 1024
|
|
BUFFER = 1024 * 1024
|
|
DENIED_DIRS = {".git", ".venv", "venv", "env", "__pycache__", ".cache",
|
|
"node_modules", ".pytest_cache", ".mypy_cache", "extension-build"}
|
|
LOG_DIRS = {"logs", "_logs"}
|
|
BINARY_SUFFIXES = {".npy", ".npz", ".feather", ".parquet", ".gz", ".zst", ".zip",
|
|
".tar", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf",
|
|
".woff", ".woff2", ".ttf", ".ico", ".so", ".o", ".a", ".bin"}
|
|
HEX = re.compile(r"^[0-9a-f]{64}$")
|
|
SECRET_RULES = (
|
|
("private-key", re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----")),
|
|
("github-token", re.compile(r"\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")),
|
|
("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
|
|
("url-user-password", re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")),
|
|
("secret-literal", re.compile(r'''(?ix)\b(?:password|passwd|api[_-]?key|secret[_-]?key)\b["']?\s*[:=]\s*(?:["']([^"'\r\n]+)["']|([A-Za-z0-9_+/@.-]{4,}))''')),
|
|
)
|
|
|
|
|
|
def canonical(value):
|
|
return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode()
|
|
|
|
|
|
def json_bytes(data):
|
|
def unique(pairs):
|
|
value = {}
|
|
for key, item in pairs:
|
|
if key in value:
|
|
raise ValueError("Duplicate JSON key")
|
|
value[key] = item
|
|
return value
|
|
return json.loads(data, object_pairs_hook=unique)
|
|
|
|
|
|
def json_read(path):
|
|
return json_bytes(Path(path).read_bytes())
|
|
|
|
|
|
def relpath(value):
|
|
if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
|
|
raise ValueError("Invalid relative path")
|
|
p = PurePosixPath(value)
|
|
if p.is_absolute() or any(x in ("", ".", "..") for x in value.split("/")) or ":" in value:
|
|
raise ValueError("Unsafe relative path")
|
|
return p.as_posix()
|
|
|
|
|
|
def forbidden(relative):
|
|
parts = PurePosixPath(relative).parts
|
|
lower = [x.lower() for x in parts]
|
|
if any(x in DENIED_DIRS or x.startswith(".venv") for x in lower):
|
|
return "git-environment-or-cache"
|
|
if any(x.startswith("phase4ar") for x in lower):
|
|
return "active-phase4ar"
|
|
if any(x in LOG_DIRS for x in lower) or lower[-1].endswith((".log", ".log.gz")):
|
|
return "log"
|
|
name = lower[-1]
|
|
if (name == ".env" or name.startswith(".env.") or name in
|
|
{"id_rsa", "id_ed25519", "id_dsa", ".git-credentials", ".netrc"}
|
|
or any(x in name for x in ("credentials", "secrets"))
|
|
or name.endswith((".pem", ".key", ".p12", ".pfx"))):
|
|
return "credential-file"
|
|
return None
|
|
|
|
|
|
def no_links(path):
|
|
if ".." in Path(path).parts:
|
|
raise ValueError("Parent path components refused")
|
|
p = Path(path).absolute()
|
|
for part in (p, *p.parents):
|
|
if part.exists() or part.is_symlink():
|
|
s = part.lstat()
|
|
if (stat.S_ISLNK(s.st_mode) or
|
|
getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)):
|
|
raise ValueError("Symlink refused: " + str(part))
|
|
return p
|
|
|
|
|
|
def identity(s):
|
|
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
|
|
|
|
|
def opened_identity(handle, expected):
|
|
actual = identity(os.fstat(handle.fileno()))
|
|
# Some Windows Python builds expose birth time via stat().ctime but change
|
|
# time via fstat().ctime. Check each clock against itself before/after; never
|
|
# omit ctime from either endpoint's stability check. Linux uses all fields.
|
|
same = actual[:-1] == expected[:-1] if os.name == "nt" else actual == expected
|
|
if not same:
|
|
raise ValueError("File changed while opening")
|
|
return actual
|
|
|
|
|
|
def sha_file(path):
|
|
p = no_links(path)
|
|
before = p.stat()
|
|
if not stat.S_ISREG(before.st_mode):
|
|
raise ValueError("Not a regular file: " + str(p))
|
|
h = hashlib.sha256()
|
|
size = 0
|
|
with p.open("rb") as f:
|
|
opened = opened_identity(f, identity(before))
|
|
while block := f.read(BUFFER):
|
|
h.update(block)
|
|
size += len(block)
|
|
if identity(os.fstat(f.fileno())) != opened:
|
|
raise ValueError("File changed while hashing: " + str(p))
|
|
if identity(p.lstat()) != identity(before) or size != before.st_size:
|
|
raise ValueError("File changed while hashing: " + str(p))
|
|
return {"sha256": h.hexdigest(), "bytes": size}
|
|
|
|
|
|
def write_new(path, data):
|
|
with Path(path).open("xb") as f:
|
|
f.write(data)
|
|
|
|
|
|
def validate_plan(plan):
|
|
if not isinstance(plan, dict) or set(plan) - {"schema", "source_root", "groups", "exclude_paths"}:
|
|
raise ValueError("Unknown plan fields")
|
|
if plan.get("schema") != "lfs-publication-plan-v1" or plan.get("source_root") != SOURCE_ROOT:
|
|
raise ValueError("Wrong plan schema/source_root")
|
|
groups = plan.get("groups")
|
|
if not isinstance(groups, list) or not groups:
|
|
raise ValueError("Nonempty groups required")
|
|
seen, all_paths = set(), []
|
|
for g in groups:
|
|
if not isinstance(g, dict) or set(g) != {"id", "paths", "license", "provenance"}:
|
|
raise ValueError("Group requires id/paths/license/provenance only")
|
|
group_id = g["id"]
|
|
if not isinstance(group_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,99}", group_id) or group_id in seen:
|
|
raise ValueError("Invalid or duplicate group id")
|
|
seen.add(group_id)
|
|
if not isinstance(g["paths"], list) or not g["paths"] or not g["license"] or not g["provenance"]:
|
|
raise ValueError("Nonempty paths/license/provenance required")
|
|
for value in g["paths"]:
|
|
p = relpath(value)
|
|
if forbidden(p):
|
|
raise ValueError("Explicit forbidden path: " + p)
|
|
if any(p == old or p.startswith(old + "/") or old.startswith(p + "/") for old in all_paths):
|
|
raise ValueError("Overlapping whitelist paths")
|
|
all_paths.append(p)
|
|
excludes = plan.get("exclude_paths", [])
|
|
if not isinstance(excludes, list):
|
|
raise ValueError("exclude_paths must be a list")
|
|
for p in excludes:
|
|
relpath(p)
|
|
canonical(plan)
|
|
|
|
|
|
def enumerate_group(root, group, excludes):
|
|
files, omitted = [], []
|
|
|
|
def visit(p):
|
|
relative = p.relative_to(root).as_posix()
|
|
s = p.lstat()
|
|
if stat.S_ISLNK(s.st_mode):
|
|
raise ValueError("Symlink refused: " + relative)
|
|
reason = forbidden(relative)
|
|
if any(relative == x or relative.startswith(x + "/") for x in excludes):
|
|
reason = "plan-exclusion"
|
|
if reason:
|
|
omitted.append({"path": relative, "reason": reason,
|
|
"scope": "subtree" if stat.S_ISDIR(s.st_mode) else "file",
|
|
"bytes": s.st_size if stat.S_ISREG(s.st_mode) else None})
|
|
elif stat.S_ISDIR(s.st_mode):
|
|
for child in sorted(p.iterdir(), key=lambda x: x.name):
|
|
visit(child)
|
|
elif stat.S_ISREG(s.st_mode):
|
|
files.append((relative, identity(s)))
|
|
else:
|
|
raise ValueError("Nonregular source refused: " + relative)
|
|
|
|
for relative in group["paths"]:
|
|
path = no_links(root / relative)
|
|
visit(path)
|
|
files.sort(key=lambda x: x[0])
|
|
omitted.sort(key=lambda x: x["path"])
|
|
if not files:
|
|
raise ValueError("Empty group refused")
|
|
return files, omitted
|
|
|
|
|
|
def scan_secrets(root, files):
|
|
for relative, original in files:
|
|
p = root / relative
|
|
if original[3] > 2 * 1024 * 1024 or p.suffix.lower() in BINARY_SUFFIXES:
|
|
continue
|
|
data = p.read_bytes()
|
|
if identity(p.lstat()) != original:
|
|
raise ValueError("Source changed during secret scan: " + relative)
|
|
try:
|
|
text = data.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
continue
|
|
if "\x00" in text:
|
|
continue
|
|
# Per-rule ASCII necessary fragments avoid repeated scans of numeric SWC
|
|
# bodies (their header includes a public URL). Non-ASCII text still uses
|
|
# every regex because Unicode IGNORECASE has broader equivalences.
|
|
lower = text.lower()
|
|
fragments = {
|
|
"private-key": ("private key",),
|
|
"github-token": ("ghp_", "github_pat_"),
|
|
"aws-access-key": ("akia",),
|
|
"url-user-password": ("://",),
|
|
"secret-literal": ("password", "passwd", "api", "secret"),
|
|
}
|
|
for rule, pattern in SECRET_RULES:
|
|
if text.isascii() and not any(x in lower for x in fragments[rule]):
|
|
continue
|
|
for match in pattern.finditer(text):
|
|
if rule == "secret-literal":
|
|
literal = match.group(1) or match.group(2)
|
|
if literal.lower() in {"none", "null", "false", "true"}:
|
|
continue
|
|
raise ValueError("Secret scan refused: " + relative + " [" + rule + "]")
|
|
|
|
|
|
def gzip_rows(path, rows):
|
|
with Path(path).open("xb") as raw:
|
|
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as out:
|
|
for row in rows:
|
|
out.write((json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode())
|
|
|
|
|
|
class HashReader:
|
|
def __init__(self, handle):
|
|
self.handle, self.hash, self.size = handle, hashlib.sha256(), 0
|
|
|
|
def read(self, count=-1):
|
|
data = self.handle.read(count)
|
|
self.hash.update(data)
|
|
self.size += len(data)
|
|
return data
|
|
|
|
|
|
def zstd_command(decompress=False, threads=2):
|
|
return ["zstd", "-q", "-d", "-c"] if decompress else ["zstd", "-q", "-3", "-T" + str(threads), "-c"]
|
|
|
|
|
|
def make_archive(root, files, destination, threads):
|
|
rows = []
|
|
with destination.open("xb") as output, tempfile.TemporaryFile() as errors:
|
|
proc = subprocess.Popen(zstd_command(threads=threads), stdin=subprocess.PIPE,
|
|
stdout=output, stderr=errors)
|
|
try:
|
|
with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.PAX_FORMAT) as archive:
|
|
for relative, original in files:
|
|
p = no_links(root / relative)
|
|
if identity(p.lstat()) != original:
|
|
raise ValueError("Source changed before archive: " + relative)
|
|
with p.open("rb") as source:
|
|
opened = opened_identity(source, original)
|
|
reader = HashReader(source)
|
|
info = tarfile.TarInfo(relative)
|
|
info.size, info.mode, info.mtime = original[3], stat.S_IMODE(original[2]) & 0o777, 0
|
|
archive.addfile(info, reader)
|
|
if (reader.size != original[3] or identity(os.fstat(source.fileno())) != opened
|
|
or identity(p.lstat()) != original):
|
|
raise ValueError("Source changed during archive: " + relative)
|
|
rows.append({"path": relative, "bytes": reader.size, "sha256": reader.hash.hexdigest()})
|
|
proc.stdin.close()
|
|
if proc.wait() != 0:
|
|
raise RuntimeError("zstd compression failed; no complete catalog published")
|
|
except BaseException:
|
|
proc.kill()
|
|
proc.wait()
|
|
raise
|
|
finally:
|
|
proc.stdin.close()
|
|
return rows
|
|
|
|
|
|
def split_archive(archive, group_dir):
|
|
chunks, full = [], hashlib.sha256()
|
|
with archive.open("rb") as source:
|
|
index = 0
|
|
while first := source.read(min(BUFFER, CHUNK_BYTES)):
|
|
name = "payload-%06d.bin" % index
|
|
h, size = hashlib.sha256(), 0
|
|
with (group_dir / name).open("xb") as out:
|
|
block = first
|
|
while block:
|
|
out.write(block)
|
|
full.update(block)
|
|
h.update(block)
|
|
size += len(block)
|
|
if size == CHUNK_BYTES:
|
|
break
|
|
block = source.read(min(BUFFER, CHUNK_BYTES - size))
|
|
chunks.append({"path": name, "bytes": size, "sha256": h.hexdigest(), "index": index})
|
|
index += 1
|
|
if not chunks:
|
|
raise ValueError("Empty compressed archive")
|
|
return chunks, full.hexdigest()
|
|
|
|
|
|
def pack(plan_path, out, work, *, threads=2):
|
|
if type(threads) is not int or not 1 <= threads <= 64:
|
|
raise ValueError("threads must be 1..64")
|
|
plan_bytes = Path(plan_path).read_bytes()
|
|
plan = json_bytes(plan_bytes)
|
|
validate_plan(plan)
|
|
root, out, work = map(no_links, (Path(plan["source_root"]), Path(out), Path(work)))
|
|
for a, b in ((root, out), (root, work), (out, work)):
|
|
if a == b or a in b.parents or b in a.parents:
|
|
raise ValueError("source, out and work must be disjoint trees")
|
|
if out.exists() or work.exists():
|
|
raise FileExistsError("out/work must both be new exclusive paths")
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
work.mkdir(parents=True, exist_ok=False)
|
|
write_new(out / "plan.json", plan_bytes)
|
|
plan_sha = hashlib.sha256(plan_bytes).hexdigest()
|
|
results = []
|
|
for group in plan["groups"]:
|
|
files, exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
|
scan_secrets(root, files)
|
|
group_dir = out / group["id"]
|
|
group_dir.mkdir()
|
|
archive = work / (group["id"] + ".tar.zst")
|
|
members = make_archive(root, files, archive, threads)
|
|
# A second metadata enumeration catches file additions/removals during pack.
|
|
after_files, after_exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
|
if after_files != files or after_exclusions != exclusions:
|
|
raise ValueError("Source inventory changed during pack")
|
|
gzip_rows(group_dir / "members.jsonl.gz", members)
|
|
gzip_rows(group_dir / "exclusions.jsonl.gz", exclusions)
|
|
chunks, combined = split_archive(archive, group_dir)
|
|
catalog = dict(schema="lfs-publication-group-v1", group_id=group["id"], source_root=str(root),
|
|
source_paths=group["paths"], source_files=len(members),
|
|
source_bytes=sum(x["bytes"] for x in members), license=group["license"],
|
|
provenance=group["provenance"], plan_sha256=plan_sha,
|
|
archive_format="tar.pax+zstd", archive_sha256=combined,
|
|
archive_bytes=sum(x["bytes"] for x in chunks), chunk_bytes=CHUNK_BYTES,
|
|
chunks=chunks, members=sha_file(group_dir / "members.jsonl.gz"),
|
|
exclusions=sha_file(group_dir / "exclusions.jsonl.gz"),
|
|
exclusion_records=len(exclusions),
|
|
verification_scope="content snapshot, not a license grant or nested-archive secret audit",
|
|
secret_scan="UTF-8 files <=2 MiB, known binary suffixes excluded; fail closed on matching rules")
|
|
write_new(group_dir / "catalog.json", canonical(catalog))
|
|
result = {"group": group["id"], "files": len(members), "source_bytes": catalog["source_bytes"],
|
|
"archive_bytes": catalog["archive_bytes"], "chunks": len(chunks),
|
|
"catalog_sha256": sha_file(group_dir / "catalog.json")["sha256"]}
|
|
results.append(result)
|
|
print(json.dumps({"group_complete": result}), flush=True)
|
|
summary = {"schema": "lfs-publication-pack-v1", "complete": True, "plan_sha256": plan_sha,
|
|
"groups": results, "source_files": sum(x["files"] for x in results),
|
|
"source_bytes": sum(x["source_bytes"] for x in results)}
|
|
write_new(out / "pack-summary.json", canonical(summary))
|
|
return summary
|
|
|
|
|
|
def check_digest_row(row):
|
|
if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not HEX.fullmatch(str(row.get("sha256", ""))):
|
|
raise ValueError("Invalid SHA/size metadata")
|
|
|
|
|
|
def iter_members(path):
|
|
previous = None
|
|
with gzip.open(path, "rt", encoding="utf-8") as inp:
|
|
for line in inp:
|
|
row = json.loads(line)
|
|
if set(row) != {"path", "bytes", "sha256"}:
|
|
raise ValueError("Invalid member row")
|
|
check_digest_row(row)
|
|
p = relpath(row["path"])
|
|
if forbidden(p) or (previous is not None and p <= previous):
|
|
raise ValueError("Forbidden/duplicate/unsorted member: " + p)
|
|
previous = p
|
|
yield row
|
|
|
|
|
|
def verify(group_dir):
|
|
group_dir = no_links(group_dir)
|
|
controls_before = {name: sha_file(group_dir / name) for name in
|
|
("catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz")}
|
|
catalog = json_read(group_dir / "catalog.json")
|
|
if catalog.get("schema") != "lfs-publication-group-v1" or catalog.get("chunk_bytes") != CHUNK_BYTES or catalog.get("archive_format") != "tar.pax+zstd":
|
|
raise ValueError("Wrong catalog schema/format/chunk size")
|
|
chunks = catalog.get("chunks")
|
|
if not isinstance(chunks, list) or not chunks:
|
|
raise ValueError("Missing payload chunks")
|
|
expected = {"catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz"}
|
|
for i, row in enumerate(chunks):
|
|
check_digest_row(row)
|
|
if set(row) != {"path", "bytes", "sha256", "index"} or type(row["index"]) is not int or row["index"] != i or row["path"] != "payload-%06d.bin" % i:
|
|
raise ValueError("Wrong chunk name/order")
|
|
if not 0 < row["bytes"] <= CHUNK_BYTES or (i < len(chunks) - 1 and row["bytes"] != CHUNK_BYTES):
|
|
raise ValueError("Wrong chunk size")
|
|
expected.add(row["path"])
|
|
if {p.name for p in group_dir.iterdir()} != expected:
|
|
raise ValueError("Extra/missing downloaded files")
|
|
for p in group_dir.iterdir():
|
|
no_links(p)
|
|
if not p.is_file():
|
|
raise ValueError("Nonfile downloaded entry")
|
|
for name, key in (("members.jsonl.gz", "members"), ("exclusions.jsonl.gz", "exclusions")):
|
|
check_digest_row(catalog[key])
|
|
if controls_before[name] != catalog[key]:
|
|
raise ValueError("Metadata digest mismatch: " + name)
|
|
with gzip.open(group_dir / "exclusions.jsonl.gz", "rt", encoding="utf-8") as source:
|
|
omitted = list(map(json.loads, source))
|
|
if len(omitted) != catalog.get("exclusion_records"):
|
|
raise ValueError("Exclusion inventory mismatch")
|
|
for row in omitted:
|
|
relpath(row["path"])
|
|
errors, combined, compressed = [], hashlib.sha256(), [0]
|
|
with tempfile.TemporaryFile() as diagnostics:
|
|
proc = subprocess.Popen(zstd_command(decompress=True), stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE, stderr=diagnostics)
|
|
|
|
def feed():
|
|
try:
|
|
for row in chunks:
|
|
p = group_dir / row["path"]
|
|
before = identity(p.lstat())
|
|
h, size = hashlib.sha256(), 0
|
|
with p.open("rb") as source:
|
|
opened = opened_identity(source, before)
|
|
while data := source.read(BUFFER):
|
|
h.update(data)
|
|
combined.update(data)
|
|
size += len(data)
|
|
compressed[0] += len(data)
|
|
proc.stdin.write(data)
|
|
if identity(os.fstat(source.fileno())) != opened or identity(p.lstat()) != before:
|
|
raise ValueError("Chunk changed during read")
|
|
if size != row["bytes"] or h.hexdigest() != row["sha256"]:
|
|
raise ValueError("Payload digest mismatch: " + row["path"])
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
finally:
|
|
try:
|
|
proc.stdin.close()
|
|
except OSError:
|
|
pass
|
|
|
|
producer = threading.Thread(target=feed, daemon=True)
|
|
producer.start()
|
|
count, total = 0, 0
|
|
expected_rows = iter(iter_members(group_dir / "members.jsonl.gz"))
|
|
try:
|
|
with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
|
|
for member in archive:
|
|
row = next(expected_rows, None)
|
|
if not member.isreg() or member.linkname or member.sparse is not None:
|
|
raise ValueError("Nonregular/sparse/link tar member refused")
|
|
path = relpath(member.name)
|
|
if forbidden(path) or row is None or path != row["path"] or member.size != row["bytes"]:
|
|
raise ValueError("Tar member set/size/path mismatch: " + path)
|
|
reader = HashReader(archive.extractfile(member))
|
|
while reader.read(BUFFER):
|
|
pass
|
|
if reader.size != row["bytes"] or reader.hash.hexdigest() != row["sha256"]:
|
|
raise ValueError("Tar member content mismatch: " + path)
|
|
count += 1
|
|
total += reader.size
|
|
# tarfile stops at the first end marker; reject a hidden second
|
|
# archive/nonzero trailer, including bytes in its read-ahead buf.
|
|
while trailer := archive.fileobj.read(BUFFER):
|
|
if any(trailer):
|
|
raise ValueError("Nonzero data after tar end marker")
|
|
if next(expected_rows, None) is not None:
|
|
raise ValueError("Missing tar members")
|
|
# Drain to validate zstd's complete stream/checksum and finish the feeder.
|
|
while proc.stdout.read(BUFFER):
|
|
pass
|
|
producer.join()
|
|
code = proc.wait()
|
|
if errors:
|
|
raise ValueError("Payload streaming failed: " + str(errors[0]))
|
|
if code != 0:
|
|
raise ValueError("zstd decompression failed")
|
|
except BaseException:
|
|
proc.kill()
|
|
proc.wait()
|
|
producer.join(timeout=10)
|
|
raise
|
|
finally:
|
|
proc.stdout.close()
|
|
expected_rows.close()
|
|
if (count != catalog.get("source_files") or total != catalog.get("source_bytes")
|
|
or compressed[0] != catalog.get("archive_bytes") or combined.hexdigest() != catalog.get("archive_sha256")):
|
|
raise ValueError("Final inventory/archive digest mismatch")
|
|
for name, before in controls_before.items():
|
|
if sha_file(group_dir / name) != before:
|
|
raise ValueError("Downloaded control changed during verification: " + name)
|
|
return {"schema": "lfs-publication-verify-v1", "pass_all": True, "group": catalog["group_id"],
|
|
"source_files": count, "source_bytes": total, "archive_bytes": compressed[0],
|
|
"archive_sha256": combined.hexdigest(), "catalog_sha256": controls_before["catalog.json"]["sha256"],
|
|
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility"}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
command = sub.add_parser("pack")
|
|
command.add_argument("--plan", required=True)
|
|
command.add_argument("--out", required=True)
|
|
command.add_argument("--work", required=True)
|
|
command.add_argument("--threads", type=int, default=2)
|
|
command = sub.add_parser("verify")
|
|
command.add_argument("--group-dir", required=True)
|
|
args = parser.parse_args()
|
|
result = (pack(args.plan, args.out, args.work, threads=args.threads) if args.command == "pack"
|
|
else verify(args.group_dir))
|
|
print(json.dumps(result, sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|