Add licensed public-resource archive plan and verified LFS tooling
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,534 @@
|
||||
"""Whitelist-only LFS archives; content verification, not a redistribution grant.
|
||||
|
||||
No source writes, cache-based trust, extraction, or Git operations. Failed packs
|
||||
remain incomplete and cannot be retried into the same output/work directories.
|
||||
File stability checks are observations, not an atomic filesystem snapshot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
SOURCE_ROOT = "/data/digital-drosophila"
|
||||
CHUNK_BYTES = 64 * 1024 * 1024
|
||||
BUFFER = 1024 * 1024
|
||||
DENIED_DIRS = {".git", ".venv", "venv", "env", "__pycache__", ".cache",
|
||||
"node_modules", ".pytest_cache", ".mypy_cache", "extension-build"}
|
||||
LOG_DIRS = {"logs", "_logs"}
|
||||
BINARY_SUFFIXES = {".npy", ".npz", ".feather", ".parquet", ".gz", ".zst", ".zip",
|
||||
".tar", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf",
|
||||
".woff", ".woff2", ".ttf", ".ico", ".so", ".o", ".a", ".bin"}
|
||||
HEX = re.compile(r"^[0-9a-f]{64}$")
|
||||
SECRET_RULES = (
|
||||
("private-key", re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----")),
|
||||
("github-token", re.compile(r"\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")),
|
||||
("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
|
||||
("url-user-password", re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")),
|
||||
("secret-literal", re.compile(r'''(?ix)\b(?:password|passwd|api[_-]?key|secret[_-]?key)\b["']?\s*[:=]\s*(?:["']([^"'\r\n]+)["']|([A-Za-z0-9_+/@.-]{4,}))''')),
|
||||
)
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode()
|
||||
|
||||
|
||||
def json_bytes(data):
|
||||
def unique(pairs):
|
||||
value = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise ValueError("Duplicate JSON key")
|
||||
value[key] = item
|
||||
return value
|
||||
return json.loads(data, object_pairs_hook=unique)
|
||||
|
||||
|
||||
def json_read(path):
|
||||
return json_bytes(Path(path).read_bytes())
|
||||
|
||||
|
||||
def relpath(value):
|
||||
if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
|
||||
raise ValueError("Invalid relative path")
|
||||
p = PurePosixPath(value)
|
||||
if p.is_absolute() or any(x in ("", ".", "..") for x in value.split("/")) or ":" in value:
|
||||
raise ValueError("Unsafe relative path")
|
||||
return p.as_posix()
|
||||
|
||||
|
||||
def forbidden(relative):
|
||||
parts = PurePosixPath(relative).parts
|
||||
lower = [x.lower() for x in parts]
|
||||
if any(x in DENIED_DIRS or x.startswith(".venv") for x in lower):
|
||||
return "git-environment-or-cache"
|
||||
if any(x.startswith("phase4ar") for x in lower):
|
||||
return "active-phase4ar"
|
||||
if any(x in LOG_DIRS for x in lower) or lower[-1].endswith((".log", ".log.gz")):
|
||||
return "log"
|
||||
name = lower[-1]
|
||||
if (name == ".env" or name.startswith(".env.") or name in
|
||||
{"id_rsa", "id_ed25519", "id_dsa", ".git-credentials", ".netrc"}
|
||||
or any(x in name for x in ("credentials", "secrets"))
|
||||
or name.endswith((".pem", ".key", ".p12", ".pfx"))):
|
||||
return "credential-file"
|
||||
return None
|
||||
|
||||
|
||||
def no_links(path):
|
||||
if ".." in Path(path).parts:
|
||||
raise ValueError("Parent path components refused")
|
||||
p = Path(path).absolute()
|
||||
for part in (p, *p.parents):
|
||||
if part.exists() or part.is_symlink():
|
||||
s = part.lstat()
|
||||
if (stat.S_ISLNK(s.st_mode) or
|
||||
getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)):
|
||||
raise ValueError("Symlink refused: " + str(part))
|
||||
return p
|
||||
|
||||
|
||||
def identity(s):
|
||||
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||||
|
||||
|
||||
def opened_identity(handle, expected):
|
||||
actual = identity(os.fstat(handle.fileno()))
|
||||
# Some Windows Python builds expose birth time via stat().ctime but change
|
||||
# time via fstat().ctime. Check each clock against itself before/after; never
|
||||
# omit ctime from either endpoint's stability check. Linux uses all fields.
|
||||
same = actual[:-1] == expected[:-1] if os.name == "nt" else actual == expected
|
||||
if not same:
|
||||
raise ValueError("File changed while opening")
|
||||
return actual
|
||||
|
||||
|
||||
def sha_file(path):
|
||||
p = no_links(path)
|
||||
before = p.stat()
|
||||
if not stat.S_ISREG(before.st_mode):
|
||||
raise ValueError("Not a regular file: " + str(p))
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
with p.open("rb") as f:
|
||||
opened = opened_identity(f, identity(before))
|
||||
while block := f.read(BUFFER):
|
||||
h.update(block)
|
||||
size += len(block)
|
||||
if identity(os.fstat(f.fileno())) != opened:
|
||||
raise ValueError("File changed while hashing: " + str(p))
|
||||
if identity(p.lstat()) != identity(before) or size != before.st_size:
|
||||
raise ValueError("File changed while hashing: " + str(p))
|
||||
return {"sha256": h.hexdigest(), "bytes": size}
|
||||
|
||||
|
||||
def write_new(path, data):
|
||||
with Path(path).open("xb") as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
def validate_plan(plan):
|
||||
if not isinstance(plan, dict) or set(plan) - {"schema", "source_root", "groups", "exclude_paths"}:
|
||||
raise ValueError("Unknown plan fields")
|
||||
if plan.get("schema") != "lfs-publication-plan-v1" or plan.get("source_root") != SOURCE_ROOT:
|
||||
raise ValueError("Wrong plan schema/source_root")
|
||||
groups = plan.get("groups")
|
||||
if not isinstance(groups, list) or not groups:
|
||||
raise ValueError("Nonempty groups required")
|
||||
seen, all_paths = set(), []
|
||||
for g in groups:
|
||||
if not isinstance(g, dict) or set(g) != {"id", "paths", "license", "provenance"}:
|
||||
raise ValueError("Group requires id/paths/license/provenance only")
|
||||
group_id = g["id"]
|
||||
if not isinstance(group_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,99}", group_id) or group_id in seen:
|
||||
raise ValueError("Invalid or duplicate group id")
|
||||
seen.add(group_id)
|
||||
if not isinstance(g["paths"], list) or not g["paths"] or not g["license"] or not g["provenance"]:
|
||||
raise ValueError("Nonempty paths/license/provenance required")
|
||||
for value in g["paths"]:
|
||||
p = relpath(value)
|
||||
if forbidden(p):
|
||||
raise ValueError("Explicit forbidden path: " + p)
|
||||
if any(p == old or p.startswith(old + "/") or old.startswith(p + "/") for old in all_paths):
|
||||
raise ValueError("Overlapping whitelist paths")
|
||||
all_paths.append(p)
|
||||
excludes = plan.get("exclude_paths", [])
|
||||
if not isinstance(excludes, list):
|
||||
raise ValueError("exclude_paths must be a list")
|
||||
for p in excludes:
|
||||
relpath(p)
|
||||
canonical(plan)
|
||||
|
||||
|
||||
def enumerate_group(root, group, excludes):
|
||||
files, omitted = [], []
|
||||
|
||||
def visit(p):
|
||||
relative = p.relative_to(root).as_posix()
|
||||
s = p.lstat()
|
||||
if stat.S_ISLNK(s.st_mode):
|
||||
raise ValueError("Symlink refused: " + relative)
|
||||
reason = forbidden(relative)
|
||||
if any(relative == x or relative.startswith(x + "/") for x in excludes):
|
||||
reason = "plan-exclusion"
|
||||
if reason:
|
||||
omitted.append({"path": relative, "reason": reason,
|
||||
"scope": "subtree" if stat.S_ISDIR(s.st_mode) else "file",
|
||||
"bytes": s.st_size if stat.S_ISREG(s.st_mode) else None})
|
||||
elif stat.S_ISDIR(s.st_mode):
|
||||
for child in sorted(p.iterdir(), key=lambda x: x.name):
|
||||
visit(child)
|
||||
elif stat.S_ISREG(s.st_mode):
|
||||
files.append((relative, identity(s)))
|
||||
else:
|
||||
raise ValueError("Nonregular source refused: " + relative)
|
||||
|
||||
for relative in group["paths"]:
|
||||
path = no_links(root / relative)
|
||||
visit(path)
|
||||
files.sort(key=lambda x: x[0])
|
||||
omitted.sort(key=lambda x: x["path"])
|
||||
if not files:
|
||||
raise ValueError("Empty group refused")
|
||||
return files, omitted
|
||||
|
||||
|
||||
def scan_secrets(root, files):
|
||||
for relative, original in files:
|
||||
p = root / relative
|
||||
if original[3] > 2 * 1024 * 1024 or p.suffix.lower() in BINARY_SUFFIXES:
|
||||
continue
|
||||
data = p.read_bytes()
|
||||
if identity(p.lstat()) != original:
|
||||
raise ValueError("Source changed during secret scan: " + relative)
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if "\x00" in text:
|
||||
continue
|
||||
# Per-rule ASCII necessary fragments avoid repeated scans of numeric SWC
|
||||
# bodies (their header includes a public URL). Non-ASCII text still uses
|
||||
# every regex because Unicode IGNORECASE has broader equivalences.
|
||||
lower = text.lower()
|
||||
fragments = {
|
||||
"private-key": ("private key",),
|
||||
"github-token": ("ghp_", "github_pat_"),
|
||||
"aws-access-key": ("akia",),
|
||||
"url-user-password": ("://",),
|
||||
"secret-literal": ("password", "passwd", "api", "secret"),
|
||||
}
|
||||
for rule, pattern in SECRET_RULES:
|
||||
if text.isascii() and not any(x in lower for x in fragments[rule]):
|
||||
continue
|
||||
for match in pattern.finditer(text):
|
||||
if rule == "secret-literal":
|
||||
literal = match.group(1) or match.group(2)
|
||||
if literal.lower() in {"none", "null", "false", "true"}:
|
||||
continue
|
||||
raise ValueError("Secret scan refused: " + relative + " [" + rule + "]")
|
||||
|
||||
|
||||
def gzip_rows(path, rows):
|
||||
with Path(path).open("xb") as raw:
|
||||
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as out:
|
||||
for row in rows:
|
||||
out.write((json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode())
|
||||
|
||||
|
||||
class HashReader:
|
||||
def __init__(self, handle):
|
||||
self.handle, self.hash, self.size = handle, hashlib.sha256(), 0
|
||||
|
||||
def read(self, count=-1):
|
||||
data = self.handle.read(count)
|
||||
self.hash.update(data)
|
||||
self.size += len(data)
|
||||
return data
|
||||
|
||||
|
||||
def zstd_command(decompress=False, threads=2):
|
||||
return ["zstd", "-q", "-d", "-c"] if decompress else ["zstd", "-q", "-3", "-T" + str(threads), "-c"]
|
||||
|
||||
|
||||
def make_archive(root, files, destination, threads):
|
||||
rows = []
|
||||
with destination.open("xb") as output, tempfile.TemporaryFile() as errors:
|
||||
proc = subprocess.Popen(zstd_command(threads=threads), stdin=subprocess.PIPE,
|
||||
stdout=output, stderr=errors)
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.PAX_FORMAT) as archive:
|
||||
for relative, original in files:
|
||||
p = no_links(root / relative)
|
||||
if identity(p.lstat()) != original:
|
||||
raise ValueError("Source changed before archive: " + relative)
|
||||
with p.open("rb") as source:
|
||||
opened = opened_identity(source, original)
|
||||
reader = HashReader(source)
|
||||
info = tarfile.TarInfo(relative)
|
||||
info.size, info.mode, info.mtime = original[3], stat.S_IMODE(original[2]) & 0o777, 0
|
||||
archive.addfile(info, reader)
|
||||
if (reader.size != original[3] or identity(os.fstat(source.fileno())) != opened
|
||||
or identity(p.lstat()) != original):
|
||||
raise ValueError("Source changed during archive: " + relative)
|
||||
rows.append({"path": relative, "bytes": reader.size, "sha256": reader.hash.hexdigest()})
|
||||
proc.stdin.close()
|
||||
if proc.wait() != 0:
|
||||
raise RuntimeError("zstd compression failed; no complete catalog published")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
finally:
|
||||
proc.stdin.close()
|
||||
return rows
|
||||
|
||||
|
||||
def split_archive(archive, group_dir):
|
||||
chunks, full = [], hashlib.sha256()
|
||||
with archive.open("rb") as source:
|
||||
index = 0
|
||||
while first := source.read(min(BUFFER, CHUNK_BYTES)):
|
||||
name = "payload-%06d.bin" % index
|
||||
h, size = hashlib.sha256(), 0
|
||||
with (group_dir / name).open("xb") as out:
|
||||
block = first
|
||||
while block:
|
||||
out.write(block)
|
||||
full.update(block)
|
||||
h.update(block)
|
||||
size += len(block)
|
||||
if size == CHUNK_BYTES:
|
||||
break
|
||||
block = source.read(min(BUFFER, CHUNK_BYTES - size))
|
||||
chunks.append({"path": name, "bytes": size, "sha256": h.hexdigest(), "index": index})
|
||||
index += 1
|
||||
if not chunks:
|
||||
raise ValueError("Empty compressed archive")
|
||||
return chunks, full.hexdigest()
|
||||
|
||||
|
||||
def pack(plan_path, out, work, *, threads=2):
|
||||
if type(threads) is not int or not 1 <= threads <= 64:
|
||||
raise ValueError("threads must be 1..64")
|
||||
plan_bytes = Path(plan_path).read_bytes()
|
||||
plan = json_bytes(plan_bytes)
|
||||
validate_plan(plan)
|
||||
root, out, work = map(no_links, (Path(plan["source_root"]), Path(out), Path(work)))
|
||||
for a, b in ((root, out), (root, work), (out, work)):
|
||||
if a == b or a in b.parents or b in a.parents:
|
||||
raise ValueError("source, out and work must be disjoint trees")
|
||||
if out.exists() or work.exists():
|
||||
raise FileExistsError("out/work must both be new exclusive paths")
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
work.mkdir(parents=True, exist_ok=False)
|
||||
write_new(out / "plan.json", plan_bytes)
|
||||
plan_sha = hashlib.sha256(plan_bytes).hexdigest()
|
||||
results = []
|
||||
for group in plan["groups"]:
|
||||
files, exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
||||
scan_secrets(root, files)
|
||||
group_dir = out / group["id"]
|
||||
group_dir.mkdir()
|
||||
archive = work / (group["id"] + ".tar.zst")
|
||||
members = make_archive(root, files, archive, threads)
|
||||
# A second metadata enumeration catches file additions/removals during pack.
|
||||
after_files, after_exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
||||
if after_files != files or after_exclusions != exclusions:
|
||||
raise ValueError("Source inventory changed during pack")
|
||||
gzip_rows(group_dir / "members.jsonl.gz", members)
|
||||
gzip_rows(group_dir / "exclusions.jsonl.gz", exclusions)
|
||||
chunks, combined = split_archive(archive, group_dir)
|
||||
catalog = dict(schema="lfs-publication-group-v1", group_id=group["id"], source_root=str(root),
|
||||
source_paths=group["paths"], source_files=len(members),
|
||||
source_bytes=sum(x["bytes"] for x in members), license=group["license"],
|
||||
provenance=group["provenance"], plan_sha256=plan_sha,
|
||||
archive_format="tar.pax+zstd", archive_sha256=combined,
|
||||
archive_bytes=sum(x["bytes"] for x in chunks), chunk_bytes=CHUNK_BYTES,
|
||||
chunks=chunks, members=sha_file(group_dir / "members.jsonl.gz"),
|
||||
exclusions=sha_file(group_dir / "exclusions.jsonl.gz"),
|
||||
exclusion_records=len(exclusions),
|
||||
verification_scope="content snapshot, not a license grant or nested-archive secret audit",
|
||||
secret_scan="UTF-8 files <=2 MiB, known binary suffixes excluded; fail closed on matching rules")
|
||||
write_new(group_dir / "catalog.json", canonical(catalog))
|
||||
result = {"group": group["id"], "files": len(members), "source_bytes": catalog["source_bytes"],
|
||||
"archive_bytes": catalog["archive_bytes"], "chunks": len(chunks),
|
||||
"catalog_sha256": sha_file(group_dir / "catalog.json")["sha256"]}
|
||||
results.append(result)
|
||||
print(json.dumps({"group_complete": result}), flush=True)
|
||||
summary = {"schema": "lfs-publication-pack-v1", "complete": True, "plan_sha256": plan_sha,
|
||||
"groups": results, "source_files": sum(x["files"] for x in results),
|
||||
"source_bytes": sum(x["source_bytes"] for x in results)}
|
||||
write_new(out / "pack-summary.json", canonical(summary))
|
||||
return summary
|
||||
|
||||
|
||||
def check_digest_row(row):
|
||||
if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not HEX.fullmatch(str(row.get("sha256", ""))):
|
||||
raise ValueError("Invalid SHA/size metadata")
|
||||
|
||||
|
||||
def iter_members(path):
|
||||
previous = None
|
||||
with gzip.open(path, "rt", encoding="utf-8") as inp:
|
||||
for line in inp:
|
||||
row = json.loads(line)
|
||||
if set(row) != {"path", "bytes", "sha256"}:
|
||||
raise ValueError("Invalid member row")
|
||||
check_digest_row(row)
|
||||
p = relpath(row["path"])
|
||||
if forbidden(p) or (previous is not None and p <= previous):
|
||||
raise ValueError("Forbidden/duplicate/unsorted member: " + p)
|
||||
previous = p
|
||||
yield row
|
||||
|
||||
|
||||
def verify(group_dir):
|
||||
group_dir = no_links(group_dir)
|
||||
controls_before = {name: sha_file(group_dir / name) for name in
|
||||
("catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz")}
|
||||
catalog = json_read(group_dir / "catalog.json")
|
||||
if catalog.get("schema") != "lfs-publication-group-v1" or catalog.get("chunk_bytes") != CHUNK_BYTES or catalog.get("archive_format") != "tar.pax+zstd":
|
||||
raise ValueError("Wrong catalog schema/format/chunk size")
|
||||
chunks = catalog.get("chunks")
|
||||
if not isinstance(chunks, list) or not chunks:
|
||||
raise ValueError("Missing payload chunks")
|
||||
expected = {"catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz"}
|
||||
for i, row in enumerate(chunks):
|
||||
check_digest_row(row)
|
||||
if set(row) != {"path", "bytes", "sha256", "index"} or type(row["index"]) is not int or row["index"] != i or row["path"] != "payload-%06d.bin" % i:
|
||||
raise ValueError("Wrong chunk name/order")
|
||||
if not 0 < row["bytes"] <= CHUNK_BYTES or (i < len(chunks) - 1 and row["bytes"] != CHUNK_BYTES):
|
||||
raise ValueError("Wrong chunk size")
|
||||
expected.add(row["path"])
|
||||
if {p.name for p in group_dir.iterdir()} != expected:
|
||||
raise ValueError("Extra/missing downloaded files")
|
||||
for p in group_dir.iterdir():
|
||||
no_links(p)
|
||||
if not p.is_file():
|
||||
raise ValueError("Nonfile downloaded entry")
|
||||
for name, key in (("members.jsonl.gz", "members"), ("exclusions.jsonl.gz", "exclusions")):
|
||||
check_digest_row(catalog[key])
|
||||
if controls_before[name] != catalog[key]:
|
||||
raise ValueError("Metadata digest mismatch: " + name)
|
||||
with gzip.open(group_dir / "exclusions.jsonl.gz", "rt", encoding="utf-8") as source:
|
||||
omitted = list(map(json.loads, source))
|
||||
if len(omitted) != catalog.get("exclusion_records"):
|
||||
raise ValueError("Exclusion inventory mismatch")
|
||||
for row in omitted:
|
||||
relpath(row["path"])
|
||||
errors, combined, compressed = [], hashlib.sha256(), [0]
|
||||
with tempfile.TemporaryFile() as diagnostics:
|
||||
proc = subprocess.Popen(zstd_command(decompress=True), stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=diagnostics)
|
||||
|
||||
def feed():
|
||||
try:
|
||||
for row in chunks:
|
||||
p = group_dir / row["path"]
|
||||
before = identity(p.lstat())
|
||||
h, size = hashlib.sha256(), 0
|
||||
with p.open("rb") as source:
|
||||
opened = opened_identity(source, before)
|
||||
while data := source.read(BUFFER):
|
||||
h.update(data)
|
||||
combined.update(data)
|
||||
size += len(data)
|
||||
compressed[0] += len(data)
|
||||
proc.stdin.write(data)
|
||||
if identity(os.fstat(source.fileno())) != opened or identity(p.lstat()) != before:
|
||||
raise ValueError("Chunk changed during read")
|
||||
if size != row["bytes"] or h.hexdigest() != row["sha256"]:
|
||||
raise ValueError("Payload digest mismatch: " + row["path"])
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
finally:
|
||||
try:
|
||||
proc.stdin.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
producer = threading.Thread(target=feed, daemon=True)
|
||||
producer.start()
|
||||
count, total = 0, 0
|
||||
expected_rows = iter(iter_members(group_dir / "members.jsonl.gz"))
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
|
||||
for member in archive:
|
||||
row = next(expected_rows, None)
|
||||
if not member.isreg() or member.linkname or member.sparse is not None:
|
||||
raise ValueError("Nonregular/sparse/link tar member refused")
|
||||
path = relpath(member.name)
|
||||
if forbidden(path) or row is None or path != row["path"] or member.size != row["bytes"]:
|
||||
raise ValueError("Tar member set/size/path mismatch: " + path)
|
||||
reader = HashReader(archive.extractfile(member))
|
||||
while reader.read(BUFFER):
|
||||
pass
|
||||
if reader.size != row["bytes"] or reader.hash.hexdigest() != row["sha256"]:
|
||||
raise ValueError("Tar member content mismatch: " + path)
|
||||
count += 1
|
||||
total += reader.size
|
||||
# tarfile stops at the first end marker; reject a hidden second
|
||||
# archive/nonzero trailer, including bytes in its read-ahead buf.
|
||||
while trailer := archive.fileobj.read(BUFFER):
|
||||
if any(trailer):
|
||||
raise ValueError("Nonzero data after tar end marker")
|
||||
if next(expected_rows, None) is not None:
|
||||
raise ValueError("Missing tar members")
|
||||
# Drain to validate zstd's complete stream/checksum and finish the feeder.
|
||||
while proc.stdout.read(BUFFER):
|
||||
pass
|
||||
producer.join()
|
||||
code = proc.wait()
|
||||
if errors:
|
||||
raise ValueError("Payload streaming failed: " + str(errors[0]))
|
||||
if code != 0:
|
||||
raise ValueError("zstd decompression failed")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
producer.join(timeout=10)
|
||||
raise
|
||||
finally:
|
||||
proc.stdout.close()
|
||||
expected_rows.close()
|
||||
if (count != catalog.get("source_files") or total != catalog.get("source_bytes")
|
||||
or compressed[0] != catalog.get("archive_bytes") or combined.hexdigest() != catalog.get("archive_sha256")):
|
||||
raise ValueError("Final inventory/archive digest mismatch")
|
||||
for name, before in controls_before.items():
|
||||
if sha_file(group_dir / name) != before:
|
||||
raise ValueError("Downloaded control changed during verification: " + name)
|
||||
return {"schema": "lfs-publication-verify-v1", "pass_all": True, "group": catalog["group_id"],
|
||||
"source_files": count, "source_bytes": total, "archive_bytes": compressed[0],
|
||||
"archive_sha256": combined.hexdigest(), "catalog_sha256": controls_before["catalog.json"]["sha256"],
|
||||
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility"}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
command = sub.add_parser("pack")
|
||||
command.add_argument("--plan", required=True)
|
||||
command.add_argument("--out", required=True)
|
||||
command.add_argument("--work", required=True)
|
||||
command.add_argument("--threads", type=int, default=2)
|
||||
command = sub.add_parser("verify")
|
||||
command.add_argument("--group-dir", required=True)
|
||||
args = parser.parse_args()
|
||||
result = (pack(args.plan, args.out, args.work, threads=args.threads) if args.command == "pack"
|
||||
else verify(args.group_dir))
|
||||
print(json.dumps(result, sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user