Add licensed public-resource archive plan and verified LFS tooling
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,85 @@
|
||||
"""Read-only source guard; write receipt only to a new publication path."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from publish_archives import no_links, sha_file, canonical
|
||||
|
||||
ROOT = Path('/data/digital-drosophila')
|
||||
REGISTERED = 'b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740'
|
||||
SOURCE_COMMIT = '80aeb2093db655329774eff7b32fb6e417b8b40b'
|
||||
|
||||
|
||||
def git(*args, cwd=ROOT):
|
||||
return subprocess.check_output(['git', '-C', str(cwd), *args], text=True).strip()
|
||||
|
||||
|
||||
def capture(plan_path, output):
|
||||
output = no_links(output)
|
||||
if output == ROOT or ROOT in output.parents or output.exists():
|
||||
raise ValueError('Receipt must be new and outside the scientific source')
|
||||
plan = json.loads(Path(plan_path).read_bytes())
|
||||
path = ROOT / 'experiments/phase4ar-normal-conditioning/protocol/preregistered.json'
|
||||
raw = path.read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != REGISTERED:
|
||||
raise ValueError('Phase 4AR registration hash mismatch')
|
||||
registration = json.loads(raw)
|
||||
records = []
|
||||
for filename, expected in sorted(registration['source_sha256'].items()):
|
||||
source = no_links(filename)
|
||||
if ROOT not in source.parents:
|
||||
raise ValueError('Registered source outside project')
|
||||
actual = sha_file(source)
|
||||
if actual['sha256'] != expected:
|
||||
raise ValueError('Phase 4AR source mismatch: ' + filename)
|
||||
records.append(dict(path=filename, **actual))
|
||||
if len(records) != 16:
|
||||
raise ValueError('Registered source inventory differs from reviewed scope')
|
||||
if git('cat-file', '-t', SOURCE_COMMIT) != 'commit':
|
||||
raise ValueError('Registered scientific commit unavailable')
|
||||
evidence = []
|
||||
for group in plan['groups']:
|
||||
licence = group['license']
|
||||
if 'evidence_file' in licence:
|
||||
source = ROOT / group['paths'][0] / licence['evidence_file']
|
||||
actual = sha_file(source)
|
||||
if actual['sha256'] != licence['evidence_sha256']:
|
||||
raise ValueError('License evidence hash mismatch: ' + group['id'])
|
||||
evidence.append(dict(group=group['id'], path=str(source), **actual))
|
||||
provenance = group['provenance']
|
||||
source = ROOT / group['paths'][0]
|
||||
if (source / '.git').exists():
|
||||
if git('rev-parse', 'HEAD', cwd=source) != provenance['commit']:
|
||||
raise ValueError('Upstream commit mismatch: ' + group['id'])
|
||||
if git('status', '--porcelain', cwd=source):
|
||||
raise ValueError('Upstream checkout changed: ' + group['id'])
|
||||
result = dict(schema='lfs-publication-source-guard-v1',
|
||||
observed_utc=datetime.now(timezone.utc).isoformat(),
|
||||
registration_sha256=REGISTERED, registered_scientific_commit=SOURCE_COMMIT,
|
||||
phase4ar_registered_sources=records, license_evidence=evidence,
|
||||
original_checkout_head=git('rev-parse', 'HEAD'),
|
||||
original_checkout_status=git('status', '--porcelain'),
|
||||
original_checkout_branch=git('branch', '--show-current'),
|
||||
disk_free_bytes={str(p): shutil.disk_usage(p).free for p in (ROOT, output.parent)},
|
||||
pass_all=True,
|
||||
scope='16 registered Phase 4AR source files, registration, pinned upstream checkout and license evidence; not a scientific rerun or full historical checksum audit')
|
||||
if result['original_checkout_status']:
|
||||
raise ValueError('Original checkout is no longer clean; preserve and investigate')
|
||||
if any(x < 100 * 1024**3 for x in result['disk_free_bytes'].values()):
|
||||
raise ValueError('Less than 100 GiB free reserve')
|
||||
with output.open('xb') as handle:
|
||||
handle.write(canonical(result))
|
||||
return dict(pass_all=True, receipt=str(output), registered_sources=len(records),
|
||||
original_checkout_head=result['original_checkout_head'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--plan', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(capture(args.plan, args.out), sort_keys=True))
|
||||
@@ -0,0 +1,534 @@
|
||||
"""Whitelist-only LFS archives; content verification, not a redistribution grant.
|
||||
|
||||
No source writes, cache-based trust, extraction, or Git operations. Failed packs
|
||||
remain incomplete and cannot be retried into the same output/work directories.
|
||||
File stability checks are observations, not an atomic filesystem snapshot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
SOURCE_ROOT = "/data/digital-drosophila"
|
||||
CHUNK_BYTES = 64 * 1024 * 1024
|
||||
BUFFER = 1024 * 1024
|
||||
DENIED_DIRS = {".git", ".venv", "venv", "env", "__pycache__", ".cache",
|
||||
"node_modules", ".pytest_cache", ".mypy_cache", "extension-build"}
|
||||
LOG_DIRS = {"logs", "_logs"}
|
||||
BINARY_SUFFIXES = {".npy", ".npz", ".feather", ".parquet", ".gz", ".zst", ".zip",
|
||||
".tar", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf",
|
||||
".woff", ".woff2", ".ttf", ".ico", ".so", ".o", ".a", ".bin"}
|
||||
HEX = re.compile(r"^[0-9a-f]{64}$")
|
||||
SECRET_RULES = (
|
||||
("private-key", re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----")),
|
||||
("github-token", re.compile(r"\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")),
|
||||
("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
|
||||
("url-user-password", re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")),
|
||||
("secret-literal", re.compile(r'''(?ix)\b(?:password|passwd|api[_-]?key|secret[_-]?key)\b["']?\s*[:=]\s*(?:["']([^"'\r\n]+)["']|([A-Za-z0-9_+/@.-]{4,}))''')),
|
||||
)
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode()
|
||||
|
||||
|
||||
def json_bytes(data):
|
||||
def unique(pairs):
|
||||
value = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise ValueError("Duplicate JSON key")
|
||||
value[key] = item
|
||||
return value
|
||||
return json.loads(data, object_pairs_hook=unique)
|
||||
|
||||
|
||||
def json_read(path):
|
||||
return json_bytes(Path(path).read_bytes())
|
||||
|
||||
|
||||
def relpath(value):
|
||||
if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
|
||||
raise ValueError("Invalid relative path")
|
||||
p = PurePosixPath(value)
|
||||
if p.is_absolute() or any(x in ("", ".", "..") for x in value.split("/")) or ":" in value:
|
||||
raise ValueError("Unsafe relative path")
|
||||
return p.as_posix()
|
||||
|
||||
|
||||
def forbidden(relative):
|
||||
parts = PurePosixPath(relative).parts
|
||||
lower = [x.lower() for x in parts]
|
||||
if any(x in DENIED_DIRS or x.startswith(".venv") for x in lower):
|
||||
return "git-environment-or-cache"
|
||||
if any(x.startswith("phase4ar") for x in lower):
|
||||
return "active-phase4ar"
|
||||
if any(x in LOG_DIRS for x in lower) or lower[-1].endswith((".log", ".log.gz")):
|
||||
return "log"
|
||||
name = lower[-1]
|
||||
if (name == ".env" or name.startswith(".env.") or name in
|
||||
{"id_rsa", "id_ed25519", "id_dsa", ".git-credentials", ".netrc"}
|
||||
or any(x in name for x in ("credentials", "secrets"))
|
||||
or name.endswith((".pem", ".key", ".p12", ".pfx"))):
|
||||
return "credential-file"
|
||||
return None
|
||||
|
||||
|
||||
def no_links(path):
|
||||
if ".." in Path(path).parts:
|
||||
raise ValueError("Parent path components refused")
|
||||
p = Path(path).absolute()
|
||||
for part in (p, *p.parents):
|
||||
if part.exists() or part.is_symlink():
|
||||
s = part.lstat()
|
||||
if (stat.S_ISLNK(s.st_mode) or
|
||||
getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)):
|
||||
raise ValueError("Symlink refused: " + str(part))
|
||||
return p
|
||||
|
||||
|
||||
def identity(s):
|
||||
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||||
|
||||
|
||||
def opened_identity(handle, expected):
|
||||
actual = identity(os.fstat(handle.fileno()))
|
||||
# Some Windows Python builds expose birth time via stat().ctime but change
|
||||
# time via fstat().ctime. Check each clock against itself before/after; never
|
||||
# omit ctime from either endpoint's stability check. Linux uses all fields.
|
||||
same = actual[:-1] == expected[:-1] if os.name == "nt" else actual == expected
|
||||
if not same:
|
||||
raise ValueError("File changed while opening")
|
||||
return actual
|
||||
|
||||
|
||||
def sha_file(path):
|
||||
p = no_links(path)
|
||||
before = p.stat()
|
||||
if not stat.S_ISREG(before.st_mode):
|
||||
raise ValueError("Not a regular file: " + str(p))
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
with p.open("rb") as f:
|
||||
opened = opened_identity(f, identity(before))
|
||||
while block := f.read(BUFFER):
|
||||
h.update(block)
|
||||
size += len(block)
|
||||
if identity(os.fstat(f.fileno())) != opened:
|
||||
raise ValueError("File changed while hashing: " + str(p))
|
||||
if identity(p.lstat()) != identity(before) or size != before.st_size:
|
||||
raise ValueError("File changed while hashing: " + str(p))
|
||||
return {"sha256": h.hexdigest(), "bytes": size}
|
||||
|
||||
|
||||
def write_new(path, data):
|
||||
with Path(path).open("xb") as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
def validate_plan(plan):
|
||||
if not isinstance(plan, dict) or set(plan) - {"schema", "source_root", "groups", "exclude_paths"}:
|
||||
raise ValueError("Unknown plan fields")
|
||||
if plan.get("schema") != "lfs-publication-plan-v1" or plan.get("source_root") != SOURCE_ROOT:
|
||||
raise ValueError("Wrong plan schema/source_root")
|
||||
groups = plan.get("groups")
|
||||
if not isinstance(groups, list) or not groups:
|
||||
raise ValueError("Nonempty groups required")
|
||||
seen, all_paths = set(), []
|
||||
for g in groups:
|
||||
if not isinstance(g, dict) or set(g) != {"id", "paths", "license", "provenance"}:
|
||||
raise ValueError("Group requires id/paths/license/provenance only")
|
||||
group_id = g["id"]
|
||||
if not isinstance(group_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,99}", group_id) or group_id in seen:
|
||||
raise ValueError("Invalid or duplicate group id")
|
||||
seen.add(group_id)
|
||||
if not isinstance(g["paths"], list) or not g["paths"] or not g["license"] or not g["provenance"]:
|
||||
raise ValueError("Nonempty paths/license/provenance required")
|
||||
for value in g["paths"]:
|
||||
p = relpath(value)
|
||||
if forbidden(p):
|
||||
raise ValueError("Explicit forbidden path: " + p)
|
||||
if any(p == old or p.startswith(old + "/") or old.startswith(p + "/") for old in all_paths):
|
||||
raise ValueError("Overlapping whitelist paths")
|
||||
all_paths.append(p)
|
||||
excludes = plan.get("exclude_paths", [])
|
||||
if not isinstance(excludes, list):
|
||||
raise ValueError("exclude_paths must be a list")
|
||||
for p in excludes:
|
||||
relpath(p)
|
||||
canonical(plan)
|
||||
|
||||
|
||||
def enumerate_group(root, group, excludes):
|
||||
files, omitted = [], []
|
||||
|
||||
def visit(p):
|
||||
relative = p.relative_to(root).as_posix()
|
||||
s = p.lstat()
|
||||
if stat.S_ISLNK(s.st_mode):
|
||||
raise ValueError("Symlink refused: " + relative)
|
||||
reason = forbidden(relative)
|
||||
if any(relative == x or relative.startswith(x + "/") for x in excludes):
|
||||
reason = "plan-exclusion"
|
||||
if reason:
|
||||
omitted.append({"path": relative, "reason": reason,
|
||||
"scope": "subtree" if stat.S_ISDIR(s.st_mode) else "file",
|
||||
"bytes": s.st_size if stat.S_ISREG(s.st_mode) else None})
|
||||
elif stat.S_ISDIR(s.st_mode):
|
||||
for child in sorted(p.iterdir(), key=lambda x: x.name):
|
||||
visit(child)
|
||||
elif stat.S_ISREG(s.st_mode):
|
||||
files.append((relative, identity(s)))
|
||||
else:
|
||||
raise ValueError("Nonregular source refused: " + relative)
|
||||
|
||||
for relative in group["paths"]:
|
||||
path = no_links(root / relative)
|
||||
visit(path)
|
||||
files.sort(key=lambda x: x[0])
|
||||
omitted.sort(key=lambda x: x["path"])
|
||||
if not files:
|
||||
raise ValueError("Empty group refused")
|
||||
return files, omitted
|
||||
|
||||
|
||||
def scan_secrets(root, files):
|
||||
for relative, original in files:
|
||||
p = root / relative
|
||||
if original[3] > 2 * 1024 * 1024 or p.suffix.lower() in BINARY_SUFFIXES:
|
||||
continue
|
||||
data = p.read_bytes()
|
||||
if identity(p.lstat()) != original:
|
||||
raise ValueError("Source changed during secret scan: " + relative)
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if "\x00" in text:
|
||||
continue
|
||||
# Per-rule ASCII necessary fragments avoid repeated scans of numeric SWC
|
||||
# bodies (their header includes a public URL). Non-ASCII text still uses
|
||||
# every regex because Unicode IGNORECASE has broader equivalences.
|
||||
lower = text.lower()
|
||||
fragments = {
|
||||
"private-key": ("private key",),
|
||||
"github-token": ("ghp_", "github_pat_"),
|
||||
"aws-access-key": ("akia",),
|
||||
"url-user-password": ("://",),
|
||||
"secret-literal": ("password", "passwd", "api", "secret"),
|
||||
}
|
||||
for rule, pattern in SECRET_RULES:
|
||||
if text.isascii() and not any(x in lower for x in fragments[rule]):
|
||||
continue
|
||||
for match in pattern.finditer(text):
|
||||
if rule == "secret-literal":
|
||||
literal = match.group(1) or match.group(2)
|
||||
if literal.lower() in {"none", "null", "false", "true"}:
|
||||
continue
|
||||
raise ValueError("Secret scan refused: " + relative + " [" + rule + "]")
|
||||
|
||||
|
||||
def gzip_rows(path, rows):
|
||||
with Path(path).open("xb") as raw:
|
||||
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as out:
|
||||
for row in rows:
|
||||
out.write((json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode())
|
||||
|
||||
|
||||
class HashReader:
|
||||
def __init__(self, handle):
|
||||
self.handle, self.hash, self.size = handle, hashlib.sha256(), 0
|
||||
|
||||
def read(self, count=-1):
|
||||
data = self.handle.read(count)
|
||||
self.hash.update(data)
|
||||
self.size += len(data)
|
||||
return data
|
||||
|
||||
|
||||
def zstd_command(decompress=False, threads=2):
|
||||
return ["zstd", "-q", "-d", "-c"] if decompress else ["zstd", "-q", "-3", "-T" + str(threads), "-c"]
|
||||
|
||||
|
||||
def make_archive(root, files, destination, threads):
|
||||
rows = []
|
||||
with destination.open("xb") as output, tempfile.TemporaryFile() as errors:
|
||||
proc = subprocess.Popen(zstd_command(threads=threads), stdin=subprocess.PIPE,
|
||||
stdout=output, stderr=errors)
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.PAX_FORMAT) as archive:
|
||||
for relative, original in files:
|
||||
p = no_links(root / relative)
|
||||
if identity(p.lstat()) != original:
|
||||
raise ValueError("Source changed before archive: " + relative)
|
||||
with p.open("rb") as source:
|
||||
opened = opened_identity(source, original)
|
||||
reader = HashReader(source)
|
||||
info = tarfile.TarInfo(relative)
|
||||
info.size, info.mode, info.mtime = original[3], stat.S_IMODE(original[2]) & 0o777, 0
|
||||
archive.addfile(info, reader)
|
||||
if (reader.size != original[3] or identity(os.fstat(source.fileno())) != opened
|
||||
or identity(p.lstat()) != original):
|
||||
raise ValueError("Source changed during archive: " + relative)
|
||||
rows.append({"path": relative, "bytes": reader.size, "sha256": reader.hash.hexdigest()})
|
||||
proc.stdin.close()
|
||||
if proc.wait() != 0:
|
||||
raise RuntimeError("zstd compression failed; no complete catalog published")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
finally:
|
||||
proc.stdin.close()
|
||||
return rows
|
||||
|
||||
|
||||
def split_archive(archive, group_dir):
|
||||
chunks, full = [], hashlib.sha256()
|
||||
with archive.open("rb") as source:
|
||||
index = 0
|
||||
while first := source.read(min(BUFFER, CHUNK_BYTES)):
|
||||
name = "payload-%06d.bin" % index
|
||||
h, size = hashlib.sha256(), 0
|
||||
with (group_dir / name).open("xb") as out:
|
||||
block = first
|
||||
while block:
|
||||
out.write(block)
|
||||
full.update(block)
|
||||
h.update(block)
|
||||
size += len(block)
|
||||
if size == CHUNK_BYTES:
|
||||
break
|
||||
block = source.read(min(BUFFER, CHUNK_BYTES - size))
|
||||
chunks.append({"path": name, "bytes": size, "sha256": h.hexdigest(), "index": index})
|
||||
index += 1
|
||||
if not chunks:
|
||||
raise ValueError("Empty compressed archive")
|
||||
return chunks, full.hexdigest()
|
||||
|
||||
|
||||
def pack(plan_path, out, work, *, threads=2):
|
||||
if type(threads) is not int or not 1 <= threads <= 64:
|
||||
raise ValueError("threads must be 1..64")
|
||||
plan_bytes = Path(plan_path).read_bytes()
|
||||
plan = json_bytes(plan_bytes)
|
||||
validate_plan(plan)
|
||||
root, out, work = map(no_links, (Path(plan["source_root"]), Path(out), Path(work)))
|
||||
for a, b in ((root, out), (root, work), (out, work)):
|
||||
if a == b or a in b.parents or b in a.parents:
|
||||
raise ValueError("source, out and work must be disjoint trees")
|
||||
if out.exists() or work.exists():
|
||||
raise FileExistsError("out/work must both be new exclusive paths")
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
work.mkdir(parents=True, exist_ok=False)
|
||||
write_new(out / "plan.json", plan_bytes)
|
||||
plan_sha = hashlib.sha256(plan_bytes).hexdigest()
|
||||
results = []
|
||||
for group in plan["groups"]:
|
||||
files, exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
||||
scan_secrets(root, files)
|
||||
group_dir = out / group["id"]
|
||||
group_dir.mkdir()
|
||||
archive = work / (group["id"] + ".tar.zst")
|
||||
members = make_archive(root, files, archive, threads)
|
||||
# A second metadata enumeration catches file additions/removals during pack.
|
||||
after_files, after_exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
||||
if after_files != files or after_exclusions != exclusions:
|
||||
raise ValueError("Source inventory changed during pack")
|
||||
gzip_rows(group_dir / "members.jsonl.gz", members)
|
||||
gzip_rows(group_dir / "exclusions.jsonl.gz", exclusions)
|
||||
chunks, combined = split_archive(archive, group_dir)
|
||||
catalog = dict(schema="lfs-publication-group-v1", group_id=group["id"], source_root=str(root),
|
||||
source_paths=group["paths"], source_files=len(members),
|
||||
source_bytes=sum(x["bytes"] for x in members), license=group["license"],
|
||||
provenance=group["provenance"], plan_sha256=plan_sha,
|
||||
archive_format="tar.pax+zstd", archive_sha256=combined,
|
||||
archive_bytes=sum(x["bytes"] for x in chunks), chunk_bytes=CHUNK_BYTES,
|
||||
chunks=chunks, members=sha_file(group_dir / "members.jsonl.gz"),
|
||||
exclusions=sha_file(group_dir / "exclusions.jsonl.gz"),
|
||||
exclusion_records=len(exclusions),
|
||||
verification_scope="content snapshot, not a license grant or nested-archive secret audit",
|
||||
secret_scan="UTF-8 files <=2 MiB, known binary suffixes excluded; fail closed on matching rules")
|
||||
write_new(group_dir / "catalog.json", canonical(catalog))
|
||||
result = {"group": group["id"], "files": len(members), "source_bytes": catalog["source_bytes"],
|
||||
"archive_bytes": catalog["archive_bytes"], "chunks": len(chunks),
|
||||
"catalog_sha256": sha_file(group_dir / "catalog.json")["sha256"]}
|
||||
results.append(result)
|
||||
print(json.dumps({"group_complete": result}), flush=True)
|
||||
summary = {"schema": "lfs-publication-pack-v1", "complete": True, "plan_sha256": plan_sha,
|
||||
"groups": results, "source_files": sum(x["files"] for x in results),
|
||||
"source_bytes": sum(x["source_bytes"] for x in results)}
|
||||
write_new(out / "pack-summary.json", canonical(summary))
|
||||
return summary
|
||||
|
||||
|
||||
def check_digest_row(row):
|
||||
if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not HEX.fullmatch(str(row.get("sha256", ""))):
|
||||
raise ValueError("Invalid SHA/size metadata")
|
||||
|
||||
|
||||
def iter_members(path):
|
||||
previous = None
|
||||
with gzip.open(path, "rt", encoding="utf-8") as inp:
|
||||
for line in inp:
|
||||
row = json.loads(line)
|
||||
if set(row) != {"path", "bytes", "sha256"}:
|
||||
raise ValueError("Invalid member row")
|
||||
check_digest_row(row)
|
||||
p = relpath(row["path"])
|
||||
if forbidden(p) or (previous is not None and p <= previous):
|
||||
raise ValueError("Forbidden/duplicate/unsorted member: " + p)
|
||||
previous = p
|
||||
yield row
|
||||
|
||||
|
||||
def verify(group_dir):
|
||||
group_dir = no_links(group_dir)
|
||||
controls_before = {name: sha_file(group_dir / name) for name in
|
||||
("catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz")}
|
||||
catalog = json_read(group_dir / "catalog.json")
|
||||
if catalog.get("schema") != "lfs-publication-group-v1" or catalog.get("chunk_bytes") != CHUNK_BYTES or catalog.get("archive_format") != "tar.pax+zstd":
|
||||
raise ValueError("Wrong catalog schema/format/chunk size")
|
||||
chunks = catalog.get("chunks")
|
||||
if not isinstance(chunks, list) or not chunks:
|
||||
raise ValueError("Missing payload chunks")
|
||||
expected = {"catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz"}
|
||||
for i, row in enumerate(chunks):
|
||||
check_digest_row(row)
|
||||
if set(row) != {"path", "bytes", "sha256", "index"} or type(row["index"]) is not int or row["index"] != i or row["path"] != "payload-%06d.bin" % i:
|
||||
raise ValueError("Wrong chunk name/order")
|
||||
if not 0 < row["bytes"] <= CHUNK_BYTES or (i < len(chunks) - 1 and row["bytes"] != CHUNK_BYTES):
|
||||
raise ValueError("Wrong chunk size")
|
||||
expected.add(row["path"])
|
||||
if {p.name for p in group_dir.iterdir()} != expected:
|
||||
raise ValueError("Extra/missing downloaded files")
|
||||
for p in group_dir.iterdir():
|
||||
no_links(p)
|
||||
if not p.is_file():
|
||||
raise ValueError("Nonfile downloaded entry")
|
||||
for name, key in (("members.jsonl.gz", "members"), ("exclusions.jsonl.gz", "exclusions")):
|
||||
check_digest_row(catalog[key])
|
||||
if controls_before[name] != catalog[key]:
|
||||
raise ValueError("Metadata digest mismatch: " + name)
|
||||
with gzip.open(group_dir / "exclusions.jsonl.gz", "rt", encoding="utf-8") as source:
|
||||
omitted = list(map(json.loads, source))
|
||||
if len(omitted) != catalog.get("exclusion_records"):
|
||||
raise ValueError("Exclusion inventory mismatch")
|
||||
for row in omitted:
|
||||
relpath(row["path"])
|
||||
errors, combined, compressed = [], hashlib.sha256(), [0]
|
||||
with tempfile.TemporaryFile() as diagnostics:
|
||||
proc = subprocess.Popen(zstd_command(decompress=True), stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=diagnostics)
|
||||
|
||||
def feed():
|
||||
try:
|
||||
for row in chunks:
|
||||
p = group_dir / row["path"]
|
||||
before = identity(p.lstat())
|
||||
h, size = hashlib.sha256(), 0
|
||||
with p.open("rb") as source:
|
||||
opened = opened_identity(source, before)
|
||||
while data := source.read(BUFFER):
|
||||
h.update(data)
|
||||
combined.update(data)
|
||||
size += len(data)
|
||||
compressed[0] += len(data)
|
||||
proc.stdin.write(data)
|
||||
if identity(os.fstat(source.fileno())) != opened or identity(p.lstat()) != before:
|
||||
raise ValueError("Chunk changed during read")
|
||||
if size != row["bytes"] or h.hexdigest() != row["sha256"]:
|
||||
raise ValueError("Payload digest mismatch: " + row["path"])
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
finally:
|
||||
try:
|
||||
proc.stdin.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
producer = threading.Thread(target=feed, daemon=True)
|
||||
producer.start()
|
||||
count, total = 0, 0
|
||||
expected_rows = iter(iter_members(group_dir / "members.jsonl.gz"))
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
|
||||
for member in archive:
|
||||
row = next(expected_rows, None)
|
||||
if not member.isreg() or member.linkname or member.sparse is not None:
|
||||
raise ValueError("Nonregular/sparse/link tar member refused")
|
||||
path = relpath(member.name)
|
||||
if forbidden(path) or row is None or path != row["path"] or member.size != row["bytes"]:
|
||||
raise ValueError("Tar member set/size/path mismatch: " + path)
|
||||
reader = HashReader(archive.extractfile(member))
|
||||
while reader.read(BUFFER):
|
||||
pass
|
||||
if reader.size != row["bytes"] or reader.hash.hexdigest() != row["sha256"]:
|
||||
raise ValueError("Tar member content mismatch: " + path)
|
||||
count += 1
|
||||
total += reader.size
|
||||
# tarfile stops at the first end marker; reject a hidden second
|
||||
# archive/nonzero trailer, including bytes in its read-ahead buf.
|
||||
while trailer := archive.fileobj.read(BUFFER):
|
||||
if any(trailer):
|
||||
raise ValueError("Nonzero data after tar end marker")
|
||||
if next(expected_rows, None) is not None:
|
||||
raise ValueError("Missing tar members")
|
||||
# Drain to validate zstd's complete stream/checksum and finish the feeder.
|
||||
while proc.stdout.read(BUFFER):
|
||||
pass
|
||||
producer.join()
|
||||
code = proc.wait()
|
||||
if errors:
|
||||
raise ValueError("Payload streaming failed: " + str(errors[0]))
|
||||
if code != 0:
|
||||
raise ValueError("zstd decompression failed")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
producer.join(timeout=10)
|
||||
raise
|
||||
finally:
|
||||
proc.stdout.close()
|
||||
expected_rows.close()
|
||||
if (count != catalog.get("source_files") or total != catalog.get("source_bytes")
|
||||
or compressed[0] != catalog.get("archive_bytes") or combined.hexdigest() != catalog.get("archive_sha256")):
|
||||
raise ValueError("Final inventory/archive digest mismatch")
|
||||
for name, before in controls_before.items():
|
||||
if sha_file(group_dir / name) != before:
|
||||
raise ValueError("Downloaded control changed during verification: " + name)
|
||||
return {"schema": "lfs-publication-verify-v1", "pass_all": True, "group": catalog["group_id"],
|
||||
"source_files": count, "source_bytes": total, "archive_bytes": compressed[0],
|
||||
"archive_sha256": combined.hexdigest(), "catalog_sha256": controls_before["catalog.json"]["sha256"],
|
||||
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility"}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
command = sub.add_parser("pack")
|
||||
command.add_argument("--plan", required=True)
|
||||
command.add_argument("--out", required=True)
|
||||
command.add_argument("--work", required=True)
|
||||
command.add_argument("--threads", type=int, default=2)
|
||||
command = sub.add_parser("verify")
|
||||
command.add_argument("--group-dir", required=True)
|
||||
args = parser.parse_args()
|
||||
result = (pack(args.plan, args.out, args.work, threads=args.threads) if args.command == "pack"
|
||||
else verify(args.group_dir))
|
||||
print(json.dumps(result, sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Temporary local fixtures only. Identity codec tests do not certify zstd.
|
||||
|
||||
The optional real-zstd test automatically runs when the executable is present.
|
||||
"""
|
||||
import contextlib
|
||||
import gzip
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("lfs_archive_tested", Path(__file__).with_name("publish_archives.py"))
|
||||
M = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(M)
|
||||
|
||||
|
||||
def codec(*args, **kwargs):
|
||||
return [sys.executable, "-c", "import sys,shutil;shutil.copyfileobj(sys.stdin.buffer,sys.stdout.buffer)"]
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.base = Path(self.temp.name)
|
||||
self.root = self.base / "source"
|
||||
(self.root / "data").mkdir(parents=True)
|
||||
(self.root / "data" / "a.txt").write_text("public source\n", encoding="utf-8")
|
||||
(self.root / "data" / "b.bin").write_bytes(bytes(range(256)) * 80)
|
||||
(self.root / "data" / "zero.txt").write_bytes(b"")
|
||||
self.plan = {"schema": "lfs-publication-plan-v1", "source_root": str(self.root),
|
||||
"groups": [{"id": "data", "paths": ["data"], "license": "test-only",
|
||||
"provenance": {"source": "synthetic"}}], "exclude_paths": []}
|
||||
self.plan_path = self.base / "plan.json"
|
||||
self.plan_path.write_bytes(M.canonical(self.plan))
|
||||
self.out, self.work = self.base / "out", self.base / "work"
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.object(M, "SOURCE_ROOT", str(self.root)).start()
|
||||
patch.object(M, "CHUNK_BYTES", 4096).start()
|
||||
self.codec_patch = patch.object(M, "zstd_command", codec)
|
||||
self.codec_patch.start()
|
||||
|
||||
def pack(self):
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return M.pack(self.plan_path, self.out, self.work)
|
||||
|
||||
def catalog(self):
|
||||
return json.loads((self.out / "data" / "catalog.json").read_bytes())
|
||||
|
||||
def save_catalog(self, cat):
|
||||
(self.out / "data" / "catalog.json").write_bytes(M.canonical(cat))
|
||||
|
||||
def rewrite_archive(self, data):
|
||||
group = self.out / "data"
|
||||
cat = self.catalog()
|
||||
for row in cat["chunks"]:
|
||||
(group / row["path"]).unlink()
|
||||
new = self.base / "mutant.tar"
|
||||
new.write_bytes(data)
|
||||
cat["chunks"], cat["archive_sha256"] = M.split_archive(new, group)
|
||||
cat["archive_bytes"] = len(data)
|
||||
self.save_catalog(cat)
|
||||
|
||||
def test_pack_verify_full_member_content_and_no_extraction(self):
|
||||
before = {p.relative_to(self.root).as_posix(): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
|
||||
summary = self.pack()
|
||||
result = M.verify(self.out / "data")
|
||||
self.assertTrue(result["pass_all"])
|
||||
self.assertEqual(result["source_files"], 3)
|
||||
self.assertEqual(result["source_bytes"], sum(map(len, before.values())))
|
||||
self.assertGreater(summary["groups"][0]["chunks"], 1)
|
||||
self.assertEqual(before, {p.relative_to(self.root).as_posix(): p.read_bytes() for p in self.root.rglob("*") if p.is_file()})
|
||||
self.assertFalse((self.out / "data" / "a.txt").exists())
|
||||
|
||||
def test_plan_executes_first_read_bytes_only(self):
|
||||
with patch.object(M, "json_read", side_effect=AssertionError("Must not read plan a second time")):
|
||||
self.pack()
|
||||
self.assertEqual(self.catalog()["plan_sha256"], hashlib.sha256(self.plan_path.read_bytes()).hexdigest())
|
||||
|
||||
def test_exclusions_logged_without_descending(self):
|
||||
for name in (".git", "venv", "__pycache__", ".cache", "node_modules", "logs", "phase4ar-normal-conditioning"):
|
||||
d = self.root / "data" / name
|
||||
d.mkdir()
|
||||
(d / "hidden.txt").write_text("skip")
|
||||
(self.root / "data" / "private.pem").write_text("private")
|
||||
(self.root / "data" / "run.log").write_text("log")
|
||||
self.pack()
|
||||
with gzip.open(self.out / "data" / "exclusions.jsonl.gz", "rt") as f:
|
||||
rows = [json.loads(x) for x in f]
|
||||
self.assertEqual(len(rows), 9)
|
||||
self.assertEqual(self.catalog()["source_files"], 3)
|
||||
self.assertTrue(M.verify(self.out / "data")["pass_all"])
|
||||
|
||||
def test_unsafe_explicit_paths(self):
|
||||
for name in ("../data", "/data", "data/../data", "data//x", "data\\x", "data/.git", "data/phase4ar-normal-conditioning"):
|
||||
plan = json.loads(json.dumps(self.plan))
|
||||
plan["groups"][0]["paths"] = [name]
|
||||
with self.subTest(name=name), self.assertRaises(ValueError):
|
||||
M.validate_plan(plan)
|
||||
|
||||
def test_overlapping_groups_refused(self):
|
||||
self.plan["groups"].append({"id": "nested", "paths": ["data/a.txt"], "license": "test", "provenance": "test"})
|
||||
with self.assertRaisesRegex(ValueError, "Overlapping"):
|
||||
M.validate_plan(self.plan)
|
||||
|
||||
def test_symlink_refused(self):
|
||||
try:
|
||||
(self.root / "data" / "link").symlink_to(self.root / "data" / "a.txt")
|
||||
except OSError:
|
||||
self.skipTest("OS does not permit temporary symlinks")
|
||||
with self.assertRaisesRegex(ValueError, "Symlink"):
|
||||
self.pack()
|
||||
|
||||
def test_exclusive_and_disjoint_outputs(self):
|
||||
self.out.mkdir()
|
||||
with self.assertRaises(FileExistsError):
|
||||
self.pack()
|
||||
with self.assertRaisesRegex(ValueError, "disjoint"):
|
||||
M.pack(self.plan_path, self.root / "output", self.work)
|
||||
|
||||
def test_parent_components_cannot_bypass_disjoint_gate(self):
|
||||
malicious = self.base / "unused" / ".." / "source" / "new-output"
|
||||
with self.assertRaisesRegex(ValueError, "Parent path components"):
|
||||
M.pack(self.plan_path, malicious, self.work)
|
||||
self.assertFalse((self.root / "new-output").exists())
|
||||
|
||||
def test_secret_rules_hide_values(self):
|
||||
# Build synthetic fixtures at runtime; keep actual credential-like
|
||||
# strings out of Git blobs so the repository secret guard stays useful.
|
||||
examples = ["-----BEGIN " + "OPENSSH PRIVATE " + "KEY-----", "ghp_" + "a" * 36,
|
||||
"github_pat_" + "b" * 40, "AKIA" + "A" * 16,
|
||||
"https://" + "account:" + "privatevalue" + "@example.org/",
|
||||
'password' + ' = "privatevalue"',
|
||||
'api_key' + ': "privatevalue"', 'secret_key' + '=ABCDEF123456',
|
||||
'pa\u017f\u017fword' + '="privatevalue"',
|
||||
'ap\u0131_key' + '="privatevalue"',
|
||||
'\u017fecret_key' + '="privatevalue"']
|
||||
path = self.root / "data" / "literal.txt"
|
||||
for value in examples:
|
||||
path.write_text(value)
|
||||
files, _ = M.enumerate_group(self.root, self.plan["groups"][0], [])
|
||||
with self.subTest(rule=value[:8]), self.assertRaisesRegex(ValueError, "Secret scan refused") as caught:
|
||||
M.scan_secrets(self.root, files)
|
||||
self.assertNotIn(value, str(caught.exception))
|
||||
self.assertNotIn("privatevalue", str(caught.exception))
|
||||
|
||||
def test_mutated_source_metadata_refused(self):
|
||||
files, _ = M.enumerate_group(self.root, self.plan["groups"][0], [])
|
||||
(self.root / "data" / "a.txt").write_text("changed bytes")
|
||||
with self.assertRaisesRegex(ValueError, "Source changed"):
|
||||
M.make_archive(self.root, files, self.base / "bad.tar", 1)
|
||||
|
||||
def test_payload_corruption_refused(self):
|
||||
self.pack()
|
||||
p = self.out / "data" / self.catalog()["chunks"][0]["path"]
|
||||
data = bytearray(p.read_bytes())
|
||||
data[-1] ^= 1
|
||||
p.write_bytes(data)
|
||||
with self.assertRaises((ValueError, tarfile.TarError)):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_missing_extra_or_reordered_payload_refused(self):
|
||||
self.pack()
|
||||
extra = self.out / "data" / "extra.bin"
|
||||
extra.write_bytes(b"x")
|
||||
with self.assertRaisesRegex(ValueError, "Extra/missing"):
|
||||
M.verify(self.out / "data")
|
||||
extra.unlink()
|
||||
cat = self.catalog()
|
||||
cat["chunks"] = list(reversed(cat["chunks"]))
|
||||
self.save_catalog(cat)
|
||||
with self.assertRaisesRegex(ValueError, "chunk name/order"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_tar_traversal_rejected_even_with_rehashed_chunks(self):
|
||||
self.pack()
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as archive:
|
||||
info = tarfile.TarInfo("../escape")
|
||||
info.size = 1
|
||||
archive.addfile(info, io.BytesIO(b"x"))
|
||||
self.rewrite_archive(buf.getvalue())
|
||||
with self.assertRaisesRegex(ValueError, "Unsafe relative path"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_tar_link_rejected_even_with_rehashed_chunks(self):
|
||||
self.pack()
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as archive:
|
||||
info = tarfile.TarInfo("data/a.txt")
|
||||
info.type, info.linkname = tarfile.SYMTYPE, "/tmp/escape"
|
||||
archive.addfile(info)
|
||||
self.rewrite_archive(buf.getvalue())
|
||||
with self.assertRaisesRegex(ValueError, "Nonregular"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_hidden_tar_after_end_refused(self):
|
||||
self.pack()
|
||||
raw = (self.work / "data.tar.zst").read_bytes()
|
||||
self.rewrite_archive(raw + b"nonzero hidden second archive")
|
||||
with self.assertRaisesRegex(ValueError, "Nonzero data after tar"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_second_tar_after_end_refused(self):
|
||||
self.pack()
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as archive:
|
||||
info = tarfile.TarInfo("unlisted-secret.txt")
|
||||
info.size = 1
|
||||
archive.addfile(info, io.BytesIO(b"x"))
|
||||
self.rewrite_archive((self.work / "data.tar.zst").read_bytes() + buf.getvalue())
|
||||
with self.assertRaisesRegex(ValueError, "Nonzero data after tar"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_member_hash_mutation_refused(self):
|
||||
self.pack()
|
||||
p = self.out / "data" / "members.jsonl.gz"
|
||||
with gzip.open(p, "rt") as f:
|
||||
rows = [json.loads(x) for x in f]
|
||||
rows[0]["sha256"] = "0" * 64
|
||||
p.unlink()
|
||||
M.gzip_rows(p, rows)
|
||||
cat = self.catalog()
|
||||
cat["members"] = M.sha_file(p)
|
||||
self.save_catalog(cat)
|
||||
with self.assertRaisesRegex(ValueError, "member content mismatch"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
@unittest.skipUnless(shutil.which("zstd"), "Real zstd unavailable; run this integration test on Linux")
|
||||
def test_real_zstd_roundtrip(self):
|
||||
self.codec_patch.stop()
|
||||
self.pack()
|
||||
self.assertTrue(M.verify(self.out / "data")["pass_all"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user