"""Whitelist-only LFS archives; content verification, not a redistribution grant.

No source writes, cache-based trust, extraction, or Git operations. Failed packs
remain incomplete and cannot be retried into the same output/work directories.
File stability checks are observations, not an atomic filesystem snapshot.
"""
from __future__ import annotations

import argparse
import gzip
import hashlib
import io
import json
import os
from pathlib import Path, PurePosixPath
import re
import stat
import subprocess
import tarfile
import tempfile
import threading

SOURCE_ROOT = "/data/digital-drosophila"
CHUNK_BYTES = 64 * 1024 * 1024
BUFFER = 1024 * 1024
DENIED_DIRS = {".git", ".venv", "venv", "env", "__pycache__", ".cache",
               "node_modules", ".pytest_cache", ".mypy_cache", "extension-build"}
LOG_DIRS = {"logs", "_logs"}
BINARY_SUFFIXES = {".npy", ".npz", ".feather", ".parquet", ".gz", ".zst", ".zip",
                   ".tar", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf",
                   ".woff", ".woff2", ".ttf", ".ico", ".so", ".o", ".a", ".bin"}
HEX = re.compile(r"^[0-9a-f]{64}$")
SECRET_RULES = (
    ("private-key", re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----")),
    ("github-token", re.compile(r"\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")),
    ("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
    ("url-user-password", re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")),
    ("secret-literal", re.compile(r'''(?ix)\b(?:password|passwd|api[_-]?key|secret[_-]?key)\b["']?\s*[:=]\s*(?:["']([^"'\r\n]+)["']|([A-Za-z0-9_+/@.-]{4,}))''')),
)


def canonical(value):
    return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode()


def json_bytes(data):
    def unique(pairs):
        value = {}
        for key, item in pairs:
            if key in value:
                raise ValueError("Duplicate JSON key")
            value[key] = item
        return value
    return json.loads(data, object_pairs_hook=unique)


def json_read(path):
    return json_bytes(Path(path).read_bytes())


def relpath(value):
    if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
        raise ValueError("Invalid relative path")
    p = PurePosixPath(value)
    if p.is_absolute() or any(x in ("", ".", "..") for x in value.split("/")) or ":" in value:
        raise ValueError("Unsafe relative path")
    return p.as_posix()


def forbidden(relative):
    parts = PurePosixPath(relative).parts
    lower = [x.lower() for x in parts]
    if any(x in DENIED_DIRS or x.startswith(".venv") for x in lower):
        return "git-environment-or-cache"
    if any(x.startswith("phase4ar") for x in lower):
        return "active-phase4ar"
    if any(x in LOG_DIRS for x in lower) or lower[-1].endswith((".log", ".log.gz")):
        return "log"
    name = lower[-1]
    if (name == ".env" or name.startswith(".env.") or name in
            {"id_rsa", "id_ed25519", "id_dsa", ".git-credentials", ".netrc"}
            or any(x in name for x in ("credentials", "secrets"))
            or name.endswith((".pem", ".key", ".p12", ".pfx"))):
        return "credential-file"
    return None


def no_links(path):
    if ".." in Path(path).parts:
        raise ValueError("Parent path components refused")
    p = Path(path).absolute()
    for part in (p, *p.parents):
        if part.exists() or part.is_symlink():
            s = part.lstat()
            if (stat.S_ISLNK(s.st_mode) or
                    getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)):
                raise ValueError("Symlink refused: " + str(part))
    return p


def identity(s):
    return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)


def opened_identity(handle, expected):
    actual = identity(os.fstat(handle.fileno()))
    # Some Windows Python builds expose birth time via stat().ctime but change
    # time via fstat().ctime. Check each clock against itself before/after; never
    # omit ctime from either endpoint's stability check. Linux uses all fields.
    same = actual[:-1] == expected[:-1] if os.name == "nt" else actual == expected
    if not same:
        raise ValueError("File changed while opening")
    return actual


def sha_file(path):
    p = no_links(path)
    before = p.stat()
    if not stat.S_ISREG(before.st_mode):
        raise ValueError("Not a regular file: " + str(p))
    h = hashlib.sha256()
    size = 0
    with p.open("rb") as f:
        opened = opened_identity(f, identity(before))
        while block := f.read(BUFFER):
            h.update(block)
            size += len(block)
        if identity(os.fstat(f.fileno())) != opened:
            raise ValueError("File changed while hashing: " + str(p))
    if identity(p.lstat()) != identity(before) or size != before.st_size:
        raise ValueError("File changed while hashing: " + str(p))
    return {"sha256": h.hexdigest(), "bytes": size}


def write_new(path, data):
    with Path(path).open("xb") as f:
        f.write(data)


def validate_plan(plan):
    if not isinstance(plan, dict) or set(plan) - {"schema", "source_root", "groups", "exclude_paths"}:
        raise ValueError("Unknown plan fields")
    if plan.get("schema") != "lfs-publication-plan-v1" or plan.get("source_root") != SOURCE_ROOT:
        raise ValueError("Wrong plan schema/source_root")
    groups = plan.get("groups")
    if not isinstance(groups, list) or not groups:
        raise ValueError("Nonempty groups required")
    seen, all_paths = set(), []
    for g in groups:
        if not isinstance(g, dict) or set(g) != {"id", "paths", "license", "provenance"}:
            raise ValueError("Group requires id/paths/license/provenance only")
        group_id = g["id"]
        if not isinstance(group_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,99}", group_id) or group_id in seen:
            raise ValueError("Invalid or duplicate group id")
        seen.add(group_id)
        if not isinstance(g["paths"], list) or not g["paths"] or not g["license"] or not g["provenance"]:
            raise ValueError("Nonempty paths/license/provenance required")
        for value in g["paths"]:
            p = relpath(value)
            if forbidden(p):
                raise ValueError("Explicit forbidden path: " + p)
            if any(p == old or p.startswith(old + "/") or old.startswith(p + "/") for old in all_paths):
                raise ValueError("Overlapping whitelist paths")
            all_paths.append(p)
    excludes = plan.get("exclude_paths", [])
    if not isinstance(excludes, list):
        raise ValueError("exclude_paths must be a list")
    for p in excludes:
        relpath(p)
    canonical(plan)


def enumerate_group(root, group, excludes):
    files, omitted = [], []

    def visit(p):
        relative = p.relative_to(root).as_posix()
        s = p.lstat()
        if stat.S_ISLNK(s.st_mode):
            raise ValueError("Symlink refused: " + relative)
        reason = forbidden(relative)
        if any(relative == x or relative.startswith(x + "/") for x in excludes):
            reason = "plan-exclusion"
        if reason:
            omitted.append({"path": relative, "reason": reason,
                            "scope": "subtree" if stat.S_ISDIR(s.st_mode) else "file",
                            "bytes": s.st_size if stat.S_ISREG(s.st_mode) else None})
        elif stat.S_ISDIR(s.st_mode):
            for child in sorted(p.iterdir(), key=lambda x: x.name):
                visit(child)
        elif stat.S_ISREG(s.st_mode):
            files.append((relative, identity(s)))
        else:
            raise ValueError("Nonregular source refused: " + relative)

    for relative in group["paths"]:
        path = no_links(root / relative)
        visit(path)
    files.sort(key=lambda x: x[0])
    omitted.sort(key=lambda x: x["path"])
    if not files:
        raise ValueError("Empty group refused")
    return files, omitted


def scan_secrets(root, files):
    for relative, original in files:
        p = root / relative
        if original[3] > 2 * 1024 * 1024 or p.suffix.lower() in BINARY_SUFFIXES:
            continue
        data = p.read_bytes()
        if identity(p.lstat()) != original:
            raise ValueError("Source changed during secret scan: " + relative)
        try:
            text = data.decode("utf-8")
        except UnicodeDecodeError:
            continue
        if "\x00" in text:
            continue
        # Per-rule ASCII necessary fragments avoid repeated scans of numeric SWC
        # bodies (their header includes a public URL). Non-ASCII text still uses
        # every regex because Unicode IGNORECASE has broader equivalences.
        lower = text.lower()
        fragments = {
            "private-key": ("private key",),
            "github-token": ("ghp_", "github_pat_"),
            "aws-access-key": ("akia",),
            "url-user-password": ("://",),
            "secret-literal": ("password", "passwd", "api", "secret"),
        }
        for rule, pattern in SECRET_RULES:
            if text.isascii() and not any(x in lower for x in fragments[rule]):
                continue
            for match in pattern.finditer(text):
                if rule == "secret-literal":
                    literal = match.group(1) or match.group(2)
                    if literal.lower() in {"none", "null", "false", "true"}:
                        continue
                raise ValueError("Secret scan refused: " + relative + " [" + rule + "]")


def gzip_rows(path, rows):
    with Path(path).open("xb") as raw:
        with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as out:
            for row in rows:
                out.write((json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode())


class HashReader:
    def __init__(self, handle):
        self.handle, self.hash, self.size = handle, hashlib.sha256(), 0

    def read(self, count=-1):
        data = self.handle.read(count)
        self.hash.update(data)
        self.size += len(data)
        return data


def zstd_command(decompress=False, threads=2):
    return ["zstd", "-q", "-d", "-c"] if decompress else ["zstd", "-q", "-3", "-T" + str(threads), "-c"]


def make_archive(root, files, destination, threads):
    rows = []
    with destination.open("xb") as output, tempfile.TemporaryFile() as errors:
        proc = subprocess.Popen(zstd_command(threads=threads), stdin=subprocess.PIPE,
                                stdout=output, stderr=errors)
        try:
            with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.PAX_FORMAT) as archive:
                for relative, original in files:
                    p = no_links(root / relative)
                    if identity(p.lstat()) != original:
                        raise ValueError("Source changed before archive: " + relative)
                    with p.open("rb") as source:
                        opened = opened_identity(source, original)
                        reader = HashReader(source)
                        info = tarfile.TarInfo(relative)
                        info.size, info.mode, info.mtime = original[3], stat.S_IMODE(original[2]) & 0o777, 0
                        archive.addfile(info, reader)
                        if (reader.size != original[3] or identity(os.fstat(source.fileno())) != opened
                                or identity(p.lstat()) != original):
                            raise ValueError("Source changed during archive: " + relative)
                        rows.append({"path": relative, "bytes": reader.size, "sha256": reader.hash.hexdigest()})
            proc.stdin.close()
            if proc.wait() != 0:
                raise RuntimeError("zstd compression failed; no complete catalog published")
        except BaseException:
            proc.kill()
            proc.wait()
            raise
        finally:
            proc.stdin.close()
    return rows


def split_archive(archive, group_dir):
    chunks, full = [], hashlib.sha256()
    with archive.open("rb") as source:
        index = 0
        while first := source.read(min(BUFFER, CHUNK_BYTES)):
            name = "payload-%06d.bin" % index
            h, size = hashlib.sha256(), 0
            with (group_dir / name).open("xb") as out:
                block = first
                while block:
                    out.write(block)
                    full.update(block)
                    h.update(block)
                    size += len(block)
                    if size == CHUNK_BYTES:
                        break
                    block = source.read(min(BUFFER, CHUNK_BYTES - size))
            chunks.append({"path": name, "bytes": size, "sha256": h.hexdigest(), "index": index})
            index += 1
    if not chunks:
        raise ValueError("Empty compressed archive")
    return chunks, full.hexdigest()


def pack(plan_path, out, work, *, threads=2):
    if type(threads) is not int or not 1 <= threads <= 64:
        raise ValueError("threads must be 1..64")
    plan_bytes = Path(plan_path).read_bytes()
    plan = json_bytes(plan_bytes)
    validate_plan(plan)
    root, out, work = map(no_links, (Path(plan["source_root"]), Path(out), Path(work)))
    for a, b in ((root, out), (root, work), (out, work)):
        if a == b or a in b.parents or b in a.parents:
            raise ValueError("source, out and work must be disjoint trees")
    if out.exists() or work.exists():
        raise FileExistsError("out/work must both be new exclusive paths")
    out.mkdir(parents=True, exist_ok=False)
    work.mkdir(parents=True, exist_ok=False)
    write_new(out / "plan.json", plan_bytes)
    plan_sha = hashlib.sha256(plan_bytes).hexdigest()
    results = []
    for group in plan["groups"]:
        files, exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
        scan_secrets(root, files)
        group_dir = out / group["id"]
        group_dir.mkdir()
        archive = work / (group["id"] + ".tar.zst")
        members = make_archive(root, files, archive, threads)
        # A second metadata enumeration catches file additions/removals during pack.
        after_files, after_exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
        if after_files != files or after_exclusions != exclusions:
            raise ValueError("Source inventory changed during pack")
        gzip_rows(group_dir / "members.jsonl.gz", members)
        gzip_rows(group_dir / "exclusions.jsonl.gz", exclusions)
        chunks, combined = split_archive(archive, group_dir)
        catalog = dict(schema="lfs-publication-group-v1", group_id=group["id"], source_root=str(root),
                       source_paths=group["paths"], source_files=len(members),
                       source_bytes=sum(x["bytes"] for x in members), license=group["license"],
                       provenance=group["provenance"], plan_sha256=plan_sha,
                       archive_format="tar.pax+zstd", archive_sha256=combined,
                       archive_bytes=sum(x["bytes"] for x in chunks), chunk_bytes=CHUNK_BYTES,
                       chunks=chunks, members=sha_file(group_dir / "members.jsonl.gz"),
                       exclusions=sha_file(group_dir / "exclusions.jsonl.gz"),
                       exclusion_records=len(exclusions),
                       verification_scope="content snapshot, not a license grant or nested-archive secret audit",
                       secret_scan="UTF-8 files <=2 MiB, known binary suffixes excluded; fail closed on matching rules")
        write_new(group_dir / "catalog.json", canonical(catalog))
        result = {"group": group["id"], "files": len(members), "source_bytes": catalog["source_bytes"],
                  "archive_bytes": catalog["archive_bytes"], "chunks": len(chunks),
                  "catalog_sha256": sha_file(group_dir / "catalog.json")["sha256"]}
        results.append(result)
        print(json.dumps({"group_complete": result}), flush=True)
    summary = {"schema": "lfs-publication-pack-v1", "complete": True, "plan_sha256": plan_sha,
               "groups": results, "source_files": sum(x["files"] for x in results),
               "source_bytes": sum(x["source_bytes"] for x in results)}
    write_new(out / "pack-summary.json", canonical(summary))
    return summary


def check_digest_row(row):
    if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not HEX.fullmatch(str(row.get("sha256", ""))):
        raise ValueError("Invalid SHA/size metadata")


def iter_members(path):
    previous = None
    with gzip.open(path, "rt", encoding="utf-8") as inp:
        for line in inp:
            row = json.loads(line)
            if set(row) != {"path", "bytes", "sha256"}:
                raise ValueError("Invalid member row")
            check_digest_row(row)
            p = relpath(row["path"])
            if forbidden(p) or (previous is not None and p <= previous):
                raise ValueError("Forbidden/duplicate/unsorted member: " + p)
            previous = p
            yield row


def verify(group_dir):
    group_dir = no_links(group_dir)
    controls_before = {name: sha_file(group_dir / name) for name in
                       ("catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz")}
    catalog = json_read(group_dir / "catalog.json")
    if catalog.get("schema") != "lfs-publication-group-v1" or catalog.get("chunk_bytes") != CHUNK_BYTES or catalog.get("archive_format") != "tar.pax+zstd":
        raise ValueError("Wrong catalog schema/format/chunk size")
    chunks = catalog.get("chunks")
    if not isinstance(chunks, list) or not chunks:
        raise ValueError("Missing payload chunks")
    expected = {"catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz"}
    for i, row in enumerate(chunks):
        check_digest_row(row)
        if set(row) != {"path", "bytes", "sha256", "index"} or type(row["index"]) is not int or row["index"] != i or row["path"] != "payload-%06d.bin" % i:
            raise ValueError("Wrong chunk name/order")
        if not 0 < row["bytes"] <= CHUNK_BYTES or (i < len(chunks) - 1 and row["bytes"] != CHUNK_BYTES):
            raise ValueError("Wrong chunk size")
        expected.add(row["path"])
    if {p.name for p in group_dir.iterdir()} != expected:
        raise ValueError("Extra/missing downloaded files")
    for p in group_dir.iterdir():
        no_links(p)
        if not p.is_file():
            raise ValueError("Nonfile downloaded entry")
    for name, key in (("members.jsonl.gz", "members"), ("exclusions.jsonl.gz", "exclusions")):
        check_digest_row(catalog[key])
        if controls_before[name] != catalog[key]:
            raise ValueError("Metadata digest mismatch: " + name)
    with gzip.open(group_dir / "exclusions.jsonl.gz", "rt", encoding="utf-8") as source:
        omitted = list(map(json.loads, source))
    if len(omitted) != catalog.get("exclusion_records"):
        raise ValueError("Exclusion inventory mismatch")
    for row in omitted:
        relpath(row["path"])
    errors, combined, compressed = [], hashlib.sha256(), [0]
    with tempfile.TemporaryFile() as diagnostics:
        proc = subprocess.Popen(zstd_command(decompress=True), stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=diagnostics)

        def feed():
            try:
                for row in chunks:
                    p = group_dir / row["path"]
                    before = identity(p.lstat())
                    h, size = hashlib.sha256(), 0
                    with p.open("rb") as source:
                        opened = opened_identity(source, before)
                        while data := source.read(BUFFER):
                            h.update(data)
                            combined.update(data)
                            size += len(data)
                            compressed[0] += len(data)
                            proc.stdin.write(data)
                        if identity(os.fstat(source.fileno())) != opened or identity(p.lstat()) != before:
                            raise ValueError("Chunk changed during read")
                    if size != row["bytes"] or h.hexdigest() != row["sha256"]:
                        raise ValueError("Payload digest mismatch: " + row["path"])
            except BaseException as exc:
                errors.append(exc)
            finally:
                try:
                    proc.stdin.close()
                except OSError:
                    pass

        producer = threading.Thread(target=feed, daemon=True)
        producer.start()
        count, total = 0, 0
        expected_rows = iter(iter_members(group_dir / "members.jsonl.gz"))
        try:
            with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
                for member in archive:
                    row = next(expected_rows, None)
                    if not member.isreg() or member.linkname or member.sparse is not None:
                        raise ValueError("Nonregular/sparse/link tar member refused")
                    path = relpath(member.name)
                    if forbidden(path) or row is None or path != row["path"] or member.size != row["bytes"]:
                        raise ValueError("Tar member set/size/path mismatch: " + path)
                    reader = HashReader(archive.extractfile(member))
                    while reader.read(BUFFER):
                        pass
                    if reader.size != row["bytes"] or reader.hash.hexdigest() != row["sha256"]:
                        raise ValueError("Tar member content mismatch: " + path)
                    count += 1
                    total += reader.size
                # tarfile stops at the first end marker; reject a hidden second
                # archive/nonzero trailer, including bytes in its read-ahead buf.
                while trailer := archive.fileobj.read(BUFFER):
                    if any(trailer):
                        raise ValueError("Nonzero data after tar end marker")
            if next(expected_rows, None) is not None:
                raise ValueError("Missing tar members")
            # Drain to validate zstd's complete stream/checksum and finish the feeder.
            while proc.stdout.read(BUFFER):
                pass
            producer.join()
            code = proc.wait()
            if errors:
                raise ValueError("Payload streaming failed: " + str(errors[0]))
            if code != 0:
                raise ValueError("zstd decompression failed")
        except BaseException:
            proc.kill()
            proc.wait()
            producer.join(timeout=10)
            raise
        finally:
            proc.stdout.close()
            expected_rows.close()
    if (count != catalog.get("source_files") or total != catalog.get("source_bytes")
            or compressed[0] != catalog.get("archive_bytes") or combined.hexdigest() != catalog.get("archive_sha256")):
        raise ValueError("Final inventory/archive digest mismatch")
    for name, before in controls_before.items():
        if sha_file(group_dir / name) != before:
            raise ValueError("Downloaded control changed during verification: " + name)
    return {"schema": "lfs-publication-verify-v1", "pass_all": True, "group": catalog["group_id"],
            "source_files": count, "source_bytes": total, "archive_bytes": compressed[0],
            "archive_sha256": combined.hexdigest(), "catalog_sha256": controls_before["catalog.json"]["sha256"],
            "scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility"}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest="command", required=True)
    command = sub.add_parser("pack")
    command.add_argument("--plan", required=True)
    command.add_argument("--out", required=True)
    command.add_argument("--work", required=True)
    command.add_argument("--threads", type=int, default=2)
    command = sub.add_parser("verify")
    command.add_argument("--group-dir", required=True)
    args = parser.parse_args()
    result = (pack(args.plan, args.out, args.work, threads=args.threads) if args.command == "pack"
              else verify(args.group_dir))
    print(json.dumps(result, sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)


if __name__ == "__main__":
    main()