Preserve CPU and GPU acceleration plans, evidence and LFS archives; mark research paused

tools/acceleration-publication/archive_results.py

@@ -0,0 +1,183 @@
+"""Read-only source packing and streamed verification for exact replay evidence."""
+import argparse
+from datetime import datetime, timezone
+import hashlib
+import io
+import json
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tarfile
+
+SOURCE_CODE = Path('/home/qishizhe/digital-drosophila-v100-acceleration/models/male-cns-lif')
+GROUPS = {
+    'gpu': (Path('/home/qishizhe/phase4ar-v100-acceleration-20261001'),
+            '6efdec34c96cdc79f786b91bd7a99b0e8ef64d0c4dcfa525646c82659a77d44f',
+            'conditioning-acceleration-v1'),
+    'pipeline': (Path('/home/qishizhe/phase4ar-pipeline-acceleration-20261001'),
+                 '4c598f51e927b3145891db42d9bb345c52d4aaa3b3a05ac257b207fff90cace4',
+                 'conditioning-pipeline-acceleration-v1'),
+}
+REGISTRY = Path('registry/acceleration-20261001')
+ARTIFACTS = Path('artifacts/acceleration-20261001')
+INSTALL_LOGS = {
+    'create-venv.stdout.log', 'create-venv.stderr.log',
+    'install-candidate.stdout.log', 'install-candidate.stderr.log',
+    'install-candidate-attempt-02.stdout.log', 'install-candidate-attempt-02.stderr.log',
+    'upgrade-candidate-pip.stdout.log', 'upgrade-candidate-pip.stderr.log',
+}
+SECRET_RULES = [
+    rb'-----BEGIN (?:OPENSSH |RSA |EC )?PRIVATE KEY-----',
+    rb'(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}|AKIA[A-Z0-9]{16})',
+    rb'https?://[^\s/]+:[^\s/@]+@',
+]
+
+
+def require(value, message):
+    if not value:
+        raise ValueError(message)
+
+
+def sha(path):
+    digest = hashlib.sha256()
+    with path.open('rb') as stream:
+        for block in iter(lambda: stream.read(8 << 20), b''):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def ident(path):
+    return dict(bytes=path.stat().st_size, sha256=sha(path))
+
+
+def save(path, obj):
+    with path.open('x') as stream:
+        json.dump(obj, stream, indent=2, sort_keys=True, allow_nan=False)
+        stream.write('\n')
+
+
+def excluded(name):
+    if name.startswith(('cupy-cache-', 'brain-replay/build/')):
+        return 'Rebuildable compiler cache/SDK intermediates, not a scientific result'
+    if name in INSTALL_LOGS:
+        return 'Environment installation output; versions and wheel report retained'
+    return None
+
+
+def scan_text(path):
+    if path.suffix.lower() not in ('.json', '.jsonl', '.log', '.md', '.txt', '.csv', '.py', '.cu', '.diff'):
+        return
+    require(path.stat().st_size <= 25 * (1 << 20), 'Oversized text requires manual review')
+    data = path.read_bytes()
+    for rule in SECRET_RULES:
+        require(not re.search(rule, data), 'Potential credential marker: ' + path.name)
+
+
+def pack(root):
+    require(root == Path('/home/qishizhe/digital-drosophila-acceleration-publication-20261001'), 'Independent publication clone required')
+    require((root / '.git').is_dir() and not root.is_symlink(), 'Ordinary independent clone required')
+    registry, artifacts = root / REGISTRY, root / ARTIFACTS
+    registry.mkdir(parents=True, exist_ok=True)
+    artifacts.mkdir(parents=True, exist_ok=False)
+    catalog = dict(schema='acceleration-publication-v1', created_utc=datetime.now(timezone.utc).isoformat(),
+        numerical_changes=False, training_resumed=False, complete_training_restore=False,
+        scope='Two completed independent engineering experiments, not the full stopped scientific run', groups={})
+    for group, (source, expected_manifest, code_dir) in GROUPS.items():
+        require(sha(source / 'artifact-manifest.json') == expected_manifest, 'Sealed source manifest changed')
+        manifest = json.loads((source / 'artifact-manifest.json').read_text())
+        for name, expected in manifest['source_files'].items():
+            require(sha(SOURCE_CODE / code_dir / name) == expected, 'Sealed source code changed')
+            require(sha(root / 'models/male-cns-lif' / code_dir / name) == expected, 'Published source differs')
+        members, omitted = {}, {}
+        for name, expected in manifest['files'].items():
+            path = source / name
+            require(path.resolve().is_relative_to(source) and not path.is_symlink(), 'Unsafe source path')
+            require(ident(path) == expected, 'Sealed input file changed: ' + name)
+            reason = excluded(name)
+            if reason:
+                omitted[name] = dict(expected, reason=reason)
+            else:
+                scan_text(path)
+                members[group + '/' + name] = dict(expected, source_relative=name)
+        for name in ('artifact-manifest.json', 'SHA256SUMS'):
+            require(group + '/' + name not in members, 'Duplicate source control file')
+            members[group + '/' + name] = dict(ident(source / name), source_relative=name)
+        archive = artifacts / (group + '.tar.zst')
+        with archive.open('xb') as destination:
+            compressor = subprocess.Popen(['zstd', '-q', '-T2', '-3', '-c'], stdin=subprocess.PIPE, stdout=destination)
+            try:
+                with tarfile.open(fileobj=compressor.stdin, mode='w|', format=tarfile.PAX_FORMAT) as tar:
+                    for name, item in sorted(members.items()):
+                        path = source / item['source_relative']
+                        info = tarfile.TarInfo(name)
+                        info.size, info.mode, info.mtime = item['bytes'], 0o644, 0
+                        with path.open('rb') as stream:
+                            tar.addfile(info, stream)
+                        require(ident(path) == {k: item[k] for k in ('bytes', 'sha256')}, 'Input changed during packing')
+                compressor.stdin.close()
+                require(compressor.wait(timeout=120) == 0, 'Compression failed')
+            finally:
+                if compressor.poll() is None:
+                    compressor.kill()
+                    compressor.wait()
+        target = registry / group
+        target.mkdir(exist_ok=False)
+        for name in ('RESULTS.md', 'final-summary.json', 'artifact-manifest.json', 'SHA256SUMS'):
+            shutil.copyfile(source / name, target / name)
+            require(sha(source / name) == sha(target / name), 'Summary copy differs')
+        catalog['groups'][group] = dict(source_root=str(source), source_manifest_sha256=expected_manifest,
+            source_files=manifest['source_files'], archive=str(archive.relative_to(root)), **ident(archive),
+            member_count=len(members), member_bytes=sum(x['bytes'] for x in members.values()),
+            members=members, excluded=omitted, excluded_count=len(omitted))
+    save(registry / 'publication-catalog.json', catalog)
+    result = verify(root)
+    save(registry / 'pack-verification.json', result)
+    print(json.dumps(dict(complete=True, groups=result['groups'], source_files_modified=False), indent=2))
+
+
+def verify(root):
+    catalog_path = root / REGISTRY / 'publication-catalog.json'
+    catalog = json.loads(catalog_path.read_text())
+    require(catalog['schema'] == 'acceleration-publication-v1', 'Wrong publication catalog')
+    results = {}
+    for group, info in catalog['groups'].items():
+        archive = root / info['archive']
+        require(archive.resolve().is_relative_to(root / ARTIFACTS) and not archive.is_symlink(), 'Unsafe archive path')
+        require(ident(archive) == {k: info[k] for k in ('bytes', 'sha256')}, 'Archive identity mismatch')
+        decoder = subprocess.Popen(['zstd', '-q', '-d', '-c', str(archive)], stdout=subprocess.PIPE)
+        found, total = set(), 0
+        try:
+            with tarfile.open(fileobj=decoder.stdout, mode='r|') as tar:
+                for entry in tar:
+                    require(entry.isfile() and entry.name in info['members'] and entry.name not in found, 'Unexpected/duplicate member')
+                    expected = info['members'][entry.name]
+                    require(entry.size == expected['bytes'], 'Member size mismatch')
+                    h, count = hashlib.sha256(), 0
+                    with tar.extractfile(entry) as stream:
+                        for block in iter(lambda: stream.read(8 << 20), b''):
+                            h.update(block)
+                            count += len(block)
+                    require(count == entry.size and h.hexdigest() == expected['sha256'], 'Member hash mismatch')
+                    found.add(entry.name)
+                    total += count
+            while decoder.stdout.read(1 << 20):
+                pass
+            require(decoder.wait(timeout=30) == 0 and found == set(info['members']), 'Archive truncated or member set differs')
+        finally:
+            decoder.stdout.close()
+            if decoder.poll() is None:
+                decoder.kill()
+                decoder.wait()
+        results[group] = dict(files=len(found), original_bytes=total, archive_bytes=info['bytes'], sha256=info['sha256'], verified=True)
+    return dict(pass_all=True, groups=results, catalog_sha256=sha(catalog_path), verification='Full streamed members, no extraction or model execution')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('mode', choices=('pack', 'verify'))
+    args = parser.parse_args()
+    if args.mode == 'pack':
+        pack(Path.cwd())
+    else:
+        print(json.dumps(verify(Path.cwd()), indent=2))

tools/acceleration-publication/verify_delivery.py

@@ -0,0 +1,68 @@
+"""Actual fresh-store remote retrieval; no scientific execution or retries."""
+import argparse
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import subprocess
+import time
+
+import archive_results as archives
+
+REMOTE = 'ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git'
+BRANCH = 'codex/v100-conditioning-acceleration'
+
+
+def cmd(args, output=False):
+    result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    archives.require(result.returncode == 0, 'Git/LFS command failed; no automatic retry; diagnostic output not published')
+    return result.stdout.decode().strip() if output else result.returncode
+
+
+def run(expected, receipt):
+    root = Path.cwd()
+    archives.require(root == Path('/home/qishizhe/digital-drosophila-acceleration-verify-20261001')
+                     and (root / '.git').is_dir(), 'New independent verification clone required')
+    archives.require(cmd(['git', 'rev-parse', 'HEAD'], True) == expected, 'Unexpected checkout commit')
+    archives.require(cmd(['git', 'remote', 'get-url', 'origin'], True) == REMOTE, 'Unexpected remote')
+    remote_ref = cmd(['git', 'ls-remote', 'origin', 'refs/heads/' + BRANCH], True)
+    archives.require(remote_ref.split()[0] == expected, 'Remote branch does not identify data commit')
+    archives.require(not cmd(['git', 'status', '--porcelain'], True), 'Unclean initial checkout')
+    objects = root / '.git/lfs/objects'
+    before = list(p for p in objects.rglob('*') if p.is_file()) if objects.exists() else []
+    archives.require(not before, 'Fresh LFS object store must contain no prior object')
+    archives.require(not receipt.exists() and receipt.parent.is_dir()
+                     and receipt.parent == Path('/home/qishizhe/digital-drosophila-acceleration-publication-20261001/registry/acceleration-20261001'),
+                     'Exclusive publication receipt required')
+    start = time.perf_counter()
+    pull = cmd(['git', 'lfs', 'pull', '--include=artifacts/acceleration-20261001/**', '--exclude=', 'origin'])
+    fsck = cmd(['git', 'lfs', 'fsck', '--dry-run'])
+    report = archives.verify(root)
+    catalog = json.loads((root / archives.REGISTRY / 'publication-catalog.json').read_text())
+    source_count = 0
+    for group, info in catalog['groups'].items():
+        directory = archives.GROUPS[group][2]
+        for name, expected_sha in info['source_files'].items():
+            archives.require(archives.sha(root / 'models/male-cns-lif' / directory / name) == expected_sha,
+                             'Published source differs from sealed version')
+            source_count += 1
+    stored = [p for p in objects.rglob('*') if p.is_file()]
+    archives.require(len(stored) == 2 and not cmd(['git', 'status', '--porcelain'], True), 'Incomplete LFS store or changed checkout')
+    archives.require(cmd(['git', 'rev-parse', 'HEAD'], True) == expected, 'Checkout changed during verification')
+    report.update(schema='acceleration-fresh-remote-delivery-v1', verified_utc=datetime.now(timezone.utc).isoformat(),
+        remote=REMOTE, branch=BRANCH, data_commit=expected, fresh_download_proven=True,
+        initial_LFS_objects=0, downloaded_LFS_objects=len(stored), downloaded_bytes=sum(p.stat().st_size for p in stored),
+        pull_exit_code=pull, fsck_exit_code=fsck, sealed_source_files_checked=source_count,
+        wall_seconds=time.perf_counter()-start, scientific_experiments_run=0, training_resumed=False,
+        complete_training_restore=False, historical_scientific_gate_reexecuted=False,
+        transport='Previously audited LAN-pinned HTTPS adapter, verified TLS and SSH short-lived authorization',
+        scope='Remote Git commit, two actual LFS downloads, complete public archive contents and 22 sealed code files; not all external scientific dependencies')
+    archives.save(receipt, report)
+    print(json.dumps(report, indent=2))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--expected-commit', required=True)
+    parser.add_argument('--receipt', type=Path, required=True)
+    args = parser.parse_args()
+    run(args.expected_commit, args.receipt)

tools/lfs-publication/lan_lfs_transfer.py

@@ -0,0 +1,336 @@
+"""Repository-local, LAN-pinned Git LFS standalone transfer agent.
+
+Protocol: https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md
+Only fixed-origin SSH authentication and HTTPS basic transfers are supported.
+No config/network changes, retries, redirects, credential files, or cache-based
+download substitution. Credentials are passed to curl exclusively on stdin.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import tempfile
+from urllib.parse import urlsplit
+
+ORIGIN = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
+HOST = "git.lain.asia"
+IP = "192.168.10.31"
+PREFIX = "/qishizhe/digital-drosophila.git/info/lfs"
+SOURCE = Path("/data/digital-drosophila")
+OID = re.compile(r"^[0-9a-f]{64}$")
+HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
+MAX_JSON = 2 * 1024 * 1024
+BLOCK = 1024 * 1024
+
+
+class TransferError(Exception):
+    """Only fixed, non-secret messages may be passed to this class."""
+
+
+def require(ok, code):
+    if not ok:
+        raise TransferError(code)
+
+
+def decode_json(data):
+    def unique(pairs):
+        result = {}
+        for key, value in pairs:
+            require(key not in result, "duplicate-json-key")
+            result[key] = value
+        return result
+    require(len(data) <= MAX_JSON, "json-response-too-large")
+    try:
+        return json.loads(data, object_pairs_hook=unique)
+    except (UnicodeError, ValueError):
+        raise TransferError("invalid-json") from None
+
+
+def safe_path(value):
+    require(isinstance(value, (str, Path)), "invalid-file-path")
+    p = Path(value)
+    require(p.is_absolute() and ".." not in p.parts, "absolute-safe-path-required")
+    for component in (p, *p.parents):
+        if component.exists() or component.is_symlink():
+            s = component.lstat()
+            require(not stat.S_ISLNK(s.st_mode) and not
+                    (getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)),
+                    "symlink-or-reparse-refused")
+    return p
+
+
+def safe_url(value):
+    require(isinstance(value, str) and len(value) <= 16384 and
+            not any(ord(c) <= 32 or ord(c) == 127 for c in value), "invalid-action-url")
+    try:
+        p = urlsplit(value)
+        valid = (p.scheme == "https" and p.netloc in (HOST, HOST + ":443") and
+                 p.hostname == HOST and p.port in (None, 443) and
+                 p.username is None and p.password is None and not p.fragment)
+    except ValueError:
+        raise TransferError("url-authority-refused") from None
+    require(valid, "url-authority-refused")
+    require(p.path == PREFIX or p.path.startswith(PREFIX + "/"), "url-repository-prefix-refused")
+    require("%" not in p.path and "\\" not in p.path and
+            all(x not in ("", ".", "..") for x in p.path.split("/")[1:]), "url-path-refused")
+    return value
+
+
+def safe_headers(value, *, allow_upload_chunked=False):
+    require(isinstance(value, dict) and len(value) <= 32, "invalid-action-headers")
+    result, seen = {}, set()
+    denied = {"host", "connection", "content-length", "transfer-encoding", "te",
+              "trailer", "upgrade", "proxy-authorization", "proxy-connection", "range"}
+    for key, item in value.items():
+        upload_chunked = (allow_upload_chunked is True and isinstance(key, str) and
+                          key.lower() == "transfer-encoding" and item == "chunked")
+        require(isinstance(key, str) and HEADER.fullmatch(key) is not None and
+                key.lower() not in seen and (key.lower() not in denied or upload_chunked),
+                "header-name-refused")
+        require(isinstance(item, str) and len(item) <= 16384 and
+                not any(ord(c) < 32 or ord(c) == 127 for c in item), "header-value-refused")
+        seen.add(key.lower())
+        result[key] = item
+    return result
+
+
+def quote(value):
+    require(isinstance(value, str) and not any(ord(c) < 32 or ord(c) == 127 for c in value),
+            "curl-config-value-refused")
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def fingerprint(s):
+    return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
+
+
+def verify_file(path, oid, size):
+    path = safe_path(path)
+    before = fingerprint(path.lstat())
+    require(stat.S_ISREG(before[2]) and before[3] == size, "object-file-type-or-size-mismatch")
+    digest, count = hashlib.sha256(), 0
+    with path.open("rb") as handle:
+        opened = fingerprint(os.fstat(handle.fileno()))
+        # Windows path stat and fstat can expose different ctime definitions.
+        require(opened[:-1] == before[:-1] if os.name == "nt" else opened == before,
+                "object-changed-at-open")
+        while data := handle.read(BLOCK):
+            digest.update(data)
+            count += len(data)
+        require(fingerprint(os.fstat(handle.fileno())) == opened, "object-changed-during-read")
+    require(fingerprint(path.lstat()) == before and count == size and digest.hexdigest() == oid,
+            "object-sha-or-stability-mismatch")
+    return before
+
+
+def git_output(cwd, *args):
+    try:
+        result = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True,
+                                timeout=15, check=False)
+        require(result.returncode == 0, "git-inspection-failed")
+        return result.stdout.decode("utf-8").strip()
+    except (OSError, UnicodeError, subprocess.TimeoutExpired):
+        raise TransferError("git-inspection-failed") from None
+
+
+class Agent:
+    def __init__(self, message, emit, cwd=None):
+        require(message.get("event") == "init" and message.get("operation") in ("upload", "download"),
+                "invalid-init")
+        require(message.get("remote") in ("origin", ORIGIN), "remote-refused")
+        concurrency = message.get("concurrenttransfers")
+        require(type(concurrency) is int and 1 <= concurrency <= 2 and
+                type(message.get("concurrent")) is bool, "concurrency-must-be-at-most-two")
+        cwd = safe_path(Path.cwd() if cwd is None else cwd)
+        require(cwd != SOURCE and SOURCE not in cwd.parents, "scientific-source-cwd-refused")
+        self.root = safe_path(git_output(cwd, "rev-parse", "--show-toplevel"))
+        require(self.root != SOURCE and SOURCE not in self.root.parents, "scientific-source-repository-refused")
+        require(git_output(cwd, "config", "--get", "remote.origin.url") == ORIGIN, "origin-refused")
+        self.git_dir = safe_path(git_output(cwd, "rev-parse", "--absolute-git-dir"))
+        require(self.git_dir == self.root / ".git" and self.git_dir.is_dir(), "ordinary-isolated-clone-required")
+        self.operation, self.emit = message["operation"], emit
+
+    def authenticate(self):
+        command = ["ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
+                   "-o", "ConnectTimeout=15", "-o", "ConnectionAttempts=1",
+                   "-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=2",
+                   "-p", "2222", "git@192.168.10.31", "git-lfs-authenticate",
+                   "qishizhe/digital-drosophila.git", self.operation]
+        try:
+            result = subprocess.run(command, capture_output=True, timeout=45, check=False)
+            require(result.returncode == 0, "ssh-authentication-failed")
+            auth = decode_json(result.stdout)
+        except (OSError, subprocess.TimeoutExpired):
+            raise TransferError("ssh-authentication-failed") from None
+        require(isinstance(auth, dict), "invalid-ssh-authentication-response")
+        href = safe_url(auth.get("href"))
+        parsed = urlsplit(href)
+        require(parsed.path == PREFIX and not parsed.query, "authentication-base-url-refused")
+        return href, safe_headers(auth.get("header", {}))
+
+    def curl(self, method, url, headers, *, body=None, upload=None, download=None, size=None):
+        safe_url(url)
+        headers = safe_headers(headers, allow_upload_chunked=(self.operation == "upload" and
+                               method == "PUT" and upload is not None and body is None and download is None))
+        require(method in ("GET", "POST", "PUT"), "unsupported-http-method")
+        present = {name.lower() for name in headers}
+        if "accept" not in present:
+            headers["Accept"] = "application/vnd.git-lfs+json"
+        if method == "PUT" and "content-type" not in present:
+            headers["Content-Type"] = "application/octet-stream"
+        if body is not None and "content-type" not in present:
+            headers["Content-Type"] = "application/vnd.git-lfs+json"
+        lines = ["url = " + quote(url), "request = " + quote(method),
+                 'noproxy = "*"', 'resolve = "' + HOST + ':443:' + IP + '"',
+                 'proto = "=https"', 'proto-redir = "=https"', "max-redirs = 0",
+                 "connect-timeout = 15", "max-time = 1800", "fail", "silent", "show-error",
+                 'write-out = "\\n%{http_code}"']
+        for name, value in headers.items():
+            lines.append("header = " + quote(name + ": " + value))
+        if body is not None:
+            encoded = json.dumps(body, separators=(",", ":"), allow_nan=False)
+            lines.append("data-binary = " + quote(encoded))
+        if upload is not None:
+            lines.append("upload-file = " + quote(str(safe_path(upload))))
+        if download is not None:
+            lines.append("output = " + quote(str(safe_path(download))))
+            require(type(size) is int and size >= 0, "invalid-download-limit")
+            # A zero-byte object still needs a full HTTP fetch, not a cache hit.
+            lines.append("max-filesize = " + str(max(size, 1)))
+        else:
+            lines.append("max-filesize = " + str(MAX_JSON))
+        config = ("\n".join(lines) + "\n").encode("utf-8")
+        try:
+            result = subprocess.run(["curl", "-q", "--config", "-"], input=config,
+                                    capture_output=True, timeout=1830, check=False)
+        except (OSError, subprocess.TimeoutExpired):
+            raise TransferError("curl-execution-failed") from None
+        # Never print stderr, response data, URLs, configuration, or headers.
+        require(result.returncode == 0, "curl-request-failed")
+        require(len(result.stdout) <= MAX_JSON + 32, "http-response-too-large")
+        content, separator, status = result.stdout.rpartition(b"\n")
+        require(separator and len(status) == 3 and status.isdigit() and 200 <= int(status) < 300,
+                "non-success-http-status")
+        return content
+
+    def batch(self, oid, size):
+        href, headers = self.authenticate()
+        headers = {k: v for k, v in headers.items() if k.lower() not in ("content-type", "accept")}
+        headers.update({"Content-Type": "application/vnd.git-lfs+json", "Accept": "application/vnd.git-lfs+json"})
+        response = decode_json(self.curl("POST", href + "/objects/batch", headers,
+                                        body={"operation": self.operation, "transfers": ["basic"],
+                                              "objects": [{"oid": oid, "size": size}]}))
+        require(isinstance(response, dict) and response.get("transfer", "basic") == "basic",
+                "non-basic-batch-response")
+        objects = response.get("objects")
+        require(isinstance(objects, list) and len(objects) == 1 and isinstance(objects[0], dict),
+                "invalid-batch-object-count")
+        item = objects[0]
+        require(item.get("oid") == oid and type(item.get("size")) is int and item["size"] == size,
+                "batch-object-identity-mismatch")
+        require("error" not in item, "batch-object-rejected")
+        actions = item.get("actions", {})
+        require(isinstance(actions, dict), "invalid-batch-actions")
+        allowed = {"upload", "verify"} if self.operation == "upload" else {"download"}
+        require(set(actions) <= allowed, "unexpected-batch-action")
+        for name, action in actions.items():
+            require(isinstance(action, dict), "invalid-batch-action")
+            safe_url(action.get("href"))
+            safe_headers(action.get("header", {}), allow_upload_chunked=(self.operation == "upload" and name == "upload"))
+        return actions
+
+    def progress(self, oid, size, previous=0):
+        self.emit({"event": "progress", "oid": oid, "bytesSoFar": size, "bytesSinceLast": size - previous})
+
+    def transfer(self, message):
+        require(message.get("event") == self.operation and message.get("action") is None,
+                "standalone-operation-required")
+        oid, size = message.get("oid"), message.get("size")
+        require(isinstance(oid, str) and OID.fullmatch(oid) is not None and
+                type(size) is int and 0 <= size <= 2**63 - 1, "invalid-object-identity")
+        source, before = None, None
+        if self.operation == "upload":
+            source = safe_path(message.get("path"))
+            require(source == self.git_dir / "lfs" / "objects" / oid[:2] / oid[2:4] / oid,
+                    "upload-outside-current-lfs-object-store")
+            before = verify_file(source, oid, size)
+        actions = self.batch(oid, size)
+        self.progress(oid, 0)
+        if self.operation == "upload":
+            if "upload" in actions:
+                action = actions["upload"]
+                self.curl("PUT", action["href"], action.get("header", {}), upload=source)
+                require(fingerprint(source.lstat()) == before, "upload-source-changed")
+                if "verify" in actions:
+                    action = actions["verify"]
+                    headers = {k: v for k, v in action.get("header", {}).items() if k.lower() != "content-type"}
+                    headers["Content-Type"] = "application/vnd.git-lfs+json"
+                    self.curl("POST", action["href"], headers, body={"oid": oid, "size": size})
+            else:
+                require(not actions, "verify-without-upload-refused")
+                require(fingerprint(source.lstat()) == before, "upload-source-changed")
+            self.progress(oid, size)
+            return {"event": "complete", "oid": oid}
+        require("download" in actions, "download-action-missing")
+        directory = safe_path(self.git_dir / "lfs" / "lan-transfer-tmp")
+        directory.mkdir(parents=True, exist_ok=True)
+        safe_path(directory)
+        fd, name = tempfile.mkstemp(prefix=oid + ".", suffix=".partial", dir=directory)
+        os.close(fd)
+        target = Path(name)
+        action = actions["download"]
+        # On failure the uniquely created partial remains for investigation;
+        # it is never used as a fallback/cache input on a later transfer.
+        self.curl("GET", action["href"], action.get("header", {}), download=target, size=size)
+        verify_file(target, oid, size)
+        self.progress(oid, size)
+        return {"event": "complete", "oid": oid, "path": str(target)}
+
+
+def emit(value):
+    sys.stdout.write(json.dumps(value, separators=(",", ":"), allow_nan=False) + "\n")
+    sys.stdout.flush()
+
+
+def serve(inp=None, send=None, factory=Agent):
+    inp, send = sys.stdin if inp is None else inp, emit if send is None else send
+    agent = None
+    while True:
+        line = inp.readline(MAX_JSON + 1)
+        if not line:
+            return 0
+        try:
+            message = decode_json(line)
+            require(isinstance(message, dict), "protocol-object-required")
+            if agent is None:
+                agent = factory(message, send)
+                send({})
+            elif message.get("event") == "terminate":
+                return 0
+            else:
+                oid = message.get("oid")
+                safe_oid = oid if isinstance(oid, str) and OID.fullmatch(oid) else ""
+                try:
+                    send(agent.transfer(message))
+                except Exception as exc:
+                    code = str(exc) if isinstance(exc, TransferError) else "transfer-internal-failure"
+                    send({"event": "complete", "oid": safe_oid,
+                          "error": {"code": 2, "message": code}})
+        except Exception as exc:
+            code = str(exc) if isinstance(exc, TransferError) else "agent-initialization-or-protocol-failure"
+            if agent is None:
+                send({"error": {"code": 32, "message": code}})
+            else:
+                # No attacker-supplied raw line or exception text is printed.
+                sys.stderr.write("lan-lfs: invalid protocol; stopping\n")
+            return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(serve())

tools/lfs-publication/test_lan_lfs_transfer.py

@@ -0,0 +1,286 @@
+"""Pure temporary/mocked protocol tests: never SSH, curl, GPU, or real Git."""
+import contextlib
+import hashlib
+import io
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import lan_lfs_transfer as M
+
+BASE = "https://" + M.HOST + M.PREFIX
+TOKEN = "synthetic-fixture-" + "credential-value"
+
+
+class AdapterTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name) / "clone"
+        self.git = self.root / ".git"
+        self.git.mkdir(parents=True)
+        self.data = b"synthetic content\x00\xff\n"
+        self.oid = hashlib.sha256(self.data).hexdigest()
+        self.size = len(self.data)
+        self.events = []
+        self.agent = object.__new__(M.Agent)
+        self.agent.root, self.agent.git_dir = self.root, self.git
+        self.agent.operation, self.agent.emit = "upload", self.events.append
+        self.source = self.git / "lfs" / "objects" / self.oid[:2] / self.oid[2:4] / self.oid
+        self.source.parent.mkdir(parents=True)
+        self.source.write_bytes(self.data)
+
+    def request(self, operation="upload", **extra):
+        value = {"event": operation, "oid": self.oid, "size": self.size, "action": None}
+        if operation == "upload":
+            value["path"] = str(self.source)
+        value.update(extra)
+        return value
+
+    def init_message(self, **extra):
+        return dict(event="init", operation="upload", remote="origin", concurrent=True,
+                    concurrenttransfers=2, **extra)
+
+    def action(self, suffix="objects/" ):
+        return {"href": BASE + "/" + suffix + self.oid,
+                "header": {"Authorization": "Bearer " + TOKEN}}
+
+    def test_strict_url_policy(self):
+        self.assertEqual(M.safe_url(BASE + "/objects/" + self.oid + "?signed=opaque"),
+                         BASE + "/objects/" + self.oid + "?signed=opaque")
+        invalid = [BASE.replace("https:", "http:"), BASE.replace(M.HOST, M.IP),
+                   BASE.replace(M.HOST, "other.example"), BASE.replace(M.HOST, M.HOST + ":444"),
+                   BASE.replace(M.HOST, "user:opaque@" + M.HOST), BASE + "-other/objects/x",
+                   BASE + "/../other", BASE + "/%2e%2e/other", BASE + "/objects/x#fragment",
+                   BASE + "//objects/x", BASE + "/objects\\x", BASE + "\n"]
+        for url in invalid:
+            with self.subTest(url=url), self.assertRaises(M.TransferError):
+                M.safe_url(url)
+
+    def test_headers_reject_injection_and_routing_overrides(self):
+        for headers in ({"Authorization": "x\r\nHost: bad"}, {"Host": "bad"},
+                        {"Authorization": "a", "authorization": "b"},
+                        {"Range": "bytes=1-"}, {"bad\nkey": "x"}, {"X": "\x00"}):
+            with self.subTest(headers=headers), self.assertRaises(M.TransferError):
+                M.safe_headers(headers)
+
+    def test_init_checks_origin_and_isolated_git_directory(self):
+        values = [str(self.root), M.ORIGIN, str(self.git)]
+        with patch.object(M, "git_output", side_effect=values):
+            actual = M.Agent(self.init_message(), self.events.append, cwd=self.root)
+        self.assertEqual(actual.git_dir, self.git)
+        with patch.object(M, "git_output", side_effect=[str(self.root), "ssh://other/repo"]):
+            with self.assertRaisesRegex(M.TransferError, "origin-refused"):
+                M.Agent(self.init_message(), self.events.append, cwd=self.root)
+
+    def test_original_source_and_excess_concurrency_refused(self):
+        with patch.object(M, "SOURCE", self.root):
+            with self.assertRaisesRegex(M.TransferError, "scientific-source-cwd"):
+                M.Agent(self.init_message(), self.events.append, cwd=self.root)
+        message = self.init_message()
+        message["concurrenttransfers"] = 3
+        with self.assertRaisesRegex(M.TransferError, "concurrency"):
+            M.Agent(message, self.events.append, cwd=self.root)
+
+    def test_ssh_authentication_is_strict_and_operation_scoped(self):
+        response = {"href": BASE, "header": {"Authorization": "Bearer " + TOKEN}}
+        result = subprocess.CompletedProcess([], 0, stdout=json.dumps(response).encode(), stderr=b"")
+        with patch.object(M.subprocess, "run", return_value=result) as call:
+            href, headers = self.agent.authenticate()
+        args = call.call_args.args[0]
+        self.assertIn("BatchMode=yes", args)
+        self.assertIn("StrictHostKeyChecking=yes", args)
+        self.assertEqual(args[-3:], ["git-lfs-authenticate", "qishizhe/digital-drosophila.git", "upload"])
+        self.assertEqual(href, BASE)
+        self.assertIn(TOKEN, headers["Authorization"])
+        self.assertNotIn(TOKEN, json.dumps(args))
+
+    def test_curl_secrets_only_in_stdin_and_secure_defaults(self):
+        result = subprocess.CompletedProcess([], 0, stdout=b"{}\n200", stderr=b"")
+        with patch.object(M.subprocess, "run", return_value=result) as call:
+            body = self.agent.curl("PUT", BASE + "/objects/" + self.oid + "?signature=" + TOKEN,
+                                   {"Authorization": "Bearer " + TOKEN}, upload=self.source)
+        self.assertEqual(body, b"{}")
+        self.assertEqual(call.call_args.args[0], ["curl", "-q", "--config", "-"])
+        self.assertNotIn(TOKEN, json.dumps(call.call_args.args[0]))
+        config = call.call_args.kwargs["input"].decode()
+        for wanted in ('noproxy = "*"', M.HOST + ":443:" + M.IP, 'proto = "=https"',
+                       "max-redirs = 0", "Accept: application/vnd.git-lfs+json",
+                       "Content-Type: application/octet-stream", TOKEN):
+            self.assertIn(wanted, config)
+        self.assertNotIn("insecure", config)
+        self.assertNotIn("location", config)
+        self.assertNotIn("retry", config)
+
+    def test_redirect_or_curl_error_is_sanitized(self):
+        for rc, output in ((0, b"redirect body\n302"), (22, b"")):
+            result = subprocess.CompletedProcess([], rc, stdout=output, stderr=TOKEN.encode())
+            with patch.object(M.subprocess, "run", return_value=result):
+                with self.assertRaises(M.TransferError) as caught:
+                    self.agent.curl("GET", BASE + "/objects/" + self.oid, {})
+            self.assertNotIn(TOKEN, str(caught.exception))
+
+    def test_post_json_sets_both_gitea_headers(self):
+        result = subprocess.CompletedProcess([], 0, stdout=b"\n204", stderr=b"")
+        with patch.object(M.subprocess, "run", return_value=result) as call:
+            self.agent.curl("POST", BASE + "/verify", {}, body={"oid": self.oid, "size": self.size})
+        config = call.call_args.kwargs["input"].decode()
+        self.assertIn("Accept: application/vnd.git-lfs+json", config)
+        self.assertIn("Content-Type: application/vnd.git-lfs+json", config)
+
+    def test_batch_requests_basic_and_checks_exact_identity(self):
+        response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": self.action()}}]}
+        with patch.object(self.agent, "authenticate", return_value=(BASE, {"Authorization": TOKEN})), \
+                patch.object(self.agent, "curl", return_value=json.dumps(response).encode()) as call:
+            self.assertIn("upload", self.agent.batch(self.oid, self.size))
+        args, kwargs = call.call_args
+        self.assertEqual(args[1], BASE + "/objects/batch")
+        self.assertEqual(args[2]["Accept"], "application/vnd.git-lfs+json")
+        self.assertEqual(kwargs["body"]["transfers"], ["basic"])
+        response["objects"][0]["size"] += 1
+        with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
+                patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
+            with self.assertRaisesRegex(M.TransferError, "identity"):
+                self.agent.batch(self.oid, self.size)
+
+    def test_batch_action_host_rejected(self):
+        action = self.action()
+        action["href"] = action["href"].replace(M.HOST, "other.example")
+        response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": action}}]}
+        with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
+                patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
+            with self.assertRaisesRegex(M.TransferError, "authority"):
+                self.agent.batch(self.oid, self.size)
+
+    def test_upload_chunked_exact_value_allowed_in_action_and_put(self):
+        action = self.action()
+        action["header"]["Transfer-Encoding"] = "chunked"
+        response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": action}}]}
+        with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
+                patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
+            self.assertEqual(self.agent.batch(self.oid, self.size)["upload"]["header"]["Transfer-Encoding"], "chunked")
+        result = subprocess.CompletedProcess([], 0, stdout=b"\n200", stderr=b"")
+        with patch.object(M.subprocess, "run", return_value=result) as call:
+            self.agent.curl("PUT", action["href"], action["header"], upload=self.source)
+        config = call.call_args.kwargs["input"].decode()
+        self.assertIn('header = "Transfer-Encoding: chunked"', config)
+        self.assertIn('request = "PUT"', config)
+        self.assertIn("upload-file = ", config)
+
+    def test_chunked_forbidden_outside_upload_or_for_other_values(self):
+        for value in ("gzip", "CHUNKED", "chunked, gzip", "chunked\r\nX: y", " chunked"):
+            with self.subTest(value=value), self.assertRaises(M.TransferError):
+                self.agent.curl("PUT", BASE + "/objects/" + self.oid,
+                                {"Transfer-Encoding": value}, upload=self.source)
+        for method, kwargs in (("GET", {}), ("POST", {"body": {}}), ("PUT", {})):
+            with self.subTest(method=method, kwargs=kwargs), self.assertRaises(M.TransferError):
+                self.agent.curl(method, BASE + "/objects/" + self.oid,
+                                {"Transfer-Encoding": "chunked"}, **kwargs)
+        with self.assertRaises(M.TransferError):
+            M.safe_headers({"Transfer-Encoding": "chunked"})
+        for operation, name in (("upload", "verify"), ("download", "download")):
+            self.agent.operation = operation
+            action = self.action()
+            action["header"]["Transfer-Encoding"] = "chunked"
+            response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {name: action}}]}
+            with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
+                    patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
+                with self.subTest(operation=operation, action=name), self.assertRaises(M.TransferError):
+                    self.agent.batch(self.oid, self.size)
+
+    def test_upload_existing_object_needs_no_put(self):
+        with patch.object(self.agent, "batch", return_value={}) as batch, patch.object(self.agent, "curl") as curl:
+            result = self.agent.transfer(self.request())
+        self.assertEqual(result, {"event": "complete", "oid": self.oid})
+        batch.assert_called_once()
+        curl.assert_not_called()
+        self.assertEqual(self.events[-1]["bytesSoFar"], self.size)
+
+    def test_upload_put_then_verification_post(self):
+        actions = {"upload": self.action(), "verify": self.action("verify/")}
+        with patch.object(self.agent, "batch", return_value=actions), \
+                patch.object(self.agent, "curl", return_value=b"") as curl:
+            self.agent.transfer(self.request())
+        self.assertEqual([c.args[0] for c in curl.call_args_list], ["PUT", "POST"])
+        self.assertEqual(curl.call_args_list[1].kwargs["body"], {"oid": self.oid, "size": self.size})
+
+    def test_upload_wrong_sha_fails_before_network(self):
+        self.source.write_bytes(b"x" * self.size)
+        with patch.object(self.agent, "batch") as batch:
+            with self.assertRaisesRegex(M.TransferError, "sha"):
+                self.agent.transfer(self.request())
+        batch.assert_not_called()
+
+    def test_upload_outside_lfs_store_refused(self):
+        other = self.root / self.oid
+        other.write_bytes(self.data)
+        with self.assertRaisesRegex(M.TransferError, "outside-current-lfs"):
+            self.agent.transfer(self.request(path=str(other)))
+
+    def test_upload_source_mutation_detected(self):
+        def change(*args, **kwargs):
+            self.source.write_bytes(b"changed")
+        with patch.object(self.agent, "batch", return_value={"upload": self.action()}), \
+                patch.object(self.agent, "curl", side_effect=change):
+            with self.assertRaisesRegex(M.TransferError, "upload-source-changed"):
+                self.agent.transfer(self.request())
+
+    def test_download_is_fresh_unique_and_verified(self):
+        self.agent.operation = "download"
+        def fetch(*args, **kwargs):
+            kwargs["download"].write_bytes(self.data)
+            return b""
+        with patch.object(self.agent, "batch", return_value={"download": self.action()}), \
+                patch.object(self.agent, "curl", side_effect=fetch) as curl:
+            first = self.agent.transfer(self.request("download"))
+            second = self.agent.transfer(self.request("download"))
+        self.assertNotEqual(first["path"], second["path"])
+        self.assertEqual(Path(first["path"]).read_bytes(), self.data)
+        self.assertEqual(curl.call_count, 2)
+        self.assertEqual(self.source.read_bytes(), self.data)
+
+    def test_download_hash_failure_retains_partial(self):
+        self.agent.operation = "download"
+        def fetch(*args, **kwargs):
+            kwargs["download"].write_bytes(b"x" * self.size)
+        with patch.object(self.agent, "batch", return_value={"download": self.action()}), \
+                patch.object(self.agent, "curl", side_effect=fetch):
+            with self.assertRaisesRegex(M.TransferError, "sha"):
+                self.agent.transfer(self.request("download"))
+        partials = list((self.git / "lfs" / "lan-transfer-tmp").glob("*.partial"))
+        self.assertEqual(len(partials), 1)
+        self.assertEqual(partials[0].read_bytes(), b"x" * self.size)
+
+    def test_protocol_init_progress_completion_terminate(self):
+        requests = [self.init_message(), self.request(), {"event": "terminate"}]
+        incoming = io.StringIO("".join(json.dumps(x) + "\n" for x in requests))
+        with patch.object(self.agent, "batch", return_value={}):
+            code = M.serve(incoming, self.events.append, factory=lambda message, send: self.agent)
+        self.assertEqual(code, 0)
+        self.assertEqual(self.events[0], {})
+        self.assertEqual(self.events[-1], {"event": "complete", "oid": self.oid})
+
+    def test_transfer_error_continues_and_never_echoes_exception_secret(self):
+        incoming = io.StringIO("\n".join(json.dumps(x) for x in
+                              [self.init_message(), self.request(), self.request(), {"event": "terminate"}]) + "\n")
+        with patch.object(self.agent, "transfer", side_effect=[RuntimeError(TOKEN), {"event": "complete", "oid": self.oid}]):
+            code = M.serve(incoming, self.events.append, factory=lambda message, send: self.agent)
+        self.assertEqual(code, 0)
+        self.assertIn("error", self.events[1])
+        self.assertNotIn(TOKEN, json.dumps(self.events))
+        self.assertNotIn("error", self.events[2])
+
+    def test_bad_initialization_uses_init_error_schema(self):
+        incoming = io.StringIO(json.dumps(self.init_message()) + "\n")
+        with patch.object(M, "git_output", side_effect=RuntimeError(TOKEN)):
+            code = M.serve(incoming, self.events.append)
+        self.assertEqual(code, 1)
+        self.assertEqual(self.events[0]["error"]["code"], 32)
+        self.assertNotIn(TOKEN, json.dumps(self.events))
+
+
+if __name__ == "__main__":
+    unittest.main()