"""Repository-local, LAN-pinned Git LFS standalone transfer agent. Protocol: https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md Only fixed-origin SSH authentication and HTTPS basic transfers are supported. No config/network changes, retries, redirects, credential files, or cache-based download substitution. Credentials are passed to curl exclusively on stdin. """ from __future__ import annotations import hashlib import json import os from pathlib import Path import re import stat import subprocess import sys import tempfile from urllib.parse import urlsplit ORIGIN = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git" HOST = "git.lain.asia" IP = "192.168.10.31" PREFIX = "/qishizhe/digital-drosophila.git/info/lfs" SOURCE = Path("/data/digital-drosophila") OID = re.compile(r"^[0-9a-f]{64}$") HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") MAX_JSON = 2 * 1024 * 1024 BLOCK = 1024 * 1024 class TransferError(Exception): """Only fixed, non-secret messages may be passed to this class.""" def require(ok, code): if not ok: raise TransferError(code) def decode_json(data): def unique(pairs): result = {} for key, value in pairs: require(key not in result, "duplicate-json-key") result[key] = value return result require(len(data) <= MAX_JSON, "json-response-too-large") try: return json.loads(data, object_pairs_hook=unique) except (UnicodeError, ValueError): raise TransferError("invalid-json") from None def safe_path(value): require(isinstance(value, (str, Path)), "invalid-file-path") p = Path(value) require(p.is_absolute() and ".." not in p.parts, "absolute-safe-path-required") for component in (p, *p.parents): if component.exists() or component.is_symlink(): s = component.lstat() require(not stat.S_ISLNK(s.st_mode) and not (getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)), "symlink-or-reparse-refused") return p def safe_url(value): require(isinstance(value, str) and len(value) <= 16384 and not any(ord(c) <= 32 or ord(c) == 127 for c in value), "invalid-action-url") try: p = urlsplit(value) valid = (p.scheme == "https" and p.netloc in (HOST, HOST + ":443") and p.hostname == HOST and p.port in (None, 443) and p.username is None and p.password is None and not p.fragment) except ValueError: raise TransferError("url-authority-refused") from None require(valid, "url-authority-refused") require(p.path == PREFIX or p.path.startswith(PREFIX + "/"), "url-repository-prefix-refused") require("%" not in p.path and "\\" not in p.path and all(x not in ("", ".", "..") for x in p.path.split("/")[1:]), "url-path-refused") return value def safe_headers(value, *, allow_upload_chunked=False): require(isinstance(value, dict) and len(value) <= 32, "invalid-action-headers") result, seen = {}, set() denied = {"host", "connection", "content-length", "transfer-encoding", "te", "trailer", "upgrade", "proxy-authorization", "proxy-connection", "range"} for key, item in value.items(): upload_chunked = (allow_upload_chunked is True and isinstance(key, str) and key.lower() == "transfer-encoding" and item == "chunked") require(isinstance(key, str) and HEADER.fullmatch(key) is not None and key.lower() not in seen and (key.lower() not in denied or upload_chunked), "header-name-refused") require(isinstance(item, str) and len(item) <= 16384 and not any(ord(c) < 32 or ord(c) == 127 for c in item), "header-value-refused") seen.add(key.lower()) result[key] = item return result def quote(value): require(isinstance(value, str) and not any(ord(c) < 32 or ord(c) == 127 for c in value), "curl-config-value-refused") return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' def fingerprint(s): return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns) def verify_file(path, oid, size): path = safe_path(path) before = fingerprint(path.lstat()) require(stat.S_ISREG(before[2]) and before[3] == size, "object-file-type-or-size-mismatch") digest, count = hashlib.sha256(), 0 with path.open("rb") as handle: opened = fingerprint(os.fstat(handle.fileno())) # Windows path stat and fstat can expose different ctime definitions. require(opened[:-1] == before[:-1] if os.name == "nt" else opened == before, "object-changed-at-open") while data := handle.read(BLOCK): digest.update(data) count += len(data) require(fingerprint(os.fstat(handle.fileno())) == opened, "object-changed-during-read") require(fingerprint(path.lstat()) == before and count == size and digest.hexdigest() == oid, "object-sha-or-stability-mismatch") return before def git_output(cwd, *args): try: result = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, timeout=15, check=False) require(result.returncode == 0, "git-inspection-failed") return result.stdout.decode("utf-8").strip() except (OSError, UnicodeError, subprocess.TimeoutExpired): raise TransferError("git-inspection-failed") from None class Agent: def __init__(self, message, emit, cwd=None): require(message.get("event") == "init" and message.get("operation") in ("upload", "download"), "invalid-init") require(message.get("remote") in ("origin", ORIGIN), "remote-refused") concurrency = message.get("concurrenttransfers") require(type(concurrency) is int and 1 <= concurrency <= 2 and type(message.get("concurrent")) is bool, "concurrency-must-be-at-most-two") cwd = safe_path(Path.cwd() if cwd is None else cwd) require(cwd != SOURCE and SOURCE not in cwd.parents, "scientific-source-cwd-refused") self.root = safe_path(git_output(cwd, "rev-parse", "--show-toplevel")) require(self.root != SOURCE and SOURCE not in self.root.parents, "scientific-source-repository-refused") require(git_output(cwd, "config", "--get", "remote.origin.url") == ORIGIN, "origin-refused") self.git_dir = safe_path(git_output(cwd, "rev-parse", "--absolute-git-dir")) require(self.git_dir == self.root / ".git" and self.git_dir.is_dir(), "ordinary-isolated-clone-required") self.operation, self.emit = message["operation"], emit def authenticate(self): command = ["ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", "-o", "ConnectionAttempts=1", "-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=2", "-p", "2222", "git@192.168.10.31", "git-lfs-authenticate", "qishizhe/digital-drosophila.git", self.operation] try: result = subprocess.run(command, capture_output=True, timeout=45, check=False) require(result.returncode == 0, "ssh-authentication-failed") auth = decode_json(result.stdout) except (OSError, subprocess.TimeoutExpired): raise TransferError("ssh-authentication-failed") from None require(isinstance(auth, dict), "invalid-ssh-authentication-response") href = safe_url(auth.get("href")) parsed = urlsplit(href) require(parsed.path == PREFIX and not parsed.query, "authentication-base-url-refused") return href, safe_headers(auth.get("header", {})) def curl(self, method, url, headers, *, body=None, upload=None, download=None, size=None): safe_url(url) headers = safe_headers(headers, allow_upload_chunked=(self.operation == "upload" and method == "PUT" and upload is not None and body is None and download is None)) require(method in ("GET", "POST", "PUT"), "unsupported-http-method") present = {name.lower() for name in headers} if "accept" not in present: headers["Accept"] = "application/vnd.git-lfs+json" if method == "PUT" and "content-type" not in present: headers["Content-Type"] = "application/octet-stream" if body is not None and "content-type" not in present: headers["Content-Type"] = "application/vnd.git-lfs+json" lines = ["url = " + quote(url), "request = " + quote(method), 'noproxy = "*"', 'resolve = "' + HOST + ':443:' + IP + '"', 'proto = "=https"', 'proto-redir = "=https"', "max-redirs = 0", "connect-timeout = 15", "max-time = 1800", "fail", "silent", "show-error", 'write-out = "\\n%{http_code}"'] for name, value in headers.items(): lines.append("header = " + quote(name + ": " + value)) if body is not None: encoded = json.dumps(body, separators=(",", ":"), allow_nan=False) lines.append("data-binary = " + quote(encoded)) if upload is not None: lines.append("upload-file = " + quote(str(safe_path(upload)))) if download is not None: lines.append("output = " + quote(str(safe_path(download)))) require(type(size) is int and size >= 0, "invalid-download-limit") # A zero-byte object still needs a full HTTP fetch, not a cache hit. lines.append("max-filesize = " + str(max(size, 1))) else: lines.append("max-filesize = " + str(MAX_JSON)) config = ("\n".join(lines) + "\n").encode("utf-8") try: result = subprocess.run(["curl", "-q", "--config", "-"], input=config, capture_output=True, timeout=1830, check=False) except (OSError, subprocess.TimeoutExpired): raise TransferError("curl-execution-failed") from None # Never print stderr, response data, URLs, configuration, or headers. require(result.returncode == 0, "curl-request-failed") require(len(result.stdout) <= MAX_JSON + 32, "http-response-too-large") content, separator, status = result.stdout.rpartition(b"\n") require(separator and len(status) == 3 and status.isdigit() and 200 <= int(status) < 300, "non-success-http-status") return content def batch(self, oid, size): href, headers = self.authenticate() headers = {k: v for k, v in headers.items() if k.lower() not in ("content-type", "accept")} headers.update({"Content-Type": "application/vnd.git-lfs+json", "Accept": "application/vnd.git-lfs+json"}) response = decode_json(self.curl("POST", href + "/objects/batch", headers, body={"operation": self.operation, "transfers": ["basic"], "objects": [{"oid": oid, "size": size}]})) require(isinstance(response, dict) and response.get("transfer", "basic") == "basic", "non-basic-batch-response") objects = response.get("objects") require(isinstance(objects, list) and len(objects) == 1 and isinstance(objects[0], dict), "invalid-batch-object-count") item = objects[0] require(item.get("oid") == oid and type(item.get("size")) is int and item["size"] == size, "batch-object-identity-mismatch") require("error" not in item, "batch-object-rejected") actions = item.get("actions", {}) require(isinstance(actions, dict), "invalid-batch-actions") allowed = {"upload", "verify"} if self.operation == "upload" else {"download"} require(set(actions) <= allowed, "unexpected-batch-action") for name, action in actions.items(): require(isinstance(action, dict), "invalid-batch-action") safe_url(action.get("href")) safe_headers(action.get("header", {}), allow_upload_chunked=(self.operation == "upload" and name == "upload")) return actions def progress(self, oid, size, previous=0): self.emit({"event": "progress", "oid": oid, "bytesSoFar": size, "bytesSinceLast": size - previous}) def transfer(self, message): require(message.get("event") == self.operation and message.get("action") is None, "standalone-operation-required") oid, size = message.get("oid"), message.get("size") require(isinstance(oid, str) and OID.fullmatch(oid) is not None and type(size) is int and 0 <= size <= 2**63 - 1, "invalid-object-identity") source, before = None, None if self.operation == "upload": source = safe_path(message.get("path")) require(source == self.git_dir / "lfs" / "objects" / oid[:2] / oid[2:4] / oid, "upload-outside-current-lfs-object-store") before = verify_file(source, oid, size) actions = self.batch(oid, size) self.progress(oid, 0) if self.operation == "upload": if "upload" in actions: action = actions["upload"] self.curl("PUT", action["href"], action.get("header", {}), upload=source) require(fingerprint(source.lstat()) == before, "upload-source-changed") if "verify" in actions: action = actions["verify"] headers = {k: v for k, v in action.get("header", {}).items() if k.lower() != "content-type"} headers["Content-Type"] = "application/vnd.git-lfs+json" self.curl("POST", action["href"], headers, body={"oid": oid, "size": size}) else: require(not actions, "verify-without-upload-refused") require(fingerprint(source.lstat()) == before, "upload-source-changed") self.progress(oid, size) return {"event": "complete", "oid": oid} require("download" in actions, "download-action-missing") directory = safe_path(self.git_dir / "lfs" / "lan-transfer-tmp") directory.mkdir(parents=True, exist_ok=True) safe_path(directory) fd, name = tempfile.mkstemp(prefix=oid + ".", suffix=".partial", dir=directory) os.close(fd) target = Path(name) action = actions["download"] # On failure the uniquely created partial remains for investigation; # it is never used as a fallback/cache input on a later transfer. self.curl("GET", action["href"], action.get("header", {}), download=target, size=size) verify_file(target, oid, size) self.progress(oid, size) return {"event": "complete", "oid": oid, "path": str(target)} def emit(value): sys.stdout.write(json.dumps(value, separators=(",", ":"), allow_nan=False) + "\n") sys.stdout.flush() def serve(inp=None, send=None, factory=Agent): inp, send = sys.stdin if inp is None else inp, emit if send is None else send agent = None while True: line = inp.readline(MAX_JSON + 1) if not line: return 0 try: message = decode_json(line) require(isinstance(message, dict), "protocol-object-required") if agent is None: agent = factory(message, send) send({}) elif message.get("event") == "terminate": return 0 else: oid = message.get("oid") safe_oid = oid if isinstance(oid, str) and OID.fullmatch(oid) else "" try: send(agent.transfer(message)) except Exception as exc: code = str(exc) if isinstance(exc, TransferError) else "transfer-internal-failure" send({"event": "complete", "oid": safe_oid, "error": {"code": 2, "message": code}}) except Exception as exc: code = str(exc) if isinstance(exc, TransferError) else "agent-initialization-or-protocol-failure" if agent is None: send({"error": {"code": 32, "message": code}}) else: # No attacker-supplied raw line or exception text is printed. sys.stderr.write("lan-lfs: invalid protocol; stopping\n") return 1 if __name__ == "__main__": raise SystemExit(serve())