Digital Drosophila Archive
337 lines
16 KiB
Python
337 lines
16 KiB
Python
"""Repository-local, LAN-pinned Git LFS standalone transfer agent.
|
|
|
|
Protocol: https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md
|
|
Only fixed-origin SSH authentication and HTTPS basic transfers are supported.
|
|
No config/network changes, retries, redirects, credential files, or cache-based
|
|
download substitution. Credentials are passed to curl exclusively on stdin.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
|
|
ORIGIN = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
|
|
HOST = "git.lain.asia"
|
|
IP = "192.168.10.31"
|
|
PREFIX = "/qishizhe/digital-drosophila.git/info/lfs"
|
|
SOURCE = Path("/data/digital-drosophila")
|
|
OID = re.compile(r"^[0-9a-f]{64}$")
|
|
HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
|
|
MAX_JSON = 2 * 1024 * 1024
|
|
BLOCK = 1024 * 1024
|
|
|
|
|
|
class TransferError(Exception):
|
|
"""Only fixed, non-secret messages may be passed to this class."""
|
|
|
|
|
|
def require(ok, code):
|
|
if not ok:
|
|
raise TransferError(code)
|
|
|
|
|
|
def decode_json(data):
|
|
def unique(pairs):
|
|
result = {}
|
|
for key, value in pairs:
|
|
require(key not in result, "duplicate-json-key")
|
|
result[key] = value
|
|
return result
|
|
require(len(data) <= MAX_JSON, "json-response-too-large")
|
|
try:
|
|
return json.loads(data, object_pairs_hook=unique)
|
|
except (UnicodeError, ValueError):
|
|
raise TransferError("invalid-json") from None
|
|
|
|
|
|
def safe_path(value):
|
|
require(isinstance(value, (str, Path)), "invalid-file-path")
|
|
p = Path(value)
|
|
require(p.is_absolute() and ".." not in p.parts, "absolute-safe-path-required")
|
|
for component in (p, *p.parents):
|
|
if component.exists() or component.is_symlink():
|
|
s = component.lstat()
|
|
require(not stat.S_ISLNK(s.st_mode) and not
|
|
(getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)),
|
|
"symlink-or-reparse-refused")
|
|
return p
|
|
|
|
|
|
def safe_url(value):
|
|
require(isinstance(value, str) and len(value) <= 16384 and
|
|
not any(ord(c) <= 32 or ord(c) == 127 for c in value), "invalid-action-url")
|
|
try:
|
|
p = urlsplit(value)
|
|
valid = (p.scheme == "https" and p.netloc in (HOST, HOST + ":443") and
|
|
p.hostname == HOST and p.port in (None, 443) and
|
|
p.username is None and p.password is None and not p.fragment)
|
|
except ValueError:
|
|
raise TransferError("url-authority-refused") from None
|
|
require(valid, "url-authority-refused")
|
|
require(p.path == PREFIX or p.path.startswith(PREFIX + "/"), "url-repository-prefix-refused")
|
|
require("%" not in p.path and "\\" not in p.path and
|
|
all(x not in ("", ".", "..") for x in p.path.split("/")[1:]), "url-path-refused")
|
|
return value
|
|
|
|
|
|
def safe_headers(value, *, allow_upload_chunked=False):
|
|
require(isinstance(value, dict) and len(value) <= 32, "invalid-action-headers")
|
|
result, seen = {}, set()
|
|
denied = {"host", "connection", "content-length", "transfer-encoding", "te",
|
|
"trailer", "upgrade", "proxy-authorization", "proxy-connection", "range"}
|
|
for key, item in value.items():
|
|
upload_chunked = (allow_upload_chunked is True and isinstance(key, str) and
|
|
key.lower() == "transfer-encoding" and item == "chunked")
|
|
require(isinstance(key, str) and HEADER.fullmatch(key) is not None and
|
|
key.lower() not in seen and (key.lower() not in denied or upload_chunked),
|
|
"header-name-refused")
|
|
require(isinstance(item, str) and len(item) <= 16384 and
|
|
not any(ord(c) < 32 or ord(c) == 127 for c in item), "header-value-refused")
|
|
seen.add(key.lower())
|
|
result[key] = item
|
|
return result
|
|
|
|
|
|
def quote(value):
|
|
require(isinstance(value, str) and not any(ord(c) < 32 or ord(c) == 127 for c in value),
|
|
"curl-config-value-refused")
|
|
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
|
|
|
|
def fingerprint(s):
|
|
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
|
|
|
|
|
def verify_file(path, oid, size):
|
|
path = safe_path(path)
|
|
before = fingerprint(path.lstat())
|
|
require(stat.S_ISREG(before[2]) and before[3] == size, "object-file-type-or-size-mismatch")
|
|
digest, count = hashlib.sha256(), 0
|
|
with path.open("rb") as handle:
|
|
opened = fingerprint(os.fstat(handle.fileno()))
|
|
# Windows path stat and fstat can expose different ctime definitions.
|
|
require(opened[:-1] == before[:-1] if os.name == "nt" else opened == before,
|
|
"object-changed-at-open")
|
|
while data := handle.read(BLOCK):
|
|
digest.update(data)
|
|
count += len(data)
|
|
require(fingerprint(os.fstat(handle.fileno())) == opened, "object-changed-during-read")
|
|
require(fingerprint(path.lstat()) == before and count == size and digest.hexdigest() == oid,
|
|
"object-sha-or-stability-mismatch")
|
|
return before
|
|
|
|
|
|
def git_output(cwd, *args):
|
|
try:
|
|
result = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True,
|
|
timeout=15, check=False)
|
|
require(result.returncode == 0, "git-inspection-failed")
|
|
return result.stdout.decode("utf-8").strip()
|
|
except (OSError, UnicodeError, subprocess.TimeoutExpired):
|
|
raise TransferError("git-inspection-failed") from None
|
|
|
|
|
|
class Agent:
|
|
def __init__(self, message, emit, cwd=None):
|
|
require(message.get("event") == "init" and message.get("operation") in ("upload", "download"),
|
|
"invalid-init")
|
|
require(message.get("remote") in ("origin", ORIGIN), "remote-refused")
|
|
concurrency = message.get("concurrenttransfers")
|
|
require(type(concurrency) is int and 1 <= concurrency <= 2 and
|
|
type(message.get("concurrent")) is bool, "concurrency-must-be-at-most-two")
|
|
cwd = safe_path(Path.cwd() if cwd is None else cwd)
|
|
require(cwd != SOURCE and SOURCE not in cwd.parents, "scientific-source-cwd-refused")
|
|
self.root = safe_path(git_output(cwd, "rev-parse", "--show-toplevel"))
|
|
require(self.root != SOURCE and SOURCE not in self.root.parents, "scientific-source-repository-refused")
|
|
require(git_output(cwd, "config", "--get", "remote.origin.url") == ORIGIN, "origin-refused")
|
|
self.git_dir = safe_path(git_output(cwd, "rev-parse", "--absolute-git-dir"))
|
|
require(self.git_dir == self.root / ".git" and self.git_dir.is_dir(), "ordinary-isolated-clone-required")
|
|
self.operation, self.emit = message["operation"], emit
|
|
|
|
def authenticate(self):
|
|
command = ["ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
|
|
"-o", "ConnectTimeout=15", "-o", "ConnectionAttempts=1",
|
|
"-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=2",
|
|
"-p", "2222", "git@192.168.10.31", "git-lfs-authenticate",
|
|
"qishizhe/digital-drosophila.git", self.operation]
|
|
try:
|
|
result = subprocess.run(command, capture_output=True, timeout=45, check=False)
|
|
require(result.returncode == 0, "ssh-authentication-failed")
|
|
auth = decode_json(result.stdout)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise TransferError("ssh-authentication-failed") from None
|
|
require(isinstance(auth, dict), "invalid-ssh-authentication-response")
|
|
href = safe_url(auth.get("href"))
|
|
parsed = urlsplit(href)
|
|
require(parsed.path == PREFIX and not parsed.query, "authentication-base-url-refused")
|
|
return href, safe_headers(auth.get("header", {}))
|
|
|
|
def curl(self, method, url, headers, *, body=None, upload=None, download=None, size=None):
|
|
safe_url(url)
|
|
headers = safe_headers(headers, allow_upload_chunked=(self.operation == "upload" and
|
|
method == "PUT" and upload is not None and body is None and download is None))
|
|
require(method in ("GET", "POST", "PUT"), "unsupported-http-method")
|
|
present = {name.lower() for name in headers}
|
|
if "accept" not in present:
|
|
headers["Accept"] = "application/vnd.git-lfs+json"
|
|
if method == "PUT" and "content-type" not in present:
|
|
headers["Content-Type"] = "application/octet-stream"
|
|
if body is not None and "content-type" not in present:
|
|
headers["Content-Type"] = "application/vnd.git-lfs+json"
|
|
lines = ["url = " + quote(url), "request = " + quote(method),
|
|
'noproxy = "*"', 'resolve = "' + HOST + ':443:' + IP + '"',
|
|
'proto = "=https"', 'proto-redir = "=https"', "max-redirs = 0",
|
|
"connect-timeout = 15", "max-time = 1800", "fail", "silent", "show-error",
|
|
'write-out = "\\n%{http_code}"']
|
|
for name, value in headers.items():
|
|
lines.append("header = " + quote(name + ": " + value))
|
|
if body is not None:
|
|
encoded = json.dumps(body, separators=(",", ":"), allow_nan=False)
|
|
lines.append("data-binary = " + quote(encoded))
|
|
if upload is not None:
|
|
lines.append("upload-file = " + quote(str(safe_path(upload))))
|
|
if download is not None:
|
|
lines.append("output = " + quote(str(safe_path(download))))
|
|
require(type(size) is int and size >= 0, "invalid-download-limit")
|
|
# A zero-byte object still needs a full HTTP fetch, not a cache hit.
|
|
lines.append("max-filesize = " + str(max(size, 1)))
|
|
else:
|
|
lines.append("max-filesize = " + str(MAX_JSON))
|
|
config = ("\n".join(lines) + "\n").encode("utf-8")
|
|
try:
|
|
result = subprocess.run(["curl", "-q", "--config", "-"], input=config,
|
|
capture_output=True, timeout=1830, check=False)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
raise TransferError("curl-execution-failed") from None
|
|
# Never print stderr, response data, URLs, configuration, or headers.
|
|
require(result.returncode == 0, "curl-request-failed")
|
|
require(len(result.stdout) <= MAX_JSON + 32, "http-response-too-large")
|
|
content, separator, status = result.stdout.rpartition(b"\n")
|
|
require(separator and len(status) == 3 and status.isdigit() and 200 <= int(status) < 300,
|
|
"non-success-http-status")
|
|
return content
|
|
|
|
def batch(self, oid, size):
|
|
href, headers = self.authenticate()
|
|
headers = {k: v for k, v in headers.items() if k.lower() not in ("content-type", "accept")}
|
|
headers.update({"Content-Type": "application/vnd.git-lfs+json", "Accept": "application/vnd.git-lfs+json"})
|
|
response = decode_json(self.curl("POST", href + "/objects/batch", headers,
|
|
body={"operation": self.operation, "transfers": ["basic"],
|
|
"objects": [{"oid": oid, "size": size}]}))
|
|
require(isinstance(response, dict) and response.get("transfer", "basic") == "basic",
|
|
"non-basic-batch-response")
|
|
objects = response.get("objects")
|
|
require(isinstance(objects, list) and len(objects) == 1 and isinstance(objects[0], dict),
|
|
"invalid-batch-object-count")
|
|
item = objects[0]
|
|
require(item.get("oid") == oid and type(item.get("size")) is int and item["size"] == size,
|
|
"batch-object-identity-mismatch")
|
|
require("error" not in item, "batch-object-rejected")
|
|
actions = item.get("actions", {})
|
|
require(isinstance(actions, dict), "invalid-batch-actions")
|
|
allowed = {"upload", "verify"} if self.operation == "upload" else {"download"}
|
|
require(set(actions) <= allowed, "unexpected-batch-action")
|
|
for name, action in actions.items():
|
|
require(isinstance(action, dict), "invalid-batch-action")
|
|
safe_url(action.get("href"))
|
|
safe_headers(action.get("header", {}), allow_upload_chunked=(self.operation == "upload" and name == "upload"))
|
|
return actions
|
|
|
|
def progress(self, oid, size, previous=0):
|
|
self.emit({"event": "progress", "oid": oid, "bytesSoFar": size, "bytesSinceLast": size - previous})
|
|
|
|
def transfer(self, message):
|
|
require(message.get("event") == self.operation and message.get("action") is None,
|
|
"standalone-operation-required")
|
|
oid, size = message.get("oid"), message.get("size")
|
|
require(isinstance(oid, str) and OID.fullmatch(oid) is not None and
|
|
type(size) is int and 0 <= size <= 2**63 - 1, "invalid-object-identity")
|
|
source, before = None, None
|
|
if self.operation == "upload":
|
|
source = safe_path(message.get("path"))
|
|
require(source == self.git_dir / "lfs" / "objects" / oid[:2] / oid[2:4] / oid,
|
|
"upload-outside-current-lfs-object-store")
|
|
before = verify_file(source, oid, size)
|
|
actions = self.batch(oid, size)
|
|
self.progress(oid, 0)
|
|
if self.operation == "upload":
|
|
if "upload" in actions:
|
|
action = actions["upload"]
|
|
self.curl("PUT", action["href"], action.get("header", {}), upload=source)
|
|
require(fingerprint(source.lstat()) == before, "upload-source-changed")
|
|
if "verify" in actions:
|
|
action = actions["verify"]
|
|
headers = {k: v for k, v in action.get("header", {}).items() if k.lower() != "content-type"}
|
|
headers["Content-Type"] = "application/vnd.git-lfs+json"
|
|
self.curl("POST", action["href"], headers, body={"oid": oid, "size": size})
|
|
else:
|
|
require(not actions, "verify-without-upload-refused")
|
|
require(fingerprint(source.lstat()) == before, "upload-source-changed")
|
|
self.progress(oid, size)
|
|
return {"event": "complete", "oid": oid}
|
|
require("download" in actions, "download-action-missing")
|
|
directory = safe_path(self.git_dir / "lfs" / "lan-transfer-tmp")
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
safe_path(directory)
|
|
fd, name = tempfile.mkstemp(prefix=oid + ".", suffix=".partial", dir=directory)
|
|
os.close(fd)
|
|
target = Path(name)
|
|
action = actions["download"]
|
|
# On failure the uniquely created partial remains for investigation;
|
|
# it is never used as a fallback/cache input on a later transfer.
|
|
self.curl("GET", action["href"], action.get("header", {}), download=target, size=size)
|
|
verify_file(target, oid, size)
|
|
self.progress(oid, size)
|
|
return {"event": "complete", "oid": oid, "path": str(target)}
|
|
|
|
|
|
def emit(value):
|
|
sys.stdout.write(json.dumps(value, separators=(",", ":"), allow_nan=False) + "\n")
|
|
sys.stdout.flush()
|
|
|
|
|
|
def serve(inp=None, send=None, factory=Agent):
|
|
inp, send = sys.stdin if inp is None else inp, emit if send is None else send
|
|
agent = None
|
|
while True:
|
|
line = inp.readline(MAX_JSON + 1)
|
|
if not line:
|
|
return 0
|
|
try:
|
|
message = decode_json(line)
|
|
require(isinstance(message, dict), "protocol-object-required")
|
|
if agent is None:
|
|
agent = factory(message, send)
|
|
send({})
|
|
elif message.get("event") == "terminate":
|
|
return 0
|
|
else:
|
|
oid = message.get("oid")
|
|
safe_oid = oid if isinstance(oid, str) and OID.fullmatch(oid) else ""
|
|
try:
|
|
send(agent.transfer(message))
|
|
except Exception as exc:
|
|
code = str(exc) if isinstance(exc, TransferError) else "transfer-internal-failure"
|
|
send({"event": "complete", "oid": safe_oid,
|
|
"error": {"code": 2, "message": code}})
|
|
except Exception as exc:
|
|
code = str(exc) if isinstance(exc, TransferError) else "agent-initialization-or-protocol-failure"
|
|
if agent is None:
|
|
send({"error": {"code": 32, "message": code}})
|
|
else:
|
|
# No attacker-supplied raw line or exception text is printed.
|
|
sys.stderr.write("lan-lfs: invalid protocol; stopping\n")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(serve())
|