Preserve CPU and GPU acceleration plans, evidence and LFS archives; mark research paused
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,336 @@
|
||||
"""Repository-local, LAN-pinned Git LFS standalone transfer agent.
|
||||
|
||||
Protocol: https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md
|
||||
Only fixed-origin SSH authentication and HTTPS basic transfers are supported.
|
||||
No config/network changes, retries, redirects, credential files, or cache-based
|
||||
download substitution. Credentials are passed to curl exclusively on stdin.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ORIGIN = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
|
||||
HOST = "git.lain.asia"
|
||||
IP = "192.168.10.31"
|
||||
PREFIX = "/qishizhe/digital-drosophila.git/info/lfs"
|
||||
SOURCE = Path("/data/digital-drosophila")
|
||||
OID = re.compile(r"^[0-9a-f]{64}$")
|
||||
HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
|
||||
MAX_JSON = 2 * 1024 * 1024
|
||||
BLOCK = 1024 * 1024
|
||||
|
||||
|
||||
class TransferError(Exception):
|
||||
"""Only fixed, non-secret messages may be passed to this class."""
|
||||
|
||||
|
||||
def require(ok, code):
|
||||
if not ok:
|
||||
raise TransferError(code)
|
||||
|
||||
|
||||
def decode_json(data):
|
||||
def unique(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
require(key not in result, "duplicate-json-key")
|
||||
result[key] = value
|
||||
return result
|
||||
require(len(data) <= MAX_JSON, "json-response-too-large")
|
||||
try:
|
||||
return json.loads(data, object_pairs_hook=unique)
|
||||
except (UnicodeError, ValueError):
|
||||
raise TransferError("invalid-json") from None
|
||||
|
||||
|
||||
def safe_path(value):
|
||||
require(isinstance(value, (str, Path)), "invalid-file-path")
|
||||
p = Path(value)
|
||||
require(p.is_absolute() and ".." not in p.parts, "absolute-safe-path-required")
|
||||
for component in (p, *p.parents):
|
||||
if component.exists() or component.is_symlink():
|
||||
s = component.lstat()
|
||||
require(not stat.S_ISLNK(s.st_mode) and not
|
||||
(getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)),
|
||||
"symlink-or-reparse-refused")
|
||||
return p
|
||||
|
||||
|
||||
def safe_url(value):
|
||||
require(isinstance(value, str) and len(value) <= 16384 and
|
||||
not any(ord(c) <= 32 or ord(c) == 127 for c in value), "invalid-action-url")
|
||||
try:
|
||||
p = urlsplit(value)
|
||||
valid = (p.scheme == "https" and p.netloc in (HOST, HOST + ":443") and
|
||||
p.hostname == HOST and p.port in (None, 443) and
|
||||
p.username is None and p.password is None and not p.fragment)
|
||||
except ValueError:
|
||||
raise TransferError("url-authority-refused") from None
|
||||
require(valid, "url-authority-refused")
|
||||
require(p.path == PREFIX or p.path.startswith(PREFIX + "/"), "url-repository-prefix-refused")
|
||||
require("%" not in p.path and "\\" not in p.path and
|
||||
all(x not in ("", ".", "..") for x in p.path.split("/")[1:]), "url-path-refused")
|
||||
return value
|
||||
|
||||
|
||||
def safe_headers(value, *, allow_upload_chunked=False):
|
||||
require(isinstance(value, dict) and len(value) <= 32, "invalid-action-headers")
|
||||
result, seen = {}, set()
|
||||
denied = {"host", "connection", "content-length", "transfer-encoding", "te",
|
||||
"trailer", "upgrade", "proxy-authorization", "proxy-connection", "range"}
|
||||
for key, item in value.items():
|
||||
upload_chunked = (allow_upload_chunked is True and isinstance(key, str) and
|
||||
key.lower() == "transfer-encoding" and item == "chunked")
|
||||
require(isinstance(key, str) and HEADER.fullmatch(key) is not None and
|
||||
key.lower() not in seen and (key.lower() not in denied or upload_chunked),
|
||||
"header-name-refused")
|
||||
require(isinstance(item, str) and len(item) <= 16384 and
|
||||
not any(ord(c) < 32 or ord(c) == 127 for c in item), "header-value-refused")
|
||||
seen.add(key.lower())
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def quote(value):
|
||||
require(isinstance(value, str) and not any(ord(c) < 32 or ord(c) == 127 for c in value),
|
||||
"curl-config-value-refused")
|
||||
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def fingerprint(s):
|
||||
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||||
|
||||
|
||||
def verify_file(path, oid, size):
|
||||
path = safe_path(path)
|
||||
before = fingerprint(path.lstat())
|
||||
require(stat.S_ISREG(before[2]) and before[3] == size, "object-file-type-or-size-mismatch")
|
||||
digest, count = hashlib.sha256(), 0
|
||||
with path.open("rb") as handle:
|
||||
opened = fingerprint(os.fstat(handle.fileno()))
|
||||
# Windows path stat and fstat can expose different ctime definitions.
|
||||
require(opened[:-1] == before[:-1] if os.name == "nt" else opened == before,
|
||||
"object-changed-at-open")
|
||||
while data := handle.read(BLOCK):
|
||||
digest.update(data)
|
||||
count += len(data)
|
||||
require(fingerprint(os.fstat(handle.fileno())) == opened, "object-changed-during-read")
|
||||
require(fingerprint(path.lstat()) == before and count == size and digest.hexdigest() == oid,
|
||||
"object-sha-or-stability-mismatch")
|
||||
return before
|
||||
|
||||
|
||||
def git_output(cwd, *args):
|
||||
try:
|
||||
result = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True,
|
||||
timeout=15, check=False)
|
||||
require(result.returncode == 0, "git-inspection-failed")
|
||||
return result.stdout.decode("utf-8").strip()
|
||||
except (OSError, UnicodeError, subprocess.TimeoutExpired):
|
||||
raise TransferError("git-inspection-failed") from None
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, message, emit, cwd=None):
|
||||
require(message.get("event") == "init" and message.get("operation") in ("upload", "download"),
|
||||
"invalid-init")
|
||||
require(message.get("remote") in ("origin", ORIGIN), "remote-refused")
|
||||
concurrency = message.get("concurrenttransfers")
|
||||
require(type(concurrency) is int and 1 <= concurrency <= 2 and
|
||||
type(message.get("concurrent")) is bool, "concurrency-must-be-at-most-two")
|
||||
cwd = safe_path(Path.cwd() if cwd is None else cwd)
|
||||
require(cwd != SOURCE and SOURCE not in cwd.parents, "scientific-source-cwd-refused")
|
||||
self.root = safe_path(git_output(cwd, "rev-parse", "--show-toplevel"))
|
||||
require(self.root != SOURCE and SOURCE not in self.root.parents, "scientific-source-repository-refused")
|
||||
require(git_output(cwd, "config", "--get", "remote.origin.url") == ORIGIN, "origin-refused")
|
||||
self.git_dir = safe_path(git_output(cwd, "rev-parse", "--absolute-git-dir"))
|
||||
require(self.git_dir == self.root / ".git" and self.git_dir.is_dir(), "ordinary-isolated-clone-required")
|
||||
self.operation, self.emit = message["operation"], emit
|
||||
|
||||
def authenticate(self):
|
||||
command = ["ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "ConnectTimeout=15", "-o", "ConnectionAttempts=1",
|
||||
"-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=2",
|
||||
"-p", "2222", "git@192.168.10.31", "git-lfs-authenticate",
|
||||
"qishizhe/digital-drosophila.git", self.operation]
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, timeout=45, check=False)
|
||||
require(result.returncode == 0, "ssh-authentication-failed")
|
||||
auth = decode_json(result.stdout)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise TransferError("ssh-authentication-failed") from None
|
||||
require(isinstance(auth, dict), "invalid-ssh-authentication-response")
|
||||
href = safe_url(auth.get("href"))
|
||||
parsed = urlsplit(href)
|
||||
require(parsed.path == PREFIX and not parsed.query, "authentication-base-url-refused")
|
||||
return href, safe_headers(auth.get("header", {}))
|
||||
|
||||
def curl(self, method, url, headers, *, body=None, upload=None, download=None, size=None):
|
||||
safe_url(url)
|
||||
headers = safe_headers(headers, allow_upload_chunked=(self.operation == "upload" and
|
||||
method == "PUT" and upload is not None and body is None and download is None))
|
||||
require(method in ("GET", "POST", "PUT"), "unsupported-http-method")
|
||||
present = {name.lower() for name in headers}
|
||||
if "accept" not in present:
|
||||
headers["Accept"] = "application/vnd.git-lfs+json"
|
||||
if method == "PUT" and "content-type" not in present:
|
||||
headers["Content-Type"] = "application/octet-stream"
|
||||
if body is not None and "content-type" not in present:
|
||||
headers["Content-Type"] = "application/vnd.git-lfs+json"
|
||||
lines = ["url = " + quote(url), "request = " + quote(method),
|
||||
'noproxy = "*"', 'resolve = "' + HOST + ':443:' + IP + '"',
|
||||
'proto = "=https"', 'proto-redir = "=https"', "max-redirs = 0",
|
||||
"connect-timeout = 15", "max-time = 1800", "fail", "silent", "show-error",
|
||||
'write-out = "\\n%{http_code}"']
|
||||
for name, value in headers.items():
|
||||
lines.append("header = " + quote(name + ": " + value))
|
||||
if body is not None:
|
||||
encoded = json.dumps(body, separators=(",", ":"), allow_nan=False)
|
||||
lines.append("data-binary = " + quote(encoded))
|
||||
if upload is not None:
|
||||
lines.append("upload-file = " + quote(str(safe_path(upload))))
|
||||
if download is not None:
|
||||
lines.append("output = " + quote(str(safe_path(download))))
|
||||
require(type(size) is int and size >= 0, "invalid-download-limit")
|
||||
# A zero-byte object still needs a full HTTP fetch, not a cache hit.
|
||||
lines.append("max-filesize = " + str(max(size, 1)))
|
||||
else:
|
||||
lines.append("max-filesize = " + str(MAX_JSON))
|
||||
config = ("\n".join(lines) + "\n").encode("utf-8")
|
||||
try:
|
||||
result = subprocess.run(["curl", "-q", "--config", "-"], input=config,
|
||||
capture_output=True, timeout=1830, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise TransferError("curl-execution-failed") from None
|
||||
# Never print stderr, response data, URLs, configuration, or headers.
|
||||
require(result.returncode == 0, "curl-request-failed")
|
||||
require(len(result.stdout) <= MAX_JSON + 32, "http-response-too-large")
|
||||
content, separator, status = result.stdout.rpartition(b"\n")
|
||||
require(separator and len(status) == 3 and status.isdigit() and 200 <= int(status) < 300,
|
||||
"non-success-http-status")
|
||||
return content
|
||||
|
||||
def batch(self, oid, size):
|
||||
href, headers = self.authenticate()
|
||||
headers = {k: v for k, v in headers.items() if k.lower() not in ("content-type", "accept")}
|
||||
headers.update({"Content-Type": "application/vnd.git-lfs+json", "Accept": "application/vnd.git-lfs+json"})
|
||||
response = decode_json(self.curl("POST", href + "/objects/batch", headers,
|
||||
body={"operation": self.operation, "transfers": ["basic"],
|
||||
"objects": [{"oid": oid, "size": size}]}))
|
||||
require(isinstance(response, dict) and response.get("transfer", "basic") == "basic",
|
||||
"non-basic-batch-response")
|
||||
objects = response.get("objects")
|
||||
require(isinstance(objects, list) and len(objects) == 1 and isinstance(objects[0], dict),
|
||||
"invalid-batch-object-count")
|
||||
item = objects[0]
|
||||
require(item.get("oid") == oid and type(item.get("size")) is int and item["size"] == size,
|
||||
"batch-object-identity-mismatch")
|
||||
require("error" not in item, "batch-object-rejected")
|
||||
actions = item.get("actions", {})
|
||||
require(isinstance(actions, dict), "invalid-batch-actions")
|
||||
allowed = {"upload", "verify"} if self.operation == "upload" else {"download"}
|
||||
require(set(actions) <= allowed, "unexpected-batch-action")
|
||||
for name, action in actions.items():
|
||||
require(isinstance(action, dict), "invalid-batch-action")
|
||||
safe_url(action.get("href"))
|
||||
safe_headers(action.get("header", {}), allow_upload_chunked=(self.operation == "upload" and name == "upload"))
|
||||
return actions
|
||||
|
||||
def progress(self, oid, size, previous=0):
|
||||
self.emit({"event": "progress", "oid": oid, "bytesSoFar": size, "bytesSinceLast": size - previous})
|
||||
|
||||
def transfer(self, message):
|
||||
require(message.get("event") == self.operation and message.get("action") is None,
|
||||
"standalone-operation-required")
|
||||
oid, size = message.get("oid"), message.get("size")
|
||||
require(isinstance(oid, str) and OID.fullmatch(oid) is not None and
|
||||
type(size) is int and 0 <= size <= 2**63 - 1, "invalid-object-identity")
|
||||
source, before = None, None
|
||||
if self.operation == "upload":
|
||||
source = safe_path(message.get("path"))
|
||||
require(source == self.git_dir / "lfs" / "objects" / oid[:2] / oid[2:4] / oid,
|
||||
"upload-outside-current-lfs-object-store")
|
||||
before = verify_file(source, oid, size)
|
||||
actions = self.batch(oid, size)
|
||||
self.progress(oid, 0)
|
||||
if self.operation == "upload":
|
||||
if "upload" in actions:
|
||||
action = actions["upload"]
|
||||
self.curl("PUT", action["href"], action.get("header", {}), upload=source)
|
||||
require(fingerprint(source.lstat()) == before, "upload-source-changed")
|
||||
if "verify" in actions:
|
||||
action = actions["verify"]
|
||||
headers = {k: v for k, v in action.get("header", {}).items() if k.lower() != "content-type"}
|
||||
headers["Content-Type"] = "application/vnd.git-lfs+json"
|
||||
self.curl("POST", action["href"], headers, body={"oid": oid, "size": size})
|
||||
else:
|
||||
require(not actions, "verify-without-upload-refused")
|
||||
require(fingerprint(source.lstat()) == before, "upload-source-changed")
|
||||
self.progress(oid, size)
|
||||
return {"event": "complete", "oid": oid}
|
||||
require("download" in actions, "download-action-missing")
|
||||
directory = safe_path(self.git_dir / "lfs" / "lan-transfer-tmp")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
safe_path(directory)
|
||||
fd, name = tempfile.mkstemp(prefix=oid + ".", suffix=".partial", dir=directory)
|
||||
os.close(fd)
|
||||
target = Path(name)
|
||||
action = actions["download"]
|
||||
# On failure the uniquely created partial remains for investigation;
|
||||
# it is never used as a fallback/cache input on a later transfer.
|
||||
self.curl("GET", action["href"], action.get("header", {}), download=target, size=size)
|
||||
verify_file(target, oid, size)
|
||||
self.progress(oid, size)
|
||||
return {"event": "complete", "oid": oid, "path": str(target)}
|
||||
|
||||
|
||||
def emit(value):
|
||||
sys.stdout.write(json.dumps(value, separators=(",", ":"), allow_nan=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def serve(inp=None, send=None, factory=Agent):
|
||||
inp, send = sys.stdin if inp is None else inp, emit if send is None else send
|
||||
agent = None
|
||||
while True:
|
||||
line = inp.readline(MAX_JSON + 1)
|
||||
if not line:
|
||||
return 0
|
||||
try:
|
||||
message = decode_json(line)
|
||||
require(isinstance(message, dict), "protocol-object-required")
|
||||
if agent is None:
|
||||
agent = factory(message, send)
|
||||
send({})
|
||||
elif message.get("event") == "terminate":
|
||||
return 0
|
||||
else:
|
||||
oid = message.get("oid")
|
||||
safe_oid = oid if isinstance(oid, str) and OID.fullmatch(oid) else ""
|
||||
try:
|
||||
send(agent.transfer(message))
|
||||
except Exception as exc:
|
||||
code = str(exc) if isinstance(exc, TransferError) else "transfer-internal-failure"
|
||||
send({"event": "complete", "oid": safe_oid,
|
||||
"error": {"code": 2, "message": code}})
|
||||
except Exception as exc:
|
||||
code = str(exc) if isinstance(exc, TransferError) else "agent-initialization-or-protocol-failure"
|
||||
if agent is None:
|
||||
send({"error": {"code": 32, "message": code}})
|
||||
else:
|
||||
# No attacker-supplied raw line or exception text is printed.
|
||||
sys.stderr.write("lan-lfs: invalid protocol; stopping\n")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(serve())
|
||||
Reference in New Issue
Block a user