Preserve CPU and GPU acceleration plans, evidence and LFS archives; mark research paused
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,336 @@
|
||||
"""Repository-local, LAN-pinned Git LFS standalone transfer agent.
|
||||
|
||||
Protocol: https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md
|
||||
Only fixed-origin SSH authentication and HTTPS basic transfers are supported.
|
||||
No config/network changes, retries, redirects, credential files, or cache-based
|
||||
download substitution. Credentials are passed to curl exclusively on stdin.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ORIGIN = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
|
||||
HOST = "git.lain.asia"
|
||||
IP = "192.168.10.31"
|
||||
PREFIX = "/qishizhe/digital-drosophila.git/info/lfs"
|
||||
SOURCE = Path("/data/digital-drosophila")
|
||||
OID = re.compile(r"^[0-9a-f]{64}$")
|
||||
HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
|
||||
MAX_JSON = 2 * 1024 * 1024
|
||||
BLOCK = 1024 * 1024
|
||||
|
||||
|
||||
class TransferError(Exception):
|
||||
"""Only fixed, non-secret messages may be passed to this class."""
|
||||
|
||||
|
||||
def require(ok, code):
|
||||
if not ok:
|
||||
raise TransferError(code)
|
||||
|
||||
|
||||
def decode_json(data):
|
||||
def unique(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
require(key not in result, "duplicate-json-key")
|
||||
result[key] = value
|
||||
return result
|
||||
require(len(data) <= MAX_JSON, "json-response-too-large")
|
||||
try:
|
||||
return json.loads(data, object_pairs_hook=unique)
|
||||
except (UnicodeError, ValueError):
|
||||
raise TransferError("invalid-json") from None
|
||||
|
||||
|
||||
def safe_path(value):
|
||||
require(isinstance(value, (str, Path)), "invalid-file-path")
|
||||
p = Path(value)
|
||||
require(p.is_absolute() and ".." not in p.parts, "absolute-safe-path-required")
|
||||
for component in (p, *p.parents):
|
||||
if component.exists() or component.is_symlink():
|
||||
s = component.lstat()
|
||||
require(not stat.S_ISLNK(s.st_mode) and not
|
||||
(getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)),
|
||||
"symlink-or-reparse-refused")
|
||||
return p
|
||||
|
||||
|
||||
def safe_url(value):
|
||||
require(isinstance(value, str) and len(value) <= 16384 and
|
||||
not any(ord(c) <= 32 or ord(c) == 127 for c in value), "invalid-action-url")
|
||||
try:
|
||||
p = urlsplit(value)
|
||||
valid = (p.scheme == "https" and p.netloc in (HOST, HOST + ":443") and
|
||||
p.hostname == HOST and p.port in (None, 443) and
|
||||
p.username is None and p.password is None and not p.fragment)
|
||||
except ValueError:
|
||||
raise TransferError("url-authority-refused") from None
|
||||
require(valid, "url-authority-refused")
|
||||
require(p.path == PREFIX or p.path.startswith(PREFIX + "/"), "url-repository-prefix-refused")
|
||||
require("%" not in p.path and "\\" not in p.path and
|
||||
all(x not in ("", ".", "..") for x in p.path.split("/")[1:]), "url-path-refused")
|
||||
return value
|
||||
|
||||
|
||||
def safe_headers(value, *, allow_upload_chunked=False):
|
||||
require(isinstance(value, dict) and len(value) <= 32, "invalid-action-headers")
|
||||
result, seen = {}, set()
|
||||
denied = {"host", "connection", "content-length", "transfer-encoding", "te",
|
||||
"trailer", "upgrade", "proxy-authorization", "proxy-connection", "range"}
|
||||
for key, item in value.items():
|
||||
upload_chunked = (allow_upload_chunked is True and isinstance(key, str) and
|
||||
key.lower() == "transfer-encoding" and item == "chunked")
|
||||
require(isinstance(key, str) and HEADER.fullmatch(key) is not None and
|
||||
key.lower() not in seen and (key.lower() not in denied or upload_chunked),
|
||||
"header-name-refused")
|
||||
require(isinstance(item, str) and len(item) <= 16384 and
|
||||
not any(ord(c) < 32 or ord(c) == 127 for c in item), "header-value-refused")
|
||||
seen.add(key.lower())
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def quote(value):
|
||||
require(isinstance(value, str) and not any(ord(c) < 32 or ord(c) == 127 for c in value),
|
||||
"curl-config-value-refused")
|
||||
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def fingerprint(s):
|
||||
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||||
|
||||
|
||||
def verify_file(path, oid, size):
|
||||
path = safe_path(path)
|
||||
before = fingerprint(path.lstat())
|
||||
require(stat.S_ISREG(before[2]) and before[3] == size, "object-file-type-or-size-mismatch")
|
||||
digest, count = hashlib.sha256(), 0
|
||||
with path.open("rb") as handle:
|
||||
opened = fingerprint(os.fstat(handle.fileno()))
|
||||
# Windows path stat and fstat can expose different ctime definitions.
|
||||
require(opened[:-1] == before[:-1] if os.name == "nt" else opened == before,
|
||||
"object-changed-at-open")
|
||||
while data := handle.read(BLOCK):
|
||||
digest.update(data)
|
||||
count += len(data)
|
||||
require(fingerprint(os.fstat(handle.fileno())) == opened, "object-changed-during-read")
|
||||
require(fingerprint(path.lstat()) == before and count == size and digest.hexdigest() == oid,
|
||||
"object-sha-or-stability-mismatch")
|
||||
return before
|
||||
|
||||
|
||||
def git_output(cwd, *args):
|
||||
try:
|
||||
result = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True,
|
||||
timeout=15, check=False)
|
||||
require(result.returncode == 0, "git-inspection-failed")
|
||||
return result.stdout.decode("utf-8").strip()
|
||||
except (OSError, UnicodeError, subprocess.TimeoutExpired):
|
||||
raise TransferError("git-inspection-failed") from None
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, message, emit, cwd=None):
|
||||
require(message.get("event") == "init" and message.get("operation") in ("upload", "download"),
|
||||
"invalid-init")
|
||||
require(message.get("remote") in ("origin", ORIGIN), "remote-refused")
|
||||
concurrency = message.get("concurrenttransfers")
|
||||
require(type(concurrency) is int and 1 <= concurrency <= 2 and
|
||||
type(message.get("concurrent")) is bool, "concurrency-must-be-at-most-two")
|
||||
cwd = safe_path(Path.cwd() if cwd is None else cwd)
|
||||
require(cwd != SOURCE and SOURCE not in cwd.parents, "scientific-source-cwd-refused")
|
||||
self.root = safe_path(git_output(cwd, "rev-parse", "--show-toplevel"))
|
||||
require(self.root != SOURCE and SOURCE not in self.root.parents, "scientific-source-repository-refused")
|
||||
require(git_output(cwd, "config", "--get", "remote.origin.url") == ORIGIN, "origin-refused")
|
||||
self.git_dir = safe_path(git_output(cwd, "rev-parse", "--absolute-git-dir"))
|
||||
require(self.git_dir == self.root / ".git" and self.git_dir.is_dir(), "ordinary-isolated-clone-required")
|
||||
self.operation, self.emit = message["operation"], emit
|
||||
|
||||
def authenticate(self):
|
||||
command = ["ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "ConnectTimeout=15", "-o", "ConnectionAttempts=1",
|
||||
"-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=2",
|
||||
"-p", "2222", "git@192.168.10.31", "git-lfs-authenticate",
|
||||
"qishizhe/digital-drosophila.git", self.operation]
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, timeout=45, check=False)
|
||||
require(result.returncode == 0, "ssh-authentication-failed")
|
||||
auth = decode_json(result.stdout)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise TransferError("ssh-authentication-failed") from None
|
||||
require(isinstance(auth, dict), "invalid-ssh-authentication-response")
|
||||
href = safe_url(auth.get("href"))
|
||||
parsed = urlsplit(href)
|
||||
require(parsed.path == PREFIX and not parsed.query, "authentication-base-url-refused")
|
||||
return href, safe_headers(auth.get("header", {}))
|
||||
|
||||
def curl(self, method, url, headers, *, body=None, upload=None, download=None, size=None):
|
||||
safe_url(url)
|
||||
headers = safe_headers(headers, allow_upload_chunked=(self.operation == "upload" and
|
||||
method == "PUT" and upload is not None and body is None and download is None))
|
||||
require(method in ("GET", "POST", "PUT"), "unsupported-http-method")
|
||||
present = {name.lower() for name in headers}
|
||||
if "accept" not in present:
|
||||
headers["Accept"] = "application/vnd.git-lfs+json"
|
||||
if method == "PUT" and "content-type" not in present:
|
||||
headers["Content-Type"] = "application/octet-stream"
|
||||
if body is not None and "content-type" not in present:
|
||||
headers["Content-Type"] = "application/vnd.git-lfs+json"
|
||||
lines = ["url = " + quote(url), "request = " + quote(method),
|
||||
'noproxy = "*"', 'resolve = "' + HOST + ':443:' + IP + '"',
|
||||
'proto = "=https"', 'proto-redir = "=https"', "max-redirs = 0",
|
||||
"connect-timeout = 15", "max-time = 1800", "fail", "silent", "show-error",
|
||||
'write-out = "\\n%{http_code}"']
|
||||
for name, value in headers.items():
|
||||
lines.append("header = " + quote(name + ": " + value))
|
||||
if body is not None:
|
||||
encoded = json.dumps(body, separators=(",", ":"), allow_nan=False)
|
||||
lines.append("data-binary = " + quote(encoded))
|
||||
if upload is not None:
|
||||
lines.append("upload-file = " + quote(str(safe_path(upload))))
|
||||
if download is not None:
|
||||
lines.append("output = " + quote(str(safe_path(download))))
|
||||
require(type(size) is int and size >= 0, "invalid-download-limit")
|
||||
# A zero-byte object still needs a full HTTP fetch, not a cache hit.
|
||||
lines.append("max-filesize = " + str(max(size, 1)))
|
||||
else:
|
||||
lines.append("max-filesize = " + str(MAX_JSON))
|
||||
config = ("\n".join(lines) + "\n").encode("utf-8")
|
||||
try:
|
||||
result = subprocess.run(["curl", "-q", "--config", "-"], input=config,
|
||||
capture_output=True, timeout=1830, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise TransferError("curl-execution-failed") from None
|
||||
# Never print stderr, response data, URLs, configuration, or headers.
|
||||
require(result.returncode == 0, "curl-request-failed")
|
||||
require(len(result.stdout) <= MAX_JSON + 32, "http-response-too-large")
|
||||
content, separator, status = result.stdout.rpartition(b"\n")
|
||||
require(separator and len(status) == 3 and status.isdigit() and 200 <= int(status) < 300,
|
||||
"non-success-http-status")
|
||||
return content
|
||||
|
||||
def batch(self, oid, size):
|
||||
href, headers = self.authenticate()
|
||||
headers = {k: v for k, v in headers.items() if k.lower() not in ("content-type", "accept")}
|
||||
headers.update({"Content-Type": "application/vnd.git-lfs+json", "Accept": "application/vnd.git-lfs+json"})
|
||||
response = decode_json(self.curl("POST", href + "/objects/batch", headers,
|
||||
body={"operation": self.operation, "transfers": ["basic"],
|
||||
"objects": [{"oid": oid, "size": size}]}))
|
||||
require(isinstance(response, dict) and response.get("transfer", "basic") == "basic",
|
||||
"non-basic-batch-response")
|
||||
objects = response.get("objects")
|
||||
require(isinstance(objects, list) and len(objects) == 1 and isinstance(objects[0], dict),
|
||||
"invalid-batch-object-count")
|
||||
item = objects[0]
|
||||
require(item.get("oid") == oid and type(item.get("size")) is int and item["size"] == size,
|
||||
"batch-object-identity-mismatch")
|
||||
require("error" not in item, "batch-object-rejected")
|
||||
actions = item.get("actions", {})
|
||||
require(isinstance(actions, dict), "invalid-batch-actions")
|
||||
allowed = {"upload", "verify"} if self.operation == "upload" else {"download"}
|
||||
require(set(actions) <= allowed, "unexpected-batch-action")
|
||||
for name, action in actions.items():
|
||||
require(isinstance(action, dict), "invalid-batch-action")
|
||||
safe_url(action.get("href"))
|
||||
safe_headers(action.get("header", {}), allow_upload_chunked=(self.operation == "upload" and name == "upload"))
|
||||
return actions
|
||||
|
||||
def progress(self, oid, size, previous=0):
|
||||
self.emit({"event": "progress", "oid": oid, "bytesSoFar": size, "bytesSinceLast": size - previous})
|
||||
|
||||
def transfer(self, message):
|
||||
require(message.get("event") == self.operation and message.get("action") is None,
|
||||
"standalone-operation-required")
|
||||
oid, size = message.get("oid"), message.get("size")
|
||||
require(isinstance(oid, str) and OID.fullmatch(oid) is not None and
|
||||
type(size) is int and 0 <= size <= 2**63 - 1, "invalid-object-identity")
|
||||
source, before = None, None
|
||||
if self.operation == "upload":
|
||||
source = safe_path(message.get("path"))
|
||||
require(source == self.git_dir / "lfs" / "objects" / oid[:2] / oid[2:4] / oid,
|
||||
"upload-outside-current-lfs-object-store")
|
||||
before = verify_file(source, oid, size)
|
||||
actions = self.batch(oid, size)
|
||||
self.progress(oid, 0)
|
||||
if self.operation == "upload":
|
||||
if "upload" in actions:
|
||||
action = actions["upload"]
|
||||
self.curl("PUT", action["href"], action.get("header", {}), upload=source)
|
||||
require(fingerprint(source.lstat()) == before, "upload-source-changed")
|
||||
if "verify" in actions:
|
||||
action = actions["verify"]
|
||||
headers = {k: v for k, v in action.get("header", {}).items() if k.lower() != "content-type"}
|
||||
headers["Content-Type"] = "application/vnd.git-lfs+json"
|
||||
self.curl("POST", action["href"], headers, body={"oid": oid, "size": size})
|
||||
else:
|
||||
require(not actions, "verify-without-upload-refused")
|
||||
require(fingerprint(source.lstat()) == before, "upload-source-changed")
|
||||
self.progress(oid, size)
|
||||
return {"event": "complete", "oid": oid}
|
||||
require("download" in actions, "download-action-missing")
|
||||
directory = safe_path(self.git_dir / "lfs" / "lan-transfer-tmp")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
safe_path(directory)
|
||||
fd, name = tempfile.mkstemp(prefix=oid + ".", suffix=".partial", dir=directory)
|
||||
os.close(fd)
|
||||
target = Path(name)
|
||||
action = actions["download"]
|
||||
# On failure the uniquely created partial remains for investigation;
|
||||
# it is never used as a fallback/cache input on a later transfer.
|
||||
self.curl("GET", action["href"], action.get("header", {}), download=target, size=size)
|
||||
verify_file(target, oid, size)
|
||||
self.progress(oid, size)
|
||||
return {"event": "complete", "oid": oid, "path": str(target)}
|
||||
|
||||
|
||||
def emit(value):
|
||||
sys.stdout.write(json.dumps(value, separators=(",", ":"), allow_nan=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def serve(inp=None, send=None, factory=Agent):
|
||||
inp, send = sys.stdin if inp is None else inp, emit if send is None else send
|
||||
agent = None
|
||||
while True:
|
||||
line = inp.readline(MAX_JSON + 1)
|
||||
if not line:
|
||||
return 0
|
||||
try:
|
||||
message = decode_json(line)
|
||||
require(isinstance(message, dict), "protocol-object-required")
|
||||
if agent is None:
|
||||
agent = factory(message, send)
|
||||
send({})
|
||||
elif message.get("event") == "terminate":
|
||||
return 0
|
||||
else:
|
||||
oid = message.get("oid")
|
||||
safe_oid = oid if isinstance(oid, str) and OID.fullmatch(oid) else ""
|
||||
try:
|
||||
send(agent.transfer(message))
|
||||
except Exception as exc:
|
||||
code = str(exc) if isinstance(exc, TransferError) else "transfer-internal-failure"
|
||||
send({"event": "complete", "oid": safe_oid,
|
||||
"error": {"code": 2, "message": code}})
|
||||
except Exception as exc:
|
||||
code = str(exc) if isinstance(exc, TransferError) else "agent-initialization-or-protocol-failure"
|
||||
if agent is None:
|
||||
send({"error": {"code": 32, "message": code}})
|
||||
else:
|
||||
# No attacker-supplied raw line or exception text is printed.
|
||||
sys.stderr.write("lan-lfs: invalid protocol; stopping\n")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(serve())
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Pure temporary/mocked protocol tests: never SSH, curl, GPU, or real Git."""
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import lan_lfs_transfer as M
|
||||
|
||||
BASE = "https://" + M.HOST + M.PREFIX
|
||||
TOKEN = "synthetic-fixture-" + "credential-value"
|
||||
|
||||
|
||||
class AdapterTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name) / "clone"
|
||||
self.git = self.root / ".git"
|
||||
self.git.mkdir(parents=True)
|
||||
self.data = b"synthetic content\x00\xff\n"
|
||||
self.oid = hashlib.sha256(self.data).hexdigest()
|
||||
self.size = len(self.data)
|
||||
self.events = []
|
||||
self.agent = object.__new__(M.Agent)
|
||||
self.agent.root, self.agent.git_dir = self.root, self.git
|
||||
self.agent.operation, self.agent.emit = "upload", self.events.append
|
||||
self.source = self.git / "lfs" / "objects" / self.oid[:2] / self.oid[2:4] / self.oid
|
||||
self.source.parent.mkdir(parents=True)
|
||||
self.source.write_bytes(self.data)
|
||||
|
||||
def request(self, operation="upload", **extra):
|
||||
value = {"event": operation, "oid": self.oid, "size": self.size, "action": None}
|
||||
if operation == "upload":
|
||||
value["path"] = str(self.source)
|
||||
value.update(extra)
|
||||
return value
|
||||
|
||||
def init_message(self, **extra):
|
||||
return dict(event="init", operation="upload", remote="origin", concurrent=True,
|
||||
concurrenttransfers=2, **extra)
|
||||
|
||||
def action(self, suffix="objects/" ):
|
||||
return {"href": BASE + "/" + suffix + self.oid,
|
||||
"header": {"Authorization": "Bearer " + TOKEN}}
|
||||
|
||||
def test_strict_url_policy(self):
|
||||
self.assertEqual(M.safe_url(BASE + "/objects/" + self.oid + "?signed=opaque"),
|
||||
BASE + "/objects/" + self.oid + "?signed=opaque")
|
||||
invalid = [BASE.replace("https:", "http:"), BASE.replace(M.HOST, M.IP),
|
||||
BASE.replace(M.HOST, "other.example"), BASE.replace(M.HOST, M.HOST + ":444"),
|
||||
BASE.replace(M.HOST, "user:opaque@" + M.HOST), BASE + "-other/objects/x",
|
||||
BASE + "/../other", BASE + "/%2e%2e/other", BASE + "/objects/x#fragment",
|
||||
BASE + "//objects/x", BASE + "/objects\\x", BASE + "\n"]
|
||||
for url in invalid:
|
||||
with self.subTest(url=url), self.assertRaises(M.TransferError):
|
||||
M.safe_url(url)
|
||||
|
||||
def test_headers_reject_injection_and_routing_overrides(self):
|
||||
for headers in ({"Authorization": "x\r\nHost: bad"}, {"Host": "bad"},
|
||||
{"Authorization": "a", "authorization": "b"},
|
||||
{"Range": "bytes=1-"}, {"bad\nkey": "x"}, {"X": "\x00"}):
|
||||
with self.subTest(headers=headers), self.assertRaises(M.TransferError):
|
||||
M.safe_headers(headers)
|
||||
|
||||
def test_init_checks_origin_and_isolated_git_directory(self):
|
||||
values = [str(self.root), M.ORIGIN, str(self.git)]
|
||||
with patch.object(M, "git_output", side_effect=values):
|
||||
actual = M.Agent(self.init_message(), self.events.append, cwd=self.root)
|
||||
self.assertEqual(actual.git_dir, self.git)
|
||||
with patch.object(M, "git_output", side_effect=[str(self.root), "ssh://other/repo"]):
|
||||
with self.assertRaisesRegex(M.TransferError, "origin-refused"):
|
||||
M.Agent(self.init_message(), self.events.append, cwd=self.root)
|
||||
|
||||
def test_original_source_and_excess_concurrency_refused(self):
|
||||
with patch.object(M, "SOURCE", self.root):
|
||||
with self.assertRaisesRegex(M.TransferError, "scientific-source-cwd"):
|
||||
M.Agent(self.init_message(), self.events.append, cwd=self.root)
|
||||
message = self.init_message()
|
||||
message["concurrenttransfers"] = 3
|
||||
with self.assertRaisesRegex(M.TransferError, "concurrency"):
|
||||
M.Agent(message, self.events.append, cwd=self.root)
|
||||
|
||||
def test_ssh_authentication_is_strict_and_operation_scoped(self):
|
||||
response = {"href": BASE, "header": {"Authorization": "Bearer " + TOKEN}}
|
||||
result = subprocess.CompletedProcess([], 0, stdout=json.dumps(response).encode(), stderr=b"")
|
||||
with patch.object(M.subprocess, "run", return_value=result) as call:
|
||||
href, headers = self.agent.authenticate()
|
||||
args = call.call_args.args[0]
|
||||
self.assertIn("BatchMode=yes", args)
|
||||
self.assertIn("StrictHostKeyChecking=yes", args)
|
||||
self.assertEqual(args[-3:], ["git-lfs-authenticate", "qishizhe/digital-drosophila.git", "upload"])
|
||||
self.assertEqual(href, BASE)
|
||||
self.assertIn(TOKEN, headers["Authorization"])
|
||||
self.assertNotIn(TOKEN, json.dumps(args))
|
||||
|
||||
def test_curl_secrets_only_in_stdin_and_secure_defaults(self):
|
||||
result = subprocess.CompletedProcess([], 0, stdout=b"{}\n200", stderr=b"")
|
||||
with patch.object(M.subprocess, "run", return_value=result) as call:
|
||||
body = self.agent.curl("PUT", BASE + "/objects/" + self.oid + "?signature=" + TOKEN,
|
||||
{"Authorization": "Bearer " + TOKEN}, upload=self.source)
|
||||
self.assertEqual(body, b"{}")
|
||||
self.assertEqual(call.call_args.args[0], ["curl", "-q", "--config", "-"])
|
||||
self.assertNotIn(TOKEN, json.dumps(call.call_args.args[0]))
|
||||
config = call.call_args.kwargs["input"].decode()
|
||||
for wanted in ('noproxy = "*"', M.HOST + ":443:" + M.IP, 'proto = "=https"',
|
||||
"max-redirs = 0", "Accept: application/vnd.git-lfs+json",
|
||||
"Content-Type: application/octet-stream", TOKEN):
|
||||
self.assertIn(wanted, config)
|
||||
self.assertNotIn("insecure", config)
|
||||
self.assertNotIn("location", config)
|
||||
self.assertNotIn("retry", config)
|
||||
|
||||
def test_redirect_or_curl_error_is_sanitized(self):
|
||||
for rc, output in ((0, b"redirect body\n302"), (22, b"")):
|
||||
result = subprocess.CompletedProcess([], rc, stdout=output, stderr=TOKEN.encode())
|
||||
with patch.object(M.subprocess, "run", return_value=result):
|
||||
with self.assertRaises(M.TransferError) as caught:
|
||||
self.agent.curl("GET", BASE + "/objects/" + self.oid, {})
|
||||
self.assertNotIn(TOKEN, str(caught.exception))
|
||||
|
||||
def test_post_json_sets_both_gitea_headers(self):
|
||||
result = subprocess.CompletedProcess([], 0, stdout=b"\n204", stderr=b"")
|
||||
with patch.object(M.subprocess, "run", return_value=result) as call:
|
||||
self.agent.curl("POST", BASE + "/verify", {}, body={"oid": self.oid, "size": self.size})
|
||||
config = call.call_args.kwargs["input"].decode()
|
||||
self.assertIn("Accept: application/vnd.git-lfs+json", config)
|
||||
self.assertIn("Content-Type: application/vnd.git-lfs+json", config)
|
||||
|
||||
def test_batch_requests_basic_and_checks_exact_identity(self):
|
||||
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": self.action()}}]}
|
||||
with patch.object(self.agent, "authenticate", return_value=(BASE, {"Authorization": TOKEN})), \
|
||||
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()) as call:
|
||||
self.assertIn("upload", self.agent.batch(self.oid, self.size))
|
||||
args, kwargs = call.call_args
|
||||
self.assertEqual(args[1], BASE + "/objects/batch")
|
||||
self.assertEqual(args[2]["Accept"], "application/vnd.git-lfs+json")
|
||||
self.assertEqual(kwargs["body"]["transfers"], ["basic"])
|
||||
response["objects"][0]["size"] += 1
|
||||
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
|
||||
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
|
||||
with self.assertRaisesRegex(M.TransferError, "identity"):
|
||||
self.agent.batch(self.oid, self.size)
|
||||
|
||||
def test_batch_action_host_rejected(self):
|
||||
action = self.action()
|
||||
action["href"] = action["href"].replace(M.HOST, "other.example")
|
||||
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": action}}]}
|
||||
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
|
||||
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
|
||||
with self.assertRaisesRegex(M.TransferError, "authority"):
|
||||
self.agent.batch(self.oid, self.size)
|
||||
|
||||
def test_upload_chunked_exact_value_allowed_in_action_and_put(self):
|
||||
action = self.action()
|
||||
action["header"]["Transfer-Encoding"] = "chunked"
|
||||
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {"upload": action}}]}
|
||||
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
|
||||
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
|
||||
self.assertEqual(self.agent.batch(self.oid, self.size)["upload"]["header"]["Transfer-Encoding"], "chunked")
|
||||
result = subprocess.CompletedProcess([], 0, stdout=b"\n200", stderr=b"")
|
||||
with patch.object(M.subprocess, "run", return_value=result) as call:
|
||||
self.agent.curl("PUT", action["href"], action["header"], upload=self.source)
|
||||
config = call.call_args.kwargs["input"].decode()
|
||||
self.assertIn('header = "Transfer-Encoding: chunked"', config)
|
||||
self.assertIn('request = "PUT"', config)
|
||||
self.assertIn("upload-file = ", config)
|
||||
|
||||
def test_chunked_forbidden_outside_upload_or_for_other_values(self):
|
||||
for value in ("gzip", "CHUNKED", "chunked, gzip", "chunked\r\nX: y", " chunked"):
|
||||
with self.subTest(value=value), self.assertRaises(M.TransferError):
|
||||
self.agent.curl("PUT", BASE + "/objects/" + self.oid,
|
||||
{"Transfer-Encoding": value}, upload=self.source)
|
||||
for method, kwargs in (("GET", {}), ("POST", {"body": {}}), ("PUT", {})):
|
||||
with self.subTest(method=method, kwargs=kwargs), self.assertRaises(M.TransferError):
|
||||
self.agent.curl(method, BASE + "/objects/" + self.oid,
|
||||
{"Transfer-Encoding": "chunked"}, **kwargs)
|
||||
with self.assertRaises(M.TransferError):
|
||||
M.safe_headers({"Transfer-Encoding": "chunked"})
|
||||
for operation, name in (("upload", "verify"), ("download", "download")):
|
||||
self.agent.operation = operation
|
||||
action = self.action()
|
||||
action["header"]["Transfer-Encoding"] = "chunked"
|
||||
response = {"objects": [{"oid": self.oid, "size": self.size, "actions": {name: action}}]}
|
||||
with patch.object(self.agent, "authenticate", return_value=(BASE, {})), \
|
||||
patch.object(self.agent, "curl", return_value=json.dumps(response).encode()):
|
||||
with self.subTest(operation=operation, action=name), self.assertRaises(M.TransferError):
|
||||
self.agent.batch(self.oid, self.size)
|
||||
|
||||
def test_upload_existing_object_needs_no_put(self):
|
||||
with patch.object(self.agent, "batch", return_value={}) as batch, patch.object(self.agent, "curl") as curl:
|
||||
result = self.agent.transfer(self.request())
|
||||
self.assertEqual(result, {"event": "complete", "oid": self.oid})
|
||||
batch.assert_called_once()
|
||||
curl.assert_not_called()
|
||||
self.assertEqual(self.events[-1]["bytesSoFar"], self.size)
|
||||
|
||||
def test_upload_put_then_verification_post(self):
|
||||
actions = {"upload": self.action(), "verify": self.action("verify/")}
|
||||
with patch.object(self.agent, "batch", return_value=actions), \
|
||||
patch.object(self.agent, "curl", return_value=b"") as curl:
|
||||
self.agent.transfer(self.request())
|
||||
self.assertEqual([c.args[0] for c in curl.call_args_list], ["PUT", "POST"])
|
||||
self.assertEqual(curl.call_args_list[1].kwargs["body"], {"oid": self.oid, "size": self.size})
|
||||
|
||||
def test_upload_wrong_sha_fails_before_network(self):
|
||||
self.source.write_bytes(b"x" * self.size)
|
||||
with patch.object(self.agent, "batch") as batch:
|
||||
with self.assertRaisesRegex(M.TransferError, "sha"):
|
||||
self.agent.transfer(self.request())
|
||||
batch.assert_not_called()
|
||||
|
||||
def test_upload_outside_lfs_store_refused(self):
|
||||
other = self.root / self.oid
|
||||
other.write_bytes(self.data)
|
||||
with self.assertRaisesRegex(M.TransferError, "outside-current-lfs"):
|
||||
self.agent.transfer(self.request(path=str(other)))
|
||||
|
||||
def test_upload_source_mutation_detected(self):
|
||||
def change(*args, **kwargs):
|
||||
self.source.write_bytes(b"changed")
|
||||
with patch.object(self.agent, "batch", return_value={"upload": self.action()}), \
|
||||
patch.object(self.agent, "curl", side_effect=change):
|
||||
with self.assertRaisesRegex(M.TransferError, "upload-source-changed"):
|
||||
self.agent.transfer(self.request())
|
||||
|
||||
def test_download_is_fresh_unique_and_verified(self):
|
||||
self.agent.operation = "download"
|
||||
def fetch(*args, **kwargs):
|
||||
kwargs["download"].write_bytes(self.data)
|
||||
return b""
|
||||
with patch.object(self.agent, "batch", return_value={"download": self.action()}), \
|
||||
patch.object(self.agent, "curl", side_effect=fetch) as curl:
|
||||
first = self.agent.transfer(self.request("download"))
|
||||
second = self.agent.transfer(self.request("download"))
|
||||
self.assertNotEqual(first["path"], second["path"])
|
||||
self.assertEqual(Path(first["path"]).read_bytes(), self.data)
|
||||
self.assertEqual(curl.call_count, 2)
|
||||
self.assertEqual(self.source.read_bytes(), self.data)
|
||||
|
||||
def test_download_hash_failure_retains_partial(self):
|
||||
self.agent.operation = "download"
|
||||
def fetch(*args, **kwargs):
|
||||
kwargs["download"].write_bytes(b"x" * self.size)
|
||||
with patch.object(self.agent, "batch", return_value={"download": self.action()}), \
|
||||
patch.object(self.agent, "curl", side_effect=fetch):
|
||||
with self.assertRaisesRegex(M.TransferError, "sha"):
|
||||
self.agent.transfer(self.request("download"))
|
||||
partials = list((self.git / "lfs" / "lan-transfer-tmp").glob("*.partial"))
|
||||
self.assertEqual(len(partials), 1)
|
||||
self.assertEqual(partials[0].read_bytes(), b"x" * self.size)
|
||||
|
||||
def test_protocol_init_progress_completion_terminate(self):
|
||||
requests = [self.init_message(), self.request(), {"event": "terminate"}]
|
||||
incoming = io.StringIO("".join(json.dumps(x) + "\n" for x in requests))
|
||||
with patch.object(self.agent, "batch", return_value={}):
|
||||
code = M.serve(incoming, self.events.append, factory=lambda message, send: self.agent)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(self.events[0], {})
|
||||
self.assertEqual(self.events[-1], {"event": "complete", "oid": self.oid})
|
||||
|
||||
def test_transfer_error_continues_and_never_echoes_exception_secret(self):
|
||||
incoming = io.StringIO("\n".join(json.dumps(x) for x in
|
||||
[self.init_message(), self.request(), self.request(), {"event": "terminate"}]) + "\n")
|
||||
with patch.object(self.agent, "transfer", side_effect=[RuntimeError(TOKEN), {"event": "complete", "oid": self.oid}]):
|
||||
code = M.serve(incoming, self.events.append, factory=lambda message, send: self.agent)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("error", self.events[1])
|
||||
self.assertNotIn(TOKEN, json.dumps(self.events))
|
||||
self.assertNotIn("error", self.events[2])
|
||||
|
||||
def test_bad_initialization_uses_init_error_schema(self):
|
||||
incoming = io.StringIO(json.dumps(self.init_message()) + "\n")
|
||||
with patch.object(M, "git_output", side_effect=RuntimeError(TOKEN)):
|
||||
code = M.serve(incoming, self.events.append)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(self.events[0]["error"]["code"], 32)
|
||||
self.assertNotIn(TOKEN, json.dumps(self.events))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user