consolidate-main-preserve-science-lfs-cpu-gpu-history
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,122 @@
|
||||
"""Explicitly authorized, fail-closed archive/retire operations in the new clone."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path('/home/qishizhe/digital-drosophila-main-integration-20261001')
|
||||
RECOVERY = Path('/home/qishizhe/digital-drosophila-branch-recovery-20261001.git')
|
||||
AUDIT = ROOT / 'registry/branch-consolidation-20261001'
|
||||
|
||||
|
||||
def require(value, message):
|
||||
if not value:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def git(*args, cwd=ROOT):
|
||||
# Existing payloads were checked separately. This operation transfers refs
|
||||
# and Git objects only; it neither replaces nor uploads any LFS object.
|
||||
env = dict(os.environ, GIT_LFS_SKIP_SMUDGE='1', GIT_LFS_SKIP_PUSH='1')
|
||||
return subprocess.check_output(['git', *args], cwd=cwd, env=env, text=True).strip()
|
||||
|
||||
|
||||
def snapshot():
|
||||
refs = {}
|
||||
default = None
|
||||
for line in git('ls-remote', '--symref', 'origin').splitlines():
|
||||
value, name = line.split('\t')
|
||||
if value.startswith('ref: '):
|
||||
if name == 'HEAD':
|
||||
default = value[5:]
|
||||
else:
|
||||
refs[name] = value
|
||||
return refs, default
|
||||
|
||||
|
||||
def write_receipt(name, **data):
|
||||
with (AUDIT / name).open('x') as f:
|
||||
json.dump(dict(schema='branch-consolidation-ref-audit-v1',
|
||||
created_utc=datetime.now(timezone.utc).isoformat(), **data),
|
||||
f, indent=2, sort_keys=True)
|
||||
f.write('\n')
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
def main(action):
|
||||
require(Path.cwd().resolve() == ROOT, 'Independent integration clone required')
|
||||
plan = json.loads((AUDIT / 'plan.json').read_text())
|
||||
require(git('remote', 'get-url', 'origin') == plan['remote'], 'Unexpected remote')
|
||||
branches = {'refs/heads/' + name: sha for name, sha in plan['branches'].items()}
|
||||
tags = {plan['archive_tag_prefix'] + name.removeprefix('codex/'): sha
|
||||
for name, sha in plan['branches'].items()}
|
||||
before, default_before = snapshot()
|
||||
|
||||
if action == 'archive':
|
||||
require(not (AUDIT / 'archive-recovery.json').exists(), 'Archive receipt already exists')
|
||||
require(not RECOVERY.exists(), 'Recovery clone already exists; inspect, do not replace')
|
||||
require({k: v for k, v in before.items() if k.startswith('refs/heads/')} == branches,
|
||||
'Remote heads changed since authorized inventory')
|
||||
for tag, sha in tags.items():
|
||||
ref = 'refs/tags/' + tag
|
||||
require(ref not in before, 'Archive tag already exists; inspect before proceeding')
|
||||
require(not git('tag', '--list', tag), 'Local archive tag already exists')
|
||||
git('tag', '-a', tag, sha, '-m',
|
||||
'Archive before user-authorized main consolidation; preserve this exact historical tip. Do not rewrite.')
|
||||
git('push', '--atomic', 'origin', *['refs/tags/' + tag + ':refs/tags/' + tag for tag in tags])
|
||||
after, _ = snapshot()
|
||||
for tag, sha in tags.items():
|
||||
require(after.get('refs/tags/' + tag + '^{}') == sha, 'Remote archived commit differs')
|
||||
git('init', '--bare', str(RECOVERY))
|
||||
git('fetch', '--no-tags', plan['remote'],
|
||||
*['refs/tags/' + tag + ':refs/tags/' + tag for tag in tags], cwd=RECOVERY)
|
||||
recovered = {tag: git('rev-parse', 'refs/tags/' + tag + '^{commit}', cwd=RECOVERY)
|
||||
for tag in tags}
|
||||
require(recovered == tags, 'Fresh remote recovery failed')
|
||||
git('fsck', '--full', '--no-reflogs', cwd=RECOVERY)
|
||||
write_receipt('archive-recovery.json', pass_all=True, remote=plan['remote'],
|
||||
remote_heads_before=branches, remote_default_before=default_before,
|
||||
tags=tags, fresh_remote_recovery_path=str(RECOVERY),
|
||||
recovered_commits=recovered, git_fsck_pass=True,
|
||||
lfs_payload_downloaded=False, old_remote_branches_deleted=False)
|
||||
|
||||
elif action == 'retire':
|
||||
require(not (AUDIT / 'remote-cleanup.json').exists(), 'Cleanup receipt already exists')
|
||||
require(json.loads((AUDIT / 'archive-recovery.json').read_text())['pass_all'], 'Recovery not verified')
|
||||
require(json.loads((AUDIT / 'merge-audit-final.json').read_text())['pass_all'], 'Merge not verified')
|
||||
require(default_before == 'refs/heads/main', 'Remote default must already be main')
|
||||
require(git('branch', '--show-current') == 'main', 'Wrong local branch')
|
||||
require(git('status', '--porcelain') == '', 'Integration clone is not clean')
|
||||
main_sha = git('rev-parse', 'HEAD')
|
||||
expected = dict(branches, **{'refs/heads/main': main_sha})
|
||||
require({k: v for k, v in before.items() if k.startswith('refs/heads/')} == expected,
|
||||
'Remote heads changed; stop cleanup')
|
||||
for tag, sha in tags.items():
|
||||
require(before.get('refs/tags/' + tag + '^{}') == sha, 'Remote archive missing/different')
|
||||
require(git('rev-parse', 'refs/tags/' + tag + '^{commit}', cwd=RECOVERY) == sha,
|
||||
'Recovery archive differs')
|
||||
git('merge-base', '--is-ancestor', sha, main_sha)
|
||||
leases = ['--force-with-lease=' + ref + ':' + sha for ref, sha in branches.items()]
|
||||
# Delete only these seven exact branch refs, atomically and with their
|
||||
# recorded old OIDs. No local refs, tags, commits, worktrees or LFS deletion.
|
||||
git('push', '--atomic', *leases, 'origin', *[':' + ref for ref in branches])
|
||||
after, default_after = snapshot()
|
||||
require({k: v for k, v in after.items() if k.startswith('refs/heads/')} ==
|
||||
{'refs/heads/main': main_sha}, 'Unexpected final heads')
|
||||
require(default_after == 'refs/heads/main', 'Default branch changed unexpectedly')
|
||||
for tag, sha in tags.items():
|
||||
require(after.get('refs/tags/' + tag + '^{}') == sha, 'Archive lost after deletion')
|
||||
write_receipt('remote-cleanup.json', pass_all=True, remote=plan['remote'],
|
||||
main_commit=main_sha, remote_default=default_after,
|
||||
deleted_remote_branch_refs=branches, preserved_archive_tags=tags,
|
||||
remaining_remote_heads={'main': main_sha},
|
||||
all_old_tips_ancestors_of_main=True, local_branches_deleted=False,
|
||||
source_worktrees_write_operations=False, lfs_objects_deleted=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('action', choices=['archive', 'retire'])
|
||||
main(p.parse_args().action)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Validate staged integration against both original trees without science runs."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def require(value, message):
|
||||
if not value:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(['git', *args])
|
||||
|
||||
|
||||
def tree(ref):
|
||||
result = {}
|
||||
for row in git('ls-tree', '-r', '-z', ref).split(b'\0'):
|
||||
if row:
|
||||
meta, path = row.split(b'\t', 1)
|
||||
mode, kind, oid = meta.decode().split()
|
||||
require(kind == 'blob', 'Unexpected tree object')
|
||||
result[path.decode()] = (mode, oid)
|
||||
return result
|
||||
|
||||
|
||||
def main(output, check_lfs):
|
||||
root = Path.cwd()
|
||||
require(root == Path('/home/qishizhe/digital-drosophila-main-integration-20261001'), 'Independent integration clone required')
|
||||
plan = json.loads((root / 'registry/branch-consolidation-20261001/plan.json').read_text())
|
||||
left, right = plan['merge_parents']
|
||||
base = git('merge-base', left, right).decode().strip()
|
||||
a, b, old = tree(left), tree(right), tree(base)
|
||||
staged = {}
|
||||
for row in git('ls-files', '--stage', '-z').split(b'\0'):
|
||||
if row:
|
||||
meta, path = row.split(b'\t', 1)
|
||||
mode, oid, stage = meta.decode().split()
|
||||
require(stage == '0', 'Unresolved merge conflict')
|
||||
staged[path.decode()] = (mode, oid)
|
||||
allowed = set(plan['allowed_existing_file_changes'])
|
||||
checked = 0
|
||||
for name in sorted(set(a) | set(b)):
|
||||
require(name in staged, 'Original file removed: ' + name)
|
||||
if name in allowed:
|
||||
continue
|
||||
av, bv, ov = a.get(name), b.get(name), old.get(name)
|
||||
if av == bv:
|
||||
expected = av
|
||||
elif av == ov:
|
||||
expected = bv
|
||||
elif bv == ov:
|
||||
expected = av
|
||||
elif av is None:
|
||||
expected = bv
|
||||
elif bv is None:
|
||||
expected = av
|
||||
else:
|
||||
raise ValueError('Unapproved conflicting original file: ' + name)
|
||||
require(staged[name] == expected, 'Original blob or mode changed: ' + name)
|
||||
checked += 1
|
||||
new_paths = sorted(set(staged) - set(a) - set(b))
|
||||
for name in new_paths:
|
||||
require(any(name.startswith(p) if p.endswith('/') else name == p for p in plan['new_paths']), 'Unapproved new path')
|
||||
artifacts = {p: v for p, v in staged.items() if p.startswith('artifacts/')}
|
||||
require(len(artifacts) == plan['expected_artifact_paths'], 'Artifact path inventory differs')
|
||||
pointers = {}
|
||||
pattern = re.compile(rb'version https://git-lfs.github.com/spec/v1\noid sha256:([0-9a-f]{64})\nsize ([0-9]+)\n?')
|
||||
for name, (mode, oid) in artifacts.items():
|
||||
data = git('cat-file', 'blob', oid)
|
||||
match = pattern.fullmatch(data)
|
||||
if match:
|
||||
pointers[name] = dict(oid=match[1].decode(), bytes=int(match[2]), git_blob=oid)
|
||||
require(len(pointers) == plan['expected_lfs_pointer_paths'], 'LFS pointer path inventory differs')
|
||||
attributes = git('check-attr', '--cached', 'filter', '--', *pointers).decode().splitlines()
|
||||
require(len(attributes) == len(pointers) and all(x.endswith(': filter: lfs') for x in attributes), 'Merged LFS rules do not cover every original object')
|
||||
lfs_report = None
|
||||
if check_lfs:
|
||||
source = root / 'tools/lfs-publication/lan_lfs_transfer.py'
|
||||
require(hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256(git('show', left + ':tools/lfs-publication/lan_lfs_transfer.py')).digest(), 'Transfer adapter changed')
|
||||
spec = importlib.util.spec_from_file_location('merge_lan', source)
|
||||
lan = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(lan)
|
||||
agent = lan.Agent(dict(event='init', operation='download', remote='origin', concurrent=True, concurrenttransfers=2), lambda _: None)
|
||||
href, headers = agent.authenticate()
|
||||
unique = {v['oid']: v['bytes'] for v in pointers.values()}
|
||||
requested = [{'oid': oid, 'size': size} for oid, size in sorted(unique.items())]
|
||||
batches = 0
|
||||
for begin in range(0, len(requested), 100):
|
||||
items = requested[begin:begin + 100]
|
||||
response = lan.decode_json(agent.curl('POST', href + '/objects/batch', headers,
|
||||
body=dict(operation='download', transfers=['basic'], objects=items)))
|
||||
require(response.get('transfer', 'basic') == 'basic', 'Unexpected LFS transfer mode')
|
||||
received = response.get('objects', [])
|
||||
require(len(received) == len(items), 'LFS metadata inventory incomplete')
|
||||
seen = set()
|
||||
for item in received:
|
||||
oid = item.get('oid')
|
||||
require(oid in unique and oid not in seen and item.get('size') == unique[oid] and 'error' not in item, 'LFS object unavailable')
|
||||
action = item.get('actions', {}).get('download')
|
||||
require(isinstance(action, dict), 'LFS object has no download action')
|
||||
lan.safe_url(action['href'])
|
||||
lan.safe_headers(action.get('header', {}))
|
||||
seen.add(oid)
|
||||
require(seen == {item['oid'] for item in items}, 'LFS response object set differs')
|
||||
batches += 1
|
||||
lfs_report = dict(unique_objects=len(unique), unique_bytes=sum(unique.values()), batches=batches,
|
||||
all_download_actions_available=True, payloads_redownloaded=False,
|
||||
scope='Authenticated read-only LFS availability; full byte validation remains the unchanged prior fresh-download receipts')
|
||||
report = dict(schema='main-integration-audit-v1', verified_utc=datetime.now(timezone.utc).isoformat(),
|
||||
pass_all=True, parents=[left, right], merge_base=base, staged_original_blobs_checked=checked,
|
||||
allowed_existing_changes=sorted(allowed), new_paths=new_paths, artifact_paths=len(artifacts),
|
||||
lfs_pointer_paths=len(pointers), pointers=pointers, lfs_availability=lfs_report,
|
||||
model_tool_registry_artifact_bytes_unchanged=True, scientific_execution=False,
|
||||
source_worktree_write_operations_by_this_tool=False,
|
||||
source_worktree_integrity_requires_separate_audit=True,
|
||||
staged_tree=git('write-tree').decode().strip())
|
||||
with output.open('x') as stream:
|
||||
json.dump(report, stream, indent=2, sort_keys=True)
|
||||
stream.write('\n')
|
||||
print(json.dumps({k: v for k, v in report.items() if k not in ('pointers', 'new_paths')}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--check-lfs', action='store_true')
|
||||
args = parser.parse_args()
|
||||
main(args.output, args.check_lfs)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Read-only source guard; write receipt only to a new publication path."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from publish_archives import no_links, sha_file, canonical
|
||||
|
||||
ROOT = Path('/data/digital-drosophila')
|
||||
REGISTERED = 'b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740'
|
||||
SOURCE_COMMIT = '80aeb2093db655329774eff7b32fb6e417b8b40b'
|
||||
|
||||
|
||||
def git(*args, cwd=ROOT):
|
||||
return subprocess.check_output(['git', '-C', str(cwd), *args], text=True).strip()
|
||||
|
||||
|
||||
def capture(plan_path, output):
|
||||
output = no_links(output)
|
||||
if output == ROOT or ROOT in output.parents or output.exists():
|
||||
raise ValueError('Receipt must be new and outside the scientific source')
|
||||
plan = json.loads(Path(plan_path).read_bytes())
|
||||
path = ROOT / 'experiments/phase4ar-normal-conditioning/protocol/preregistered.json'
|
||||
raw = path.read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != REGISTERED:
|
||||
raise ValueError('Phase 4AR registration hash mismatch')
|
||||
registration = json.loads(raw)
|
||||
records = []
|
||||
for filename, expected in sorted(registration['source_sha256'].items()):
|
||||
source = no_links(filename)
|
||||
if ROOT not in source.parents:
|
||||
raise ValueError('Registered source outside project')
|
||||
actual = sha_file(source)
|
||||
if actual['sha256'] != expected:
|
||||
raise ValueError('Phase 4AR source mismatch: ' + filename)
|
||||
records.append(dict(path=filename, **actual))
|
||||
if len(records) != 16:
|
||||
raise ValueError('Registered source inventory differs from reviewed scope')
|
||||
if git('cat-file', '-t', SOURCE_COMMIT) != 'commit':
|
||||
raise ValueError('Registered scientific commit unavailable')
|
||||
evidence = []
|
||||
for group in plan['groups']:
|
||||
licence = group['license']
|
||||
if 'evidence_file' in licence:
|
||||
source = ROOT / group['paths'][0] / licence['evidence_file']
|
||||
actual = sha_file(source)
|
||||
if actual['sha256'] != licence['evidence_sha256']:
|
||||
raise ValueError('License evidence hash mismatch: ' + group['id'])
|
||||
evidence.append(dict(group=group['id'], path=str(source), **actual))
|
||||
provenance = group['provenance']
|
||||
source = ROOT / group['paths'][0]
|
||||
if (source / '.git').exists():
|
||||
if git('rev-parse', 'HEAD', cwd=source) != provenance['commit']:
|
||||
raise ValueError('Upstream commit mismatch: ' + group['id'])
|
||||
if git('status', '--porcelain', cwd=source):
|
||||
raise ValueError('Upstream checkout changed: ' + group['id'])
|
||||
result = dict(schema='lfs-publication-source-guard-v1',
|
||||
observed_utc=datetime.now(timezone.utc).isoformat(),
|
||||
registration_sha256=REGISTERED, registered_scientific_commit=SOURCE_COMMIT,
|
||||
phase4ar_registered_sources=records, license_evidence=evidence,
|
||||
original_checkout_head=git('rev-parse', 'HEAD'),
|
||||
original_checkout_status=git('status', '--porcelain'),
|
||||
original_checkout_branch=git('branch', '--show-current'),
|
||||
disk_free_bytes={str(p): shutil.disk_usage(p).free for p in (ROOT, output.parent)},
|
||||
pass_all=True,
|
||||
scope='16 registered Phase 4AR source files, registration, pinned upstream checkout and license evidence; not a scientific rerun or full historical checksum audit')
|
||||
if result['original_checkout_status']:
|
||||
raise ValueError('Original checkout is no longer clean; preserve and investigate')
|
||||
if any(x < 100 * 1024**3 for x in result['disk_free_bytes'].values()):
|
||||
raise ValueError('Less than 100 GiB free reserve')
|
||||
with output.open('xb') as handle:
|
||||
handle.write(canonical(result))
|
||||
return dict(pass_all=True, receipt=str(output), registered_sources=len(records),
|
||||
original_checkout_head=result['original_checkout_head'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--plan', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(capture(args.plan, args.out), sort_keys=True))
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Fresh-cache transport verification for one independent publication checkout.
|
||||
|
||||
Run only from its top level. This is not a scientific audit or authorization to
|
||||
publish. Failed attempts retain their cache/partial files and are never retried.
|
||||
Git/LFS output is suppressed; receipts contain no environment or auth headers.
|
||||
The caller supplies the trusted commit. Tool source SHA pins are observations,
|
||||
not signatures; executable versions are not executable-byte identity proofs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import publish_archives as archives
|
||||
import verify_publication as content
|
||||
|
||||
REMOTE = "ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git"
|
||||
PAYLOAD = "artifacts/public-resources-20261001"
|
||||
PROTECTED = Path("/data/digital-drosophila")
|
||||
TRANSFER_OVERRIDES = (r"^(lfs\.(url|pushurl|standalonetransferagent|customtransfer\..*|"
|
||||
r"remote\.(autodetect|searchall))|remote\.(lfsdefault|origin\.lfsurl))$")
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def within(path, root):
|
||||
return path == root or root in path.parents
|
||||
|
||||
|
||||
def command(args, *, text=False, bytes_output=False):
|
||||
"""Never return or print command diagnostics, which may contain credentials."""
|
||||
result = subprocess.run(args, stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE if text or bytes_output else subprocess.DEVNULL,
|
||||
check=False)
|
||||
if text or bytes_output:
|
||||
require(result.returncode == 0, "Read-only Git/tool identity command failed")
|
||||
if bytes_output:
|
||||
return result.stdout
|
||||
return result.stdout.decode("utf-8", errors="strict").strip()
|
||||
return result.returncode
|
||||
|
||||
|
||||
def identity(expected, checkout):
|
||||
require(command(["git", "rev-parse", "--verify", "HEAD"], text=True) == expected,
|
||||
"HEAD differs from expected commit")
|
||||
require(archives.no_links(command(["git", "rev-parse", "--show-toplevel"], text=True)) == checkout,
|
||||
"Run from the independent checkout top level")
|
||||
require(archives.no_links(command(["git", "rev-parse", "--absolute-git-dir"], text=True)) == checkout / ".git",
|
||||
"External/linked Git directory refused")
|
||||
require(command(["git", "remote", "get-url", "--all", "origin"], text=True) == REMOTE,
|
||||
"Origin must exactly match the approved remote")
|
||||
|
||||
|
||||
def transfer_policy(checkout, expected_commit, allow_lan_adapter=False):
|
||||
require(not (checkout / ".lfsconfig").exists() and not (checkout / ".lfsconfig").is_symlink(),
|
||||
"Repository LFS endpoint overrides refused")
|
||||
if not allow_lan_adapter:
|
||||
# Exit 1 means no matches; potential credential values are never read.
|
||||
require(command(["git", "config", "--get-regexp", TRANSFER_OVERRIDES]) == 1,
|
||||
"LFS endpoint/custom-transfer override or unreadable configuration refused")
|
||||
return None
|
||||
relative = "tools/lfs-publication/lan_lfs_transfer.py"
|
||||
adapter = archives.no_links(checkout / relative)
|
||||
require(not any(c.isspace() for c in str(adapter)), "LAN adapter path must not require argument quoting")
|
||||
prefix = "lfs.customtransfer.malecns-lan-https."
|
||||
allowed = {"lfs.standalonetransferagent": "malecns-lan-https",
|
||||
prefix + "path": "/usr/bin/python3", prefix + "args": "-B " + str(adapter),
|
||||
prefix + "concurrent": "true", prefix + "direction": "both"}
|
||||
keys = command(["git", "config", "--name-only", "--get-regexp", TRANSFER_OVERRIDES], text=True).splitlines()
|
||||
require(len(keys) == len(allowed) and set(keys) == set(allowed),
|
||||
"Only the exact approved LAN adapter configuration is permitted")
|
||||
for key, value in allowed.items():
|
||||
require(command(["git", "config", "--local", "--get-all", key], text=True) == value
|
||||
and command(["git", "config", "--get-all", key], text=True) == value,
|
||||
"LAN adapter configuration must be exact and exclusively repository-local")
|
||||
pin = archives.sha_file(adapter)
|
||||
blob = command(["git", "show", expected_commit + ":" + relative], bytes_output=True)
|
||||
require(len(blob) == pin["bytes"] and hashlib.sha256(blob).hexdigest() == pin["sha256"],
|
||||
"LAN adapter must exactly match the expected HEAD blob")
|
||||
return {"path": str(adapter), **pin}
|
||||
|
||||
|
||||
def new_output(path, checkout, cache, payload):
|
||||
p = archives.no_links(path)
|
||||
require(not within(p, PROTECTED.absolute()), "Output under original project refused")
|
||||
require(within(p, checkout.parent), "Output must stay in the independent workspace")
|
||||
require(not within(p, payload) and not within(p, checkout / ".git")
|
||||
and not within(p, checkout / "artifacts") and not within(p, cache),
|
||||
"Receipt must be outside payload, artifacts, Git and cache trees")
|
||||
require(not p.exists() and p.parent.is_dir(), "Receipt must be new with an existing parent")
|
||||
return p
|
||||
|
||||
|
||||
def cache_inventory(cache):
|
||||
count = total = 0
|
||||
objects = cache / "objects"
|
||||
if not objects.exists():
|
||||
return {"objects": 0, "bytes": 0}
|
||||
archives.no_links(objects)
|
||||
require(objects.is_dir(), "LFS objects must be a directory")
|
||||
for p in objects.rglob("*"):
|
||||
archives.no_links(p)
|
||||
s = p.lstat()
|
||||
if stat.S_ISDIR(s.st_mode):
|
||||
continue
|
||||
require(stat.S_ISREG(s.st_mode), "Nonregular cache object refused")
|
||||
parts = p.relative_to(objects).parts
|
||||
require(len(parts) == 3 and archives.HEX.fullmatch(parts[2]) is not None
|
||||
and parts[0] == parts[2][:2] and parts[1] == parts[2][2:4],
|
||||
"Unexpected LFS object layout")
|
||||
count += 1
|
||||
total += s.st_size
|
||||
return {"objects": count, "bytes": total}
|
||||
|
||||
|
||||
def source_pins(adapter=None):
|
||||
result = {name: {"path": str(Path(path).absolute()), **archives.sha_file(path)}
|
||||
for name, path in (("download_verify.py", __file__),
|
||||
("verify_publication.py", content.__file__),
|
||||
("publish_archives.py", archives.__file__))}
|
||||
if adapter is not None:
|
||||
result["lan_lfs_transfer.py"] = {"path": adapter["path"], **archives.sha_file(adapter["path"])}
|
||||
return result
|
||||
|
||||
|
||||
def run(expected_commit, cache, receipt, content_receipt, historical_inventory, *, allow_lan_adapter=False):
|
||||
require(isinstance(expected_commit, str) and re.fullmatch(r"[0-9a-f]{40}", expected_commit),
|
||||
"Expected commit must be a complete lowercase SHA-1")
|
||||
checkout = archives.no_links(Path.cwd())
|
||||
require(not within(checkout, PROTECTED.absolute()), "Original project checkout refused")
|
||||
gitdir = archives.no_links(checkout / ".git")
|
||||
require(gitdir.is_dir(), "Only an independent clone with its own .git directory is accepted")
|
||||
identity(expected_commit, checkout)
|
||||
adapter = transfer_policy(checkout, expected_commit, allow_lan_adapter)
|
||||
require(not command(["git", "status", "--porcelain", "--untracked-files=no"], text=True),
|
||||
"Tracked checkout changes refused")
|
||||
cache = archives.no_links(cache)
|
||||
require(not within(cache, PROTECTED.absolute()), "Cache under original project refused")
|
||||
require(within(cache, gitdir) or (within(cache, checkout.parent) and not within(cache, checkout)),
|
||||
"Cache must be inside this .git or outside checkout in the independent workspace")
|
||||
require(not cache.exists() and cache.parent.is_dir(), "Cache must never have existed for this attempt")
|
||||
payload = archives.no_links(checkout / PAYLOAD)
|
||||
receipt = new_output(receipt, checkout, cache, payload)
|
||||
content_receipt = new_output(content_receipt, checkout, cache, payload)
|
||||
history = archives.no_links(historical_inventory)
|
||||
require(len({receipt, content_receipt, history, cache}) == 4, "Inputs/outputs must be distinct")
|
||||
history_pin = archives.sha_file(history)
|
||||
pins = source_pins(adapter)
|
||||
versions = {"git": command(["git", "--version"], text=True),
|
||||
"git_lfs": command(["git", "lfs", "version"], text=True),
|
||||
"zstd": command(["zstd", "--version"], text=True),
|
||||
"python": sys.version.split()[0]}
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
cache.mkdir(exist_ok=False)
|
||||
initial = cache_inventory(cache)
|
||||
require(initial == {"objects": 0, "bytes": 0}, "New cache is not empty")
|
||||
print(json.dumps({"stage": "fresh-cache-created", "objects": 0, "bytes": 0}), flush=True)
|
||||
git = ["git", "-c", "lfs.storage=" + str(cache), "-c", "lfs.concurrenttransfers=2"]
|
||||
pull = command(git + ["lfs", "pull", "--include=artifacts/**", "--exclude=", "origin"])
|
||||
require(pull == 0, "LFS pull failed; cache retained, no automatic retry")
|
||||
print(json.dumps({"stage": "pull-exited", "exit_code": pull}), flush=True)
|
||||
fsck = command(git + ["lfs", "fsck", "--dry-run"])
|
||||
require(fsck == 0, "LFS fsck failed; cache retained, no automatic retry")
|
||||
print(json.dumps({"stage": "fsck-exited", "exit_code": fsck}), flush=True)
|
||||
objects = cache_inventory(cache)
|
||||
require(objects["objects"] > 0 and objects["bytes"] > 0, "Fresh LFS cache contains no objects")
|
||||
verified = content.verify_publication(payload, content_receipt, history)
|
||||
require(verified.get("pass_all") is True
|
||||
and archives.HEX.fullmatch(str(verified.get("aggregate_inventory_sha256", ""))) is not None,
|
||||
"Content verification did not report success")
|
||||
content_pin = archives.sha_file(content_receipt)
|
||||
require(content_pin["sha256"] == verified.get("receipt_sha256"), "Content receipt SHA mismatch")
|
||||
require(archives.sha_file(history) == history_pin and source_pins(adapter) == pins,
|
||||
"History or verification tool changed during attempt")
|
||||
identity(expected_commit, checkout)
|
||||
require(transfer_policy(checkout, expected_commit, allow_lan_adapter) == adapter,
|
||||
"LAN adapter identity/configuration changed during attempt")
|
||||
require(cache_inventory(cache) == objects, "Cache object inventory changed during content verification")
|
||||
report = {"schema": "lfs-publication-download-verification-v1", "pass_all": True,
|
||||
"started_utc": started, "verified_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"checkout": str(checkout), "expected_commit": expected_commit,
|
||||
"head": expected_commit, "origin": REMOTE,
|
||||
"explicit_pull_remote": "origin", "lfs_endpoint_overrides_refused": True,
|
||||
"lan_adapter": {"enabled": adapter is not None, "source": adapter,
|
||||
"configuration_scope": "repository-local only" if adapter else None,
|
||||
"temporary_download_directory": str(gitdir / "lfs/lan-transfer-tmp") if adapter else None},
|
||||
"cache": {"path": str(cache), "initially_absent": True,
|
||||
"before": initial, "after": objects},
|
||||
"pull_exit_code": pull, "fsck_dry_run_exit_code": fsck,
|
||||
"include": "artifacts/**", "exclude": "", "concurrent_transfers": 2,
|
||||
"content_receipt": {"path": str(content_receipt), **content_pin},
|
||||
"aggregate_inventory_sha256": verified["aggregate_inventory_sha256"],
|
||||
"historical_inventory": {"path": str(history), **history_pin,
|
||||
"status": verified.get("historical_status")},
|
||||
"tool_sources": pins, "tool_versions": versions, "fresh_download_proven": True,
|
||||
"learning_pass_claimed": False, "original_project_write_requested": False,
|
||||
"scope": "Fresh isolated LFS storage, actual pull/fsck exits and complete publication member verification; smoke covered by pull/fsck. Not a scientific gate, all-history audit, or executable-byte identity proof."}
|
||||
archives.write_new(receipt, archives.canonical(report))
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--expected-commit", required=True)
|
||||
parser.add_argument("--cache", required=True)
|
||||
parser.add_argument("--receipt", required=True)
|
||||
parser.add_argument("--content-receipt", required=True)
|
||||
parser.add_argument("--historical-inventory", required=True)
|
||||
parser.add_argument("--allow-lan-adapter", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
run(args.expected_commit, args.cache, args.receipt, args.content_receipt, args.historical_inventory,
|
||||
allow_lan_adapter=args.allow_lan_adapter)
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
print(json.dumps({"status": "FAILED-NOT-RETRIED", "error_type": type(exc).__name__}), flush=True)
|
||||
return 1
|
||||
print(json.dumps({"status": "PASS", "receipt": str(Path(args.receipt).absolute())}), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,534 @@
|
||||
"""Whitelist-only LFS archives; content verification, not a redistribution grant.
|
||||
|
||||
No source writes, cache-based trust, extraction, or Git operations. Failed packs
|
||||
remain incomplete and cannot be retried into the same output/work directories.
|
||||
File stability checks are observations, not an atomic filesystem snapshot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
SOURCE_ROOT = "/data/digital-drosophila"
|
||||
CHUNK_BYTES = 64 * 1024 * 1024
|
||||
BUFFER = 1024 * 1024
|
||||
DENIED_DIRS = {".git", ".venv", "venv", "env", "__pycache__", ".cache",
|
||||
"node_modules", ".pytest_cache", ".mypy_cache", "extension-build"}
|
||||
LOG_DIRS = {"logs", "_logs"}
|
||||
BINARY_SUFFIXES = {".npy", ".npz", ".feather", ".parquet", ".gz", ".zst", ".zip",
|
||||
".tar", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf",
|
||||
".woff", ".woff2", ".ttf", ".ico", ".so", ".o", ".a", ".bin"}
|
||||
HEX = re.compile(r"^[0-9a-f]{64}$")
|
||||
SECRET_RULES = (
|
||||
("private-key", re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----")),
|
||||
("github-token", re.compile(r"\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")),
|
||||
("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
|
||||
("url-user-password", re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")),
|
||||
("secret-literal", re.compile(r'''(?ix)\b(?:password|passwd|api[_-]?key|secret[_-]?key)\b["']?\s*[:=]\s*(?:["']([^"'\r\n]+)["']|([A-Za-z0-9_+/@.-]{4,}))''')),
|
||||
)
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode()
|
||||
|
||||
|
||||
def json_bytes(data):
|
||||
def unique(pairs):
|
||||
value = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise ValueError("Duplicate JSON key")
|
||||
value[key] = item
|
||||
return value
|
||||
return json.loads(data, object_pairs_hook=unique)
|
||||
|
||||
|
||||
def json_read(path):
|
||||
return json_bytes(Path(path).read_bytes())
|
||||
|
||||
|
||||
def relpath(value):
|
||||
if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
|
||||
raise ValueError("Invalid relative path")
|
||||
p = PurePosixPath(value)
|
||||
if p.is_absolute() or any(x in ("", ".", "..") for x in value.split("/")) or ":" in value:
|
||||
raise ValueError("Unsafe relative path")
|
||||
return p.as_posix()
|
||||
|
||||
|
||||
def forbidden(relative):
|
||||
parts = PurePosixPath(relative).parts
|
||||
lower = [x.lower() for x in parts]
|
||||
if any(x in DENIED_DIRS or x.startswith(".venv") for x in lower):
|
||||
return "git-environment-or-cache"
|
||||
if any(x.startswith("phase4ar") for x in lower):
|
||||
return "active-phase4ar"
|
||||
if any(x in LOG_DIRS for x in lower) or lower[-1].endswith((".log", ".log.gz")):
|
||||
return "log"
|
||||
name = lower[-1]
|
||||
if (name == ".env" or name.startswith(".env.") or name in
|
||||
{"id_rsa", "id_ed25519", "id_dsa", ".git-credentials", ".netrc"}
|
||||
or any(x in name for x in ("credentials", "secrets"))
|
||||
or name.endswith((".pem", ".key", ".p12", ".pfx"))):
|
||||
return "credential-file"
|
||||
return None
|
||||
|
||||
|
||||
def no_links(path):
|
||||
if ".." in Path(path).parts:
|
||||
raise ValueError("Parent path components refused")
|
||||
p = Path(path).absolute()
|
||||
for part in (p, *p.parents):
|
||||
if part.exists() or part.is_symlink():
|
||||
s = part.lstat()
|
||||
if (stat.S_ISLNK(s.st_mode) or
|
||||
getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)):
|
||||
raise ValueError("Symlink refused: " + str(part))
|
||||
return p
|
||||
|
||||
|
||||
def identity(s):
|
||||
return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns)
|
||||
|
||||
|
||||
def opened_identity(handle, expected):
|
||||
actual = identity(os.fstat(handle.fileno()))
|
||||
# Some Windows Python builds expose birth time via stat().ctime but change
|
||||
# time via fstat().ctime. Check each clock against itself before/after; never
|
||||
# omit ctime from either endpoint's stability check. Linux uses all fields.
|
||||
same = actual[:-1] == expected[:-1] if os.name == "nt" else actual == expected
|
||||
if not same:
|
||||
raise ValueError("File changed while opening")
|
||||
return actual
|
||||
|
||||
|
||||
def sha_file(path):
|
||||
p = no_links(path)
|
||||
before = p.stat()
|
||||
if not stat.S_ISREG(before.st_mode):
|
||||
raise ValueError("Not a regular file: " + str(p))
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
with p.open("rb") as f:
|
||||
opened = opened_identity(f, identity(before))
|
||||
while block := f.read(BUFFER):
|
||||
h.update(block)
|
||||
size += len(block)
|
||||
if identity(os.fstat(f.fileno())) != opened:
|
||||
raise ValueError("File changed while hashing: " + str(p))
|
||||
if identity(p.lstat()) != identity(before) or size != before.st_size:
|
||||
raise ValueError("File changed while hashing: " + str(p))
|
||||
return {"sha256": h.hexdigest(), "bytes": size}
|
||||
|
||||
|
||||
def write_new(path, data):
|
||||
with Path(path).open("xb") as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
def validate_plan(plan):
|
||||
if not isinstance(plan, dict) or set(plan) - {"schema", "source_root", "groups", "exclude_paths"}:
|
||||
raise ValueError("Unknown plan fields")
|
||||
if plan.get("schema") != "lfs-publication-plan-v1" or plan.get("source_root") != SOURCE_ROOT:
|
||||
raise ValueError("Wrong plan schema/source_root")
|
||||
groups = plan.get("groups")
|
||||
if not isinstance(groups, list) or not groups:
|
||||
raise ValueError("Nonempty groups required")
|
||||
seen, all_paths = set(), []
|
||||
for g in groups:
|
||||
if not isinstance(g, dict) or set(g) != {"id", "paths", "license", "provenance"}:
|
||||
raise ValueError("Group requires id/paths/license/provenance only")
|
||||
group_id = g["id"]
|
||||
if not isinstance(group_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,99}", group_id) or group_id in seen:
|
||||
raise ValueError("Invalid or duplicate group id")
|
||||
seen.add(group_id)
|
||||
if not isinstance(g["paths"], list) or not g["paths"] or not g["license"] or not g["provenance"]:
|
||||
raise ValueError("Nonempty paths/license/provenance required")
|
||||
for value in g["paths"]:
|
||||
p = relpath(value)
|
||||
if forbidden(p):
|
||||
raise ValueError("Explicit forbidden path: " + p)
|
||||
if any(p == old or p.startswith(old + "/") or old.startswith(p + "/") for old in all_paths):
|
||||
raise ValueError("Overlapping whitelist paths")
|
||||
all_paths.append(p)
|
||||
excludes = plan.get("exclude_paths", [])
|
||||
if not isinstance(excludes, list):
|
||||
raise ValueError("exclude_paths must be a list")
|
||||
for p in excludes:
|
||||
relpath(p)
|
||||
canonical(plan)
|
||||
|
||||
|
||||
def enumerate_group(root, group, excludes):
|
||||
files, omitted = [], []
|
||||
|
||||
def visit(p):
|
||||
relative = p.relative_to(root).as_posix()
|
||||
s = p.lstat()
|
||||
if stat.S_ISLNK(s.st_mode):
|
||||
raise ValueError("Symlink refused: " + relative)
|
||||
reason = forbidden(relative)
|
||||
if any(relative == x or relative.startswith(x + "/") for x in excludes):
|
||||
reason = "plan-exclusion"
|
||||
if reason:
|
||||
omitted.append({"path": relative, "reason": reason,
|
||||
"scope": "subtree" if stat.S_ISDIR(s.st_mode) else "file",
|
||||
"bytes": s.st_size if stat.S_ISREG(s.st_mode) else None})
|
||||
elif stat.S_ISDIR(s.st_mode):
|
||||
for child in sorted(p.iterdir(), key=lambda x: x.name):
|
||||
visit(child)
|
||||
elif stat.S_ISREG(s.st_mode):
|
||||
files.append((relative, identity(s)))
|
||||
else:
|
||||
raise ValueError("Nonregular source refused: " + relative)
|
||||
|
||||
for relative in group["paths"]:
|
||||
path = no_links(root / relative)
|
||||
visit(path)
|
||||
files.sort(key=lambda x: x[0])
|
||||
omitted.sort(key=lambda x: x["path"])
|
||||
if not files:
|
||||
raise ValueError("Empty group refused")
|
||||
return files, omitted
|
||||
|
||||
|
||||
def scan_secrets(root, files):
|
||||
for relative, original in files:
|
||||
p = root / relative
|
||||
if original[3] > 2 * 1024 * 1024 or p.suffix.lower() in BINARY_SUFFIXES:
|
||||
continue
|
||||
data = p.read_bytes()
|
||||
if identity(p.lstat()) != original:
|
||||
raise ValueError("Source changed during secret scan: " + relative)
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if "\x00" in text:
|
||||
continue
|
||||
# Per-rule ASCII necessary fragments avoid repeated scans of numeric SWC
|
||||
# bodies (their header includes a public URL). Non-ASCII text still uses
|
||||
# every regex because Unicode IGNORECASE has broader equivalences.
|
||||
lower = text.lower()
|
||||
fragments = {
|
||||
"private-key": ("private key",),
|
||||
"github-token": ("ghp_", "github_pat_"),
|
||||
"aws-access-key": ("akia",),
|
||||
"url-user-password": ("://",),
|
||||
"secret-literal": ("password", "passwd", "api", "secret"),
|
||||
}
|
||||
for rule, pattern in SECRET_RULES:
|
||||
if text.isascii() and not any(x in lower for x in fragments[rule]):
|
||||
continue
|
||||
for match in pattern.finditer(text):
|
||||
if rule == "secret-literal":
|
||||
literal = match.group(1) or match.group(2)
|
||||
if literal.lower() in {"none", "null", "false", "true"}:
|
||||
continue
|
||||
raise ValueError("Secret scan refused: " + relative + " [" + rule + "]")
|
||||
|
||||
|
||||
def gzip_rows(path, rows):
|
||||
with Path(path).open("xb") as raw:
|
||||
with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as out:
|
||||
for row in rows:
|
||||
out.write((json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode())
|
||||
|
||||
|
||||
class HashReader:
|
||||
def __init__(self, handle):
|
||||
self.handle, self.hash, self.size = handle, hashlib.sha256(), 0
|
||||
|
||||
def read(self, count=-1):
|
||||
data = self.handle.read(count)
|
||||
self.hash.update(data)
|
||||
self.size += len(data)
|
||||
return data
|
||||
|
||||
|
||||
def zstd_command(decompress=False, threads=2):
|
||||
return ["zstd", "-q", "-d", "-c"] if decompress else ["zstd", "-q", "-3", "-T" + str(threads), "-c"]
|
||||
|
||||
|
||||
def make_archive(root, files, destination, threads):
|
||||
rows = []
|
||||
with destination.open("xb") as output, tempfile.TemporaryFile() as errors:
|
||||
proc = subprocess.Popen(zstd_command(threads=threads), stdin=subprocess.PIPE,
|
||||
stdout=output, stderr=errors)
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.PAX_FORMAT) as archive:
|
||||
for relative, original in files:
|
||||
p = no_links(root / relative)
|
||||
if identity(p.lstat()) != original:
|
||||
raise ValueError("Source changed before archive: " + relative)
|
||||
with p.open("rb") as source:
|
||||
opened = opened_identity(source, original)
|
||||
reader = HashReader(source)
|
||||
info = tarfile.TarInfo(relative)
|
||||
info.size, info.mode, info.mtime = original[3], stat.S_IMODE(original[2]) & 0o777, 0
|
||||
archive.addfile(info, reader)
|
||||
if (reader.size != original[3] or identity(os.fstat(source.fileno())) != opened
|
||||
or identity(p.lstat()) != original):
|
||||
raise ValueError("Source changed during archive: " + relative)
|
||||
rows.append({"path": relative, "bytes": reader.size, "sha256": reader.hash.hexdigest()})
|
||||
proc.stdin.close()
|
||||
if proc.wait() != 0:
|
||||
raise RuntimeError("zstd compression failed; no complete catalog published")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
finally:
|
||||
proc.stdin.close()
|
||||
return rows
|
||||
|
||||
|
||||
def split_archive(archive, group_dir):
|
||||
chunks, full = [], hashlib.sha256()
|
||||
with archive.open("rb") as source:
|
||||
index = 0
|
||||
while first := source.read(min(BUFFER, CHUNK_BYTES)):
|
||||
name = "payload-%06d.bin" % index
|
||||
h, size = hashlib.sha256(), 0
|
||||
with (group_dir / name).open("xb") as out:
|
||||
block = first
|
||||
while block:
|
||||
out.write(block)
|
||||
full.update(block)
|
||||
h.update(block)
|
||||
size += len(block)
|
||||
if size == CHUNK_BYTES:
|
||||
break
|
||||
block = source.read(min(BUFFER, CHUNK_BYTES - size))
|
||||
chunks.append({"path": name, "bytes": size, "sha256": h.hexdigest(), "index": index})
|
||||
index += 1
|
||||
if not chunks:
|
||||
raise ValueError("Empty compressed archive")
|
||||
return chunks, full.hexdigest()
|
||||
|
||||
|
||||
def pack(plan_path, out, work, *, threads=2):
|
||||
if type(threads) is not int or not 1 <= threads <= 64:
|
||||
raise ValueError("threads must be 1..64")
|
||||
plan_bytes = Path(plan_path).read_bytes()
|
||||
plan = json_bytes(plan_bytes)
|
||||
validate_plan(plan)
|
||||
root, out, work = map(no_links, (Path(plan["source_root"]), Path(out), Path(work)))
|
||||
for a, b in ((root, out), (root, work), (out, work)):
|
||||
if a == b or a in b.parents or b in a.parents:
|
||||
raise ValueError("source, out and work must be disjoint trees")
|
||||
if out.exists() or work.exists():
|
||||
raise FileExistsError("out/work must both be new exclusive paths")
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
work.mkdir(parents=True, exist_ok=False)
|
||||
write_new(out / "plan.json", plan_bytes)
|
||||
plan_sha = hashlib.sha256(plan_bytes).hexdigest()
|
||||
results = []
|
||||
for group in plan["groups"]:
|
||||
files, exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
||||
scan_secrets(root, files)
|
||||
group_dir = out / group["id"]
|
||||
group_dir.mkdir()
|
||||
archive = work / (group["id"] + ".tar.zst")
|
||||
members = make_archive(root, files, archive, threads)
|
||||
# A second metadata enumeration catches file additions/removals during pack.
|
||||
after_files, after_exclusions = enumerate_group(root, group, plan.get("exclude_paths", []))
|
||||
if after_files != files or after_exclusions != exclusions:
|
||||
raise ValueError("Source inventory changed during pack")
|
||||
gzip_rows(group_dir / "members.jsonl.gz", members)
|
||||
gzip_rows(group_dir / "exclusions.jsonl.gz", exclusions)
|
||||
chunks, combined = split_archive(archive, group_dir)
|
||||
catalog = dict(schema="lfs-publication-group-v1", group_id=group["id"], source_root=str(root),
|
||||
source_paths=group["paths"], source_files=len(members),
|
||||
source_bytes=sum(x["bytes"] for x in members), license=group["license"],
|
||||
provenance=group["provenance"], plan_sha256=plan_sha,
|
||||
archive_format="tar.pax+zstd", archive_sha256=combined,
|
||||
archive_bytes=sum(x["bytes"] for x in chunks), chunk_bytes=CHUNK_BYTES,
|
||||
chunks=chunks, members=sha_file(group_dir / "members.jsonl.gz"),
|
||||
exclusions=sha_file(group_dir / "exclusions.jsonl.gz"),
|
||||
exclusion_records=len(exclusions),
|
||||
verification_scope="content snapshot, not a license grant or nested-archive secret audit",
|
||||
secret_scan="UTF-8 files <=2 MiB, known binary suffixes excluded; fail closed on matching rules")
|
||||
write_new(group_dir / "catalog.json", canonical(catalog))
|
||||
result = {"group": group["id"], "files": len(members), "source_bytes": catalog["source_bytes"],
|
||||
"archive_bytes": catalog["archive_bytes"], "chunks": len(chunks),
|
||||
"catalog_sha256": sha_file(group_dir / "catalog.json")["sha256"]}
|
||||
results.append(result)
|
||||
print(json.dumps({"group_complete": result}), flush=True)
|
||||
summary = {"schema": "lfs-publication-pack-v1", "complete": True, "plan_sha256": plan_sha,
|
||||
"groups": results, "source_files": sum(x["files"] for x in results),
|
||||
"source_bytes": sum(x["source_bytes"] for x in results)}
|
||||
write_new(out / "pack-summary.json", canonical(summary))
|
||||
return summary
|
||||
|
||||
|
||||
def check_digest_row(row):
|
||||
if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not HEX.fullmatch(str(row.get("sha256", ""))):
|
||||
raise ValueError("Invalid SHA/size metadata")
|
||||
|
||||
|
||||
def iter_members(path):
|
||||
previous = None
|
||||
with gzip.open(path, "rt", encoding="utf-8") as inp:
|
||||
for line in inp:
|
||||
row = json.loads(line)
|
||||
if set(row) != {"path", "bytes", "sha256"}:
|
||||
raise ValueError("Invalid member row")
|
||||
check_digest_row(row)
|
||||
p = relpath(row["path"])
|
||||
if forbidden(p) or (previous is not None and p <= previous):
|
||||
raise ValueError("Forbidden/duplicate/unsorted member: " + p)
|
||||
previous = p
|
||||
yield row
|
||||
|
||||
|
||||
def verify(group_dir):
|
||||
group_dir = no_links(group_dir)
|
||||
controls_before = {name: sha_file(group_dir / name) for name in
|
||||
("catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz")}
|
||||
catalog = json_read(group_dir / "catalog.json")
|
||||
if catalog.get("schema") != "lfs-publication-group-v1" or catalog.get("chunk_bytes") != CHUNK_BYTES or catalog.get("archive_format") != "tar.pax+zstd":
|
||||
raise ValueError("Wrong catalog schema/format/chunk size")
|
||||
chunks = catalog.get("chunks")
|
||||
if not isinstance(chunks, list) or not chunks:
|
||||
raise ValueError("Missing payload chunks")
|
||||
expected = {"catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz"}
|
||||
for i, row in enumerate(chunks):
|
||||
check_digest_row(row)
|
||||
if set(row) != {"path", "bytes", "sha256", "index"} or type(row["index"]) is not int or row["index"] != i or row["path"] != "payload-%06d.bin" % i:
|
||||
raise ValueError("Wrong chunk name/order")
|
||||
if not 0 < row["bytes"] <= CHUNK_BYTES or (i < len(chunks) - 1 and row["bytes"] != CHUNK_BYTES):
|
||||
raise ValueError("Wrong chunk size")
|
||||
expected.add(row["path"])
|
||||
if {p.name for p in group_dir.iterdir()} != expected:
|
||||
raise ValueError("Extra/missing downloaded files")
|
||||
for p in group_dir.iterdir():
|
||||
no_links(p)
|
||||
if not p.is_file():
|
||||
raise ValueError("Nonfile downloaded entry")
|
||||
for name, key in (("members.jsonl.gz", "members"), ("exclusions.jsonl.gz", "exclusions")):
|
||||
check_digest_row(catalog[key])
|
||||
if controls_before[name] != catalog[key]:
|
||||
raise ValueError("Metadata digest mismatch: " + name)
|
||||
with gzip.open(group_dir / "exclusions.jsonl.gz", "rt", encoding="utf-8") as source:
|
||||
omitted = list(map(json.loads, source))
|
||||
if len(omitted) != catalog.get("exclusion_records"):
|
||||
raise ValueError("Exclusion inventory mismatch")
|
||||
for row in omitted:
|
||||
relpath(row["path"])
|
||||
errors, combined, compressed = [], hashlib.sha256(), [0]
|
||||
with tempfile.TemporaryFile() as diagnostics:
|
||||
proc = subprocess.Popen(zstd_command(decompress=True), stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=diagnostics)
|
||||
|
||||
def feed():
|
||||
try:
|
||||
for row in chunks:
|
||||
p = group_dir / row["path"]
|
||||
before = identity(p.lstat())
|
||||
h, size = hashlib.sha256(), 0
|
||||
with p.open("rb") as source:
|
||||
opened = opened_identity(source, before)
|
||||
while data := source.read(BUFFER):
|
||||
h.update(data)
|
||||
combined.update(data)
|
||||
size += len(data)
|
||||
compressed[0] += len(data)
|
||||
proc.stdin.write(data)
|
||||
if identity(os.fstat(source.fileno())) != opened or identity(p.lstat()) != before:
|
||||
raise ValueError("Chunk changed during read")
|
||||
if size != row["bytes"] or h.hexdigest() != row["sha256"]:
|
||||
raise ValueError("Payload digest mismatch: " + row["path"])
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
finally:
|
||||
try:
|
||||
proc.stdin.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
producer = threading.Thread(target=feed, daemon=True)
|
||||
producer.start()
|
||||
count, total = 0, 0
|
||||
expected_rows = iter(iter_members(group_dir / "members.jsonl.gz"))
|
||||
try:
|
||||
with tarfile.open(fileobj=proc.stdout, mode="r|") as archive:
|
||||
for member in archive:
|
||||
row = next(expected_rows, None)
|
||||
if not member.isreg() or member.linkname or member.sparse is not None:
|
||||
raise ValueError("Nonregular/sparse/link tar member refused")
|
||||
path = relpath(member.name)
|
||||
if forbidden(path) or row is None or path != row["path"] or member.size != row["bytes"]:
|
||||
raise ValueError("Tar member set/size/path mismatch: " + path)
|
||||
reader = HashReader(archive.extractfile(member))
|
||||
while reader.read(BUFFER):
|
||||
pass
|
||||
if reader.size != row["bytes"] or reader.hash.hexdigest() != row["sha256"]:
|
||||
raise ValueError("Tar member content mismatch: " + path)
|
||||
count += 1
|
||||
total += reader.size
|
||||
# tarfile stops at the first end marker; reject a hidden second
|
||||
# archive/nonzero trailer, including bytes in its read-ahead buf.
|
||||
while trailer := archive.fileobj.read(BUFFER):
|
||||
if any(trailer):
|
||||
raise ValueError("Nonzero data after tar end marker")
|
||||
if next(expected_rows, None) is not None:
|
||||
raise ValueError("Missing tar members")
|
||||
# Drain to validate zstd's complete stream/checksum and finish the feeder.
|
||||
while proc.stdout.read(BUFFER):
|
||||
pass
|
||||
producer.join()
|
||||
code = proc.wait()
|
||||
if errors:
|
||||
raise ValueError("Payload streaming failed: " + str(errors[0]))
|
||||
if code != 0:
|
||||
raise ValueError("zstd decompression failed")
|
||||
except BaseException:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
producer.join(timeout=10)
|
||||
raise
|
||||
finally:
|
||||
proc.stdout.close()
|
||||
expected_rows.close()
|
||||
if (count != catalog.get("source_files") or total != catalog.get("source_bytes")
|
||||
or compressed[0] != catalog.get("archive_bytes") or combined.hexdigest() != catalog.get("archive_sha256")):
|
||||
raise ValueError("Final inventory/archive digest mismatch")
|
||||
for name, before in controls_before.items():
|
||||
if sha_file(group_dir / name) != before:
|
||||
raise ValueError("Downloaded control changed during verification: " + name)
|
||||
return {"schema": "lfs-publication-verify-v1", "pass_all": True, "group": catalog["group_id"],
|
||||
"source_files": count, "source_bytes": total, "archive_bytes": compressed[0],
|
||||
"archive_sha256": combined.hexdigest(), "catalog_sha256": controls_before["catalog.json"]["sha256"],
|
||||
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility"}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
command = sub.add_parser("pack")
|
||||
command.add_argument("--plan", required=True)
|
||||
command.add_argument("--out", required=True)
|
||||
command.add_argument("--work", required=True)
|
||||
command.add_argument("--threads", type=int, default=2)
|
||||
command = sub.add_parser("verify")
|
||||
command.add_argument("--group-dir", required=True)
|
||||
args = parser.parse_args()
|
||||
result = (pack(args.plan, args.out, args.work, threads=args.threads) if args.command == "pack"
|
||||
else verify(args.group_dir))
|
||||
print(json.dumps(result, sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""One real archive chunk upload and fresh download; never modifies source data."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import lan_lfs_transfer as lan
|
||||
import publish_archives as archives
|
||||
|
||||
|
||||
def run(output):
|
||||
root = Path.cwd()
|
||||
receipt = archives.no_links(output)
|
||||
if receipt.exists() or root / 'registry/publication-20261001' not in receipt.parents:
|
||||
raise ValueError('New publication receipt required')
|
||||
catalog = archives.json_read(root / 'artifacts/public-resources-20261001/malecns-synaptic-partners/catalog.json')
|
||||
chunk = catalog['chunks'][0]
|
||||
oid, size = chunk['sha256'], chunk['bytes']
|
||||
source = root / '.git/lfs/objects' / oid[:2] / oid[2:4] / oid
|
||||
records = []
|
||||
for operation in ('upload', 'download'):
|
||||
agent = lan.Agent(dict(event='init', operation=operation, remote='origin',
|
||||
concurrent=True, concurrenttransfers=2), lambda value: None)
|
||||
original = agent.curl
|
||||
methods = []
|
||||
def observe(method, url, headers, **kwargs):
|
||||
result = original(method, url, headers, **kwargs)
|
||||
methods.append(method)
|
||||
return result
|
||||
agent.curl = observe
|
||||
start = time.perf_counter()
|
||||
result = agent.transfer(dict(event=operation, action=None, oid=oid, size=size,
|
||||
**({'path': str(source)} if operation == 'upload' else {})))
|
||||
record = dict(operation=operation, seconds=time.perf_counter()-start, methods=methods,
|
||||
sha256=oid, bytes=size, complete=result['event'] == 'complete')
|
||||
if operation == 'download':
|
||||
record['new_download_file'] = result['path']
|
||||
records.append(record)
|
||||
report = dict(schema='lfs-lan-roundtrip-smoke-v1', pass_all=True,
|
||||
observed_utc=datetime.now(timezone.utc).isoformat(),
|
||||
ip=lan.IP, hostname=lan.HOST, port=443, certificate_verification=True,
|
||||
proxy_used=False, transfers=records,
|
||||
adapter=archives.sha_file(lan.__file__))
|
||||
archives.write_new(receipt, archives.canonical(report))
|
||||
print(json.dumps(report), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--receipt', required=True)
|
||||
run(parser.parse_args().receipt)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Generate checksums for this publication only, after real download success."""
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
from publish_archives import canonical, no_links, sha_file, write_new
|
||||
|
||||
|
||||
def main():
|
||||
root = no_links(Path.cwd())
|
||||
protected = Path('/data/digital-drosophila')
|
||||
if root == protected or protected in root.parents or not (root / '.git').is_dir():
|
||||
raise ValueError('Only an isolated publication checkout is allowed')
|
||||
folder = root / 'registry/publication-20261001'
|
||||
output, checksums = folder / 'publication-manifest.json', folder / 'SHA256SUMS'
|
||||
if output.exists() or checksums.exists():
|
||||
raise ValueError('Never overwrite a publication manifest')
|
||||
transport = json.loads((folder / 'fresh-download-verification.json').read_bytes())
|
||||
guard = json.loads((folder / 'source-guard-final.json').read_bytes())
|
||||
head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
|
||||
if not (transport['pass_all'] and transport['fresh_download_proven'] and guard['pass_all']
|
||||
and transport['expected_commit'] == head):
|
||||
raise ValueError('Actual transport/source evidence or commit binding is missing')
|
||||
files = [root / p for p in ('.gitattributes', '.gitignore', 'NOTICE.md')]
|
||||
for prefix in ('artifacts/public-resources-20261001', 'artifacts/lfs-smoke',
|
||||
'registry/publication-20261001', 'tools/lfs-publication'):
|
||||
for p in (root / prefix).rglob('*'):
|
||||
no_links(p)
|
||||
if '__pycache__' not in p.parts and p.is_file():
|
||||
files.append(p)
|
||||
rows = [dict(path=p.relative_to(root).as_posix(), **sha_file(p)) for p in sorted(files)]
|
||||
if len({row['path'] for row in rows}) != len(rows):
|
||||
raise ValueError('Duplicate publication path')
|
||||
manifest = dict(schema='lfs-publication-final-manifest-v1', pass_all=True,
|
||||
generated_utc=datetime.now(timezone.utc).isoformat(), data_commit=head,
|
||||
branch='codex/lfs-public-resources', repository_public=True,
|
||||
aggregate_inventory_sha256=transport['aggregate_inventory_sha256'],
|
||||
logical_files=len(rows), logical_bytes=sum(r['bytes'] for r in rows),
|
||||
files=rows,
|
||||
scope='Publication payload, tools, notices and evidence only; original file SHA are nested in members.jsonl.gz. Not an entire server backup or scientific pass.',
|
||||
self_hash_policy='Manifest is covered by SHA256SUMS; SHA256SUMS self-hash is omitted and its bytes are bound by the final Git commit.')
|
||||
write_new(output, canonical(manifest))
|
||||
rows.append(dict(path=output.relative_to(root).as_posix(), **sha_file(output)))
|
||||
write_new(checksums, ''.join(r['sha256'] + ' ' + r['path'] + '\n' for r in sorted(rows, key=lambda r: r['path'])).encode())
|
||||
print(json.dumps(dict(pass_all=True, files=len(rows), manifest=sha_file(output), checksums=sha_file(checksums))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Synthetic temporary fixtures and mocked Git/content operations; no transfer."""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import download_verify as D
|
||||
|
||||
|
||||
class DownloadTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.base = Path(self.temp.name)
|
||||
self.checkout = self.base / "verification"
|
||||
(self.checkout / ".git").mkdir(parents=True)
|
||||
(self.checkout / "registry").mkdir()
|
||||
(self.checkout / D.PAYLOAD).mkdir(parents=True)
|
||||
self.cache = self.checkout / ".git" / "brand-new-lfs"
|
||||
self.receipt = self.checkout / "registry" / "transport.json"
|
||||
self.content_receipt = self.checkout / "registry" / "content.json"
|
||||
self.history = self.base / "history.gz"
|
||||
self.history.write_bytes(b"synthetic history placeholder")
|
||||
self.commit = "a" * 40
|
||||
self.calls = []
|
||||
self.pull_code = self.fsck_code = 0
|
||||
self.remote = D.REMOTE
|
||||
self.head = self.commit
|
||||
self.dirty = ""
|
||||
self.content_failure = False
|
||||
self.config_exit = 1
|
||||
self.adapter_config = {}
|
||||
self.local_adapter_config = {}
|
||||
self.adapter_blob = b"# Synthetic LAN adapter fixture; never executed.\n"
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.object(D.Path, "cwd", return_value=self.checkout).start()
|
||||
patch.object(D, "command", side_effect=self.command).start()
|
||||
self.verifier = patch.object(D.content, "verify_publication", side_effect=self.verify).start()
|
||||
self.capture = contextlib.redirect_stdout(io.StringIO())
|
||||
self.capture.__enter__()
|
||||
self.addCleanup(self.capture.__exit__, None, None, None)
|
||||
|
||||
def command(self, args, *, text=False, bytes_output=False):
|
||||
self.calls.append(args)
|
||||
if "rev-parse" in args:
|
||||
if "HEAD" in args:
|
||||
return self.head
|
||||
return str(self.checkout / ".git") if "--absolute-git-dir" in args else str(self.checkout)
|
||||
if "remote" in args:
|
||||
return self.remote
|
||||
if "status" in args:
|
||||
return self.dirty
|
||||
if "--get-regexp" in args:
|
||||
if "--name-only" in args:
|
||||
return "\n".join(self.adapter_config)
|
||||
return self.config_exit
|
||||
if "--get-all" in args:
|
||||
values = self.local_adapter_config if "--local" in args else self.adapter_config
|
||||
return values.get(args[-1], "")
|
||||
if "show" in args:
|
||||
return self.adapter_blob
|
||||
if "pull" in args:
|
||||
oid = "f" * 64
|
||||
obj = self.cache / "objects" / oid[:2] / oid[2:4] / oid
|
||||
obj.parent.mkdir(parents=True)
|
||||
obj.write_bytes(b"synthetic object")
|
||||
return self.pull_code
|
||||
if "fsck" in args:
|
||||
return self.fsck_code
|
||||
return "synthetic-version"
|
||||
|
||||
def verify(self, root, receipt, history):
|
||||
self.assertEqual(root, self.checkout / D.PAYLOAD)
|
||||
self.assertEqual(history, self.history)
|
||||
if self.content_failure:
|
||||
raise ValueError("synthetic failure")
|
||||
D.archives.write_new(receipt, b'{"synthetic":true}\n')
|
||||
return {"pass_all": True, "receipt_sha256": D.archives.sha_file(receipt)["sha256"],
|
||||
"aggregate_inventory_sha256": "b" * 64, "historical_status": "FULL_CURRENT_MEMBER_MATCH"}
|
||||
|
||||
def run_attempt(self):
|
||||
return D.run(self.commit, self.cache, self.receipt, self.content_receipt, self.history)
|
||||
|
||||
def prepare_adapter(self):
|
||||
adapter = self.checkout / "tools/lfs-publication/lan_lfs_transfer.py"
|
||||
adapter.parent.mkdir(parents=True)
|
||||
adapter.write_bytes(self.adapter_blob)
|
||||
prefix = "lfs.customtransfer.malecns-lan-https."
|
||||
self.adapter_config = {"lfs.standalonetransferagent": "malecns-lan-https",
|
||||
prefix + "path": "/usr/bin/python3", prefix + "args": "-B " + str(adapter),
|
||||
prefix + "concurrent": "true", prefix + "direction": "both"}
|
||||
self.local_adapter_config = self.adapter_config.copy()
|
||||
self.config_exit = 0
|
||||
return adapter
|
||||
|
||||
def assert_not_started(self):
|
||||
self.assertFalse(self.cache.exists())
|
||||
self.assertFalse(self.receipt.exists())
|
||||
self.assertFalse(any("pull" in args for args in self.calls))
|
||||
|
||||
def test_success_binds_new_cache_exits_content_and_source_sha(self):
|
||||
result = self.run_attempt()
|
||||
self.assertEqual(json.loads(self.receipt.read_bytes()), result)
|
||||
self.assertTrue(result["cache"]["initially_absent"])
|
||||
self.assertEqual(result["cache"]["before"], {"objects": 0, "bytes": 0})
|
||||
self.assertEqual(result["cache"]["after"], {"objects": 1, "bytes": 16})
|
||||
self.assertEqual(result["content_receipt"]["sha256"], D.archives.sha_file(self.content_receipt)["sha256"])
|
||||
self.assertEqual(len(result["tool_sources"]), 3)
|
||||
self.assertFalse(result["learning_pass_claimed"])
|
||||
pull = next(args for args in self.calls if "pull" in args)
|
||||
fsck = next(args for args in self.calls if "fsck" in args)
|
||||
self.assertIn("--include=artifacts/**", pull)
|
||||
self.assertIn("--exclude=", pull)
|
||||
self.assertEqual(pull[-1], "origin")
|
||||
self.assertIn("--dry-run", fsck)
|
||||
for args in (pull, fsck):
|
||||
self.assertIn("lfs.storage=" + str(self.cache), args)
|
||||
self.assertIn("lfs.concurrenttransfers=2", args)
|
||||
|
||||
def test_existing_cache_receipt_and_content_receipt_are_never_reused(self):
|
||||
for path in (self.cache, self.receipt, self.content_receipt):
|
||||
with self.subTest(path=path):
|
||||
path.write_bytes(b"keep")
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assertEqual(path.read_bytes(), b"keep")
|
||||
self.assertFalse(any("pull" in args for args in self.calls))
|
||||
path.unlink()
|
||||
|
||||
def test_wrong_head_or_remote_or_dirty_tree_refused(self):
|
||||
for attr, value in (("head", "c" * 40), ("remote", "https://wrong.invalid/repo.git"), ("dirty", " M file")):
|
||||
original = getattr(self, attr)
|
||||
setattr(self, attr, value)
|
||||
with self.subTest(attr=attr), self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assert_not_started()
|
||||
setattr(self, attr, original)
|
||||
|
||||
def test_unsafe_paths_refused_before_transfer(self):
|
||||
original = self.cache
|
||||
for unsafe in (self.checkout / "payload-cache", self.base / ".." / "escape",
|
||||
D.PROTECTED / "should-never-create"):
|
||||
self.cache = unsafe
|
||||
with self.subTest(path=unsafe), self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assertFalse(any("pull" in args for args in self.calls))
|
||||
self.cache = original
|
||||
self.receipt = self.checkout / D.PAYLOAD / "receipt.json"
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assert_not_started()
|
||||
|
||||
def test_pull_failure_retains_partial_and_never_fscks_or_retries(self):
|
||||
self.pull_code = 9
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assertTrue(self.cache.exists())
|
||||
self.assertFalse(self.receipt.exists())
|
||||
self.assertFalse(self.content_receipt.exists())
|
||||
self.assertFalse(any("fsck" in args for args in self.calls))
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assertEqual(sum("pull" in args for args in self.calls), 1)
|
||||
|
||||
def test_endpoint_and_custom_transfer_overrides_fail_closed(self):
|
||||
for code in (0, 2):
|
||||
self.config_exit = code
|
||||
with self.subTest(code=code), self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assert_not_started()
|
||||
self.config_exit = 1
|
||||
(self.checkout / ".lfsconfig").write_text("[lfs]\nurl = https://not-approved.invalid\n")
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assert_not_started()
|
||||
|
||||
def test_optional_lan_adapter_is_exact_local_and_head_bound(self):
|
||||
adapter = self.prepare_adapter()
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assert_not_started()
|
||||
report = D.run(self.commit, self.cache, self.receipt, self.content_receipt,
|
||||
self.history, allow_lan_adapter=True)
|
||||
self.assertTrue(report["lan_adapter"]["enabled"])
|
||||
self.assertEqual(report["tool_sources"]["lan_lfs_transfer.py"]["sha256"],
|
||||
D.archives.sha_file(adapter)["sha256"])
|
||||
|
||||
def test_lan_wrong_blob_global_override_or_other_endpoint_rejected(self):
|
||||
self.prepare_adapter()
|
||||
self.adapter_blob = b"not the actual file"
|
||||
with self.assertRaises(ValueError):
|
||||
D.transfer_policy(self.checkout, self.commit, True)
|
||||
self.adapter_blob = (self.checkout / "tools/lfs-publication/lan_lfs_transfer.py").read_bytes()
|
||||
original = self.local_adapter_config["lfs.standalonetransferagent"]
|
||||
self.local_adapter_config["lfs.standalonetransferagent"] = ""
|
||||
with self.assertRaises(ValueError):
|
||||
D.transfer_policy(self.checkout, self.commit, True)
|
||||
self.local_adapter_config["lfs.standalonetransferagent"] = original
|
||||
self.adapter_config["lfs.url"] = "https://wrong.invalid"
|
||||
with self.assertRaises(ValueError):
|
||||
D.transfer_policy(self.checkout, self.commit, True)
|
||||
self.assert_not_started()
|
||||
|
||||
def test_fsck_failure_does_not_call_content_or_publish_receipt(self):
|
||||
self.fsck_code = 8
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.verifier.assert_not_called()
|
||||
self.assertTrue(self.cache.exists())
|
||||
self.assertFalse(self.receipt.exists())
|
||||
|
||||
def test_content_failure_leaves_no_transport_pass(self):
|
||||
self.content_failure = True
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assertFalse(self.receipt.exists())
|
||||
self.assertEqual(sum("pull" in args for args in self.calls), 1)
|
||||
|
||||
def test_head_changed_during_verification_preserves_content_but_no_transport_pass(self):
|
||||
def changed(*args):
|
||||
result = self.verify(*args)
|
||||
self.head = "d" * 40
|
||||
return result
|
||||
self.verifier.side_effect = changed
|
||||
with self.assertRaises(ValueError):
|
||||
self.run_attempt()
|
||||
self.assertTrue(self.content_receipt.exists())
|
||||
self.assertFalse(self.receipt.exists())
|
||||
|
||||
def test_cli_failure_never_prints_exception_diagnostics(self):
|
||||
with patch.object(D, "run", side_effect=RuntimeError("secret should not escape")), \
|
||||
patch.object(D.sys, "argv", ["download_verify", "--expected-commit", self.commit,
|
||||
"--cache", str(self.cache), "--receipt", str(self.receipt),
|
||||
"--content-receipt", str(self.content_receipt), "--historical-inventory", str(self.history)]), \
|
||||
contextlib.redirect_stdout(io.StringIO()) as output:
|
||||
self.assertEqual(D.main(), 1)
|
||||
self.assertNotIn("secret should not escape", output.getvalue())
|
||||
self.assertIn("FAILED-NOT-RETRIED", output.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Temporary local fixtures only. Identity codec tests do not certify zstd.
|
||||
|
||||
The optional real-zstd test automatically runs when the executable is present.
|
||||
"""
|
||||
import contextlib
|
||||
import gzip
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location("lfs_archive_tested", Path(__file__).with_name("publish_archives.py"))
|
||||
M = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(M)
|
||||
|
||||
|
||||
def codec(*args, **kwargs):
|
||||
return [sys.executable, "-c", "import sys,shutil;shutil.copyfileobj(sys.stdin.buffer,sys.stdout.buffer)"]
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.base = Path(self.temp.name)
|
||||
self.root = self.base / "source"
|
||||
(self.root / "data").mkdir(parents=True)
|
||||
(self.root / "data" / "a.txt").write_text("public source\n", encoding="utf-8")
|
||||
(self.root / "data" / "b.bin").write_bytes(bytes(range(256)) * 80)
|
||||
(self.root / "data" / "zero.txt").write_bytes(b"")
|
||||
self.plan = {"schema": "lfs-publication-plan-v1", "source_root": str(self.root),
|
||||
"groups": [{"id": "data", "paths": ["data"], "license": "test-only",
|
||||
"provenance": {"source": "synthetic"}}], "exclude_paths": []}
|
||||
self.plan_path = self.base / "plan.json"
|
||||
self.plan_path.write_bytes(M.canonical(self.plan))
|
||||
self.out, self.work = self.base / "out", self.base / "work"
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.object(M, "SOURCE_ROOT", str(self.root)).start()
|
||||
patch.object(M, "CHUNK_BYTES", 4096).start()
|
||||
self.codec_patch = patch.object(M, "zstd_command", codec)
|
||||
self.codec_patch.start()
|
||||
|
||||
def pack(self):
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return M.pack(self.plan_path, self.out, self.work)
|
||||
|
||||
def catalog(self):
|
||||
return json.loads((self.out / "data" / "catalog.json").read_bytes())
|
||||
|
||||
def save_catalog(self, cat):
|
||||
(self.out / "data" / "catalog.json").write_bytes(M.canonical(cat))
|
||||
|
||||
def rewrite_archive(self, data):
|
||||
group = self.out / "data"
|
||||
cat = self.catalog()
|
||||
for row in cat["chunks"]:
|
||||
(group / row["path"]).unlink()
|
||||
new = self.base / "mutant.tar"
|
||||
new.write_bytes(data)
|
||||
cat["chunks"], cat["archive_sha256"] = M.split_archive(new, group)
|
||||
cat["archive_bytes"] = len(data)
|
||||
self.save_catalog(cat)
|
||||
|
||||
def test_pack_verify_full_member_content_and_no_extraction(self):
|
||||
before = {p.relative_to(self.root).as_posix(): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
|
||||
summary = self.pack()
|
||||
result = M.verify(self.out / "data")
|
||||
self.assertTrue(result["pass_all"])
|
||||
self.assertEqual(result["source_files"], 3)
|
||||
self.assertEqual(result["source_bytes"], sum(map(len, before.values())))
|
||||
self.assertGreater(summary["groups"][0]["chunks"], 1)
|
||||
self.assertEqual(before, {p.relative_to(self.root).as_posix(): p.read_bytes() for p in self.root.rglob("*") if p.is_file()})
|
||||
self.assertFalse((self.out / "data" / "a.txt").exists())
|
||||
|
||||
def test_plan_executes_first_read_bytes_only(self):
|
||||
with patch.object(M, "json_read", side_effect=AssertionError("Must not read plan a second time")):
|
||||
self.pack()
|
||||
self.assertEqual(self.catalog()["plan_sha256"], hashlib.sha256(self.plan_path.read_bytes()).hexdigest())
|
||||
|
||||
def test_exclusions_logged_without_descending(self):
|
||||
for name in (".git", "venv", "__pycache__", ".cache", "node_modules", "logs", "phase4ar-normal-conditioning"):
|
||||
d = self.root / "data" / name
|
||||
d.mkdir()
|
||||
(d / "hidden.txt").write_text("skip")
|
||||
(self.root / "data" / "private.pem").write_text("private")
|
||||
(self.root / "data" / "run.log").write_text("log")
|
||||
self.pack()
|
||||
with gzip.open(self.out / "data" / "exclusions.jsonl.gz", "rt") as f:
|
||||
rows = [json.loads(x) for x in f]
|
||||
self.assertEqual(len(rows), 9)
|
||||
self.assertEqual(self.catalog()["source_files"], 3)
|
||||
self.assertTrue(M.verify(self.out / "data")["pass_all"])
|
||||
|
||||
def test_unsafe_explicit_paths(self):
|
||||
for name in ("../data", "/data", "data/../data", "data//x", "data\\x", "data/.git", "data/phase4ar-normal-conditioning"):
|
||||
plan = json.loads(json.dumps(self.plan))
|
||||
plan["groups"][0]["paths"] = [name]
|
||||
with self.subTest(name=name), self.assertRaises(ValueError):
|
||||
M.validate_plan(plan)
|
||||
|
||||
def test_overlapping_groups_refused(self):
|
||||
self.plan["groups"].append({"id": "nested", "paths": ["data/a.txt"], "license": "test", "provenance": "test"})
|
||||
with self.assertRaisesRegex(ValueError, "Overlapping"):
|
||||
M.validate_plan(self.plan)
|
||||
|
||||
def test_symlink_refused(self):
|
||||
try:
|
||||
(self.root / "data" / "link").symlink_to(self.root / "data" / "a.txt")
|
||||
except OSError:
|
||||
self.skipTest("OS does not permit temporary symlinks")
|
||||
with self.assertRaisesRegex(ValueError, "Symlink"):
|
||||
self.pack()
|
||||
|
||||
def test_exclusive_and_disjoint_outputs(self):
|
||||
self.out.mkdir()
|
||||
with self.assertRaises(FileExistsError):
|
||||
self.pack()
|
||||
with self.assertRaisesRegex(ValueError, "disjoint"):
|
||||
M.pack(self.plan_path, self.root / "output", self.work)
|
||||
|
||||
def test_parent_components_cannot_bypass_disjoint_gate(self):
|
||||
malicious = self.base / "unused" / ".." / "source" / "new-output"
|
||||
with self.assertRaisesRegex(ValueError, "Parent path components"):
|
||||
M.pack(self.plan_path, malicious, self.work)
|
||||
self.assertFalse((self.root / "new-output").exists())
|
||||
|
||||
def test_secret_rules_hide_values(self):
|
||||
# Build synthetic fixtures at runtime; keep actual credential-like
|
||||
# strings out of Git blobs so the repository secret guard stays useful.
|
||||
examples = ["-----BEGIN " + "OPENSSH PRIVATE " + "KEY-----", "ghp_" + "a" * 36,
|
||||
"github_pat_" + "b" * 40, "AKIA" + "A" * 16,
|
||||
"https://" + "account:" + "privatevalue" + "@example.org/",
|
||||
'password' + ' = "privatevalue"',
|
||||
'api_key' + ': "privatevalue"', 'secret_key' + '=ABCDEF123456',
|
||||
'pa\u017f\u017fword' + '="privatevalue"',
|
||||
'ap\u0131_key' + '="privatevalue"',
|
||||
'\u017fecret_key' + '="privatevalue"']
|
||||
path = self.root / "data" / "literal.txt"
|
||||
for value in examples:
|
||||
path.write_text(value)
|
||||
files, _ = M.enumerate_group(self.root, self.plan["groups"][0], [])
|
||||
with self.subTest(rule=value[:8]), self.assertRaisesRegex(ValueError, "Secret scan refused") as caught:
|
||||
M.scan_secrets(self.root, files)
|
||||
self.assertNotIn(value, str(caught.exception))
|
||||
self.assertNotIn("privatevalue", str(caught.exception))
|
||||
|
||||
def test_mutated_source_metadata_refused(self):
|
||||
files, _ = M.enumerate_group(self.root, self.plan["groups"][0], [])
|
||||
(self.root / "data" / "a.txt").write_text("changed bytes")
|
||||
with self.assertRaisesRegex(ValueError, "Source changed"):
|
||||
M.make_archive(self.root, files, self.base / "bad.tar", 1)
|
||||
|
||||
def test_payload_corruption_refused(self):
|
||||
self.pack()
|
||||
p = self.out / "data" / self.catalog()["chunks"][0]["path"]
|
||||
data = bytearray(p.read_bytes())
|
||||
data[-1] ^= 1
|
||||
p.write_bytes(data)
|
||||
with self.assertRaises((ValueError, tarfile.TarError)):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_missing_extra_or_reordered_payload_refused(self):
|
||||
self.pack()
|
||||
extra = self.out / "data" / "extra.bin"
|
||||
extra.write_bytes(b"x")
|
||||
with self.assertRaisesRegex(ValueError, "Extra/missing"):
|
||||
M.verify(self.out / "data")
|
||||
extra.unlink()
|
||||
cat = self.catalog()
|
||||
cat["chunks"] = list(reversed(cat["chunks"]))
|
||||
self.save_catalog(cat)
|
||||
with self.assertRaisesRegex(ValueError, "chunk name/order"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_tar_traversal_rejected_even_with_rehashed_chunks(self):
|
||||
self.pack()
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as archive:
|
||||
info = tarfile.TarInfo("../escape")
|
||||
info.size = 1
|
||||
archive.addfile(info, io.BytesIO(b"x"))
|
||||
self.rewrite_archive(buf.getvalue())
|
||||
with self.assertRaisesRegex(ValueError, "Unsafe relative path"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_tar_link_rejected_even_with_rehashed_chunks(self):
|
||||
self.pack()
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as archive:
|
||||
info = tarfile.TarInfo("data/a.txt")
|
||||
info.type, info.linkname = tarfile.SYMTYPE, "/tmp/escape"
|
||||
archive.addfile(info)
|
||||
self.rewrite_archive(buf.getvalue())
|
||||
with self.assertRaisesRegex(ValueError, "Nonregular"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_hidden_tar_after_end_refused(self):
|
||||
self.pack()
|
||||
raw = (self.work / "data.tar.zst").read_bytes()
|
||||
self.rewrite_archive(raw + b"nonzero hidden second archive")
|
||||
with self.assertRaisesRegex(ValueError, "Nonzero data after tar"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_second_tar_after_end_refused(self):
|
||||
self.pack()
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as archive:
|
||||
info = tarfile.TarInfo("unlisted-secret.txt")
|
||||
info.size = 1
|
||||
archive.addfile(info, io.BytesIO(b"x"))
|
||||
self.rewrite_archive((self.work / "data.tar.zst").read_bytes() + buf.getvalue())
|
||||
with self.assertRaisesRegex(ValueError, "Nonzero data after tar"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
def test_member_hash_mutation_refused(self):
|
||||
self.pack()
|
||||
p = self.out / "data" / "members.jsonl.gz"
|
||||
with gzip.open(p, "rt") as f:
|
||||
rows = [json.loads(x) for x in f]
|
||||
rows[0]["sha256"] = "0" * 64
|
||||
p.unlink()
|
||||
M.gzip_rows(p, rows)
|
||||
cat = self.catalog()
|
||||
cat["members"] = M.sha_file(p)
|
||||
self.save_catalog(cat)
|
||||
with self.assertRaisesRegex(ValueError, "member content mismatch"):
|
||||
M.verify(self.out / "data")
|
||||
|
||||
@unittest.skipUnless(shutil.which("zstd"), "Real zstd unavailable; run this integration test on Linux")
|
||||
def test_real_zstd_roundtrip(self):
|
||||
self.codec_patch.stop()
|
||||
self.pack()
|
||||
self.assertTrue(M.verify(self.out / "data")["pass_all"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Aggregate verification tests: temporary files and an identity codec only.
|
||||
|
||||
These test metadata/content binding, not zstd itself. The separate archive-tool
|
||||
suite includes its optional real-zstd integration test. No real project data,
|
||||
network, Git mutation, GPU, or persistent outputs are used here.
|
||||
"""
|
||||
import contextlib
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import publish_archives as P
|
||||
import verify_publication as V
|
||||
|
||||
|
||||
def identity_codec(*args, **kwargs):
|
||||
return [sys.executable, "-c",
|
||||
"import sys,shutil;shutil.copyfileobj(sys.stdin.buffer,sys.stdout.buffer)"]
|
||||
|
||||
|
||||
class AggregateTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.base = Path(self.temp.name)
|
||||
self.source = self.base / "source"
|
||||
self.data = self.source / "data"
|
||||
self.data.mkdir(parents=True)
|
||||
(self.data / "LICENSE").write_bytes(b"Synthetic license\n")
|
||||
(self.data / "one.feather").write_bytes(b"fake binary bytes\x00")
|
||||
self.pins = {
|
||||
p.relative_to(self.source).as_posix(): dict(
|
||||
path=p.relative_to(self.source).as_posix(), kind="file", **P.sha_file(p))
|
||||
for p in sorted(self.data.iterdir())
|
||||
}
|
||||
self.plan = {
|
||||
"schema": "lfs-publication-plan-v1", "source_root": str(self.source),
|
||||
"groups": [{
|
||||
"id": "malecns-synaptic-partners", "paths": ["data"],
|
||||
"license": {"evidence_file": "LICENSE",
|
||||
"evidence_sha256": self.pins["data/LICENSE"]["sha256"]},
|
||||
"provenance": {"registered_file_sha256": self.pins["data/one.feather"]["sha256"]},
|
||||
}],
|
||||
"exclude_paths": [],
|
||||
}
|
||||
self.plan_file = self.base / "plan.json"
|
||||
self.root, self.work = self.base / "payload", self.base / "work"
|
||||
self.receipt = self.base / "receipt.json"
|
||||
self.history_file = self.base / "history.jsonl.gz"
|
||||
for item in (patch.object(P, "SOURCE_ROOT", str(self.source)),
|
||||
patch.object(P, "CHUNK_BYTES", 4096),
|
||||
patch.object(P, "zstd_command", identity_codec)):
|
||||
item.start()
|
||||
self.addCleanup(item.stop)
|
||||
|
||||
def pack(self):
|
||||
self.plan_file.write_bytes(P.canonical(self.plan))
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
P.pack(self.plan_file, self.root, self.work)
|
||||
|
||||
def verify(self, history=False):
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return V.verify_publication(self.root, self.receipt,
|
||||
self.history_file if history else None)
|
||||
|
||||
def write_history(self, rows):
|
||||
with gzip.open(self.history_file, "wt", encoding="utf-8") as handle:
|
||||
for row in rows:
|
||||
handle.write(json.dumps(row) + "\n")
|
||||
|
||||
def test_full_without_history_and_exclusive_receipt(self):
|
||||
self.pack()
|
||||
result = self.verify()
|
||||
self.assertTrue(result["pass_all"])
|
||||
self.assertFalse(result["fresh_download_proven"])
|
||||
self.assertEqual(result["historical_status"], "NOT_REQUESTED")
|
||||
self.assertEqual(result["source_files"], 2)
|
||||
with self.assertRaises(ValueError):
|
||||
self.verify()
|
||||
|
||||
def test_full_history_absolute_path_and_skipped_nonfile(self):
|
||||
self.pack()
|
||||
rows = [dict(row) for row in self.pins.values()]
|
||||
# Use the exact registered source_root prefix on either Windows/Linux.
|
||||
rows[0]["path"] = str(self.source) + "/" + rows[0]["path"]
|
||||
rows.append({"path": "outside/link", "kind": "symlink"})
|
||||
self.write_history(rows)
|
||||
result = self.verify(history=True)
|
||||
report = json.loads(self.receipt.read_bytes())
|
||||
self.assertEqual(result["historical_status"], "FULL_CURRENT_MEMBER_MATCH")
|
||||
self.assertEqual(report["historical_inventory"]["nonfile_records_skipped"], 1)
|
||||
|
||||
def test_partial_history_is_not_full_historical_pass(self):
|
||||
self.pack()
|
||||
self.write_history([self.pins["data/LICENSE"],
|
||||
{"path": "data/one.feather", "kind": "symlink"}])
|
||||
self.verify(history=True)
|
||||
history = json.loads(self.receipt.read_bytes())["historical_inventory"]
|
||||
self.assertEqual(history["status"], "PARTIAL_CURRENT_MEMBER_COVERAGE")
|
||||
self.assertFalse(history["all_current_members_historically_matched"])
|
||||
self.assertEqual(history["current_members_without_regular_history"], 1)
|
||||
self.assertEqual(history["current_members_with_nonfile_history"], 1)
|
||||
|
||||
def test_history_mismatch_writes_no_receipt(self):
|
||||
self.pack()
|
||||
self.write_history([dict(self.pins["data/LICENSE"], sha256="0" * 64)])
|
||||
with self.assertRaisesRegex(ValueError, "Historical member mismatch"):
|
||||
self.verify(history=True)
|
||||
self.assertFalse(self.receipt.exists())
|
||||
|
||||
def test_registered_feather_sha_mismatch(self):
|
||||
self.plan["groups"][0]["provenance"]["registered_file_sha256"] = "0" * 64
|
||||
self.pack()
|
||||
with self.assertRaisesRegex(ValueError, "Registered feather SHA"):
|
||||
self.verify()
|
||||
|
||||
def test_multiple_feather_members_refused(self):
|
||||
(self.data / "two.feather").write_bytes(b"another fixture")
|
||||
self.pack()
|
||||
with self.assertRaisesRegex(ValueError, "exactly one feather"):
|
||||
self.verify()
|
||||
|
||||
def test_license_evidence_sha_mismatch(self):
|
||||
self.plan["groups"][0]["license"]["evidence_sha256"] = "0" * 64
|
||||
self.pack()
|
||||
with self.assertRaisesRegex(ValueError, "License evidence SHA"):
|
||||
self.verify()
|
||||
|
||||
def test_extra_root_entry_refused(self):
|
||||
self.pack()
|
||||
(self.root / "unlisted").write_bytes(b"x")
|
||||
with self.assertRaisesRegex(ValueError, "extra/missing"):
|
||||
self.verify()
|
||||
|
||||
def test_wrong_catalog_pin_refused(self):
|
||||
self.pack()
|
||||
summary = P.json_read(self.root / "pack-summary.json")
|
||||
summary["groups"][0]["catalog_sha256"] = "0" * 64
|
||||
(self.root / "pack-summary.json").write_bytes(P.canonical(summary))
|
||||
with self.assertRaisesRegex(ValueError, "Catalog SHA"):
|
||||
self.verify()
|
||||
|
||||
def test_root_file_count_mismatch(self):
|
||||
self.pack()
|
||||
summary = P.json_read(self.root / "pack-summary.json")
|
||||
summary["source_files"] = 3
|
||||
(self.root / "pack-summary.json").write_bytes(P.canonical(summary))
|
||||
with self.assertRaisesRegex(ValueError, "Root source file count"):
|
||||
self.verify()
|
||||
|
||||
def test_receipt_inside_payload_refused(self):
|
||||
self.pack()
|
||||
self.receipt = self.root / "receipt.json"
|
||||
with self.assertRaisesRegex(ValueError, "outside"):
|
||||
self.verify()
|
||||
|
||||
def test_plan_raw_sha_mismatch(self):
|
||||
self.pack()
|
||||
path = self.root / "plan.json"
|
||||
path.write_bytes(path.read_bytes() + b"\n")
|
||||
with self.assertRaisesRegex(ValueError, "plan SHA"):
|
||||
self.verify()
|
||||
|
||||
|
||||
class GitMetadataTests(unittest.TestCase):
|
||||
def test_ascii_head_with_non_ascii_stderr(self):
|
||||
result = subprocess.CompletedProcess([], 0, stdout=b"a" * 40 + b"\n", stderr=bytes([255]))
|
||||
with patch.object(V.subprocess, "run", return_value=result):
|
||||
self.assertEqual(V.git_identity()["head"], "a" * 40)
|
||||
|
||||
def test_failed_git_with_non_ascii_stderr_is_optional(self):
|
||||
result = subprocess.CompletedProcess([], 128, stdout=b"", stderr=bytes([255]))
|
||||
with patch.object(V.subprocess, "run", return_value=result):
|
||||
self.assertIsNone(V.git_identity()["head"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Verify a complete downloaded publication and exclusively write a new receipt.
|
||||
|
||||
This verifies the supplied bytes, not how they arrived or their external trust
|
||||
anchor. No extraction, LFS-cache lookup, source modification, or Git mutation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
import publish_archives as archives
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def equal(left, right):
|
||||
return archives.canonical(left) == archives.canonical(right)
|
||||
|
||||
|
||||
def history_path(value, source_root):
|
||||
require(isinstance(value, str), "Historical path must be text")
|
||||
prefix = source_root.rstrip("/") + "/"
|
||||
if value.startswith(prefix):
|
||||
value = value[len(prefix):]
|
||||
return archives.relpath(value)
|
||||
|
||||
|
||||
def load_history(path, source_root):
|
||||
files, nonfiles, seen = {}, {}, set()
|
||||
total = 0
|
||||
with gzip.open(path, "rt", encoding="utf-8") as inp:
|
||||
for line in inp:
|
||||
row = archives.json_bytes(line)
|
||||
require(isinstance(row, dict), "Historical row must be an object")
|
||||
relative = history_path(row.get("path"), source_root)
|
||||
require(relative not in seen, "Duplicate historical path: " + relative)
|
||||
seen.add(relative)
|
||||
total += 1
|
||||
if row.get("kind", "file") != "file":
|
||||
nonfiles[relative] = row.get("kind")
|
||||
continue
|
||||
archives.check_digest_row(row)
|
||||
files[relative] = {"bytes": row["bytes"], "sha256": row["sha256"]}
|
||||
return files, nonfiles, total
|
||||
|
||||
|
||||
def git_identity():
|
||||
try:
|
||||
process = subprocess.run(["git", "rev-parse", "--verify", "HEAD"],
|
||||
capture_output=True, check=False)
|
||||
head = process.stdout.decode("ascii").strip()
|
||||
if process.returncode == 0 and len(head) in (40, 64) and all(x in "0123456789abcdef" for x in head):
|
||||
return {"cwd": str(Path.cwd()), "head": head,
|
||||
"scope": "HEAD observed at invocation cwd; not a clean-tree or payload-to-commit proof"}
|
||||
except (OSError, UnicodeError):
|
||||
pass
|
||||
return {"cwd": str(Path.cwd()), "head": None, "scope": "Git HEAD unavailable"}
|
||||
|
||||
|
||||
def verify_publication(payload_root, receipt, historical_inventory=None):
|
||||
root, receipt = archives.no_links(payload_root), archives.no_links(receipt)
|
||||
require(root.is_dir(), "Payload root is not a directory")
|
||||
require(receipt != root and root not in receipt.parents,
|
||||
"Receipt must be outside the immutable payload tree")
|
||||
require(not receipt.exists(), "Receipt already exists; refusing overwrite/retry")
|
||||
require(receipt.parent.is_dir(), "Receipt parent must already exist")
|
||||
controls = {name: archives.sha_file(root / name) for name in ("plan.json", "pack-summary.json")}
|
||||
plan = archives.json_read(root / "plan.json")
|
||||
summary = archives.json_read(root / "pack-summary.json")
|
||||
archives.validate_plan(plan)
|
||||
require(summary.get("schema") == "lfs-publication-pack-v1" and summary.get("complete") is True,
|
||||
"Pack summary is not complete")
|
||||
require(summary.get("plan_sha256") == controls["plan.json"]["sha256"], "Root plan SHA mismatch")
|
||||
plan_groups = {g["id"]: g for g in plan["groups"]}
|
||||
summary_groups = summary.get("groups")
|
||||
require(isinstance(summary_groups, list), "Missing summary groups")
|
||||
require(all(isinstance(g, dict) and isinstance(g.get("group"), str) for g in summary_groups),
|
||||
"Invalid summary group")
|
||||
listed = [g["group"] for g in summary_groups]
|
||||
require(listed == list(plan_groups), "Summary group inventory/order differs from plan")
|
||||
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
|
||||
"Payload root has extra/missing entries; preserve and investigate")
|
||||
for p in root.iterdir():
|
||||
archives.no_links(p)
|
||||
historical = None
|
||||
if historical_inventory is not None:
|
||||
hpath = archives.no_links(historical_inventory)
|
||||
require(hpath != receipt, "Receipt cannot replace historical inventory")
|
||||
hpin = archives.sha_file(hpath)
|
||||
hfiles, hnonfiles, htotal = load_history(hpath, plan["source_root"])
|
||||
historical = {"path": str(hpath), "sha256": hpin["sha256"], "bytes": hpin["bytes"],
|
||||
"records": htotal, "regular_file_records": len(hfiles),
|
||||
"nonfile_records_skipped": len(hnonfiles)}
|
||||
else:
|
||||
hfiles, hnonfiles = {}, {}
|
||||
matched, missing, nonfile_current = 0, [], 0
|
||||
publication_paths, results, manifest_controls = set(), [], {}
|
||||
for group_summary in summary_groups:
|
||||
group_id = group_summary["group"]
|
||||
group, directory = plan_groups[group_id], root / group_id
|
||||
require(directory.is_dir(), "Missing group directory: " + group_id)
|
||||
cat_path = directory / "catalog.json"
|
||||
cat_pin = archives.sha_file(cat_path)
|
||||
require(cat_pin["sha256"] == group_summary.get("catalog_sha256"), "Catalog SHA mismatch: " + group_id)
|
||||
catalog = archives.json_read(cat_path)
|
||||
require(catalog.get("group_id") == group_id and catalog.get("source_root") == plan["source_root"]
|
||||
and catalog.get("plan_sha256") == controls["plan.json"]["sha256"],
|
||||
"Catalog identity/plan differs: " + group_id)
|
||||
for key, wanted in (("source_paths", group["paths"]), ("license", group["license"]),
|
||||
("provenance", group["provenance"])):
|
||||
require(equal(catalog.get(key), wanted), "Catalog plan metadata mismatch: " + group_id + "/" + key)
|
||||
actual = archives.verify(directory)
|
||||
expected_summary = {"group": group_id, "files": actual["source_files"],
|
||||
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"],
|
||||
"chunks": len(catalog["chunks"]), "catalog_sha256": cat_pin["sha256"]}
|
||||
require(equal(group_summary, expected_summary), "Group summary differs from actual payload: " + group_id)
|
||||
members_path = directory / "members.jsonl.gz"
|
||||
manifest_controls[str(members_path)] = catalog["members"]
|
||||
manifest_controls[str(cat_path)] = cat_pin
|
||||
manifest_controls[str(directory / "exclusions.jsonl.gz")] = catalog["exclusions"]
|
||||
feather, member_lookup, group_matched, group_missing = [], {}, 0, 0
|
||||
for row in archives.iter_members(members_path):
|
||||
name = row["path"]
|
||||
require(name not in publication_paths, "Duplicate member across groups: " + name)
|
||||
require(any(name == p or name.startswith(p + "/") for p in group["paths"]),
|
||||
"Member outside group whitelist: " + name)
|
||||
require(not any(name == p or name.startswith(p + "/") for p in plan.get("exclude_paths", [])),
|
||||
"Excluded path appears in payload: " + name)
|
||||
publication_paths.add(name)
|
||||
member_lookup[name] = row
|
||||
if name.lower().endswith(".feather"):
|
||||
feather.append(row)
|
||||
if historical is not None:
|
||||
if name in hfiles:
|
||||
require(equal(hfiles[name], {"bytes": row["bytes"], "sha256": row["sha256"]}),
|
||||
"Historical member mismatch: " + name)
|
||||
matched += 1
|
||||
group_matched += 1
|
||||
else:
|
||||
missing.append(name)
|
||||
group_missing += 1
|
||||
nonfile_current += int(name in hnonfiles)
|
||||
require(len(member_lookup) == actual["source_files"], "Member inventory count changed")
|
||||
provenance = group["provenance"]
|
||||
registered = provenance.get("registered_file_sha256") if isinstance(provenance, dict) else None
|
||||
if group_id == "malecns-synaptic-partners":
|
||||
require(registered is not None, "Synaptic-partners registered file SHA missing")
|
||||
file_anchor = None
|
||||
if registered is not None:
|
||||
require(isinstance(registered, str) and archives.HEX.fullmatch(registered) is not None,
|
||||
"Invalid registered file SHA")
|
||||
require(len(feather) == 1, "Registered-file group must contain exactly one feather: " + group_id)
|
||||
require(feather[0]["sha256"] == registered, "Registered feather SHA mismatch: " + group_id)
|
||||
file_anchor = feather[0]
|
||||
license_anchor = None
|
||||
license_meta = group["license"]
|
||||
if isinstance(license_meta, dict) and "evidence_file" in license_meta:
|
||||
relative = archives.relpath(license_meta["evidence_file"])
|
||||
path = group["paths"][0] + "/" + relative
|
||||
require(path in member_lookup, "License evidence absent from payload: " + path)
|
||||
require(member_lookup[path]["sha256"] == license_meta.get("evidence_sha256"),
|
||||
"License evidence SHA mismatch: " + path)
|
||||
license_anchor = member_lookup[path]
|
||||
result = dict(actual, catalog_path=str(cat_path), registered_file_verified=file_anchor,
|
||||
license_evidence_verified=license_anchor,
|
||||
historical_matched=group_matched if historical is not None else None,
|
||||
historical_missing=group_missing if historical is not None else None)
|
||||
results.append(result)
|
||||
print(json.dumps({"group_verified": group_id, "source_files": actual["source_files"],
|
||||
"source_bytes": actual["source_bytes"], "archive_bytes": actual["archive_bytes"]}), flush=True)
|
||||
files = sum(r["source_files"] for r in results)
|
||||
source_bytes = sum(r["source_bytes"] for r in results)
|
||||
archive_bytes = sum(r["archive_bytes"] for r in results)
|
||||
require(type(summary.get("source_files")) is int and summary["source_files"] == files,
|
||||
"Root source file count mismatch")
|
||||
require(type(summary.get("source_bytes")) is int and summary["source_bytes"] == source_bytes,
|
||||
"Root source byte count mismatch")
|
||||
for name, pin in controls.items():
|
||||
require(archives.sha_file(root / name) == pin, "Root control changed during verification: " + name)
|
||||
for path, pin in manifest_controls.items():
|
||||
require(archives.sha_file(path) == pin, "Group control changed during verification: " + path)
|
||||
require({p.name for p in root.iterdir()} == set(plan_groups) | {"plan.json", "pack-summary.json"},
|
||||
"Payload root inventory changed during verification")
|
||||
if historical is not None:
|
||||
require(archives.sha_file(hpath) == hpin, "Historical inventory changed during verification")
|
||||
historical.update(status="FULL_CURRENT_MEMBER_MATCH" if not missing else "PARTIAL_CURRENT_MEMBER_COVERAGE",
|
||||
all_current_members_historically_matched=not missing,
|
||||
current_members_matched=matched, current_members_without_regular_history=len(missing),
|
||||
current_members_with_nonfile_history=nonfile_current,
|
||||
missing_sample=missing[:20], missing_sample_truncated=len(missing) > 20,
|
||||
historical_regular_files_outside_publication=len(set(hfiles) - publication_paths),
|
||||
scope="Only publication members with regular-file history compared; not a full historical source-tree audit")
|
||||
else:
|
||||
historical = {"status": "NOT_REQUESTED", "all_current_members_historically_matched": None}
|
||||
aggregate = [{"group": r["group"], "catalog_sha256": r["catalog_sha256"],
|
||||
"archive_sha256": r["archive_sha256"], "source_files": r["source_files"],
|
||||
"source_bytes": r["source_bytes"], "archive_bytes": r["archive_bytes"]} for r in results]
|
||||
report = {"schema": "lfs-publication-aggregate-verification-v1", "pass_all": True,
|
||||
"verified_utc": datetime.now(timezone.utc).isoformat(), "payload_root": str(root),
|
||||
"receipt": str(receipt), "plan": {"path": str(root / "plan.json"), **controls["plan.json"]},
|
||||
"pack_summary": {"path": str(root / "pack-summary.json"), **controls["pack-summary.json"]},
|
||||
"groups": results, "source_files": files, "source_bytes": source_bytes,
|
||||
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": hashlib.sha256(archives.canonical(aggregate)).hexdigest(),
|
||||
"historical_inventory": historical, "git": git_identity(),
|
||||
"fresh_download_proven": False,
|
||||
"scope": "Given payload bytes verified against plan/catalog/members and optional historical overlap; provenance/plan authenticity and fresh-clone transfer evidence are caller responsibilities"}
|
||||
archives.write_new(receipt, archives.canonical(report))
|
||||
return {"pass_all": True, "receipt": str(receipt), "receipt_sha256": archives.sha_file(receipt)["sha256"],
|
||||
"groups": len(results), "source_files": files, "source_bytes": source_bytes,
|
||||
"archive_bytes": archive_bytes, "aggregate_inventory_sha256": report["aggregate_inventory_sha256"],
|
||||
"historical_status": historical["status"], "fresh_download_proven": False}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--payload-root", required=True)
|
||||
parser.add_argument("--receipt", required=True)
|
||||
parser.add_argument("--historical-inventory")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(verify_publication(args.payload_root, args.receipt, args.historical_inventory),
|
||||
sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user