consolidate-main-preserve-science-lfs-cpu-gpu-history
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,122 @@
|
||||
"""Explicitly authorized, fail-closed archive/retire operations in the new clone."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path('/home/qishizhe/digital-drosophila-main-integration-20261001')
|
||||
RECOVERY = Path('/home/qishizhe/digital-drosophila-branch-recovery-20261001.git')
|
||||
AUDIT = ROOT / 'registry/branch-consolidation-20261001'
|
||||
|
||||
|
||||
def require(value, message):
|
||||
if not value:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def git(*args, cwd=ROOT):
|
||||
# Existing payloads were checked separately. This operation transfers refs
|
||||
# and Git objects only; it neither replaces nor uploads any LFS object.
|
||||
env = dict(os.environ, GIT_LFS_SKIP_SMUDGE='1', GIT_LFS_SKIP_PUSH='1')
|
||||
return subprocess.check_output(['git', *args], cwd=cwd, env=env, text=True).strip()
|
||||
|
||||
|
||||
def snapshot():
|
||||
refs = {}
|
||||
default = None
|
||||
for line in git('ls-remote', '--symref', 'origin').splitlines():
|
||||
value, name = line.split('\t')
|
||||
if value.startswith('ref: '):
|
||||
if name == 'HEAD':
|
||||
default = value[5:]
|
||||
else:
|
||||
refs[name] = value
|
||||
return refs, default
|
||||
|
||||
|
||||
def write_receipt(name, **data):
|
||||
with (AUDIT / name).open('x') as f:
|
||||
json.dump(dict(schema='branch-consolidation-ref-audit-v1',
|
||||
created_utc=datetime.now(timezone.utc).isoformat(), **data),
|
||||
f, indent=2, sort_keys=True)
|
||||
f.write('\n')
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
def main(action):
|
||||
require(Path.cwd().resolve() == ROOT, 'Independent integration clone required')
|
||||
plan = json.loads((AUDIT / 'plan.json').read_text())
|
||||
require(git('remote', 'get-url', 'origin') == plan['remote'], 'Unexpected remote')
|
||||
branches = {'refs/heads/' + name: sha for name, sha in plan['branches'].items()}
|
||||
tags = {plan['archive_tag_prefix'] + name.removeprefix('codex/'): sha
|
||||
for name, sha in plan['branches'].items()}
|
||||
before, default_before = snapshot()
|
||||
|
||||
if action == 'archive':
|
||||
require(not (AUDIT / 'archive-recovery.json').exists(), 'Archive receipt already exists')
|
||||
require(not RECOVERY.exists(), 'Recovery clone already exists; inspect, do not replace')
|
||||
require({k: v for k, v in before.items() if k.startswith('refs/heads/')} == branches,
|
||||
'Remote heads changed since authorized inventory')
|
||||
for tag, sha in tags.items():
|
||||
ref = 'refs/tags/' + tag
|
||||
require(ref not in before, 'Archive tag already exists; inspect before proceeding')
|
||||
require(not git('tag', '--list', tag), 'Local archive tag already exists')
|
||||
git('tag', '-a', tag, sha, '-m',
|
||||
'Archive before user-authorized main consolidation; preserve this exact historical tip. Do not rewrite.')
|
||||
git('push', '--atomic', 'origin', *['refs/tags/' + tag + ':refs/tags/' + tag for tag in tags])
|
||||
after, _ = snapshot()
|
||||
for tag, sha in tags.items():
|
||||
require(after.get('refs/tags/' + tag + '^{}') == sha, 'Remote archived commit differs')
|
||||
git('init', '--bare', str(RECOVERY))
|
||||
git('fetch', '--no-tags', plan['remote'],
|
||||
*['refs/tags/' + tag + ':refs/tags/' + tag for tag in tags], cwd=RECOVERY)
|
||||
recovered = {tag: git('rev-parse', 'refs/tags/' + tag + '^{commit}', cwd=RECOVERY)
|
||||
for tag in tags}
|
||||
require(recovered == tags, 'Fresh remote recovery failed')
|
||||
git('fsck', '--full', '--no-reflogs', cwd=RECOVERY)
|
||||
write_receipt('archive-recovery.json', pass_all=True, remote=plan['remote'],
|
||||
remote_heads_before=branches, remote_default_before=default_before,
|
||||
tags=tags, fresh_remote_recovery_path=str(RECOVERY),
|
||||
recovered_commits=recovered, git_fsck_pass=True,
|
||||
lfs_payload_downloaded=False, old_remote_branches_deleted=False)
|
||||
|
||||
elif action == 'retire':
|
||||
require(not (AUDIT / 'remote-cleanup.json').exists(), 'Cleanup receipt already exists')
|
||||
require(json.loads((AUDIT / 'archive-recovery.json').read_text())['pass_all'], 'Recovery not verified')
|
||||
require(json.loads((AUDIT / 'merge-audit-final.json').read_text())['pass_all'], 'Merge not verified')
|
||||
require(default_before == 'refs/heads/main', 'Remote default must already be main')
|
||||
require(git('branch', '--show-current') == 'main', 'Wrong local branch')
|
||||
require(git('status', '--porcelain') == '', 'Integration clone is not clean')
|
||||
main_sha = git('rev-parse', 'HEAD')
|
||||
expected = dict(branches, **{'refs/heads/main': main_sha})
|
||||
require({k: v for k, v in before.items() if k.startswith('refs/heads/')} == expected,
|
||||
'Remote heads changed; stop cleanup')
|
||||
for tag, sha in tags.items():
|
||||
require(before.get('refs/tags/' + tag + '^{}') == sha, 'Remote archive missing/different')
|
||||
require(git('rev-parse', 'refs/tags/' + tag + '^{commit}', cwd=RECOVERY) == sha,
|
||||
'Recovery archive differs')
|
||||
git('merge-base', '--is-ancestor', sha, main_sha)
|
||||
leases = ['--force-with-lease=' + ref + ':' + sha for ref, sha in branches.items()]
|
||||
# Delete only these seven exact branch refs, atomically and with their
|
||||
# recorded old OIDs. No local refs, tags, commits, worktrees or LFS deletion.
|
||||
git('push', '--atomic', *leases, 'origin', *[':' + ref for ref in branches])
|
||||
after, default_after = snapshot()
|
||||
require({k: v for k, v in after.items() if k.startswith('refs/heads/')} ==
|
||||
{'refs/heads/main': main_sha}, 'Unexpected final heads')
|
||||
require(default_after == 'refs/heads/main', 'Default branch changed unexpectedly')
|
||||
for tag, sha in tags.items():
|
||||
require(after.get('refs/tags/' + tag + '^{}') == sha, 'Archive lost after deletion')
|
||||
write_receipt('remote-cleanup.json', pass_all=True, remote=plan['remote'],
|
||||
main_commit=main_sha, remote_default=default_after,
|
||||
deleted_remote_branch_refs=branches, preserved_archive_tags=tags,
|
||||
remaining_remote_heads={'main': main_sha},
|
||||
all_old_tips_ancestors_of_main=True, local_branches_deleted=False,
|
||||
source_worktrees_write_operations=False, lfs_objects_deleted=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('action', choices=['archive', 'retire'])
|
||||
main(p.parse_args().action)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Validate staged integration against both original trees without science runs."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def require(value, message):
|
||||
if not value:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.check_output(['git', *args])
|
||||
|
||||
|
||||
def tree(ref):
|
||||
result = {}
|
||||
for row in git('ls-tree', '-r', '-z', ref).split(b'\0'):
|
||||
if row:
|
||||
meta, path = row.split(b'\t', 1)
|
||||
mode, kind, oid = meta.decode().split()
|
||||
require(kind == 'blob', 'Unexpected tree object')
|
||||
result[path.decode()] = (mode, oid)
|
||||
return result
|
||||
|
||||
|
||||
def main(output, check_lfs):
|
||||
root = Path.cwd()
|
||||
require(root == Path('/home/qishizhe/digital-drosophila-main-integration-20261001'), 'Independent integration clone required')
|
||||
plan = json.loads((root / 'registry/branch-consolidation-20261001/plan.json').read_text())
|
||||
left, right = plan['merge_parents']
|
||||
base = git('merge-base', left, right).decode().strip()
|
||||
a, b, old = tree(left), tree(right), tree(base)
|
||||
staged = {}
|
||||
for row in git('ls-files', '--stage', '-z').split(b'\0'):
|
||||
if row:
|
||||
meta, path = row.split(b'\t', 1)
|
||||
mode, oid, stage = meta.decode().split()
|
||||
require(stage == '0', 'Unresolved merge conflict')
|
||||
staged[path.decode()] = (mode, oid)
|
||||
allowed = set(plan['allowed_existing_file_changes'])
|
||||
checked = 0
|
||||
for name in sorted(set(a) | set(b)):
|
||||
require(name in staged, 'Original file removed: ' + name)
|
||||
if name in allowed:
|
||||
continue
|
||||
av, bv, ov = a.get(name), b.get(name), old.get(name)
|
||||
if av == bv:
|
||||
expected = av
|
||||
elif av == ov:
|
||||
expected = bv
|
||||
elif bv == ov:
|
||||
expected = av
|
||||
elif av is None:
|
||||
expected = bv
|
||||
elif bv is None:
|
||||
expected = av
|
||||
else:
|
||||
raise ValueError('Unapproved conflicting original file: ' + name)
|
||||
require(staged[name] == expected, 'Original blob or mode changed: ' + name)
|
||||
checked += 1
|
||||
new_paths = sorted(set(staged) - set(a) - set(b))
|
||||
for name in new_paths:
|
||||
require(any(name.startswith(p) if p.endswith('/') else name == p for p in plan['new_paths']), 'Unapproved new path')
|
||||
artifacts = {p: v for p, v in staged.items() if p.startswith('artifacts/')}
|
||||
require(len(artifacts) == plan['expected_artifact_paths'], 'Artifact path inventory differs')
|
||||
pointers = {}
|
||||
pattern = re.compile(rb'version https://git-lfs.github.com/spec/v1\noid sha256:([0-9a-f]{64})\nsize ([0-9]+)\n?')
|
||||
for name, (mode, oid) in artifacts.items():
|
||||
data = git('cat-file', 'blob', oid)
|
||||
match = pattern.fullmatch(data)
|
||||
if match:
|
||||
pointers[name] = dict(oid=match[1].decode(), bytes=int(match[2]), git_blob=oid)
|
||||
require(len(pointers) == plan['expected_lfs_pointer_paths'], 'LFS pointer path inventory differs')
|
||||
attributes = git('check-attr', '--cached', 'filter', '--', *pointers).decode().splitlines()
|
||||
require(len(attributes) == len(pointers) and all(x.endswith(': filter: lfs') for x in attributes), 'Merged LFS rules do not cover every original object')
|
||||
lfs_report = None
|
||||
if check_lfs:
|
||||
source = root / 'tools/lfs-publication/lan_lfs_transfer.py'
|
||||
require(hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256(git('show', left + ':tools/lfs-publication/lan_lfs_transfer.py')).digest(), 'Transfer adapter changed')
|
||||
spec = importlib.util.spec_from_file_location('merge_lan', source)
|
||||
lan = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(lan)
|
||||
agent = lan.Agent(dict(event='init', operation='download', remote='origin', concurrent=True, concurrenttransfers=2), lambda _: None)
|
||||
href, headers = agent.authenticate()
|
||||
unique = {v['oid']: v['bytes'] for v in pointers.values()}
|
||||
requested = [{'oid': oid, 'size': size} for oid, size in sorted(unique.items())]
|
||||
batches = 0
|
||||
for begin in range(0, len(requested), 100):
|
||||
items = requested[begin:begin + 100]
|
||||
response = lan.decode_json(agent.curl('POST', href + '/objects/batch', headers,
|
||||
body=dict(operation='download', transfers=['basic'], objects=items)))
|
||||
require(response.get('transfer', 'basic') == 'basic', 'Unexpected LFS transfer mode')
|
||||
received = response.get('objects', [])
|
||||
require(len(received) == len(items), 'LFS metadata inventory incomplete')
|
||||
seen = set()
|
||||
for item in received:
|
||||
oid = item.get('oid')
|
||||
require(oid in unique and oid not in seen and item.get('size') == unique[oid] and 'error' not in item, 'LFS object unavailable')
|
||||
action = item.get('actions', {}).get('download')
|
||||
require(isinstance(action, dict), 'LFS object has no download action')
|
||||
lan.safe_url(action['href'])
|
||||
lan.safe_headers(action.get('header', {}))
|
||||
seen.add(oid)
|
||||
require(seen == {item['oid'] for item in items}, 'LFS response object set differs')
|
||||
batches += 1
|
||||
lfs_report = dict(unique_objects=len(unique), unique_bytes=sum(unique.values()), batches=batches,
|
||||
all_download_actions_available=True, payloads_redownloaded=False,
|
||||
scope='Authenticated read-only LFS availability; full byte validation remains the unchanged prior fresh-download receipts')
|
||||
report = dict(schema='main-integration-audit-v1', verified_utc=datetime.now(timezone.utc).isoformat(),
|
||||
pass_all=True, parents=[left, right], merge_base=base, staged_original_blobs_checked=checked,
|
||||
allowed_existing_changes=sorted(allowed), new_paths=new_paths, artifact_paths=len(artifacts),
|
||||
lfs_pointer_paths=len(pointers), pointers=pointers, lfs_availability=lfs_report,
|
||||
model_tool_registry_artifact_bytes_unchanged=True, scientific_execution=False,
|
||||
source_worktree_write_operations_by_this_tool=False,
|
||||
source_worktree_integrity_requires_separate_audit=True,
|
||||
staged_tree=git('write-tree').decode().strip())
|
||||
with output.open('x') as stream:
|
||||
json.dump(report, stream, indent=2, sort_keys=True)
|
||||
stream.write('\n')
|
||||
print(json.dumps({k: v for k, v in report.items() if k not in ('pointers', 'new_paths')}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--check-lfs', action='store_true')
|
||||
args = parser.parse_args()
|
||||
main(args.output, args.check_lfs)
|
||||
Reference in New Issue
Block a user