"""Fetch exactly the externally pinned AN registry, never remote brain bulk. No network at import. ``fetch`` accepts an authenticated read-only SFTP client; the CLI requires existing host trust and environment-only credentials. Local paths preserve registry relative names. A partial destination is not resumed. This checks selected bytes and publication cross-links, not scientific gates or the untransferred experiment/model payload. """ from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path, PurePosixPath import re import stat REMOTE_ROOT = '/data/digital-drosophila' EXP_REL = 'experiments/phase4an-engineering-validation' MODEL_REL = 'models/male-cns-lif/olfactory-conditioning-lab-v1' REGISTRY_REL = 'registry/phase4an-engineering-validation' REGISTRY_ROOT = REMOTE_ROOT + '/' + REGISTRY_REL DEFAULT_DESTINATION = Path(__file__).resolve().parents[2] / 'reports/phase4an-engineering-validation' PREREG_SHA = '7317441126f191ccc6e6d28438870b3921537ebe282a6acb163716a690057945' MAX_FILE_BYTES = 4 * 1024**2 MAX_TOTAL_BYTES = 1024**3 SUFFIXES = frozenset(('.json', '.csv', '.md', '.log', '.sha256')) MODULES = ('runtime_light', 'runtime_continuation', 'engineering_inputs', 'engineering_plan', 'engineering_compare', 'engineering_queue_audit', 'engineering_prepare', 'engineering_workflow', 'engineering_scopes', 'engineering_analysis') SOURCE_FILES = tuple(sorted([m + '.py' for m in MODULES] + ['test_' + m + '.py' for m in MODULES] + ['ENGINEERING_GATE_DRAFT.md'])) GENERATED = frozenset(('seal-verification.json', 'evidence-index.json', 'source-identity.json')) CONTROLS = ('manifest.json', 'checksums.sha256') PASS = 'PASS-EXACT-LIGHT-CONTINUATION-ENGINEERING-NOT-LEARNING' STATUSES = frozenset((PASS, 'FAIL-NUMERICAL', 'INCOMPLETE-COVERAGE')) SHA = re.compile('[0-9a-f]{64}\\Z') def require(ok, message): if not ok: raise ValueError(message) def sha(data): return hashlib.sha256(data).hexdigest() def json_bytes(data): def unique(pairs): out = {} for key, value in pairs: require(key not in out, 'Duplicate JSON key') out[key] = value return out def bad(value): raise ValueError('Nonfinite JSON constant') return json.loads(data.decode('utf-8'), object_pairs_hook=unique, parse_constant=bad) def relative_name(value): require(type(value) is str and value and not any(ord(c) < 32 or c in '\\:"<>|?*' for c in value), 'Invalid relative filename') path = PurePosixPath(value) require(not path.is_absolute() and path.as_posix() == value and all(x not in ('.', '..', '__pycache__') and not x.endswith((' ', '.')) for x in path.parts), 'Noncanonical relative filename') devices = {'CON', 'PRN', 'AUX', 'NUL', *(prefix + number for prefix in ('COM', 'LPT') for number in '123456789¹²³')} require(all(x.split('.')[0].upper() not in devices for x in path.parts), 'Reserved local filename') return path def file_row(row, small=False): require(type(row) is dict and set(row) == {'path', 'bytes', 'sha256'}, 'Unexpected file row') relative_name(row['path']) require(type(row['bytes']) is int and row['bytes'] >= 0 and type(row['sha256']) is str and SHA.fullmatch(row['sha256']), 'Invalid size/SHA') require(not small or row['bytes'] <= MAX_FILE_BYTES, 'File exceeds 4 MiB') return row def _collision_check(names): folded = [relative_name(name).as_posix().casefold() for name in names] values = set(folded) require(len(values) == len(folded) and not any(parent.as_posix().casefold() in values for name in names for parent in PurePosixPath(name).parents if str(parent) != '.'), 'Local case/prefix path collision') def selected(row): path = PurePosixPath(row['path']) if path.parts[0] == 'extension-build': return None, 'build_artifact' if row['bytes'] > MAX_FILE_BYTES: return None, 'exceeds_4_MiB' if path.parts[:2] == ('protocol', 'source-snapshot') and len(path.parts) == 3 and path.name in SOURCE_FILES: return 'source-snapshot/' + path.name, None if path.suffix.lower() not in SUFFIXES: return None, 'bulk_binary_or_unselected_suffix_including_jsonl' return 'evidence/' + row['path'], None def validate_receipt(raw, expected_sha): require(sha(raw) == expected_sha, 'External receipt SHA mismatch') receipt = json_bytes(raw) require(type(receipt) is dict and receipt.get('schema') == 'phase4an-engineering-publication-v1' and receipt.get('publication_complete') is True and receipt.get('project_root') == REMOTE_ROOT and receipt.get('registry') == REGISTRY_REL and receipt.get('declared_status') in STATUSES, 'Wrong/incomplete publication receipt') for key in ('used_sources_verified_before_and_after', 'complete_checksums_reference_bulk_not_copied', 'publication_success_is_not_engineering_or_learning_pass'): require(receipt.get(key) is True, 'Missing publication qualification: ' + key) for key in ('model_directory_sealed', 'model_files_copied', 'complete_bulk_backup', 'numerical_gate_rerun_by_publisher', 'training', 'candidate_generated', 'learning_pass_claimed', 'existing_files_modified', 'helper_globals_modified'): require(receipt.get(key) is False, 'Unexpected publication claim: ' + key) require(receipt.get('selection_policy') == dict(max_file_bytes=MAX_FILE_BYTES, suffixes=sorted(SUFFIXES), source_snapshot_names=list(SOURCE_FILES), excluded_tree='extension-build', all_jsonl_excluded=True), 'Selection policy changed') require(receipt.get('used_source_count') == 21 and receipt.get('completed_calls') == 32 and receipt.get('executed_ticks') == 11074 and receipt.get('preregistration_sha256') == PREREG_SHA and receipt.get('source_identity') == 'source-identity.json' and receipt.get('bulk_remains_at') == EXP_REL, 'Receipt authority/scope changed') for key in ('experiment_manifest_sha256', 'experiment_checksums_sha256', 'final_sha256', 'analysis_sha256', 'publisher_sha256', 'helper_sha256'): require(type(receipt.get(key)) is str and SHA.fullmatch(receipt[key]), 'Missing SHA anchor: ' + key) rows = {} require(type(receipt.get('files')) is list and receipt['files'], 'Empty publication inventory') for row in receipt['files']: file_row(row, small=True); name = row['path'] allowed = name in GENERATED or name in {'source-snapshot/' + n for n in SOURCE_FILES} or ( name.startswith('evidence/') and PurePosixPath(name).suffix.lower() in SUFFIXES and not name.startswith('evidence/extension-build/')) require(allowed and name not in rows, 'Unexpected/duplicate registry file') rows[name] = row required = {*GENERATED, *('source-snapshot/' + n for n in SOURCE_FILES), 'evidence/manifest.json', 'evidence/checksums.sha256', 'evidence/protocol/preregistered.json', 'evidence/audit/final-status.json', 'evidence/audit/engineering-analysis.json'} require(required <= set(rows), 'Missing required evidence/source/control files') _collision_check([*rows, 'publication-receipt.json', 'fetch-verification.json', 'fetch-failure.json']) return receipt, rows def validate_manifest(manifest_raw, checksums_raw, receipt): require(sha(manifest_raw) == receipt['experiment_manifest_sha256'] and sha(checksums_raw) == receipt['experiment_checksums_sha256'], 'Manifest/control anchor mismatch') manifest = json_bytes(manifest_raw) require(type(manifest) is dict and set(manifest) == {'root', 'files'} and manifest['root'] == REMOTE_ROOT + '/' + EXP_REL and type(manifest['files']) is list and manifest['files'], 'Wrong experiment manifest') rows = {} for row in manifest['files']: file_row(row); require(row['path'] not in rows and row['path'] not in CONTROLS, 'Duplicate/control manifest payload') rows[row['path']] = row checks = {} for line in checksums_raw.decode('utf-8').splitlines(): require(len(line) >= 67 and line[64:66] == ' ' and SHA.fullmatch(line[:64]), 'Malformed checksum line') name = line[66:]; relative_name(name) require(name not in checks, 'Duplicate checksum path'); checks[name] = line[:64] require(checks == {**{k: row['sha256'] for k, row in rows.items()}, 'manifest.json': receipt['experiment_manifest_sha256']}, 'Manifest self-SHA/checksum inventory mismatch') return rows def validate_links(cached, receipt, rows, manifest_rows): """Cross-link selected metadata; never pretend untransferred bulk is local.""" expected, omitted = {}, {} controls = [dict(path=name, bytes=len(cached['evidence/' + name]), sha256=sha(cached['evidence/' + name])) for name in CONTROLS] for row in [*manifest_rows.values(), *controls]: destination, reason = selected(row) if destination: require(destination not in expected, 'Selection maps multiple files to one destination') expected[destination] = dict(path=destination, bytes=row['bytes'], sha256=row['sha256']) else: omitted[row['path']] = dict(row, reason=reason) require({k: v for k, v in rows.items() if k not in GENERATED} == expected, 'Receipt selection is not the exact sealed policy inventory') actual_omitted = {} require(type(receipt.get('experiment_payload_not_copied')) is list, 'Missing omitted inventory') for row in receipt['experiment_payload_not_copied']: require(type(row) is dict and set(row) == {'path', 'bytes', 'sha256', 'reason'} and row['path'] not in actual_omitted, 'Malformed omitted inventory') file_row({k: row[k] for k in ('path', 'bytes', 'sha256')}); actual_omitted[row['path']] = row require(actual_omitted == omitted, 'Omitted inventory contradicts selection policy') index = json_bytes(cached['evidence-index.json']); indexed = {} require(type(index) is list, 'Invalid evidence index') for item in index: require(type(item) is dict and set(item) == {'source', 'copy', 'bytes', 'sha256'} and item['copy'] in expected and item['copy'] not in indexed, 'Invalid/duplicate indexed file') name = item['copy']; original = ('protocol/' + name if name.startswith('source-snapshot/') else name[len('evidence/'):]) require(item == dict(source=EXP_REL + '/' + original, copy=name, bytes=expected[name]['bytes'], sha256=expected[name]['sha256']), 'Evidence index source/hash mismatch') indexed[name] = item require(set(indexed) == set(expected) and receipt.get('copied_evidence_files') == len(expected) and receipt.get('copied_evidence_bytes') == sum(x['bytes'] for x in expected.values()), 'Evidence index coverage/count mismatch') proof = json_bytes(cached['seal-verification.json']) require(proof.get('pass_all') is True and proof.get('root') == REMOTE_ROOT + '/' + EXP_REL and proof.get('manifest_sha256') == receipt['experiment_manifest_sha256'] and proof.get('checksums_sha256') == receipt['experiment_checksums_sha256'] and proof.get('payload_file_count') == len(manifest_rows) and proof.get('payload_bytes') == sum(x['bytes'] for x in manifest_rows.values()), 'Seal proof disagrees with controls') reg_raw, final_raw, analysis_raw = (cached['evidence/' + name] for name in ('protocol/preregistered.json', 'audit/final-status.json', 'audit/engineering-analysis.json')) require(sha(reg_raw) == PREREG_SHA and sha(final_raw) == receipt['final_sha256'] and sha(analysis_raw) == receipt['analysis_sha256'], 'Prereg/final/analysis anchor mismatch') reg, final, analysis = map(json_bytes, (reg_raw, final_raw, analysis_raw)) require(final.get('status') == analysis.get('status') == receipt['declared_status'] and final.get('analysis_sha256') == receipt['analysis_sha256'] and final.get('registration_sha256') == analysis.get('registration_sha256') == PREREG_SHA and final.get('analysis_recomputed_exactly') is True and final.get('completed_calls') == 32 and final.get('executed_ticks') == 11074 and analysis.get('expected_calls') == analysis.get('analyzed_calls') == 32 and analysis.get('executed_ticks') == 11074, 'Final/analysis scope or linkage mismatch') require(final.get('training') is False and final.get('learned_candidate') is False and final.get('learning_pass_claimed') is False and analysis.get('training') is False and analysis.get('candidate_generated') is False and analysis.get('learning_pass_claimed') is False, 'Final/analysis asserts learning or training') numerical, covered = analysis.get('numerical_exact'), analysis.get('nonvacuous_temporal_coverage') require(type(numerical) is type(covered) is bool and receipt.get('declared_numerical_exact') is numerical and receipt.get('declared_live_coverage') is covered and analysis.get('pass_all') is (numerical and covered) and receipt.get('declared_engineering_pass') is analysis['pass_all'] and final['status'] == ('FAIL-NUMERICAL' if not numerical else (PASS if covered else 'INCOMPLETE-COVERAGE')), 'Engineering status was promoted or contradicted') sources = json_bytes(cached['source-identity.json']) require(type(sources) is list and len(sources) == 21, 'Source identity inventory missing') expected_sources, seen = {}, set() for item in sources: require(type(item) is dict and set(item) == {'model_source', 'sealed_source', 'copy', 'bytes', 'sha256'}, 'Invalid source identity row') name = item['copy']; require(name in {'source-snapshot/' + n for n in SOURCE_FILES} and name not in seen, 'Unexpected/duplicate source identity') seen.add(name); filename = PurePosixPath(name).name expected_item = dict(model_source=MODEL_REL + '/' + filename, sealed_source=EXP_REL + '/protocol/' + name, copy=name, bytes=rows[name]['bytes'], sha256=rows[name]['sha256']) require(item == expected_item, 'Source identity disagrees with registered snapshot') expected_sources[REMOTE_ROOT + '/' + item['model_source']] = item['sha256'] require(reg.get('input_sha256', {}).get(REMOTE_ROOT + '/' + item['sealed_source']) == item['sha256'], 'Snapshot missing from preregistered input authority') require(reg.get('source_sha256') == analysis.get('source_sha256') == expected_sources, 'Original 21-source inventory differs from source identities') class BoundedSFTP: def __init__(self, sftp, byte_budget=MAX_TOTAL_BYTES): self.sftp, self.byte_budget, self.bytes_read = sftp, byte_budget, 0 def read(self, remote, size=None, checksum=None): require(remote.startswith(REGISTRY_ROOT + '/'), 'Only the fixed AN registry may be downloaded') relative_name(remote[len(REGISTRY_ROOT) + 1:]) parent = PurePosixPath(remote).parent for directory in reversed((parent, *parent.parents)): require(stat.S_ISDIR(self.sftp.lstat(str(directory)).st_mode), 'Remote symlink/non-directory') info = self.sftp.lstat(remote) require(stat.S_ISREG(info.st_mode), 'Remote symlink/nonregular file') require(0 <= info.st_size <= MAX_FILE_BYTES and (size is None or info.st_size == size), 'Remote size outside bounded inventory') require(self.bytes_read + info.st_size <= self.byte_budget, 'Total SFTP read budget exceeded') chunks, total = [], 0 with self.sftp.open(remote, 'rb') as stream: while total < info.st_size: data = stream.read(min(1 << 20, info.st_size - total, self.byte_budget - self.bytes_read)) require(data, 'Remote file ended before its declared size') total += len(data); self.bytes_read += len(data) require(total <= MAX_FILE_BYTES and self.bytes_read <= self.byte_budget, 'Remote file grew beyond download limit'); chunks.append(data) raw = b''.join(chunks) after = self.sftp.lstat(remote) require(stat.S_ISREG(after.st_mode) and after.st_size == info.st_size and len(raw) == info.st_size and (size is None or len(raw) == size), 'Remote type/size changed during read') require(checksum is None or sha(raw) == checksum, 'Remote file SHA mismatch') return raw def check_destination(path): require(path.is_absolute() and '..' not in path.parts, 'Destination must be canonical and absolute') for node in (path, *path.parents): require(not node.is_symlink() and not getattr(node, 'is_junction', lambda: False)(), 'Local symlink/junction destination') require(not path.exists(), 'Destination exists; no overwrite or partial retry') def write_bytes(path, raw): path.parent.mkdir(parents=True, exist_ok=True) with path.open('xb') as stream: stream.write(raw) require(path.stat().st_size == len(raw) and sha(path.read_bytes()) == sha(raw), 'Local byte/SHA verification failed') def fetch(sftp, destination, receipt_sha256, *, max_total_bytes=MAX_TOTAL_BYTES): require(type(receipt_sha256) is str and SHA.fullmatch(receipt_sha256), 'External receipt SHA required') require(type(max_total_bytes) is int and 0 < max_total_bytes <= MAX_TOTAL_BYTES, 'Download cap cannot exceed 1 GiB') destination = Path(destination); check_destination(destination) remote = BoundedSFTP(sftp, max_total_bytes) receipt_raw = remote.read(REGISTRY_ROOT + '/publication-receipt.json', checksum=receipt_sha256) receipt, rows = validate_receipt(receipt_raw, receipt_sha256) total = len(receipt_raw) + sum(row['bytes'] for row in rows.values()) require(total + len(receipt_raw) <= max_total_bytes, 'Exact inventory plus final anchor reread exceeds total download budget') essentials = {*GENERATED, 'evidence/manifest.json', 'evidence/checksums.sha256', 'evidence/protocol/preregistered.json', 'evidence/audit/final-status.json', 'evidence/audit/engineering-analysis.json'} cached = {name: remote.read(REGISTRY_ROOT + '/' + name, rows[name]['bytes'], rows[name]['sha256']) for name in sorted(essentials)} manifest_rows = validate_manifest(cached['evidence/manifest.json'], cached['evidence/checksums.sha256'], receipt) validate_links(cached, receipt, rows, manifest_rows) check_destination(destination); destination.mkdir(parents=True, exist_ok=False) try: write_bytes(destination / 'publication-receipt.json', receipt_raw) downloaded = [dict(path='publication-receipt.json', bytes=len(receipt_raw), sha256=receipt_sha256)] for name, row in sorted(rows.items()): raw = cached[name] if name in cached else remote.read(REGISTRY_ROOT + '/' + name, row['bytes'], row['sha256']) require(len(raw) == row['bytes'] and sha(raw) == row['sha256'], 'Cached/downloaded byte mismatch') write_bytes(destination / name, raw); downloaded.append(dict(row)) remote.read(REGISTRY_ROOT + '/publication-receipt.json', len(receipt_raw), receipt_sha256) expected_names = {row['path'] for row in downloaded} require({x.relative_to(destination).as_posix() for x in destination.rglob('*') if x.is_file()} == expected_names and not any(x.is_symlink() or getattr(x, 'is_junction', lambda: False)() for x in destination.rglob('*')), 'Local unexpected/linked payload') require(all((destination / row['path']).stat().st_size == row['bytes'] and sha((destination / row['path']).read_bytes()) == row['sha256'] for row in downloaded), 'Local payload changed during fetch') result = dict(fetch_complete=True, publication_receipt_sha256=receipt_sha256, declared_status=receipt['declared_status'], experiment_manifest_sha256=receipt['experiment_manifest_sha256'], final_sha256=receipt['final_sha256'], preregistration_sha256=PREREG_SHA, registry_relative_names_preserved=True, exact_receipt_inventory_downloaded=True, files=downloaded, downloaded_file_count=len(downloaded), downloaded_bytes=total, total_download_limit_bytes=max_total_bytes, sftp_bytes_read_including_anchor_reread=remote.bytes_read, manifest_self_checksum_and_selected_links_verified=True, source_snapshot_count=21, original_model_sources_downloaded=False, remote_bulk_downloaded=False, remote_bulk_rehashed_locally=False, full_remote_seal_verification_is_publisher_claim=True, complete_bulk_backup=False, engineering_gate_executed=False, fetch_success_is_not_engineering_or_learning_pass=True, learning_pass_claimed=False, candidate_generated=False, existing_files_modified=False) raw = (json.dumps(result, sort_keys=True, indent=2, allow_nan=False) + '\n').encode() require(len(raw) <= MAX_FILE_BYTES, 'Fetch receipt exceeds 4 MiB') write_bytes(destination / 'fetch-verification.json', raw) return result except Exception as error: path = destination / 'fetch-failure.json' if not path.exists(): write_bytes(path, (json.dumps(dict(fetch_complete=False, error_type=type(error).__name__, message=str(error), partial_outputs_preserved=True, existing_files_modified=False), allow_nan=False) + '\n').encode()) raise def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--receipt-sha256', required=True) parser.add_argument('--destination', type=Path, default=DEFAULT_DESTINATION) parser.add_argument('--host', default='192.168.10.51'); parser.add_argument('--port', type=int, default=22) parser.add_argument('--user-env', default='MALECNS_REMOTE_USER') parser.add_argument('--password-env', default='MALECNS_REMOTE_PASSWORD') parser.add_argument('--known-hosts', type=Path, default=Path.home() / '.ssh/known_hosts') args = parser.parse_args() require(SHA.fullmatch(args.receipt_sha256), 'External receipt SHA required'); check_destination(args.destination) require(args.known_hosts.is_file(), 'Existing known_hosts required; trust must be configured separately') username, password = os.environ.get(args.user_env), os.environ.get(args.password_env) require(username and password, 'Credential environment variables are required') import paramiko client = paramiko.SSHClient() client.load_system_host_keys(str(args.known_hosts)) client.set_missing_host_key_policy(paramiko.RejectPolicy()) try: client.connect(args.host, port=args.port, username=username, password=password, allow_agent=False, look_for_keys=False, timeout=15, auth_timeout=15, banner_timeout=15) with client.open_sftp() as sftp: result = fetch(sftp, args.destination, args.receipt_sha256) print(json.dumps({key: result[key] for key in ('fetch_complete', 'declared_status', 'publication_receipt_sha256', 'downloaded_file_count')}), flush=True) finally: client.close() if __name__ == '__main__': main()