Digital Drosophila Archive
271 lines
14 KiB
Python
271 lines
14 KiB
Python
"""Publish small evidence only after independent verification of both seals.
|
|
|
|
No prior verification helper is imported or executed. Publishing verifies
|
|
bytes and declared completion; it does not rerun or certify a scientific gate.
|
|
Every destination is exclusive. An interrupted publication is retained, not
|
|
deleted, overwritten, or silently resumed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import stat
|
|
|
|
|
|
ROOT = Path('/data/digital-drosophila')
|
|
MODEL_RELATIVE = Path('models/male-cns-lif/pn-recruitment-audit-v1')
|
|
EXPERIMENT_RELATIVE = Path('experiments/phase4ai-pn-recruitment-audit')
|
|
REGISTRY_RELATIVE = Path('registry/phase4ai-pn-recruitment-audit')
|
|
FINAL_STATUS = 'PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT'
|
|
MAX_EVIDENCE_BYTES = 4 * 1024 ** 2
|
|
EVIDENCE_SUFFIXES = frozenset(('.json', '.csv', '.md', '.log', '.sha256'))
|
|
CONTROLS = frozenset(('manifest.json', 'checksums.sha256'))
|
|
SHA256 = re.compile(r'^[0-9a-f]{64}$')
|
|
|
|
|
|
def _require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def _unique_object(pairs):
|
|
result = {}
|
|
for name, value in pairs:
|
|
_require(name not in result, 'Duplicate JSON object key: ' + name)
|
|
result[name] = value
|
|
return result
|
|
|
|
|
|
def _bad_constant(value):
|
|
raise ValueError('Nonfinite JSON constant: ' + value)
|
|
|
|
|
|
def _json_bytes(data):
|
|
return json.loads(data.decode('utf-8'), object_pairs_hook=_unique_object,
|
|
parse_constant=_bad_constant)
|
|
|
|
|
|
def _digest(path):
|
|
checksum = hashlib.sha256()
|
|
with Path(path).open('rb') as handle:
|
|
for block in iter(lambda: handle.read(8 << 20), b''):
|
|
checksum.update(block)
|
|
return checksum.hexdigest()
|
|
|
|
|
|
def _write_json(path, value):
|
|
with Path(path).open('x', encoding='utf-8') as handle:
|
|
json.dump(value, handle, indent=2, ensure_ascii=False, allow_nan=False)
|
|
handle.write('\n')
|
|
|
|
|
|
def _relative_name(value):
|
|
_require(isinstance(value, str) and bool(value) and '\\' not in value and
|
|
'\x00' not in value and '\n' not in value and '\r' not in value,
|
|
'Invalid manifest/checksum relative path')
|
|
path = PurePosixPath(value)
|
|
_require(not path.is_absolute() and str(path) == value and
|
|
all(part not in ('', '.', '..') and ':' not in part for part in path.parts),
|
|
'Noncanonical or escaping relative path: ' + value)
|
|
return path
|
|
|
|
|
|
def _regular_file(base, relative):
|
|
rel = _relative_name(relative)
|
|
target = base
|
|
for part in rel.parts:
|
|
target = target / part
|
|
_require(not target.is_symlink(), 'Symlink not allowed in sealed path: ' + str(target))
|
|
_require(target.exists() and stat.S_ISREG(target.stat().st_mode),
|
|
'Missing/nonregular sealed file: ' + str(target))
|
|
_require(target.resolve().is_relative_to(base), 'Sealed file escapes its root')
|
|
return target
|
|
|
|
|
|
def verify_seal(directory):
|
|
"""Independently verify every listed byte/hash and the checksum inventory.
|
|
|
|
Unlisted files fail, apart from __pycache__ files explicitly excluded by
|
|
the producer's seal contract. Such caches are counted, never published.
|
|
The checksum file must describe exactly the manifest payload plus the
|
|
manifest itself. Symlinks, duplicate paths and path traversal fail.
|
|
"""
|
|
supplied = Path(directory)
|
|
_require(supplied.is_absolute() and supplied.is_dir() and not supplied.is_symlink(),
|
|
'A sealed root must be an absolute real directory')
|
|
base = supplied.resolve(strict=True)
|
|
manifest_path = _regular_file(base, 'manifest.json')
|
|
checksums_path = _regular_file(base, 'checksums.sha256')
|
|
manifest_bytes = manifest_path.read_bytes()
|
|
manifest_sha = hashlib.sha256(manifest_bytes).hexdigest()
|
|
manifest = _json_bytes(manifest_bytes)
|
|
_require(isinstance(manifest, dict) and set(manifest) == {'root', 'files'}, 'Unexpected seal manifest schema')
|
|
_require(isinstance(manifest['root'], str) and Path(manifest['root']).is_absolute() and
|
|
Path(manifest['root']).resolve(strict=True) == base, 'Manifest names another root')
|
|
_require(isinstance(manifest['files'], list) and bool(manifest['files']), 'Empty/invalid manifest payload')
|
|
rows, expected = [], {}
|
|
for row in manifest['files']:
|
|
_require(isinstance(row, dict) and set(row) == {'path', 'bytes', 'sha256'}, 'Invalid manifest file row')
|
|
relative = row['path']
|
|
parsed = _relative_name(relative)
|
|
_require(relative not in CONTROLS and '__pycache__' not in parsed.parts,
|
|
'Control/cache file incorrectly listed as payload: ' + relative)
|
|
_require(relative not in expected, 'Duplicate manifest path: ' + relative)
|
|
_require(type(row['bytes']) is int and row['bytes'] >= 0 and isinstance(row['sha256'], str) and
|
|
SHA256.fullmatch(row['sha256']), 'Invalid byte count or SHA-256: ' + relative)
|
|
path = _regular_file(base, relative)
|
|
_require(path.stat().st_size == row['bytes'], 'Payload byte count mismatch: ' + relative)
|
|
_require(_digest(path) == row['sha256'], 'Payload SHA-256 mismatch: ' + relative)
|
|
_require(path.stat().st_size == row['bytes'], 'Payload changed during verification: ' + relative)
|
|
expected[relative] = row['sha256']
|
|
rows.append(dict(row))
|
|
observed, excluded_caches = set(), 0
|
|
for path in base.rglob('*'):
|
|
_require(not path.is_symlink(), 'Symlink found in sealed tree: ' + str(path))
|
|
if path.is_dir():
|
|
continue
|
|
_require(stat.S_ISREG(path.stat().st_mode), 'Nonregular file in sealed tree: ' + str(path))
|
|
relative = path.relative_to(base).as_posix()
|
|
if '__pycache__' in path.relative_to(base).parts:
|
|
excluded_caches += 1
|
|
else:
|
|
observed.add(relative)
|
|
_require(observed == set(expected) | CONTROLS,
|
|
'Sealed tree has unlisted/missing files: ' + repr(sorted(observed ^ (set(expected) | CONTROLS))))
|
|
checksums_bytes = checksums_path.read_bytes()
|
|
checksum_rows = {}
|
|
for line in checksums_bytes.decode('utf-8').splitlines():
|
|
_require(len(line) >= 67 and line[64:66] == ' ' and SHA256.fullmatch(line[:64]),
|
|
'Malformed checksums.sha256 line')
|
|
relative = line[66:]
|
|
_relative_name(relative)
|
|
_require(relative not in checksum_rows, 'Duplicate checksum path: ' + relative)
|
|
checksum_rows[relative] = line[:64]
|
|
_require(checksum_rows == {**expected, 'manifest.json': manifest_sha},
|
|
'Checksum inventory or manifest self-checksum mismatch')
|
|
return dict(pass_all=True, root=str(base), payload_file_count=len(rows),
|
|
payload_bytes=sum(row['bytes'] for row in rows), manifest_sha256=manifest_sha,
|
|
checksums_sha256=hashlib.sha256(checksums_bytes).hexdigest(),
|
|
excluded_pycache_files=excluded_caches, files=sorted(rows, key=lambda row: row['path']),
|
|
verification_scope='All sealed payload bytes/SHA-256, exact checksum inventory, and manifest self-SHA; no scientific gate execution')
|
|
|
|
|
|
def _copy_verified(source, destination, byte_count, checksum):
|
|
_require(source.stat().st_size == byte_count, 'Source size changed before evidence copy')
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.open('rb') as src, destination.open('xb') as dst:
|
|
shutil.copyfileobj(src, dst, length=1 << 20)
|
|
_require(destination.stat().st_size == byte_count and _digest(destination) == checksum,
|
|
'Copied evidence failed its sealed byte/hash check')
|
|
_require(source.stat().st_size == byte_count and _digest(source) == checksum,
|
|
'Sealed source changed during evidence copy')
|
|
|
|
|
|
def _control_rows(root, verification):
|
|
return [dict(path='manifest.json', bytes=(root / 'manifest.json').stat().st_size,
|
|
sha256=verification['manifest_sha256']),
|
|
dict(path='checksums.sha256', bytes=(root / 'checksums.sha256').stat().st_size,
|
|
sha256=verification['checksums_sha256'])]
|
|
|
|
|
|
def publish(root=ROOT):
|
|
"""Create the one new registry exclusively; call only after finalize seals.
|
|
|
|
``root`` exists for isolated synthetic tests. Production uses the fixed
|
|
default project root and fixed Phase 4AI relative directories above.
|
|
"""
|
|
root = Path(root)
|
|
_require(root.is_absolute() and root.is_dir() and not root.is_symlink(), 'Invalid project root')
|
|
root = root.resolve(strict=True)
|
|
model, experiment, registry = (root / part for part in
|
|
(MODEL_RELATIVE, EXPERIMENT_RELATIVE, REGISTRY_RELATIVE))
|
|
_require(not registry.exists() and not registry.is_symlink(), 'Registry already exists; no overwrite/resume')
|
|
for path in (model, experiment, registry.parent):
|
|
current = root
|
|
for part in path.relative_to(root).parts:
|
|
current = current / part
|
|
_require(not current.is_symlink(), 'Symlink in publication root path: ' + str(current))
|
|
verified = {'model': verify_seal(model), 'experiment': verify_seal(experiment)}
|
|
experiment_rows = {row['path']: row for row in verified['experiment']['files']}
|
|
_require('audit/final-status.json' in experiment_rows,
|
|
'Final status must exist inside the sealed experiment payload')
|
|
status = _json_bytes((experiment / 'audit/final-status.json').read_bytes())
|
|
_require(isinstance(status, dict) and status.get('status') == FINAL_STATUS and
|
|
status.get('pass_all') is True, 'Finalize completion status is missing or unexpected')
|
|
for flag in ('candidate_generated', 'learning_achieved', 'new_brain_simulation', 'GPU_used'):
|
|
_require(status.get(flag) is False, 'Unexpected finalized scope flag: ' + flag)
|
|
plans, omitted = [], []
|
|
for row in verified['experiment']['files'] + _control_rows(experiment, verified['experiment']):
|
|
allowed = PurePosixPath(row['path']).suffix.lower() in EVIDENCE_SUFFIXES
|
|
if allowed and row['bytes'] <= MAX_EVIDENCE_BYTES:
|
|
plans.append((experiment, row, 'evidence/' + row['path']))
|
|
else:
|
|
omitted.append(dict(path=row['path'], bytes=row['bytes'],
|
|
reason='exceeds_4_MiB' if row['bytes'] > MAX_EVIDENCE_BYTES else 'bulk_or_non_evidence_extension'))
|
|
for row in _control_rows(model, verified['model']):
|
|
_require(row['bytes'] <= MAX_EVIDENCE_BYTES, 'Model seal control exceeds small-evidence limit')
|
|
plans.append((model, row, 'model/' + row['path']))
|
|
for control in ('manifest.json', 'checksums.sha256'):
|
|
_require(any(target == 'evidence/' + control for _, _, target in plans),
|
|
'Experiment seal control exceeds small-evidence limit')
|
|
_require(any(target == 'evidence/audit/final-status.json' for _, _, target in plans),
|
|
'Final status exceeds small-evidence limit')
|
|
registry.parent.mkdir(parents=True, exist_ok=True)
|
|
registry.mkdir(exist_ok=False)
|
|
try:
|
|
records = []
|
|
for source_root, row, relative in plans:
|
|
source = _regular_file(source_root.resolve(), row['path'])
|
|
target = registry / relative
|
|
_copy_verified(source, target, row['bytes'], row['sha256'])
|
|
records.append(dict(source=source.relative_to(root).as_posix(), copy=relative,
|
|
bytes=row['bytes'], sha256=row['sha256']))
|
|
# Revalidate after copying, including payloads deliberately not copied.
|
|
after = {'model': verify_seal(model), 'experiment': verify_seal(experiment)}
|
|
_require(after == verified, 'A sealed tree changed during publication')
|
|
proof = {name: {key: value for key, value in report.items() if key != 'files'}
|
|
for name, report in verified.items()}
|
|
_write_json(registry / 'seal-verification.json', proof)
|
|
_write_json(registry / 'evidence-index.json', records)
|
|
receipt_files = []
|
|
for path in sorted(registry.rglob('*')):
|
|
if path.is_file():
|
|
receipt_files.append(dict(path=path.relative_to(registry).as_posix(),
|
|
bytes=path.stat().st_size, sha256=_digest(path)))
|
|
receipt = dict(publication_complete=True, registry=registry.relative_to(root).as_posix(),
|
|
project_root=str(root), declared_audit_status=status['status'],
|
|
model_manifest_sha256=verified['model']['manifest_sha256'],
|
|
experiment_manifest_sha256=verified['experiment']['manifest_sha256'],
|
|
copied_evidence_files=len(records), copied_evidence_bytes=sum(row['bytes'] for row in records),
|
|
files=receipt_files, experiment_payload_not_copied=omitted,
|
|
model_payload_not_duplicated=True, model_payload_remains_at=MODEL_RELATIVE.as_posix(),
|
|
bulk_remains_at=EXPERIMENT_RELATIVE.as_posix(),
|
|
complete_bulk_backup=False, scientific_gate_executed_by_publisher=False,
|
|
scientific_gate_certified_by_publication=False, candidate_generated=False,
|
|
learning_achieved=False, existing_files_modified=False,
|
|
receipt_self_hash='Not self-referential; returned by publish() and printed by main()',
|
|
limits=dict(max_selected_evidence_file_bytes=MAX_EVIDENCE_BYTES,
|
|
selected_suffixes=sorted(EVIDENCE_SUFFIXES)))
|
|
receipt_path = registry / 'publication-receipt.json'
|
|
_write_json(receipt_path, receipt)
|
|
return dict(publication_complete=True, registry=str(registry), copied_evidence_files=len(records),
|
|
receipt_sha256=_digest(receipt_path), scientific_gate_executed_by_publisher=False)
|
|
except Exception as error:
|
|
failure = registry / 'publication-failure.json'
|
|
if not failure.exists():
|
|
_write_json(failure, dict(publication_complete=False, type=type(error).__name__,
|
|
message=str(error), partial_outputs_preserved=True,
|
|
existing_files_modified=False))
|
|
raise
|
|
|
|
|
|
def main():
|
|
print(json.dumps(publish(), ensure_ascii=False, allow_nan=False), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|