Audit conditional PN recruitment and graded APL source timing
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,270 @@
|
||||
"""Publish small evidence only after independent verification of both seals.
|
||||
|
||||
No prior verification helper is imported or executed. Publishing verifies
|
||||
bytes and declared completion; it does not rerun or certify a scientific gate.
|
||||
Every destination is exclusive. An interrupted publication is retained, not
|
||||
deleted, overwritten, or silently resumed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
|
||||
|
||||
ROOT = Path('/data/digital-drosophila')
|
||||
MODEL_RELATIVE = Path('models/male-cns-lif/pn-recruitment-audit-v1')
|
||||
EXPERIMENT_RELATIVE = Path('experiments/phase4ai-pn-recruitment-audit')
|
||||
REGISTRY_RELATIVE = Path('registry/phase4ai-pn-recruitment-audit')
|
||||
FINAL_STATUS = 'PASS-CONDITIONAL-PN-APL-SOURCE-AUDIT'
|
||||
MAX_EVIDENCE_BYTES = 4 * 1024 ** 2
|
||||
EVIDENCE_SUFFIXES = frozenset(('.json', '.csv', '.md', '.log', '.sha256'))
|
||||
CONTROLS = frozenset(('manifest.json', 'checksums.sha256'))
|
||||
SHA256 = re.compile(r'^[0-9a-f]{64}$')
|
||||
|
||||
|
||||
def _require(condition, message):
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _unique_object(pairs):
|
||||
result = {}
|
||||
for name, value in pairs:
|
||||
_require(name not in result, 'Duplicate JSON object key: ' + name)
|
||||
result[name] = value
|
||||
return result
|
||||
|
||||
|
||||
def _bad_constant(value):
|
||||
raise ValueError('Nonfinite JSON constant: ' + value)
|
||||
|
||||
|
||||
def _json_bytes(data):
|
||||
return json.loads(data.decode('utf-8'), object_pairs_hook=_unique_object,
|
||||
parse_constant=_bad_constant)
|
||||
|
||||
|
||||
def _digest(path):
|
||||
checksum = hashlib.sha256()
|
||||
with Path(path).open('rb') as handle:
|
||||
for block in iter(lambda: handle.read(8 << 20), b''):
|
||||
checksum.update(block)
|
||||
return checksum.hexdigest()
|
||||
|
||||
|
||||
def _write_json(path, value):
|
||||
with Path(path).open('x', encoding='utf-8') as handle:
|
||||
json.dump(value, handle, indent=2, ensure_ascii=False, allow_nan=False)
|
||||
handle.write('\n')
|
||||
|
||||
|
||||
def _relative_name(value):
|
||||
_require(isinstance(value, str) and bool(value) and '\\' not in value and
|
||||
'\x00' not in value and '\n' not in value and '\r' not in value,
|
||||
'Invalid manifest/checksum relative path')
|
||||
path = PurePosixPath(value)
|
||||
_require(not path.is_absolute() and str(path) == value and
|
||||
all(part not in ('', '.', '..') and ':' not in part for part in path.parts),
|
||||
'Noncanonical or escaping relative path: ' + value)
|
||||
return path
|
||||
|
||||
|
||||
def _regular_file(base, relative):
|
||||
rel = _relative_name(relative)
|
||||
target = base
|
||||
for part in rel.parts:
|
||||
target = target / part
|
||||
_require(not target.is_symlink(), 'Symlink not allowed in sealed path: ' + str(target))
|
||||
_require(target.exists() and stat.S_ISREG(target.stat().st_mode),
|
||||
'Missing/nonregular sealed file: ' + str(target))
|
||||
_require(target.resolve().is_relative_to(base), 'Sealed file escapes its root')
|
||||
return target
|
||||
|
||||
|
||||
def verify_seal(directory):
|
||||
"""Independently verify every listed byte/hash and the checksum inventory.
|
||||
|
||||
Unlisted files fail, apart from __pycache__ files explicitly excluded by
|
||||
the producer's seal contract. Such caches are counted, never published.
|
||||
The checksum file must describe exactly the manifest payload plus the
|
||||
manifest itself. Symlinks, duplicate paths and path traversal fail.
|
||||
"""
|
||||
supplied = Path(directory)
|
||||
_require(supplied.is_absolute() and supplied.is_dir() and not supplied.is_symlink(),
|
||||
'A sealed root must be an absolute real directory')
|
||||
base = supplied.resolve(strict=True)
|
||||
manifest_path = _regular_file(base, 'manifest.json')
|
||||
checksums_path = _regular_file(base, 'checksums.sha256')
|
||||
manifest_bytes = manifest_path.read_bytes()
|
||||
manifest_sha = hashlib.sha256(manifest_bytes).hexdigest()
|
||||
manifest = _json_bytes(manifest_bytes)
|
||||
_require(isinstance(manifest, dict) and set(manifest) == {'root', 'files'}, 'Unexpected seal manifest schema')
|
||||
_require(isinstance(manifest['root'], str) and Path(manifest['root']).is_absolute() and
|
||||
Path(manifest['root']).resolve(strict=True) == base, 'Manifest names another root')
|
||||
_require(isinstance(manifest['files'], list) and bool(manifest['files']), 'Empty/invalid manifest payload')
|
||||
rows, expected = [], {}
|
||||
for row in manifest['files']:
|
||||
_require(isinstance(row, dict) and set(row) == {'path', 'bytes', 'sha256'}, 'Invalid manifest file row')
|
||||
relative = row['path']
|
||||
parsed = _relative_name(relative)
|
||||
_require(relative not in CONTROLS and '__pycache__' not in parsed.parts,
|
||||
'Control/cache file incorrectly listed as payload: ' + relative)
|
||||
_require(relative not in expected, 'Duplicate manifest path: ' + relative)
|
||||
_require(type(row['bytes']) is int and row['bytes'] >= 0 and isinstance(row['sha256'], str) and
|
||||
SHA256.fullmatch(row['sha256']), 'Invalid byte count or SHA-256: ' + relative)
|
||||
path = _regular_file(base, relative)
|
||||
_require(path.stat().st_size == row['bytes'], 'Payload byte count mismatch: ' + relative)
|
||||
_require(_digest(path) == row['sha256'], 'Payload SHA-256 mismatch: ' + relative)
|
||||
_require(path.stat().st_size == row['bytes'], 'Payload changed during verification: ' + relative)
|
||||
expected[relative] = row['sha256']
|
||||
rows.append(dict(row))
|
||||
observed, excluded_caches = set(), 0
|
||||
for path in base.rglob('*'):
|
||||
_require(not path.is_symlink(), 'Symlink found in sealed tree: ' + str(path))
|
||||
if path.is_dir():
|
||||
continue
|
||||
_require(stat.S_ISREG(path.stat().st_mode), 'Nonregular file in sealed tree: ' + str(path))
|
||||
relative = path.relative_to(base).as_posix()
|
||||
if '__pycache__' in path.relative_to(base).parts:
|
||||
excluded_caches += 1
|
||||
else:
|
||||
observed.add(relative)
|
||||
_require(observed == set(expected) | CONTROLS,
|
||||
'Sealed tree has unlisted/missing files: ' + repr(sorted(observed ^ (set(expected) | CONTROLS))))
|
||||
checksums_bytes = checksums_path.read_bytes()
|
||||
checksum_rows = {}
|
||||
for line in checksums_bytes.decode('utf-8').splitlines():
|
||||
_require(len(line) >= 67 and line[64:66] == ' ' and SHA256.fullmatch(line[:64]),
|
||||
'Malformed checksums.sha256 line')
|
||||
relative = line[66:]
|
||||
_relative_name(relative)
|
||||
_require(relative not in checksum_rows, 'Duplicate checksum path: ' + relative)
|
||||
checksum_rows[relative] = line[:64]
|
||||
_require(checksum_rows == {**expected, 'manifest.json': manifest_sha},
|
||||
'Checksum inventory or manifest self-checksum mismatch')
|
||||
return dict(pass_all=True, root=str(base), payload_file_count=len(rows),
|
||||
payload_bytes=sum(row['bytes'] for row in rows), manifest_sha256=manifest_sha,
|
||||
checksums_sha256=hashlib.sha256(checksums_bytes).hexdigest(),
|
||||
excluded_pycache_files=excluded_caches, files=sorted(rows, key=lambda row: row['path']),
|
||||
verification_scope='All sealed payload bytes/SHA-256, exact checksum inventory, and manifest self-SHA; no scientific gate execution')
|
||||
|
||||
|
||||
def _copy_verified(source, destination, byte_count, checksum):
|
||||
_require(source.stat().st_size == byte_count, 'Source size changed before evidence copy')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with source.open('rb') as src, destination.open('xb') as dst:
|
||||
shutil.copyfileobj(src, dst, length=1 << 20)
|
||||
_require(destination.stat().st_size == byte_count and _digest(destination) == checksum,
|
||||
'Copied evidence failed its sealed byte/hash check')
|
||||
_require(source.stat().st_size == byte_count and _digest(source) == checksum,
|
||||
'Sealed source changed during evidence copy')
|
||||
|
||||
|
||||
def _control_rows(root, verification):
|
||||
return [dict(path='manifest.json', bytes=(root / 'manifest.json').stat().st_size,
|
||||
sha256=verification['manifest_sha256']),
|
||||
dict(path='checksums.sha256', bytes=(root / 'checksums.sha256').stat().st_size,
|
||||
sha256=verification['checksums_sha256'])]
|
||||
|
||||
|
||||
def publish(root=ROOT):
|
||||
"""Create the one new registry exclusively; call only after finalize seals.
|
||||
|
||||
``root`` exists for isolated synthetic tests. Production uses the fixed
|
||||
default project root and fixed Phase 4AI relative directories above.
|
||||
"""
|
||||
root = Path(root)
|
||||
_require(root.is_absolute() and root.is_dir() and not root.is_symlink(), 'Invalid project root')
|
||||
root = root.resolve(strict=True)
|
||||
model, experiment, registry = (root / part for part in
|
||||
(MODEL_RELATIVE, EXPERIMENT_RELATIVE, REGISTRY_RELATIVE))
|
||||
_require(not registry.exists() and not registry.is_symlink(), 'Registry already exists; no overwrite/resume')
|
||||
for path in (model, experiment, registry.parent):
|
||||
current = root
|
||||
for part in path.relative_to(root).parts:
|
||||
current = current / part
|
||||
_require(not current.is_symlink(), 'Symlink in publication root path: ' + str(current))
|
||||
verified = {'model': verify_seal(model), 'experiment': verify_seal(experiment)}
|
||||
experiment_rows = {row['path']: row for row in verified['experiment']['files']}
|
||||
_require('audit/final-status.json' in experiment_rows,
|
||||
'Final status must exist inside the sealed experiment payload')
|
||||
status = _json_bytes((experiment / 'audit/final-status.json').read_bytes())
|
||||
_require(isinstance(status, dict) and status.get('status') == FINAL_STATUS and
|
||||
status.get('pass_all') is True, 'Finalize completion status is missing or unexpected')
|
||||
for flag in ('candidate_generated', 'learning_achieved', 'new_brain_simulation', 'GPU_used'):
|
||||
_require(status.get(flag) is False, 'Unexpected finalized scope flag: ' + flag)
|
||||
plans, omitted = [], []
|
||||
for row in verified['experiment']['files'] + _control_rows(experiment, verified['experiment']):
|
||||
allowed = PurePosixPath(row['path']).suffix.lower() in EVIDENCE_SUFFIXES
|
||||
if allowed and row['bytes'] <= MAX_EVIDENCE_BYTES:
|
||||
plans.append((experiment, row, 'evidence/' + row['path']))
|
||||
else:
|
||||
omitted.append(dict(path=row['path'], bytes=row['bytes'],
|
||||
reason='exceeds_4_MiB' if row['bytes'] > MAX_EVIDENCE_BYTES else 'bulk_or_non_evidence_extension'))
|
||||
for row in _control_rows(model, verified['model']):
|
||||
_require(row['bytes'] <= MAX_EVIDENCE_BYTES, 'Model seal control exceeds small-evidence limit')
|
||||
plans.append((model, row, 'model/' + row['path']))
|
||||
for control in ('manifest.json', 'checksums.sha256'):
|
||||
_require(any(target == 'evidence/' + control for _, _, target in plans),
|
||||
'Experiment seal control exceeds small-evidence limit')
|
||||
_require(any(target == 'evidence/audit/final-status.json' for _, _, target in plans),
|
||||
'Final status exceeds small-evidence limit')
|
||||
registry.parent.mkdir(parents=True, exist_ok=True)
|
||||
registry.mkdir(exist_ok=False)
|
||||
try:
|
||||
records = []
|
||||
for source_root, row, relative in plans:
|
||||
source = _regular_file(source_root.resolve(), row['path'])
|
||||
target = registry / relative
|
||||
_copy_verified(source, target, row['bytes'], row['sha256'])
|
||||
records.append(dict(source=source.relative_to(root).as_posix(), copy=relative,
|
||||
bytes=row['bytes'], sha256=row['sha256']))
|
||||
# Revalidate after copying, including payloads deliberately not copied.
|
||||
after = {'model': verify_seal(model), 'experiment': verify_seal(experiment)}
|
||||
_require(after == verified, 'A sealed tree changed during publication')
|
||||
proof = {name: {key: value for key, value in report.items() if key != 'files'}
|
||||
for name, report in verified.items()}
|
||||
_write_json(registry / 'seal-verification.json', proof)
|
||||
_write_json(registry / 'evidence-index.json', records)
|
||||
receipt_files = []
|
||||
for path in sorted(registry.rglob('*')):
|
||||
if path.is_file():
|
||||
receipt_files.append(dict(path=path.relative_to(registry).as_posix(),
|
||||
bytes=path.stat().st_size, sha256=_digest(path)))
|
||||
receipt = dict(publication_complete=True, registry=registry.relative_to(root).as_posix(),
|
||||
project_root=str(root), declared_audit_status=status['status'],
|
||||
model_manifest_sha256=verified['model']['manifest_sha256'],
|
||||
experiment_manifest_sha256=verified['experiment']['manifest_sha256'],
|
||||
copied_evidence_files=len(records), copied_evidence_bytes=sum(row['bytes'] for row in records),
|
||||
files=receipt_files, experiment_payload_not_copied=omitted,
|
||||
model_payload_not_duplicated=True, model_payload_remains_at=MODEL_RELATIVE.as_posix(),
|
||||
bulk_remains_at=EXPERIMENT_RELATIVE.as_posix(),
|
||||
complete_bulk_backup=False, scientific_gate_executed_by_publisher=False,
|
||||
scientific_gate_certified_by_publication=False, candidate_generated=False,
|
||||
learning_achieved=False, existing_files_modified=False,
|
||||
receipt_self_hash='Not self-referential; returned by publish() and printed by main()',
|
||||
limits=dict(max_selected_evidence_file_bytes=MAX_EVIDENCE_BYTES,
|
||||
selected_suffixes=sorted(EVIDENCE_SUFFIXES)))
|
||||
receipt_path = registry / 'publication-receipt.json'
|
||||
_write_json(receipt_path, receipt)
|
||||
return dict(publication_complete=True, registry=str(registry), copied_evidence_files=len(records),
|
||||
receipt_sha256=_digest(receipt_path), scientific_gate_executed_by_publisher=False)
|
||||
except Exception as error:
|
||||
failure = registry / 'publication-failure.json'
|
||||
if not failure.exists():
|
||||
_write_json(failure, dict(publication_complete=False, type=type(error).__name__,
|
||||
message=str(error), partial_outputs_preserved=True,
|
||||
existing_files_modified=False))
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
print(json.dumps(publish(), ensure_ascii=False, allow_nan=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user