"""Serial, immutable artifact accounting; not a process runner or science gate. The long-lived parent hashes this cycle's declared inputs/outputs and the known ledger tail. It does NOT rehash all old artifacts per cycle. Reopening, inspect and final_verify perform full historical verification. Never infer a process's terminal state from a PID, missing file, elapsed time, or a live handle. """ from __future__ import annotations from contextlib import contextmanager import hashlib import json import os from pathlib import Path, PurePosixPath import re import shutil import uuid SCHEMA = 'normal-append-ledger-v1' CONTROL = '.append-ledger' REGISTRATION = 'protocol/preregistered.json' SHA = re.compile('[0-9a-f]{64}') ID = re.compile('[A-Za-z0-9][A-Za-z0-9_-]{0,127}') def require(ok, message): if not ok: raise ValueError(message) def raw_json(value): return (json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False)+'\n').encode('utf-8') def read_json(path): def unique(items): result = {} for key, value in items: require(key not in result, 'Duplicate JSON key'); result[key] = value return result return json.loads(Path(path).read_bytes().decode('utf-8'), object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON'))) def digest(path): h = hashlib.sha256() with Path(path).open('rb') as stream: for block in iter(lambda: stream.read(8 << 20), b''): h.update(block) return h.hexdigest() def relative_name(value): require(type(value) is str and bool(value) and '\\' not in value and '\x00' not in value and '\n' not in value and '\r' not in value, 'Invalid relative path') path = PurePosixPath(value) require(not path.is_absolute() and str(path) == value and all(p not in ('.', '..', '') and ':' not in p for p in path.parts), 'Escaping/noncanonical path') return value def unlinked(path): p = Path(path) require(p.is_absolute() and '..' not in p.parts, 'Absolute nonescaping path required') require(not any(q.is_symlink() or (hasattr(q, 'is_junction') and q.is_junction()) for q in (p, *p.parents)), 'Linked path forbidden') return p def identity(path): p = unlinked(path) require(p.is_file(), 'Missing regular file: '+str(p)) size = p.stat().st_size; sha = digest(p) require(p.stat().st_size == size, 'File changed while hashing') return dict(bytes=size, sha256=sha) def validate_identity(value): require(type(value) is dict and set(value) == {'bytes', 'sha256'} and type(value['bytes']) is int and value['bytes'] >= 0 and type(value['sha256']) is str and SHA.fullmatch(value['sha256']), 'Invalid SHA/byte identity') def _sync_directory(directory): # Windows has no portable directory fsync. File bytes are fsynced there. if os.name != 'nt': descriptor = os.open(directory, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) def publish_exclusive(path, value): """Atomic no-replace publication on filesystems supporting hard links. No unsafe rename/replace fallback. An interrupted temporary is preserved; inspect/final audit rejects it. A failed destination collision also never deletes or replaces the pre-existing destination. """ path = unlinked(path); require(path.parent.is_dir(), 'Missing publication parent') require(not path.exists(), 'Immutable destination already exists; no retry') raw = raw_json(value); temporary = path.parent / ('.publishing-'+uuid.uuid4().hex) descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, 'wb') as stream: stream.write(raw); stream.flush(); os.fsync(stream.fileno()) os.link(temporary, path) # Exclusive destination, same bytes/inode; never overwrite. _sync_directory(path.parent) require(path.read_bytes() == raw, 'Published bytes changed') temporary.unlink() # Only this function's exact, newly created temporary. _sync_directory(path.parent) return dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) class Ledger: """One long-lived instance per serial parent, never parallel scheduling. Inventories map absolute or root-relative paths to {bytes, sha256}; initial inventory is root-relative and must contain the exact registration file. Call begin BEFORE spawn, then append only after the parent has observed actual exit 0 and saved its independent process receipt. Existing starts are never reissued. An interrupted append may be independently inspected, but automatic resume/rerun is not provided. """ def __init__(self, root, registration_sha, initial_inventory, max_output_bytes, reserve_bytes, shared_build_relative='extension-build'): self.root = unlinked(Path(root)).resolve(strict=True) require(self.root.is_dir(), 'Existing experiment root required') require(type(registration_sha) is str and SHA.fullmatch(registration_sha), 'External registration SHA required') require(type(max_output_bytes) is int and max_output_bytes > 0 and type(reserve_bytes) is int and reserve_bytes >= 0, 'Explicit nonnegative reserve and positive output cap required') self.registration_sha = registration_sha; self.max_output_bytes = max_output_bytes; self.reserve_bytes = reserve_bytes self.build_name = relative_name(shared_build_relative) require(self.build_name != CONTROL and not self.build_name.startswith(CONTROL+'/'), 'Build overlaps ledger controls') self.control = self.root / CONTROL; self.header_path = self.control / 'header.json' self.starts = self.control / 'started'; self.entries = self.control / 'entries' require(type(initial_inventory) is dict and initial_inventory, 'Explicit initial inventory required') initial = {} for name, expected in initial_inventory.items(): relative_name(name); self._payload_path(name); validate_identity(expected) require(identity(self.root/name) == expected, 'Initial file identity differs: '+name) initial[name] = dict(expected) require(initial.get(REGISTRATION, {}).get('sha256') == registration_sha, 'Initial registration binding missing') header = dict(schema=SCHEMA, root=str(self.root), registration_sha256=registration_sha, initial_inventory=initial, max_output_bytes=max_output_bytes, reserve_bytes=reserve_bytes, shared_mutable_build=self.build_name, initial_inventory_scope='caller supplied; entire tree checked only by final_verify', scientific_gate_claimed=False) if self.header_path.exists(): require(read_json(self.header_path) == header, 'Existing ledger authority/options differ') else: require(not self.control.exists(), 'Partial ledger initialization; no automatic retry') self.control.mkdir(); self.starts.mkdir(); self.entries.mkdir() publish_exclusive(self.header_path, header) self.header_identity = identity(self.header_path); self.initial = initial self._tracked = dict(initial); self._controls = {CONTROL+'/header.json': self.header_identity} self._completed = []; self._completed_set = set(); self._pending = {}; self._tail = None self._immutable_bytes = sum(v['bytes'] for v in initial.values()) self._control_bytes = self.header_identity['bytes'] self._write_failed = False self._replay() self._budget() def _payload_path(self, value): path = Path(value) if not path.is_absolute(): path = self.root / relative_name(str(value)) path = unlinked(path) require(path.is_relative_to(self.root), 'Output escapes experiment') rel = path.relative_to(self.root).as_posix(); relative_name(rel) require(rel != CONTROL and not rel.startswith(CONTROL+'/') and rel != self.build_name and not rel.startswith(self.build_name+'/'), 'Immutable payload overlaps controls/shared build') return path, rel def _rows(self, mapping, *, output, verified=None): require(type(mapping) is dict, 'Explicit inventory dictionary required') result = []; used = set() for text, expected in mapping.items(): validate_identity(expected); p = Path(text) if output: p, rel = self._payload_path(text) else: if not p.is_absolute(): p = self.root / relative_name(str(text)) p = unlinked(p); rel = p.relative_to(self.root).as_posix() if p.is_relative_to(self.root) else None p = p.resolve(strict=True) rel = p.relative_to(self.root).as_posix() if p.is_relative_to(self.root) else None require(str(p) not in used, 'Duplicate path aliases'); used.add(str(p)) if not output and rel in self._tracked: require(expected == self._tracked[rel], 'Consumed immutable input differs from committed identity') observed = self._identity_once(p, verified) require(observed == expected, 'Actual file SHA/bytes differ: '+str(p)) result.append(dict(path=str(p), relative_path=rel, **expected)) return sorted(result, key=lambda row: row['path']) @staticmethod def _identity_once(path, verified): """One *invocation-local* actual SHA read per unique file, no stat cache.""" if verified is None: return identity(path) key = str(path) if key not in verified: verified[key] = identity(path) return verified[key] def _check_tail(self): require(not self._write_failed, 'Previous publication failed; independent inspection required, no automatic retry') require(identity(self.header_path) == self.header_identity, 'Ledger header changed') if self._completed: p = self.entries / f'{len(self._completed):016d}.json' require(identity(p)['sha256'] == self._tail, 'Known ledger tail changed') require(not (self.entries / f'{len(self._completed)+1:016d}.json').exists(), 'Unobserved appended entry; independently reopen/inspect') @contextmanager def _writer(self): lock = self.control/'writer.lock'; token = uuid.uuid4().hex.encode() unlinked(lock) descriptor = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: with os.fdopen(descriptor, 'wb') as stream: stream.write(token); stream.flush(); os.fsync(stream.fileno()) self._check_tail() yield finally: # Do not remove another actor's/stale lock or broadly delete a tree. if lock.is_file() and lock.read_bytes() == token: lock.unlink() def _build_inventory(self): root = unlinked(self.root/self.build_name); result = {} if root.exists(): require(root.is_dir(), 'Shared build must be a directory') for p in sorted(root.rglob('*')): unlinked(p) if p.is_file(): result[p.relative_to(self.root).as_posix()] = identity(p) else: require(p.is_dir(), 'Nonregular shared build file') return result def _budget(self, *, outputs=(), extra_control=0): # O(new outputs), not O(all previous payload rows), for the serial path. added = {row['relative_path']:row['bytes'] for row in outputs} provisional = {} for attempt in self._pending.values(): job = attempt['job']; rel = job['relative_path'] if rel is not None and rel not in self._tracked and rel not in added: provisional[rel] = job['bytes'] build = self._build_inventory() immutable = self._immutable_bytes+sum(added.values()) control = self._control_bytes+extra_control build_bytes = sum(row['bytes'] for row in build.values()) total = immutable+control+build_bytes+sum(provisional.values()) require(total <= self.max_output_bytes, 'Tracked outputs/control/shared build exceed cap') require(shutil.disk_usage(self.root).free >= self.reserve_bytes+extra_control, 'Free-space reserve would be crossed') return dict(immutable_payload_bytes=immutable, ledger_control_bytes=control, shared_mutable_build_bytes=build_bytes, provisional_job_bytes=sum(provisional.values()), total_accounted_bytes=total, shared_mutable_build_inventory=build, unlisted_files_included=False, scope='tracked immutable payload + ledger controls + pending job files + current shared mutable build; unlisted files checked at final audit') def begin(self, entry_id, job_file, metadata=None): """Persist BEFORE spawning; success does not authorize automatic retry.""" require(type(entry_id) is str and ID.fullmatch(entry_id), 'Unsafe entry ID') require(metadata is None or type(metadata) is dict, 'Begin metadata must be a JSON dictionary') with self._writer(): require(not self._pending, 'Serial ledger has an uncommitted start; do not rerun it') require(entry_id not in self._completed_set and not (self.starts/(entry_id+'.json')).exists(), 'Entry already started/committed; no retry') path = Path(job_file) if not path.is_absolute(): path = self.root / relative_name(str(job_file)) rows = self._rows({str(path):identity(path)}, output=False) value = dict(schema=SCHEMA+'-start', entry_id=entry_id, registration_sha256=self.registration_sha, next_sequence=len(self._completed)+1, prev_entry_sha256=self._tail, job=rows[0], metadata=metadata or {}, terminal_state='unknown', retry_authorized=False) raw_json(value); self._pending[entry_id] = value try: self._budget(extra_control=len(raw_json(value))) info = publish_exclusive(self.starts/(entry_id+'.json'),value) except BaseException: self._write_failed = True self._pending.pop(entry_id, None); raise self._controls[f'{CONTROL}/started/{entry_id}.json'] = info self._control_bytes += info['bytes'] return dict(entry_id=entry_id,start_sha256=info['sha256'],sequence=value['next_sequence'],terminal_state='unknown',retry_authorized=False) def append(self, entry_id, inputs, outputs, metadata): """Commit exactly one observed-successful serial parent job. metadata requires job_file/process_receipt_file/scope/observed_returncode. The independent parent process JSON must declare entry_id, registration SHA, job SHA, complete=true, exit_code integer 0, timed_out=false, exit_observed_by_parent=true. These are authenticated recorded facts, not a new OS observation by this ledger. """ require(type(metadata) is dict and all(k in metadata for k in ('job_file','process_receipt_file','scope','observed_returncode')), 'Complete parent metadata required') require(type(metadata['scope']) is str and bool(metadata['scope'].strip()) and type(metadata['observed_returncode']) is int and metadata['observed_returncode'] == 0, 'Live/failed process cannot be committed as successful') raw_json(metadata) with self._writer(): require(entry_id in self._pending and entry_id not in self._completed_set, 'No unique uncommitted start; no retry') start = self._pending[entry_id]; startfile = self.starts/(entry_id+'.json') require(identity(startfile) == self._controls[f'{CONTROL}/started/{entry_id}.json'] and read_json(startfile) == start, 'Start authority changed') incoming = self._rows(inputs, output=False); outgoing = self._rows(outputs, output=True) require(outgoing, 'Empty completed output inventory') require(all(row['relative_path'] not in self._tracked for row in outgoing), 'Output overwrites previously tracked file') self._receipt(entry_id,start,incoming,outgoing,metadata) cumulative = self._immutable_bytes+sum(row['bytes'] for row in outgoing) budget = self._budget(outputs=outgoing) value = dict(schema=SCHEMA+'-entry', sequence=len(self._completed)+1, entry_id=entry_id, registration_sha256=self.registration_sha, prev_entry_sha256=self._tail, start_sha256=identity(startfile)['sha256'], consumed_inputs=incoming, new_outputs=outgoing, metadata=metadata, cumulative_immutable_bytes=cumulative, shared_mutable_build=budget['shared_mutable_build_inventory'], hash_scope='actual declared files and current head; historical payload is rehashed at inspect/reopen/final_verify', scientific_gate_claimed=False, OS_exit_independently_observed_here=False) budget = self._budget(outputs=outgoing,extra_control=len(raw_json(value))) require(budget['shared_mutable_build_inventory'] == value['shared_mutable_build'], 'Shared build changed during append accounting') try: info = publish_exclusive(self.entries/f'{value["sequence"]:016d}.json',value) except BaseException: self._write_failed = True; raise self._controls[f'{CONTROL}/entries/{value["sequence"]:016d}.json'] = info self._control_bytes += info['bytes']; self._immutable_bytes = cumulative self._tracked.update({r['relative_path']:{k:r[k] for k in ('bytes','sha256')} for r in outgoing}) self._completed.append(entry_id); self._completed_set.add(entry_id); self._pending.pop(entry_id); self._tail = info['sha256'] return dict(sequence=value['sequence'],entry_id=entry_id,entry_sha256=self._tail, cumulative_immutable_bytes=cumulative,budget=budget,scientific_gate_claimed=False) def _receipt(self,entry_id,start,inputs,outputs,metadata): def path(value): p=Path(value) return str(unlinked(p if p.is_absolute() else self.root/relative_name(str(value)))) jobpath=path(metadata['job_file']);receiptpath=path(metadata['process_receipt_file']) supplied={r['path']:r for r in inputs+outputs} require(jobpath == start['job']['path'] and jobpath in supplied and all(supplied[jobpath][k] == start['job'][k] for k in ('bytes','sha256')), 'Job not bound to original start') require(type(metadata.get('observed_returncode')) is int and metadata['observed_returncode'] == 0 and type(metadata.get('scope')) is str and bool(metadata['scope'].strip()), 'Invalid parent observation/scope') require(receiptpath in {r['path'] for r in outputs}, 'Parent process receipt must be a new output') receipt=read_json(receiptpath) require(receipt.get('entry_id') == entry_id and receipt.get('registration_sha256') == self.registration_sha and receipt.get('job_sha256') == start['job']['sha256'] and receipt.get('complete') is True and type(receipt.get('exit_code')) is int and receipt['exit_code'] == 0 and receipt.get('timed_out') is False and receipt.get('exit_observed_by_parent') is True, 'Missing bound observed-successful parent process receipt') def _replay(self, verified=None): verified = {} if verified is None else verified header_observed = identity(self.header_path) require(header_observed == self.header_identity, 'Ledger header changed') verified[str(self.header_path)] = header_observed if self._completed: require(identity(self.entries/f'{len(self._completed):016d}.json')['sha256'] == self._tail, 'Previously observed ledger head changed') require(not (self.control/'writer.lock').exists(), 'Writer/stale lock present; state unknown, never clear automatically') tracked=dict(self.initial);controls={CONTROL+'/header.json':self.header_identity};completed=[];completed_set=set();tail=None cumulative=sum(v['bytes'] for v in tracked.values()) starts={} require(self.starts.is_dir() and self.entries.is_dir(), 'Missing ledger directories') for p in sorted(self.starts.iterdir()): unlinked(p);require(p.is_file() and p.suffix=='.json' and ID.fullmatch(p.stem), 'Unknown/partial start artifact') row=read_json(p);require(row.get('schema')==SCHEMA+'-start' and row.get('entry_id')==p.stem and row.get('registration_sha256')==self.registration_sha and row.get('retry_authorized') is False, 'Invalid start schema/authority') require(row['job']==self._rows({row['job']['path']:{k:row['job'][k] for k in ('bytes','sha256')}},output=False,verified=verified)[0], 'Start job bytes changed') starts[p.stem]=row;controls[p.relative_to(self.root).as_posix()]=self._identity_once(p,verified) for index,p in enumerate(sorted(self.entries.iterdir()),1): unlinked(p);require(p.is_file() and p.name==f'{index:016d}.json','Missing/out-of-order/temporary ledger entry') row=read_json(p);entry_id=row.get('entry_id');require(entry_id in starts and entry_id not in completed_set,'Missing/duplicate start binding') start=starts[entry_id] require(row.get('schema')==SCHEMA+'-entry' and type(row.get('sequence')) is int and row['sequence']==index and row.get('registration_sha256')==self.registration_sha and row.get('prev_entry_sha256')==tail and start['next_sequence']==index and start['prev_entry_sha256']==tail and row['start_sha256']==controls[f'{CONTROL}/started/{entry_id}.json']['sha256'] and row.get('scientific_gate_claimed') is False,'Entry chain/start identity differs') for field,output in (('consumed_inputs',False),('new_outputs',True)): items=row[field];require(type(items) is list and len({r['path'] for r in items})==len(items),'Invalid stored file rows') observed=self._rows({r['path']:{k:r[k] for k in ('bytes','sha256')} for r in items},output=output,verified=verified) require(items==observed,'Stored absolute/relative/hash identity differs') require(row['new_outputs'] and all(r['relative_path'] not in tracked for r in row['new_outputs']), 'Immutable output repeated') self._receipt(entry_id,start,row['consumed_inputs'],row['new_outputs'],row['metadata']) tracked.update({r['relative_path']:{k:r[k] for k in ('bytes','sha256')} for r in row['new_outputs']}) cumulative+=sum(r['bytes'] for r in row['new_outputs']) require(type(row['cumulative_immutable_bytes']) is int and row['cumulative_immutable_bytes']==cumulative,'Cumulative byte accounting differs') completed.append(entry_id);completed_set.add(entry_id);info=self._identity_once(p,verified);controls[p.relative_to(self.root).as_posix()]=info;tail=info['sha256'] pending={k:v for k,v in starts.items() if k not in completed_set} require(len(pending)<=1 and all(v['next_sequence']==len(completed)+1 and v['prev_entry_sha256']==tail for v in pending.values()), 'Uncommitted start does not follow actual head') for name,info in tracked.items():require(self._identity_once(self.root/name,verified)==info,'Historical payload changed: '+name) allowed=set(controls) for p in self.control.rglob('*'): unlinked(p) require(p.is_dir() or (p.is_file() and p.relative_to(self.root).as_posix() in allowed),'Unlisted/partial ledger control artifact') # Fresh head reads bracket this audit. This is not an atomic FS snapshot. require(identity(self.header_path)==self.header_identity,'Ledger header changed during audit') require(not (self.control/'writer.lock').exists() and not (self.entries/f'{len(completed)+1:016d}.json').exists(), 'Concurrent writer/append detected; audit not terminal') if completed:require(identity(self.entries/f'{len(completed):016d}.json')['sha256']==tail,'Ledger head changed during audit') self._tracked=tracked;self._controls=controls;self._completed=completed;self._completed_set=completed_set;self._pending=pending;self._tail=tail self._immutable_bytes=cumulative self._control_bytes=sum(v['bytes'] for v in controls.values()) def inspect(self): """Full historical read verification; no restart or retry authorization.""" self._replay() return dict(schema=SCHEMA+'-inspection',registration_sha256=self.registration_sha,entries=len(self._completed), entry_ids=list(self._completed),head_sha256=self._tail,uncommitted_entry_ids=sorted(self._pending), uncommitted_state='STARTED-UNCOMMITTED-STATE-UNKNOWN' if self._pending else None, retry_authorized=False,all_tracked_history_rehashed=True,entire_experiment_tree_checked=False, SHA_scope='each unique payload path read during this invocation; no cross-invocation cache; not an atomic snapshot', budget=self._budget(),scientific_gate_claimed=False) def final_verify(self,expected_entry_ids): """Full tree at this instant, before caller writes any future final report. expected IDs must be derived by caller from the externally pinned registration. Shared build is mutable and separately hashed/accounted, never silently accepted as an immutable payload. """ require(type(expected_entry_ids) is list and all(type(x) is str and ID.fullmatch(x) for x in expected_entry_ids) and len(set(expected_entry_ids))==len(expected_entry_ids),'Exact registered expected entry order required') errors=[];verified={} try:self._replay(verified) except (ValueError,OSError,KeyError,TypeError) as exc:errors.append(type(exc).__name__+': '+str(exc)) expected={**self._tracked,**self._controls};actual={};links=[] for p in sorted(self.root.rglob('*')): rel=p.relative_to(self.root).as_posix() try: unlinked(p) if p.is_file():actual[rel]=self._identity_once(p,verified) else:require(p.is_dir(),'Nonregular artifact') except (ValueError,OSError) as exc:links.append(rel);errors.append(str(exc)) build={k:v for k,v in actual.items() if k.startswith(self.build_name+'/')} immutable={k:v for k,v in actual.items() if k not in build} extra=sorted(set(immutable)-set(expected));missing=sorted(set(expected)-set(immutable)) changed=sorted(k for k in set(expected)&set(immutable) if expected[k]!=immutable[k]) exact_order=self._completed==expected_entry_ids actual_bytes=sum(v['bytes'] for v in actual.values());within_cap=actual_bytes<=self.max_output_bytes free_reserve=shutil.disk_usage(self.root).free>=self.reserve_bytes # Bracket the *entire* tree walk as well as replay. These fresh small # authority reads deliberately bypass the per-invocation payload cache. # They detect changed/appended heads, not an atomic filesystem snapshot. try: self._check_tail() require(not (self.control/'writer.lock').exists(), 'Concurrent writer detected at final audit end') except (ValueError,OSError) as exc:errors.append(type(exc).__name__+': '+str(exc)) passed=not(errors or extra or missing or changed or self._pending) and exact_order and within_cap and free_reserve return dict(schema=SCHEMA+'-final-audit',pass_all=passed,registration_sha256=self.registration_sha, entry_ids=list(self._completed),expected_entry_ids=expected_entry_ids,expected_order_exact=exact_order, head_sha256=self._tail,extra_files=extra,missing_files=missing,changed_files=changed,unsafe_files=links,errors=errors, uncommitted_entry_ids=sorted(self._pending),uncommitted_state_unknown=bool(self._pending),retry_authorized=False, actual_total_bytes=actual_bytes,immutable_and_ledger_bytes=actual_bytes-sum(v['bytes'] for v in build.values()), shared_mutable_build_bytes=sum(v['bytes'] for v in build.values()),shared_mutable_build_inventory=build, output_cap_respected=within_cap,free_reserve_respected=free_reserve, all_current_files_rehashed=True,checked_before_future_final_report=True, SHA_scope='each unique path read during this invocation; fresh header/head checks bracket full audit; no cross-invocation cache; not an atomic snapshot', expected_inventory_authority='caller derives IDs from pinned registration; ledger does not interpret scientific plan', scientific_gate_claimed=False)