diff --git a/models/male-cns-lif/normal-conditioning-v1/PROTOCOL.md b/models/male-cns-lif/normal-conditioning-v1/PROTOCOL.md new file mode 100644 index 0000000..1ea422c --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/PROTOCOL.md @@ -0,0 +1,91 @@ +# Normal olfactory acquisition: independent D18 branch + +This phase does not modify MaleCNS reference/runtime, Phase 1–4AQ data, the +base graph, the 14,232 candidate edges, eligibility-v2, APL gain, cell thresholds, +transmitter signs or any previously sealed failure. An engineering interface +PASS is not evidence of learning. + +## Fixed science + +The original AO file (SHA-256 +`a7bda850ee87df4d9f647994e1a1a4074dfea5793184edb1822ab3d8c02e8216`) +supplies master 20260930, all semantic seeds, ordered cues and control schedules. +The old frozen evaluation policy supplies completed-epoch snapshot boundaries. +There are 40 normal blocks: physiological/matched-energy × PAM15/PPL103 × +reciprocal reinforced odor × five training replicates. A is 4-ethylphenol, B is +3-hexanone. Evaluation uses 80 independently registered input streams. + +All seven arms are real: paired, D=0, frozen, random reward, unpaired, CS-only +and DAN-only. 30,000 complete fresh training/control epochs and 3,920 real +fresh A/B probes are required. Even identical baselines are executed, not +relabelled copies. Snapshots 0/1/2/5/10/20/50 never authorize outcome selection. +Only the 50-pairing terminal outcome determines acquisition. Every arm finishes +its entire original schedule, including trailing neutral or blank epochs. + +Epochs are 120 ms, CS is [0,100) ms, scheduled artificial US is at 90 ms. +PAM15 and PPL103 use only their existing confirmed compartment-specific DAN +identities. The positive/negative behavioral predictions are explicit model +hypotheses, not a claim that every PAM/PPL neuron has that function. Scheduled +D is an artificial dimensionless interface, not measured dopamine concentration +or the number of accepted DAN spikes. US attempts/rejections and actual events +remain in the original runtime artifacts. No resampling compensates for a +rejected pulse. + +Fresh neural state is reset each epoch; only the separately authenticated +plastic delta carries forward. Original E/U compute the end-of-epoch update, +then a separate CPU process recomputes it from sealed actual KC events. Probes +have no US, no E/update and no feedback. Brain math is untouched. + +## Readout and gates + +The frozen AQ `learning_gates.py` and original `metrics.py` are pinned before +sampling. All six controls, raw signed MBON balance, functional count/latency +change, MBON separation and both downstream readouts remain mandatory. The +same replicate must satisfy the conjunction; at least three of five pass in +each of the eight cells. PAM/PPL or input modes are never pooled to rescue a +failure. Original H mean-control rules are reported separately for each DAN +control source, both required. Delta change alone cannot imply learning. + +Readout populations are anatomical, never inferred from active/plastic subsets. +The original H actual MBON downstream edge table defines legacyDownstream; +nonMBONDownstream is its exact set difference with the full MBON population. +Missing identity or an altered frozen source stops preparation. + +## Execution, resources and integrity + +Use only the registered 4090 UUID and serial execution. ComfyUI admission must +be empty; never stop or evict a service. Source code is committed before +registration. Static tables invoke the original L PN-rate transform, preserving +all original PN columns for both modes. Shared CS aliases reuse saved input +bytes only. The full 63-bit CUDA sampler, full matrix draw and CS window are +unchanged; there is no CPU-RNG replacement. + +The engineering ceiling is 256 GiB of new outputs, at least 100 GiB disk reserve, +1,800 seconds per child and an absolute 21-day registration deadline. The AQ +linear estimate is only planning arithmetic from limited warm measurements, +not a measured forecast across all conditions. Exceeding a limit means an +incomplete preserved experiment, not shortened trials or changed parameters. + +Each child has an exclusive start record, actual parent-observed exit, +dependency hashes and immutable output inventory. A stopped or uncommitted +child is never silently retried. A persistent incremental ledger avoids +rehashing every historical trial on every iteration; each invocation still +rehashes its actual consumed dependencies. Full audits read every unique file +and reject missing, extra, changed, linked or incomplete output. They are not +claimed to be atomic filesystem snapshots. Mutable compiler cache is accounted +separately. CPU/source tests are not described as GPU numerical regression. + +A detached supervisor keeps the serial controller alive across SSH disconnects. +Administrative logs live in `/data/digital-drosophila/logs/phase4ar-normal-conditioning/`, +outside the immutable per-trial ledger. The final outer auditor must retain +these logs and its actual exit receipt. A launch PID alone proves no completion. + +All original event/state/queue/update artifacts are retained; no dense wholebrain +time-by-neuron output is added. Complete historical sealed integrity is checked +at the boundaries. The five historical reference trials are preserved evidence, +not claimed as newly rerun here. + +Normal acquisition is the only authorized scientific scope of this registration. +No delayed/reversal/retention assay starts automatically, and no candidate is +issued from normal acquisition alone. There is no game, action decoder, RL, +visual input, training of base weights, web service or 3D interface. diff --git a/models/male-cns-lif/normal-conditioning-v1/append_ledger.py b/models/male-cns-lif/normal-conditioning-v1/append_ledger.py new file mode 100644 index 0000000..dd8ae56 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/append_ledger.py @@ -0,0 +1,442 @@ +"""Serial, immutable artifact accounting; not a process runner or science gate. + +The long-lived parent hashes this cycle's declared inputs/outputs and the known +ledger tail. It does NOT rehash all old artifacts per cycle. Reopening, inspect +and final_verify perform full historical verification. Never infer a process's +terminal state from a PID, missing file, elapsed time, or a live handle. +""" +from __future__ import annotations + +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import uuid + +SCHEMA = 'normal-append-ledger-v1' +CONTROL = '.append-ledger' +REGISTRATION = 'protocol/preregistered.json' +SHA = re.compile('[0-9a-f]{64}') +ID = re.compile('[A-Za-z0-9][A-Za-z0-9_-]{0,127}') + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def raw_json(value): + return (json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False)+'\n').encode('utf-8') + + +def read_json(path): + def unique(items): + result = {} + for key, value in items: + require(key not in result, 'Duplicate JSON key'); result[key] = value + return result + return json.loads(Path(path).read_bytes().decode('utf-8'), object_pairs_hook=unique, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON'))) + + +def digest(path): + h = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(8 << 20), b''): h.update(block) + return h.hexdigest() + + +def relative_name(value): + require(type(value) is str and bool(value) and '\\' not in value and '\x00' not in value and + '\n' not in value and '\r' not in value, 'Invalid relative path') + path = PurePosixPath(value) + require(not path.is_absolute() and str(path) == value and + all(p not in ('.', '..', '') and ':' not in p for p in path.parts), 'Escaping/noncanonical path') + return value + + +def unlinked(path): + p = Path(path) + require(p.is_absolute() and '..' not in p.parts, 'Absolute nonescaping path required') + require(not any(q.is_symlink() or (hasattr(q, 'is_junction') and q.is_junction()) for q in (p, *p.parents)), + 'Linked path forbidden') + return p + + +def identity(path): + p = unlinked(path) + require(p.is_file(), 'Missing regular file: '+str(p)) + size = p.stat().st_size; sha = digest(p) + require(p.stat().st_size == size, 'File changed while hashing') + return dict(bytes=size, sha256=sha) + + +def validate_identity(value): + require(type(value) is dict and set(value) == {'bytes', 'sha256'} and type(value['bytes']) is int and + value['bytes'] >= 0 and type(value['sha256']) is str and SHA.fullmatch(value['sha256']), 'Invalid SHA/byte identity') + + +def _sync_directory(directory): + # Windows has no portable directory fsync. File bytes are fsynced there. + if os.name != 'nt': + descriptor = os.open(directory, os.O_RDONLY) + try: os.fsync(descriptor) + finally: os.close(descriptor) + + +def publish_exclusive(path, value): + """Atomic no-replace publication on filesystems supporting hard links. + + No unsafe rename/replace fallback. An interrupted temporary is preserved; + inspect/final audit rejects it. A failed destination collision also never + deletes or replaces the pre-existing destination. + """ + path = unlinked(path); require(path.parent.is_dir(), 'Missing publication parent') + require(not path.exists(), 'Immutable destination already exists; no retry') + raw = raw_json(value); temporary = path.parent / ('.publishing-'+uuid.uuid4().hex) + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, 'wb') as stream: + stream.write(raw); stream.flush(); os.fsync(stream.fileno()) + os.link(temporary, path) # Exclusive destination, same bytes/inode; never overwrite. + _sync_directory(path.parent) + require(path.read_bytes() == raw, 'Published bytes changed') + temporary.unlink() # Only this function's exact, newly created temporary. + _sync_directory(path.parent) + return dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + + +class Ledger: + """One long-lived instance per serial parent, never parallel scheduling. + + Inventories map absolute or root-relative paths to {bytes, sha256}; initial + inventory is root-relative and must contain the exact registration file. + Call begin BEFORE spawn, then append only after the parent has observed + actual exit 0 and saved its independent process receipt. Existing starts + are never reissued. An interrupted append may be independently inspected, + but automatic resume/rerun is not provided. + """ + def __init__(self, root, registration_sha, initial_inventory, max_output_bytes, reserve_bytes, + shared_build_relative='extension-build'): + self.root = unlinked(Path(root)).resolve(strict=True) + require(self.root.is_dir(), 'Existing experiment root required') + require(type(registration_sha) is str and SHA.fullmatch(registration_sha), 'External registration SHA required') + require(type(max_output_bytes) is int and max_output_bytes > 0 and type(reserve_bytes) is int and reserve_bytes >= 0, + 'Explicit nonnegative reserve and positive output cap required') + self.registration_sha = registration_sha; self.max_output_bytes = max_output_bytes; self.reserve_bytes = reserve_bytes + self.build_name = relative_name(shared_build_relative) + require(self.build_name != CONTROL and not self.build_name.startswith(CONTROL+'/'), 'Build overlaps ledger controls') + self.control = self.root / CONTROL; self.header_path = self.control / 'header.json' + self.starts = self.control / 'started'; self.entries = self.control / 'entries' + require(type(initial_inventory) is dict and initial_inventory, 'Explicit initial inventory required') + initial = {} + for name, expected in initial_inventory.items(): + relative_name(name); self._payload_path(name); validate_identity(expected) + require(identity(self.root/name) == expected, 'Initial file identity differs: '+name) + initial[name] = dict(expected) + require(initial.get(REGISTRATION, {}).get('sha256') == registration_sha, 'Initial registration binding missing') + header = dict(schema=SCHEMA, root=str(self.root), registration_sha256=registration_sha, + initial_inventory=initial, max_output_bytes=max_output_bytes, reserve_bytes=reserve_bytes, + shared_mutable_build=self.build_name, initial_inventory_scope='caller supplied; entire tree checked only by final_verify', + scientific_gate_claimed=False) + if self.header_path.exists(): + require(read_json(self.header_path) == header, 'Existing ledger authority/options differ') + else: + require(not self.control.exists(), 'Partial ledger initialization; no automatic retry') + self.control.mkdir(); self.starts.mkdir(); self.entries.mkdir() + publish_exclusive(self.header_path, header) + self.header_identity = identity(self.header_path); self.initial = initial + self._tracked = dict(initial); self._controls = {CONTROL+'/header.json': self.header_identity} + self._completed = []; self._completed_set = set(); self._pending = {}; self._tail = None + self._immutable_bytes = sum(v['bytes'] for v in initial.values()) + self._control_bytes = self.header_identity['bytes'] + self._write_failed = False + self._replay() + self._budget() + + def _payload_path(self, value): + path = Path(value) + if not path.is_absolute(): path = self.root / relative_name(str(value)) + path = unlinked(path) + require(path.is_relative_to(self.root), 'Output escapes experiment') + rel = path.relative_to(self.root).as_posix(); relative_name(rel) + require(rel != CONTROL and not rel.startswith(CONTROL+'/') and rel != self.build_name and + not rel.startswith(self.build_name+'/'), 'Immutable payload overlaps controls/shared build') + return path, rel + + def _rows(self, mapping, *, output, verified=None): + require(type(mapping) is dict, 'Explicit inventory dictionary required') + result = []; used = set() + for text, expected in mapping.items(): + validate_identity(expected); p = Path(text) + if output: p, rel = self._payload_path(text) + else: + if not p.is_absolute(): p = self.root / relative_name(str(text)) + p = unlinked(p); rel = p.relative_to(self.root).as_posix() if p.is_relative_to(self.root) else None + p = p.resolve(strict=True) + rel = p.relative_to(self.root).as_posix() if p.is_relative_to(self.root) else None + require(str(p) not in used, 'Duplicate path aliases'); used.add(str(p)) + if not output and rel in self._tracked: + require(expected == self._tracked[rel], 'Consumed immutable input differs from committed identity') + observed = self._identity_once(p, verified) + require(observed == expected, 'Actual file SHA/bytes differ: '+str(p)) + result.append(dict(path=str(p), relative_path=rel, **expected)) + return sorted(result, key=lambda row: row['path']) + + @staticmethod + def _identity_once(path, verified): + """One *invocation-local* actual SHA read per unique file, no stat cache.""" + if verified is None: return identity(path) + key = str(path) + if key not in verified: verified[key] = identity(path) + return verified[key] + + def _check_tail(self): + require(not self._write_failed, 'Previous publication failed; independent inspection required, no automatic retry') + require(identity(self.header_path) == self.header_identity, 'Ledger header changed') + if self._completed: + p = self.entries / f'{len(self._completed):016d}.json' + require(identity(p)['sha256'] == self._tail, 'Known ledger tail changed') + require(not (self.entries / f'{len(self._completed)+1:016d}.json').exists(), 'Unobserved appended entry; independently reopen/inspect') + + @contextmanager + def _writer(self): + lock = self.control/'writer.lock'; token = uuid.uuid4().hex.encode() + unlinked(lock) + descriptor = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, 'wb') as stream: stream.write(token); stream.flush(); os.fsync(stream.fileno()) + self._check_tail() + yield + finally: + # Do not remove another actor's/stale lock or broadly delete a tree. + if lock.is_file() and lock.read_bytes() == token: lock.unlink() + + def _build_inventory(self): + root = unlinked(self.root/self.build_name); result = {} + if root.exists(): + require(root.is_dir(), 'Shared build must be a directory') + for p in sorted(root.rglob('*')): + unlinked(p) + if p.is_file(): result[p.relative_to(self.root).as_posix()] = identity(p) + else: require(p.is_dir(), 'Nonregular shared build file') + return result + + def _budget(self, *, outputs=(), extra_control=0): + # O(new outputs), not O(all previous payload rows), for the serial path. + added = {row['relative_path']:row['bytes'] for row in outputs} + provisional = {} + for attempt in self._pending.values(): + job = attempt['job']; rel = job['relative_path'] + if rel is not None and rel not in self._tracked and rel not in added: provisional[rel] = job['bytes'] + build = self._build_inventory() + immutable = self._immutable_bytes+sum(added.values()) + control = self._control_bytes+extra_control + build_bytes = sum(row['bytes'] for row in build.values()) + total = immutable+control+build_bytes+sum(provisional.values()) + require(total <= self.max_output_bytes, 'Tracked outputs/control/shared build exceed cap') + require(shutil.disk_usage(self.root).free >= self.reserve_bytes+extra_control, 'Free-space reserve would be crossed') + return dict(immutable_payload_bytes=immutable, ledger_control_bytes=control, shared_mutable_build_bytes=build_bytes, + provisional_job_bytes=sum(provisional.values()), total_accounted_bytes=total, + shared_mutable_build_inventory=build, unlisted_files_included=False, + scope='tracked immutable payload + ledger controls + pending job files + current shared mutable build; unlisted files checked at final audit') + + def begin(self, entry_id, job_file, metadata=None): + """Persist BEFORE spawning; success does not authorize automatic retry.""" + require(type(entry_id) is str and ID.fullmatch(entry_id), 'Unsafe entry ID') + require(metadata is None or type(metadata) is dict, 'Begin metadata must be a JSON dictionary') + with self._writer(): + require(not self._pending, 'Serial ledger has an uncommitted start; do not rerun it') + require(entry_id not in self._completed_set and not (self.starts/(entry_id+'.json')).exists(), 'Entry already started/committed; no retry') + path = Path(job_file) + if not path.is_absolute(): path = self.root / relative_name(str(job_file)) + rows = self._rows({str(path):identity(path)}, output=False) + value = dict(schema=SCHEMA+'-start', entry_id=entry_id, registration_sha256=self.registration_sha, + next_sequence=len(self._completed)+1, prev_entry_sha256=self._tail, job=rows[0], metadata=metadata or {}, + terminal_state='unknown', retry_authorized=False) + raw_json(value); self._pending[entry_id] = value + try: + self._budget(extra_control=len(raw_json(value))) + info = publish_exclusive(self.starts/(entry_id+'.json'),value) + except BaseException: + self._write_failed = True + self._pending.pop(entry_id, None); raise + self._controls[f'{CONTROL}/started/{entry_id}.json'] = info + self._control_bytes += info['bytes'] + return dict(entry_id=entry_id,start_sha256=info['sha256'],sequence=value['next_sequence'],terminal_state='unknown',retry_authorized=False) + + def append(self, entry_id, inputs, outputs, metadata): + """Commit exactly one observed-successful serial parent job. + + metadata requires job_file/process_receipt_file/scope/observed_returncode. + The independent parent process JSON must declare entry_id, registration + SHA, job SHA, complete=true, exit_code integer 0, timed_out=false, + exit_observed_by_parent=true. These are authenticated recorded facts, + not a new OS observation by this ledger. + """ + require(type(metadata) is dict and all(k in metadata for k in + ('job_file','process_receipt_file','scope','observed_returncode')), 'Complete parent metadata required') + require(type(metadata['scope']) is str and bool(metadata['scope'].strip()) and + type(metadata['observed_returncode']) is int and metadata['observed_returncode'] == 0, + 'Live/failed process cannot be committed as successful') + raw_json(metadata) + with self._writer(): + require(entry_id in self._pending and entry_id not in self._completed_set, 'No unique uncommitted start; no retry') + start = self._pending[entry_id]; startfile = self.starts/(entry_id+'.json') + require(identity(startfile) == self._controls[f'{CONTROL}/started/{entry_id}.json'] and read_json(startfile) == start, + 'Start authority changed') + incoming = self._rows(inputs, output=False); outgoing = self._rows(outputs, output=True) + require(outgoing, 'Empty completed output inventory') + require(all(row['relative_path'] not in self._tracked for row in outgoing), 'Output overwrites previously tracked file') + self._receipt(entry_id,start,incoming,outgoing,metadata) + cumulative = self._immutable_bytes+sum(row['bytes'] for row in outgoing) + budget = self._budget(outputs=outgoing) + value = dict(schema=SCHEMA+'-entry', sequence=len(self._completed)+1, entry_id=entry_id, + registration_sha256=self.registration_sha, prev_entry_sha256=self._tail, + start_sha256=identity(startfile)['sha256'], consumed_inputs=incoming, new_outputs=outgoing, + metadata=metadata, cumulative_immutable_bytes=cumulative, shared_mutable_build=budget['shared_mutable_build_inventory'], + hash_scope='actual declared files and current head; historical payload is rehashed at inspect/reopen/final_verify', + scientific_gate_claimed=False, OS_exit_independently_observed_here=False) + budget = self._budget(outputs=outgoing,extra_control=len(raw_json(value))) + require(budget['shared_mutable_build_inventory'] == value['shared_mutable_build'], 'Shared build changed during append accounting') + try: info = publish_exclusive(self.entries/f'{value["sequence"]:016d}.json',value) + except BaseException: + self._write_failed = True; raise + self._controls[f'{CONTROL}/entries/{value["sequence"]:016d}.json'] = info + self._control_bytes += info['bytes']; self._immutable_bytes = cumulative + self._tracked.update({r['relative_path']:{k:r[k] for k in ('bytes','sha256')} for r in outgoing}) + self._completed.append(entry_id); self._completed_set.add(entry_id); self._pending.pop(entry_id); self._tail = info['sha256'] + return dict(sequence=value['sequence'],entry_id=entry_id,entry_sha256=self._tail, + cumulative_immutable_bytes=cumulative,budget=budget,scientific_gate_claimed=False) + + def _receipt(self,entry_id,start,inputs,outputs,metadata): + def path(value): + p=Path(value) + return str(unlinked(p if p.is_absolute() else self.root/relative_name(str(value)))) + jobpath=path(metadata['job_file']);receiptpath=path(metadata['process_receipt_file']) + supplied={r['path']:r for r in inputs+outputs} + require(jobpath == start['job']['path'] and jobpath in supplied and + all(supplied[jobpath][k] == start['job'][k] for k in ('bytes','sha256')), 'Job not bound to original start') + require(type(metadata.get('observed_returncode')) is int and metadata['observed_returncode'] == 0 and + type(metadata.get('scope')) is str and bool(metadata['scope'].strip()), 'Invalid parent observation/scope') + require(receiptpath in {r['path'] for r in outputs}, 'Parent process receipt must be a new output') + receipt=read_json(receiptpath) + require(receipt.get('entry_id') == entry_id and receipt.get('registration_sha256') == self.registration_sha and + receipt.get('job_sha256') == start['job']['sha256'] and receipt.get('complete') is True and + type(receipt.get('exit_code')) is int and receipt['exit_code'] == 0 and receipt.get('timed_out') is False and + receipt.get('exit_observed_by_parent') is True, 'Missing bound observed-successful parent process receipt') + + def _replay(self, verified=None): + verified = {} if verified is None else verified + header_observed = identity(self.header_path) + require(header_observed == self.header_identity, 'Ledger header changed') + verified[str(self.header_path)] = header_observed + if self._completed: + require(identity(self.entries/f'{len(self._completed):016d}.json')['sha256'] == self._tail, + 'Previously observed ledger head changed') + require(not (self.control/'writer.lock').exists(), 'Writer/stale lock present; state unknown, never clear automatically') + tracked=dict(self.initial);controls={CONTROL+'/header.json':self.header_identity};completed=[];completed_set=set();tail=None + cumulative=sum(v['bytes'] for v in tracked.values()) + starts={} + require(self.starts.is_dir() and self.entries.is_dir(), 'Missing ledger directories') + for p in sorted(self.starts.iterdir()): + unlinked(p);require(p.is_file() and p.suffix=='.json' and ID.fullmatch(p.stem), 'Unknown/partial start artifact') + row=read_json(p);require(row.get('schema')==SCHEMA+'-start' and row.get('entry_id')==p.stem and + row.get('registration_sha256')==self.registration_sha and row.get('retry_authorized') is False, + 'Invalid start schema/authority') + require(row['job']==self._rows({row['job']['path']:{k:row['job'][k] for k in ('bytes','sha256')}},output=False,verified=verified)[0], 'Start job bytes changed') + starts[p.stem]=row;controls[p.relative_to(self.root).as_posix()]=self._identity_once(p,verified) + for index,p in enumerate(sorted(self.entries.iterdir()),1): + unlinked(p);require(p.is_file() and p.name==f'{index:016d}.json','Missing/out-of-order/temporary ledger entry') + row=read_json(p);entry_id=row.get('entry_id');require(entry_id in starts and entry_id not in completed_set,'Missing/duplicate start binding') + start=starts[entry_id] + require(row.get('schema')==SCHEMA+'-entry' and type(row.get('sequence')) is int and row['sequence']==index and + row.get('registration_sha256')==self.registration_sha and row.get('prev_entry_sha256')==tail and + start['next_sequence']==index and start['prev_entry_sha256']==tail and + row['start_sha256']==controls[f'{CONTROL}/started/{entry_id}.json']['sha256'] and + row.get('scientific_gate_claimed') is False,'Entry chain/start identity differs') + for field,output in (('consumed_inputs',False),('new_outputs',True)): + items=row[field];require(type(items) is list and len({r['path'] for r in items})==len(items),'Invalid stored file rows') + observed=self._rows({r['path']:{k:r[k] for k in ('bytes','sha256')} for r in items},output=output,verified=verified) + require(items==observed,'Stored absolute/relative/hash identity differs') + require(row['new_outputs'] and all(r['relative_path'] not in tracked for r in row['new_outputs']), 'Immutable output repeated') + self._receipt(entry_id,start,row['consumed_inputs'],row['new_outputs'],row['metadata']) + tracked.update({r['relative_path']:{k:r[k] for k in ('bytes','sha256')} for r in row['new_outputs']}) + cumulative+=sum(r['bytes'] for r in row['new_outputs']) + require(type(row['cumulative_immutable_bytes']) is int and row['cumulative_immutable_bytes']==cumulative,'Cumulative byte accounting differs') + completed.append(entry_id);completed_set.add(entry_id);info=self._identity_once(p,verified);controls[p.relative_to(self.root).as_posix()]=info;tail=info['sha256'] + pending={k:v for k,v in starts.items() if k not in completed_set} + require(len(pending)<=1 and all(v['next_sequence']==len(completed)+1 and v['prev_entry_sha256']==tail for v in pending.values()), + 'Uncommitted start does not follow actual head') + for name,info in tracked.items():require(self._identity_once(self.root/name,verified)==info,'Historical payload changed: '+name) + allowed=set(controls) + for p in self.control.rglob('*'): + unlinked(p) + require(p.is_dir() or (p.is_file() and p.relative_to(self.root).as_posix() in allowed),'Unlisted/partial ledger control artifact') + # Fresh head reads bracket this audit. This is not an atomic FS snapshot. + require(identity(self.header_path)==self.header_identity,'Ledger header changed during audit') + require(not (self.control/'writer.lock').exists() and not (self.entries/f'{len(completed)+1:016d}.json').exists(), + 'Concurrent writer/append detected; audit not terminal') + if completed:require(identity(self.entries/f'{len(completed):016d}.json')['sha256']==tail,'Ledger head changed during audit') + self._tracked=tracked;self._controls=controls;self._completed=completed;self._completed_set=completed_set;self._pending=pending;self._tail=tail + self._immutable_bytes=cumulative + self._control_bytes=sum(v['bytes'] for v in controls.values()) + + def inspect(self): + """Full historical read verification; no restart or retry authorization.""" + self._replay() + return dict(schema=SCHEMA+'-inspection',registration_sha256=self.registration_sha,entries=len(self._completed), + entry_ids=list(self._completed),head_sha256=self._tail,uncommitted_entry_ids=sorted(self._pending), + uncommitted_state='STARTED-UNCOMMITTED-STATE-UNKNOWN' if self._pending else None, + retry_authorized=False,all_tracked_history_rehashed=True,entire_experiment_tree_checked=False, + SHA_scope='each unique payload path read during this invocation; no cross-invocation cache; not an atomic snapshot', + budget=self._budget(),scientific_gate_claimed=False) + + def final_verify(self,expected_entry_ids): + """Full tree at this instant, before caller writes any future final report. + + expected IDs must be derived by caller from the externally pinned + registration. Shared build is mutable and separately hashed/accounted, + never silently accepted as an immutable payload. + """ + require(type(expected_entry_ids) is list and all(type(x) is str and ID.fullmatch(x) for x in expected_entry_ids) and + len(set(expected_entry_ids))==len(expected_entry_ids),'Exact registered expected entry order required') + errors=[];verified={} + try:self._replay(verified) + except (ValueError,OSError,KeyError,TypeError) as exc:errors.append(type(exc).__name__+': '+str(exc)) + expected={**self._tracked,**self._controls};actual={};links=[] + for p in sorted(self.root.rglob('*')): + rel=p.relative_to(self.root).as_posix() + try: + unlinked(p) + if p.is_file():actual[rel]=self._identity_once(p,verified) + else:require(p.is_dir(),'Nonregular artifact') + except (ValueError,OSError) as exc:links.append(rel);errors.append(str(exc)) + build={k:v for k,v in actual.items() if k.startswith(self.build_name+'/')} + immutable={k:v for k,v in actual.items() if k not in build} + extra=sorted(set(immutable)-set(expected));missing=sorted(set(expected)-set(immutable)) + changed=sorted(k for k in set(expected)&set(immutable) if expected[k]!=immutable[k]) + exact_order=self._completed==expected_entry_ids + actual_bytes=sum(v['bytes'] for v in actual.values());within_cap=actual_bytes<=self.max_output_bytes + free_reserve=shutil.disk_usage(self.root).free>=self.reserve_bytes + # Bracket the *entire* tree walk as well as replay. These fresh small + # authority reads deliberately bypass the per-invocation payload cache. + # They detect changed/appended heads, not an atomic filesystem snapshot. + try: + self._check_tail() + require(not (self.control/'writer.lock').exists(), 'Concurrent writer detected at final audit end') + except (ValueError,OSError) as exc:errors.append(type(exc).__name__+': '+str(exc)) + passed=not(errors or extra or missing or changed or self._pending) and exact_order and within_cap and free_reserve + return dict(schema=SCHEMA+'-final-audit',pass_all=passed,registration_sha256=self.registration_sha, + entry_ids=list(self._completed),expected_entry_ids=expected_entry_ids,expected_order_exact=exact_order, + head_sha256=self._tail,extra_files=extra,missing_files=missing,changed_files=changed,unsafe_files=links,errors=errors, + uncommitted_entry_ids=sorted(self._pending),uncommitted_state_unknown=bool(self._pending),retry_authorized=False, + actual_total_bytes=actual_bytes,immutable_and_ledger_bytes=actual_bytes-sum(v['bytes'] for v in build.values()), + shared_mutable_build_bytes=sum(v['bytes'] for v in build.values()),shared_mutable_build_inventory=build, + output_cap_respected=within_cap,free_reserve_respected=free_reserve, + all_current_files_rehashed=True,checked_before_future_final_report=True, + SHA_scope='each unique path read during this invocation; fresh header/head checks bracket full audit; no cross-invocation cache; not an atomic snapshot', + expected_inventory_authority='caller derives IDs from pinned registration; ledger does not interpret scientific plan', + scientific_gate_claimed=False) diff --git a/models/male-cns-lif/normal-conditioning-v1/normal_inputs.py b/models/male-cns-lif/normal-conditioning-v1/normal_inputs.py new file mode 100644 index 0000000..40e7a4e --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/normal_inputs.py @@ -0,0 +1,131 @@ +"""Exact frozen L rate transform and full-shape CUDA sampler, no brain/update. + +Static table derivation is CPU/read-only. Sampling is a separate explicit job. +Common-random aliases share only saved immutable CS bytes, never brain outputs. +""" +from __future__ import annotations +import argparse +import os +from pathlib import Path +import sys +import time +import numpy as np +from normal_io import * + +PINS = { + 'conditioning_inputs.py': '19382e066e450140bb653a3661f980ffe7cc62f412020ef57df0b6e6b3cbb8ba', + 'conditioning_preflight_inputs.py': '846d2141b472f18b1bb728eed4aa33876896d4b33dece364787c81204f62995f', + 'trial_plan.py': '6fed3f494d075339a7e5b001d43b2598640b189e038952fbd4d41200bacac587', + 'seed_registry.py': '0da033b132095ec3087b5d9f64e53c2e391fa50ab0b7653b31296eeb84799b7e', +} + +def dependencies(): + verify({str(LAB/n): s for n,s in PINS.items()}) + require(not any(n in sys.modules for n in ('runtime_light','conditioning_epoch')), 'No neural modules in sampler') + sys.path.insert(0, str(LAB)) + import conditioning_preflight_inputs as source + import conditioning_inputs as inputs + require(Path(source.__file__) == LAB/'conditioning_preflight_inputs.py' and + Path(inputs.__file__) == LAB/'conditioning_inputs.py', 'Wrong input module identity') + return source, inputs + +def derive_rates(root=ROOT): + """Calls the original pinned pn_drive for both registered modes.""" + import pandas as pd + import yaml + source, inputs = dependencies() + loaded = source.load_rate_tables(root) + static = Path(root)/source.STATIC + selection = read(static/'odor-encoder/selection.json') + panel = pd.read_parquet(static/'odor-encoder/frozen-odor-panel.parquet') + mapping = pd.read_parquet(static/'receptor-mapping/olfactory-mapping.parquet') + config = yaml.safe_load((Path(root)/source.CONFIG_REL).read_text()) + drive, ast_sha = source._extract_drive((Path(root)/source.SOURCE_REL).read_bytes(), str(Path(root)/source.SOURCE_REL)) + vectors = npz(static/'odor-encoder/mapped-glomerular-vectors.npz') + result = {} + for mode in ('physiological','matched_energy'): + for cue in ('A','B'): + original = loaded['tables'][cue] + targets, rates, frame = drive(original['odor_id'], vectors[original['odor_id']], mapping, + selection['glomeruli_order'], mode, loaded['audit']['median_mapped_response_sum'], config) + inputs.probabilities_f32(targets, rates) + require(np.array_equal(frame.neuron_idx.to_numpy(np.int32), targets) and + frame.rate_hz.to_numpy(np.float32).tobytes() == rates.tobytes(), 'Original PN order differs') + if mode == 'physiological': + require(all(original[n].tobytes() == a.tobytes() for n,a in (('targets',targets),('rates_hz',rates))), + 'Physiological reconstruction differs') + prior = npz(Path(root)/f'experiments/phase4ao-conditioning-interface/inputs/static/{cue}-rates.npz') + require(all(prior[n].dtype == a.dtype and prior[n].shape == a.shape and prior[n].tobytes() == a.tobytes() + for n,a in (('targets',targets),('rates_hz',rates))), 'Prior AO rate bytes differ') + result[mode+'--'+cue] = dict(targets=targets, rates_hz=rates) + verify({str(Path(root)/p):s for p,s in source.SOURCE_PINS.items()}) + return result, dict(original_source_sha256=source.SOURCE_PINS, original_AST_sha256=ast_sha, + physiological_matches_original_AO=True, original_PN_column_order_preserved=True, + modes=['physiological','matched_energy'], clipping=False, gain_tuning=False, + historical_dynamic_results_read=False, random_draws=0) + +def descriptor(task): + r = task['recipe'] + require(task['steps'] == 1200 and task['mode'] in ('physiological','matched_energy'), 'Wrong normal clock/mode') + if task['kind'] == 'train': + cs = r['cs'] + if cs is None: return None + cue, seed, start, stop = cs['cue'],cs['seed'],cs['start_tick'],cs['stop_tick_exclusive'] + stream = cs['source_stream_id'] + else: + require(task['kind'] == 'probe' and r['US_requested'] is False and r['weight_updates_enabled'] is False, + 'Probe contains US/update') + cue, seed = r['cue'], r['input']['seed'] + start, stop, stream = r['cs_start_tick'], r['cs_stop_tick_exclusive'], r['input']['stream_id'] + require(cue in ('A','B') and type(seed) is int and 0 <= seed < 2**63 and (start,stop) == (0,1000), + 'CS timing/full seed differs') + return dict(mode=task['mode'],cue=cue,seed=seed,steps=1200,cs_start_tick=start,cs_stop_tick_exclusive=stop, + source_stream_id=stream,table_id=task['mode']+'--'+cue) + +def inventory_from_plan(plan): + descriptors, task_inputs = {}, {} + for task in plan['tasks']: + d = descriptor(task) + key = None if d is None else content_sha(d) + if d is not None: + require(key not in descriptors or descriptors[key] == d, 'Conflicting shared CS descriptor') + descriptors[key] = d + require(task['task_id'] not in task_inputs, 'Duplicate planned task') + task_inputs[task['task_id']] = key + require(len(descriptors) == 4080, 'Expected 4,000 training + 80 evaluation streams') + return dict(schema='normal-CS-input-inventory-v1',descriptors=descriptors,task_inputs=task_inputs, + brain_output_deduplication=False,CS_only_saved=True,US_composed_from_each_original_epoch=True) + +def sample(reg_sha): + require(sys.dont_write_bytecode, 'Use -B') + file = EXP/'protocol/preregistered.json'; require(digest(file) == reg_sha, 'Registration differs') + reg = read(file); verify(reg['source_sha256']); verify(reg['static_input_sha256']) + require(reg['scientific_scope'] == 'normal-acquisition-only', 'Wrong scientific scope') + out = unlinked(EXP/'inputs/generated'); require(not out.exists(), 'Sampler cannot retry existing output') + h = load_source('_normal_sampler_admission', AP/'isolated_workers.py', AP_SHA) + os.environ['CUDA_VISIBLE_DEVICES'] = GPU + admission = h._admit_brain() + source, inputs = dependencies() + inv = read(EXP/'protocol/input-inventory.json') + require(digest(EXP/'protocol/input-inventory.json') == reg['prepared_sha256'][str(EXP/'protocol/input-inventory.json')], + 'Input inventory differs') + out.mkdir(); start = time.perf_counter(); completed = {} + for index,(key,d) in enumerate(inv['descriptors'].items()): + if index % 100 == 0: h._admit_brain() + table = npz(EXP/f"inputs/static/{d['table_id']}.npz") + require(digest(EXP/f"inputs/static/{d['table_id']}.npz") == + reg['prepared_sha256'][str(EXP/f"inputs/static/{d['table_id']}.npz")], 'Rate file changed') + result = inputs.generate_cs_requests(table['targets'],table['rates_hz'],seed=d['seed'],steps=1200, + cs_start_tick=0,cs_stop_tick=1000,device='cuda:0',max_uniform_draws=reg['budget']['max_uniform_draws']) + save_npz(out/(key+'.npz'),result['requests']) + save(out/(key+'.json'),dict(descriptor=d,audit=result['audit'],requests_sha256=digest(out/(key+'.npz')))) + completed[key] = dict(requests=pin(out/(key+'.npz')),audit=pin(out/(key+'.json'))) + verify(reg['source_sha256']); verify(reg['static_input_sha256']) + save(EXP/'receipts/input-generation.json',dict(complete=True,registration_sha256=reg_sha, + input_inventory_sha256=digest(EXP/'protocol/input-inventory.json'),inputs=completed, + descriptors=len(completed),wall_seconds=time.perf_counter()-start,admission=admission, + brain_executed=False,weight_updates_executed=False)) + +if __name__ == '__main__': + p=argparse.ArgumentParser();p.add_argument('--registration-sha256',required=True) + sample(p.parse_args().registration_sha256) diff --git a/models/male-cns-lif/normal-conditioning-v1/normal_io.py b/models/male-cns-lif/normal-conditioning-v1/normal_io.py new file mode 100644 index 0000000..e40a9df --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/normal_io.py @@ -0,0 +1,88 @@ +"""Small exclusive file helpers. No neural imports, mutation or retry policy.""" +from __future__ import annotations +import hashlib +import importlib.util +import json +from pathlib import Path +import sys + +ROOT = Path('/data/digital-drosophila') +MODEL = Path(__file__).resolve().parent +LAB = MODEL.parent/'olfactory-conditioning-lab-v1' +AQ = MODEL.parent/'conditioning-execution-v1' +AP = MODEL.parent/'conditioning-interface-v2' +EXP = ROOT/'experiments/phase4ar-normal-conditioning' +GPU = 'GPU-79f3a85d-adce-7833-7003-6488140ca596' +AP_SHA = 'e47651a8ea99d7c44115107ff6b5dc71312183609ae11c891db65bb21ed61e30' + +def require(ok, message): + if not ok: raise ValueError(message) + +def digest(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for data in iter(lambda: f.read(8 << 20), b''): h.update(data) + return h.hexdigest() + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False) + +def content_sha(value): + return hashlib.sha256(canonical(value).encode()).hexdigest() + +def read(path): + def pairs(items): + result = {} + for key, value in items: + require(key not in result, 'Duplicate JSON key'); result[key] = value + return result + return json.loads(Path(path).read_text(encoding='utf-8'), object_pairs_hook=pairs, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON'))) + +def save(path, value): + with unlinked(path).open('x', encoding='utf-8', newline='\n') as f: + json.dump(value, f, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False); f.write('\n') + +def unlinked(path): + p = Path(path) + require(p.is_absolute() and '..' not in p.parts and not any(x.is_symlink() or + (hasattr(x, 'is_junction') and x.is_junction()) for x in (p, *p.parents)), 'Unsafe linked path') + return p + +def identity(path): + p = unlinked(path); require(p.is_file(), 'Missing regular file: '+str(p)) + return dict(sha256=digest(p), bytes=p.stat().st_size) + +def pin(path): + return dict(path=str(unlinked(path)), sha256=digest(path)) + +def verify(pins): + for p, sha in pins.items(): require(identity(p)['sha256'] == sha, 'Pinned input changed: '+str(p)) + +def inventory(root): + result = {} + for p in sorted(unlinked(root).rglob('*')): + unlinked(p) + if p.is_file(): result[str(p)] = identity(p) + else: require(p.is_dir(), 'Nonregular artifact') + return result + +def load_source(name, path, sha): + path = unlinked(path); raw = path.read_bytes() + require(hashlib.sha256(raw).hexdigest() == sha and name not in sys.modules, 'Source/module identity differs') + spec = importlib.util.spec_from_file_location(name, path); module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + exec(compile(raw, str(path), 'exec'), module.__dict__) + return module + +def npz(path): + import numpy as np + with np.load(path, allow_pickle=False) as f: return {k: f[k].copy() for k in f.files} + +def save_npz(path, values): + import numpy as np + with unlinked(path).open('xb') as f: np.savez_compressed(f, **values) + +def task_key(task): + # IDs remain verbatim in receipts; filesystem IDs never incorporate slashes. + return 'task-'+hashlib.sha256(task['task_id'].encode()).hexdigest() diff --git a/models/male-cns-lif/normal-conditioning-v1/normal_metrics.py b/models/male-cns-lif/normal-conditioning-v1/normal_metrics.py new file mode 100644 index 0000000..78f03ab --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/normal_metrics.py @@ -0,0 +1,528 @@ +"""Read-only normal analysis, conditional on independent final integrity. + +analyze(exp, registration_sha256, *, verified_inventory, final_ledger) consumes +an externally authenticated absolute-path -> {bytes, sha256} inventory and the +final append-ledger audit. Neither caller Boolean nor digest alone is treated +as an OS observation: actual ledger/start/job/parent/worker receipts and every +consumed file are rebound to bytes. SHA caching is invocation-local, never +mtime-based. The outer verifier still owns complete tree/historical integrity, +anatomical/source authority, concurrency exclusion and the final science claim. +No writes, sampling, subprocesses, follow-up assays, candidates or new gates. +""" +from __future__ import annotations + +from collections import defaultdict +import hashlib +import io +import json +from pathlib import Path +import types + +import numpy as np + +GATES_SHA256 = '8ae9bcb77fe14dbdda164f86ffdbfc1e2976be752766ef526777577223b4dd99' +PLAN_SHA256 = 'd1a6b840a547cb10b00df40f4d8b287867d09f355c9777e5f7a8b9489f5555ea' +ARM_NAMES = dict(paired='paired', d_zero='D=0', frozen='frozen', random_reward='random reward', + unpaired='unpaired', cs_only='CS-only', dan_only='DAN-only') +ODORS = {'A': '4-ethylphenol', 'B': '3-hexanone'} +POINTS = (0, 1, 2, 5, 10, 20, 50) + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False) + + +def sha(value): + return hashlib.sha256(canonical(value).encode()).hexdigest() + + +def _module(name, path, expected): + raw = path.read_bytes() + require(hashlib.sha256(raw).hexdigest() == expected, 'Analysis dependency pin differs: ' + str(path)) + module = types.ModuleType(name) + module.__file__ = str(path) + exec(compile(raw, str(path), 'exec'), module.__dict__) + return module + + +_HERE = Path(__file__).resolve().parent +G = _module('_normal_frozen_gates', _HERE.parent / 'conditioning-execution-v1/learning_gates.py', GATES_SHA256) +P = _module('_normal_frozen_plan', _HERE / 'normal_plan.py', PLAN_SHA256) +M = G.M + + +class Reader: + """Actual SHA once per unique path per invocation; not an atomic snapshot.""" + def __init__(self, inventory): + require(type(inventory) is dict and inventory, 'Externally verified checksum inventory required') + self.inventory = inventory + self.seen = {} + + def path(self, path): + path = Path(path) + require(path.is_absolute() and '..' not in path.parts and not any(p.is_symlink() or + (hasattr(p, 'is_junction') and p.is_junction()) for p in (path, *path.parents)), 'Unsafe artifact path') + key = str(path) + require(key in self.inventory, 'Artifact absent from external checksum authority: ' + key) + expected = self.inventory[key] + require(type(expected) is dict and set(expected) == {'bytes', 'sha256'} and + type(expected['bytes']) is int and expected['bytes'] >= 0 and M._sha(expected['sha256']), 'Invalid checksum authority') + return path, expected + + def check(self, path): + path, expected = self.path(path) + key = str(path) + if key not in self.seen: + require(path.is_file(), 'Missing artifact: ' + key) + digest = hashlib.sha256() + size = 0 + with path.open('rb') as stream: + for part in iter(lambda: stream.read(8 << 20), b''): + digest.update(part); size += len(part) + require(dict(bytes=size, sha256=digest.hexdigest()) == expected, 'Actual file checksum differs: ' + key) + self.seen[key] = dict(expected) + require(self.seen[key] == expected, 'Checksum authority changed during analysis') + return expected + + def raw(self, path): + self.check(path) + # Parsed data are read from the now-bound file, not retained for every + # trajectory. Concurrent mutation remains an outer quiescence obligation. + return Path(path).read_bytes() + + def json(self, path): + def unique(items): + result = {} + for key, value in items: + require(key not in result, 'Duplicate JSON field'); result[key] = value + return result + return json.loads(self.raw(path).decode('utf-8'), object_pairs_hook=unique, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON'))) + + def arrays(self, path): + with np.load(io.BytesIO(self.raw(path)), allow_pickle=False) as archive: + return {key: archive[key].copy() for key in archive.files} + + +def task_key(task): + return 'task-' + hashlib.sha256(task['task_id'].encode()).hexdigest() + + +def expected_entries(plan): + result = ['input-generation'] + for task in plan['tasks']: + result.extend(task_key(task) + suffix for suffix in + (('--brain', '--update', '--verify') if task['kind'] == 'train' else ('--brain',))) + return result + + +def audit_case(reader, exp, task, job, *, template, inputs, sampler, populations, population_file, output_cap): + """Independent raw CS/US composition and reset/spec checks; no brain call. + + A scheduled US request is not an accepted spike or a successful behavior. + The artificial D scalar remains the original recipe, never an event count. + """ + exp = Path(exp); case = exp / 'cases' / task_key(task) + require(job['case_dir'] == str(case), 'Case path differs from original task') + spec = reader.json(case / 'spec.json') + removed = ('validation_cohort', 'odor_index', 'expected_external_events', 'training', 'cs_input_seed', 'cs_input_seed_bits', + 'seed_scope', 'odor_id', 'odor_name', 'source_panel_row', 'source_panel_split', 'rate_sum_hz', 'target_count', + 'expected_CS_requests', 'CS_window', 'outer_epoch_id', 'outer_arm', 'outer_protocol') + fixed = {k: v for k, v in template.items() if k not in removed} + fixed.update(name=task_key(task), steps=1200, duration_ms=120., max_steps=1200, budget_root=str(exp), + output_budget_bytes=output_cap, mode=task['mode'], within_trial_weight_updates=False, + outer_protocol='normal-acquisition-v1', outer_task_id=task['task_id'], outer_arm=task['arm']) + require(canonical(spec) == canonical(fixed) and spec['seed'] == template['seed'] and spec['dt_ms'] == .1 and + spec['apl_delay_steps'] == 18 and spec['precision'] == 'float64' and spec['apl_precision'] == 'float32' and + spec['record_history'] is False and spec['kind'] == 'fullgraph', 'Scientific model/reset/clock spec changed') + recipe = task['recipe'] + cs_recipe = recipe['cs'] if task['kind'] == 'train' else recipe + expected = None + if cs_recipe is not None: + cue = cs_recipe['cue'] + seed = cs_recipe['seed'] if task['kind'] == 'train' else cs_recipe['input']['seed'] + stream = cs_recipe['source_stream_id'] if task['kind'] == 'train' else cs_recipe['input']['stream_id'] + require(type(seed) is int and 0 <= seed < 2**63, 'Original full63 seed required') + expected = dict(mode=task['mode'], cue=cue, seed=seed, steps=1200, cs_start_tick=0, + cs_stop_tick_exclusive=1000, source_stream_id=stream, table_id=task['mode'] + '--' + cue) + csid = None if expected is None else sha(expected) + require(inputs['task_inputs'][task['task_id']] == csid, 'Original task/CS descriptor mapping differs') + binding = reader.json(case / 'input-binding.json') + require(binding['task_id'] == task['task_id'] and binding['CS_descriptor_id'] == csid and + binding['DAN_type'] == task['dan_type'] and binding['population_file'] == + dict(path=population_file, sha256=reader.check(population_file)['sha256']), 'Case input/population binding differs') + empty = dict(tick=np.empty(0, np.int32), neuron_idx=np.empty(0, np.int32)) + source = empty + if csid is None: + require(binding['CS_source'] is None, 'No-CS epoch acquired an odor source') + else: + npzpath = exp / f'inputs/generated/{csid}.npz'; jsonpath = exp / f'inputs/generated/{csid}.json' + generated = reader.json(jsonpath); audit = generated['audit'] + require(inputs['descriptors'][csid] == generated['descriptor'] == expected and + generated['requests_sha256'] == reader.check(npzpath)['sha256'] and + binding['CS_source'] == dict(path=str(npzpath), sha256=reader.check(npzpath)['sha256']) and + sampler['inputs'][csid] == dict(requests=binding['CS_source'], audit=dict(path=str(jsonpath), sha256=reader.check(jsonpath)['sha256'])), + 'Saved sampler receipt/descriptor differs') + require(audit['seed'] == audit['generator_initial_seed'] == seed and audit['seed_bits'] == 63 and + audit['seed_conversion'] == 'exact integer; no truncation/modulo/offset' and audit['private_generator'] is True and + audit['matrix_draw_calls'] == 1 and audit['uniform_matrix_shape'][0] == 1200 and audit['CS_window'] == [0, 1000], + 'Original sampler seed/full-shape/window evidence differs') + source = reader.arrays(npzpath) + + def keys(request): + require(set(request) == {'tick', 'neuron_idx'} and all(v.dtype == np.int32 and v.ndim == 1 for v in request.values()) and + request['tick'].shape == request['neuron_idx'].shape and np.all((request['tick'] >= 0) & (request['tick'] < 1200)) and + np.all((request['neuron_idx'] >= 0) & (request['neuron_idx'] < 166700)), 'Invalid raw request arrays') + result = request['tick'].astype(np.int64) * 166700 + request['neuron_idx'] + require(np.all(result[1:] > result[:-1]), 'Requests are not sorted unique') + return result + + source_keys = keys(source) + require(np.all(source['tick'] < 1000), 'CS escaped original window') + cs, us, combined = (reader.arrays(case / name) for name in ('cs_requests.npz', 'us_requests.npz', 'requests.npz')) + cs_keys, us_keys, combined_keys = keys(cs), keys(us), keys(combined) + require(np.array_equal(source_keys, cs_keys), 'Actual case CS differs from original saved input') + requested = task['kind'] == 'train' and recipe['us']['requested'] + dan = populations[task['dan_type']] + require(dan.dtype == np.int32 and dan.ndim == 1 and np.all(dan[1:] > dan[:-1]), 'Actual DAN mapping differs') + require(binding['DAN_membership_sha256'] == hashlib.sha256(dan.tobytes()).hexdigest(), 'DAN membership binding differs') + expected_us = 900 * 166700 + dan.astype(np.int64) if requested else np.empty(0, np.int64) + require(np.array_equal(us_keys, expected_us) and not np.intersect1d(cs_keys, us_keys).size and + np.array_equal(combined_keys, np.union1d(cs_keys, us_keys)), 'Actual scheduled US/CS composition differs') + masks = reader.arrays(case / 'source-mask.npz') + expected_mask = np.where(np.isin(combined_keys, cs_keys), 1, 2).astype(np.uint8) + require(set(masks) == {'source_mask'} and masks['source_mask'].dtype == np.uint8 and + np.array_equal(masks['source_mask'], expected_mask) and binding['composition']['policy'] == 'error' and + binding['composition']['cross_source_conflict_count'] == 0, 'Raw composition/source-mask differs') + + +class Evidence: + def __init__(self, reader, exp, registration_sha, expected, final): + require(final.get('schema') == 'normal-append-ledger-v1-final-audit' and + final.get('registration_sha256') == registration_sha and final.get('pass_all') is True and + final.get('entry_ids') == final.get('expected_entry_ids') == expected and + final.get('expected_order_exact') is True, 'Final ledger authority/complete order differs') + require(all(final.get(k) == [] for k in ('extra_files', 'missing_files', 'changed_files', + 'unsafe_files', 'errors', 'uncommitted_entry_ids')) and final.get('uncommitted_state_unknown') is False, + 'Incomplete/failed final ledger audit') + self.r, self.exp, self.regsha, self.expected = reader, Path(exp), registration_sha, expected + self.index, self.tail, self.final = 0, None, final + + def process(self, entry_id, kind): + self.index += 1 + require(self.expected[self.index - 1] == entry_id, 'Analysis process order differs') + entry_path = self.exp / f'.append-ledger/entries/{self.index:016d}.json' + entry = self.r.json(entry_path) + start_path = self.exp / f'.append-ledger/started/{entry_id}.json' + start = self.r.json(start_path) + job_path = self.exp / f'jobs/{entry_id}.json' + process_path = self.exp / 'receipts' / entry_id / 'process.json' + require(entry['entry_id'] == start['entry_id'] == entry_id and entry['sequence'] == start['next_sequence'] == self.index and + entry['registration_sha256'] == start['registration_sha256'] == self.regsha and + entry['prev_entry_sha256'] == start['prev_entry_sha256'] == self.tail and + entry['start_sha256'] == self.r.check(start_path)['sha256'] and start['retry_authorized'] is False, + 'Ledger/start chain differs') + incoming = {row['path']: row for row in entry['consumed_inputs']} + outgoing = {row['path']: row for row in entry['new_outputs']} + require(len(incoming) == len(entry['consumed_inputs']) and len(outgoing) == len(entry['new_outputs']), 'Duplicate ledger files') + for path, row in {**incoming, **outgoing}.items(): + _, bound = self.r.path(path) + require(all(row[k] == bound[k] for k in ('bytes', 'sha256')), 'Ledger/checksum authority differs') + metadata = entry['metadata'] + require(metadata['job_file'] == str(job_path) and metadata['process_receipt_file'] == str(process_path) and + type(metadata['observed_returncode']) is int and metadata['observed_returncode'] == 0 and + str(process_path) in outgoing and str(job_path) in incoming and + start['job']['path'] == str(job_path) and start['job']['sha256'] == self.r.check(job_path)['sha256'], + 'Process/job not owned by committed ledger entry') + process, job = self.r.json(process_path), self.r.json(job_path) + require(process['entry_id'] == entry_id and process['registration_sha256'] == job['registration_sha256'] == self.regsha and + process['job_sha256'] == self.r.check(job_path)['sha256'] and process['complete'] is True and + type(process['exit_code']) is int and process['exit_code'] == 0 and process['timed_out'] is False and + process['exit_observed_by_parent'] is True and job['kind'] == kind, 'Actual parent exit/job receipt differs') + for path, digest in job['input_sha256'].items(): + require(path in incoming and self.r.check(path)['sha256'] == digest, 'Actual consumed job input differs') + for label in ('stdout', 'stderr'): + path = self.exp / f'logs/{entry_id}.{label}.log' + require(str(path) in outgoing and self.r.check(path)['sha256'] == process[label + '_sha256'], 'Parent log differs') + if kind in ('brain', 'update'): + require(job['receipt_file'] in outgoing and job['receipt_file'] + '.attempt.json' in outgoing, + 'Worker/attempt receipt not ledger owned') + worker = self.r.json(job['receipt_file']) + closure = dict(job['input_sha256']) + closure[str(job_path)] = self.r.check(job_path)['sha256'] + # The AP worker includes its own exact script source in its closure. + script = process['argv'][2] + closure[script] = self.r.check(script)['sha256'] + for value in job['sources'].values(): + require(self.r.check(value['path'])['sha256'] == value['sha256'], 'Worker source differs') + closure[value['path']] = value['sha256'] + require(worker['schema'] == 'phase4ap-isolated-worker-receipt-v1' and worker['complete'] is True and + worker['job_id'] == entry_id and worker['kind'] == kind and worker['registration_sha256'] == self.regsha and + worker['job_sha256'] == self.r.check(job_path)['sha256'] and worker['verified_input_sha256'] == closure and + worker['attempt_sha256'] == self.r.check(job['receipt_file'] + '.attempt.json')['sha256'], 'Worker/attempt binding differs') + declared = {str(Path(p).relative_to(job['output_dir'])): row['sha256'] for p, row in outgoing.items() + if Path(p).is_relative_to(job['output_dir'])} + require(worker['output_files_sha256'] == declared, 'Worker output coverage differs') + self.tail = self.r.check(entry_path)['sha256'] + return dict(job=job, outgoing=outgoing, process_sha256=self.r.check(process_path)['sha256'], process_path=str(process_path)) + + def finish(self): + require(self.index == len(self.expected) and self.tail == self.final['head_sha256'], 'Missing process or terminal ledger head differs') + + +def audit_records(reader, exp, reg, regsha, plan, evidence, *, template, inputs, populations): + """Bind all 33,920 tasks and actual update chains; retain only small indexes.""" + sample = evidence.process('input-generation', 'sample') + sampler = reader.json(Path(exp) / 'receipts/input-generation.json') + require(sampler['complete'] is True and sampler['registration_sha256'] == regsha and + str(Path(exp) / 'receipts/input-generation.json') in sample['outgoing'], 'Sampler receipt not ledger owned') + require(sampler['descriptors'] == len(inputs['descriptors']) == 4080 and set(sampler['inputs']) == set(inputs['descriptors']) and + sampler['input_inventory_sha256'] == reader.check(Path(exp) / 'protocol/input-inventory.json')['sha256'], 'Actual sampler inventory incomplete') + previous, probes, arms, record_hashes = {}, {}, defaultdict(list), [] + for task in plan['tasks']: + key = task_key(task); result_path = Path(exp) / f'results/{key}.json' + record = reader.json(result_path) + require(record['schema'] == 'normal-task-result-v1' and record['complete'] is True and + record['registration_sha256'] == regsha and canonical(record['task']) == canonical(task), 'Actual result/task differs') + branch = task['block_id'], task['arm'] + overlay = previous.get(branch, reg['initial_overlay']) + require(record['overlay_before'] == overlay, 'Delta ancestry changed or probe fed back') + brain = evidence.process(key + '--brain', 'brain') + audit_case(reader, exp, task, brain['job'], template=template, inputs=inputs, sampler=sampler, + populations=populations, population_file=reg['population_file'], output_cap=reg['budget']['output_cap_bytes']) + job = brain['job']; output = Path(exp) / 'runs' / key + require(record['brain_process_receipt_path'] == brain['process_path'] and + record['checkpoint_dir'] == job['output_dir'] == str(output) and + all(job[k] == overlay[k] for k in ('overlay_file', 'overlay_manifest')) and + job['diagnostic_mode'] == 'light', 'Brain result/overlay path differs') + manifest = reader.json(output / 'checkpoint-manifest.json') + require(record['checkpoint_manifest_sha256'] == record['brain_audit']['checkpoint_manifest_sha256'] == + reader.check(output / 'checkpoint-manifest.json')['sha256'] and + record['brain_audit']['task_id'] == task['task_id'], 'Checkpoint binding differs') + for name, digest in manifest['files_sha256'].items(): + require(reader.check(output / name)['sha256'] == digest, 'Checkpoint payload differs') + record_hashes.append([task['task_id'], reader.check(result_path)['sha256']]) + if task['kind'] == 'probe': + require(str(result_path) in brain['outgoing'], 'Probe result not ledger owned') + probes[(task['block_id'], task['arm'], task['checkpoint'], task['recipe']['cue'])] = record + continue + update = evidence.process(key + '--update', 'update') + cpu = evidence.process(key + '--verify', 'cpu_verify') + require(record['update_process_receipt_path'] == update['process_path'] and record['cpu_process_receipt_path'] == cpu['process_path'] and + str(result_path) in cpu['outgoing'], 'Training result/parent process ownership differs') + authority = reader.json(record['epoch_authority']['path']) + require(reader.check(record['epoch_authority']['path'])['sha256'] == record['epoch_authority']['sha256'] == + update['job']['authority_sha256'] == cpu['job']['authority_sha256'] and authority['epoch_id'] == task['task_id'] and + authority['checkpoint_dir'] == str(output) and authority['checkpoint_manifest_sha256'] == record['checkpoint_manifest_sha256'], + 'Original E authority differs') + command = dict(steps=1200, compartment=task['compartment_verbatim'], us_tick=task['recipe']['us']['tick'], + d_command=task['recipe']['D_command'], frozen=not task['recipe']['update_enabled'], trace_backend='torch') + require(authority['command'] == command, 'Original E command changed') + valid = reader.json(record['cpu_validation']['path']) + after = dict(overlay_file=str(Path(exp) / 'updates' / key / 'delta-after.npz'), + overlay_manifest=str(Path(exp) / 'updates' / key / 'overlay-manifest.json')) + require(record['update_dir'] == update['job']['output_dir'] == cpu['job']['update_dir'] == str(Path(exp) / 'updates' / key) and + record['cpu_validation']['path'] == cpu['job']['receipt_file'] and record['cpu_validation']['path'] in cpu['outgoing'] and + record['epoch_authority']['path'] == update['job']['authority_file'] == cpu['job']['authority_file'] and + authority['files']['state'] == dict(path=str(output / 'state.npz'), sha256=reader.check(output / 'state.npz')['sha256']) and + authority['files']['overlay_before'] == dict(path=overlay['overlay_file'], sha256=reader.check(overlay['overlay_file'])['sha256']) and + authority['files']['overlay_manifest_before'] == dict(path=overlay['overlay_manifest'], sha256=reader.check(overlay['overlay_manifest'])['sha256']) and + record['after_overlay'] == after and reader.check(record['cpu_validation']['path'])['sha256'] == record['cpu_validation']['sha256'] and + valid['complete'] is True and valid['frozen_U_verify_receipt_recomputed'] is True and valid['actual_sealed_events_used'] is True and + valid['epoch_authority_sha256'] == record['epoch_authority']['sha256'] and + valid['after_overlay_sha256'] == reader.check(after['overlay_file'])['sha256'], 'Independent CPU update validation differs') + if task['arm'] in ('d_zero', 'frozen', 'cs_only'): + require(not reader.arrays(after['overlay_file'])['plastic_delta_mv'].view(np.uint32).any(), 'Zero control contains nonzero delta') + previous[branch] = after + arms[branch].append(dict(result=record_hashes[-1], brain=brain['process_sha256'], + update=update['process_sha256'], cpu=cpu['process_sha256'], requested=task['recipe']['us']['requested'])) + evidence.finish() + require(len(probes) == 3920 and len(arms) == 280 and len(record_hashes) == 33920, 'Incomplete actual task inventory') + return probes, arms, sha(record_hashes) + + +def build_registry(trial, plan, *, nodes, populations, population_binding_sha, regsha, input_hashes): + membership = M.validate_populations(nodes, populations) + blocks = {} + inputs = {(r['block_id'], r['cue']): r for r in plan['evaluation_inputs']} + for block in trial['plan']['blocks']: + identity = block['block_id'] + source_rows = [r for r in trial['seed_manifest']['independent_streams'] + if r['key'][0] == identity and r['key'][1] == 'normal' and r['split'] in ('training', 'design')] + sources = [dict(stream_id=P._FROZEN['trial_plan']._stream_id(P.MASTER, r['split'], r['key']), + seed=r['seed'], split=r['split']) for r in source_rows] + blocks[identity] = dict(mode=block['mode'], dan_type=block['dan_type'], compartment_verbatim=block['compartment_verbatim'], + conditioned_cue=ODORS[block['initial_reinforced_cue']], replicate=block['replicate'], training_sources=sources, + evaluation_sources={ODORS[c]: {k: inputs[(identity, c)][k] for k in ('stream_id', 'seed', 'split')} for c in ODORS}, + evaluation_input_sha256={ODORS[c]: input_hashes[(identity, c)] for c in ODORS}) + registry = dict(schema=G.REGISTRY_SCHEMA, protocol_sha256=regsha, population_binding_sha256=population_binding_sha, + nodes_raw_sha256=M._hash(nodes), population_membership_sha256=membership['membership_sha256'], blocks=blocks) + G.validate_registry(registry, expected_sha256=G.descriptor_sha256(registry)) + return registry + + +def compact_metric(metric): + return dict(rawS=metric['rawS'], rawB=metric['rawB'], separation=metric['separation'], + cues={cue: dict(state_pre=summary['pre_state_raw_sha256'], state_post=summary['post_state_raw_sha256'], + populations={name: {k: v for k, v in group.items() if not isinstance(v, list)} + for name, group in summary['populations'].items()}) for cue, summary in metric['cues'].items()}) + + +def scientific_reports(probes, arms, *, registry, authority, nodes, populations, load_state): + """One cell's terminal vectors at a time; curve outputs contain no raw vectors.""" + curves, compact_reports = [], [] + + def cells(): + for mode in M.MODES: + for dan in M.DAN_TYPES: + for conditioned in M.CUES: + rows = [] + selected = [(key, b) for key, b in registry['blocks'].items() + if (b['mode'], b['dan_type'], b['conditioned_cue']) == (mode, dan, conditioned)] + for identity, block in selected: + arm_values = {} + for name, legacy_name in ARM_NAMES.items(): + pre = {ODORS[c]: load_state(probes[(identity, name, 0, c)]) for c in ODORS} + terminal = None + for point in POINTS: + post = pre if point == 0 else {ODORS[c]: load_state(probes[(identity, name, point, c)]) for c in ODORS} + metric = M.arm_metrics(pre, post, conditioned_cue=conditioned, nodes=nodes, populations=populations) + curves.append(dict(block_id=identity, arm=name, checkpoint=point, metrics=compact_metric(metric))) + if point == 50: + terminal = metric + history = arms[(identity, name)] + require(len(history) == (150 if name == 'unpaired' else 100), 'Missing actual training/control epochs') + execution = dict(complete=True, recording_verified=True, receipt_sha256=sha(history), assay='normal', + epoch_duration_ms=120., cs_duration_ms=100., us_tick=900, completed_epochs=len(history), + reinforcement_opportunities=sum(r['requested'] for r in history), terminal_pairings=50) + arm_values[legacy_name] = dict(metrics=terminal, execution=execution) + rows.append(dict(replicate_id=identity, replicate_unit='independent_training', + training_stream_ids=[x['stream_id'] for x in block['training_sources']], + evaluation_stream_ids=[block['evaluation_sources'][c]['stream_id'] for c in M.CUES], + assay='normal', mode=mode, dan_type=dan, conditioned_cue=conditioned, arms=arm_values)) + report = G.evaluate_acquisition(rows, assay='normal', mode=mode, dan_type=dan, + conditioned_cue=conditioned, registry=registry, authority=authority) + compact_reports.append({k: v for k, v in report.items() + if k not in ('acquisition_metric_inputs', 'frozen_acquisition_result')}) + yield report + + cohort = G.summarize_cohort(cells(), registry=registry, authority=authority, required_assays=('normal',)) + require(cohort['complete_inventory'], 'Missing normal cells') + return dict(cells=compact_reports, cohort=cohort, curves=curves) + + +def legacy_audit(reports, declared_mapping): + """G.H_MAPPING source layout; call the unchanged M scalar twice per group.""" + require(declared_mapping == G.H_MAPPING, 'Explicit original registered H mapping required') + lookup = {(r['mode'], r['dan_type'], r['conditioned_cue']): r for r in reports} + require(len(reports) == len(lookup) == 8, 'Exact eight-cell legacy input required') + groups = [] + for mode in M.MODES: + for cue in M.CUES: + selected = {d: lookup[(mode, d, cue)] for d in M.DAN_TYPES} + outcomes = {} + for source in M.DAN_TYPES: + gains = {label: [r['population_separation_gains']['MBON'] for r in selected[dan]['replicates']] + for dan, label in (('PAM15', 'appetitive-conditioning'), ('PPL103', 'aversive-conditioning'))} + for name, label in (('D=0', 'd-zero-control'), ('random reward', 'random-reward-control'), ('frozen', 'frozen-brain-control')): + controls = selected[source]['legacy_control_MBON_gains'][name] + require(set(controls) == {r['replicate_id'] for r in selected[source]['replicates']}, 'Legacy replicate control source differs') + gains[label] = [controls[r['replicate_id']] for r in selected[source]['replicates']] + outcomes[source] = M.legacy_h_mean_control_audit(gains) + groups.append(dict(mode=mode, conditioned_cue=cue, control_sources=outcomes, + both_pass=all(v['predicate_pass'] for v in outcomes.values()))) + return dict(schema=G.SCHEMA, status='CONDITIONAL-AUDIT', declared_mapping=declared_mapping, groups=groups, + both_pass=all(g['both_pass'] for g in groups), original_H_data_layout_reproduced=False, + new_stricter_control_source_mapping=True, **G._scope()) + + +def analyze(exp, registration_sha256, *, verified_inventory, final_ledger): + exp, reader = Path(exp), Reader(verified_inventory) + regpath = exp / 'protocol/preregistered.json' + require(reader.check(regpath)['sha256'] == registration_sha256, 'Registration authority differs') + reg = reader.json(regpath) + require(reg['schema'] == 'normal-conditioning-registration-v1' and reg['scientific_scope'] == 'normal-acquisition-only' and + reg['subsequent_assays_authorized'] is False, 'Wrong scientific authority') + for collection in ('source_sha256', 'input_sha256', 'prepared_sha256'): + for path, digest in reg[collection].items(): + require(reader.check(path)['sha256'] == digest, 'Registered dependency differs') + plan = reader.json(exp / 'protocol/normal-plan.json') + full = reader.json(reg['original_full_trial_plan']['path']) + require(reader.check(reg['original_full_trial_plan']['path'])['sha256'] == reg['original_full_trial_plan']['sha256'] == P.AO_TRIAL_FILE_SHA256, + 'Original AO full plan differs') + evaluation = P.expand_pinned_evaluation(full) + P.validate_normal_plan(plan, full, evaluation) + gates = reader.json(exp / 'protocol/gates.json') + require(gates['H_mapping'] == G.H_MAPPING and gates['all_eight_cells_required'] is True and + gates['same_replicate_conjunction_at_least'] == 3 and gates['training_replicates'] == 5 and gates['terminal_checkpoint'] == 50 and + gates['intermediate_checkpoints_cannot_rescue_terminal'] is True, 'Registered scientific policy differs') + template = reader.json(exp / 'protocol/case-template.json') + nodes_path = Path(template['graph_dir']) / 'nodes.npy' + nodes = np.load(io.BytesIO(reader.raw(nodes_path)), allow_pickle=False) + require(nodes.dtype == np.int32 and len(nodes) == 166700, 'Full original graph nodes required') + require('readout_population_file' in reg, 'Registered full legacy/non-MBON readout populations missing') + populations = reader.arrays(reg['readout_population_file']) + original = reader.arrays(reg['population_file']) + for name in ('MBON', 'MBON01', 'MBON12'): + require(np.array_equal(original[name], populations[name]), 'Original anatomical population changed') + M.validate_populations(nodes, populations) + require('readout_population_binding_file' in reg, 'Registered anatomical binding file missing') + population_binding_sha = reader.check(reg['readout_population_binding_file'])['sha256'] + evidence = Evidence(reader, exp, registration_sha256, expected_entries(plan), final_ledger) + inputs = reader.json(exp / 'protocol/input-inventory.json') + probes, arms, inventory_sha = audit_records(reader, exp, reg, registration_sha256, plan, evidence, + template=template, inputs=inputs, populations=original) + input_hashes, seen_receipts = {}, set() + + def load_state(record): + task = record['task']; key = task_key(task) + receipt = record['brain_process_receipt_path'] + require(receipt not in seen_receipts, 'Probe execution reused across snapshots') + seen_receipts.add(receipt) + csid = inputs['task_inputs'][task['task_id']] + recipe = task['recipe'] + expected_descriptor = dict(mode=task['mode'], cue=recipe['cue'], seed=recipe['input']['seed'], steps=1200, + cs_start_tick=0, cs_stop_tick_exclusive=1000, source_stream_id=recipe['input']['stream_id'], + table_id=task['mode'] + '--' + recipe['cue']) + require(csid == sha(expected_descriptor) and inputs['descriptors'][csid] == expected_descriptor, + 'Evaluation seed/stream/clock mapping differs from original plan') + source = exp / f'inputs/generated/{csid}.npz' + generated = reader.json(exp / f'inputs/generated/{csid}.json') + require(generated['descriptor'] == inputs['descriptors'][csid] and generated['requests_sha256'] == reader.check(source)['sha256'], + 'Actual evaluation input descriptor differs') + case = exp / 'cases' / key + binding = reader.json(case / 'input-binding.json') + require(binding['CS_source'] == dict(path=str(source), sha256=reader.check(source)['sha256']) and + binding['CS_descriptor_id'] == csid and binding['task_id'] == task['task_id'], 'Probe shared input binding differs') + requests, cs, us = (reader.arrays(path) for path in (case / 'requests.npz', source, case / 'us_requests.npz')) + require(set(requests) == set(cs) == set(us) == {'tick', 'neuron_idx'} and + all(not len(v) for v in us.values()) and all(np.array_equal(requests[k], cs[k]) for k in requests), 'Probe includes US or changed CS') + state = reader.arrays(Path(record['checkpoint_dir']) / 'state.npz') + actual = M.validate_state(state, nodes=nodes, steps=1200) + require(actual['hashes'] == record['brain_audit']['state_raw_sha256'], 'Actual probe raw arrays differ') + return state + + for item in plan['evaluation_inputs']: + record = probes[(item['block_id'], 'paired', 0, item['cue'])] + csid = inputs['task_inputs'][record['task']['task_id']] + input_hashes[(item['block_id'], item['cue'])] = reader.check(exp / f'inputs/generated/{csid}.npz')['sha256'] + registry = build_registry(full, plan, nodes=nodes, populations=populations, population_binding_sha=population_binding_sha, + regsha=registration_sha256, input_hashes=input_hashes) + authority = dict(schema=G.AUTHORITY_SCHEMA, registry_sha256=G.descriptor_sha256(registry), + verification_receipt_sha256=sha(final_ledger), execution_inventory_sha256=inventory_sha, + population_binding_sha256=population_binding_sha) + result = scientific_reports(probes, arms, registry=registry, authority=authority, nodes=nodes, populations=populations, load_state=load_state) + require(len(seen_receipts) == 3920, 'Not every independently executed probe was analyzed') + result['legacy_H'] = legacy_audit(result['cells'], gates['H_mapping']) + return dict(schema='normal-scientific-analysis-v1', status='CONDITIONAL-PENDING-OUTER-INTEGRITY', + registration_sha256=registration_sha256, normal_plan_sha256=plan['payload_sha256'], registry=registry, authority=authority, + analysis=result, analyzed_probe_calls=3920, curve_points=1960, consumed_file_count=len(reader.seen), + subsequent_assays_status='NOT-RUN', subsequent_assays_authorized=False, candidate_generated=False, + learning_pass_claimed=False, scientific_learning_pass_claimed=False, data_backed_claimed=False, + external_full_tree_anatomy_historical_integrity_verified_here=False, + retained_raw_evidence='original state and receipt files; terminal raw metric vectors recomputable, not duplicated in report') diff --git a/models/male-cns-lif/normal-conditioning-v1/normal_plan.py b/models/male-cns-lif/normal-conditioning-v1/normal_plan.py new file mode 100644 index 0000000..02c4cae --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/normal_plan.py @@ -0,0 +1,199 @@ +"""Pure normal-only adapter of the AO-pinned, master-20260930 inventory. + +No sampling, brain execution, file writes, biological identity certification, +or execution authorization. Import reads and SHA-checks six frozen sources. +The trial argument must reproduce the ORIGINAL AO saved-file bytes under its +documented JSON encoding. The evaluation argument is a new exact expansion of +the old pinned policy, NOT an independently archived AO evaluation artifact. +Physical inputs, actual DAN mapping, checkpoints, model and process receipts +remain the registering executor's responsibility. No scientific gate changes. +""" +from __future__ import annotations + +import builtins +from collections import Counter +import hashlib +import json +from pathlib import Path +import types + + +SCHEMA = "normal-conditioning-plan-v1" +MASTER = 20260930 +AO_TRIAL_FILE_SHA256 = "a7bda850ee87df4d9f647994e1a1a4074dfea5793184edb1822ab3d8c02e8216" +AO_REGISTRATION_SHA256 = "9e2ed05879ce279da7dc0b4cac09a44d67665c230b56db94e2ca39c645bdc250" +AP_REGISTRATION_SHA256 = "02a03b536a9f14196ad4b92fa9565b592732c6c863e72eeaee89b3b0ea8ed37d" +AO_TRIAL_PATH = "/data/digital-drosophila/experiments/phase4ao-conditioning-interface/protocol/full-trial-plan.json" +FROZEN_DIRECTORY = Path(__file__).resolve().parent.parent / "olfactory-conditioning-lab-v1" +SOURCE_PINS = { + "seed_registry.py": "0da033b132095ec3087b5d9f64e53c2e391fa50ab0b7653b31296eeb84799b7e", + "trial_plan.py": "6fed3f494d075339a7e5b001d43b2598640b189e038952fbd4d41200bacac587", + "evaluation_plan.py": "991c813ff3322b230947f302b12b04ec382c3b632817f94b698101e4b17efa7d", + "evaluation_policy.py": "21331a9ff3567dc6f6399e16bb91ef486d74231f5db51799beb6dd1bf71d022b", + "PROTOCOL_DRAFT.md": "8ebecb1f3d012ddc29a1cb727269dd277629c53565e09978bf9dbf7140cb8508", + "EVALUATION_POLICY_DRAFT.md": "a84170a61564715a1532f20df4c6a330f0f9725c383db48b8f3e1a4b5fba0bc8", +} + + +def _require(condition, message): + if not condition: + raise ValueError(message) + + +def _canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False) + + +def _sha(value): + return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() + + +def _load_frozen(): + """Private module graph; never change sys.path or ordinary module caches.""" + sources = {} + for filename, expected in SOURCE_PINS.items(): + raw = (FROZEN_DIRECTORY / filename).read_bytes() + _require(hashlib.sha256(raw).hexdigest() == expected, + "Frozen source pin mismatch: " + filename) + sources[filename] = raw + modules = {} + ordinary_import = builtins.__import__ + + def private_import(name, globals=None, locals=None, fromlist=(), level=0): + if level == 0 and name in modules: + return modules[name] + return ordinary_import(name, globals, locals, fromlist, level) + + for name in ("seed_registry", "trial_plan", "evaluation_plan", "evaluation_policy"): + module = types.ModuleType("_normal_frozen_" + name) + module.__file__ = str(FROZEN_DIRECTORY / (name + ".py")) + module.__dict__["__builtins__"] = dict(vars(builtins), __import__=private_import) + exec(compile(sources[name + ".py"], module.__file__, "exec"), module.__dict__) + modules[name] = module + return modules + + +_FROZEN = _load_frozen() + + +def _validate_original_trial(trial): + _require(type(trial) is dict and set(trial) == + {"plan", "seed_manifest", "budget", "payload_sha256"}, "Full original trial payload required") + _require(type(trial["plan"]["master"]) is int and trial["plan"]["master"] == MASTER, + "Original AO master is fixed; no seed selection") + # This is the exact LF serialization used by AO engineering_prepare.save. + # Its registered file pin is stronger than reconstructing with a new seed. + encoded = json.dumps(trial, sort_keys=True, indent=2, ensure_ascii=False, + allow_nan=False) + "\n" + _require(hashlib.sha256(encoded.encode("utf-8")).hexdigest() == AO_TRIAL_FILE_SHA256, + "Trial does not reproduce the AO registered full-trial-plan file") + _require(_sha({k: v for k, v in trial.items() if k != "payload_sha256"}) == + trial["payload_sha256"], "Original trial self-hash mismatch") + + +def _expand_validated_trial(trial): + # Private frozen assembly is safe AFTER the exact archived-file pin above; + # avoid the old public APIs' redundant 68,000-epoch trial reconstruction. + policy = _FROZEN["evaluation_policy"]._assemble(trial) + return _FROZEN["evaluation_plan"]._assemble(trial, policy["config"]) + + +def expand_pinned_evaluation(full_trial_payload): + """Expand the fixed historical policy; no new choices or sampled inputs.""" + _validate_original_trial(full_trial_payload) + return _expand_validated_trial(full_trial_payload) + + +def build_normal_plan(full_trial_payload, full_evaluation_payload): + """Return 33,920 ordered tasks; caller supplies both complete authorities.""" + trial, evaluation = full_trial_payload, full_evaluation_payload + _validate_original_trial(trial) + expected_evaluation = _expand_validated_trial(trial) + _require(_canonical(evaluation) == _canonical(expected_evaluation), + "Evaluation must exactly expand the AO-pinned evaluation policy") + snapshots = {(s["block_id"], s["arm"], s["completed_epochs"]): s + for s in evaluation["snapshots"] if s["assay"] == "normal"} + probes = {(p["snapshot_id"], p["cue"]): p + for p in evaluation["probes"] if p["assay"] == "normal"} + tasks, inputs, blocks = [], [], [] + for block in trial["plan"]["blocks"]: + common = dict(block_id=block["block_id"], mode=block["mode"], dan_type=block["dan_type"], + compartment_verbatim=block["compartment_verbatim"], + reinforced_cue=block["initial_reinforced_cue"], replicate=block["replicate"], + assay="normal", steps=1200, artifact_validation_required=True) + blocks.append(dict(common, DAN_target_binding=block["DAN_target_binding"])) + for cue in ("A", "B"): + sid = block["block_id"] + "/normal/paired/checkpoint-00" + inputs.append(dict(block_id=block["block_id"], cue=cue, **probes[(sid, cue)]["input"])) + suite = next(s for s in block["suites"] if s["assay"] == "normal") + for arm in suite["arms"]: + name = arm["arm"] + delta = dict(arm["delta_start"]) + + def append_probes(completed): + snapshot = snapshots.get((block["block_id"], name, completed)) + if snapshot is None: + return + _require(snapshot["delta_source"] == delta, "Snapshot delta dependency differs") + for cue in ("A", "B"): + recipe = probes[(snapshot["snapshot_id"], cue)] + tasks.append(dict(common, kind="probe", arm=name, task_id=recipe["probe_id"], + probe_id=recipe["probe_id"], snapshot_id=snapshot["snapshot_id"], + checkpoint=snapshot["checkpoint"], completed_epochs=completed, + evaluation_input_id=recipe["input"]["input_id"], delta_dependency=dict(delta), + delta_output_id=None, neural_state_policy="common_frozen_reset_independent_probe", + carry_between_epochs=[], original_E_required=False, cpu_validation_required=False, + feedback_to_training_or_idle=False, recipe=recipe, snapshot_binding=snapshot, + source_binding=dict(payload_sha256=evaluation["payload_sha256"], + record_id=recipe["probe_id"], record_sha256=_sha(recipe)))) + + append_probes(0) + for completed, epoch in enumerate(arm["epochs"], 1): + tasks.append(dict(common, kind="train", arm=name, task_id=epoch["epoch_id"], + epoch_id=epoch["epoch_id"], epoch_index=epoch["epoch_index"], + delta_dependency=dict(delta), delta_output_id=epoch["epoch_id"] + "/delta-after", + neural_state_policy=epoch["initial_neural_state"], + carry_between_epochs=list(arm["carry_between_epochs"]), + original_E_required=True, cpu_validation_required=True, + feedback_to_training_or_idle=True, recipe=epoch, + source_binding=dict(payload_sha256=trial["payload_sha256"], + record_id=epoch["epoch_id"], record_sha256=_sha(epoch)))) + delta = dict(kind="epoch_delta_after", source=epoch["epoch_id"] + "/delta-after") + append_probes(completed) + counts = Counter(row["kind"] for row in tasks) + _require(counts == {"train": 30000, "probe": 3920} and len(inputs) == 80 and len(blocks) == 40, + "Normal inventory changed") + _require(len({t["task_id"] for t in tasks}) == len(tasks), "Task IDs must be unique") + result = dict(schema=SCHEMA, master=MASTER, assay="normal", blocks=blocks, + tasks=tasks, evaluation_inputs=inputs, + inventory=dict(blocks=40, training_replicates_per_cell=5, arms_per_block=7, + nominal_reinforcements=50, checkpoints=[0, 1, 2, 5, 10, 20, 50], + train_calls=30000, probe_calls=3920, brain_calls=33920, + original_E_calls=30000, training_cpu_validation_calls=30000, + brain_steps=40704000, biological_seconds=4070.4, evaluation_inputs=80, + baseline_execution="execute_each_snapshot", baseline_output_deduplication=False), + provenance=dict(trial_payload_sha256=trial["payload_sha256"], + seed_manifest_sha256=trial["seed_manifest"]["payload_sha256"], + evaluation_payload_sha256=evaluation["payload_sha256"], + evaluation_config_sha256=evaluation["config_sha256"], + ao_trial_file_sha256=AO_TRIAL_FILE_SHA256, ao_trial_path=AO_TRIAL_PATH, + ao_registration_sha256=AO_REGISTRATION_SHA256, ap_registration_sha256=AP_REGISTRATION_SHA256, + frozen_source_sha256=SOURCE_PINS, + evaluation_authority="exact expansion of AO-pinned policy; not a historical saved evaluation artifact", + external_inputs_mapping_processes_and_physical_source_files_verified=False), + execution_authorized=False, inputs_generated=False, new_seed_streams=0, + learning_pass_claimed=False, subsequent_assays_included=False) + result["payload_sha256"] = _sha(result) + # No returned nested structure aliases either caller argument or module pins. + return json.loads(_canonical(result)) + + +def validate_normal_plan(payload, full_trial_payload, full_evaluation_payload): + """Fail closed even if a modified plan has been given a fresh self-hash.""" + _require(type(payload) is dict and "payload_sha256" in payload, "Normal payload required") + _require(_sha({k: v for k, v in payload.items() if k != "payload_sha256"}) == + payload["payload_sha256"], "Normal plan self-hash mismatch") + expected = build_normal_plan(full_trial_payload, full_evaluation_payload) + _require(_canonical(payload) == _canonical(expected), "Normal plan exact reconstruction mismatch") + return True diff --git a/models/male-cns-lif/normal-conditioning-v1/normal_prepare.py b/models/male-cns-lif/normal-conditioning-v1/normal_prepare.py new file mode 100644 index 0000000..08ec612 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/normal_prepare.py @@ -0,0 +1,198 @@ +"""Register only normal acquisition; never sample inputs or execute a brain. + +Prior sealed files stay read-only. Complete historical integrity is checked at +the phase boundary; each trial rehashes its actual runtime dependency closure, +not every historical engineering result. No source or budget changes after +registration. Outcome files have no invented pre-outcome hashes. +""" +from __future__ import annotations +import argparse +import copy +from datetime import datetime, timezone, timedelta +import shutil +import subprocess +import sys +from normal_io import * +import normal_plan as PLAN + +AQ_EXP = ROOT/'experiments/phase4aq-conditioning-execution' +AP_EXP = ROOT/'experiments/phase4ap-conditioning-interface-v2' +AO_EXP = ROOT/'experiments/phase4ao-conditioning-interface' +AQ_ANCHORS = { + 'protocol/preregistered.json':'dad0ba33ded8c6725662c419525ef4185a6ff169042a402c18d0bd760dad7358', + 'manifest.json':'2dc0feca75f718f2ee0d7c9240da503221823573fd25f1344d2fd97d97eb2db7', + 'audit/final-status.json':'dcbb58f296bd0d51e3c8757f0c45dd5585b3a76f8c2a0b5881f4c14076721882', +} +H_ANCHORS = { + 'audit/phase4h.py':'28bb2ab7c4c95af3d95b0bf0bc45a5a9bea791996c59ce296ec66b71e01b1695', + 'mappings/mbon-to-downstream-actual-edges.parquet':'76bac27c50cec41c48fd6414bcef5a6e270753ac4062b92537f38226018e9278', + 'manifests/checksums.sha256':'1ea0394fd0477bb40d656ed0854108964f3809d633b5561bed9de43cda630ba1', + 'manifests/manifest.json':'d1f187f164f7775a1a02fff259c8f5e31ae035929d808c3876cfc0013d19c73c', +} + +def readout_populations(population_file): + import numpy as np + import pyarrow.parquet as pq + h=ROOT/'experiments/phase4h-dopamine-plasticity' + verify({str(h/p):s for p,s in H_ANCHORS.items()}) + rows=pq.read_table(h/'mappings/mbon-to-downstream-actual-edges.parquet',columns=['target_neuron_idx'])['target_neuron_idx'].to_numpy() + require(rows.dtype==np.int32 and len(rows)==39279 and np.all((rows>=0)&(rows<166700)), 'Original H endpoint schema differs') + old=npz(population_file) + result={k:old[k] for k in ('MBON','MBON01','MBON12')} + result['legacyDownstream']=np.unique(rows).astype(np.int32) + result['nonMBONDownstream']=np.setdiff1d(result['legacyDownstream'],result['MBON']) + require(not len(np.setdiff1d(result['MBON'],result['legacyDownstream'])),'H downstream no longer includes all MBON') + hashes={k:hashlib.sha256(v.tobytes()).hexdigest() for k,v in result.items()} + require(hashes['legacyDownstream']=='ce9edbb56854be8f6891bfb83559dd7529c70acac6867864947f32f933175dbc' and + hashes['nonMBONDownstream']=='01e3fbdde8bf2558665f142cdf546d8b89b4898427b90e2f4368f21f4e241a7f', + 'Actual H readout membership changed') + return result,dict(source_sha256={str(h/p):s for p,s in H_ANCHORS.items()}, + original_populations=pin(population_file),definition='unique actual H MBON-edge target_neuron_idx, int32; no plastic/active subset', + nonMBON_definition='exact set difference legacyDownstream minus full anatomical MBON', + counts={k:len(v) for k,v in result.items()},raw_membership_sha256=hashes, + historical_current_model_source_used=False,original_sealed_audit_source_used=True) + +def historical_integrity(): + verify({str(AQ_EXP/p):s for p,s in AQ_ANCHORS.items()}) + ar = read(AQ_EXP/'protocol/preregistered.json') + verify(ar['source_sha256']) + # This function is executed in a separate preparation/integrity CPU process. + for p in (AQ, AP, LAB): sys.path.insert(0,str(p)) + old = load_source('_normal_previous_integrity',AQ/'prepare_execution.py',ar['source_sha256'][str(AQ/'prepare_execution.py')]) + prior = old.integrity() + helper = load_source('_normal_AQ_seal_reader', MODEL.parent/'pn-recruitment-audit-v1/publication.py', + '7a79c499098b74163e7bb16adabcbc54e7069dafa204d0b99c7f4c03d79f8016') + sealed = helper.verify_seal(AQ_EXP) + require(read(AQ_EXP/'audit/final-status.json')['status'] == + 'PASS-EXACT-UPDATED-BOUNDARY-ENGINEERING-NOT-LEARNING','AQ gate differs') + return dict(complete=True,prior=prior,AQ=sealed,old_experiments_reexecuted=False, + frozen_five_reexecuted=False,scope='actual historical seal bytes reverified, preserved numerical evidence; not a new numerical trial') + +def resource_policy(): + """Engineering ceiling, not a measured all-condition forecast or stop-for-success.""" + return dict(root=str(EXP),output_cap_bytes=256*1024**3,minimum_free_reserve_bytes=100*1024**3, + child_timeout_seconds=1800,workflow_timeout_seconds=21*86400,max_uniform_draws=10_000_000, + next_child_headroom_bytes=4*1024**3, + execution='serial_single_4090',no_automatic_retry=True, + forecast_scope='rough planning arithmetic from one AQ warm training cycle and one probe; not condition-complete measurement', + AQ_measured_training_cycle_seconds=24.771224757998425,AQ_measured_probe_seconds=10.394962482001574, + AQ_linear_wall_scenario_seconds=30000*24.771224757998425+3920*10.394962482001574, + uncertainties=['startup and materialization','event-dependent output size','all control/DAN/odor activity', + 'ledger growth','shared service contention','final full integrity audit'], + cap_or_timeout_policy='preserve all outputs, stop incomplete; no shortening epochs or selecting favorable outcomes') + +def prepare(): + require(sys.dont_write_bytecode and unlinked(EXP).is_dir() and not any(EXP.iterdir()), 'Exact new empty EXP and -B required') + files=sorted(MODEL.iterdir()) + require(files and all(p.is_file() and not p.is_symlink() and p.suffix in ('.py','.md') for p in files), 'Source-only model directory') + sources={str(p):digest(p) for p in files} + commit=subprocess.check_output(['git','-C',str(ROOT),'rev-parse','HEAD']).decode().strip() + for path,sha in sources.items(): + blob=subprocess.check_output(['git','-C',str(ROOT),'show',commit+':'+Path(path).relative_to(ROOT).as_posix()]) + require(hashlib.sha256(blob).hexdigest()==sha, 'Uncommitted source '+path) + budget=resource_policy() + budget['registered_utc']=datetime.now(timezone.utc).isoformat() + budget['not_after_utc']=(datetime.now(timezone.utc)+timedelta(seconds=budget['workflow_timeout_seconds'])).isoformat() + require(shutil.disk_usage(EXP).free >= budget['output_cap_bytes']+budget['minimum_free_reserve_bytes'], 'Insufficient reserved capacity') + for name in ('protocol/source-snapshot','inputs/static','jobs','cases','runs','updates','validation','results','receipts','logs','audit','regression'): + unlinked(EXP/name).mkdir(parents=True) + tests=subprocess.run([sys.executable,'-B',str(MODEL/'run_cpu_tests.py'),'--output',str(EXP/'audit/cpu-tests.json')], + capture_output=True,text=True,timeout=1800,env={**dict(__import__('os').environ),'CUDA_VISIBLE_DEVICES':''}) + save(EXP/'audit/cpu-tests.process.json',dict(exit_code=tests.returncode,stdout=tests.stdout,stderr=tests.stderr, + actual_parent_observation=True)) + require(tests.returncode==0 and read(EXP/'audit/cpu-tests.json')['pass_all'] is True, + 'CPU/source tests failed; no sampler or brain started') + # Old integrity readers may own a private R/C module graph. Keep them in an + # exec-isolated CPU process, never the table/registration interpreter. + integrity_command=[sys.executable,'-B',str(MODEL/'normal_prepare.py'),'integrity', + '--output',str(EXP/'regression/integrity-before.json')] + checked=subprocess.run(integrity_command,capture_output=True,text=True,timeout=1800, + env={**dict(__import__('os').environ),'CUDA_VISIBLE_DEVICES':''}) + save(EXP/'regression/integrity-before.process.json',dict(argv=integrity_command,exit_code=checked.returncode, + stdout=checked.stdout,stderr=checked.stderr,actual_parent_observation=True)) + require(checked.returncode==0,'Historical integrity child failed; preserve artifacts') + before=read(EXP/'regression/integrity-before.json');require(before['complete'] is True,'Integrity child incomplete') + aq=read(AQ_EXP/'protocol/preregistered.json') + full=read(PLAN.AO_TRIAL_PATH) + require(digest(PLAN.AO_TRIAL_PATH)==PLAN.AO_TRIAL_FILE_SHA256,'Original full plan pin differs') + evaluation=PLAN.expand_pinned_evaluation(full) + plan=PLAN.build_normal_plan(full,evaluation) + from normal_inputs import derive_rates, inventory_from_plan, PINS, dependencies + rate_tables,rate_audit=derive_rates() + inputs=inventory_from_plan(plan) + template_path=AP_EXP/'cases/D1--reference--epoch-001/spec.json' + authority_path=AP_EXP/'jobs/D1--reference--epoch-001.authority.json' + template=read(template_path); epoch=read(authority_path) + pinned={**aq['input_sha256'],**aq['source_sha256'],**aq['prepared_file_sha256']} + for path in (template_path,authority_path,AO_EXP/'inputs/static/populations.npz'): + require(str(path) in pinned and digest(path)==pinned[str(path)],'Original template/population not pinned') + for path in sources: + with Path(path).open('rb') as i,(EXP/'protocol/source-snapshot'/Path(path).name).open('xb') as o:shutil.copyfileobj(i,o) + save(EXP/'protocol/normal-plan.json',plan) + save(EXP/'protocol/evaluation-policy.json',evaluation['config']) + save(EXP/'protocol/input-inventory.json',inputs) + save(EXP/'protocol/case-template.json',template) + # Freeze scientific predicates before any new CS sample or outcome. + gate_sources={str(AQ/'learning_gates.py'):'8ae9bcb77fe14dbdda164f86ffdbfc1e2976be752766ef526777577223b4dd99', + str(LAB/'metrics.py'):'47319e21db3c8b030b1bafbf0703cd86fb588d23bb29436accbacdba73d08c3c'} + verify(gate_sources) + save(EXP/'protocol/gates.json',dict(source_sha256=gate_sources,all_eight_cells_required=True, + same_replicate_conjunction_at_least=3,training_replicates=5,terminal_checkpoint=50, + normal_failure_prevents_followups=True,intermediate_checkpoints_cannot_rescue_terminal=True, + downstream_policy='nonMBONDownstream-pre-outcome-operationalization-v1', + H_mapping='each-DAN-control-source-separately-both-pass-v1', + physiological_and_matched_energy_not_pooled=True,unknown_compartments_not_guessed=True)) + save(EXP/'inputs/static/rate-audit.json',rate_audit) + for key,table in rate_tables.items():save_npz(EXP/f'inputs/static/{key}.npz',table) + populations,pop_audit=readout_populations(AO_EXP/'inputs/static/populations.npz') + save_npz(EXP/'inputs/static/readout-populations.npz',populations) + save(EXP/'inputs/static/readout-population-binding.json',pop_audit) + # Actual consumed neural dependency closure, all inherited byte pins. + required=set() + for role in ('graph_dir','apl_bundle_dir'): + required.update(str(p) for p in Path(template[role]).iterdir() if p.suffix=='.npy') + required.add(template['coefficients_path']) + original_sources={p:s for p,s in epoch['frozen_inputs'].items() if p.startswith(str(ROOT/'models/male-cns-lif/') ) + and ('author-' in p or 'apl-timing-lab-v1/' in p)} + runtime={p:pinned[p] for p in required} + runtime.update(original_sources) + roles={k:v for k,v in epoch['files'].items() if k not in ('spec','requests','state','overlay_before','overlay_manifest_before')} + runtime.update({v['path']:v['sha256'] for v in roles.values()}) + save(EXP/'protocol/epoch-template.json',dict(schema=epoch['schema'],files=roles,frozen_inputs=runtime)) + static={str(LAB/n):s for n,s in {**PLAN.SOURCE_PINS,**PINS}.items()} + static.update({str(ROOT/p):s for p,s in rate_audit['original_source_sha256'].items()}) + external={**runtime,**static,**gate_sources,**pop_audit['source_sha256'],**{str(AQ_EXP/p):s for p,s in AQ_ANCHORS.items()}, + str(PLAN.AO_TRIAL_PATH):PLAN.AO_TRIAL_FILE_SHA256,str(AP/'isolated_workers.py'):AP_SHA, + str(authority_path):pinned[str(authority_path)],str(template_path):pinned[str(template_path)], + str(AO_EXP/'inputs/static/populations.npz'):pinned[str(AO_EXP/'inputs/static/populations.npz')], + str(AQ/'validate_update_cpu.py'):aq['source_sha256'][str(AQ/'validate_update_cpu.py')]} + for name in ('conditioning_epoch.py','conditioning_preflight_compare.py','engineering_compare.py'): + path=str(LAB/name);require(path in pinned,'Missing original verifier source pin');external[path]=pinned[path] + zero=aq['initial_overlay'] + external.update({p:digest(p) for p in zero.values()}) + verify(external) + prepared={p:row['sha256'] for p,row in inventory(EXP).items()} + reg=dict(schema='normal-conditioning-registration-v1',scientific_scope='normal-acquisition-only', + subsequent_assays_authorized=False,source_code_commit=commit,source_sha256=sources,input_sha256=external, + static_input_sha256=static,runtime_input_sha256=runtime,prepared_sha256=prepared,budget=budget, + initial_overlay=zero,population_file=str(AO_EXP/'inputs/static/populations.npz'), + readout_population_file=str(EXP/'inputs/static/readout-populations.npz'), + readout_population_binding_file=str(EXP/'inputs/static/readout-population-binding.json'), + original_full_trial_plan=pin(PLAN.AO_TRIAL_PATH),normal_inventory=plan['inventory'], + evidence_scope='AQ validates exact interface, not learning; this new registration admits actual normal outcomes', + brain_parameters_changed=False,learning_success_claimed=False, + execution_order='original fixed block/arm order; actual A/B baseline each arm; no outcome selection', + biological_hypotheses='D18 branch; PAM15/PPL103 compartment hypotheses only, not all-PAM/PPL functional assignment', + historical_integrity_scope=before['scope'],failure_policy='stop preserve partials, never auto retry', + administrative_supervisor_directory=str(ROOT/'logs/phase4ar-normal-conditioning'), + supervisor_logging_scope='outside immutable EXP ledger; real top-level exit and logs must be included by final outer auditor', + full_plan_in_old_frozen_location=True,future_artifact_hashes_fabricated=False) + save(EXP/'protocol/preregistered.json',reg) + return digest(EXP/'protocol/preregistered.json') + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('mode',choices=('prepare','integrity'));p.add_argument('--output') + a=p.parse_args() + if a.mode=='prepare':print('REGISTRATION_SHA256',prepare(),flush=True) + else: + require(a.output,'Exclusive report path required');save(a.output,historical_integrity()) diff --git a/models/male-cns-lif/normal-conditioning-v1/normal_runner.py b/models/male-cns-lif/normal-conditioning-v1/normal_runner.py new file mode 100644 index 0000000..92f95dc --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/normal_runner.py @@ -0,0 +1,341 @@ +"""Serial normal conditioning. Original R/E/U executables stay unchanged. + +Every fresh brain trial is followed by original E and independent original-U +recomputation in separate exec processes. Probes never run E or feed back. +Only fully committed steps may be skipped on explicit resume. An uncommitted +start, failed child or partial task always stops; no inferred OS exit or retry. +""" +from __future__ import annotations +import argparse +import copy +from datetime import datetime, timezone +import os +import shutil +import subprocess +import sys +import time +import traceback +import numpy as np +from normal_io import * +from append_ledger import Ledger + +def registration(expected): + require(digest(EXP/'protocol/preregistered.json')==expected,'Registration hash differs') + reg=read(EXP/'protocol/preregistered.json') + require(reg['schema']=='normal-conditioning-registration-v1' and reg['scientific_scope']=='normal-acquisition-only' + and reg['subsequent_assays_authorized'] is False,'Wrong authority') + verify(reg['source_sha256']);verify(reg['input_sha256']);verify(reg['prepared_sha256']) + return reg + +def derive_spec(original,task,*,budget_root,output_cap): + require(original['kind']=='fullgraph' and original['dt_ms']==.1 and original['apl_delay_steps']==18 and + original['precision']=='float64' and original['apl_precision']=='float32' and original['record_history'] is False, + 'Original fixed D18 spec required') + require(task['steps']==1200 and task['kind'] in ('train','probe'),'Wrong task') + allowed=('validation_cohort','odor_index','expected_external_events','training','cs_input_seed','cs_input_seed_bits', + 'seed_scope','odor_id','odor_name','source_panel_row','source_panel_split','rate_sum_hz','target_count', + 'expected_CS_requests','CS_window','outer_epoch_id','outer_arm','outer_protocol') + result={k:copy.deepcopy(v) for k,v in original.items() if k not in allowed} + result.update(name=task_key(task),steps=1200,duration_ms=120.,max_steps=1200,budget_root=str(budget_root), + output_budget_bytes=output_cap,mode=task['mode'],within_trial_weight_updates=False, + outer_protocol='normal-acquisition-v1',outer_task_id=task['task_id'],outer_arm=task['arm']) + # The frozen independent reset-control seed is retained, not derived from CS. + require(result['seed']==original['seed'],'Reset seed changed') + return result + +def compose_requests(cs,dan,task,inputs): + require(task['steps']==1200,'Only normal120ms tasks') + if task['kind']=='probe': + recipe=task['recipe'];require(recipe['US_requested'] is False and recipe['D_command']==0 and + recipe['weight_updates_enabled'] is False and task['feedback_to_training_or_idle'] is False,'Probe command differs') + nodes=np.empty(0,np.int32) + else: + recipe=task['recipe'];us=recipe['us'] + require(us['nominal_slot_tick']==900 and us['rng_seed'] is None and us['rng_draws']==0 and + ((us['requested'] is True and us['tick']==900) or (us['requested'] is False and us['tick'] is None)), + 'Scheduled artificial US differs') + require(recipe['D_command']==(1. if us['requested'] and task['arm']!='d_zero' else 0.) and + recipe['update_enabled'] is (task['arm']!='frozen'),'D/update command differs') + require(dan.dtype==np.int32 and dan.ndim==1 and len(dan) in (2,4) and np.all(dan[1:]>dan[:-1]),'Bound DAN IDs required') + nodes=dan.copy() if us['requested'] else np.empty(0,np.int32) + us=dict(tick=np.full(len(nodes),900,np.int32),neuron_idx=nodes) + composed=inputs.merge_requests(cs,us,steps=1200,conflict_policy='error') + return composed,us + +class Workflow: + def __init__(self,sha): + self.sha=sha;self.reg=registration(sha);self.start=time.perf_counter() + require(not (EXP/'audit/stopped.json').exists() and not (EXP/'manifest.json').exists(), 'Stopped or sealed phase cannot run') + self.plan=read(EXP/'protocol/normal-plan.json');self.input_plan=read(EXP/'protocol/input-inventory.json') + self.tasks={x['task_id']:x for x in self.plan['tasks']} + self.h=load_source('_normal_AP_helpers',AP/'isolated_workers.py',AP_SHA) + self.inputs=load_source('_normal_request_composer',LAB/'conditioning_inputs.py',self.reg['static_input_sha256'][str(LAB/'conditioning_inputs.py')]) + self.metric=load_source('_normal_CPU_state_validation',LAB/'metrics.py',self.reg['input_sha256'][str(LAB/'metrics.py')]) + self.nodes=np.load(Path(read(EXP/'protocol/case-template.json')['graph_dir'])/'nodes.npy',allow_pickle=False) + self.populations=npz(self.reg['population_file']) + initial={Path(p).relative_to(EXP).as_posix():identity(p) for p in self.reg['prepared_sha256']} + initial['protocol/preregistered.json']=identity(EXP/'protocol/preregistered.json') + self.ledger=Ledger(EXP,sha,initial,self.reg['budget']['output_cap_bytes'],self.reg['budget']['minimum_free_reserve_bytes']) + status=self.ledger.inspect();require(not status['uncommitted_entry_ids'],'Uncommitted OS state unknown; no automatic retry') + self.completed=set(status['entry_ids']);self.expected=[] + + def basepins(self): + return {**self.reg['runtime_input_sha256'],str(EXP/'protocol/preregistered.json'):self.sha} + + def immutable_pin(self,path): + path=unlinked(path);rel=path.relative_to(EXP).as_posix() + require(rel in self.ledger._tracked and identity(path)==self.ledger._tracked[rel], + 'Consumed immutable file differs from committed bytes: '+rel) + return dict(path=str(path),sha256=self.ledger._tracked[rel]['sha256']) + + def budget(self): + require(time.perf_counter()-self.start < self.reg['budget']['workflow_timeout_seconds'],'Invocation wall cap exceeded') + require(datetime.now(timezone.utc)=self.reg['budget']['minimum_free_reserve_bytes'],'Free space reserve exceeded') + # Running totals account only committed payload, ledger and shared build. + # Each child also gets a fresh per-child output allowance; full tree at final. + accounting=self.ledger._budget() + require(accounting['total_accounted_bytes']+self.reg['budget']['next_child_headroom_bytes'] <= + self.reg['budget']['output_cap_bytes'],'No registered headroom for another complete child') + + def launch(self,name,job,argv,outputs,*,pre_outputs=(),post=None,ap=False): + self.expected.append(name) + require(name not in self.completed,'Internal attempt to rerun committed job') + self.budget() + unlinked(EXP/'receipts'/name).mkdir() + jobfile=EXP/f'jobs/{name}.json';save(jobfile,job);jobsha=digest(jobfile) + argv=[*argv] + if ap:argv+=['--job',str(jobfile),'--job-sha256',jobsha] + start=self.ledger.begin(name,jobfile,dict(kind=job['kind'])) + stdout=EXP/f'logs/{name}.stdout.log';stderr=EXP/f'logs/{name}.stderr.log' + process=EXP/f'receipts/{name}/process.json' + started=time.perf_counter();rc=None;timed=False;launch_error=None + env=dict(os.environ);env['PYTHONDONTWRITEBYTECODE']='1' + if job['kind'] not in ('brain','sample'):env['CUDA_VISIBLE_DEVICES']='' + with unlinked(stdout).open('xb') as out,unlinked(stderr).open('xb') as err: + try:rc=subprocess.run(argv,cwd=MODEL,env=env,stdout=out,stderr=err, + timeout=self.reg['budget']['child_timeout_seconds']).returncode + except subprocess.TimeoutExpired:timed=True + except OSError as error:launch_error=type(error).__name__+': '+str(error) + base=dict(entry_id=name,registration_sha256=self.sha,job_sha256=jobsha,argv=argv, + exit_code=rc,timed_out=timed,exit_observed_by_parent=rc is not None or timed,launch_error=launch_error, + child_wall_seconds=time.perf_counter()-started,observed_utc=datetime.now(timezone.utc).isoformat(), + stdout_sha256=digest(stdout),stderr_sha256=digest(stderr)) + try: + require(type(rc)is int and rc==0 and not timed,'Child failed; retain outputs and stop') + verify(job['input_sha256']) + if ap: + receipt=read(job['receipt_file']) + require(receipt['schema']=='phase4ap-isolated-worker-receipt-v1' and receipt['complete'] is True and + receipt['job_sha256']==jobsha and receipt['registration_sha256']==self.sha and + receipt['kind']==job['kind'] and receipt['job_id']==name,'Actual child receipt differs') + expected={**job['input_sha256'],str(jobfile):jobsha,str(AP/'isolated_workers.py'):AP_SHA, + **{v['path']:v['sha256'] for v in job['sources'].values()}} + require(receipt['verified_input_sha256']==expected,'Worker input closure differs') + actual={str(Path(p).relative_to(job['output_dir'])):v['sha256'] for p,v in inventory(Path(job['output_dir'])).items()} + require(actual==receipt['output_files_sha256'],'Worker output inventory differs') + require(digest(job['receipt_file']+'.attempt.json')==receipt['attempt_sha256'],'Attempt changed') + if post is not None:post() + base.update(complete=True,end_to_end_seconds=time.perf_counter()-started) + save(process,base) + files=[*pre_outputs,jobfile,stdout,stderr,process] + for p in outputs: + p=Path(p);files.extend(Path(x) for x in inventory(p)) if p.is_dir() else files.append(p) + outgoing={str(p):identity(p) for p in files} + incoming={p:identity(p) for p in job['input_sha256']} + incoming[str(jobfile)]=identity(jobfile) + self.ledger.append(name,incoming,outgoing,dict(job_file=str(jobfile),process_receipt_file=str(process), + observed_returncode=rc,scope='actual serial child exit and immutable artifact verification')) + self.completed.add(name) + print(json.dumps(dict(complete=True,entry_id=name,wall_seconds=base['end_to_end_seconds']),sort_keys=True),flush=True) + except BaseException as error: + if not process.exists():save(process,dict(base,complete=False,error=str(error))) + raise + + def input_generation(self): + name='input-generation' + if name in self.completed:self.expected.append(name);return + pins={**self.reg['source_sha256'],**self.reg['static_input_sha256'], + **{p:s for p,s in self.reg['prepared_sha256'].items() if '/inputs/static/' in p or p.endswith('/input-inventory.json')}, + str(EXP/'protocol/preregistered.json'):self.sha} + job=dict(kind='sample',registration_sha256=self.sha,input_sha256=pins) + def validate_sample(): + report=read(EXP/'receipts/input-generation.json') + require(report['complete'] is True and report['registration_sha256']==self.sha and + set(report['inputs'])==set(self.input_plan['descriptors']) and report['descriptors']==4080, + 'Incomplete original sampling inventory') + for key,d in self.input_plan['descriptors'].items(): + f=EXP/f'inputs/generated/{key}.npz';a=EXP/f'inputs/generated/{key}.json' + audit=read(a) + require(audit['descriptor']==d and audit['requests_sha256']==digest(f) and + report['inputs'][key]==dict(requests=pin(f),audit=pin(a)), 'Saved sampler binding differs') + self.inputs.validate_requests(npz(f),1200) + self.launch(name,job,[sys.executable,'-B',str(MODEL/'normal_inputs.py'),'--registration-sha256',self.sha], + [EXP/'inputs/generated',EXP/'receipts/input-generation.json'],post=validate_sample) + + def overlay(self,task): + self.overlay_dependency_pins={} + dependency=task['delta_dependency'] + if dependency==dict(kind='zero',source=None):return copy.deepcopy(self.reg['initial_overlay']) + require(dependency['kind']=='epoch_delta_after' and dependency['source'].endswith('/delta-after'),'Unknown delta dependency') + previous=self.tasks[dependency['source'][:-len('/delta-after')]] + key=task_key(previous);require(key+'--verify' in self.completed,'Previous real E+CPU validation missing') + result_path=EXP/f'results/{key}.json' + def committed(path): + checked=self.immutable_pin(path) + self.overlay_dependency_pins[checked['path']]=checked['sha256'] + committed(result_path);record=read(result_path) + require(record['complete'] is True and record['task']==previous and previous['kind']=='train' and + all(previous[k]==task[k] for k in ('block_id','arm')),'Delta cannot cross unrelated branch') + validation=Path(record['cpu_validation']['path']);committed(validation) + require(digest(validation)==record['cpu_validation']['sha256'],'CPU receipt changed') + valid=read(validation) + committed(Path(record['cpu_process_receipt_path'])) + process=read(record['cpu_process_receipt_path']) + require(process['complete'] is True and type(process['exit_code']) is int and process['exit_code']==0 and + process['entry_id']==key+'--verify' and process['registration_sha256']==self.sha, + 'Previous CPU exit missing') + for role,filename,field in (('overlay_file','delta-after.npz','after_overlay_sha256'), + ('overlay_manifest','overlay-manifest.json','after_manifest_sha256')): + path=Path(record['after_overlay'][role]);committed(path) + require(path==Path(record['update_dir'])/filename and digest(path)==valid[field], + 'Next installed delta is not the actual independently verified output') + return copy.deepcopy(record['after_overlay']) + + def prepare_case(self,task): + key=task_key(task);case=unlinked(EXP/'cases'/key);case.mkdir() + template_file=EXP/'protocol/case-template.json' + checked=self.immutable_pin(template_file) + self.case_dependency_pins={checked['path']:checked['sha256'], + self.reg['population_file']:self.reg['input_sha256'][self.reg['population_file']]} + template=read(template_file) + spec=derive_spec(template,task,budget_root=EXP,output_cap=self.reg['budget']['output_cap_bytes']) + csid=self.input_plan['task_inputs'][task['task_id']] + if csid is None:cs=dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32));cs_source=None + else: + cs_source=EXP/f'inputs/generated/{csid}.npz' + audit_file=EXP/'inputs/generated'/(csid+'.json') + for path in (cs_source,audit_file): + checked=self.immutable_pin(path);self.case_dependency_pins[checked['path']]=checked['sha256'] + generated=read(audit_file) + require(generated['descriptor']==self.input_plan['descriptors'][csid] and + generated['requests_sha256']==digest(cs_source),'Saved shared CS changed') + cs=npz(cs_source) + combined,us=compose_requests(cs,self.populations[task['dan_type']],task,self.inputs) + save(case/'spec.json',spec);save_npz(case/'requests.npz',combined['requests']) + save_npz(case/'cs_requests.npz',cs);save_npz(case/'us_requests.npz',us) + save_npz(case/'source-mask.npz',dict(source_mask=combined['source_mask'])) + save(case/'input-binding.json',dict(task_id=task['task_id'],CS_source=None if cs_source is None else pin(cs_source), + CS_descriptor_id=csid,composition=combined['audit'],DAN_type=task['dan_type'], + DAN_membership_sha256=hashlib.sha256(self.populations[task['dan_type']].tobytes()).hexdigest(), + population_file=pin(self.reg['population_file']),artificial_US_not_dopamine_concentration=True)) + return case + + def ap_job(self,name,kind,output,*,case=None,overlay=None,authority=None): + pins=self.basepins() + job=dict(schema='phase4ap-isolated-worker-job-v1',job_id=name,kind=kind,registration_sha256=self.sha, + worker_sha256=AP_SHA,output_dir=str(output),receipt_file=str(EXP/f'receipts/{name}/worker.json')) + if kind=='brain': + pins.update(self.overlay_dependency_pins) + pins.update(self.case_dependency_pins) + for p in (case/'spec.json',case/'requests.npz',overlay['overlay_file'],overlay['overlay_manifest']):pins[str(p)]=digest(p) + job.update(case_dir=str(case),build_root=str(EXP/'extension-build'),**overlay,diagnostic_mode='light', + admission=dict(gpu_uuid=GPU,comfy_queue_urls=list(self.h.COMFY_URLS)), + sources={n:dict(path=str(LAB/(n+'.py')),sha256=self.h.SOURCE_PINS[n]) for n in ('runtime_light','runtime_continuation')}) + else: + auth=read(authority);pins[str(authority)]=digest(authority) + pins.update({v['path']:v['sha256'] for v in auth['files'].values()}) + pins.update(auth['frozen_inputs']);pins[str(Path(auth['checkpoint_dir'])/'checkpoint-manifest.json')]=auth['checkpoint_manifest_sha256'] + job.update(authority_file=str(authority),authority_sha256=digest(authority), + sources={'conditioning_epoch':dict(path=str(LAB/'conditioning_epoch.py'),sha256=self.h.SOURCE_PINS['conditioning_epoch'])}) + job['input_sha256']=pins;return job + + def brain_verify(self,task,case,output): + summary=read(output/'summary.json');state=npz(output/'state.npz') + checked=self.metric.validate_state(state,nodes=self.nodes,steps=1200) + require(summary['all_finite'] is True and summary['steps']==1200 and summary['diagnostic_mode']=='light','Nonfinite/incomplete brain') + require(int(state['counts'].sum())==summary['total_spikes'],'Spike event/count mismatch') + manifest=read(output/'checkpoint-manifest.json') + for rel,sha in manifest['files_sha256'].items():require(digest(output/rel)==sha,'Checkpoint payload changed') + return dict(complete=True,state_raw_sha256=checked['hashes'],finite=True, + checkpoint_manifest_sha256=digest(output/'checkpoint-manifest.json'),task_id=task['task_id']) + + def task(self,task): + key=task_key(task);brain=key+'--brain';up=key+'--update';cpu=key+'--verify' + planned=[brain,up,cpu] if task['kind']=='train' else [brain] + if all(x in self.completed for x in planned):self.expected.extend(planned);return + require(not any(x in self.completed for x in planned),'Partially completed task: preserve and inspect; no implicit replay') + overlay=self.overlay(task);case=self.prepare_case(task);output=EXP/'runs'/key + record=dict(schema='normal-task-result-v1',complete=True,registration_sha256=self.sha,task=task, + checkpoint_dir=str(output),overlay_before=overlay,brain_process_receipt_path=str(EXP/f'receipts/{brain}/process.json')) + def after_brain(): + record['brain_audit']=self.brain_verify(task,case,output) + record['checkpoint_manifest_sha256']=digest(output/'checkpoint-manifest.json') + # Parent process receipt is produced later and is bound by the ledger, + # avoiding a self-reference in a result written before process exit audit. + if task['kind']=='probe':save(EXP/f'results/{key}.json',record) + job=self.ap_job(brain,'brain',output,case=case,overlay=overlay) + outs=[output,Path(job['receipt_file']),Path(job['receipt_file']+'.attempt.json')] + if task['kind']=='probe':outs.append(EXP/f'results/{key}.json') + self.launch(brain,job,[sys.executable,'-B',str(AP/'isolated_workers.py'),'brain'],outs, + pre_outputs=[Path(p) for p in inventory(case)],post=after_brain,ap=True) + if task['kind']=='probe':return + template_pin=self.immutable_pin(EXP/'protocol/epoch-template.json') + template=read(template_pin['path']);auth=copy.deepcopy(template) + r=task['recipe'];auth.update(epoch_id=task['task_id'],diagnostic_mode='light',checkpoint_dir=str(output), + checkpoint_manifest_sha256=digest(output/'checkpoint-manifest.json'), + command=dict(steps=1200,compartment=task['compartment_verbatim'],us_tick=r['us']['tick'],d_command=r['D_command'], + frozen=not r['update_enabled'],trace_backend='torch')) + for role,path in dict(spec=case/'spec.json',requests=case/'requests.npz',state=output/'state.npz', + overlay_before=overlay['overlay_file'],overlay_manifest_before=overlay['overlay_manifest']).items():auth['files'][role]=pin(path) + authority=EXP/f'jobs/{key}.authority.json';save(authority,auth);update=EXP/'updates'/key + job=self.ap_job(up,'update',update,authority=authority) + job['input_sha256'][template_pin['path']]=template_pin['sha256'] + for path in (EXP/f'receipts/{brain}/process.json',EXP/f'receipts/{brain}/worker.json'): + checked=self.immutable_pin(path);job['input_sha256'][checked['path']]=checked['sha256'] + self.launch(up,job,[sys.executable,'-B',str(AP/'isolated_workers.py'),'update'], + [update,Path(job['receipt_file']),Path(job['receipt_file']+'.attempt.json')],pre_outputs=[authority],ap=True) + out=unlinked(EXP/'validation'/key);out.mkdir();validation=out/'validation.json' + script=AQ/'validate_update_cpu.py';verify({str(script):self.reg['input_sha256'][str(script)]}) + pins={**job['input_sha256'],str(script):self.reg['input_sha256'][str(script)]} + for name in ('conditioning_epoch.py','conditioning_preflight_compare.py','engineering_compare.py'): + path=str(LAB/name);pins[path]=self.reg['input_sha256'][path] + pins.update({p:v['sha256'] for p,v in inventory(update).items()}) + job=dict(kind='cpu_verify',registration_sha256=self.sha,input_sha256=pins,authority_file=str(authority), + authority_sha256=digest(authority),update_dir=str(update),chain_sha256=digest(update/'chain-receipt.json'), + receipt_file=str(validation),validator_sha256=self.reg['input_sha256'][str(script)]) + def after_cpu(): + valid=read(validation) + require(valid['complete'] is True and valid['frozen_U_verify_receipt_recomputed'] is True and + valid['actual_sealed_events_used'] is True and valid['after_overlay_sha256']==digest(update/'delta-after.npz') and + valid['epoch_authority_sha256']==digest(authority),'Independent update verification differs') + if task['arm'] in ('d_zero','frozen','cs_only'): + require(not npz(update/'delta-after.npz')['plastic_delta_mv'].view(np.uint32).any(),'Zero-update control became nonzero') + record.update(epoch_authority=pin(authority),update_dir=str(update),cpu_validation=pin(validation), + after_overlay=dict(overlay_file=str(update/'delta-after.npz'),overlay_manifest=str(update/'overlay-manifest.json')), + update_process_receipt_path=str(EXP/f'receipts/{up}/process.json'), + cpu_process_receipt_path=str(EXP/f'receipts/{cpu}/process.json')) + save(EXP/f'results/{key}.json',record) + self.launch(cpu,job,[sys.executable,'-B',str(script),'--authority',str(authority),'--authority-sha256',digest(authority), + '--update-dir',str(update),'--chain-sha256',digest(update/'chain-receipt.json'),'--receipt',str(validation)], + [out,EXP/f'results/{key}.json'],post=after_cpu) + + def run(self): + try: + self.input_generation() + for task in self.plan['tasks']:self.task(task) + report=self.ledger.final_verify(self.expected) + require(report['pass_all'],'Final ledger coverage/integrity failed') + save(EXP/'audit/execution-complete.json',dict(complete=True,registration_sha256=self.sha, + inventory=self.plan['inventory'],ledger=report,learning_success_claimed=False, + next='independent scientific analysis and frozen integrity, no subsequent assay')) + except BaseException as error: + if not (EXP/'audit/stopped.json').exists():save(EXP/'audit/stopped.json',dict(complete=False, + registration_sha256=self.sha,error_type=type(error).__name__,error=str(error), + traceback=traceback.format_exc(),partial_files_preserved=True,automatic_retry=False)) + raise + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--registration-sha256',required=True) + Workflow(p.parse_args().registration_sha256).run() diff --git a/models/male-cns-lif/normal-conditioning-v1/run_cpu_tests.py b/models/male-cns-lif/normal-conditioning-v1/run_cpu_tests.py new file mode 100644 index 0000000..a3d6406 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/run_cpu_tests.py @@ -0,0 +1,30 @@ +"""Exclusive CPU/source test receipt. No scientific execution or CUDA allowed.""" +import argparse +import io +import os +from pathlib import Path +import sys +import time +import unittest +from normal_io import digest, save, require, MODEL + +def run(output): + require(sys.dont_write_bytecode,'Use -B') + os.environ['CUDA_VISIBLE_DEVICES']='' + before={p.name:digest(p) for p in MODEL.iterdir() if p.is_file()} + suite=unittest.defaultTestLoader.discover(str(MODEL),pattern='test_*.py') + text=io.StringIO();start=time.perf_counter() + result=unittest.TextTestRunner(stream=text,verbosity=2).run(suite) + after={p.name:digest(p) for p in MODEL.iterdir() if p.is_file()} + no_neural_modules=not any(n in sys.modules for n in ('torch','runtime_light','conditioning_epoch')) + success=result.wasSuccessful() and not result.skipped and before==after and no_neural_modules + save(output,dict(schema='normal-CPU-tests-v1',pass_all=success,tests=result.testsRun, + failures=len(result.failures),errors=len(result.errors),skips=len(result.skipped), + source_sha256=before,sources_unchanged=before==after,wall_seconds=time.perf_counter()-start, + stdout=text.getvalue(),fresh_process=True,CUDA_hidden=True,no_neural_modules=no_neural_modules, + scope='CPU source and mock-process tests; not real GPU numerical or scientific evidence')) + return success + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--output',required=True) + sys.exit(0 if run(p.parse_args().output) else 1) diff --git a/models/male-cns-lif/normal-conditioning-v1/run_normal.py b/models/male-cns-lif/normal-conditioning-v1/run_normal.py new file mode 100644 index 0000000..f90cd5d --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/run_normal.py @@ -0,0 +1,55 @@ +"""Detached parent supervisor, with real final exit receipts outside EXP. + +Administrative stdout is separate from the immutable per-trial ledger. Launch +is exclusive, never a retry policy. A launch PID is NOT a completion receipt. +""" +import argparse +from datetime import datetime, timezone +import os +import subprocess +import sys +import time +from normal_io import * + +CONTROL=ROOT/'logs/phase4ar-normal-conditioning' + +def authority(sha): + require(sys.dont_write_bytecode,'Use -B') + require(digest(EXP/'protocol/preregistered.json')==sha,'Registration differs') + reg=read(EXP/'protocol/preregistered.json');verify(reg['source_sha256']) + require(reg['scientific_scope']=='normal-acquisition-only' and unlinked(CONTROL).is_dir(),'Wrong scope/control directory') + require(not (EXP/'audit/stopped.json').exists() and not (EXP/'audit/execution-complete.json').exists(), + 'Stopped/completed phase cannot launch') + return reg + +def launch(sha): + authority(sha) + require(not any(CONTROL.iterdir()),'Control directory nonempty: inspect, do not retry') + save(CONTROL/'launch-attempt.json',dict(registration_sha256=sha,started_utc=datetime.now(timezone.utc).isoformat(), + complete=False,terminal_state='UNKNOWN',automatic_retry=False)) + command=[sys.executable,'-B',str(MODEL/'run_normal.py'),'supervise','--registration-sha256',sha] + with unlinked(CONTROL/'stdout.log').open('xb') as out,unlinked(CONTROL/'stderr.log').open('xb') as err: + process=subprocess.Popen(command,cwd=MODEL,stdin=subprocess.DEVNULL,stdout=out,stderr=err,start_new_session=True, + env={**os.environ,'PYTHONDONTWRITEBYTECODE':'1'}) + record=dict(supervisor_pid=process.pid,argv=command,registration_sha256=sha,launch_observed=True, + final_exit_observed=False,not_evidence_of_completion=True) + save(CONTROL/'launch.json',record);return record + +def supervise(sha): + authority(sha) + save(CONTROL/'supervisor-start.json',dict(pid=os.getpid(),registration_sha256=sha, + started_utc=datetime.now(timezone.utc).isoformat(),complete=False)) + command=[sys.executable,'-B',str(MODEL/'normal_runner.py'),'--registration-sha256',sha] + started=time.perf_counter();code=None;error=None + try:code=subprocess.run(command,cwd=MODEL,stdin=subprocess.DEVNULL).returncode + except BaseException as exc:error=type(exc).__name__+': '+str(exc) + save(CONTROL/'exit.json',dict(registration_sha256=sha,argv=command,exit_code=code, + final_exit_observed=type(code)is int,wall_seconds=time.perf_counter()-started,error=error, + complete=code==0,learning_success_claimed=False,scientific_analysis_required=True)) + return 0 if code==0 else 1 + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('mode',choices=('launch','supervise')) + parser.add_argument('--registration-sha256',required=True);args=parser.parse_args() + if args.mode=='launch':print(canonical(launch(args.registration_sha256)),flush=True) + else:sys.exit(supervise(args.registration_sha256)) diff --git a/models/male-cns-lif/normal-conditioning-v1/test_append_ledger.py b/models/male-cns-lif/normal-conditioning-v1/test_append_ledger.py new file mode 100644 index 0000000..3a7fb4c --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/test_append_ledger.py @@ -0,0 +1,241 @@ +"""Temporary filesystem tests only; no process/brain/GPU execution.""" +import copy +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import append_ledger as L + + +def put(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value if type(value) is bytes else L.raw_json(value)) + return L.identity(path) + + +class LedgerTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.root=Path(self.temp.name).resolve()/'normal';self.root.mkdir() + self.reg=self.root/L.REGISTRATION; info=put(self.reg,{'only':'synthetic-registration'}) + self.initial={L.REGISTRATION:info};self.sha=info['sha256'] + self.ledger=L.Ledger(self.root,self.sha,self.initial,1<<20,0) + + def job(self,name): + job=self.root/'jobs'/(name+'.json');put(job,{'entry_id':name}) + self.ledger.begin(name,job,{'synthetic':True}) + return job + + def completion(self,name,job,*,payload=b'output',changes=None): + out=self.root/'runs'/name/'payload.dat';put(out,payload) + process=self.root/'receipts'/(name+'.json') + value=dict(entry_id=name,registration_sha256=self.sha,job_sha256=L.digest(job),complete=True, + exit_code=0,timed_out=False,exit_observed_by_parent=True) + value.update(changes or {});put(process,value) + inputs={str(self.reg):L.identity(self.reg),str(job):L.identity(job)} + outputs={str(p):L.identity(p) for p in (job,out,process)} + metadata=dict(job_file=str(job),process_receipt_file=str(process),scope='synthetic completed parent job only',observed_returncode=0) + return inputs,outputs,metadata + + def commit(self,name): + job=self.job(name);args=self.completion(name,job) + return self.ledger.append(name,*args) + + def test_two_entries_chain_full_tree_and_cumulative_bytes(self): + one=self.commit('one');two=self.commit('two') + second=L.read_json(self.root/L.CONTROL/'entries/0000000000000002.json') + self.assertEqual(second['prev_entry_sha256'],one['entry_sha256']) + self.assertEqual(two['entry_sha256'],L.digest(self.root/L.CONTROL/'entries/0000000000000002.json')) + inspected=self.ledger.inspect();self.assertEqual(inspected['entry_ids'],['one','two']) + self.assertFalse(inspected['retry_authorized']);self.assertTrue(inspected['all_tracked_history_rehashed']) + final=self.ledger.final_verify(['one','two']);self.assertTrue(final['pass_all']) + self.assertEqual(final['actual_total_bytes'],sum(p.stat().st_size for p in self.root.rglob('*') if p.is_file())) + self.assertFalse(final['scientific_gate_claimed']) + + def test_reopen_independent_read_verifies_no_reexecution_or_duplicate_append(self): + self.commit('one') + reopened=L.Ledger(self.root,self.sha,self.initial,1<<20,0) + self.assertEqual(reopened.inspect()['entry_ids'],['one']) + with self.assertRaisesRegex(ValueError,'already started/committed'): + reopened.begin('one',self.root/'jobs/one.json') + with self.assertRaisesRegex(ValueError,'No unique uncommitted'): + reopened.append('one',{}, {},dict(job_file='x',process_receipt_file='y',scope='s',observed_returncode=0)) + + def test_pending_or_live_handle_is_unknown_and_never_retry_authority(self): + job=self.job('live');state=self.ledger.inspect() + self.assertEqual(state['uncommitted_state'],'STARTED-UNCOMMITTED-STATE-UNKNOWN') + self.assertFalse(state['retry_authorized']) + with self.assertRaisesRegex(ValueError,'uncommitted start'):self.ledger.begin('another',job) + args=list(self.completion('live',job));args[2]['observed_returncode']=None + with self.assertRaisesRegex(ValueError,'Live/failed'):self.ledger.append('live',*args) + final=self.ledger.final_verify(['live']);self.assertFalse(final['pass_all']);self.assertTrue(final['uncommitted_state_unknown']) + reopened=L.Ledger(self.root,self.sha,self.initial,1<<20,0) + self.assertEqual(reopened.inspect()['uncommitted_entry_ids'],['live']) + + def test_actual_parent_receipt_required_not_worker_success_or_bool_exit(self): + job=self.job('one') + for mutation in ({'exit_code':False},{'complete':False},{'exit_observed_by_parent':False}, + {'timed_out':True},{'registration_sha256':'0'*64},{'job_sha256':'0'*64},{'entry_id':'other'}): + args=self.completion('one',job,changes=mutation) + with self.subTest(mutation=mutation),self.assertRaisesRegex(ValueError,'parent process receipt'): + self.ledger.append('one',*args) + + def test_same_size_same_mtime_mutation_fails_actual_SHA(self): + job=self.job('one');args=self.completion('one',job,payload=b'aaaa') + out=self.root/'runs/one/payload.dat';stat=out.stat();out.write_bytes(b'bbbb') + os.utime(out,ns=(stat.st_atime_ns,stat.st_mtime_ns)) + with self.assertRaisesRegex(ValueError,'Actual file SHA/bytes'):self.ledger.append('one',*args) + + def test_missing_extra_changed_files_reported_at_full_audit(self): + self.commit('one');out=self.root/'runs/one/payload.dat' + extra=self.root/'unexpected.log';put(extra,b'extra') + final=self.ledger.final_verify(['one']);self.assertFalse(final['pass_all']);self.assertIn('unexpected.log',final['extra_files']) + out.unlink();final=self.ledger.final_verify(['one']) + self.assertIn('runs/one/payload.dat',final['missing_files']) + put(out,b'different');final=self.ledger.final_verify(['one']) + self.assertIn('runs/one/payload.dat',final['changed_files']) + + def test_tail_and_middle_tamper_not_trusted_by_inspect(self): + self.commit('one');self.commit('two') + first=self.root/L.CONTROL/'entries/0000000000000001.json' + value=L.read_json(first);value['metadata']['scope']='tampered' + put(first,value) + with self.assertRaisesRegex(ValueError,'chain/start identity'):self.ledger.inspect() + self.assertFalse(self.ledger.final_verify(['one','two'])['pass_all']) + + def test_previously_observed_head_rewrite_rejected_even_new_valid_JSON(self): + self.commit('one');path=self.root/L.CONTROL/'entries/0000000000000001.json' + value=L.read_json(path);value['metadata']['scope']='changed';put(path,value) + with self.assertRaisesRegex(ValueError,'head changed'):self.ledger.inspect() + + def test_registered_final_entry_order_and_coverage_are_exact(self): + self.commit('one');self.commit('two') + for expected in (['two','one'],['one'],['one','two','three']): + report=self.ledger.final_verify(expected) + self.assertFalse(report['pass_all']);self.assertFalse(report['expected_order_exact']) + with self.assertRaisesRegex(ValueError,'expected entry order'):self.ledger.final_verify(['one','one']) + + def test_path_escape_aliases_and_overwrite_rejected(self): + self.commit('one');job=self.job('two');inputs,outputs,meta=self.completion('two',job) + outside=self.root.parent/'outside.dat';put(outside,b'external') + outputs[str(outside)]=L.identity(outside) + with self.assertRaisesRegex(ValueError,'escapes experiment'):self.ledger.append('two',inputs,outputs,meta) + del outputs[str(outside)];old=self.root/'runs/one/payload.dat';outputs[str(old)]=L.identity(old) + with self.assertRaisesRegex(ValueError,'previously tracked'):self.ledger.append('two',inputs,outputs,meta) + del outputs[str(old)];outputs['runs/two/payload.dat']=outputs[str(self.root/'runs/two/payload.dat')] + with self.assertRaisesRegex(ValueError,'Duplicate path aliases'):self.ledger.append('two',inputs,outputs,meta) + with self.assertRaisesRegex(ValueError,'Escaping/noncanonical'):L.relative_name('../escape') + + def test_consumed_tracked_input_cannot_be_silently_reauthorized_with_new_SHA(self): + self.commit('one');job=self.job('two');inputs,outputs,meta=self.completion('two',job) + prior=self.root/'runs/one/payload.dat';put(prior,b'overwritten') + inputs[str(prior)]=L.identity(prior) + with self.assertRaisesRegex(ValueError,'committed identity'):self.ledger.append('two',inputs,outputs,meta) + + def test_external_inputs_allowed_but_declared_bytes_reverified(self): + job=self.job('one');inputs,outputs,meta=self.completion('one',job) + external=self.root.parent/'frozen-reference.json';put(external,{'read_only':'data'}) + inputs[str(external)]=L.identity(external);self.ledger.append('one',inputs,outputs,meta) + row=L.read_json(self.root/L.CONTROL/'entries/0000000000000001.json') + self.assertIsNone(next(r for r in row['consumed_inputs'] if r['path']==str(external))['relative_path']) + put(external,{'read_only':'changed'}) + with self.assertRaisesRegex(ValueError,'Actual file SHA/bytes'):self.ledger.inspect() + + def test_full_audit_hashes_shared_external_input_once_per_invocation(self): + external=self.root.parent/'large-shared-reference';put(external,b'fixed external bytes') + expected=L.identity(external) + for name in ('one','two','three'): + job=self.job(name);inputs,outputs,meta=self.completion(name,job) + inputs[str(external)]=expected;self.ledger.append(name,inputs,outputs,meta) + original=L.digest;counts={} + def counted(path): + key=str(Path(path));counts[key]=counts.get(key,0)+1 + return original(path) + with patch.object(L,'digest',side_effect=counted): + self.assertTrue(self.ledger.final_verify(['one','two','three'])['pass_all']) + self.assertEqual(counts[str(external)],1) + counts.clear();self.ledger.inspect();self.assertEqual(counts[str(external)],1) + counts.clear();self.ledger.inspect();self.assertEqual(counts[str(external)],1) + oldstat=external.stat();external.write_bytes(b'other external bytes') + os.utime(external,ns=(oldstat.st_atime_ns,oldstat.st_mtime_ns)) + with self.assertRaisesRegex(ValueError,'Actual file SHA/bytes'):self.ledger.inspect() + + def test_append_does_not_iterate_or_sum_historical_maps_or_search_ordered_ids(self): + self.commit('one') + class LookupOnlyDict(dict): + def __iter__(self):raise AssertionError('Historical map iteration in fast append') + def keys(self):raise AssertionError('Historical map copy in fast append') + def values(self):raise AssertionError('Historical map sum in fast append') + def items(self):raise AssertionError('Historical map iteration in fast append') + class NoLinearMembership(list): + def __contains__(self,value):raise AssertionError('Linear completed-ID membership') + self.ledger._tracked=LookupOnlyDict(self.ledger._tracked) + self.ledger._controls=LookupOnlyDict(self.ledger._controls) + self.ledger._completed=NoLinearMembership(self.ledger._completed) + self.commit('two') + self.assertEqual(self.ledger._completed,['one','two']) + self.assertTrue(self.ledger.final_verify(['one','two'])['pass_all']) + + def test_head_is_freshly_checked_after_cached_full_tree_walk(self): + self.commit('one');tail=self.root/L.CONTROL/'entries/0000000000000001.json' + original_replay=self.ledger._replay;original_identity=self.ledger._identity_once + state={'walking':False,'mutated':False} + def replay(verified): + original_replay(verified);state['walking']=True + def during_walk(path,verified): + if state['walking'] and not state['mutated']: + value=L.read_json(tail);value['metadata']['scope']='mutation during final walk' + put(tail,value);state['mutated']=True + return original_identity(path,verified) + with patch.object(self.ledger,'_replay',side_effect=replay),\ + patch.object(self.ledger,'_identity_once',side_effect=during_walk): + report=self.ledger.final_verify(['one']) + self.assertTrue(state['mutated']);self.assertFalse(report['pass_all']) + self.assertTrue(any('Known ledger tail changed' in message for message in report['errors'])) + + def test_shared_mutable_build_separate_current_inventory_and_cap(self): + build=self.root/'extension-build/cache.bin';put(build,b'first');self.commit('one') + put(build,b'changed build is allowed');self.commit('two') + report=self.ledger.final_verify(['one','two']);self.assertTrue(report['pass_all']) + self.assertEqual(report['shared_mutable_build_bytes'],len(b'changed build is allowed')) + self.assertNotIn('extension-build/cache.bin',report['extra_files']) + put(build,b'x'*(1<<20));report=self.ledger.final_verify(['one','two']) + self.assertFalse(report['pass_all']);self.assertFalse(report['output_cap_respected']) + + def test_stale_writer_lock_and_partial_publication_never_auto_cleared(self): + lock=self.root/L.CONTROL/'writer.lock';put(lock,b'unknown owner') + with self.assertRaises(FileExistsError):self.ledger.begin('one',self.reg) + with self.assertRaisesRegex(ValueError,'state unknown'):self.ledger.inspect() + self.assertEqual(lock.read_bytes(),b'unknown owner') + + def test_exclusive_atomic_publication_and_preserved_failed_temporary(self): + target=self.root/'standalone.json';info=L.publish_exclusive(target,{'a':1}) + self.assertEqual(info,L.identity(target)) + with self.assertRaisesRegex(ValueError,'already exists'):L.publish_exclusive(target,{'a':2}) + self.assertEqual(L.read_json(target),{'a':1}) + target2=self.root/'failed.json' + with patch.object(L.os,'link',side_effect=OSError('synthetic unsupported filesystem')): + with self.assertRaises(OSError):L.publish_exclusive(target2,{'a':2}) + self.assertFalse(target2.exists());self.assertEqual(len(list(self.root.glob('.publishing-*'))),1) + + def test_linked_output_rejected_if_platform_supports_links(self): + source=self.root.parent/'other';source.mkdir();link=self.root/'linked' + try:link.symlink_to(source,target_is_directory=True) + except OSError:self.skipTest('Platform disallows synthetic symlink') + job=self.job('one');inputs,outputs,meta=self.completion('one',job) + put(source/'payload',b'x');outputs[str(link/'payload')]=L.identity(source/'payload') + with self.assertRaisesRegex(ValueError,'Linked'):self.ledger.append('one',inputs,outputs,meta) + + def test_initial_and_header_authority_are_not_optional(self): + for change in ('registration','options'): + with self.subTest(change=change),self.assertRaises(ValueError): + L.Ledger(self.root,'0'*64 if change=='registration' else self.sha,self.initial, + 2<<20 if change=='options' else 1<<20,0) + header=self.root/L.CONTROL/'header.json';value=L.read_json(header);value['max_output_bytes']+=1;put(header,value) + with self.assertRaisesRegex(ValueError,'header changed'):self.ledger.inspect() + + +if __name__=='__main__':unittest.main() diff --git a/models/male-cns-lif/normal-conditioning-v1/test_normal_inputs.py b/models/male-cns-lif/normal-conditioning-v1/test_normal_inputs.py new file mode 100644 index 0000000..7269d2f --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/test_normal_inputs.py @@ -0,0 +1,73 @@ +"""CPU request descriptors and pinned static transform, never a CUDA test.""" +import copy +import unittest +import numpy as np +import normal_inputs as I +from normal_io import load_source, LAB +from normal_runner import compose_requests, derive_spec + +class InputsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.inputs=load_source('_normal_test_CI', LAB/'conditioning_inputs.py',I.PINS['conditioning_inputs.py']) + + def task(self,arm='paired',cs=True): + return dict(task_id='block/normal/'+arm+'/000',kind='train',arm=arm,steps=1200,mode='physiological', + recipe=dict(cs=None if not cs else dict(cue='A',seed=2**62+37,start_tick=0,stop_tick_exclusive=1000,source_stream_id='s'), + us=dict(nominal_slot_tick=900,rng_seed=None,rng_draws=0,requested=True,tick=900), + D_command=0. if arm=='d_zero' else 1.,update_enabled=arm!='frozen')) + + def test_full63_seed_and_aliases(self): + a=self.task();b=self.task('d_zero') + self.assertEqual(I.descriptor(a),I.descriptor(b)) + self.assertEqual(I.descriptor(a)['seed'],2**62+37) + + def test_empty_CS_is_not_resampled(self): + self.assertIsNone(I.descriptor(self.task('dan_only',False))) + + def test_all_DAN_ids_and_counterfactual_US(self): + dan=np.array([10,20],np.int32);empty=dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32)) + for arm in ('paired','d_zero','frozen'): + composed,us=compose_requests(empty,dan,self.task(arm),self.inputs) + np.testing.assert_array_equal(composed['requests']['neuron_idx'],dan) + self.assertEqual(us['tick'].tolist(),[900,900]) + + def test_no_US_and_zero_command(self): + task=self.task('cs_only');task['recipe']['us'].update(requested=False,tick=None);task['recipe']['D_command']=0. + cs=dict(tick=np.array([1],np.int32),neuron_idx=np.array([5],np.int32)) + combined,us=compose_requests(cs,np.array([10,20],np.int32),task,self.inputs) + self.assertEqual(len(us['tick']),0);self.assertEqual(combined['requests']['neuron_idx'].tolist(),[5]) + + def test_conflict_and_command_fail_closed(self): + cs=dict(tick=np.array([900],np.int32),neuron_idx=np.array([10],np.int32)) + with self.assertRaises(ValueError):compose_requests(cs,np.array([10,20],np.int32),self.task(),self.inputs) + task=self.task('d_zero');task['recipe']['D_command']=1. + with self.assertRaises(ValueError):compose_requests(cs,np.array([10,20],np.int32),task,self.inputs) + + def test_probe_no_US_no_update(self): + p=dict(kind='probe',mode='matched_energy',steps=1200,feedback_to_training_or_idle=False, + recipe=dict(cue='B',input=dict(seed=2**61,stream_id='eval'),cs_start_tick=0,cs_stop_tick_exclusive=1000, + US_requested=False,D_command=0.,weight_updates_enabled=False)) + d=I.descriptor(p);self.assertEqual(d['table_id'],'matched_energy--B') + empty=dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32)) + combined,us=compose_requests(empty,np.array([10,20],np.int32),p,self.inputs) + self.assertEqual(len(combined['requests']['tick']),0) + p['feedback_to_training_or_idle']=True + with self.assertRaises(ValueError):compose_requests(empty,np.array([10,20],np.int32),p,self.inputs) + + def test_clock_rejected(self): + task=self.task();task['steps']=1800 + with self.assertRaises(ValueError):I.descriptor(task) + + def test_case_math_config_retained(self): + original=dict(kind='fullgraph',dt_ms=.1,apl_delay_steps=18,precision='float64',apl_precision='float32', + record_history=False,seed=1073741825,model=dict(tau=20),apl=dict(gain=.01),graph_dir='/frozen/graph', + rate_sum_hz=1,mode='physiological') + task=self.task();task['mode']='matched_energy';before=copy.deepcopy(original) + spec=derive_spec(original,task,budget_root='/new/exp',output_cap=1234) + self.assertEqual(original,before) + for key in ('dt_ms','apl_delay_steps','precision','apl_precision','seed','model','apl','graph_dir'): + self.assertEqual(spec[key],original[key]) + self.assertEqual(spec['steps'],1200);self.assertFalse(spec['within_trial_weight_updates']) + +if __name__=='__main__':unittest.main() diff --git a/models/male-cns-lif/normal-conditioning-v1/test_normal_metrics.py b/models/male-cns-lif/normal-conditioning-v1/test_normal_metrics.py new file mode 100644 index 0000000..a9a65e9 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/test_normal_metrics.py @@ -0,0 +1,274 @@ +"""CPU arrays and mocked artifact I/O; these are not scientific observations.""" +import copy +import hashlib +import importlib.util +import io +from pathlib import Path +import unittest +from unittest.mock import patch + +import numpy as np + +spec = importlib.util.spec_from_file_location('_test_normal_metrics', Path(__file__).with_name('normal_metrics.py')) +N = importlib.util.module_from_spec(spec) +spec.loader.exec_module(N) + + +def fixture(): + nodes = np.arange(6, dtype=np.int32) + pops = {k: np.asarray(v, np.int32) for k, v in dict(MBON=[0, 1], MBON01=[0], MBON12=[1], + legacyDownstream=[0, 1, 2, 3], nonMBONDownstream=[2, 3]).items()} + blocks, probes, arms = {}, {}, {} + seed = 1 + for mode in N.M.MODES: + for dan in N.M.DAN_TYPES: + for cue in ('A', 'B'): + for rep in range(1, 6): + identity = f'{mode}__{dan}__reinforce_{cue}__rep{rep:02d}' + sources = [] + for split in ('training', 'design'): + sources.append(dict(stream_id=identity + '/' + split, seed=seed, split=split)); seed += 1 + evaluation = {} + for c in ('A', 'B'): + evaluation[N.ODORS[c]] = dict(stream_id=identity + '/evaluation/' + c, seed=seed, split='evaluation'); seed += 1 + blocks[identity] = dict(mode=mode, dan_type=dan, compartment_verbatim=N.G.DAN_COMPARTMENTS[dan], + conditioned_cue=N.ODORS[cue], replicate=rep, training_sources=sources, evaluation_sources=evaluation, + evaluation_input_sha256={N.ODORS[c]: N.sha([identity, c]) for c in ('A', 'B')}) + for arm in N.ARM_NAMES: + arms[(identity, arm)] = [dict(result=[identity, arm, e], requested=e < 50 and arm != 'cs_only') + for e in range(150 if arm == 'unpaired' else 100)] + for point in N.POINTS: + for c in ('A', 'B'): + probes[(identity, arm, point, c)] = dict(block=identity, arm=arm, point=point, cue=c, + active=arm == 'paired' and point > 0 and c == cue and rep <= 3, dan=dan) + membership = N.M.validate_populations(nodes, pops) + registry = dict(schema=N.G.REGISTRY_SCHEMA, protocol_sha256='1' * 64, population_binding_sha256='2' * 64, + nodes_raw_sha256=N.M._hash(nodes), population_membership_sha256=membership['membership_sha256'], blocks=blocks) + authority = dict(schema=N.G.AUTHORITY_SCHEMA, registry_sha256=N.G.descriptor_sha256(registry), + verification_receipt_sha256='3' * 64, execution_inventory_sha256='4' * 64, population_binding_sha256='2' * 64) + return nodes, pops, registry, authority, probes, arms + + +def state_for(record): + ids = np.asarray(sorted([1 if record['dan'] == 'PAM15' else 0, 2]) if record['active'] else [], np.int32) + state = dict(step=np.full(len(ids), 10, np.int32), neuron_idx=ids, + counts=np.bincount(ids, minlength=6).astype(np.int32), v=np.zeros(6, np.float64), + g=np.zeros(6, np.float64), ref=np.zeros(6, np.int32)) + if record['active']: + state['g'][ids[0]] = .1 + return state + + +class MockReader: + def __init__(self, values): + self.values = values + self.inventory = {str(p): dict(bytes=len(N.canonical(v)), sha256=N.sha(v)) for p, v in values.items()} + def json(self, path): return copy.deepcopy(self.values[str(path)]) + def check(self, path): return self.inventory[str(path)] + def path(self, path): return Path(path), self.inventory[str(path)] + + +def process_fixture(): + exp = Path.cwd() / 'mock-normal-evidence' + entry_id = 'input-generation'; regsha = '1' * 64 + job_path = exp / 'jobs/input-generation.json' + proc_path = exp / 'receipts/input-generation/process.json' + start_path = exp / '.append-ledger/started/input-generation.json' + entry_path = exp / '.append-ledger/entries/0000000000000001.json' + values = {str(job_path): dict(kind='sample', registration_sha256=regsha, input_sha256={})} + for label in ('stdout', 'stderr'): + values[str(exp / f'logs/input-generation.{label}.log')] = {'log': label} + values[str(proc_path)] = dict(entry_id=entry_id, registration_sha256=regsha, job_sha256=N.sha(values[str(job_path)]), + complete=True, exit_code=0, timed_out=False, exit_observed_by_parent=True, + stdout_sha256=N.sha({'log': 'stdout'}), stderr_sha256=N.sha({'log': 'stderr'})) + r = MockReader(values) + row = lambda p: dict(path=str(p), relative_path=Path(p).relative_to(exp).as_posix(), **r.inventory[str(p)]) + values[str(start_path)] = dict(entry_id=entry_id, registration_sha256=regsha, next_sequence=1, + prev_entry_sha256=None, retry_authorized=False, job=row(job_path)) + values[str(entry_path)] = dict(entry_id=entry_id, registration_sha256=regsha, sequence=1, + prev_entry_sha256=None, start_sha256=N.sha(values[str(start_path)]), consumed_inputs=[row(job_path)], + new_outputs=[row(p) for p in list(values) if p != str(start_path)], + metadata=dict(job_file=str(job_path), process_receipt_file=str(proc_path), observed_returncode=0)) + final = dict(schema='normal-append-ledger-v1-final-audit', registration_sha256=regsha, pass_all=True, + entry_ids=[entry_id], expected_entry_ids=[entry_id], expected_order_exact=True, + extra_files=[], missing_files=[], changed_files=[], unsafe_files=[], errors=[], uncommitted_entry_ids=[], + uncommitted_state_unknown=False, head_sha256=N.sha(values[str(entry_path)])) + return exp, values, final, proc_path, entry_path + + +def case_fixture(no_cs=False): + exp = Path.cwd() / 'mock-normal-case'; population_file = str(exp / 'population.npz') + seed = (1 << 62) + 17 + task = dict(task_id='real-recipe/epoch-000', kind='train', mode='physiological', arm='dan_only' if no_cs else 'paired', + dan_type='PAM15', recipe=dict(cs=None if no_cs else dict(cue='A', seed=seed, source_stream_id='independent-original'), + us=dict(requested=True), D_command=1.0)) + case = exp / 'cases' / N.task_key(task) + template = dict(kind='fullgraph', steps=1200, dt_ms=.1, apl_delay_steps=18, precision='float64', + apl_precision='float32', record_history=False, seed=98765, synaptic_tau=5.0, cs_input_seed=999) + spec = {k: v for k, v in template.items() if k != 'cs_input_seed'} + spec.update(name=N.task_key(task), steps=1200, duration_ms=120., max_steps=1200, budget_root=str(exp), + output_budget_bytes=123, mode=task['mode'], within_trial_weight_updates=False, + outer_protocol='normal-acquisition-v1', outer_task_id=task['task_id'], outer_arm=task['arm']) + descriptor = dict(mode=task['mode'], cue='A', seed=seed, steps=1200, cs_start_tick=0, + cs_stop_tick_exclusive=1000, source_stream_id='independent-original', table_id='physiological--A') + csid = None if no_cs else N.sha(descriptor) + arrays = {} + source = dict(tick=np.asarray([] if no_cs else [10], np.int32), neuron_idx=np.asarray([] if no_cs else [3], np.int32)) + dan = np.asarray([4, 5, 6, 7], np.int32) + us = dict(tick=np.full(4, 900, np.int32), neuron_idx=dan) + combined = {k: np.concatenate((source[k], us[k])) for k in source} + for name, value in (('cs_requests.npz', source), ('us_requests.npz', us), ('requests.npz', combined), + ('source-mask.npz', dict(source_mask=np.asarray(([1] if not no_cs else []) + [2]*4, np.uint8)))): + arrays[str(case / name)] = value + values = {str(case / 'spec.json'): spec, population_file: {'artifact': 'actual-static-populations'}} + source_pin = None; sampler = dict(inputs={}) + if not no_cs: + source_path = exp / f'inputs/generated/{csid}.npz'; audit_path = exp / f'inputs/generated/{csid}.json' + values[str(source_path)] = {'array-artifact': 'mock-sparse-CS'} + arrays[str(source_path)] = source + source_pin = dict(path=str(source_path), sha256=N.sha(values[str(source_path)])) + values[str(audit_path)] = dict(descriptor=descriptor, requests_sha256=source_pin['sha256'], audit=dict(seed=seed, + generator_initial_seed=seed, seed_bits=63, seed_conversion='exact integer; no truncation/modulo/offset', + private_generator=True, matrix_draw_calls=1, uniform_matrix_shape=[1200, 1], CS_window=[0, 1000])) + sampler['inputs'][csid] = dict(requests=source_pin, audit=dict(path=str(audit_path), sha256=N.sha(values[str(audit_path)]))) + values[str(case / 'input-binding.json')] = dict(task_id=task['task_id'], CS_descriptor_id=csid, CS_source=source_pin, + DAN_type='PAM15', population_file=dict(path=population_file, sha256=N.sha(values[population_file])), + DAN_membership_sha256=hashlib.sha256(dan.tobytes()).hexdigest(), composition=dict(policy='error', cross_source_conflict_count=0)) + reader = MockReader(values) + reader.arrays = lambda p: {k: v.copy() for k, v in arrays[str(p)].items()} + args = dict(template=template, inputs=dict(task_inputs={task['task_id']: csid}, descriptors={} if no_cs else {csid: descriptor}), + sampler=sampler, populations=dict(PAM15=dan), population_file=population_file, output_cap=123) + return reader, exp, task, dict(case_dir=str(case)), args, values, arrays, case + + +class NormalMetricTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.nodes, cls.pops, cls.registry, cls.authority, cls.probes, cls.arms = fixture() + cls.full_reports = {} + original = N.G.evaluate_acquisition + def capture(*args, **kwargs): + result = original(*args, **kwargs) + cls.full_reports[(result['mode'], result['dan_type'], result['conditioned_cue'])] = result + return result + with patch.object(N.G, 'evaluate_acquisition', side_effect=capture): + cls.report = N.scientific_reports(cls.probes, cls.arms, registry=cls.registry, authority=cls.authority, + nodes=cls.nodes, populations=cls.pops, load_state=state_for) + + def test_full_fixed_curves_and_eight_same_replicate_gates(self): + self.assertEqual(len(self.report['cells']), 8) + self.assertEqual(len(self.report['curves']), 1960) + self.assertTrue(self.report['cohort']['all_functional_predicates_pass']) + self.assertTrue(all(r['successful_same_replicate_conjunctions'] == 3 for r in self.report['cells'])) + self.assertTrue(all('acquisition_metric_inputs' not in r for r in self.report['cells'])) + self.assertTrue(all(not r['scientific_learning_pass_claimed'] for r in self.report['cells'])) + self.assertEqual({r['checkpoint'] for r in self.report['curves']}, set(N.POINTS)) + self.assertTrue(all(r['metrics']['rawS'] == 0 for r in self.report['curves'] if r['checkpoint'] == 0)) + self.assertTrue(all(not isinstance(v, list) for r in self.report['curves'] + for cue in r['metrics']['cues'].values() for group in cue['populations'].values() for v in group.values())) + + def test_legacy_mapping_exactly_matches_frozen_wrapper(self): + expected = N.G.legacy_h_cohort_audit(list(self.full_reports.values()), registry=self.registry, + authority=self.authority, declared_mapping=N.G.H_MAPPING) + actual = N.legacy_audit(self.report['cells'], N.G.H_MAPPING) + self.assertEqual(actual, expected) + self.assertEqual(len(actual['groups']), 4) + self.assertTrue(all(set(g['control_sources']) == {'PAM15', 'PPL103'} for g in actual['groups'])) + with self.assertRaises(ValueError): N.legacy_audit(self.report['cells'], None) + + def test_training_evaluation_seed_collision_and_missing_control_fail_closed(self): + registry = copy.deepcopy(self.registry) + block = next(iter(registry['blocks'].values())) + block['evaluation_sources'][N.M.CUES[0]]['seed'] = block['training_sources'][0]['seed'] + with self.assertRaisesRegex(ValueError, 'collision'): + N.G.validate_registry(registry, expected_sha256=N.G.descriptor_sha256(registry)) + report = next(iter(self.full_reports.values())) + rows = copy.deepcopy(report['acquisition_metric_inputs']) + del rows[0]['arms']['DAN-only'] + with self.assertRaisesRegex(ValueError, 'six actual controls'): + N.G.evaluate_acquisition(rows, assay='normal', mode=report['mode'], dan_type=report['dan_type'], + conditioned_cue=report['conditioned_cue'], registry=self.registry, authority=self.authority) + + def test_state_mutation_and_absent_probe_never_become_zero_metrics(self): + probes = dict(self.probes); del probes[next(iter(probes))] + with self.assertRaises((KeyError, ValueError)): + N.scientific_reports(probes, self.arms, registry=self.registry, authority=self.authority, + nodes=self.nodes, populations=self.pops, load_state=state_for) + def corrupt(record): + state = state_for(record); state['g'][0] = np.nan; return state + with self.assertRaisesRegex(ValueError, 'Nonfinite'): + N.scientific_reports(self.probes, self.arms, registry=self.registry, authority=self.authority, + nodes=self.nodes, populations=self.pops, load_state=corrupt) + + def test_actual_parent_receipt_nested_layout_and_exit_failure(self): + exp, values, final, proc, _ = process_fixture() + evidence = N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final) + observed = evidence.process('input-generation', 'sample'); evidence.finish() + self.assertEqual(observed['process_path'], str(exp / 'receipts/input-generation/process.json')) + values[str(proc)]['exit_code'] = 1 + # Even a caller report saying pass_all cannot rescue a real exit failure. + r = MockReader(values) + entry = next(v for v in values.values() if 'new_outputs' in v) + for row in entry['new_outputs']: + if row['path'] == str(proc): row.update(r.inventory[str(proc)]) + with self.assertRaisesRegex(ValueError, 'Actual parent exit'): + N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final).process('input-generation', 'sample') + + def test_result_process_must_be_ledger_owned_not_only_metadata(self): + exp, values, final, proc, entry_path = process_fixture() + entry = values[str(entry_path)] + entry['new_outputs'] = [r for r in entry['new_outputs'] if r['path'] != str(proc)] + with self.assertRaisesRegex(ValueError, 'not owned'): + N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], final).process('input-generation', 'sample') + bad = dict(final, uncommitted_entry_ids=['unknown-child']) + with self.assertRaisesRegex(ValueError, 'Incomplete'): + N.Evidence(MockReader(values), exp, '1' * 64, ['input-generation'], bad) + + def test_reader_per_invocation_actual_sha_cache_and_no_boolean_substitution(self): + path = Path.cwd() / 'mock-artifact.json'; raw = b'{"actual":true}' + identity = dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + with patch.object(Path, 'is_symlink', return_value=False), patch.object(Path, 'is_junction', return_value=False, create=True), \ + patch.object(Path, 'is_file', return_value=True), patch.object(Path, 'open', side_effect=lambda *a, **k: io.BytesIO(raw)) as opened: + reader = N.Reader({str(path): identity}) + reader.check(path); reader.check(path) + self.assertEqual(opened.call_count, 1) + N.Reader({str(path): identity}).check(path) + self.assertEqual(opened.call_count, 2) + with self.assertRaisesRegex(ValueError, 'Actual file checksum'): + N.Reader({str(path): dict(bytes=len(raw), sha256='0' * 64)}).check(path) + + def test_real_inventory_process_count_and_no_followup(self): + tasks = [dict(task_id='train-' + str(i), kind='train') for i in range(30000)] + tasks += [dict(task_id='probe-' + str(i), kind='probe') for i in range(3920)] + entries = N.expected_entries(dict(tasks=tasks)) + self.assertEqual(len(entries), 93921) + self.assertEqual(len(set(entries)), 93921) + self.assertEqual(entries[0], 'input-generation') + self.assertTrue(entries[-1].endswith('--brain')) + + def test_original_train_cs_us_seed_and_model_checked_independently(self): + reader, exp, task, job, args, values, arrays, case = case_fixture() + N.audit_case(reader, exp, task, job, **args) + arrays[str(case / 'us_requests.npz')]['tick'][0] = 901 + with self.assertRaisesRegex(ValueError, 'sorted unique|scheduled US'): + N.audit_case(reader, exp, task, job, **args) + arrays[str(case / 'us_requests.npz')]['tick'][0] = 900 + values[str(case / 'spec.json')]['seed'] += 1 + with self.assertRaisesRegex(ValueError, 'model/reset/clock'): + N.audit_case(reader, exp, task, job, **args) + values[str(case / 'spec.json')]['seed'] -= 1 + generated = next(v for v in values.values() if 'audit' in v) + generated['audit']['generator_initial_seed'] %= 2**32 + with self.assertRaisesRegex(ValueError, 'sampler seed'): + N.audit_case(reader, exp, task, job, **args) + + def test_no_cs_is_empty_and_dan_only_still_uses_original_fixed_us(self): + reader, exp, task, job, args, values, arrays, case = case_fixture(no_cs=True) + N.audit_case(reader, exp, task, job, **args) + arrays[str(case / 'cs_requests.npz')] = dict(tick=np.asarray([10], np.int32), neuron_idx=np.asarray([3], np.int32)) + with self.assertRaisesRegex(ValueError, 'case CS'): + N.audit_case(reader, exp, task, job, **args) + + +if __name__ == '__main__': + unittest.main() diff --git a/models/male-cns-lif/normal-conditioning-v1/test_normal_plan.py b/models/male-cns-lif/normal-conditioning-v1/test_normal_plan.py new file mode 100644 index 0000000..3d1a188 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/test_normal_plan.py @@ -0,0 +1,160 @@ +"""CPU-only fixed inventory, authority, and dependency regression tests.""" +from collections import Counter, defaultdict +import importlib.util +from pathlib import Path +import random +import sys +import unittest +from unittest.mock import patch + + +_SPEC = importlib.util.spec_from_file_location("normal_plan_test_subject", Path(__file__).with_name("normal_plan.py")) +P = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(P) + + +class NormalPlanTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Deterministic recipe construction, NOT neural random sampling. + cls.trial = P._FROZEN["trial_plan"].build_trial_plan(P.MASTER) + cls.evaluation = P.expand_pinned_evaluation(cls.trial) + with patch.object(random, "random", side_effect=AssertionError("RNG draw")), \ + patch.object(random, "Random", side_effect=AssertionError("RNG instance")): + cls.plan = P.build_normal_plan(cls.trial, cls.evaluation) + + def test_complete_inventory_and_order(self): + plan = self.plan + self.assertEqual(Counter(t["kind"] for t in plan["tasks"]), {"train": 30000, "probe": 3920}) + self.assertEqual(sum(t["steps"] for t in plan["tasks"]), 40704000) + self.assertEqual(len(plan["blocks"]), 40) + self.assertEqual(Counter((b["mode"], b["dan_type"], b["reinforced_cue"]) for b in plan["blocks"]), + {(m, d, c): 5 for m in ("physiological", "matched_energy") + for d in ("PAM15", "PPL103") for c in ("A", "B")}) + groups = defaultdict(list) + for row in plan["tasks"]: + groups[(row["block_id"], row["arm"])].append(row) + self.assertEqual(list(groups), [(b["block_id"], a) for b in self.trial["plan"]["blocks"] + for a in P._FROZEN["trial_plan"].ARMS]) + for (_, arm), rows in groups.items(): + self.assertEqual([r["kind"] for r in rows[:3]], ["probe", "probe", "train"]) + self.assertEqual([r["recipe"]["cue"] for r in rows[:2]], ["A", "B"]) + self.assertEqual(len(rows), 164 if arm == "unpaired" else 114) + self.assertEqual([r["checkpoint"] for r in rows if r["kind"] == "probe"], + [n for n in (0, 1, 2, 5, 10, 20, 50) for _ in range(2)]) + self.assertEqual(len(groups), 280) + + def test_every_dependency_is_earlier_same_arm_delta_and_probes_do_not_feed_back(self): + previous, completed = {}, Counter() + for row in self.plan["tasks"]: + key = row["block_id"], row["arm"] + expected = previous.get(key, {"kind": "zero", "source": None}) + self.assertEqual(row["delta_dependency"], expected) + self.assertTrue(row["artifact_validation_required"]) + if row["kind"] == "train": + self.assertEqual(row["epoch_index"], completed[key]) + completed[key] += 1 + self.assertEqual(row["carry_between_epochs"], ["plastic_delta_only"]) + self.assertEqual(row["neural_state_policy"], "frozen_reset_including_KC_trace_and_both_delay_queues") + self.assertTrue(row["original_E_required"] and row["cpu_validation_required"]) + previous[key] = {"kind": "epoch_delta_after", "source": row["delta_output_id"]} + else: + self.assertEqual(row["completed_epochs"], completed[key]) + self.assertEqual(row["snapshot_binding"]["completed_epochs"], completed[key]) + self.assertFalse(row["feedback_to_training_or_idle"]) + self.assertFalse(row["original_E_required"] or row["cpu_validation_required"]) + self.assertFalse(row["recipe"]["US_requested"] or row["recipe"]["weight_updates_enabled"]) + self.assertEqual(row["recipe"]["D_command"], 0.0) + self.assertIsNone(row["delta_output_id"]) + + def test_frozen_control_recipes_not_nominal_boundaries(self): + actual = {r["task_id"]: r for r in self.plan["tasks"]} + for block in self.trial["plan"]["blocks"]: + normal = next(s for s in block["suites"] if s["assay"] == "normal") + for arm in normal["arms"]: + for recipe in arm["epochs"]: + row = actual[recipe["epoch_id"]] + self.assertEqual(row["recipe"], recipe) + self.assertEqual(row["source_binding"]["record_sha256"], P._sha(recipe)) + self.assertEqual(row["source_binding"]["payload_sha256"], self.trial["payload_sha256"]) + self.assertEqual(sum(e["us"]["requested"] for e in arm["epochs"]), + 0 if arm["arm"] == "cs_only" else 50) + if arm["arm"] == "unpaired": + self.assertTrue(all(not (e["cs"] and e["us"]["requested"]) for e in arm["epochs"])) + for original in self.evaluation["probes"]: + if original["assay"] == "normal": + self.assertEqual(actual[original["probe_id"]]["recipe"], original) + self.assertTrue(any(r["kind"] == "probe" and r["arm"] == "unpaired" and + r["checkpoint"] == 50 and r["completed_epochs"] == 150 + for r in actual.values())) + + def test_evaluation_inputs_reused_not_redrawn_or_counted_as_replicates(self): + inputs = self.plan["evaluation_inputs"] + self.assertEqual(len(inputs), 80) + self.assertEqual(len({r["stream_id"] for r in inputs}), 80) + seeds = {r["seed"] for r in inputs} + training = {r["recipe"]["cs"]["seed"] for r in self.plan["tasks"] + if r["kind"] == "train" and r["recipe"]["cs"] is not None} + self.assertFalse(seeds & training) + self.assertTrue(all(type(s) is int and 0 <= s < 2**63 for s in seeds | training)) + reuse = Counter(r["evaluation_input_id"] for r in self.plan["tasks"] if r["kind"] == "probe") + self.assertEqual(set(reuse.values()), {49}) + self.assertEqual(set(reuse), {r["input_id"] for r in inputs}) + self.assertTrue(all(not r["recipe"]["adds_training_replicate"] for r in self.plan["tasks"] if r["kind"] == "probe")) + self.assertEqual(self.plan["new_seed_streams"], 0) + self.assertFalse(self.plan["learning_pass_claimed"] or self.plan["execution_authorized"]) + + def test_no_mutable_alias_to_authorities(self): + row = next(r for r in self.plan["tasks"] if r["kind"] == "train") + before = row["recipe"]["D_command"] + try: + row["recipe"]["D_command"] = 999 + original = self.trial["plan"]["blocks"][0]["suites"][0]["arms"][0]["epochs"][0] + self.assertEqual(original["D_command"], before) + finally: + row["recipe"]["D_command"] = before + self.assertIsNot(self.plan["provenance"]["frozen_source_sha256"], P.SOURCE_PINS) + + def test_private_loader_ignores_ordinary_module_cache_and_fails_closed_on_source_pin(self): + sentinels = {name: object() for name in ("seed_registry", "trial_plan", "evaluation_plan", "evaluation_policy", "R", "E")} + with patch.dict(sys.modules, sentinels): + loaded = P._load_frozen() + self.assertIs(loaded["trial_plan"].seed_registry, loaded["seed_registry"]) + self.assertIs(loaded["evaluation_policy"].evaluation_plan, loaded["evaluation_plan"]) + self.assertTrue(all(sys.modules[name] is value for name, value in sentinels.items())) + with patch.object(Path, "read_bytes", return_value=b"unregistered mutation"): + with self.assertRaisesRegex(ValueError, "Frozen source pin mismatch"): + P._load_frozen() + + def test_original_trial_and_policy_mutation_rejected(self): + master = self.trial["plan"]["master"] + try: + self.trial["plan"]["master"] = master + 1 + with self.assertRaisesRegex(ValueError, "master"): + P.build_normal_plan(self.trial, self.evaluation) + finally: + self.trial["plan"]["master"] = master + binding = self.evaluation["config"]["snapshot_bindings"][1] + count = binding["completed_epochs"] + try: + binding["completed_epochs"] += 1 + with self.assertRaisesRegex(ValueError, "exactly expand"): + P.build_normal_plan(self.trial, self.evaluation) + finally: + binding["completed_epochs"] = count + + def test_validates_exact_reconstruction_and_rejects_rehashed_task_mutation(self): + self.assertTrue(P.validate_normal_plan(self.plan, self.trial, self.evaluation)) + row, original_hash = self.plan["tasks"][0], self.plan["payload_sha256"] + try: + row["feedback_to_training_or_idle"] = True + self.plan["payload_sha256"] = P._sha({k: v for k, v in self.plan.items() if k != "payload_sha256"}) + with self.assertRaisesRegex(ValueError, "exact reconstruction"): + P.validate_normal_plan(self.plan, self.trial, self.evaluation) + finally: + row["feedback_to_training_or_idle"] = False + self.plan["payload_sha256"] = original_hash + + +if __name__ == "__main__": + unittest.main() diff --git a/models/male-cns-lif/normal-conditioning-v1/test_normal_runner.py b/models/male-cns-lif/normal-conditioning-v1/test_normal_runner.py new file mode 100644 index 0000000..0fbe9e8 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/test_normal_runner.py @@ -0,0 +1,371 @@ +"""Pure temporary-file and mock-process tests; no sampler, Torch, brain or GPU.""" +import copy +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np +import normal_runner as R + + +def load_readonly(name, path): + spec=importlib.util.spec_from_file_location(name,path) + module=importlib.util.module_from_spec(spec) + exec(compile(path.read_bytes(),str(path),'exec'),module.__dict__) + return module + + +LAB=Path(__file__).resolve().parent.parent/'olfactory-conditioning-lab-v1' +COMPOSER=load_readonly('_runner_test_original_composer',LAB/'conditioning_inputs.py') +WORKER=load_readonly('_runner_test_original_worker',LAB.parent/'conditioning-interface-v2/isolated_workers.py') + + +def task(arm='paired', *, requested=True, kind='train', dan='PAM15', number=1): + value=dict(task_id=f'block/normal/{arm}/epoch-{number:03d}',kind=kind,steps=1200, + block_id='block',arm=arm,mode='physiological',dan_type=dan, + compartment_verbatim="y5B'2a" if dan=='PAM15' else "y2a'1", + delta_dependency=dict(kind='zero',source=None),feedback_to_training_or_idle=kind=='train') + if kind=='probe': + value['recipe']=dict(US_requested=False,D_command=0,weight_updates_enabled=False) + else: + value['recipe']=dict(us=dict(nominal_slot_tick=900,rng_seed=None,rng_draws=0, + requested=requested,tick=900 if requested else None), + D_command=1. if requested and arm!='d_zero' else 0.,update_enabled=arm!='frozen') + return value + + +class RunnerTests(unittest.TestCase): + def setUp(self): + temporary=tempfile.TemporaryDirectory();self.addCleanup(temporary.cleanup) + self.root=Path(temporary.name).resolve() + for rel in ('protocol','jobs','cases','runs','updates','validation','results','receipts','logs','audit'): + (self.root/rel).mkdir() + self.addCleanup(patch.stopall) + patch.object(R,'EXP',self.root).start() + self.regfile=self.root/'protocol/preregistered.json';R.save(self.regfile,{'synthetic':True}) + self.sha=R.digest(self.regfile) + self.w=R.Workflow.__new__(R.Workflow);self.w.sha=self.sha + self.w.start=time.perf_counter();self.w.completed=set();self.w.expected=[];self.w.tasks={} + self.w.overlay_dependency_pins={};self.w.case_dependency_pins={} + self.w.reg=dict(budget=dict(output_cap_bytes=8<<20,minimum_free_reserve_bytes=0, + workflow_timeout_seconds=60,child_timeout_seconds=10,not_after_utc='2099-01-01T00:00:00+00:00', + next_child_headroom_bytes=65536),runtime_input_sha256={}, + input_sha256={},initial_overlay=dict(overlay_file=str(self.root/'protocol/zero.npz'), + overlay_manifest=str(self.root/'protocol/zero.json'))) + R.save_npz(self.root/'protocol/zero.npz',dict(plastic_delta_mv=np.zeros(2,np.float32))) + R.save(self.root/'protocol/zero.json',dict(synthetic=True)) + initial={p.relative_to(self.root).as_posix():R.identity(p) for p in (self.root/'protocol').iterdir()} + self.w.ledger=R.Ledger(self.root,self.sha,initial,8<<20,0) + + def test_original_composer_no_US_D0_frozen_and_PPL_requests(self): + cs=dict(tick=np.array([10,20],np.int32),neuron_idx=np.array([3,4],np.int32)) + for arm,requested,dan in (('paired',False,'PAM15'),('d_zero',True,'PAM15'), + ('frozen',True,'PAM15'),('paired',True,'PPL103')): + item=task(arm,requested=requested,dan=dan) + ids=np.array([10,11,12,13] if dan=='PAM15' else [14,15],np.int32) + composed,us=R.compose_requests(cs,ids,item,COMPOSER) + self.assertEqual(len(us['tick']),len(ids) if requested else 0) + self.assertEqual(composed['audit']['US_attempts'],len(us['tick'])) + self.assertEqual(item['recipe']['D_command'],0. if not requested or arm=='d_zero' else 1.) + self.assertEqual(item['recipe']['update_enabled'],arm!='frozen') + if requested:self.assertTrue(np.array_equal(us['neuron_idx'],ids)) + bad=task('d_zero');bad['recipe']['D_command']=1. + with self.assertRaisesRegex(ValueError,'D/update'):R.compose_requests(cs,ids,bad,COMPOSER) + + def test_probe_refuses_US_update_or_feedback(self): + cs=dict(tick=np.array([7],np.int32),neuron_idx=np.array([3],np.int32)) + value=task(kind='probe');combined,us=R.compose_requests(cs,np.array([9,10],np.int32),value,COMPOSER) + self.assertEqual(len(us['tick']),0);self.assertEqual(combined['audit']['US_attempts'],0) + for role in ('US_requested','weight_updates_enabled','feedback_to_training_or_idle'): + bad=copy.deepcopy(value) + (bad if role=='feedback_to_training_or_idle' else bad['recipe'])[role]=True + with self.assertRaisesRegex(ValueError,'Probe command'):R.compose_requests(cs,np.array([9,10],np.int32),bad,COMPOSER) + + def test_frozen_clock_and_reset_seed_retained(self): + original=dict(kind='fullgraph',dt_ms=.1,apl_delay_steps=18,precision='float64', + apl_precision='float32',record_history=False,seed=1234,training=False,cs_input_seed=999) + actual=R.derive_spec(original,task(),budget_root=self.root,output_cap=123456) + self.assertEqual(actual['seed'],1234);self.assertEqual(actual['steps'],1200) + self.assertEqual(actual['duration_ms'],120.);self.assertNotIn('cs_input_seed',actual) + self.assertFalse(actual['within_trial_weight_updates']);self.assertFalse(original['training']) + with self.assertRaisesRegex(ValueError,'fixed D18'):R.derive_spec(dict(original,apl_delay_steps=0),task(),budget_root=self.root,output_cap=1) + + def test_actual_successful_parent_receipt_commits_ledger(self): + output=self.root/'runs/synthetic.txt' + job=dict(kind='cpu_verify',input_sha256={str(self.regfile):self.sha}) + def fake_run(argv,**kwargs): + self.assertEqual(kwargs['env']['CUDA_VISIBLE_DEVICES'],'') + self.assertEqual(kwargs['env']['PYTHONDONTWRITEBYTECODE'],'1') + kwargs['stdout'].write(b'synthetic child only\n');output.write_bytes(b'new immutable output') + return SimpleNamespace(returncode=0) + with patch.object(R.subprocess,'run',side_effect=fake_run) as run: + self.w.launch('unit',job,['not-a-real-executable'],[output]) + self.assertEqual(run.call_count,1);self.assertIn('unit',self.w.completed) + self.assertTrue(self.w.ledger.final_verify(['unit'])['pass_all']) + process=R.read(self.root/'receipts/unit/process.json') + self.assertEqual(process['exit_code'],0);self.assertTrue(process['complete']) + with self.assertRaisesRegex(ValueError,'rerun committed'): + self.w.launch('unit',job,['not-real'],[output]) + + def test_failed_child_is_uncommitted_unknown_and_cannot_rerun(self): + job=dict(kind='cpu_verify',input_sha256={str(self.regfile):self.sha}) + with patch.object(R.subprocess,'run',return_value=SimpleNamespace(returncode=9)): + with self.assertRaisesRegex(ValueError,'Child failed'):self.w.launch('failed',job,['not-real'],[]) + receipt=R.read(self.root/'receipts/failed/process.json') + self.assertFalse(receipt['complete']);self.assertEqual(receipt['exit_code'],9) + self.assertEqual(self.w.ledger.inspect()['uncommitted_entry_ids'],['failed']) + with self.assertRaises(FileExistsError):self.w.launch('failed',job,['not-real'],[]) + + def test_timed_out_child_never_gets_success_commit(self): + job=dict(kind='cpu_verify',input_sha256={str(self.regfile):self.sha}) + with patch.object(R.subprocess,'run',side_effect=subprocess.TimeoutExpired('not-real',10)): + with self.assertRaisesRegex(ValueError,'Child failed'):self.w.launch('timeout',job,['not-real'],[]) + receipt=R.read(self.root/'receipts/timeout/process.json') + self.assertTrue(receipt['timed_out']);self.assertIsNone(receipt['exit_code']) + self.assertNotIn('timeout',self.w.completed) + + def test_spawn_OSerror_retains_failed_receipt_without_fake_exit(self): + job=dict(kind='cpu_verify',input_sha256={str(self.regfile):self.sha}) + with patch.object(R.subprocess,'run',side_effect=OSError('synthetic spawn failure')): + with self.assertRaisesRegex(ValueError,'Child failed'):self.w.launch('spawn-error',job,['not-real'],[]) + receipt=R.read(self.root/'receipts/spawn-error/process.json') + self.assertFalse(receipt['complete']);self.assertIsNone(receipt['exit_code']) + self.assertFalse(receipt['exit_observed_by_parent']);self.assertIn('synthetic spawn failure',receipt['launch_error']) + self.assertEqual(self.w.ledger.inspect()['uncommitted_entry_ids'],['spawn-error']) + + def test_after_process_input_mutation_stops_before_commit(self): + dependency=self.root/'protocol/input.json';R.save(dependency,dict(value=1)) + job=dict(kind='cpu_verify',input_sha256={str(dependency):R.digest(dependency)}) + def fake_run(*a,**k): + dependency.write_bytes(b'{"value":2}');return SimpleNamespace(returncode=0) + with patch.object(R.subprocess,'run',side_effect=fake_run): + with self.assertRaisesRegex(ValueError,'Pinned input changed'):self.w.launch('mutation',job,['not-real'],[]) + self.assertNotIn('mutation',self.w.completed) + + def _delta_fixture(self): + old=task();new=task(number=2);new['delta_dependency']=dict(kind='epoch_delta_after',source=old['task_id']+'/delta-after') + self.w.tasks={old['task_id']:old};key=R.task_key(old) + self.w.completed.add(key+'--verify') + output=self.root/'updates'/key;output.mkdir() + R.save_npz(output/'delta-after.npz',dict(plastic_delta_mv=np.array([-.25,0.],np.float32))) + R.save(output/'overlay-manifest.json',dict(synthetic=True)) + validation=self.root/'validation/previous.json';R.save(validation,dict(complete=True, + after_overlay_sha256=R.digest(output/'delta-after.npz'),after_manifest_sha256=R.digest(output/'overlay-manifest.json'))) + process=self.root/f'receipts/{key}--verify/process.json';process.parent.mkdir() + R.save(process,dict(complete=True,exit_code=0, + entry_id=key+'--verify',registration_sha256=self.sha)) + overlay=dict(overlay_file=str(output/'delta-after.npz'),overlay_manifest=str(output/'overlay-manifest.json')) + result=self.root/f'results/{key}.json' + R.save(result,dict(complete=True,task=old,cpu_validation=R.pin(validation),after_overlay=overlay, + cpu_process_receipt_path=str(process),update_dir=str(output))) + for path in (result,validation,process,output/'delta-after.npz',output/'overlay-manifest.json'): + self.w.ledger._tracked[path.relative_to(self.root).as_posix()]=R.identity(path) + return old,new,overlay,result,validation + + def test_delta_dependency_requires_same_branch_completed_verify_and_receipt(self): + old,new,overlay,result,validation=self._delta_fixture() + self.w.completed.clear() + with self.assertRaisesRegex(ValueError,'CPU validation missing'):self.w.overlay(new) + self.w.completed.add(R.task_key(old)+'--verify') + self.assertEqual(self.w.overlay(new),overlay) + self.assertIn(str(result),self.w.overlay_dependency_pins) + self.assertIn(str(validation),self.w.overlay_dependency_pins) + for field in ('arm','block_id'): + altered=copy.deepcopy(new);altered[field]='unrelated' + with self.assertRaisesRegex(ValueError,'unrelated branch'):self.w.overlay(altered) + validation.write_bytes(b'{}') + with self.assertRaisesRegex(ValueError,'committed bytes'):self.w.overlay(new) + + def test_alternate_valid_overlay_cannot_replace_previous_verified_delta(self): + _,new,_,result,_=self._delta_fixture() + alternative=self.root/'updates/other-valid';alternative.mkdir() + R.save_npz(alternative/'delta-after.npz',dict(plastic_delta_mv=np.array([-.5,0.],np.float32))) + R.save(alternative/'overlay-manifest.json',dict(synthetic=True,other=True)) + for path in alternative.iterdir():self.w.ledger._tracked[path.relative_to(self.root).as_posix()]=R.identity(path) + record=R.read(result);record['update_dir']=str(alternative) + record['after_overlay']={k:str(alternative/name) for k,name in + (('overlay_file','delta-after.npz'),('overlay_manifest','overlay-manifest.json'))} + result.write_text(R.canonical(record),encoding='utf-8') + with self.assertRaisesRegex(ValueError,'committed bytes'):self.w.overlay(new) + # Even a synthetic "committed" bad result must fail its independent CPU output binding. + self.w.ledger._tracked[result.relative_to(self.root).as_posix()]=R.identity(result) + with self.assertRaisesRegex(ValueError,'independently verified output'):self.w.overlay(new) + + def test_zero_dependency_does_not_carry_previous_lineage_pins(self): + self.w.overlay_dependency_pins={'stale':'0'*64} + self.assertEqual(self.w.overlay(task()),self.w.reg['initial_overlay']) + self.assertEqual(self.w.overlay_dependency_pins,{}) + + def _case_fixture(self): + item=task();template=self.root/'protocol/case-template.json' + R.save(template,dict(kind='fullgraph',dt_ms=.1,apl_delay_steps=18,precision='float64', + apl_precision='float32',record_history=False,seed=1234)) + population=self.root/'protocol/populations.npz' + self.w.populations=dict(PAM15=np.array([10,11,12,13],np.int32),PPL103=np.array([14,15],np.int32)) + R.save_npz(population,self.w.populations);self.w.reg['population_file']=str(population) + self.w.reg['input_sha256'][str(population)]=R.digest(population) + directory=self.root/'inputs/generated';directory.mkdir(parents=True) + source=directory/'source.npz';audit=directory/'source.json' + R.save_npz(source,dict(tick=np.array([5],np.int32),neuron_idx=np.array([3],np.int32))) + descriptor=dict(synthetic=True) + R.save(audit,dict(descriptor=descriptor,requests_sha256=R.digest(source))) + self.w.input_plan=dict(task_inputs={item['task_id']:'source'},descriptors={'source':descriptor}) + self.w.inputs=COMPOSER + for path in (template,population,source,audit): + self.w.ledger._tracked[path.relative_to(self.root).as_posix()]=R.identity(path) + return item,template,source,audit + + def test_prepare_case_reads_only_committed_template_and_shared_CS(self): + item,template,source,audit=self._case_fixture() + directory=self.w.prepare_case(item) + self.assertEqual(R.read(directory/'spec.json')['seed'],1234) + self.assertTrue({str(template),str(source),str(audit)} <= set(self.w.case_dependency_pins)) + self.assertEqual(R.npz(directory/'us_requests.npz')['tick'].tolist(),[900]*4) + + def test_changed_case_template_is_rejected_before_new_spec_write(self): + item,template,_,_=self._case_fixture();value=R.read(template);value['seed']=9876 + template.write_text(R.canonical(value),encoding='utf-8') + with self.assertRaisesRegex(ValueError,'committed bytes'):self.w.prepare_case(item) + self.assertFalse((self.root/'cases'/R.task_key(item)/'spec.json').exists()) + + def test_changed_CS_and_self_rehashed_audit_are_both_rejected(self): + item,_,source,audit=self._case_fixture() + with source.open('wb') as stream:np.savez_compressed(stream,tick=np.array([9],np.int32),neuron_idx=np.array([4],np.int32)) + value=R.read(audit);value['requests_sha256']=R.digest(source) + audit.write_text(R.canonical(value),encoding='utf-8') + with self.assertRaisesRegex(ValueError,'committed bytes'):self.w.prepare_case(item) + self.assertFalse((self.root/'cases'/R.task_key(item)/'requests.npz').exists()) + + def test_exclusive_helpers_reject_linked_parent_before_writes(self): + destination=self.root/'external';destination.mkdir();link=self.root/'alias' + try:link.symlink_to(destination,target_is_directory=True) + except OSError:self.skipTest('Platform disallows temporary symlinks') + for filename,write in (('x.json',lambda p:R.save(p,{'a':1})), + ('x.npz',lambda p:R.save_npz(p,{'a':np.array([1],np.int32)}))): + with self.assertRaisesRegex(ValueError,'linked path'):write(link/filename) + self.assertFalse((destination/filename).exists()) + + def _mock_task(self,item): + """Exercise real orchestration/control closures, with no scientific child.""" + validator=self.root/'validator.py';validator.write_bytes(b'# not executed\n') + self.w.reg['input_sha256'][str(validator)]=R.digest(validator) + patch.object(R,'AQ',self.root).start() + target=self.root/'validate_update_cpu.py';target.write_bytes(validator.read_bytes()) + self.w.reg['input_sha256'][str(target)]=R.digest(target) + for name in ('conditioning_epoch.py','conditioning_preflight_compare.py','engineering_compare.py'): + self.w.reg['input_sha256'][str(LAB/name)]=R.digest(LAB/name) + R.save(self.root/'protocol/epoch-template.json',dict(schema='synthetic',files={},frozen_inputs={})) + self.w.ledger._tracked['protocol/epoch-template.json']=R.identity(self.root/'protocol/epoch-template.json') + calls=[];authorities=[] + def case(task_value): + result=self.root/'cases'/R.task_key(task_value);result.mkdir() + R.save(result/'spec.json',dict(steps=1200));R.save_npz(result/'requests.npz',dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32))) + return result + def ap_job(name,kind,output,**kwargs): + return dict(kind=kind,output_dir=str(output),receipt_file=str(self.root/f'receipts/{name}/worker.json'), + input_sha256={},authority_file=str(kwargs.get('authority',''))) + def launch(name,job,argv,outputs,*,pre_outputs=(),post=None,ap=False): + calls.append(job['kind']) + if job['kind']=='brain': + out=Path(job['output_dir']);out.mkdir();R.save(out/'checkpoint-manifest.json',dict(synthetic=True)) + R.save_npz(out/'state.npz',dict(step=np.empty(0,np.int32))) + for filename in ('process.json','worker.json'): + receipt=self.root/f'receipts/{name}/{filename}';receipt.parent.mkdir(exist_ok=True) + R.save(receipt,dict(synthetic=True)) + self.w.ledger._tracked[receipt.relative_to(self.root).as_posix()]=R.identity(receipt) + elif job['kind']=='update': + authorities.append(R.read(job['authority_file'])) + out=Path(job['output_dir']);out.mkdir() + R.save_npz(out/'delta-after.npz',dict(plastic_delta_mv=np.zeros(2,np.float32))) + R.save(out/'overlay-manifest.json',{});R.save(out/'chain-receipt.json',{}) + else: + out=Path(job['update_dir']) + R.save(job['receipt_file'],dict(complete=True,frozen_U_verify_receipt_recomputed=True, + actual_sealed_events_used=True,after_overlay_sha256=R.digest(out/'delta-after.npz'), + epoch_authority_sha256=job['authority_sha256'])) + if post is not None:post() + self.w.completed.add(name) + with patch.object(self.w,'prepare_case',side_effect=case),patch.object(self.w,'ap_job',side_effect=ap_job),\ + patch.object(self.w,'launch',side_effect=launch),patch.object(self.w,'brain_verify',return_value={'synthetic':True}): + self.w.task(item) + return calls,authorities,R.read(self.root/f'results/{R.task_key(item)}.json') + + def test_probe_orchestration_never_calls_E_or_CPU_update(self): + calls,authorities,record=self._mock_task(task(kind='probe')) + self.assertEqual(calls,['brain']);self.assertEqual(authorities,[]) + self.assertNotIn('after_overlay',record);self.assertNotIn('cpu_validation',record) + + def test_training_no_US_preserves_None_and_zero_D(self): + calls,authorities,record=self._mock_task(task(requested=False)) + self.assertEqual(calls,['brain','update','cpu_verify']) + self.assertIsNone(authorities[0]['command']['us_tick']) + self.assertEqual(authorities[0]['command']['d_command'],0.) + self.assertFalse(authorities[0]['command']['frozen']);self.assertIn('after_overlay',record) + + def test_D0_keeps_US_but_passes_zero_D_to_original_E(self): + calls,authorities,_=self._mock_task(task('d_zero')) + self.assertEqual(calls,['brain','update','cpu_verify']) + self.assertEqual(authorities[0]['command']['us_tick'],900) + self.assertEqual(authorities[0]['command']['d_command'],0.) + self.assertFalse(authorities[0]['command']['frozen']) + + def test_frozen_keeps_D_request_but_explicitly_disables_update(self): + _,authorities,_=self._mock_task(task('frozen')) + self.assertEqual(authorities[0]['command']['d_command'],1.) + self.assertTrue(authorities[0]['command']['frozen']) + + def test_PPL_compartment_not_replaced_by_PAM_or_sign_flip(self): + _,authorities,_=self._mock_task(task(dan='PPL103')) + self.assertEqual(authorities[0]['command']['compartment'],"y2a'1") + self.assertEqual(authorities[0]['command']['d_command'],1.) + self.assertEqual(authorities[0]['command']['trace_backend'],'torch') + + def test_partial_task_cannot_silently_resume_remaining_children(self): + item=task();self.w.completed.add(R.task_key(item)+'--brain') + with patch.object(self.w,'launch') as launch: + with self.assertRaisesRegex(ValueError,'Partially completed task'):self.w.task(item) + launch.assert_not_called() + + def test_real_AP_validation_accepts_brain_job_and_rejects_protected_outputs(self): + # Actual frozen parser only, using small synthetic files, no runtime load. + frozen=self.root/'frozen';frozen.mkdir();graph=frozen/'graph';graph.mkdir();apl=frozen/'apl';apl.mkdir() + for name in WORKER.GRAPH_FILES:(graph/name).write_bytes(b'synthetic graph bytes') + for name in WORKER.APL_FILES:(apl/name).write_bytes(b'synthetic APL bytes') + for name in ('mask.npy','base.npy','coeff.json'):(frozen/name).write_bytes(b'synthetic authority') + case=self.root/'cases/first';case.mkdir() + R.save(case/'spec.json',dict(graph_dir=str(graph),apl_bundle_dir=str(apl),coefficients_path=str(frozen/'coeff.json'))) + R.save_npz(case/'requests.npz',dict(tick=np.empty(0,np.int32),neuron_idx=np.empty(0,np.int32))) + overlay=self.w.reg['initial_overlay'];manifest=Path(overlay['overlay_manifest']) + am=LAB.parent/'wholebrain-plastic-interface-v1/overlay.py' + manifest.write_text(R.canonical(dict(mask_path=str(frozen/'mask.npy'),mask_sha256=R.digest(frozen/'mask.npy'), + base_candidate_path=str(frozen/'base.npy'),base_candidate_sha256=R.digest(frozen/'base.npy'), + am_overlay_path=str(am),am_overlay_sha256=R.digest(am))),encoding='utf-8') + self.w.reg['runtime_input_sha256']={p:v['sha256'] for p,v in R.inventory(frozen).items()} + self.w.reg['runtime_input_sha256'][str(am)]=R.digest(am);self.w.h=WORKER + name='parser-brain';job=self.w.ap_job(name,'brain',self.root/'runs/first',case=case,overlay=overlay) + Path(job['receipt_file']).parent.mkdir() + file=self.root/'jobs/parser.json';R.save(file,job) + parsed,paths,_,_=WORKER.validate_job(file,R.digest(file),'brain') + self.assertEqual(parsed['diagnostic_mode'],'light');self.assertFalse(paths['output'].exists()) + for field in ('output_dir','receipt_file','build_root'): + altered=copy.deepcopy(job);altered[field]=str(case/('forbidden-'+field)) + alternative=self.root/'jobs'/(field+'.json');R.save(alternative,altered) + with self.assertRaisesRegex(ValueError,'protected input/source'): + WORKER.validate_job(alternative,R.digest(alternative),'brain') + self.assertFalse((case/'forbidden-output_dir').exists()) + + def test_separate_receipt_directories_allow_original_worker_lineage_protection(self): + # Real AP path validator should not interpret a previous receipt as + # authorizing writes into its parent. New receipts must be siblings. + prior=self.root/'receipts/previous';prior.mkdir() + previous=prior/'process.json';R.save(previous,dict(synthetic=True)) + self.w.overlay_dependency_pins={str(previous):R.digest(previous)} + self.test_real_AP_validation_accepts_brain_job_and_rejects_protected_outputs() + + +if __name__=='__main__':unittest.main() diff --git a/models/male-cns-lif/normal-conditioning-v1/test_run_normal.py b/models/male-cns-lif/normal-conditioning-v1/test_run_normal.py new file mode 100644 index 0000000..6032a34 --- /dev/null +++ b/models/male-cns-lif/normal-conditioning-v1/test_run_normal.py @@ -0,0 +1,37 @@ +"""Pure temporary-file supervisor bookkeeping tests, no actual child launch.""" +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import run_normal as R + +class SupervisorTests(unittest.TestCase): + def test_launch_is_exclusive_not_completion(self): + with tempfile.TemporaryDirectory() as temp,patch.object(R,'CONTROL',Path(temp)),patch.object(R,'authority',return_value={}): + with patch.object(R.subprocess,'Popen',return_value=SimpleNamespace(pid=123)) as spawn: + record=R.launch('a'*64) + self.assertEqual(spawn.call_count,1) + self.assertTrue(spawn.call_args.kwargs['start_new_session']) + self.assertFalse(record['final_exit_observed']) + self.assertFalse((Path(temp)/'exit.json').exists()) + with self.assertRaises(ValueError):R.launch('a'*64) + self.assertEqual(spawn.call_count,1) + + def test_final_exit_is_parent_observed_not_learning(self): + with tempfile.TemporaryDirectory() as temp,patch.object(R,'CONTROL',Path(temp)),patch.object(R,'authority',return_value={}): + with patch.object(R.subprocess,'run',return_value=SimpleNamespace(returncode=0)): + self.assertEqual(R.supervise('b'*64),0) + record=R.read(Path(temp)/'exit.json') + self.assertTrue(record['complete'] and record['final_exit_observed']) + self.assertFalse(record['learning_success_claimed']) + + def test_failed_spawn_has_no_fabricated_exit(self): + with tempfile.TemporaryDirectory() as temp,patch.object(R,'CONTROL',Path(temp)),patch.object(R,'authority',return_value={}): + with patch.object(R.subprocess,'run',side_effect=OSError('fixture')): + self.assertEqual(R.supervise('b'*64),1) + record=R.read(Path(temp)/'exit.json') + self.assertIsNone(record['exit_code']) + self.assertFalse(record['complete'] or record['final_exit_observed']) + +if __name__=='__main__':unittest.main()