Add isolated CPU residency performance probe without changing AR

2026-09-30 23:23:22 +08:00
parent e7b2be7ee0
commit 7f341d7ffb
4 changed files with 784 additions and 0 deletions
@@ -0,0 +1,370 @@
+"""Bounded CPU-only residency experiment; never an AR runtime replacement."""
+from __future__ import annotations
+import argparse
+import contextlib
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import selectors
+import shutil
+import statistics
+import subprocess
+import sys
+import time
+import traceback
+
+ROOT = Path('/data/digital-drosophila')
+LAB = ROOT/'models/male-cns-lif/olfactory-conditioning-lab-v1'
+AQ = ROOT/'experiments/phase4aq-conditioning-execution'
+VALIDATOR = ROOT/'models/male-cns-lif/conditioning-execution-v1/validate_update_cpu.py'
+E_SHA = '757935a4815d9f625640df685184ed3fe44a8845323562796da7aa43cbb194e1'
+V_SHA = '44e95eca9e39e9cd25a1d5d2d46b2cea76e839143865a196f865a61fac700b9f'
+MANIFEST_SHA = '2dc0feca75f718f2ee0d7c9240da503221823573fd25f1344d2fd97d97eb2db7'
+FIXTURES = {
+    'M': ('mechanical--fresh-parent', '4616a42c255ab839346368aa0e5e4306bb3878e55fcc95b176f058b1993c9b0c'),
+    'N': ('performance--natural-light-fresh', '785e3d26d06b47efe7f054883d38fe2e13711195359bf9f4b03aa88887fbfb37'),
+}
+SEQUENCE = ('M', 'N', 'M', 'N_D0', 'N_frozen', 'M_D0', 'M')
+UPDATE_FILES = ('delta-after.npz', 'overlay-manifest.json', 'update-receipt.json', 'chain-receipt.json')
+TIMEOUT = 180
+
+
+def require(ok, message):
+    if not ok:
+        raise ValueError(message)
+
+
+def digest(path):
+    h = hashlib.sha256()
+    with Path(path).open('rb') as stream:
+        for chunk in iter(lambda: stream.read(8 << 20), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False)
+
+
+def read(path):
+    return json.loads(Path(path).read_text(encoding='utf-8'))
+
+
+def save(path, value):
+    with Path(path).open('x', encoding='utf-8', newline='\n') as stream:
+        json.dump(value, stream, sort_keys=True, indent=2, allow_nan=False)
+        stream.write('\n')
+
+
+def safe(path):
+    require('..' not in Path(path).parts, 'Parent traversal forbidden')
+    path = Path(path).absolute()
+    require(not any(p.is_symlink() for p in (path, *path.parents)), 'Symlink path forbidden')
+    return path.resolve(strict=False)
+
+
+def pinned_load(name, path, sha):
+    path = safe(path)
+    raw = path.read_bytes()
+    require(hashlib.sha256(raw).hexdigest() == sha and name not in sys.modules, 'Source/module identity differs')
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    exec(compile(raw, str(path), 'exec'), module.__dict__)
+    return module
+
+
+def worker(role, mode, plan_file, plan_sha):
+    require(sys.dont_write_bytecode and os.environ.get('CUDA_VISIBLE_DEVICES') == '', 'CPU-only -B worker required')
+    require(digest(plan_file) == plan_sha, 'External plan SHA mismatch')
+    plan = read(plan_file)
+    out = safe(plan['output_root'])
+    pins = plan['prototype_sources']
+    require(all(digest(p) == s for p, s in pins.items()), 'Prototype source changed')
+    module = None
+    seq = 0
+    print(canonical({'ready': True, 'pid': os.getpid(), 'role': role}), flush=True)
+    for line in sys.stdin:
+        job = json.loads(line)
+        require(set(job) == {'sequence', 'authority', 'authority_sha256', 'output', 'update_dir', 'chain_sha256'}, 'Wrong request fields')
+        require(seq < len(SEQUENCE) and job['sequence'] == seq and
+            job['authority'] in plan['authorities'].values() and
+            plan['input_sha256'][job['authority']] == job['authority_sha256'] and
+            digest(job['authority']) == job['authority_sha256'], 'Sequence/authority differs')
+        output = safe(job['output'])
+        require(output.is_relative_to(out) and not output.exists() and output.parent.is_dir(), 'Output must be exclusive and within experiment')
+        started = time.perf_counter()
+        with contextlib.redirect_stdout(sys.stderr):
+            if module is None:
+                if role == 'update':
+                    module = pinned_load('_perf_E', LAB/'conditioning_epoch.py', E_SHA)
+                elif role == 'verify-reference':
+                    module = pinned_load('_perf_original_V', VALIDATOR, V_SHA)
+                else:
+                    source = Path(__file__).parent/'resident_validator.py'
+                    module = pinned_load('_perf_resident_V', source, pins[str(source)])
+            if role == 'update':
+                module.postprocess_epoch(job['authority'], job['authority_sha256'], output)
+                hashes = {name: digest(output/name) for name in UPDATE_FILES}
+            else:
+                module.validate(job['authority'], job['authority_sha256'], job['update_dir'], job['chain_sha256'], output)
+                hashes = {'receipt': digest(output)}
+            import torch
+            require(os.environ['CUDA_VISIBLE_DEVICES'] == '' and not torch.cuda.is_initialized(), 'CUDA use forbidden')
+            require('runtime_light' not in sys.modules, 'Ordinary runtime module collision')
+        require(all(digest(p) == s for p, s in pins.items()), 'Prototype source changed during task')
+        print(canonical({'ack': True, 'sequence': seq, 'pid': os.getpid(), 'role': role,
+            'request_sha256': hashlib.sha256(canonical(job).encode()).hexdigest(),
+            'output_sha256': hashes, 'task_seconds': time.perf_counter()-started,
+            'CUDA_initialized': False, 'task_ack_is_not_process_exit': True}), flush=True)
+        seq += 1
+        if mode == 'once':
+            break
+
+
+class Process:
+    def __init__(self, role, mode, plan, root, label, deadline):
+        self.role, self.mode, self.seq = role, mode, 0
+        self.deadline = deadline
+        self.label, self.root, self.records = label, root, []
+        self.stderr_path = root/'logs'/(label+'.stderr.log')
+        self.stderr = self.stderr_path.open('xb')
+        env = dict(os.environ, CUDA_VISIBLE_DEVICES='', PYTHONDONTWRITEBYTECODE='1',
+            OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1')
+        argv = [sys.executable, '-B', str(Path(__file__).resolve()), 'worker', '--role', role,
+            '--mode', mode, '--plan', str(plan), '--plan-sha', digest(plan)]
+        self.started = time.perf_counter(); self.argv = argv
+        self.selector = selectors.DefaultSelector()
+        self.p = None
+        try:
+            self.remaining()
+            self.p = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                stderr=self.stderr, text=True, bufsize=1, env=env)
+            self.selector.register(self.p.stdout, selectors.EVENT_READ)
+            self.ready = self.line()
+            require(self.ready == {'ready': True, 'pid': self.p.pid, 'role': role}, 'Wrong worker ready record')
+            self.startup_seconds = time.perf_counter()-self.started
+        except BaseException:
+            self.abort('Failed startup/ready')
+            raise
+
+    def remaining(self):
+        remaining = self.deadline-time.perf_counter()
+        require(remaining > 0, 'Probe absolute wall deadline exceeded')
+        return min(TIMEOUT, remaining)
+
+    def abort(self, reason):
+        if self.p is not None:
+            if self.p.poll() is None:
+                self.p.terminate()
+                try: self.p.wait(timeout=10)
+                except subprocess.TimeoutExpired: self.p.kill(); self.p.wait(timeout=10)
+            for stream in (self.p.stdin, self.p.stdout):
+                if stream is not None: stream.close()
+            path = self.root/'processes'/(self.label+'.aborted.json')
+            if not path.exists():
+                save(path, dict(pid=self.p.pid, argv=self.argv, actual_parent_observed_exit=self.p.returncode,
+                    reason=reason, partial_preserved=True, benchmark_failed=True, no_retry=True))
+        self.stderr.close(); self.selector.close()
+
+    def line(self):
+        require(self.selector.select(self.remaining()), 'Worker response timeout; no retry')
+        raw = self.p.stdout.readline()
+        require(raw, 'Worker EOF before expected response; inspect stderr')
+        return json.loads(raw)
+
+    def request(self, authority, output, update_dir=None):
+        request_started = time.perf_counter()
+        job = dict(sequence=self.seq, authority=str(authority), authority_sha256=digest(authority),
+            output=str(output), update_dir=None if update_dir is None else str(update_dir),
+            chain_sha256=None if update_dir is None else digest(update_dir/'chain-receipt.json'))
+        save(self.root/'requests'/(self.label+f'-{self.seq:02d}.json'), job)
+        started = time.perf_counter()
+        self.p.stdin.write(canonical(job)+'\n'); self.p.stdin.flush()
+        ack = self.line()
+        require(ack.get('ack') is True and ack['sequence'] == self.seq and ack['pid'] == self.p.pid and
+            ack['role'] == self.role and ack['request_sha256'] == hashlib.sha256(canonical(job).encode()).hexdigest() and
+            ack['CUDA_initialized'] is False and ack['task_ack_is_not_process_exit'] is True, 'Worker acknowledgment differs')
+        hashes = ({name: digest(output/name) for name in UPDATE_FILES} if self.role == 'update' else {'receipt': digest(output)})
+        require(hashes == ack['output_sha256'], 'Worker output changed or acknowledgment differs')
+        record = dict(ack=ack, rpc_parent_seconds=time.perf_counter()-started)
+        save(self.root/'responses'/(self.label+f'-{self.seq:02d}.json'), record)
+        record['parent_request_seconds'] = time.perf_counter()-request_started
+        self.records.append(record); self.seq += 1
+        return record
+
+    def close(self):
+        self.p.stdin.close()
+        rc = self.p.wait(timeout=self.remaining())
+        elapsed = time.perf_counter()-self.started
+        trailing = self.p.stdout.read()
+        self.p.stdout.close(); self.stderr.close(); self.selector.close()
+        receipt = dict(pid=self.p.pid, argv=self.argv, role=self.role, mode=self.mode,
+            actual_parent_observed_exit=rc, tasks_acknowledged=self.seq, startup_seconds=self.startup_seconds,
+            lifetime_parent_seconds=elapsed, stderr_sha256=digest(self.stderr_path),
+            task_ACKs_are_not_individual_OS_exits=True)
+        save(self.root/'processes'/(self.label+'.json'), receipt)
+        require(rc == 0 and not trailing, 'Worker actual nonzero exit or unsolicited output')
+        return receipt
+
+
+def compare_updates(a, b):
+    import numpy as np
+    require(set(x.name for x in a.iterdir()) == set(UPDATE_FILES) == set(x.name for x in b.iterdir()), 'Wrong update inventory')
+    for name in UPDATE_FILES:
+        require(digest(a/name) == digest(b/name), 'Exact output file differs: '+name)
+    with np.load(a/'delta-after.npz', allow_pickle=False) as x, np.load(b/'delta-after.npz', allow_pickle=False) as y:
+        require(x.files == y.files, 'Array schema differs')
+        for key in x.files:
+            require(x[key].dtype == y[key].dtype and x[key].shape == y[key].shape and x[key].tobytes() == y[key].tobytes(), 'Raw array differs')
+    return {'all_four_files_SHA_identical': True, 'arrays_bitwise_identical': True,
+            'full_update_receipt_and_trace_digests_identical': True}
+
+
+def compare_validation(reference, resident):
+    a, b = read(reference), read(resident)
+    require(a.pop('schema') == 'conditioning-update-cpu-validation-v1', 'Wrong original verifier schema')
+    require(b.pop('schema') == 'conditioning-resident-cpu-validation-prototype-v1', 'Wrong prototype verifier schema')
+    original = a.pop('validator_source')
+    require(original == {'path': str(VALIDATOR), 'sha256': V_SHA}, 'Wrong original validator identity')
+    require(b.pop('original_validator_source') == original and b.pop('resident_prototype') is True, 'Missing prototype original-source provenance')
+    require(b.pop('resident_cache_scope') == 'SHA-verified imported modules only; no input/hash/state/delta/checkpoint caching', 'Wrong cache scope')
+    identity = b.pop('validator_source')
+    require(identity == {'path': str(Path(__file__).resolve().parent/'resident_validator.py'),
+                         'sha256': digest(Path(__file__).parent/'resident_validator.py')}, 'Wrong resident source identity')
+    require(a == b, 'Scientific verifier receipt differs beyond enumerated provenance fields')
+    return {'all_scientific_fields_identical': True, 'explicit_differing_fields':
+            ['schema', 'validator_source', 'original_validator_source', 'resident_prototype', 'resident_cache_scope']}
+
+
+def prepare(root):
+    require(sys.dont_write_bytecode and root.is_dir() and not list(root.iterdir()), 'New empty output root and -B required')
+    require(not root.is_relative_to(ROOT/'experiments') and not root.is_relative_to(ROOT/'models'), 'Prototype may not write experimental/model trees')
+    require(shutil.disk_usage(root).free > (10 << 30), 'Insufficient temporary free space')
+    for name in ('inputs', 'updates', 'validation', 'logs', 'requests', 'responses', 'processes'):
+        (root/name).mkdir()
+    require(digest(AQ/'manifest.json') == MANIFEST_SHA, 'AQ sealed manifest differs')
+    manifest = {x['path']: x for x in read(AQ/'manifest.json')['files']}
+    pins = {str(AQ/'manifest.json'): MANIFEST_SHA, str(LAB/'conditioning_epoch.py'): E_SHA, str(VALIDATOR): V_SHA}
+    authorities = {}
+    for label, (name, expected) in FIXTURES.items():
+        authority = AQ/'authorities'/(name+'.epoch.json')
+        require(manifest[authority.relative_to(AQ).as_posix()]['sha256'] == expected and digest(authority) == expected, 'AQ fixture authority seal differs')
+        authorities[label] = authority
+        a = read(authority); pins[str(authority)] = expected
+        pins.update(a['frozen_inputs'])
+        pins.update({x['path']: x['sha256'] for x in a['files'].values()})
+        checkpoint = Path(a['checkpoint_dir']); cp = checkpoint/'checkpoint-manifest.json'
+        require(manifest[cp.relative_to(AQ).as_posix()]['sha256'] == a['checkpoint_manifest_sha256'], 'Checkpoint not in fixed AQ seal')
+        pins[str(cp)] = a['checkpoint_manifest_sha256']
+        for f in checkpoint.iterdir():
+            require(f.is_file() and not f.is_symlink(), 'Unexpected checkpoint entry')
+            pins[str(f)] = manifest[f.relative_to(AQ).as_posix()]['sha256']
+        for filename in UPDATE_FILES:
+            f = AQ/'updates'/name/filename
+            pins[str(f)] = manifest[f.relative_to(AQ).as_posix()]['sha256']
+    for label in ('N_D0', 'N_frozen', 'M_D0'):
+        a = read(authorities[label[0]])
+        a['epoch_id'] = 'engineering-residency-control-'+label
+        if label.endswith('_D0'):
+            a['command']['d_command'] = 0.0
+        else:
+            a['command']['frozen'] = True
+        path = root/'inputs'/(label+'.authority.json'); save(path, a)
+        authorities[label] = path; pins[str(path)] = digest(path)
+    require(all(digest(p) == sha for p, sha in pins.items()), 'Frozen fixture input bytes differ')
+    sources = {str(p.resolve()): digest(p) for p in Path(__file__).parent.glob('*.py')}
+    plan = dict(schema='conditioning-cpu-residency-probe-v1', output_root=str(root), sequence=list(SEQUENCE),
+        authorities={k: str(v) for k, v in authorities.items()}, input_sha256=pins, prototype_sources=sources,
+        fixture_selection='Two predeclared complete frozen AQ engineering fixtures; never AR interim results',
+        derived_controls='Only D=0/frozen command variants over fixed sealed events; not new brain runs',
+        threads=1, CPU_only=True, no_brain_execution=True, no_weight_installation=True,
+        current_AR_untouched=True, no_automatic_retry=True, max_wall_seconds=900,
+        benchmark_scope='CPU epoch postprocessing and verification only; excludes AP wrapper/ledger and all GPU/brain work',
+        acceptance='Exact four update files, bitwise delta, full trace/receipt identity, fresh original verifier exit0, separate resident verifier identity')
+    save(root/'benchmark-plan.json', plan)
+    return plan
+
+
+def run(root):
+    require(root.is_dir() and not list(root.iterdir()), 'New empty output directory required')
+    started = time.perf_counter(); deadline = started+900
+    active = []
+    try:
+        plan = prepare(root); planfile = root/'benchmark-plan.json'
+        # Yield to the live experiment; only one worker computes at a time.
+        os.nice(19)
+        e = Process('update', 'server', planfile, root, 'resident-E', deadline); active.append(e)
+        v = Process('verify-resident', 'server', planfile, root, 'resident-V', deadline); active.append(v)
+        require(e.p.pid != v.p.pid, 'Independent workers required')
+        rows = []
+        for number, label in enumerate(SEQUENCE):
+            require(time.perf_counter()-started < 900, 'Probe wall cap exceeded')
+            authority = Path(plan['authorities'][label]); prefix = f'{number:02d}-{label}'
+            refout, resout = (root/'updates'/(prefix+'-'+x) for x in ('fresh', 'resident'))
+            reference = Process('update', 'once', planfile, root, prefix+'-fresh-E', deadline); active.append(reference)
+            reference.request(authority, refout); ereceipt = reference.close(); active.remove(reference)
+            er = e.request(authority, resout)
+            equality = compare_updates(refout, resout)
+            if label in FIXTURES:
+                compare_updates(AQ/'updates'/FIXTURES[label][0], resout)
+            freshv, residentv = (root/'validation'/(prefix+'-'+x+'.json') for x in ('fresh', 'resident'))
+            validator = Process('verify-reference', 'once', planfile, root, prefix+'-fresh-V', deadline); active.append(validator)
+            validator.request(authority, freshv, resout); vreceipt = validator.close(); active.remove(validator)
+            vr = v.request(authority, residentv, resout)
+            vequality = compare_validation(freshv, residentv)
+            if label.endswith('_D0') or label.endswith('_frozen'):
+                import numpy as np
+                before = Path(read(authority)['files']['overlay_before']['path'])
+                with np.load(before, allow_pickle=False) as a, np.load(resout/'delta-after.npz', allow_pickle=False) as b:
+                    require(a['plastic_delta_mv'].tobytes() == b['plastic_delta_mv'].tobytes(), 'D0/frozen failed bitwise identity')
+            row = dict(label=label, position=number, fresh_E_seconds=ereceipt['lifetime_parent_seconds'],
+                resident_E_seconds=er['parent_request_seconds'], fresh_V_seconds=vreceipt['lifetime_parent_seconds'],
+                resident_V_seconds=vr['parent_request_seconds'], update_comparison=equality, validator_comparison=vequality,
+                original_fresh_validator_actual_exit=vreceipt['actual_parent_observed_exit'])
+            rows.append(row); save(root/(prefix+'-comparison.json'), row)
+            print(canonical({'completed': number+1, 'total': len(SEQUENCE), 'label': label, 'exact_equal': True}), flush=True)
+        ep, vp = e.close(), v.close(); active.clear()
+        require(all(digest(p) == sha for p, sha in plan['input_sha256'].items()), 'Consumed frozen inputs changed')
+        require(all(digest(p) == sha for p, sha in plan['prototype_sources'].items()), 'Prototype changed')
+        require(time.perf_counter() < deadline, 'Probe absolute wall deadline exceeded before PASS')
+        warm = rows[1:]
+        timing = {}
+        for field in ('fresh_E_seconds','resident_E_seconds','fresh_V_seconds','resident_V_seconds'):
+            timing[field] = dict(all_mean=statistics.mean(r[field] for r in rows),
+                warm_mean=statistics.mean(r[field] for r in warm), warm_median=statistics.median(r[field] for r in warm))
+        timing['warm_combined_speedup_ratio'] = sum(r['fresh_E_seconds']+r['fresh_V_seconds'] for r in warm)/sum(r['resident_E_seconds']+r['resident_V_seconds'] for r in warm)
+        timing['warm_E_only_with_original_fresh_verifier_speedup_ratio'] = sum(r['fresh_E_seconds']+r['fresh_V_seconds'] for r in warm)/sum(r['resident_E_seconds']+r['fresh_V_seconds'] for r in warm)
+        report = dict(status='PASS-CPU-RESIDENCY-PROTOTYPE-ONLY', trials=rows, timing=timing,
+            resident_process_exits=[ep['actual_parent_observed_exit'], vp['actual_parent_observed_exit']],
+            source_and_consumed_input_SHA_unchanged=True, full_history_reaudit=False,
+            prototype_plan_sha256=digest(planfile), total_wall_seconds=time.perf_counter()-started,
+            timing_boundary='Parent request includes input/chain digest, request+response persistence and output verification. Fresh also includes spawn/ready/exit; resident lifetime/startup separately recorded.',
+            AR_modified=False, brain_run=False, GPU_benchmarked=False, static_hash_cache_removed=False,
+            learning_test=False, candidate_generated=False, end_to_end_AR_speedup_not_measured=True,
+            scope='7 fixed engineering replays; CPU postprocessing only. Not authorized or validated as AR replacement.')
+        save(root/'report.json', report)
+        files = {str(p.relative_to(root)): {'sha256': digest(p), 'bytes': p.stat().st_size} for p in root.rglob('*') if p.is_file()}
+        save(root/'manifest.json', {'files': files, 'manifest_self_excluded': True})
+        print(canonical({'report': str(root/'report.json'), 'status': report['status'], 'timing': timing}), flush=True)
+    except BaseException as error:
+        for process in active:
+            process.abort(type(error).__name__+': '+str(error))
+        if not (root/'stopped.json').exists():
+            save(root/'stopped.json', dict(error_type=type(error).__name__, error=str(error), traceback=traceback.format_exc(),
+                partial_preserved=True, no_automatic_retry=True, registered_AR_not_touched=True))
+        raise
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest='mode', required=True)
+    r = sub.add_parser('run'); r.add_argument('--output', required=True)
+    w = sub.add_parser('worker'); w.add_argument('--role', choices=('update','verify-reference','verify-resident'), required=True)
+    w.add_argument('--mode', dest='worker_mode', choices=('once','server'), required=True)
+    w.add_argument('--plan', required=True); w.add_argument('--plan-sha', required=True)
+    args = parser.parse_args()
+    if args.mode == 'run': run(safe(args.output))
+    else: worker(args.role, args.worker_mode, args.plan, args.plan_sha)