qishizhe/digital-drosophila
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish sealed updated-boundary audit and truthful learning status

Browse Source
This commit is contained in:
Digital Drosophila Archive
2026-09-30 20:17:50 +08:00
parent a9c5918445
commit 3e30934a55
574 changed files with 218421 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
registry/phase4aq-conditioning-execution/source-snapshot/updated_continuation.py
+601
View File
@@ -0,0 +1,601 @@
"""Externally authorized, single-update same-state continuation; not learning proof.
Only epoch_update from a sealed fresh AN checkpoint and its original E four-file
postprocess is admitted. Later idle transitions retain the installed snapshot.
The input policy is empty_idle or externally_pinned_CS_probe. Probe CS/seed/no-US
semantics and branch lineage belong to the registered coordinator, not this module.
E/U are NEVER imported/executed here. An external CPU validation receipt binds U's
recomputation; its executor/process/registration authority belongs to the caller.
No public small-graph mode. CPU synthetic tests exercise private pure helpers.
"""
from __future__ import annotations
import hashlib
import importlib.util
import json
from pathlib import Path
import sys
import numpy as np
SOURCE_PINS = {
'runtime_continuation_source': '0b8e8e0a6a23a079dc62a88355ce355d53b3ae22f6665519a8f6aef2dcfc5aba',
'runtime_light_source': '522b8c31ea938890fe15364fccd25ccc5bded54101754909f255532e7ad01aae',
'conditioning_epoch_source': '757935a4815d9f625640df685184ed3fe44a8845323562796da7aa43cbb194e1',
'conditioning_update_source': '96efff678cff910654eb08e86f1a17789e4e5589e340345f419c7027a7d547e2',
'conditioning_preflight_compare_source': 'b88fd9a88d9f92d5c4d042b90f2129b015e8d4bd810c41351ee573241e2d4c2e',
'engineering_compare_source': '5a2403f11bc08d65a9d242ac2cf9686b850e0c3680d65659f785e9b6b50c0238',
'eligibility_bridge_source': '174fb8d14619189e74b095e77a6db18b14c93046dc431c0b55710a5aa2dd2e61',
'phase4j_source': 'abb4574da85f00bd9ef61024c65993ef81d42651ff8d8d63c15dd69c6222471e',
'am_overlay_source': '7f891d0ba79b393c96e75fd2dee2eddc51e57d8213eaac2779703bc2c18712b4',
}
SCHEMA = 'conditioning-updated-boundary-v1'
CHECKPOINT_SCHEMA = 'conditioning-updated-checkpoint-v1'
PROVENANCE_SCHEMA = 'conditioning-updated-runtime-v1'
RECEIPT = 'updated-continuation-receipt.json'
MANIFEST = 'checkpoint-manifest.json'
GRAPH_FILES = ('nodes.npy', 'edge_id.npy', 'src.npy', 'dst.npy', 'signed_raw.npy', 'ptr.npy', 'weights.npy')
APL_FILES = ('input_matrix.npy', 'output_matrix.npy', 'kc_globals.npy', 'apl_globals.npy')
UPDATE_FILES = ('chain-receipt.json', 'update-receipt.json', 'delta-after.npz', 'overlay-manifest.json')
E_SOURCE_ROLES = ('runtime_light_source', 'runtime_continuation_source', 'conditioning_update_source',
'eligibility_bridge_source', 'phase4j_source', 'am_overlay_source')
E_ROLES = frozenset(('spec', 'requests', 'overlay_before', 'overlay_manifest_before', 'state', 'kc',
'canonical_kc', 'candidate_map', 'mask', 'base_candidate', *E_SOURCE_ROLES))
def _require(ok, message):
if not ok:
raise ValueError(message)
def _digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as stream:
for block in iter(lambda: stream.read(8 << 20), b''):
h.update(block)
return h.hexdigest()
def _continuation():
path = Path(__file__).resolve().parent.parent / 'olfactory-conditioning-lab-v1/runtime_continuation.py'
raw = path.read_bytes()
expected = SOURCE_PINS['runtime_continuation_source']
_require(hashlib.sha256(raw).hexdigest() == expected, 'Frozen C source pin changed')
name = 'conditioning_updated_pinned_C'
existing = sys.modules.get(name)
if existing is not None:
_require(getattr(existing, '__updated_pin__', None) == expected and
Path(existing.__file__).resolve() == path, 'C module identity collision')
return existing
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
try:
exec(compile(raw, str(path), 'exec'), module.__dict__)
except BaseException:
sys.modules.pop(name, None)
raise
module.__updated_pin__ = expected
return module
C = _continuation()
R = C.R # The ONLY runtime/helper cache owner. Never import ordinary R or E.
def _canonical(value):
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False)
def _same(a, b):
return a.dtype == b.dtype and a.shape == b.shape and a.tobytes() == b.tobytes()
def _entry(entry, pins):
R.require(type(entry) is dict and set(entry) == {'path', 'sha256'} and R._sha(entry['sha256']), 'Malformed file pin')
raw = Path(entry['path'])
R.require(raw.is_absolute() and raw.is_file() and not raw.is_symlink(), 'Expected absolute regular unlinked file')
path = raw.resolve(strict=True)
R.require(str(path) not in pins or pins[str(path)] == entry['sha256'], 'Conflicting file pins')
pins[str(path)] = entry['sha256']
return path
def _check(pins):
for path, sha in pins.items():
R.require(R._sha(sha) and _digest(path) == sha, 'Source/input changed: ' + str(path))
def _evidence(value):
return dict(dtype=value.dtype.str, shape=list(value.shape), nbytes=int(value.nbytes),
raw_c_order_sha256=R._raw(value))
def _Q(sources):
R._load('engineering_compare', sources['engineering_compare_source'], SOURCE_PINS['engineering_compare_source'])
return R._load('conditioning_updated_pinned_Q', sources['conditioning_preflight_compare_source'],
SOURCE_PINS['conditioning_preflight_compare_source'])
def _requests(path, spec, policy):
values = C._npz(path, ('tick', 'neuron_idx'))
tick, node = values['tick'], values['neuron_idx']
R.require(tick.dtype == node.dtype == np.int32 and tick.ndim == node.ndim == 1 and tick.shape == node.shape,
'Invalid request dtype/shape')
R.require(np.all((tick >= 0) & (tick < spec['steps'])) and np.all((node >= 0) & (node < C.N)), 'Request outside case')
keys = tick.astype(np.int64) * C.N + node
R.require(len(keys) < 2 or np.all(keys[1:] > keys[:-1]), 'Requests must be sorted unique')
R.require(policy in ('empty_idle', 'externally_pinned_CS_probe'), 'Invalid input policy')
R.require(policy != 'empty_idle' or len(tick) == 0, 'empty_idle contains requests')
return values
def _expected_effective(helper, plan, spec):
maps = {}
try:
graph = Path(spec['graph_dir'])
for name, dtype, size in (('nodes', 'int32', C.N), ('edge_id', 'int64', C.E),
('src', 'int32', C.E), ('weights', 'float64', C.E)):
a = np.load(graph / (name + '.npy'), mmap_mode='r', allow_pickle=False)
maps[name] = a
R.require(a.dtype == np.dtype(dtype) and a.shape == (size,), 'Full graph dtype/shape changed: ' + name)
R.require(np.array_equal(maps['nodes'], np.arange(C.N, dtype=np.int32)), 'Noncanonical node order')
apl = np.load(Path(spec['apl_bundle_dir']) / 'apl_globals.npy', allow_pickle=False)
R.require(apl.dtype == np.int32 and apl.shape == (2,), 'Invalid APL IDs')
apl_mask = np.isin(maps['src'], apl)
R.require(int(apl_mask.sum()) == 5845, 'APL output mask changed')
mapping = helper.map_installation(maps['weights'], maps['edge_id'], apl_mask, plan)
R.require(mapping['missing_candidate_count'] == 0 and mapping['mapped_candidate_count'] == 14232, 'Full candidate mapping required')
return helper.expected_hashes(maps['weights'], apl_mask, mapping)[1]
finally:
for a in maps.values():
a._mmap.close()
def _epoch_update(value, paths, sources, pins, parent, before, after, before_meta, after_meta):
update = value['update']
R.require(type(update) is dict and set(update) == {'epoch_authority', 'files'}, 'Missing epoch update authority')
ep = _entry(update['epoch_authority'], pins)
paths['epoch_authority'] = ep
R.require(type(update['files']) is dict and set(update['files']) == set(UPDATE_FILES), 'All four original E outputs required')
files = {name: _entry(update['files'][name], pins) for name in UPDATE_FILES}
directory = files['chain-receipt.json'].parent
R.require(all(path == directory / name for name, path in files.items()) and
{p.name for p in directory.iterdir()} == set(UPDATE_FILES), 'E output must be exact four-file directory')
R.require(files['delta-after.npz'] == paths['after_overlay'] and files['overlay-manifest.json'] == paths['after_manifest'],
'Installed after snapshot is not the original E output')
_check(pins)
authority = R._read_json(ep)
R.require(type(authority) is dict and set(authority) == {'schema', 'epoch_id', 'diagnostic_mode', 'command',
'checkpoint_dir', 'checkpoint_manifest_sha256', 'files', 'frozen_inputs'} and
authority['schema'] == 'phase4an-epoch-postprocess-authority-v1', 'Original E authority schema changed')
R.require(parent['fresh'] is True and parent['global_tick_offset'] == 0 and
Path(authority['checkpoint_dir']).resolve(strict=True) == parent['directory'] and
authority['checkpoint_manifest_sha256'] == parent['manifest_sha256'] and
authority['diagnostic_mode'] == parent['summary']['diagnostic_mode'], 'Wrong fresh E parent')
R.require(type(authority['files']) is dict and set(authority['files']) == E_ROLES, 'Incomplete E file roles')
epins = {str(ep): pins[str(ep)], str(parent['directory'] / MANIFEST): parent['manifest_sha256']}
epaths = {role: _entry(entry, epins) for role, entry in authority['files'].items()}
for role in E_SOURCE_ROLES:
R.require(epins[str(epaths[role])] == SOURCE_PINS[role] and epaths[role] == sources[role], 'E source role mismatch: ' + role)
R.require(type(authority['frozen_inputs']) is dict and authority['frozen_inputs'], 'Missing E frozen input pins')
for path, sha in authority['frozen_inputs'].items():
_entry(dict(path=path, sha256=sha), epins)
R.require(epaths['state'] == parent['directory'] / 'state.npz' and
parent['manifest']['files_sha256']['state.npz'] == epins[str(epaths['state'])], 'E state is not sealed parent')
for role, actual in (('overlay_before', paths['before_overlay']), ('overlay_manifest_before', paths['before_manifest'])):
R.require(epaths[role] == actual, 'E before overlay path differs')
for role in ('spec', 'requests', 'overlay_before', 'overlay_manifest_before'):
R.require(parent['provenance']['input_sha256'].get(str(epaths[role])) == epins[str(epaths[role])], 'E/runtime input differs: ' + role)
for path, sha in epins.items():
_entry(dict(path=path, sha256=sha), pins)
_check(pins)
chain = R._read_json(files['chain-receipt.json'])
receipt = R._read_json(files['update-receipt.json'])
command = authority['command']
R.require(type(command) is dict and set(command) == {'steps', 'compartment', 'us_tick', 'd_command', 'frozen', 'trace_backend'}, 'E command schema')
R.require(type(command['steps']) is int and command['steps'] == parent['steps'], 'E command clock mismatch')
R.require(type(command['d_command']) in (int, float) and command['d_command'] in (0, 1) and
type(command['frozen']) is bool and command['trace_backend'] in ('numpy', 'torch') and
command['compartment'] in ("B2B'2a", 'a1', "y2a'1", "y5B'2a") and
(command['us_tick'] is None or (type(command['us_tick']) is int and 0 <= command['us_tick'] < command['steps'])) and
(command['us_tick'] is not None or command['d_command'] == 0), 'E command value/type mismatch')
required_chain = dict(schema='phase4an-epoch-postprocess-chain-v1', complete=True,
epoch_id=authority['epoch_id'], authority_file=str(ep), authority_sha256=pins[str(ep)],
coordinator_source_sha256=SOURCE_PINS['conditioning_epoch_source'], command=command,
diagnostic_mode=authority['diagnostic_mode'], source_neural_checkpoint_dir=str(parent['directory']),
source_neural_checkpoint_manifest_sha256=parent['manifest_sha256'],
before_overlay_file_sha256=pins[str(paths['before_overlay'])],
before_overlay_manifest_sha256=pins[str(paths['before_manifest'])],
after_overlay_file_sha256=pins[str(paths['after_overlay'])],
after_overlay_manifest_sha256=pins[str(paths['after_manifest'])],
source_state_file_sha256=epins[str(epaths['state'])], source_spec_sha256=epins[str(epaths['spec'])],
source_requests_sha256=epins[str(epaths['requests'])], input_files_sha256=epins,
output_files_sha256={name: pins[str(files[name])] for name in UPDATE_FILES if name != 'chain-receipt.json'},
runtime_delta_before_raw_sha256=R._raw(before['plastic_delta_mv']),
runtime_effective_weights_raw_sha256=parent['summary']['effective_weights_raw_sha256'],
update_receipt_payload_sha256=receipt.get('payload_sha256'), runtime_learned_manifest_loader_verified=True,
input_and_checkpoint_bytes_reverified=True, candidate_endpoints_equal_graph=True,
KC_equals_supplied_canonical_membership=True, weights_installed=False, brain_executed=False,
learning_pass_claimed=False, candidate_generated=False)
R.require(all(_canonical(chain.get(k)) == _canonical(v) for k, v in required_chain.items()), 'Original E chain mismatch')
for role, prefix in (('mask', 'mask'), ('base_candidate', 'base_candidate'), ('am_overlay_source', 'am_overlay')):
R.require(Path(before_meta[prefix + '_path']).resolve() == epaths[role] and
before_meta[prefix + '_sha256'] == epins[str(epaths[role])], 'E mapping/base authority mismatch')
mapping = C._npz(epaths['candidate_map'], ('edge_id', 'source_neuron_idx', 'target_neuron_idx', 'target_compartment_verbatim'))
kc = np.load(epaths['kc'], allow_pickle=False)
canonical_kc = np.load(epaths['canonical_kc'], allow_pickle=False)
base = np.load(epaths['base_candidate'], allow_pickle=False)
R.require(epaths['canonical_kc'] == Path(C._parent_spec(parent)['apl_bundle_dir']).resolve() / 'kc_globals.npy' and
kc.dtype == np.int32 and kc.shape == (4064,) and _same(kc, canonical_kc), 'Full canonical KC mismatch')
R.require(_same(mapping['edge_id'], before['edge_id']) and _same(after['edge_id'], before['edge_id']) and
_same(base, before['base_candidate_mv']) and _same(base, after['base_candidate_mv']), 'Full before/after mask/base mismatch')
graph = Path(C._parent_spec(parent)['graph_dir'])
parent_spec = C._parent_spec(parent)
required_epins = [graph / name for name in GRAPH_FILES] + [Path(parent_spec['apl_bundle_dir']) / name for name in APL_FILES]
required_epins.append(Path(parent_spec['coefficients_path']))
required_epins.extend(Path(before_meta['aj_model_dir']).parent / name for name in R.PINS)
R.require(all(str(path.resolve(strict=True)) in authority['frozen_inputs'] for path in required_epins),
'Original E authority omitted frozen graph/APL/compiled sources')
for role, filename in (('source_neuron_idx', 'src.npy'), ('target_neuron_idx', 'dst.npy')):
a = np.load(graph / filename, mmap_mode='r', allow_pickle=False)
try:
R.require(mapping[role].dtype == np.int32 and mapping[role].shape == (14232,) and
_same(mapping[role], a[before['edge_id']]), 'Actual candidate endpoint mismatch')
finally:
a._mmap.close()
compartments = mapping['target_compartment_verbatim']
R.require(compartments.dtype.kind == 'U' and compartments.shape == (14232,), 'Invalid full compartment metadata')
arrays = dict(step=parent['state']['step'], neuron_idx=parent['state']['neuron_idx'], kc=kc,
candidate_edge_id=before['edge_id'], candidate_sources=mapping['source_neuron_idx'],
candidate_targets=mapping['target_neuron_idx'], candidate_compartments=compartments,
delta_before=before['plastic_delta_mv'], base_candidate=base)
R.require(_canonical(receipt.get('input_arrays')) == _canonical({k: _evidence(a) for k, a in arrays.items()}), 'U receipt actual array binding failed')
expected_command = dict(steps=command['steps'], n_neurons=C.N, compartment=command['compartment'],
us_tick=command['us_tick'], D_scheduled=float(command['d_command']), frozen=command['frozen'], trace_backend=command['trace_backend'])
R.require(_canonical(receipt.get('command')) == _canonical(expected_command), 'U receipt command differs')
binding = _Q(sources).audit_delta_binding(before['plastic_delta_mv'], after['plastic_delta_mv'], after['plastic_delta_mv'],
update_receipt=receipt, parent_snapshot_sha256=pins[str(paths['before_overlay'])], child_manifest=after_meta)
R.require(binding['pass_all'] is True, 'Q actual before/after delta binding failed')
cp = _entry(value['cpu_validation'], pins)
paths['cpu_validation'] = cp
cpu = R._read_json(cp)
expected_cpu = dict(schema='conditioning-update-cpu-validation-v1', complete=True,
epoch_authority_sha256=pins[str(ep)], parent_checkpoint_manifest_sha256=parent['manifest_sha256'],
update_files_sha256={name: pins[str(files[name])] for name in UPDATE_FILES},
before_overlay_sha256=pins[str(paths['before_overlay'])], before_manifest_sha256=pins[str(paths['before_manifest'])],
after_overlay_sha256=pins[str(paths['after_overlay'])], after_manifest_sha256=pins[str(paths['after_manifest'])],
before_delta_raw_sha256=R._raw(before['plastic_delta_mv']), after_delta_raw_sha256=R._raw(after['plastic_delta_mv']),
update_receipt_payload_sha256=receipt['payload_sha256'],
validator_source=dict(path=str(sources['cpu_validator_source']), sha256=pins[str(sources['cpu_validator_source'])]),
frozen_U_source_sha256=SOURCE_PINS['conditioning_update_source'], frozen_U_verify_receipt_recomputed=True,
actual_sealed_events_used=True, full_candidate_mapping_verified=True, brain_executed=False, learning_pass_claimed=False)
R.require(_canonical(cpu) == _canonical(expected_cpu), 'External CPU recomputation receipt mismatch')
return binding
def validate_authority(authority_json, authority_sha256, *, _seen=None):
"""Read-only CPU admission. Does not recompute U, run E, import Torch or restore.
Returns value/authority/authority_sha256, paths, pins (absolute str -> SHA),
sources, parent, case_dir/spec, helper, before_plan/after_plan/before_meta/
after_meta; suitable for an independent monolithic oracle without restore.
"""
pins = {}
authority_path = _entry(dict(path=str(authority_json), sha256=authority_sha256), pins)
_check(pins)
value = R._read_json(authority_path)
fields = {'schema', 'boundary_id', 'registration_sha256', 'transition', 'diagnostic_mode', 'input_policy',
'case', 'parent', 'global_tick_offset', 'overlay_before', 'overlay_after',
'expected_effective_weight_before_sha256', 'expected_effective_weight_after_sha256', 'update',
'cpu_validation', 'source_files', 'frozen_inputs'}
R.require(type(value) is dict and set(value) == fields and value['schema'] == SCHEMA, 'Boundary authority schema mismatch')
R.require(type(value['boundary_id']) is str and value['boundary_id'].strip() == value['boundary_id'] and value['boundary_id'], 'Invalid boundary ID')
R.require(R._sha(value['registration_sha256']) and value['transition'] in ('epoch_update', 'idle') and
value['diagnostic_mode'] in ('reference', 'light'), 'Invalid boundary scope')
R.require(type(value['case']) is dict and set(value['case']) == {'spec', 'requests'}, 'Exact case pins required')
paths = {key: _entry(value['case'][role], pins) for key, role in (('case_spec', 'spec'), ('requests', 'requests'))}
for moment in ('before', 'after'):
pair = value['overlay_' + moment]
R.require(type(pair) is dict and set(pair) == {'file', 'manifest'}, 'Exact overlay pair required')
paths[moment + '_overlay'] = _entry(pair['file'], pins)
paths[moment + '_manifest'] = _entry(pair['manifest'], pins)
source_values = value['source_files']
R.require(type(source_values) is dict and set(source_values) == set(SOURCE_PINS) |
{'updated_continuation_source', 'cpu_validator_source'}, 'Exact source roles required')
sources = {name: _entry(entry, pins) for name, entry in source_values.items()}
for name, sha in SOURCE_PINS.items():
R.require(pins[str(sources[name])] == sha, 'Frozen source may not be replaced: ' + name)
R.require(sources['updated_continuation_source'] == Path(__file__).resolve() and
sources['runtime_continuation_source'] == Path(C.__file__).resolve() and
sources['runtime_light_source'] == Path(R.__file__).resolve() and
sources['cpu_validator_source'] == Path(__file__).resolve().parent / 'validate_update_cpu.py',
'Loaded runtime/CPU validator source path differs')
R.require(type(value['frozen_inputs']) is dict and value['frozen_inputs'], 'Full frozen inputs required')
for path, sha in value['frozen_inputs'].items():
_entry(dict(path=path, sha256=sha), pins)
p = value['parent']
R.require(type(p) is dict and set(p) == {'directory', 'checkpoint_manifest_sha256'} and Path(p['directory']).is_absolute(), 'Exact parent authority required')
parentdir = Path(p['directory']).resolve(strict=True)
_entry(dict(path=str(parentdir / MANIFEST), sha256=p['checkpoint_manifest_sha256']), pins)
_check(pins)
parent = load_checkpoint(parentdir, p['checkpoint_manifest_sha256'], _seen=_seen)
R.require(type(value['global_tick_offset']) is int and value['global_tick_offset'] == parent['global_next_tick'], 'Wrong absolute continuation clock')
case = paths['case_spec'].parent
R.require(paths['case_spec'].name == 'spec.json' and paths['requests'] == case / 'requests.npz', 'Case filenames differ')
spec = R._read_json(paths['case_spec'])
R.require(spec.get('parent_checkpoint_manifest_sha256') == p['checkpoint_manifest_sha256'] and
type(spec.get('global_tick_offset')) is int and spec['global_tick_offset'] == value['global_tick_offset'], 'Case parent/clock differs')
fresh = dict(spec)
fresh.pop('parent_checkpoint_manifest_sha256')
R.validate_spec(fresh, case)
R.require(parent['global_next_tick'] + spec['steps'] <= np.iinfo(np.int64).max, 'Global clock overflow')
_requests(paths['requests'], spec, value['input_policy'])
previous = C._parent_spec(parent)
for key in ('dt_ms', 'model', 'apl', 'apl_delay_steps'):
R.require(_canonical(spec[key]) == _canonical(previous[key]), 'Frozen dynamics differ: ' + key)
for key in ('graph_dir', 'apl_bundle_dir'):
R.require(Path(spec[key]).resolve(strict=True) == Path(previous[key]).resolve(strict=True), 'Frozen directory differs')
R.require(_digest(spec['coefficients_path']) == parent['summary']['input_sha256']['coefficients.json'] and
_digest(Path(spec['graph_dir']) / 'weights.npy') == parent['summary']['input_sha256']['base_weights.npy'], 'Base/coefficients differ')
required = [Path(spec['graph_dir']).resolve() / n for n in GRAPH_FILES] + [Path(spec['apl_bundle_dir']).resolve() / n for n in APL_FILES]
required.append(Path(spec['coefficients_path']).resolve(strict=True))
R.require(all(str(path) in value['frozen_inputs'] for path in required), 'Full graph/APL/coeff pins missing')
helper, before, bm, bpins = R.load_overlay(paths['before_overlay'], paths['before_manifest'])
helper2, after, am, apins = R.load_overlay(paths['after_overlay'], paths['after_manifest'])
R.require(helper2 is helper, 'Overlay cache owner differs')
for path, sha in {**bpins, **apins}.items():
_entry(dict(path=str(path), sha256=sha), pins)
for relative, sha in R.PINS.items():
path = (Path(am['aj_model_dir']).resolve().parent / relative).resolve(strict=True)
R.require(value['frozen_inputs'].get(str(path)) == sha, 'Compiled/source dependency authority missing: ' + relative)
for key in ('mask_path', 'mask_sha256', 'base_candidate_path', 'base_candidate_sha256', 'aj_model_dir', 'am_overlay_path', 'am_overlay_sha256'):
R.require(bm[key] == am[key], 'Before/after anatomical/source authority differs: ' + key)
R.require(_digest(parentdir / 'overlay-input.npz') == pins[str(paths['before_overlay'])] and
_digest(parentdir / 'overlay-manifest.json') == pins[str(paths['before_manifest'])], 'Before snapshot differs from actual neural parent')
R.require(parent['overlay']['delta_raw_sha256'] == R._raw(before['plastic_delta_mv']), 'Parent delta raw identity mismatch')
for key in ('expected_effective_weight_before_sha256', 'expected_effective_weight_after_sha256'):
R.require(R._sha(value[key]), 'Expected effective SHA required')
before_effective = _expected_effective(helper, before, spec)
after_effective = _expected_effective(helper, after, spec)
R.require(before_effective == value['expected_effective_weight_before_sha256'] == parent['summary']['effective_weights_raw_sha256'], 'Before effective weights differ')
R.require(after_effective == value['expected_effective_weight_after_sha256'], 'After effective weights differ')
paths.update(epoch_authority=None, cpu_validation=None)
binding = None
if value['transition'] == 'epoch_update':
R.require(parent['fresh'] is True, 'An update boundary is allowed only once, from fresh AN parent')
binding = _epoch_update(value, paths, sources, pins, parent, before, after, bm, am)
root = authority_sha256
else:
R.require(parent['manifest']['schema'] == CHECKPOINT_SCHEMA and value['update'] is None and value['cpu_validation'] is None,
'Idle requires own updated parent and no new update authority')
R.require(pins[str(paths['before_overlay'])] == pins[str(paths['after_overlay'])] and
pins[str(paths['before_manifest'])] == pins[str(paths['after_manifest'])] and
before_effective == after_effective, 'Idle changed the installed snapshot')
R.require(parent['input_policy'] == 'empty_idle', 'A probe branch cannot feed back into a continuation lineage')
root = parent['update_root_authority_sha256']
_check(pins)
return dict(value=value, authority=authority_path, authority_sha256=authority_sha256, paths=paths, pins=pins,
sources=sources, parent=parent, case_dir=case, spec=spec, helper=helper, before_plan=before, after_plan=after,
before_meta=bm, after_meta=am, delta_binding=binding, update_root_authority_sha256=root)
class UpdatedRuntime(C.ContinuedRuntime):
"""Only weight admission differs from C; restoration and kernels unchanged."""
def __init__(self, *args, authority, **kwargs):
super().__init__(*args, **kwargs)
self.authority = authority
self.restore_audit = None
def install(self, *args, **kwargs):
R.require(self.installations == 0 and self.restorations == 0, 'Double/belated weight installation')
R._Runtime.install(self, *args, **kwargs) # Original base + full delta once; not old effective + delta.
R.require(self.effective_sha == self.authority['value']['expected_effective_weight_after_sha256'], 'Installed weight differs from authorized after')
def restore(self, v, g, ref, counts, ring, apl_ring, apl, upload):
super().restore(v, g, ref, counts, ring, apl_ring, apl, upload)
expected = dict(v=self.parent['state']['v'], g=self.parent['state']['g'], ref=self.parent['state']['ref'],
spike_ring=C.rotate_spike_ring(self.parent['terminal']['spike_ring'], int(self.parent['terminal']['next_tick'])),
apl_pending=self.parent['terminal']['apl_pending'], apl_state=self.parent['terminal']['apl_state'])
actual = dict(v=v, g=g, ref=ref, spike_ring=ring, apl_pending=apl_ring, apl_state=apl.state)
for key, tensor in actual.items():
R.require(_same(tensor.detach().cpu().numpy(), expected[key]), 'Restoration changed actual state bytes: ' + key)
self.restore_audit = dict(restored_raw_sha256={key: R._raw(a) for key, a in expected.items()},
all_state_and_queue_bytes_exact=True, existing_g_rescaled=False, retroactive_arrival_reweighting=False,
pending_spike_arrivals_use_current_installed_weight=True, pending_APL_samples_preserved=True,
boundary_installations=1, restorations=1, learning_pass_claimed=False)
def summary(self, value):
result = super().summary(value)
result.update(updated_boundary_schema=SCHEMA, boundary_authority_sha256=self.authority['authority_sha256'],
update_root_authority_sha256=self.authority['update_root_authority_sha256'],
boundary_transition=self.authority['value']['transition'], input_policy=self.authority['value']['input_policy'],
weight_update_at_boundary_authorized=True, frozen_same_weight_retention_claimed=False)
return result
def compile_updated(aj, runtime, mode):
tree, audit = C.derive_continued(Path(aj.__file__).read_bytes(), mode)
namespace = dict(aj.__dict__)
namespace.update(__name__=__name__ + '.derived_' + mode, __file__=str(Path(__file__).resolve()),
_runtime=runtime, _continuation=runtime, write_json=runtime.write_json, audit_sequence=runtime.audit_sequence)
exec(compile(tree, str(Path(__file__).resolve()) + '::updated-continuation-' + mode, 'exec'), namespace)
return namespace['run_gpu'], audit
def _record(directory, seen):
provenance = R._read_json(directory / 'runtime-provenance.json')
R.require(provenance.get('schema') == PROVENANCE_SCHEMA and provenance.get('complete') is True and
provenance.get('source_sha256') == _digest(__file__) and
Path(provenance['source_file']).resolve() == Path(__file__).resolve(), 'Not a genuine new continuation record')
R.require(provenance.get('old_globals_patched') is False and provenance.get('weights_updated_during_run') is False and
provenance.get('U_recomputed_here') is False and provenance.get('learning_pass_claimed') is False, 'Invalid continuation scope')
C._verify_receipt_hashes(directory, provenance)
R.require(provenance.get('state_sha256') == _digest(directory / 'state.npz'), 'Actual state is not bound')
authority = validate_authority(provenance['authority_file'], provenance['authority_sha256'], _seen=seen)
parent, spec, value = authority['parent'], authority['spec'], authority['value']
steps, offset = spec['steps'], parent['global_next_tick']
R.require(provenance['input_sha256'] == authority['pins'], 'Receipt input closure differs from authority')
R.require(provenance.get('mode') == value['diagnostic_mode'] and
provenance.get('parent_checkpoint_manifest_sha256') == parent['manifest_sha256'] and
provenance.get('global_tick_offset') == offset and provenance.get('global_next_tick') == offset+steps and
provenance.get('external_CPU_validation_receipt_bound') is (value['transition'] == 'epoch_update') and
provenance.get('external_registration_process_and_probe_semantics_verified_here') is False,
'Runtime provenance boundary/clock/scope differs')
_, expected_ast = C.derive_continued((Path(authority['after_meta']['aj_model_dir']) / 'gpu_timing.py').read_bytes(), value['diagnostic_mode'])
R.require(provenance.get('ast') == expected_ast, 'Derived source AST provenance differs')
summary = R._read_json(directory / 'summary.json')
finite = R._read_json(directory / 'finite-audit.json')
for name, expected in dict(complete=True, pass_all=True, all_finite=True, steps=steps, completed_ticks=steps,
nodes=C.N, edges=C.E, precision='float64', apl_precision='float32', within_trial_weight_updates=False,
checkpoint_continuation_used=True, global_tick_offset=offset, global_next_tick=offset+steps,
explicit_steps_executed=steps, max_steps=spec['max_steps'], hidden_early_stop=False,
entire_neural_trial_executed=True, record_history=False, counts_scope='segment_only_not_lifetime',
event_tick_scope='segment_local', weight_update_at_boundary_authorized=True, frozen_same_weight_retention_claimed=False,
parent_checkpoint_manifest_sha256=parent['manifest_sha256'], diagnostic_mode=value['diagnostic_mode'],
boundary_authority_sha256=authority['authority_sha256'], update_root_authority_sha256=authority['update_root_authority_sha256'],
updated_boundary_schema=SCHEMA, boundary_transition=value['transition'], input_policy=value['input_policy'],
effective_weights_raw_sha256=value['expected_effective_weight_after_sha256']).items():
R.require(_canonical(summary.get(name)) == _canonical(expected), 'Summary binding mismatch: ' + name)
R.require(finite.get('all_finite') is True and finite.get('first_failure_tick') == -1 and
finite.get('finite_observed_ticks') == steps, 'Missing full finite coverage')
expected_input = {'spec.json': authority['pins'][str(authority['paths']['case_spec'])],
'requests.npz': authority['pins'][str(authority['paths']['requests'])],
'base_weights.npy': _digest(Path(spec['graph_dir']) / 'weights.npy'), 'coefficients.json': _digest(spec['coefficients_path'])}
R.require(summary.get('input_sha256') == expected_input, 'Summary case/base inputs differ')
state = C._npz(directory / 'state.npz', ('step', 'neuron_idx', 'counts', 'v', 'g', 'ref'))
terminal = C._npz(directory / 'terminal-queues.npz', ('spike_ring', 'apl_state', 'apl_pending', 'next_tick'))
C.validate_state(state, terminal, steps, fresh=False)
if steps < 19:
restored = C.rotate_spike_ring(parent['terminal']['spike_ring'], int(parent['terminal']['next_tick']))
R.require(_same(terminal['spike_ring'][steps:], restored[steps:]), 'Unwritten carried spike slots differ')
if steps < 18:
R.require(_same(terminal['apl_pending'][:18-steps], parent['terminal']['apl_pending'][steps:]), 'Carried APL tail differs')
pending = np.load(directory / 'apl-pending-tail.float32.npy', allow_pickle=False)
timing = R._read_json(directory / 'apl-timing-audit.json')
R.require(_same(pending, terminal['apl_pending']) and timing.get('pending_raw_sha256') == R._raw(pending) and
timing.get('apl_delay_steps') == 18 and timing.get('brain_tail_advanced') is False and
timing.get('final_apl_state_u32_bits') == terminal['apl_state'].view(np.uint32).tolist() and
timing.get('global_tick_offset') == offset and timing.get('global_next_tick') == offset+steps and
timing.get('initial_queue_raw_sha256') == R._raw(parent['terminal']['apl_pending']), 'APL metadata/terminal mismatch')
overlay = R._read_json(directory / 'overlay-audit.json')
R.require(overlay.get('complete') is True and overlay.get('weights_updated_during_run') is False and
overlay.get('effective_weights_raw_sha256') == value['expected_effective_weight_after_sha256'] and
overlay.get('delta_raw_sha256') == R._raw(authority['after_plan']['plastic_delta_mv']) and
overlay.get('mask_sha256') == R.MASK_SHA and overlay.get('am_helper_sha256') == R.AM_OVERLAY_SHA, 'Installed snapshot receipt mismatch')
for dest, source in (('overlay-input.npz', 'after_overlay'), ('overlay-manifest.json', 'after_manifest')):
R.require(_digest(directory / dest) == authority['pins'][str(authority['paths'][source])], 'Copied after snapshot differs')
receipt = R._read_json(directory / RECEIPT)
payload = C._tree_hashes(directory)
payload.pop(RECEIPT, None)
expected_receipt = dict(schema='conditioning-updated-receipt-v1', complete=True,
authority_file=str(authority['authority']), authority_sha256=authority['authority_sha256'],
parent_checkpoint_dir=str(parent['directory']), parent_checkpoint_manifest_sha256=parent['manifest_sha256'],
global_tick_offset=offset, global_next_tick=offset+steps, segment_steps=steps,
update_root_authority_sha256=authority['update_root_authority_sha256'], input_policy=value['input_policy'],
payload_sha256=payload, weights_updated_during_run=False, counts_scope='segment_only_not_lifetime',
checkpoint_continuation_verified_by_GPU_comparison=False, learning_pass_claimed=False)
R.require(_canonical(receipt) == _canonical(expected_receipt), 'New receipt payload/ancestry mismatch')
restored = dict(v=parent['state']['v'], g=parent['state']['g'], ref=parent['state']['ref'],
spike_ring=C.rotate_spike_ring(parent['terminal']['spike_ring'], int(parent['terminal']['next_tick'])),
apl_pending=parent['terminal']['apl_pending'], apl_state=parent['terminal']['apl_state'])
boundary = R._read_json(directory / 'boundary-restore-audit.json')
R.require(boundary == dict(restored_raw_sha256={k:R._raw(a) for k,a in restored.items()},
all_state_and_queue_bytes_exact=True, existing_g_rescaled=False, retroactive_arrival_reweighting=False,
pending_spike_arrivals_use_current_installed_weight=True, pending_APL_samples_preserved=True,
boundary_installations=1, restorations=1, learning_pass_claimed=False), 'Boundary restore audit differs from real parent')
return dict(directory=directory, summary=summary, provenance=provenance, overlay=overlay, state=state, terminal=terminal,
steps=steps, global_tick_offset=offset, global_next_tick=offset+steps, fresh=False,
ancestry_directories=[directory, *parent['ancestry_directories']], input_policy=value['input_policy'],
update_root_authority_sha256=authority['update_root_authority_sha256'])
def load_checkpoint(checkpoint_dir, expected_sha, *, _seen=None):
directory = Path(checkpoint_dir).resolve(strict=True)
R.require(R._sha(expected_sha) and _digest(directory / MANIFEST) == expected_sha, 'External checkpoint pin differs')
manifest = R._read_json(directory / MANIFEST)
if manifest.get('schema') == 'phase4an-checkpoint-v1':
return C.load_checkpoint(directory, expected_sha, _seen=_seen)
seen = set() if _seen is None else set(_seen)
R.require(directory not in seen, 'Updated checkpoint ancestry cycle')
seen.add(directory)
R.require(set(manifest) == {'schema', 'complete', 'files_sha256', 'segment_steps', 'global_tick_offset', 'global_next_tick',
'counts_scope', 'update_root_authority_sha256', 'input_policy', 'learning_pass_claimed',
'checkpoint_continuation_verified_by_GPU_comparison'} and
manifest.get('schema') == CHECKPOINT_SCHEMA and manifest.get('complete') is True and
manifest.get('learning_pass_claimed') is False and manifest.get('checkpoint_continuation_verified_by_GPU_comparison') is False and
manifest.get('counts_scope') == 'segment_only' and manifest.get('files_sha256') == C._tree_hashes(directory), 'New checkpoint exact inventory differs')
record = _record(directory, seen)
for key in ('global_tick_offset', 'global_next_tick', 'update_root_authority_sha256', 'input_policy'):
R.require(manifest.get(key) == record[key], 'Checkpoint metadata mismatch: ' + key)
R.require(manifest.get('segment_steps') == record['steps'] and manifest['files_sha256'] == C._tree_hashes(directory) and
_digest(directory / MANIFEST) == expected_sha, 'Checkpoint changed while loading')
record.update(manifest=manifest, manifest_sha256=expected_sha)
return record
def seal_checkpoint(checkpoint_dir):
directory = Path(checkpoint_dir).resolve(strict=True)
R.require(not (directory / MANIFEST).exists(), 'Checkpoint is already sealed')
before = C._tree_hashes(directory)
record = _record(directory, {directory})
R.require(before == C._tree_hashes(directory), 'Checkpoint changed while sealing')
value = dict(schema=CHECKPOINT_SCHEMA, complete=True, files_sha256=before, segment_steps=record['steps'],
global_tick_offset=record['global_tick_offset'], global_next_tick=record['global_next_tick'],
counts_scope='segment_only', update_root_authority_sha256=record['update_root_authority_sha256'],
input_policy=record['input_policy'], learning_pass_claimed=False,
checkpoint_continuation_verified_by_GPU_comparison=False)
R._save(directory / MANIFEST, value)
return dict(path=str(directory / MANIFEST), sha256=_digest(directory / MANIFEST), **value)
def run(authority_json, authority_sha256, out_dir, build_root, *, mode='light'):
R.require(sys.dont_write_bytecode, 'Use python -B')
admitted = validate_authority(authority_json, authority_sha256)
value, parent, spec = admitted['value'], admitted['parent'], admitted['spec']
R.require(mode == value['diagnostic_mode'], 'Mode differs from authority')
protected = [*parent['ancestry_directories'], *(Path(path).parent for path in admitted['pins'])]
case, out, build = R.execution_paths(admitted['case_dir'], out_dir, build_root, protected)
ajdir = Path(admitted['after_meta']['aj_model_dir'])
R._load('timing_queue', ajdir / 'timing_queue.py', R.PINS['apl-timing-lab-v1/timing_queue.py'])
aj = R._load('conditioning_updated_sealed_AJ', ajdir / 'gpu_timing.py', R.AJ_SHA)
runtime = UpdatedRuntime(mode, spec, admitted['helper'], admitted['after_plan'], admitted['after_meta'],
admitted['pins'], aj.torch, admitted['paths']['after_overlay'], admitted['paths']['after_manifest'],
parent=parent, authority=admitted)
function, ast_audit = compile_updated(aj, runtime, mode)
try:
inherited = function(case, out, build)
R.require(runtime.finished and runtime.restorations == runtime.installations == 1 and inherited['complete'], 'Incomplete new continuation')
R.require(inherited['effective_weights_raw_sha256'] == value['expected_effective_weight_after_sha256'], 'Runtime effective identity differs')
_check(admitted['pins'])
load_checkpoint(parent['directory'], parent['manifest_sha256'])
summary = R._read_json(out / 'summary.json')
R.require(summary == runtime.summary(inherited), 'Stored updated summary differs')
R._save(out / 'boundary-restore-audit.json', runtime.restore_audit)
provenance = dict(schema=PROVENANCE_SCHEMA, complete=True, mode=mode,
source_file=str(Path(__file__).resolve()), source_sha256=_digest(__file__),
authority_file=str(admitted['authority']), authority_sha256=authority_sha256,
ast=ast_audit, input_sha256=admitted['pins'], old_globals_patched=False, weights_updated_during_run=False,
global_tick_offset=parent['global_next_tick'], global_next_tick=parent['global_next_tick']+spec['steps'],
parent_checkpoint_manifest_sha256=parent['manifest_sha256'], U_recomputed_here=False,
external_CPU_validation_receipt_bound=value['transition'] == 'epoch_update',
external_registration_process_and_probe_semantics_verified_here=False, learning_pass_claimed=False)
for key, name in (('summary_sha256','summary.json'), ('finite_audit_sha256','finite-audit.json'),
('terminal_queues_sha256','terminal-queues.npz'), ('overlay_audit_sha256','overlay-audit.json'), ('state_sha256','state.npz')):
provenance[key] = _digest(out / name)
R._save(out / 'runtime-provenance.json', provenance)
receipt = dict(schema='conditioning-updated-receipt-v1', complete=True,
authority_file=str(admitted['authority']), authority_sha256=authority_sha256,
parent_checkpoint_dir=str(parent['directory']), parent_checkpoint_manifest_sha256=parent['manifest_sha256'],
global_tick_offset=parent['global_next_tick'], global_next_tick=parent['global_next_tick']+spec['steps'],
segment_steps=spec['steps'], update_root_authority_sha256=admitted['update_root_authority_sha256'],
input_policy=value['input_policy'], payload_sha256=C._tree_hashes(out), weights_updated_during_run=False,
counts_scope='segment_only_not_lifetime', checkpoint_continuation_verified_by_GPU_comparison=False, learning_pass_claimed=False)
R._save(out / RECEIPT, receipt)
sealed = seal_checkpoint(out)
return dict(summary, checkpoint_manifest_sha256=sealed['sha256'])
except Exception as error:
if out.is_dir() and not (out / 'updated-error.json').exists():
R._save(out / 'updated-error.json', dict(complete=False, type=type(error).__name__, message=str(error), retry_forbidden=True))
raise