Seal AO interface failure and verified isolated postprocessing diagnosis
This commit is contained in:
Digital Drosophila Archive
@@ -0,0 +1,312 @@
|
||||
"""Exclusive postprocessing of one externally authorized sealed fresh epoch.
|
||||
|
||||
No brain execution, source selection, input generation or weight installation.
|
||||
The external caller must derive and pin authority from its registered row and
|
||||
successful process receipt; a SHA supplied to this helper is not self-proving
|
||||
registration authority. Production uses the actual pinned C/R checkpoint and
|
||||
overlay readers. Small synthetic tests exercise only the private core.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from types import ModuleType
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
SCHEMA = 'phase4an-epoch-postprocess-authority-v1'
|
||||
SOURCE_PINS = {
|
||||
'runtime_light_source': '522b8c31ea938890fe15364fccd25ccc5bded54101754909f255532e7ad01aae',
|
||||
'runtime_continuation_source': '0b8e8e0a6a23a079dc62a88355ce355d53b3ae22f6665519a8f6aef2dcfc5aba',
|
||||
'conditioning_update_source': '96efff678cff910654eb08e86f1a17789e4e5589e340345f419c7027a7d547e2',
|
||||
'eligibility_bridge_source': '174fb8d14619189e74b095e77a6db18b14c93046dc431c0b55710a5aa2dd2e61',
|
||||
'phase4j_source': 'abb4574da85f00bd9ef61024c65993ef81d42651ff8d8d63c15dd69c6222471e',
|
||||
'am_overlay_source': '7f891d0ba79b393c96e75fd2dee2eddc51e57d8213eaac2779703bc2c18712b4',
|
||||
}
|
||||
ROLES = frozenset(('spec', 'requests', 'overlay_before', 'overlay_manifest_before', 'state',
|
||||
'kc', 'canonical_kc', 'candidate_map', 'mask', 'base_candidate', *SOURCE_PINS))
|
||||
GRAPH_FILES = ('nodes.npy', 'edge_id.npy', 'src.npy', 'dst.npy', 'signed_raw.npy', 'ptr.npy', 'weights.npy')
|
||||
APL_FILES = ('input_matrix.npy', 'output_matrix.npy', 'kc_globals.npy', 'apl_globals.npy')
|
||||
DIMENSIONS = (166700, 25582938, 4064, 14232)
|
||||
_MODULES = {}
|
||||
|
||||
|
||||
def _require(ok, message):
|
||||
if not ok:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _sha(value):
|
||||
return type(value) is str and len(value) == 64 and all(c in '0123456789abcdef' for c in value)
|
||||
|
||||
|
||||
def _digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as stream:
|
||||
for chunk in iter(lambda: stream.read(8 << 20), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _read(path):
|
||||
def pairs(items):
|
||||
value = {}
|
||||
for key, item in items:
|
||||
_require(key not in value, 'Duplicate JSON key: ' + key)
|
||||
value[key] = item
|
||||
return value
|
||||
return json.loads(Path(path).read_text(encoding='utf-8'), object_pairs_hook=pairs,
|
||||
parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Nonfinite JSON: ' + value)))
|
||||
|
||||
|
||||
def _save(path, value):
|
||||
with Path(path).open('x', encoding='utf-8') as stream:
|
||||
json.dump(value, stream, sort_keys=True, indent=2, allow_nan=False)
|
||||
stream.write('\n')
|
||||
|
||||
|
||||
def _path(value):
|
||||
_require(type(value) is str and Path(value).is_absolute(), 'Input paths must be explicit and absolute')
|
||||
path = Path(value)
|
||||
_require(path.is_file() and not path.is_symlink(), 'Input must be an existing regular, unlinked file')
|
||||
return path.resolve(strict=True)
|
||||
|
||||
|
||||
def _check_files(pins):
|
||||
for path, expected in pins.items():
|
||||
_require(_sha(expected) and _digest(path) == expected, 'Input/source SHA mismatch: ' + str(path))
|
||||
|
||||
|
||||
def _authority(path, expected):
|
||||
path = _path(str(path))
|
||||
_require(_sha(expected) and _digest(path) == expected, 'Authority differs from external SHA')
|
||||
value = _read(path)
|
||||
keys = {'schema', 'epoch_id', 'diagnostic_mode', 'command', 'checkpoint_dir',
|
||||
'checkpoint_manifest_sha256', 'files', 'frozen_inputs'}
|
||||
_require(type(value) is dict and set(value) == keys and value['schema'] == SCHEMA, 'Invalid authority fields/schema')
|
||||
_require(type(value['epoch_id']) is str and value['epoch_id'] and value['epoch_id'].strip() == value['epoch_id'], 'Invalid epoch identity')
|
||||
_require(value['diagnostic_mode'] in ('light', 'reference'), 'Invalid diagnostic mode')
|
||||
command = value['command']
|
||||
_require(type(command) is dict and set(command) == {'steps', 'compartment', 'us_tick', 'd_command', 'frozen', 'trace_backend'}, 'Invalid command fields')
|
||||
_require(type(command['steps']) is int and command['steps'] > 0, 'Invalid command steps')
|
||||
_require(command['us_tick'] is None or (type(command['us_tick']) is int and 0 <= command['us_tick'] < command['steps']), 'Invalid US tick')
|
||||
_require(type(command['d_command']) in (int, float) and math.isfinite(command['d_command']) and command['d_command'] in (0, 1), 'Invalid scheduled D')
|
||||
_require(command['us_tick'] is not None or command['d_command'] == 0, 'No US requires D=0')
|
||||
_require(type(command['frozen']) is bool and command['trace_backend'] in ('numpy', 'torch'), 'Explicit frozen/backend required')
|
||||
_require(command['compartment'] in ("B2B'2a", 'a1', "y2a'1", "y5B'2a"), 'Explicit frozen compartment required')
|
||||
_require(type(value['files']) is dict and set(value['files']) == ROLES, 'Missing/unknown authority file roles')
|
||||
paths, pins = {}, {path: expected}
|
||||
for role, entry in value['files'].items():
|
||||
_require(type(entry) is dict and set(entry) == {'path', 'sha256'} and _sha(entry['sha256']), 'Invalid file authority: ' + role)
|
||||
location = _path(entry['path'])
|
||||
_require(location not in pins or pins[location] == entry['sha256'], 'Conflicting hashes for the same file')
|
||||
paths[role], pins[location] = location, entry['sha256']
|
||||
_require(type(value['frozen_inputs']) is dict and value['frozen_inputs'], 'Full frozen input pins are required')
|
||||
for name, sha in value['frozen_inputs'].items():
|
||||
location = _path(name)
|
||||
_require(_sha(sha) and (location not in pins or pins[location] == sha), 'Invalid/conflicting frozen input pin')
|
||||
pins[location] = sha
|
||||
checkpoint = Path(value['checkpoint_dir'])
|
||||
_require(checkpoint.is_absolute() and checkpoint.is_dir() and not checkpoint.is_symlink(), 'Explicit existing checkpoint directory required')
|
||||
checkpoint = checkpoint.resolve(strict=True)
|
||||
manifest = checkpoint / 'checkpoint-manifest.json'
|
||||
_require(_sha(value['checkpoint_manifest_sha256']), 'External checkpoint manifest SHA required')
|
||||
_require(manifest not in pins or pins[manifest] == value['checkpoint_manifest_sha256'], 'Conflicting checkpoint manifest pins')
|
||||
pins[manifest] = value['checkpoint_manifest_sha256']
|
||||
_require(paths['state'] == checkpoint / 'state.npz', 'Events must come from this checkpoint state.npz')
|
||||
_check_files(pins)
|
||||
return value, paths, pins, checkpoint, path
|
||||
|
||||
|
||||
def _load_source(path, expected, role):
|
||||
raw = path.read_bytes()
|
||||
_require(hashlib.sha256(raw).hexdigest() == expected, 'Pinned source changed: ' + role)
|
||||
name = '_conditioning_epoch_' + role
|
||||
if name in _MODULES:
|
||||
saved_path, saved_sha, module = _MODULES[name]
|
||||
_require(saved_path == path and saved_sha == expected and sys.modules.get(name) is module,
|
||||
'Source module identity collision: ' + role)
|
||||
return module
|
||||
_require(name not in sys.modules, 'Unverified source module collision: ' + role)
|
||||
module = ModuleType(name)
|
||||
module.__file__ = str(path)
|
||||
sys.modules[name] = module
|
||||
try:
|
||||
exec(compile(raw, str(path), 'exec'), module.__dict__)
|
||||
except BaseException:
|
||||
sys.modules.pop(name, None)
|
||||
raise
|
||||
_MODULES[name] = (path, expected, module)
|
||||
return module
|
||||
|
||||
|
||||
def _production_dependencies(paths, pins):
|
||||
for role, expected in SOURCE_PINS.items():
|
||||
_require(pins[paths[role]] == expected, 'Authority cannot replace frozen source: ' + role)
|
||||
_require(paths['runtime_continuation_source'].parent / 'runtime_light.py' == paths['runtime_light_source'],
|
||||
'Continuation must load its declared adjacent frozen runtime')
|
||||
c = _load_source(paths['runtime_continuation_source'], SOURCE_PINS['runtime_continuation_source'], 'continuation')
|
||||
_require(Path(c.R.__file__).resolve() == paths['runtime_light_source'] and
|
||||
_digest(c.R.__file__) == SOURCE_PINS['runtime_light_source'], 'Loaded actual runtime differs from authority')
|
||||
u = _load_source(paths['conditioning_update_source'], SOURCE_PINS['conditioning_update_source'], 'update')
|
||||
return c, c.R, u
|
||||
|
||||
|
||||
def _npz(path, keys):
|
||||
with np.load(path, allow_pickle=False) as archive:
|
||||
_require(len(archive.files) == len(keys) and set(archive.files) == set(keys), 'Unexpected NPZ schema: ' + str(path))
|
||||
return {key: archive[key].copy() for key in keys}
|
||||
|
||||
|
||||
def _same(a, b):
|
||||
return a.dtype == b.dtype and a.shape == b.shape and a.tobytes() == b.tobytes()
|
||||
|
||||
|
||||
def _vector(a, dtype, count, label):
|
||||
_require(a.dtype == np.dtype(dtype) and a.shape == (count,), 'Invalid dtype/shape: ' + label)
|
||||
|
||||
|
||||
def _coordinate(value, paths, pins, checkpoint, authority_path, out_dir, dependencies, dimensions):
|
||||
"""Private core: explicit tiny-fixture dependencies only in synthetic tests."""
|
||||
c, r, u = dependencies
|
||||
n, edges, kc_count, candidates = dimensions
|
||||
out = Path(out_dir).resolve(strict=False)
|
||||
protected = {checkpoint, *(path.parent for path in pins)}
|
||||
_require(not out.exists() and not any(out.is_relative_to(root) for root in protected),
|
||||
'Output must be new and outside checkpoint/source/data/input parent directories')
|
||||
_require(out.parent.is_dir(), 'Output parent must already exist')
|
||||
source_sha = _digest(__file__)
|
||||
spec = r.validate_spec(_read(paths['spec']), paths['spec'].parent)
|
||||
command = value['command']
|
||||
_require(type(spec['steps']) is int and spec['steps'] == command['steps'], 'Command steps differ from authorized spec')
|
||||
_require(paths['spec'].name == 'spec.json' and paths['requests'] == paths['spec'].parent / 'requests.npz', 'Actual case input filenames differ')
|
||||
graph = Path(spec['graph_dir']).resolve(strict=True)
|
||||
apl = Path(spec['apl_bundle_dir']).resolve(strict=True)
|
||||
required_inputs = [graph / name for name in GRAPH_FILES] + [apl / name for name in APL_FILES] + [Path(spec['coefficients_path']).resolve(strict=True)]
|
||||
_require(all(path in pins for path in required_inputs), 'Full graph/APL/coefficient authority is incomplete')
|
||||
_require(paths['canonical_kc'] == apl / 'kc_globals.npy', 'Canonical KC must be the declared frozen APL/KC membership')
|
||||
old_meta = _read(paths['overlay_manifest_before'])
|
||||
for role, prefix in (('mask', 'mask'), ('base_candidate', 'base_candidate'), ('am_overlay_source', 'am_overlay')):
|
||||
_require(Path(old_meta[prefix + '_path']).resolve(strict=True) == paths[role] and
|
||||
old_meta[prefix + '_sha256'] == pins[paths[role]], 'Overlay authority differs: ' + role)
|
||||
aj = Path(old_meta['aj_model_dir']).resolve(strict=True)
|
||||
for relative, expected in r.PINS.items():
|
||||
path = (aj.parent / relative).resolve(strict=True)
|
||||
_require(pins.get(path) == expected, 'Missing exact frozen dependency pin: ' + relative)
|
||||
record = c.load_checkpoint(checkpoint, value['checkpoint_manifest_sha256'])
|
||||
_require(record['fresh'] is True and record['global_tick_offset'] == 0 and record['steps'] == command['steps'], 'Only complete fresh epochs may produce an update')
|
||||
summary, provenance = record['summary'], record['provenance']
|
||||
_require(summary['nodes'] == n and summary['edges'] == edges and summary['diagnostic_mode'] == value['diagnostic_mode'], 'Runtime population/mode differs from authority')
|
||||
_require(record['manifest']['files_sha256']['state.npz'] == pins[paths['state']], 'State/events are not bound by the expected checkpoint seal')
|
||||
for role in ('spec', 'requests', 'overlay_before', 'overlay_manifest_before'):
|
||||
_require(provenance['input_sha256'].get(str(paths[role])) == pins[paths[role]], 'Runtime did not bind authorized ' + role)
|
||||
_require(_digest(checkpoint / 'overlay-input.npz') == pins[paths['overlay_before']] and
|
||||
_digest(checkpoint / 'overlay-manifest.json') == pins[paths['overlay_manifest_before']], 'Sealed runtime used a different overlay or manifest')
|
||||
helper, before, checked_meta, _ = r.load_overlay(paths['overlay_before'], paths['overlay_manifest_before'])
|
||||
_require(checked_meta == old_meta, 'Overlay manifest changed during validation')
|
||||
mapping = _npz(paths['candidate_map'], ('edge_id', 'source_neuron_idx', 'target_neuron_idx', 'target_compartment_verbatim'))
|
||||
mask = np.load(paths['mask'], allow_pickle=False)
|
||||
kc = np.load(paths['kc'], allow_pickle=False)
|
||||
canonical_kc = np.load(paths['canonical_kc'], allow_pickle=False)
|
||||
base = np.load(paths['base_candidate'], allow_pickle=False)
|
||||
_vector(mask, 'int64', candidates, 'complete mask')
|
||||
_vector(kc, 'int32', kc_count, 'KC')
|
||||
_vector(canonical_kc, 'int32', kc_count, 'canonical KC')
|
||||
_vector(base, 'float32', candidates, 'original base mV')
|
||||
_require(_same(kc, canonical_kc), 'Supplied KC differs from canonical membership bytes')
|
||||
_require(_same(mapping['edge_id'], mask) and _same(before['edge_id'], mask), 'Candidate mapping/overlay differ from exact mask')
|
||||
_require(_same(before['base_candidate_mv'], base), 'Overlay used different original base bytes')
|
||||
for key in ('source_neuron_idx', 'target_neuron_idx'):
|
||||
_vector(mapping[key], 'int32', candidates, key)
|
||||
comps = mapping['target_compartment_verbatim']
|
||||
_require(comps.shape == (candidates,) and comps.dtype.kind == 'U', 'Compartment mapping must be an exact Unicode vector')
|
||||
maps = {}
|
||||
try:
|
||||
for filename, dtype, length in (('nodes.npy', 'int32', n), ('edge_id.npy', 'int64', edges),
|
||||
('src.npy', 'int32', edges), ('dst.npy', 'int32', edges)):
|
||||
array = np.load(graph / filename, allow_pickle=False, mmap_mode='r')
|
||||
maps[filename] = array
|
||||
_vector(array, dtype, length, 'full graph ' + filename)
|
||||
_require(np.array_equal(maps['nodes.npy'], np.arange(n, dtype=np.int32)), 'Global nodes are not canonical')
|
||||
_require(np.all((mask >= 0) & (mask < edges)) and np.all(mask[1:] > mask[:-1]), 'Invalid canonical edge mask')
|
||||
_require(_same(maps['edge_id.npy'][mask], mask), 'Mask is not indexed by original global edge ID')
|
||||
_require(_same(mapping['source_neuron_idx'], maps['src.npy'][mask]) and
|
||||
_same(mapping['target_neuron_idx'], maps['dst.npy'][mask]), 'Candidate endpoints differ from actual graph')
|
||||
finally:
|
||||
for array in maps.values():
|
||||
array._mmap.close()
|
||||
state = _npz(paths['state'], ('step', 'neuron_idx', 'counts', 'v', 'g', 'ref'))
|
||||
_require(all(_same(state[key], record['state'][key]) for key in state), 'Event/state arrays differ from actual sealed reader')
|
||||
_require(record['overlay']['delta_raw_sha256'] == r._raw(before['plastic_delta_mv']) and
|
||||
record['overlay']['effective_weights_raw_sha256'] == summary['effective_weights_raw_sha256'], 'Runtime weight installation is not bound to delta_before')
|
||||
args = dict(step=state['step'], neuron_idx=state['neuron_idx'], kc=kc,
|
||||
candidate_edge_id=mask, candidate_sources=mapping['source_neuron_idx'],
|
||||
candidate_targets=mapping['target_neuron_idx'], candidate_compartments=comps,
|
||||
delta_before=before['plastic_delta_mv'], base_candidate=base, steps=command['steps'], n_neurons=n,
|
||||
compartment=command['compartment'], us_tick=command['us_tick'], d_command=command['d_command'],
|
||||
frozen=command['frozen'], bridge_source=paths['eligibility_bridge_source'],
|
||||
phase4j_source=paths['phase4j_source'], trace_backend=command['trace_backend'])
|
||||
result = u.update_epoch(**args)
|
||||
_check_files(pins)
|
||||
# The directory creation is the exclusive claim. A later failure remains a
|
||||
# partial directory and can never silently be retried or overwritten.
|
||||
out.mkdir()
|
||||
try:
|
||||
with (out / 'delta-after.npz').open('xb') as stream:
|
||||
np.savez_compressed(stream, edge_id=mask, plastic_delta_mv=result.delta_after)
|
||||
metadata = {key: old_meta[key] for key in ('schema', 'mask_path', 'mask_sha256',
|
||||
'base_candidate_path', 'base_candidate_sha256', 'aj_model_dir', 'am_overlay_path', 'am_overlay_sha256')}
|
||||
metadata.update(overlay_name='learned', overlay_sha256=_digest(out / 'delta-after.npz'),
|
||||
parent_snapshot_sha256=pins[paths['overlay_before']], eligibility_v2_source_sha256=SOURCE_PINS['phase4j_source'])
|
||||
_save(out / 'overlay-manifest.json', metadata)
|
||||
_save(out / 'update-receipt.json', result.receipt)
|
||||
_, after, _, _ = r.load_overlay(out / 'delta-after.npz', out / 'overlay-manifest.json')
|
||||
_require(_same(after['plastic_delta_mv'], result.delta_after) and _same(after['edge_id'], mask), 'Written learned snapshot failed actual runtime loader')
|
||||
# Re-read the sealed neural state after writing, without altering its
|
||||
# seal or appending coordinator output inside the runtime directory.
|
||||
c.load_checkpoint(checkpoint, value['checkpoint_manifest_sha256'])
|
||||
_check_files(pins)
|
||||
_require(_digest(__file__) == source_sha, 'Coordinator source changed')
|
||||
output_hashes = {name: _digest(out / name) for name in ('delta-after.npz', 'overlay-manifest.json', 'update-receipt.json')}
|
||||
chain = dict(schema='phase4an-epoch-postprocess-chain-v1', complete=True,
|
||||
epoch_id=value['epoch_id'], authority_file=str(authority_path), authority_sha256=pins[authority_path],
|
||||
coordinator_source_sha256=source_sha, command=dict(command), diagnostic_mode=value['diagnostic_mode'],
|
||||
before_overlay_file_sha256=pins[paths['overlay_before']],
|
||||
before_overlay_manifest_sha256=pins[paths['overlay_manifest_before']],
|
||||
after_overlay_file_sha256=output_hashes['delta-after.npz'],
|
||||
after_overlay_manifest_sha256=output_hashes['overlay-manifest.json'],
|
||||
parent_snapshot_sha256_semantics='delta-before overlay NPZ file SHA256, not neural checkpoint or manifest SHA',
|
||||
source_neural_checkpoint_dir=str(checkpoint),
|
||||
source_neural_checkpoint_manifest_sha256=value['checkpoint_manifest_sha256'],
|
||||
source_state_file_sha256=pins[paths['state']], source_spec_sha256=pins[paths['spec']],
|
||||
source_requests_sha256=pins[paths['requests']],
|
||||
runtime_delta_before_raw_sha256=record['overlay']['delta_raw_sha256'],
|
||||
runtime_effective_weights_raw_sha256=summary['effective_weights_raw_sha256'],
|
||||
input_files_sha256={str(path): sha for path, sha in sorted(pins.items(), key=lambda item: str(item[0]))},
|
||||
output_files_sha256=output_hashes,
|
||||
update_receipt_payload_sha256=result.receipt['payload_sha256'],
|
||||
runtime_learned_manifest_loader_verified=True, input_and_checkpoint_bytes_reverified=True,
|
||||
candidate_endpoints_equal_graph=True, KC_equals_supplied_canonical_membership=True,
|
||||
authority_scope='caller must bind this externally pinned authority to its preregistered row, successful process, anatomical metadata and complete mask audit',
|
||||
registration_authority_independently_proven=False, anatomical_identity_inferred_from_lengths=False,
|
||||
neural_checkpoint_and_plastic_parent_are_distinct=True,
|
||||
weights_installed=False, brain_executed=False, learning_pass_claimed=False, candidate_generated=False)
|
||||
_save(out / 'chain-receipt.json', chain)
|
||||
return chain
|
||||
except BaseException as error:
|
||||
if not (out / 'postprocess-error.json').exists():
|
||||
_save(out / 'postprocess-error.json', dict(complete=False, error_type=type(error).__name__,
|
||||
message=str(error), partial_preserved=True, retry_forbidden=True, learning_pass_claimed=False))
|
||||
raise
|
||||
|
||||
|
||||
def postprocess_epoch(authority_json, expected_authority_sha256, out_dir):
|
||||
"""Production entry: full 166700/25582938/4064/14232, no small-mode flag."""
|
||||
_require(sys.dont_write_bytecode, 'Use python -B; frozen sources must not receive bytecode writes')
|
||||
value, paths, pins, checkpoint, authority_path = _authority(authority_json, expected_authority_sha256)
|
||||
dependencies = _production_dependencies(paths, pins)
|
||||
return _coordinate(value, paths, pins, checkpoint, authority_path, out_dir, dependencies, DIMENSIONS)
|
||||
Reference in New Issue
Block a user