Seal AO interface failure and verified isolated postprocessing diagnosis

This commit is contained in:
Digital Drosophila Archive
2026-09-30 17:57:02 +08:00
parent 4ff241bdc9
commit 8e9f8f4d56
126 changed files with 92124 additions and 3 deletions
@@ -0,0 +1,373 @@
"""Deterministic, read-only CPU analysis of the registered 32-call AN gate.
analyze(reg, reg_sha, verified_records) accepts the actual runtime receipts
already verified by engineering_workflow.verify_execution. It does not run
that workflow, publish, seal, save, retry or import a GPU runtime. All external
AM seals/checkpoint chains/process receipts remain the caller's prior duty.
This module rehashes the files it reads, executes every canonical descriptor,
and independently reconstructs the reference queues and effective SI weights.
"""
from __future__ import annotations
import hashlib
import json
import math
from pathlib import Path
import types
import numpy as np
import engineering_compare as compare
import engineering_plan as plan_module
import engineering_queue_audit as queue_audit
import engineering_scopes as scopes
N, E, K = 166700, 25582938, 4064
CANDIDATES, FAST_APL_EDGES = 14232, 5845
AM_OVERLAY_SHA = '7f891d0ba79b393c96e75fd2dee2eddc51e57d8213eaac2779703bc2c18712b4'
MASK_SHA = '8b0fb6eb2e6c31da4d17dc20d890aa3f367e514f05642de5bad6b42a1611c05b'
SUCCESS = 'PASS-EXACT-LIGHT-CONTINUATION-ENGINEERING-NOT-LEARNING'
def require(value, message):
if not value:
raise ValueError(message)
def _canonical(value):
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False)
def _sha_file(path):
h = hashlib.sha256()
with Path(path).open('rb') as stream:
for data in iter(lambda: stream.read(8 << 20), b''):
h.update(data)
return h.hexdigest()
class _Reader:
"""Fresh byte hashes, not stat caching; deterministic read manifest."""
def __init__(self, registered):
self.registered = registered
self.reads = {}
self.arrays = []
def bind(self, path, *, registered=True, expected=None):
path = Path(path)
require(path.is_absolute() and path.is_file() and not path.is_symlink(), 'Missing/linked/relative evidence: ' + str(path))
path = path.resolve(strict=True)
if registered:
require(str(path) in self.registered, 'Unregistered dependency: ' + str(path))
require(expected is None or expected == self.registered[str(path)], 'Conflicting external hash authority')
expected = self.registered[str(path)]
actual = _sha_file(path)
require(expected is None or (compare._sha(expected) and actual == expected), 'Evidence bytes changed: ' + str(path))
value = dict(path=str(path), bytes=path.stat().st_size, sha256=actual)
require(str(path) not in self.reads or self.reads[str(path)] == value, 'File changed during analysis')
self.reads[str(path)] = value
return path
def json(self, path, **kwargs):
return compare._json(self.bind(path, **kwargs))
def npy(self, path):
value = np.load(self.bind(path), mmap_mode='r', allow_pickle=False)
self.arrays.append(value)
return value
def npz(self, path, keys):
with np.load(self.bind(path), allow_pickle=False) as archive:
require(len(archive.files) == len(keys) and set(archive.files) == set(keys), 'Unexpected input NPZ keys')
return {name: archive[name].copy() for name in keys}
def run(self, directory, *, legacy):
directory = Path(directory)
names = ['summary.json', 'state.npz', 'apl-timing-audit.json', 'apl-pending-tail.float32.npy', 'requests.csv']
if legacy:
names.append('adapter-provenance.json')
else:
names.append('terminal-queues.npz')
names += [name for name in [*compare.TRACE_FILES.values(), 'tick-state-hashes.jsonl', 'queue.jsonl']
if (directory / name).is_file()]
for name in names:
self.bind(directory / name, registered=legacy)
return compare.load_run(directory, allow_legacy_am=legacy)
def finish(self):
for row in self.reads.values():
path = Path(row['path'])
require(path.is_file() and path.stat().st_size == row['bytes'] and _sha_file(path) == row['sha256'],
'Read evidence changed during analysis: ' + str(path))
return [self.reads[key] for key in sorted(self.reads)]
def close(self):
for value in self.arrays:
mmap = getattr(value, '_mmap', None)
if mmap is not None:
mmap.close()
def _registration(reg, reg_sha, verified):
require(type(reg) is dict and reg.get('schema') == 'phase4an-engineering-registration-v1' and
reg.get('training') is False and compare._sha(reg_sha), 'Invalid engineering registration')
plan = plan_module.validate_plan(reg['plan'])
ids = [row['row_id'] for row in plan['rows']]
require(type(verified) is dict and set(verified) == set(ids) and set(reg['cases']) == set(ids),
'Exactly all 32 registered runtime receipts/cases are required')
exp = None
for row in plan['rows']:
identity = row['row_id']; path = Path(reg['cases'][identity]['case_dir'])
require(path.is_absolute() and path.name == identity and path.parent.name == 'cases', 'Invalid registered case layout')
if exp is None:
exp = path.parent.parent
require(path == exp / 'cases' / identity, 'Cases span inconsistent experiment roots')
receipt = verified[identity]
require(receipt.get('schema') == 'phase4an-engineering-runtime-record-v1' and receipt.get('complete') is True and
receipt.get('training') is False and receipt.get('registration_sha256') == reg_sha and
_canonical(receipt.get('row')) == _canonical(row) and compare._sha(receipt.get('checkpoint_manifest_sha256')),
'Runtime receipt is not the exact verified registered row: ' + identity)
require(exp.name == 'phase4an-engineering-validation' and exp.parent.name == 'experiments', 'Unexpected fixed engineering output root')
require(set(reg['overlays']) == set(reg['selected_case']['AM_outputs']) == {'zero', 'static25'}, 'Incomplete overlay/AM authority')
return plan, exp
def _materialize(reader, reg, graph, stage):
"""Use the exact pinned pure AM helper; no original globals are patched."""
declaration = reg['overlays'][stage]
meta = reader.json(declaration['overlay_manifest'])
require(meta.get('schema') == 'phase4an-overlay-v1' and meta.get('overlay_name') == stage and
meta.get('am_overlay_sha256') == AM_OVERLAY_SHA and meta.get('mask_sha256') == MASK_SHA,
'Original fixed overlay authority changed')
helper_path = reader.bind(meta['am_overlay_path'], expected=AM_OVERLAY_SHA)
module = types.ModuleType('phase4an_analysis_pinned_overlay')
module.__file__ = str(helper_path)
exec(compile(helper_path.read_bytes(), str(helper_path), 'exec'), module.__dict__)
mask = reader.npy(meta['mask_path']); base_candidate = reader.npy(meta['base_candidate_path'])
require(reader.reads[str(Path(meta['mask_path']).resolve())]['sha256'] == meta['mask_sha256'] and
reader.reads[str(Path(meta['base_candidate_path']).resolve())]['sha256'] == meta['base_candidate_sha256'],
'Overlay external mask/base hash mismatch')
overlay_file = reader.bind(declaration['overlay_file'], expected=meta['overlay_sha256'])
arrays = reader.npz(overlay_file, {'edge_id', 'plastic_delta_mv'})
require(mask.dtype == np.int64 and mask.shape == (CANDIDATES,), 'Complete frozen plastic mask required')
values = module.validate_arrays(arrays['edge_id'], arrays['plastic_delta_mv'], mask, base_candidate)
if stage == 'zero':
require(not np.any(values['plastic_delta_mv'] != 0), 'Zero overlay is nonzero')
else:
require(module.bit_equal(values['plastic_delta_mv'], np.multiply(np.float32(-.25), base_candidate)),
'static25 differs from the original float32-mV floor')
effective, mapping = module.materialize_numpy(graph['weights'], graph['edge_id'], graph['apl_out_mask'], values)
require(mapping['mapped_candidate_count'] == CANDIDATES and mapping['missing_candidate_count'] == 0 and
mapping['apl_overlap_count'] == 0, 'Incomplete/overlapping candidate mapping')
base_sha, effective_sha = module.expected_hashes(graph['weights'], graph['apl_out_mask'], mapping)
require(queue_audit._raw_sha(effective) == effective_sha, 'CPU materialization/hash disagreement')
return effective, dict(effective_weights_raw_sha256=effective_sha, base_weights_raw_sha256=base_sha,
candidate_count=CANDIDATES, installed_nonzero_count=mapping['installed_nonzero_count'],
apl_fast_edges_zeroed=FAST_APL_EDGES, bridge=module.BRIDGE,
helper_source_sha256=AM_OVERLAY_SHA, weights_updated_during_run=False)
def _load_context(reg, reg_sha, plan, exp):
reader = _Reader(reg['input_sha256'])
records, legacy = {}, {}
try:
actual_reg = reader.json(exp / 'protocol/preregistered.json', registered=False, expected=reg_sha)
require(_canonical(actual_reg) == _canonical(reg), 'Supplied registration differs from actual registered bytes')
for path, expected in sorted(reg['source_sha256'].items()):
reader.bind(path, registered=False, expected=expected)
original = Path(reg['selected_case']['original_dir'])
spec = reader.json(original / 'spec.json')
require(spec.get('kind') == 'fullgraph' and spec.get('steps') == 1000 and spec.get('dt_ms') == .1 and
spec.get('apl_delay_steps') == 18, 'Original fixed fullgraph D18 spec changed')
requests = reader.npz(original / 'requests.npz', {'tick', 'neuron_idx'})
graph_dir, bundle = Path(spec['graph_dir']), Path(spec['apl_bundle_dir'])
graph = {key: reader.npy(graph_dir / (key + '.npy')) for key in ('nodes', 'ptr', 'dst', 'edge_id', 'weights')}
require(graph['nodes'].dtype == np.int32 and np.array_equal(graph['nodes'], np.arange(N, dtype=np.int32)),
'Graph is not the complete canonical neuron set')
n, e = queue_audit._graph(graph['ptr'], graph['dst'], graph['edge_id'], graph['weights'])
require((n, e) == (N, E), 'Original graph dimensions changed')
apl = reader.npy(bundle / 'apl_globals.npy')
require(apl.dtype == np.int32 and apl.shape == (2,) and np.all((apl >= 0) & (apl < N)) and
apl[0] < apl[1], 'Invalid canonical APL source identity')
mask = np.zeros(E, np.bool_)
for source in apl:
mask[graph['ptr'][source]:graph['ptr'][source + 1]] = True
require(int(mask.sum()) == FAST_APL_EDGES, 'Frozen fast-APL source-edge mask changed')
graph['apl_out_mask'] = mask
storage = {}
for row in plan['rows']:
identity = row['row_id']; folder = exp / 'runs' / identity
require(folder.is_dir() and folder.resolve() == folder and not any(p.is_symlink() for p in folder.rglob('*')),
'Missing/linked runtime evidence folder')
records[identity] = reader.run(folder, legacy=False)
storage[identity] = sum(p.stat().st_size for p in folder.rglob('*') if p.is_file())
for stage in ('zero', 'static25'):
legacy[stage] = reader.run(reg['selected_case']['AM_outputs'][stage], legacy=True)
return dict(reader=reader, records=records, legacy=legacy, graph=graph, requests=requests,
storage=storage, input_scope='read-only authenticated selected original inputs and AM outputs')
except BaseException:
for record in [*records.values(), *legacy.values()]:
compare.close_run(record)
reader.close()
raise
def _pair(candidate, reference):
checks, observed = compare._pair_checks(candidate, reference)
return dict(numerical_exact=all(checks.values()), checks=checks,
failed_checks=[key for key, value in checks.items() if not value],
reference_comparisons_available=observed, learning_pass_claimed=False)
def _dispatch(descriptor, records, legacy, overlay):
candidates = [records[name] for name in descriptor['candidate_row_ids']]
reference = records[descriptor['reference_row_id']] if descriptor['reference_row_id'] is not None else None
kind = descriptor['check']
if kind == 'am_exact':
return compare.compare_fresh(candidates[0], legacy[overlay])
if kind == 'fresh_exact':
return compare.compare_fresh(candidates[0], reference)
if kind in ('prefix_of_reference', 'segment_of_reference'):
return scopes.compare_slice(candidates[0], reference)
if kind == 'segment_exact':
return _pair(candidates[0], reference)
if kind == 'checkpoint_exact':
return scopes.compare_checkpoint(candidates[0], reference)
if kind == 'parent_carry':
return scopes.compare_parent_carry(candidates[0], reference)
if kind == 'chain_exact':
return compare.compare_segments(reference, candidates)
raise ValueError('Unimplemented registered comparison: ' + str(kind))
def _number(value, name, integer=False):
if value is None:
return None
require((type(value) is int if integer else type(value) in (int, float)) and math.isfinite(value) and value >= 0,
'Malformed observed metric: ' + name)
return value
def _run_checks(row, record, expected_hash, reference):
compare.validate_run(record)
summary = record['summary']; identity = row['row_id']
require(len(record['state']['v']) == N and record['terminal']['apl_pending'].shape == (18, K), 'Incomplete full brain state: ' + identity)
checks = {key: summary.get(key) is True for key in ('complete', 'pass_all', 'all_finite', 'counts_match_sparse_events')}
fixed = dict(nodes=N, edges=E, steps=row['steps'], completed_ticks=row['steps'], apl_delay_steps=18,
requested=len(record['requests']), total_spikes=len(record['state']['step']),
accepted_requests=int(record['requests'][:, 3].sum(dtype=np.int64)),
rejected_requests=int(record['requests'][:, 4].sum(dtype=np.int64)),
apl_fast_edges_zeroed=FAST_APL_EDGES, kc_targets_present=K)
for key, value in fixed.items():
checks[key] = type(summary.get(key)) is int and summary[key] == value
checks['registered_clock'] = record['steps'] == row['steps'] and record['global_tick_offset'] == row['start']
checks['recording_mode'] = summary.get('diagnostic_mode') == row['mode']
checks['effective_weight_bytes'] = summary.get('effective_weights_raw_sha256') == expected_hash
checks['within_trial_weights_fixed'] = summary.get('within_trial_weight_updates') is False
checks['learning_not_claimed'] = summary.get('learning_pass_claimed') is False
expected_queue = reference['queues'][row['start']:row['stop']]
for field, queue_key in [('arriving_edges', 'count'), ('accepted_arriving_edges', 'accepted_count')]:
expected = sum(q[queue_key] for q in expected_queue)
checks[field] = type(summary.get(field)) is int and summary[field] == expected
return {name: bool(value) for name, value in checks.items()}
def analyze(reg, reg_sha, verified_records):
"""No writes/runs. The caller rechecks registration and receipts afterward."""
plan, exp = _registration(reg, reg_sha, verified_records)
context = _load_context(reg, reg_sha, plan, exp)
records, legacy = context['records'], context['legacy']
try:
require(set(records) == {r['row_id'] for r in plan['rows']} and set(legacy) == {'zero', 'static25'},
'Loaded evidence inventory is incomplete')
comparisons = []
for row in plan['rows']:
for number, descriptor in enumerate(row['expected_comparisons']):
try:
result = _dispatch(descriptor, records, legacy, row['overlay'])
require(type(result.get('numerical_exact')) is bool, 'Comparator lacks an explicit Boolean numerical gate')
except (ValueError, KeyError, TypeError, IndexError, OverflowError) as error:
result = dict(numerical_exact=False, error_type=type(error).__name__, error=str(error),
incomplete_observation_not_fabricated=True)
comparisons.append(dict(descriptor_id=row['row_id'] + '/' + str(number),
descriptor=descriptor, result=result))
expected_descriptors = sum(len(r['expected_comparisons']) for r in plan['rows'])
require(len(comparisons) == expected_descriptors, 'Not every registered descriptor was evaluated')
overlay_reports, metric_rows, all_row_checks = {}, [], {}
for stage in ('zero', 'static25'):
effective, installation = _materialize(context['reader'], reg, context['graph'], stage)
stage_rows = [r for r in plan['rows'] if r['overlay'] == stage]
ref_row = next(r for r in stage_rows if r['kind'] == 'fresh' and r['mode'] == 'reference')
reference = records[ref_row['row_id']]
reference_chain = [records[r['row_id']] for r in stage_rows if r['kind'] == 'chain' and r['mode'] == 'reference']
cuts = [r['start'] for r in stage_rows if r['kind'] == 'chain' and r['mode'] == 'reference' and r['start'] > 0]
independent = queue_audit.audit_queues(reference, reference_chain, ptr=context['graph']['ptr'], dst=context['graph']['dst'],
edge_id=context['graph']['edge_id'], effective_weights=effective, requests=context['requests'], cuts=cuts)
del effective
reference_result = next(item['result'] for item in comparisons if item['descriptor']['check'] == 'chain_exact' and
item['descriptor']['reference_row_id'] == ref_row['row_id'])
light_result = next(item['result'] for item in comparisons if item['descriptor']['check'] == 'chain_exact' and
item['descriptor']['reference_row_id'] == plan_module._fresh_id(stage, 'light'))
coverage = reference_result.get('coverage', {})
apl_values = coverage.get('carried_APL_nonzero_applied_values')
simultaneous_carry = any(type(row.get('pending_source_spike_events')) is int and row['pending_source_spike_events'] > 0 and
type(row.get('carried_APL_nonzero_pending_values')) is int and row['carried_APL_nonzero_pending_values'] > 0
for row in coverage.get('boundaries', []))
coverage_pass = (reference_result.get('nonvacuous_temporal_coverage') is True and
type(apl_values) is int and apl_values > 0 and simultaneous_carry and
independent.get('accepted_nonzero_crossing_witness_verified') is True)
legacy_checks = dict(full_dimensions=type(legacy[stage]['summary'].get('nodes')) is int and legacy[stage]['summary']['nodes'] == N and
type(legacy[stage]['summary'].get('edges')) is int and legacy[stage]['summary']['edges'] == E,
effective_weight_bytes=legacy[stage]['summary'].get('effective_weights_raw_sha256') == installation['effective_weights_raw_sha256'])
overlay_reports[stage] = dict(installation=installation, independent_queue_audit=independent,
reference_numerical_exact=reference_result.get('numerical_exact') is True,
light_numerical_exact=light_result.get('numerical_exact') is True,
live_coverage_pass=coverage_pass, legacy_checks=legacy_checks,
same_internal_boundary_has_spike_and_APL_carry=simultaneous_carry,
carried_APL_nonzero_applied_values=apl_values,
light_standalone_coverage=light_result.get('coverage'),
light_per_tick_float_state_directly_observed=False,
reference_coverage_does_not_fabricate_light_observations=True)
for row in stage_rows:
identity = row['row_id']; record = records[identity]; summary = record['summary']; receipt = verified_records[identity]
checks = _run_checks(row, record, installation['effective_weights_raw_sha256'], reference)
all_row_checks[identity] = checks
metric_rows.append(dict(row_id=identity, overlay=stage, mode=row['mode'], start=row['start'], stop=row['stop'],
biological_time_ms=row['steps'] * .1, runtime_and_immediate_seal_wall_seconds=_number(receipt.get('runtime_and_immediate_seal_wall_seconds'), 'runtime wall'),
runtime_reported_wall_seconds=_number(summary.get('wall_seconds'), 'runtime reported wall'),
diagnostic_simulation_seconds=_number(summary.get('diagnostic_simulation_seconds'), 'simulation'),
build_or_cache_seconds=_number(summary.get('build_or_cache_seconds'), 'build/cache'),
peak_allocated_vram_bytes=_number(summary.get('peak_allocated_vram_bytes'), 'allocated VRAM', True),
peak_reserved_vram_bytes=_number(summary.get('peak_reserved_vram_bytes'), 'reserved VRAM', True),
sealed_output_file_bytes=context['storage'][identity], compressed_storage=receipt.get('storage'),
gpu_utilization=None, cpu_utilization=None, unobserved_metrics=['gpu_utilization', 'cpu_utilization'],
checkpoint_manifest_sha256=receipt['checkpoint_manifest_sha256'],
verified_runtime_receipt_canonical_sha256=hashlib.sha256(_canonical(receipt).encode('utf-8')).hexdigest()))
numerical = (all(item['result']['numerical_exact'] for item in comparisons) and
all(all(checks.values()) for checks in all_row_checks.values()) and
all(all(r['legacy_checks'].values()) and r['independent_queue_audit']['queue_ref_request_exact'] for r in overlay_reports.values()))
covered = all(row['live_coverage_pass'] for row in overlay_reports.values())
read_manifest = context['reader'].finish()
report = dict(schema='phase4an-engineering-analysis-v1', registration_sha256=reg_sha,
status='FAIL-NUMERICAL' if not numerical else (SUCCESS if covered else 'INCOMPLETE-COVERAGE'),
pass_all=bool(numerical and covered), numerical_exact=bool(numerical), nonvacuous_temporal_coverage=bool(covered),
expected_calls=32, analyzed_calls=len(records), executed_ticks=plan['total_ticks'],
expected_comparisons=expected_descriptors, evaluated_comparisons=len(comparisons), comparisons=comparisons,
run_checks=all_row_checks, failed_run_checks={identity: [k for k, ok in checks.items() if not ok] for identity, checks in all_row_checks.items() if not all(checks.values())},
overlay_reports=overlay_reports, metrics=metric_rows, read_manifest=read_manifest,
source_sha256=reg['source_sha256'], process_receipts_and_checkpoint_chains_verified_by_caller=True,
external_AM_full_seals_reverified_here=False, no_new_brain_runs_during_analysis=True,
GPU_used=False, training=False, learning_pass_claimed=False, candidate_generated=False,
physiology_or_memory_success_claimed=False,
limitations=['Fixed zero/static25 engineering trajectories only; not conditioning or learned memory.',
'Light intermediate V/g/ref, APL histories and per-tick queue arrays were not directly recorded.',
'Independent queue/ref reconstruction is conditional on recorded spikes; V/g dynamics are not independently re-simulated.',
'Missing performance metrics are null, not zero. No universal performance claim is made.',
'Caller verifies all process receipts, parent chains, registration authority and full AM/frozen seals.'])
_canonical(report) # Fail rather than return non-JSON/nonfinite output.
return report
finally:
for record in [*records.values(), *legacy.values()]:
compare.close_run(record)
context['reader'].close()