Digital Drosophila Archive
223 lines
11 KiB
Python
223 lines
11 KiB
Python
"""Read-only sparse plastic-overlay validation and independent SI arithmetic.
|
|
|
|
This module does not calculate eligibility, generate learning updates, simulate
|
|
neurons, or alter a graph. File loading enforces the complete frozen 14,232-row
|
|
candidate identity; the array helpers also accept small synthetic identities.
|
|
The externally anchored manifest is a provenance input, not its own authority.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
|
|
import numpy as np
|
|
|
|
|
|
CANDIDATE_COUNT = 14232
|
|
EDGE_COUNT = 25582938
|
|
SCHEMA = 'phase4am-overlay-v1'
|
|
BRIDGE = 'baseSI64 + (delta32 widened to float64) * float64(0.001); nonzero rows only'
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def file_hash(path):
|
|
digest = hashlib.sha256()
|
|
with Path(path).open('rb') as source:
|
|
for block in iter(lambda: source.read(8 << 20), b''):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def raw_hash(value):
|
|
require(isinstance(value, np.ndarray), 'Raw hash requires an ndarray')
|
|
dtype = value.dtype.newbyteorder('<')
|
|
return hashlib.sha256(np.ascontiguousarray(value, dtype=dtype).tobytes()).hexdigest()
|
|
|
|
|
|
def bit_equal(first, second):
|
|
return (isinstance(first, np.ndarray) and isinstance(second, np.ndarray) and
|
|
first.dtype == second.dtype and first.shape == second.shape and
|
|
first.tobytes(order='C') == second.tobytes(order='C'))
|
|
|
|
|
|
def _vector(value, dtype, name, size=None):
|
|
require(isinstance(value, np.ndarray) and value.dtype == np.dtype(dtype) and value.ndim == 1,
|
|
name + ' requires exact ' + dtype + ' vector; no implicit conversion')
|
|
require(size is None or value.shape == (size,), name + ' length differs')
|
|
return value
|
|
|
|
|
|
def _ids(value, name):
|
|
_vector(value, 'int64', name)
|
|
require(np.all((value >= 0) & (value < EDGE_COUNT)) and np.all(value[1:] > value[:-1]),
|
|
name + ' must be sorted unique frozen-range global edge IDs')
|
|
return value
|
|
|
|
|
|
def _owned(value):
|
|
result = np.array(value, copy=True, order='C')
|
|
result.flags.writeable = False
|
|
return result
|
|
|
|
|
|
def validate_arrays(edge_id, plastic_delta_mv, mask_edge_id, base_candidate_mv=None):
|
|
"""Validate/copy arrays; small masks are permitted only for pure unit helpers.
|
|
|
|
The production file loader separately insists on CANDIDATE_COUNT. Every
|
|
nonzero delta needs the original, positive float32 base in legacy mV.
|
|
Its bound is the frozen float32 expression -0.25 * base_candidate, NOT
|
|
a fraction recomputed from the author's float64 SI runtime weights.
|
|
"""
|
|
ids, mask = _ids(edge_id, 'edge_id'), _ids(mask_edge_id, 'mask_edge_id')
|
|
delta = _vector(plastic_delta_mv, 'float32', 'plastic_delta_mv', len(ids))
|
|
require(len(ids) > 0 and bit_equal(ids, mask), 'Candidate IDs must equal the complete anchored mask in order')
|
|
require(np.isfinite(delta).all() and np.all(delta <= np.float32(0)), 'Delta must be finite, nonpositive float32 mV')
|
|
base = None
|
|
if base_candidate_mv is not None:
|
|
base = _vector(base_candidate_mv, 'float32', 'base_candidate_mv', len(ids))
|
|
require(np.isfinite(base).all() and np.all(base > np.float32(0)), 'Original candidate base must be finite, positive float32 mV')
|
|
lower = np.multiply(np.float32(-.25), base)
|
|
require(np.all(delta >= lower), 'Delta exceeds the frozen original-float32 -25% bound')
|
|
else:
|
|
require(not np.any(delta != np.float32(0)), 'Nonzero delta requires the authoritative original float32 base')
|
|
return {'edge_id': _owned(ids), 'plastic_delta_mv': _owned(delta),
|
|
'base_candidate_mv': None if base is None else _owned(base)}
|
|
|
|
|
|
def _checked_file(manifest, path_key, hash_key):
|
|
path = Path(manifest[path_key])
|
|
expected = manifest[hash_key]
|
|
require(path.is_absolute() and path.is_file(), 'Missing absolute input: ' + path_key)
|
|
require(isinstance(expected, str) and re.fullmatch('[0-9a-f]{64}', expected) is not None,
|
|
'Malformed SHA256: ' + hash_key)
|
|
require(file_hash(path) == expected, 'Input SHA256 mismatch: ' + path_key)
|
|
return path
|
|
|
|
|
|
def load_overlay(overlay_file, overlay_manifest):
|
|
"""Load immutable, hash-bound production inputs without importing Torch.
|
|
|
|
Manifest keys: schema, overlay_name (zero/static25), overlay_sha256,
|
|
mask_path/mask_sha256, base_candidate_path/base_candidate_sha256 (both
|
|
null only for zero), aj_model_dir. Extra provenance fields are retained
|
|
in the copied original manifest, not treated as executable instructions.
|
|
NPZ keys: edge_id, plastic_delta_mv; optional base_candidate_mv must be
|
|
bit-identical to the authoritative external .npy, never a replacement.
|
|
"""
|
|
overlay_file, manifest_file = Path(overlay_file).absolute(), Path(overlay_manifest).absolute()
|
|
require(overlay_file.is_file() and manifest_file.is_file(), 'Missing overlay or manifest')
|
|
manifest_bytes = manifest_file.read_bytes()
|
|
manifest_sha = hashlib.sha256(manifest_bytes).hexdigest()
|
|
metadata = json.loads(manifest_bytes.decode('utf-8'))
|
|
required = {'schema', 'overlay_name', 'overlay_sha256', 'mask_path', 'mask_sha256',
|
|
'base_candidate_path', 'base_candidate_sha256', 'aj_model_dir'}
|
|
require(isinstance(metadata, dict) and required <= metadata.keys(), 'Incomplete overlay manifest')
|
|
require(metadata['schema'] == SCHEMA and metadata['overlay_name'] in ('zero', 'static25'), 'Unregistered interface overlay kind')
|
|
require(file_hash(overlay_file) == metadata['overlay_sha256'], 'Overlay file SHA256 mismatch')
|
|
aj_dir = Path(metadata['aj_model_dir'])
|
|
require(aj_dir.is_absolute() and aj_dir.is_dir(), 'aj_model_dir must name the real sealed AJ directory')
|
|
mask_path = _checked_file(metadata, 'mask_path', 'mask_sha256')
|
|
mask = np.load(mask_path, allow_pickle=False)
|
|
_ids(mask, 'authoritative mask')
|
|
require(mask.shape == (CANDIDATE_COUNT,), 'Frozen mask must contain exactly 14,232 candidates')
|
|
base_path, base = None, None
|
|
if metadata['base_candidate_path'] is None:
|
|
require(metadata['base_candidate_sha256'] is None, 'Absent base needs a null base hash')
|
|
else:
|
|
base_path = _checked_file(metadata, 'base_candidate_path', 'base_candidate_sha256')
|
|
base = np.load(base_path, allow_pickle=False)
|
|
with np.load(overlay_file, allow_pickle=False) as archive:
|
|
require({'edge_id', 'plastic_delta_mv'} <= set(archive.files) <=
|
|
{'edge_id', 'plastic_delta_mv', 'base_candidate_mv'}, 'Unexpected overlay NPZ schema')
|
|
values = validate_arrays(archive['edge_id'], archive['plastic_delta_mv'], mask, base)
|
|
if 'base_candidate_mv' in archive.files:
|
|
require(base is not None and bit_equal(archive['base_candidate_mv'], base),
|
|
'Embedded base is not the authoritative external original float32 base')
|
|
delta = values['plastic_delta_mv']
|
|
if metadata['overlay_name'] == 'zero':
|
|
require(not np.any(delta != 0), 'Named zero overlay is not zero')
|
|
else:
|
|
require(base is not None and bit_equal(delta, np.multiply(np.float32(-.25), base)),
|
|
'static25 must exactly equal the original float32 floor, not 25% of float64 runtime base')
|
|
values.update(metadata=metadata, input={
|
|
'overlay_file': str(overlay_file), 'overlay_sha256': metadata['overlay_sha256'],
|
|
'manifest_file': str(manifest_file), 'manifest_sha256': manifest_sha,
|
|
'mask_file': str(mask_path), 'mask_sha256': metadata['mask_sha256'],
|
|
'base_candidate_file': None if base_path is None else str(base_path),
|
|
'base_candidate_sha256': metadata['base_candidate_sha256']})
|
|
verify_inputs(values)
|
|
return values
|
|
|
|
|
|
def verify_inputs(plan):
|
|
for name in ('overlay', 'manifest', 'mask', 'base_candidate'):
|
|
path, expected = plan['input'][name + '_file'], plan['input'][name + '_sha256']
|
|
require((path is None and expected is None) or (path is not None and file_hash(path) == expected),
|
|
'Overlay input changed: ' + name)
|
|
|
|
|
|
def map_installation(base_si, graph_edge_id, apl_out_mask, plan):
|
|
"""Return sparse nonzero installation rows and independently computed values.
|
|
|
|
Missing zero-delta mask members are permitted in a small fixture. Missing
|
|
nonzero members fail closed. APL/candidate overlap is always rejected.
|
|
Neither base nor any caller array is changed. Zero delta has no write row.
|
|
"""
|
|
ids = _ids(graph_edge_id, 'graph_edge_id')
|
|
base = _vector(base_si, 'float64', 'base_si', len(ids))
|
|
apl = _vector(apl_out_mask, 'bool', 'apl_out_mask', len(ids))
|
|
require(np.isfinite(base).all(), 'Nonfinite graph SI base')
|
|
candidates = plan['edge_id']
|
|
delta = plan['plastic_delta_mv']
|
|
rows = np.searchsorted(ids, candidates)
|
|
present = rows < len(ids)
|
|
present[present] &= ids[rows[present]] == candidates[present]
|
|
nonzero = delta != np.float32(0)
|
|
require(not np.any(nonzero & ~present), 'A nonzero plastic candidate is absent from this graph')
|
|
mapped = rows[present]
|
|
require(not np.any(apl[mapped]), 'Plastic candidate overlaps the frozen fast-APL output mask')
|
|
require(np.all(base[mapped] > np.float64(0)), 'Candidate graph SI base must remain positive')
|
|
selected = rows[nonzero].astype(np.int64)
|
|
delta_si = np.multiply(delta[nonzero].astype(np.float64), np.float64(.001))
|
|
expected = np.add(base[selected], delta_si)
|
|
require(np.isfinite(expected).all() and np.all(expected > 0), 'Overlay would change a candidate sign or create nonfinite weights')
|
|
return {'rows': _owned(selected), 'delta_si': _owned(delta_si), 'expected_si': _owned(expected),
|
|
'mapped_candidate_count': int(present.sum()), 'missing_candidate_count': int((~present).sum()),
|
|
'installed_nonzero_count': int(nonzero.sum()), 'apl_overlap_count': 0}
|
|
|
|
|
|
def materialize_numpy(base_si, graph_edge_id, apl_out_mask, plan):
|
|
"""Independent pure CPU reference; used by unit tests, not a brain run."""
|
|
mapping = map_installation(base_si, graph_edge_id, apl_out_mask, plan)
|
|
result = np.array(base_si, copy=True, order='C')
|
|
result[apl_out_mask] = np.float64(0)
|
|
result[mapping['rows']] = mapping['expected_si']
|
|
return result, mapping
|
|
|
|
|
|
def expected_hashes(base_si, apl_out_mask, mapping, chunk_elements=1 << 20):
|
|
"""Bounded-memory base/effective raw hashes, preserving untouched signed bits."""
|
|
require(type(chunk_elements) is int and chunk_elements > 0, 'Invalid hash chunk size')
|
|
base_hash, effective_hash = hashlib.sha256(), hashlib.sha256()
|
|
rows = mapping['rows']
|
|
for start in range(0, len(base_si), chunk_elements):
|
|
stop = min(start + chunk_elements, len(base_si))
|
|
original = np.ascontiguousarray(base_si[start:stop], dtype='<f8')
|
|
base_hash.update(original.tobytes())
|
|
effective = original.copy()
|
|
effective[apl_out_mask[start:stop]] = np.float64(0)
|
|
lo, hi = np.searchsorted(rows, (start, stop))
|
|
effective[rows[lo:hi] - start] = mapping['expected_si'][lo:hi]
|
|
effective_hash.update(effective.tobytes())
|
|
return base_hash.hexdigest(), effective_hash.hexdigest()
|
|
|
|
|
|
def array_metadata(value):
|
|
return {'dtype': value.dtype.name, 'shape': list(value.shape), 'raw_sha256': raw_hash(value)}
|