Digital Drosophila Archive
220 lines
15 KiB
Python
220 lines
15 KiB
Python
"""Diagnostic live-boundary oracle, NOT a production/learned brain backend.
|
|
|
|
Re-execute a fresh parent's exact inputs/old overlay, without calling restore.
|
|
At the declared epoch boundary compare live state to the sealed parent, replace
|
|
only existing candidate weights with base+delta-after, and continue the original
|
|
reference loop. One whitelisted hook is independently removable from its AST.
|
|
The diagnostic has an explicit within-run boundary switch; old C must NOT load
|
|
its output as an unchanged-weight checkpoint.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import numpy as np
|
|
|
|
LAB=Path(__file__).resolve().parent.parent/'olfactory-conditioning-lab-v1'
|
|
C_SHA='0b8e8e0a6a23a079dc62a88355ce355d53b3ae22f6665519a8f6aef2dcfc5aba'
|
|
HOOK="if tick == _oracle.boundary_tick:\n _oracle.switch(tick, weights, v, g, ref, counts, ring, apl_ring, apl, out_dir)"
|
|
|
|
|
|
def require(ok,message):
|
|
if not ok:raise ValueError(message)
|
|
|
|
|
|
def digest(path):
|
|
h=hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for b in iter(lambda:stream.read(8<<20),b''):h.update(b)
|
|
return h.hexdigest()
|
|
|
|
|
|
def load(name,path,sha):
|
|
raw=Path(path).read_bytes()
|
|
require(hashlib.sha256(raw).hexdigest()==sha and name not in sys.modules,'Unverified oracle source/module binding')
|
|
spec=importlib.util.spec_from_file_location(name,path);module=importlib.util.module_from_spec(spec)
|
|
sys.modules[name]=module;exec(compile(raw,str(path),'exec'),module.__dict__)
|
|
return module
|
|
|
|
|
|
def frozen():
|
|
return load('_conditioning_live_oracle_C',LAB/'runtime_continuation.py',C_SHA)
|
|
|
|
|
|
def derive_oracle(c,source_bytes):
|
|
tree,inherited=c.R.derive_ast(source_bytes,'reference');before=copy.deepcopy(tree)
|
|
loops=[n for n in ast.walk(tree) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='tick']
|
|
require(len(loops)==1,'Unique frozen timestep loop required')
|
|
loops[0].body.insert(0,ast.parse(HOOK).body[0]);tree=ast.fix_missing_locations(tree)
|
|
rollback=copy.deepcopy(tree)
|
|
matches=[n for n in ast.walk(rollback) if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='tick']
|
|
require(ast.unparse(matches[0].body[0])==HOOK,'Oracle hook changed')
|
|
del matches[0].body[0]
|
|
require(c.R._dump(rollback)==c.R._dump(before),'Unlisted oracle mathematical AST edit')
|
|
return tree,dict(base=inherited,explicit_boundary_hook=HOOK,full_AST_rollback_identical=True,
|
|
frozen_within_epoch_math_unchanged=True,restore_called=False,
|
|
diagnostic_live_weight_switch=True,production_backend=False,
|
|
ast_sha256=hashlib.sha256(c.R._dump(tree).encode()).hexdigest())
|
|
|
|
|
|
def expected_candidate_values(base_si,edge_ids,after_delta):
|
|
"""Base+f32-mV delta with the frozen separate f64 multiply/add contract."""
|
|
require(base_si.dtype==np.float64 and base_si.ndim==1 and edge_ids.dtype==np.int64 and edge_ids.ndim==1 and
|
|
after_delta.dtype==np.float32 and after_delta.shape==edge_ids.shape,'Wrong candidate bridge dtype/shape')
|
|
require(np.all((edge_ids>=0)&(edge_ids<len(base_si))) and np.all(edge_ids[1:]>edge_ids[:-1]),'Invalid existing edge IDs')
|
|
require(np.isfinite(after_delta).all() and np.all(after_delta<=0),'Invalid depression delta')
|
|
result=base_si[edge_ids].copy();nonzero=after_delta!=np.float32(0)
|
|
result[nonzero]=np.add(result[nonzero],np.multiply(after_delta[nonzero].astype(np.float64),np.float64(.001)))
|
|
require(np.isfinite(result).all() and np.all(result>0),'Candidate sign changed')
|
|
return result
|
|
|
|
|
|
def boundary_arrays(parent,tick,v,g,ref,counts,ring,apl_ring,apl_state):
|
|
require(type(tick)is int and tick==parent['steps']==parent['global_next_tick'] and parent['fresh'] is True,
|
|
'Oracle needs the exact fresh-parent boundary')
|
|
arrays=dict(v=v,g=g,ref=ref,counts=counts,spike_ring=ring,apl_state=apl_state,
|
|
apl_pending=np.concatenate((apl_ring[tick%18:],apl_ring[:tick%18]),axis=0))
|
|
expected={**{k:parent['state'][k] for k in ('v','g','ref','counts')},
|
|
**{k:parent['terminal'][k] for k in ('spike_ring','apl_state','apl_pending')}}
|
|
checks={k:(a.dtype==expected[k].dtype and a.shape==expected[k].shape and a.tobytes()==expected[k].tobytes())
|
|
for k,a in arrays.items()}
|
|
require(all(checks.values()),'Live prefix differs from sealed parent: '+str([k for k,v in checks.items() if not v]))
|
|
return arrays,checks
|
|
|
|
|
|
def runtime_class(c):
|
|
r=c.R
|
|
class Oracle(r._Runtime):
|
|
def __init__(self,*args,boundary,**kwargs):
|
|
super().__init__(*args,**kwargs);self.boundary=boundary;self.parent=boundary['parent']
|
|
self.boundary_tick=self.parent['steps'];self.switches=0
|
|
|
|
def tensor_sha(self,value):
|
|
h=hashlib.sha256()
|
|
for start in range(0,value.numel(),1<<20):h.update(value[start:start+(1<<20)].cpu().numpy().astype('<f8',copy=False).tobytes())
|
|
return h.hexdigest()
|
|
|
|
def install(self,weights,graph,apl_mask,upload,out_dir,inherited,device):
|
|
require(self.installations==0 and len(graph['nodes'])==166700 and len(graph['edge_id'])==25582938 and
|
|
int(apl_mask.sum())==5845,'Only one intact frozen fullgraph installation')
|
|
prior=self.helper.map_installation(graph['weights'],graph['edge_id'],apl_mask,self.plan)
|
|
following=self.helper.map_installation(graph['weights'],graph['edge_id'],apl_mask,self.boundary['after_plan'])
|
|
require(prior['missing_candidate_count']==following['missing_candidate_count']==0,'Incomplete candidate mapping')
|
|
if len(prior['rows']):
|
|
idx=upload(prior['rows'],self.torch.int64)
|
|
change=upload(self.plan['plastic_delta_mv'][self.plan['plastic_delta_mv']!=np.float32(0)],self.torch.float64)*.001
|
|
weights[idx]=weights[idx]+change
|
|
require(self.helper.bit_equal(weights[idx].cpu().numpy(),prior['expected_si']),'Initial SI bridge mismatch')
|
|
base_sha,before_sha=self.helper.expected_hashes(graph['weights'],apl_mask,prior)
|
|
_,after_sha=self.helper.expected_hashes(graph['weights'],apl_mask,following)
|
|
require(before_sha==self.parent['summary']['effective_weights_raw_sha256'] and self.tensor_sha(weights)==before_sha,
|
|
'Oracle before weights differ from parent')
|
|
auth=self.boundary['value']
|
|
require(before_sha==auth['expected_effective_weight_before_sha256'] and after_sha==auth['expected_effective_weight_after_sha256'],
|
|
'External expected weights differ')
|
|
mask=self.plan['edge_id'];require(np.array_equal(graph['edge_id'][mask],mask),'Noncanonical candidate positions')
|
|
expected=expected_candidate_values(graph['weights'],mask,self.boundary['after_plan']['plastic_delta_mv'])
|
|
nz=self.boundary['after_plan']['plastic_delta_mv']!=np.float32(0)
|
|
require(self.helper.bit_equal(expected[nz],following['expected_si']),'Independent after bridge differs')
|
|
self.rows_gpu=upload(mask,self.torch.int64);self.after_gpu=upload(expected,self.torch.float64)
|
|
self.before_sha,self.after_sha=before_sha,after_sha;self.effective_sha=before_sha
|
|
self.observer=r.FiniteObserver(self.torch,device);self.installations=1
|
|
for role,target in (('before_overlay','overlay-before-input.npz'),('before_manifest','overlay-before-manifest.json'),
|
|
('after_overlay','overlay-after-input.npz'),('after_manifest','overlay-after-manifest.json')):
|
|
source=self.boundary['paths'][role];r._copy_input(source,out_dir/target,r.digest(source))
|
|
r._save(out_dir/'overlay-audit.json',dict(schema='conditioning-live-oracle-overlay-v1',complete=True,
|
|
base_weights_raw_sha256=base_sha,initial_effective_weights_raw_sha256=before_sha,
|
|
planned_final_effective_weights_raw_sha256=after_sha,weights_updated_during_run=True,
|
|
one_explicit_epoch_boundary=True,existing_candidates_only=True,production_checkpoint=False))
|
|
|
|
def switch(self,tick,weights,v,g,ref,counts,ring,apl_ring,apl,out_dir):
|
|
require(self.installations==1 and self.switches==0 and tick==self.boundary_tick,'Wrong/repeated boundary switch')
|
|
arrays,checks=boundary_arrays(self.parent,tick,*[a.cpu().numpy().copy() for a in
|
|
(v,g,ref,counts,ring,apl_ring,apl.state)])
|
|
require(self.tensor_sha(weights)==self.before_sha,'Weights changed before authorized boundary')
|
|
# Replacement from ORIGINAL base; never increment by delta-after again.
|
|
weights[self.rows_gpu]=self.after_gpu
|
|
require(self.tensor_sha(weights)==self.after_sha,'Live boundary after-weight mismatch')
|
|
self.effective_sha=self.after_sha;self.switches=1
|
|
with (out_dir/'boundary-snapshot.npz').open('xb') as f:np.savez_compressed(f,**arrays)
|
|
r._save(out_dir/'boundary-comparison.json',dict(complete=True,boundary_tick=tick,checks=checks,
|
|
parent_checkpoint_manifest_sha256=self.parent['manifest_sha256'],restore_called=False,
|
|
initial_state_replaced=False,queues_replaced=False,existing_weights_replaced_once=True,
|
|
before_effective_sha256=self.before_sha,after_effective_sha256=self.after_sha))
|
|
|
|
def finish(self,*args,**kwargs):
|
|
require(self.switches==1,'Expected diagnostic boundary did not execute')
|
|
return super().finish(*args,**kwargs)
|
|
|
|
def summary(self,value):
|
|
result=super().summary(value)
|
|
result.update(within_trial_weight_updates=True,diagnostic_live_boundary=True,
|
|
boundary_tick=self.boundary_tick,explicit_weight_switches=self.switches,
|
|
effective_weights_raw_sha256=self.after_sha,initial_effective_weights_raw_sha256=self.before_sha,
|
|
restored_from_checkpoint=False,production_backend=False,
|
|
pass_all_scope='One diagnostic live boundary and finite integrity; no learning/equivalence claim')
|
|
return result
|
|
return Oracle
|
|
|
|
|
|
def run(authority_file,authority_sha,updated_source_sha,oracle_case,spec_sha,requests_sha,out_dir,build_root):
|
|
require(sys.dont_write_bytecode,'Use Python -B')
|
|
d=load('_conditioning_live_oracle_updated',Path(__file__).with_name('updated_continuation.py'),updated_source_sha)
|
|
boundary=d.validate_authority(authority_file,authority_sha)
|
|
c=d.C;r=c.R;parent=boundary['parent'];paths=boundary['paths'];value=boundary['value']
|
|
require(value['transition']=='epoch_update' and parent['fresh'] is True,'Live oracle supports one verified fresh epoch update only')
|
|
case=Path(oracle_case).resolve(strict=True)
|
|
require(digest(case/'spec.json')==spec_sha and digest(case/'requests.npz')==requests_sha,'Combined oracle input differs from external pins')
|
|
spec=r._read_json(case/'spec.json');previous=c._parent_spec(parent);suffix=boundary['spec']
|
|
require(spec.get('within_trial_weight_updates') is True and spec.get('diagnostic_boundary_tick')==parent['steps'],
|
|
'Oracle must explicitly declare its one within-run boundary')
|
|
r.validate_spec({**spec,'within_trial_weight_updates':False},case)
|
|
require(spec['steps']==parent['steps']+suffix['steps'],'Combined oracle clock differs')
|
|
for key in ('dt_ms','model','apl','apl_delay_steps','graph_dir','apl_bundle_dir','coefficients_path'):
|
|
require(spec[key]==previous[key]==suffix[key],'Oracle changes frozen input/dynamics: '+key)
|
|
def requests(path):
|
|
with np.load(path,allow_pickle=False) as f:
|
|
require(set(f.files)=={'tick','neuron_idx'},'Invalid request archive')
|
|
a={k:f[k].copy() for k in f.files}
|
|
require(all(v.dtype==np.int32 and v.ndim==1 for v in a.values()),'Invalid request dtype')
|
|
return a
|
|
epoch=r._read_json(paths['epoch_authority'])
|
|
old=requests(epoch['files']['requests']['path']);new=requests(paths['requests']);combined=requests(case/'requests.npz')
|
|
expected={'tick':np.concatenate((old['tick'],new['tick']+parent['steps'])),
|
|
'neuron_idx':np.concatenate((old['neuron_idx'],new['neuron_idx']))}
|
|
require(all(np.array_equal(combined[k],expected[k]) for k in expected),'Oracle prefix/suffix requests are not exact')
|
|
protected={*parent['ancestry_directories'],*[Path(p).parent for p in boundary['pins']]}
|
|
case,out,build=r.execution_paths(case,out_dir,build_root,protected)
|
|
inputs={**boundary['pins'],str(case/'spec.json'):spec_sha,str(case/'requests.npz'):requests_sha,str(Path(__file__).resolve()):digest(__file__)}
|
|
aj_dir=Path(boundary['before_meta']['aj_model_dir']).resolve()
|
|
r._load('timing_queue',aj_dir/'timing_queue.py',r.PINS['apl-timing-lab-v1/timing_queue.py'])
|
|
aj=r._load('conditioning_live_oracle_AJ',aj_dir/'gpu_timing.py',r.AJ_SHA)
|
|
runtime=runtime_class(c)('reference',spec,boundary['helper'],boundary['before_plan'],boundary['before_meta'],
|
|
inputs,aj.torch,paths['before_overlay'],paths['before_manifest'],boundary=boundary)
|
|
tree,audit=derive_oracle(c,Path(aj.__file__).read_bytes())
|
|
namespace=dict(aj.__dict__);namespace.update(__name__=__name__+'.derived',__file__=str(Path(__file__).resolve()),
|
|
_runtime=runtime,_oracle=runtime,write_json=runtime.write_json)
|
|
exec(compile(tree,str(Path(__file__).resolve())+'::live-boundary-AST','exec'),namespace)
|
|
inherited=namespace['run_gpu'](case,out,build)
|
|
require(runtime.finished and runtime.switches==1 and inherited['complete'],'Incomplete live-boundary oracle')
|
|
with np.load(out/'state.npz',allow_pickle=False) as state:
|
|
prefix=state['step']<parent['steps']
|
|
require(np.array_equal(state['step'][prefix],parent['state']['step']) and
|
|
np.array_equal(state['neuron_idx'][prefix],parent['state']['neuron_idx']),'Oracle prefix events differ from actual parent')
|
|
require(all(digest(p)==sha for p,sha in inputs.items()),'Oracle source/input drift')
|
|
c.load_checkpoint(parent['directory'],parent['manifest_sha256'])
|
|
summary=r._read_json(out/'summary.json');require(summary==runtime.summary(inherited),'Oracle summary mismatch')
|
|
r._save(out/'oracle-provenance.json',dict(schema='conditioning-live-boundary-oracle-v1',complete=True,
|
|
source_sha256=digest(__file__),updated_source_sha256=updated_source_sha,authority_sha256=authority_sha,
|
|
input_sha256=inputs,ast=audit,restore_called=False,old_globals_patched=False,
|
|
prefix_sparse_events_exact=True,production_checkpoint=False,learning_pass_claimed=False))
|
|
files={p.relative_to(out).as_posix():digest(p) for p in out.rglob('*') if p.is_file()}
|
|
r._save(out/'oracle-manifest.json',dict(schema='conditioning-live-oracle-files-v1',complete=True,files_sha256=files))
|
|
return summary
|