Digital Drosophila Archive
153 lines
8.8 KiB
Python
153 lines
8.8 KiB
Python
"""Pure exact input derivations for a future preregistered engineering gate.
|
|
|
|
No I/O, random draws, scientific execution or registration. The coordinator
|
|
must bind the original case/overlay and every static dependency to the sealed
|
|
AM manifest before using these helpers. A self-consistent dictionary is not
|
|
independent input authority. Derivation outputs are copies; validate_requests
|
|
only checks and returns its original, unmodified input.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import re
|
|
from pathlib import PurePosixPath
|
|
|
|
import numpy as np
|
|
|
|
|
|
N = 166700
|
|
AM_HELPER_SHA = '7f891d0ba79b393c96e75fd2dee2eddc51e57d8213eaac2779703bc2c18712b4'
|
|
MASK_SHA = '8b0fb6eb2e6c31da4d17dc20d890aa3f367e514f05642de5bad6b42a1611c05b'
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def _positive_int(value, name):
|
|
require(type(value) is int and 0 < value <= np.iinfo(np.int32).max, 'Invalid ' + name)
|
|
|
|
|
|
def _sha(value):
|
|
return type(value) is str and re.fullmatch('[0-9a-f]{64}', value) is not None
|
|
|
|
|
|
def _absolute(value):
|
|
return (type(value) is str and value.startswith('/') and not value.startswith('//') and
|
|
PurePosixPath(value).is_absolute() and
|
|
'..' not in PurePosixPath(value).parts)
|
|
|
|
|
|
def validate_requests(requests, steps):
|
|
_positive_int(steps, 'steps')
|
|
require(type(requests) is dict and set(requests) == {'tick', 'neuron_idx'}, 'Exact sparse request schema required')
|
|
for key, value in requests.items():
|
|
require(isinstance(value, np.ndarray) and value.dtype == np.int32 and value.ndim == 1,
|
|
'Request requires int32 vector: ' + key)
|
|
tick, neuron = requests['tick'], requests['neuron_idx']
|
|
require(tick.shape == neuron.shape, 'Request shape mismatch')
|
|
require(np.all((tick >= 0) & (tick < steps)) and np.all((neuron >= 0) & (neuron < N)), 'Request outside fixed trial/canonical set')
|
|
key = tick.astype(np.int64) * N + neuron
|
|
require(len(key) < 2 or np.all(key[1:] > key[:-1]), 'Requests must be sorted and unique; no implicit deduplication')
|
|
return requests
|
|
|
|
|
|
def slice_requests(requests, steps, start, stop):
|
|
"""Half-open exact slice; translate ticks only, never redraw or reorder."""
|
|
validate_requests(requests, steps)
|
|
require(type(start) is int and type(stop) is int and 0 <= start < stop <= steps, 'Invalid half-open input interval')
|
|
lo, hi = np.searchsorted(requests['tick'], (start, stop))
|
|
result = {'tick': (requests['tick'][lo:hi] - np.int32(start)).copy(),
|
|
'neuron_idx': requests['neuron_idx'][lo:hi].copy()}
|
|
validate_requests(result, stop - start)
|
|
return result
|
|
|
|
|
|
def verify_partition(original, steps, intervals, parts):
|
|
"""Verify complete coverage plus exact request reconstruction, even blanks."""
|
|
validate_requests(original, steps)
|
|
require(type(intervals) in (list, tuple) and type(parts) in (list, tuple) and
|
|
len(intervals) == len(parts) > 0, 'Partition needs all intervals and parts')
|
|
end, reconstructed = 0, {'tick': [], 'neuron_idx': []}
|
|
for interval, part in zip(intervals, parts):
|
|
require(type(interval) in (list, tuple) and len(interval) == 2, 'Invalid interval schema')
|
|
start, stop = interval
|
|
require(type(start) is int and type(stop) is int and start == end and start < stop <= steps,
|
|
'Partition gap/overlap/nonpositive interval')
|
|
validate_requests(part, stop - start)
|
|
expected = slice_requests(original, steps, start, stop)
|
|
require(all(part[k].tobytes() == expected[k].tobytes() for k in part), 'Partition changes actual requests')
|
|
reconstructed['tick'].append(part['tick'].astype(np.int64) + start)
|
|
reconstructed['neuron_idx'].append(part['neuron_idx'])
|
|
end = stop
|
|
require(end == steps, 'Partition omits the terminal interval')
|
|
require(np.array_equal(np.concatenate(reconstructed['tick']), original['tick']) and
|
|
np.array_equal(np.concatenate(reconstructed['neuron_idx']), original['neuron_idx']), 'Reconstructed input mismatch')
|
|
return {'complete': True, 'intervals': [list(x) for x in intervals], 'steps': steps,
|
|
'requests': len(original['tick']), 'random_draws': 0, 'input_reselected': False}
|
|
|
|
|
|
def derive_spec(original, *, start, stop, budget_root, output_budget_bytes, parent_sha=None):
|
|
"""Explicit new case admission/budget labels; frozen model fields unchanged.
|
|
|
|
New budget_root must be outside the OLD budget tree. Full source/data/
|
|
ancestor containment checks remain the runtime/coordinator's responsibility.
|
|
A nonzero interval requires the actual parent manifest's external SHA;
|
|
future parent hashes must not be fabricated during preregistration.
|
|
"""
|
|
require(type(original) is dict and original.get('kind') == 'fullgraph', 'Original fullgraph spec required')
|
|
steps = original.get('steps')
|
|
_positive_int(steps, 'original steps')
|
|
require(original.get('dt_ms') == .1 and original.get('duration_ms') == steps * .1 and
|
|
type(original.get('apl_delay_steps')) is int and original['apl_delay_steps'] == 18 and
|
|
original.get('record_history') is False and original.get('training') is False and
|
|
original.get('within_trial_weight_updates', False) is False, 'Original fixed D18 trial contract required')
|
|
require(not any('checkpoint' in k.lower() or k in ('global_tick_offset', 'resume', 'continuation', 'initial_state')
|
|
for k in original), 'Original must be a fresh case')
|
|
require(type(start) is int and type(stop) is int and 0 <= start < stop <= steps, 'Invalid case interval')
|
|
require(_absolute(budget_root) and _absolute(original.get('budget_root')), 'Explicit absolute budget roots required')
|
|
new, old = PurePosixPath(budget_root), PurePosixPath(original['budget_root'])
|
|
require(not new.is_relative_to(old) and not old.is_relative_to(new), 'New output budget must not overlap old experiment tree')
|
|
require(type(output_budget_bytes) is int and output_budget_bytes > 0, 'Explicit positive byte budget required')
|
|
require((start == 0 and parent_sha is None) or (start > 0 and _sha(parent_sha)), 'Only a continued interval needs a real parent SHA')
|
|
result = copy.deepcopy(original)
|
|
result.update(steps=stop - start, duration_ms=(stop - start) * .1, max_steps=steps,
|
|
budget_root=budget_root, output_budget_bytes=output_budget_bytes,
|
|
within_trial_weight_updates=False,
|
|
outer_protocol='phase4an-engineering-validation-no-training')
|
|
if start:
|
|
result.update(global_tick_offset=start, parent_checkpoint_manifest_sha256=parent_sha)
|
|
changes = {key: {'present_before': key in original, 'before': copy.deepcopy(original.get(key)),
|
|
'after': copy.deepcopy(value)} for key, value in result.items()
|
|
if key not in original or value != original[key]}
|
|
allowed = {'steps', 'duration_ms', 'max_steps', 'budget_root', 'output_budget_bytes',
|
|
'within_trial_weight_updates', 'outer_protocol', 'global_tick_offset',
|
|
'parent_checkpoint_manifest_sha256'}
|
|
require(set(changes) <= allowed and all(result[k] == v for k, v in original.items() if k not in allowed),
|
|
'Unexpected scientific spec edit')
|
|
return result, {'input_interval': [start, stop], 'explicit_field_changes': changes,
|
|
'all_other_fields_unchanged': True, 'random_draws': 0, 'training': False}
|
|
|
|
|
|
def adapt_overlay_manifest(original, am_helper_path):
|
|
"""Translate observed AM manifest fields; preserve original overlay bytes.
|
|
|
|
This function validates declarations only. Callers independently rehash
|
|
every referenced file against external AM/source authority before use.
|
|
"""
|
|
require(type(original) is dict and original.get('schema') == 'phase4am-overlay-v1' and
|
|
original.get('overlay_name') in ('zero', 'static25') and
|
|
original.get('training') is False and original.get('fixed_for_whole_trial') is True,
|
|
'Only the fixed AM zero/static25 overlays are in this engineering gate')
|
|
require(original.get('mask_sha256') == MASK_SHA and _absolute(am_helper_path), 'Original mask/helper identity required')
|
|
names = ('overlay_name', 'overlay_sha256', 'mask_path', 'mask_sha256',
|
|
'base_candidate_path', 'base_candidate_sha256', 'aj_model_dir')
|
|
require(all(k in original for k in names), 'Missing observed AM manifest field')
|
|
require(all(_sha(original[k]) for k in ('overlay_sha256', 'mask_sha256', 'base_candidate_sha256')),
|
|
'Invalid external overlay/base SHA')
|
|
require(all(_absolute(original[k]) for k in ('mask_path', 'base_candidate_path', 'aj_model_dir')),
|
|
'Invalid external overlay/base path')
|
|
return dict(schema='phase4an-overlay-v1', **{k: copy.deepcopy(original[k]) for k in names},
|
|
am_overlay_path=am_helper_path, am_overlay_sha256=AM_HELPER_SHA)
|