Digital Drosophila Archive
248 lines
17 KiB
Python
248 lines
17 KiB
Python
"""Pure CPU comparisons for a separately authorized, finite input/update test.
|
|
|
|
No files, checkpoint seals, source pins, parent trees or GPU are read here.
|
|
The coordinator must first verify subprocess receipts, externally pinned
|
|
checkpoint manifests, registered source/input files and learned-parent chains.
|
|
It must independently recompute update receipts with conditioning_update's
|
|
verify_receipt. These functions bind supplied arrays and declared metadata;
|
|
they cannot replace those external checks or prove functional learning.
|
|
|
|
The frozen engineering comparator is reused unchanged. Reference diagnostics
|
|
must be complete. Light has events, final state and queues, not per-tick V/g
|
|
or APL histories; missing light observations are never fabricated as exact.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
|
|
import numpy as np
|
|
|
|
import engineering_compare as C
|
|
|
|
|
|
N, E, N_KC, CANDIDATE_COUNT = 166700, 25582938, 4064, 14232
|
|
COMPARATOR_SHA256 = "5a2403f11bc08d65a9d242ac2cf9686b850e0c3680d65659f785e9b6b50c0238"
|
|
PHASE4J_SHA256 = "abb4574da85f00bd9ef61024c65993ef81d42651ff8d8d63c15dd69c6222471e"
|
|
ELIGIBILITY_BRIDGE_SHA256 = "174fb8d14619189e74b095e77a6db18b14c93046dc431c0b55710a5aa2dd2e61"
|
|
MASK_SHA256 = "8b0fb6eb2e6c31da4d17dc20d890aa3f367e514f05642de5bad6b42a1611c05b"
|
|
AM_OVERLAY_SHA256 = "7f891d0ba79b393c96e75fd2dee2eddc51e57d8213eaac2779703bc2c18712b4"
|
|
MANIFEST_FIELDS = frozenset(("schema", "overlay_name", "overlay_sha256", "mask_path", "mask_sha256",
|
|
"base_candidate_path", "base_candidate_sha256", "aj_model_dir", "am_overlay_path", "am_overlay_sha256",
|
|
"parent_snapshot_sha256", "eligibility_v2_source_sha256"))
|
|
|
|
|
|
def _require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def _canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False)
|
|
|
|
|
|
def _sha(value):
|
|
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _valid_sha(value):
|
|
return type(value) is str and re.fullmatch("[0-9a-f]{64}", value) is not None
|
|
|
|
|
|
def _evidence(value):
|
|
return dict(dtype=value.dtype.str, shape=list(value.shape), nbytes=int(value.nbytes),
|
|
raw_c_order_sha256=hashlib.sha256(value.tobytes(order="C")).hexdigest())
|
|
|
|
|
|
def _integer(mapping, key, expected):
|
|
return type(mapping.get(key)) is int and mapping[key] == expected
|
|
|
|
|
|
def _request_input(value, steps):
|
|
_require(isinstance(value, np.ndarray) and value.dtype == np.int32 and value.ndim == 2 and value.shape[1] == 2,
|
|
"Expected input requests must be exact int32 [M,2] (local tick, global neuron)")
|
|
_require(np.all((value[:, 0] >= 0) & (value[:, 0] < steps)) and
|
|
np.all((value[:, 1] >= 0) & (value[:, 1] < N)), "Expected input request is out of range")
|
|
keys = value[:, 0].astype(np.int64) * N + value[:, 1]
|
|
_require(len(keys) < 2 or np.all(keys[1:] > keys[:-1]), "Expected requests must already be sorted unique")
|
|
|
|
|
|
def compare_epoch_pair(reference_record, light_record, *, expected_requests, expected_steps,
|
|
expected_effective_weight_sha256):
|
|
"""Compare fresh reference/light records, not paths or self-asserted seals.
|
|
|
|
Inputs are dictionaries returned by frozen engineering_compare.load_run.
|
|
The coordinator supplies the original registered request array and the
|
|
independently expected effective-weight raw SHA, not a value copied from
|
|
the summary being checked. Corrupt array schemas raise ValueError. Valid
|
|
but different records return numerical_exact/pass_all=False. pass_all is
|
|
ONLY the supplied-record identity check, not a learning or activity gate.
|
|
"""
|
|
_require(type(expected_steps) is int and 0 < expected_steps <= np.iinfo(np.int32).max,
|
|
"Explicit expected epoch steps required")
|
|
_require(_valid_sha(expected_effective_weight_sha256), "Expected effective-weight raw SHA is required")
|
|
_request_input(expected_requests, expected_steps)
|
|
_require(type(reference_record) is dict and type(light_record) is dict,
|
|
"Supply already authenticated loaded records, not paths")
|
|
C.validate_run(reference_record)
|
|
C.validate_run(light_record)
|
|
inherited = C.compare_fresh(light_record, reference_record)
|
|
checks = dict(inherited["checks"])
|
|
for label, record, mode in (("reference", reference_record, "reference"), ("light", light_record, "light")):
|
|
summary = record.get("summary")
|
|
_require(type(summary) is dict, "Missing runtime summary")
|
|
checks[label + ".full_population"] = len(record["state"]["v"]) == N and _integer(summary, "nodes", N) and _integer(summary, "edges", E)
|
|
checks[label + ".expected_clock"] = (record["steps"] == expected_steps and record["global_tick_offset"] == 0 and
|
|
_integer(summary, "steps", expected_steps) and _integer(summary, "completed_ticks", expected_steps) and
|
|
_integer(summary, "explicit_steps_executed", expected_steps))
|
|
checks[label + ".fresh_fixed_weight_scope"] = (summary.get("checkpoint_continuation_used") is False and
|
|
summary.get("within_trial_weight_updates") is False and summary.get("hidden_early_stop") is False and
|
|
summary.get("entire_neural_trial_executed") is True and summary.get("record_history") is False and
|
|
summary.get("parent_checkpoint_manifest_sha256") is None)
|
|
checks[label + ".mode_precision_delay"] = (summary.get("diagnostic_mode") == mode and
|
|
summary.get("precision") == "float64" and summary.get("apl_precision") == "float32" and
|
|
_integer(summary, "apl_delay_steps", 18) and summary.get("state_units") == "SI volts")
|
|
checks[label + ".complete_reported_finite"] = (summary.get("complete") is True and summary.get("pass_all") is True and
|
|
summary.get("all_finite") is True and _integer(summary, "first_failure_tick", -1))
|
|
checks[label + ".effective_weight_authority"] = summary.get("effective_weights_raw_sha256") == expected_effective_weight_sha256
|
|
checks[label + ".input_request_array"] = np.array_equal(record["requests"][:, :2], expected_requests.astype(np.int64))
|
|
checks[label + ".request_and_spike_accounting"] = (summary.get("counts_match_sparse_events") is True and
|
|
_integer(summary, "total_spikes", len(record["state"]["step"])) and
|
|
_integer(summary, "requested", len(expected_requests)) and
|
|
_integer(summary, "accepted_requests", int(record["requests"][:, 3].sum(dtype=np.int64))) and
|
|
_integer(summary, "rejected_requests", int(record["requests"][:, 4].sum(dtype=np.int64))))
|
|
checks[label + ".recorded_full_APL_terminal"] = (record.get("terminal_origin") == "recorded_terminal_queues" and
|
|
record["terminal"]["apl_pending"].shape == (18, N_KC))
|
|
full = reference_record
|
|
full_summary = full["summary"]
|
|
checks["reference.full_diagnostics"] = (set(full["traces"]) == set(C.TRACE_FILES) and
|
|
full.get("state_hashes") is not None and full.get("queues") is not None and
|
|
all(full_summary.get(key) is True for key in
|
|
("full_APL_history_recorded", "full_tick_state_hashes_recorded", "full_queue_hashes_recorded", "full_sequence_bitwise_audit_performed")))
|
|
if full.get("queues") is not None:
|
|
checks["reference.queue_summary_accounting"] = (
|
|
_integer(full_summary, "arriving_edges", sum(row["count"] for row in full["queues"])) and
|
|
_integer(full_summary, "accepted_arriving_edges", sum(row["accepted_count"] for row in full["queues"])))
|
|
else:
|
|
checks["reference.queue_summary_accounting"] = False
|
|
light_summary = light_record["summary"]
|
|
checks["light.declared_recording_scope"] = (not light_record["traces"] and
|
|
light_record.get("state_hashes") is None and light_record.get("queues") is None and
|
|
all(light_summary.get(key) is False for key in
|
|
("full_APL_history_recorded", "full_tick_state_hashes_recorded", "full_queue_hashes_recorded", "full_sequence_bitwise_audit_performed")))
|
|
for label, summary in (("reference", full_summary), ("light", light_summary)):
|
|
inputs = summary.get("input_sha256")
|
|
checks[label + ".input_hash_schema"] = (type(inputs) is dict and
|
|
set(inputs) == {"base_weights.npy", "coefficients.json", "requests.npz", "spec.json"} and
|
|
all(_valid_sha(value) for value in inputs.values()))
|
|
for name in ("base_weights.npy", "coefficients.json"):
|
|
left = full_summary.get("input_sha256", {}).get(name)
|
|
right = light_summary.get("input_sha256", {}).get(name)
|
|
checks["pair.input." + name] = _valid_sha(left) and left == right
|
|
for name in ("arriving_edges", "accepted_arriving_edges"):
|
|
checks["pair.summary." + name] = (type(full_summary.get(name)) is int and
|
|
type(light_summary.get(name)) is int and full_summary[name] == light_summary[name])
|
|
checks = {key: bool(value) for key, value in checks.items()}
|
|
exact = all(checks.values())
|
|
return dict(schema="conditioning-preflight-epoch-comparison-v1", numerical_exact=exact, pass_all=exact,
|
|
status="EXACT-RECORDED-EPOCH-PAIR-NOT-LEARNING" if exact else "FAIL-EPOCH-COMPARISON",
|
|
checks=checks, failed_checks=[key for key, ok in checks.items() if not ok],
|
|
expected_steps=expected_steps, expected_requests=_evidence(expected_requests),
|
|
expected_effective_weight_raw_sha256=expected_effective_weight_sha256,
|
|
recorded_pair_availability=inherited["reference_comparisons_available"],
|
|
activity=inherited["activity"], nonvacuous_activity=inherited["nonvacuous_activity"],
|
|
activity_is_descriptive_not_a_new_gate=True,
|
|
full_reference_diagnostics_required=True, light_intermediate_states_observed=False,
|
|
light_per_tick_V_g_ref_identity_claimed=False, independent_finite_latch_replay_performed=False,
|
|
independent_CSR_queue_reconstruction_performed=False,
|
|
independent_wholebrain_CPU_oracle_performed=False,
|
|
expected_comparator_source_sha256=COMPARATOR_SHA256, source_pin_verified_here=False,
|
|
external_file_seals_process_and_input_authority_verified_here=False,
|
|
input_generator_or_parent_delta_chain_verified_here=False,
|
|
learning_pass_claimed=False, candidate_generated=False,
|
|
scope="Supplied fresh epoch records and original requests only; prerequisite external checkpoint/source/parent verification belongs to coordinator")
|
|
|
|
|
|
def _delta(value, name):
|
|
_require(isinstance(value, np.ndarray) and value.dtype == np.float32 and value.shape == (CANDIDATE_COUNT,) and
|
|
np.isfinite(value).all(), name + " must be the full finite float32 candidate delta vector")
|
|
|
|
|
|
def audit_delta_binding(delta_before, delta_after, installed_delta, *, update_receipt,
|
|
parent_snapshot_sha256, child_manifest):
|
|
"""Bind already computed/installed arrays to receipt and learned metadata.
|
|
|
|
parent_snapshot_sha256 means the actual PARENT OVERLAY NPZ file hash,
|
|
externally established by the coordinator, NOT a checkpoint manifest hash
|
|
and NOT a raw delta-array hash. installed_delta must come from the child
|
|
overlay subsequently checked by frozen runtime_light.load_overlay.
|
|
|
|
This function does not read those files, authenticate anatomical IDs,
|
|
recompute eligibility-v2, establish clipping bounds, or prove a causal
|
|
behavioral effect. Call conditioning_update.verify_receipt separately
|
|
against actual sealed epoch events and complete pinned mappings.
|
|
"""
|
|
for name, value in (("delta_before", delta_before), ("delta_after", delta_after), ("installed_delta", installed_delta)):
|
|
_delta(value, name)
|
|
_require(_valid_sha(parent_snapshot_sha256), "Externally established parent overlay FILE SHA is required")
|
|
_require(type(update_receipt) is dict and type(child_manifest) is dict, "Receipt and manifest must be dictionaries")
|
|
_require(type(update_receipt.get("input_arrays")) is dict and type(update_receipt.get("outputs")) is dict and
|
|
type(update_receipt.get("source_pins")) is dict and type(update_receipt.get("command")) is dict and
|
|
type(update_receipt.get("semantics")) is dict and type(update_receipt.get("counts")) is dict,
|
|
"Incomplete epoch update receipt schema")
|
|
before, after, installed = (_evidence(value) for value in (delta_before, delta_after, installed_delta))
|
|
command = update_receipt["command"]
|
|
counts = update_receipt["counts"]
|
|
checks = {
|
|
"receipt.self_hash": update_receipt.get("payload_sha256") == _sha({k: v for k, v in update_receipt.items() if k != "payload_sha256"}),
|
|
"receipt.schema": update_receipt.get("schema") == "phase4an-conditioning-update-draft-v1" and
|
|
update_receipt.get("status") == "DRAFT-COMPUTED-EPOCH-UPDATE-NOT-INSTALLED",
|
|
"receipt.original_source_pins": update_receipt["source_pins"] == {
|
|
"eligibility_bridge_sha256": ELIGIBILITY_BRIDGE_SHA256, "phase4j_sha256": PHASE4J_SHA256},
|
|
"receipt.delta_before": _canonical(update_receipt["input_arrays"].get("delta_before")) == _canonical(before),
|
|
"receipt.delta_after": _canonical(update_receipt["outputs"].get("delta_after")) == _canonical(after),
|
|
"computed_installed_delta_bitwise": C.bit_equal(delta_after, installed_delta),
|
|
"delta.nonpositive": bool(np.all(delta_before <= 0) and np.all(delta_after <= 0)),
|
|
"delta.no_potentiation": bool(np.all(delta_after <= delta_before)),
|
|
"receipt.candidate_count": _integer(counts, "candidates", CANDIDATE_COUNT),
|
|
"receipt.changed_bit_count": _integer(counts, "actual_bitwise_changed_edges", int(np.count_nonzero(delta_before.view(np.uint32) != delta_after.view(np.uint32)))),
|
|
"receipt.changed_value_count": _integer(counts, "actual_value_changed_edges", int(np.count_nonzero(delta_before != delta_after))),
|
|
"receipt.explicit_scheduled_D": type(command.get("D_scheduled")) is float and command["D_scheduled"] in (0.0, 1.0) and
|
|
type(command.get("frozen")) is bool,
|
|
"receipt.epoch_boundary_only": update_receipt["semantics"].get("application") == "epoch_boundary_after_all_recorded_ticks; affects subsequent epoch only" and
|
|
update_receipt["semantics"].get("delta_dtype") == "float32" and update_receipt["semantics"].get("delta_units") == "mV",
|
|
"receipt.not_a_DAN_measurement": update_receipt["semantics"].get("D_definition") == "artificial scheduled scalar" and
|
|
update_receipt["semantics"].get("actual_DAN_delivery_observed") is False and
|
|
update_receipt["semantics"].get("actual_DAN_delivery_used_to_gate") is False,
|
|
"manifest.exact_fields": set(child_manifest) == MANIFEST_FIELDS,
|
|
"manifest.learned_schema": child_manifest.get("schema") == "phase4an-overlay-v1" and child_manifest.get("overlay_name") == "learned",
|
|
"manifest.parent_overlay_file_sha": child_manifest.get("parent_snapshot_sha256") == parent_snapshot_sha256,
|
|
"manifest.frozen_rule_mask_helper": child_manifest.get("eligibility_v2_source_sha256") == PHASE4J_SHA256 and
|
|
child_manifest.get("mask_sha256") == MASK_SHA256 and child_manifest.get("am_overlay_sha256") == AM_OVERLAY_SHA256,
|
|
"manifest.declared_file_hashes": all(_valid_sha(child_manifest.get(key)) for key in ("overlay_sha256", "base_candidate_sha256")),
|
|
"manifest.declared_paths": all(type(child_manifest.get(key)) is str and bool(child_manifest[key])
|
|
for key in ("mask_path", "base_candidate_path", "aj_model_dir", "am_overlay_path")),
|
|
}
|
|
no_update = command.get("frozen") is True or command.get("D_scheduled") == 0.0
|
|
checks["zero_D_or_frozen_identity"] = not no_update or C.bit_equal(delta_before, delta_after)
|
|
for name in ("weights_installed", "base_modified", "brain_dynamics_executed", "scientific_learning_pass_claimed", "candidate_generated"):
|
|
checks["receipt.scope." + name] = update_receipt.get(name) is False
|
|
checks = {key: bool(value) for key, value in checks.items()}
|
|
exact = all(checks.values())
|
|
return dict(schema="conditioning-preflight-delta-binding-v1", pass_all=exact,
|
|
status="EXACT-DELTA-ARRAY-BINDING-NOT-LEARNING" if exact else "FAIL-DELTA-BINDING",
|
|
checks=checks, failed_checks=[key for key, ok in checks.items() if not ok],
|
|
delta_before=before, computed_delta_after=after, installed_delta=installed,
|
|
parent_overlay_file_sha256=parent_snapshot_sha256,
|
|
declared_child_overlay_file_sha256=child_manifest.get("overlay_sha256"),
|
|
update_receipt_payload_sha256=update_receipt.get("payload_sha256"),
|
|
actual_changed_edges=int(np.count_nonzero(delta_before != delta_after)),
|
|
actual_bitwise_changed_edges=int(np.count_nonzero(delta_before.view(np.uint32) != delta_after.view(np.uint32))),
|
|
delta_nonzero_or_change_is_not_learning_evidence=True,
|
|
eligibility_v2_independently_recomputed_here=False, full_mask_identity_verified_here=False,
|
|
candidate_order_or_compartment_mapping_verified_here=False, floor_bound_independently_verified_here=False,
|
|
file_bytes_and_parent_snapshot_authority_verified_here=False, installation_in_runtime_verified_here=False,
|
|
actual_DAN_delivery_verified_here=False, learning_pass_claimed=False, candidate_generated=False,
|
|
scope="Supplied float32 mV arrays and declared parent/update metadata only; coordinator must verify real files, v2 recomputation and next-epoch installation")
|