Digital Drosophila Archive
206 lines
12 KiB
Python
206 lines
12 KiB
Python
"""Read-only exact-table contracts for the proposed conditioning branch.
|
|
|
|
No body identity, compartment or behavioral function is inferred from a name.
|
|
The specific two interfaces are explicit protocol hypotheses checked against
|
|
the frozen H maps and actual graph edges. No training or file write occurs.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
|
|
INTERFACES = {"PAM15": ("y5B'2a", "MBON01"), "PPL103": ("y2a'1", "MBON12")}
|
|
N = 166700
|
|
E = 25582938
|
|
INPUT_PINS = {
|
|
"datasets/male-cns-v1.0/processed/neuron-only/neuron-index/unified-metadata.parquet":
|
|
"922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63",
|
|
"experiments/phase4h-dopamine-plasticity/plastic-mask/plastic-edge-candidates.parquet":
|
|
"e5075ee3e5a0edf3a4d197453847163cc295b254f21c67203159bba204b12a76",
|
|
"experiments/phase4h-dopamine-plasticity/mappings/dan-to-compartment.parquet":
|
|
"a16f72c3121ed0ec638c1a2b249ace446e6ccee020b49679818eb42589d00662",
|
|
"experiments/phase4h-dopamine-plasticity/mappings/mbon-to-compartment.parquet":
|
|
"54f371f7e35b9def44becd1ba9bd2872861f4876b255b2c8ec46feee6473b70c",
|
|
"experiments/phase4h-dopamine-plasticity/mappings/dan-to-mbon-confirmed.parquet":
|
|
"730d000f25667b59181d2ee6703c64591229a8beab0856197ed71e68d799d191",
|
|
}
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open("rb") as stream:
|
|
for block in iter(lambda: stream.read(8 << 20), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def _columns(frame, names, label):
|
|
require(isinstance(frame, pd.DataFrame) and set(names) <= set(frame.columns),
|
|
"Missing observed schema columns: " + label)
|
|
|
|
|
|
def _identity(frame, meta, label):
|
|
_columns(frame, ("neuron_idx", "body_id", "annotation_type", "annotation_instance"), label)
|
|
require(frame.neuron_idx.dtype == np.int32 and frame.body_id.dtype == np.uint64,
|
|
"Canonical identity dtypes changed: " + label)
|
|
ids = frame.neuron_idx.to_numpy()
|
|
require(len(np.unique(ids)) == len(ids) and np.all((ids >= 0) & (ids < len(meta))),
|
|
"Ambiguous/out-of-range identity: " + label)
|
|
for field in ("body_id", "annotation_type", "annotation_instance"):
|
|
actual = frame[field].reset_index(drop=True)
|
|
expected = meta.iloc[ids][field].reset_index(drop=True)
|
|
require(actual.equals(expected), "Identity field mismatch: " + label + "/" + field)
|
|
|
|
|
|
def _edges(frame, meta, src, dst, signed_raw, label):
|
|
names = ("edge_id", "source_neuron_idx", "source_body_id", "target_neuron_idx", "target_body_id",
|
|
"raw_synapse_count", "source_annotation_type", "source_annotation_instance",
|
|
"target_annotation_type", "target_annotation_instance")
|
|
_columns(frame, names, label)
|
|
require(frame.edge_id.dtype == np.int64 and frame.edge_id.is_unique, "Invalid edge IDs: " + label)
|
|
ids = frame.edge_id.to_numpy()
|
|
require(np.all((ids >= 0) & (ids < len(src))), "Edge outside graph: " + label)
|
|
require(frame.raw_synapse_count.dtype == np.int64 and (frame.raw_synapse_count > 0).all(),
|
|
"Invalid anatomical contact count: " + label)
|
|
require(np.array_equal(np.abs(signed_raw[ids]), frame.raw_synapse_count.to_numpy()),
|
|
"Anatomical edge weight mismatch: " + label)
|
|
for side, expected_idx in (("source", src[ids]), ("target", dst[ids])):
|
|
require(frame[side + "_neuron_idx"].dtype == np.int32 and
|
|
frame[side + "_body_id"].dtype == np.uint64,
|
|
"Canonical edge identity dtype mismatch: " + label)
|
|
require(np.array_equal(frame[side + "_neuron_idx"].to_numpy(), expected_idx),
|
|
"Actual graph endpoint mismatch: " + label)
|
|
for field in ("body_id", "annotation_type", "annotation_instance"):
|
|
actual = frame[side + "_" + field].reset_index(drop=True)
|
|
expected = meta.iloc[expected_idx][field].reset_index(drop=True)
|
|
require(actual.equals(expected), "Actual edge identity mismatch: " + label + "/" + side + "/" + field)
|
|
|
|
|
|
def audit_interface_tables(meta, dan, mbon, confirmed, candidates, src, dst, signed_raw):
|
|
"""Pure table audit; small synthetic graphs are allowed in unit tests only.
|
|
|
|
The production loader below separately requires the complete N/E identity.
|
|
Supplied annotation population membership is never filled from a prefix.
|
|
"""
|
|
_columns(meta, ("neuron_idx", "body_id", "annotation_class", "annotation_type", "annotation_instance"), "metadata")
|
|
require(meta.neuron_idx.dtype == np.int32 and meta.body_id.dtype == np.uint64,
|
|
"Canonical metadata dtype mismatch")
|
|
require(np.array_equal(meta.neuron_idx.to_numpy(), np.arange(len(meta), dtype=np.int32)) and meta.body_id.is_unique,
|
|
"Metadata must have canonical complete ordered identities")
|
|
require(src.dtype == dst.dtype == np.int32 and signed_raw.dtype == np.int64 and
|
|
src.ndim == dst.ndim == signed_raw.ndim == 1 and len(src) == len(dst) == len(signed_raw),
|
|
"Graph endpoint/weight schema mismatch")
|
|
require(np.all((src >= 0) & (src < len(meta))) and np.all((dst >= 0) & (dst < len(meta))),
|
|
"Graph endpoint out of range")
|
|
_identity(dan, meta, "DAN map")
|
|
_identity(mbon, meta, "MBON map")
|
|
for table, klass, label in ((dan, "DAN", "DAN"), (mbon, "MBON", "MBON")):
|
|
require(np.array_equal(np.sort(table.neuron_idx.to_numpy()), np.flatnonzero(meta.annotation_class.eq(klass))),
|
|
"Mapping must include the complete exact-annotation population: " + label)
|
|
_columns(table, ("mb_compartment_verbatim", "mb_compartment_mapping_status"), label)
|
|
_edges(confirmed, meta, src, dst, signed_raw, "confirmed DAN-MBON")
|
|
_edges(candidates, meta, src, dst, signed_raw, "plastic candidates")
|
|
_columns(confirmed, ("source_compartment_verbatim", "target_compartment_verbatim", "exact_compartment_match"), "confirmed DAN-MBON")
|
|
_columns(candidates, ("target_compartment_verbatim", "plasticity_polarity"), "plastic candidates")
|
|
require(confirmed.exact_compartment_match.dtype == np.bool_ and confirmed.exact_compartment_match.all(),
|
|
"Confirmed table contains an unknown compartment relation")
|
|
require(confirmed.source_compartment_verbatim.notna().all() and
|
|
confirmed.source_compartment_verbatim.equals(confirmed.target_compartment_verbatim),
|
|
"Confirmed exact compartment relation differs")
|
|
require(meta.iloc[candidates.source_neuron_idx.to_numpy()].annotation_class.eq("Kenyon_Cell").all() and
|
|
meta.iloc[candidates.target_neuron_idx.to_numpy()].annotation_class.eq("MBON").all(),
|
|
"Candidate is not a real KC-MBON edge")
|
|
require((signed_raw[candidates.edge_id.to_numpy()] > 0).all() and candidates.plasticity_polarity.eq(-1).all(),
|
|
"Frozen KC base sign/polarity changed")
|
|
pop = {klass: np.flatnonzero(meta.annotation_class.eq(klass)).astype(np.int32)
|
|
for klass in ("Kenyon_Cell", "MBON", "DAN")}
|
|
reports = {}
|
|
for dan_type, (compartment, mbon_type) in INTERFACES.items():
|
|
dd = dan[dan.annotation_type.eq(dan_type)].sort_values("neuron_idx")
|
|
mm = mbon[mbon.annotation_type.eq(mbon_type)].sort_values("neuron_idx")
|
|
require(len(dd) > 0 and len(mm) > 0, "Missing exact type: " + dan_type)
|
|
for table in (dd, mm):
|
|
require(table.mb_compartment_mapping_status.eq("explicit_single_parenthesized_token").all() and
|
|
table.mb_compartment_verbatim.eq(compartment).all(),
|
|
"Unknown or mismatched compartment: " + dan_type)
|
|
edges = confirmed[confirmed.source_neuron_idx.isin(dd.neuron_idx)]
|
|
require(len(edges) > 0 and edges.source_compartment_verbatim.eq(compartment).all() and
|
|
edges.target_compartment_verbatim.eq(compartment).all() and
|
|
edges.target_neuron_idx.isin(mm.neuron_idx).all(),
|
|
"Specific DAN interface has unknown or mismatched actual target")
|
|
require(set(edges.source_neuron_idx) == set(dd.neuron_idx) and
|
|
set(edges.target_neuron_idx) == set(mm.neuron_idx),
|
|
"Incomplete actual DAN/MBON type coverage")
|
|
plastic = candidates[candidates.target_compartment_verbatim.eq(compartment)]
|
|
require(len(plastic) > 0 and plastic.target_neuron_idx.isin(mm.neuron_idx).all() and
|
|
set(plastic.target_neuron_idx) == set(mm.neuron_idx),
|
|
"Frozen plastic subset is not the exact mapped target population")
|
|
pop[dan_type] = dd.neuron_idx.to_numpy(dtype=np.int32)
|
|
pop[mbon_type] = mm.neuron_idx.to_numpy(dtype=np.int32)
|
|
def members(frame):
|
|
return frame[["neuron_idx", "body_id", "annotation_type", "annotation_instance", "mb_compartment_verbatim"]].to_dict(orient="records")
|
|
reports[dan_type] = dict(compartment=compartment, MBON_type=mbon_type,
|
|
DAN_members=members(dd), MBON_members=members(mm),
|
|
confirmed_DAN_MBON_edge_ids=edges.edge_id.tolist(),
|
|
confirmed_DAN_MBON_edges=len(edges), confirmed_DAN_MBON_contacts=int(edges.raw_synapse_count.sum()),
|
|
candidate_edges=len(plastic), candidate_contacts=int(plastic.raw_synapse_count.sum()),
|
|
unknown_selected_interface_relations=0,
|
|
functional_valence="cross-dataset type-level hypothesis, not measured MaleCNS body function",
|
|
artificial_US=True, D_equals_measured_dopamine=False)
|
|
return pop, dict(interfaces=reports, whole_DAN_mapping_unknown=int(dan.mb_compartment_verbatim.isna().sum()),
|
|
whole_MBON_mapping_unknown=int(mbon.mb_compartment_verbatim.isna().sum()),
|
|
all_candidate_edges_retained=len(candidates), populations={k: len(v) for k, v in pop.items()},
|
|
global_unknowns_filled=False, training=False, new_identity_inferred=False)
|
|
|
|
|
|
def _complete_graph_identity(meta, graph, n=N, e=E):
|
|
"""Exact full-graph dimensions; small explicit sizes are for unit tests."""
|
|
require(len(meta) == n and graph["nodes"].dtype == np.int32 and
|
|
np.array_equal(graph["nodes"], np.arange(n, dtype=np.int32)), "Complete canonical neuron set required")
|
|
require(graph["edge_id"].dtype == np.int64 and
|
|
np.array_equal(graph["edge_id"], np.arange(e, dtype=np.int64)), "Complete canonical edge set required")
|
|
require(all(graph[name].shape == (e,) for name in ("src", "dst", "signed_raw")),
|
|
"Complete graph endpoint/weight lengths required")
|
|
|
|
|
|
def load_bound_tables(root, graph_manifest):
|
|
"""Require external, registered graph hashes, not self-authorized paths.
|
|
|
|
graph_manifest maps the absolute graph paths to their authorized SHA-256.
|
|
The coordinator must bind this dictionary to the preregistration/parent.
|
|
"""
|
|
root = Path(root).resolve()
|
|
observed = {}
|
|
tables = []
|
|
for relative, sha in INPUT_PINS.items():
|
|
path = root / relative
|
|
require(digest(path) == sha, "Frozen table SHA mismatch: " + relative)
|
|
observed[str(path)] = sha
|
|
tables.append(pd.read_parquet(path))
|
|
meta, candidates, dan, mbon, confirmed = tables
|
|
graph_dir = root / "experiments/phase4ag-author-apl-r2/graphs/full"
|
|
graph = {}
|
|
for name in ("nodes", "edge_id", "src", "dst", "signed_raw"):
|
|
path = graph_dir / (name + ".npy")
|
|
require(str(path) in graph_manifest and digest(path) == graph_manifest[str(path)],
|
|
"Unbound/changed complete graph input: " + name)
|
|
observed[str(path)] = graph_manifest[str(path)]
|
|
graph[name] = np.load(path, mmap_mode="r", allow_pickle=False)
|
|
_complete_graph_identity(meta, graph)
|
|
require(len(candidates) == 14232, "Complete frozen mask required")
|
|
populations, audit = audit_interface_tables(meta, dan, mbon, confirmed, candidates,
|
|
graph["src"], graph["dst"], graph["signed_raw"])
|
|
require(all(digest(path) == sha for path, sha in observed.items()), "Input bytes changed during audit")
|
|
return populations, dict(audit, input_sha256=observed, schema={
|
|
name: table.dtypes.astype(str).to_dict() for name, table in
|
|
(("metadata", meta), ("candidates", candidates), ("DAN", dan), ("MBON", mbon), ("confirmed", confirmed))})
|