Digital Drosophila Archive
140 lines
9.1 KiB
Python
140 lines
9.1 KiB
Python
"""Pure, disjoint source labels from frozen anatomical metadata evidence.
|
|
|
|
No files, trajectories, Torch, simulation, or parameters are touched. The
|
|
caller must verify context.META against its registered SHA-256 before loading.
|
|
Class membership follows Phase 4AG/4X; transmitter metadata is an orthogonal
|
|
diagnostic, never anatomical DAN identity or a reward/valence assertion.
|
|
"""
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
|
|
N_NEURONS = 166700
|
|
REQUIRED_COLUMNS = ("neuron_idx", "body_id", "annotation_class", "annotation_type",
|
|
"neurotransmitter", "nt_ground_truth")
|
|
GROUPS = ("KC", "APL", "ALPN", "MBON", "anatomical_DAN",
|
|
"dopamine_annotation_only", "other", "unclassified")
|
|
METADATA_SHA256 = "922af16846a065c3dbfca7c11146b83ca217fcbd7392edc745f444a04eae8c63"
|
|
FROZEN_4X_ROLES_SHA256 = "76ba0cae6c5db417ec030456797f7935216c6877bd339d5e435133a0061e3930"
|
|
FROZEN_4X_SIDECAR_SHA256 = "8b51d12d84d66cdca66aeb7d052a24e520b9c28f462bd6bf87f37dda85c33284"
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def _text_column(meta, name):
|
|
column = meta[name]
|
|
# Do not silently turn corrupt numeric metadata into a plausible label.
|
|
require(bool(column.map(lambda value: pd.isna(value) or isinstance(value, str)).all()),
|
|
"Expected nullable string metadata: " + name)
|
|
return column.fillna("").astype(str).to_numpy()
|
|
|
|
|
|
def classify(meta):
|
|
"""Return (Unicode labels[N], JSON-serializable report), indexed by global ID.
|
|
|
|
The full metadata must already be in exact neuron_idx order. Conflicting
|
|
exact identities and absent/explicitly unknown identities are retained as
|
|
unclassified instead of silently resolving them by assignment precedence.
|
|
DPM is reported explicitly but belongs to other (unless its annotations
|
|
conflict). A PAM/PPL1/DAN name prefix alone never creates a DAN identity.
|
|
"""
|
|
require(isinstance(meta, pd.DataFrame), "classify expects a pandas DataFrame")
|
|
missing = set(REQUIRED_COLUMNS) - set(meta.columns)
|
|
require(not missing, "Missing actual metadata schema fields: " + ", ".join(sorted(missing)))
|
|
require(meta.columns.is_unique, "Duplicate metadata columns")
|
|
require(len(meta) == N_NEURONS, "Expected the complete 166700-neuron metadata")
|
|
ids = meta["neuron_idx"].to_numpy()
|
|
bodies = meta["body_id"].to_numpy()
|
|
require(ids.dtype == np.int32 and np.array_equal(ids, np.arange(N_NEURONS, dtype=np.int32)),
|
|
"neuron_idx must be int32, complete, unique and in canonical row order")
|
|
require(bodies.dtype == np.uint64 and len(np.unique(bodies)) == N_NEURONS,
|
|
"body_id must be uint64 and unique")
|
|
cls, typ = _text_column(meta, "annotation_class"), _text_column(meta, "annotation_type")
|
|
nt_raw = _text_column(meta, "neurotransmitter")
|
|
ground_truth = _text_column(meta, "nt_ground_truth")
|
|
# Match frozen 4X normalization for the transmitter axis only. Anatomical
|
|
# class/type membership retains exact, case-sensitive original values.
|
|
nt = np.char.lower(np.char.strip(nt_raw.astype(str)))
|
|
dopamine = nt == "dopamine"
|
|
dpm = typ == "DPM"
|
|
masks = {"KC": cls == "Kenyon_Cell", "APL": typ == "APL", "ALPN": cls == "ALPN",
|
|
"MBON": cls == "MBON", "anatomical_DAN": cls == "DAN"}
|
|
hits = np.stack(list(masks.values())).sum(axis=0)
|
|
# Preserve the frozen 4X DAN/DPM conflict explicitly, and do not silently
|
|
# prefer an exact APL type over an incompatible exact anatomical class.
|
|
conflict = (hits > 1) | (dpm & (hits > 0))
|
|
labels = np.full(N_NEURONS, "other", dtype="<U32")
|
|
for name, mask in masks.items():
|
|
labels[mask & ~conflict] = name
|
|
unassigned = hits == 0
|
|
diagnostic_dopamine = dopamine & unassigned & ~dpm & ~conflict
|
|
labels[diagnostic_dopamine] = "dopamine_annotation_only"
|
|
# These are explicit missingness/uncertainty markers, not inferred types.
|
|
unknown_tokens = ("", "unknown", "unclear", "unclassified")
|
|
class_unknown = np.isin(np.char.lower(np.char.strip(cls.astype(str))), unknown_tokens)
|
|
type_unknown = np.isin(np.char.lower(np.char.strip(typ.astype(str))), unknown_tokens)
|
|
no_anatomical_annotation = class_unknown & type_unknown
|
|
dan_name_hint = np.char.startswith(typ.astype(str), ("PAM")) | \
|
|
np.char.startswith(typ.astype(str), ("PPL1")) | \
|
|
np.char.startswith(typ.astype(str), ("DAN"))
|
|
uncertain_dan_prefix = dan_name_hint & ~masks["anatomical_DAN"]
|
|
unresolved = unassigned & ~dpm & ~diagnostic_dopamine & (
|
|
no_anatomical_annotation | (class_unknown & uncertain_dan_prefix))
|
|
labels[unresolved | conflict] = "unclassified"
|
|
# Orthogonal flags remain auditable even when the primary label is disjoint.
|
|
flag_masks = {"annotation_DPM": dpm, "metadata_dopamine": dopamine,
|
|
"annotation_DAN": masks["anatomical_DAN"], "identity_conflict": conflict,
|
|
"multiple_exact_group_hits": hits > 1,
|
|
"no_anatomical_annotation": no_anatomical_annotation,
|
|
"DAN_name_prefix_without_DAN_class": uncertain_dan_prefix,
|
|
"metadata_dopamine_not_annotation_DAN": dopamine & ~masks["anatomical_DAN"],
|
|
"annotation_DAN_not_metadata_dopamine": masks["anatomical_DAN"] & ~dopamine,
|
|
"has_raw_ground_truth": np.char.strip(ground_truth.astype(str)) != ""}
|
|
counts = {name: int(np.count_nonzero(labels == name)) for name in GROUPS}
|
|
require(sum(counts.values()) == N_NEURONS and set(np.unique(labels)).issubset(GROUPS),
|
|
"Output groups do not partition the complete graph")
|
|
require(not np.any((labels == "anatomical_DAN") & dpm), "DPM must never be assigned anatomical_DAN")
|
|
report = {"neurons": N_NEURONS, "groups": counts, "group_order": list(GROUPS),
|
|
"all_neurons_assigned_once": True, "labels_index": "global neuron_idx, identical to canonical row index",
|
|
"identity_conflicts_present": bool(np.any(conflict)),
|
|
"pass_all": bool(not np.any(conflict)),
|
|
"pass_all_scope": "schema, complete disjoint accounting and no conflicting exact identities; not biological validation",
|
|
"flags": {name: int(mask.sum()) for name, mask in flag_masks.items()},
|
|
"flag_counts_by_group": {name: {group: int(np.count_nonzero(mask & (labels == group)))
|
|
for group in GROUPS} for name, mask in flag_masks.items()},
|
|
"DPM_neuron_idx": ids[dpm].tolist(), "identity_conflict_neuron_idx": ids[conflict].tolist(),
|
|
"uncertain_DAN_prefix_neuron_idx": ids[uncertain_dan_prefix].tolist(),
|
|
"definitions": {
|
|
"KC": "annotation_class exactly Kenyon_Cell",
|
|
"APL": "annotation_type exactly APL",
|
|
"ALPN": "annotation_class exactly ALPN; annotation_class olfactory is NOT included",
|
|
"MBON": "annotation_class exactly MBON",
|
|
"anatomical_DAN": "annotation_class exactly DAN, no exact identity conflict; not inferred from NT or PAM/PPL1 prefix",
|
|
"dopamine_annotation_only": "normalized neurotransmitter=dopamine, no named exact group, no DPM; transmitter-evidence diagnostic, NOT DAN",
|
|
"other": "remaining annotated neurons, explicitly including non-conflicting annotation_type DPM",
|
|
"unclassified": "conflicting exact identity, no anatomical class/type evidence, or only uncertain DAN-name hint without class/NT evidence"},
|
|
"uncertainty_policy": {
|
|
"explicit_unknown_tokens_casefolded": list(unknown_tokens),
|
|
"PAM_PPL1_DAN_prefix_never_assigns_identity": True,
|
|
"transmitter_role_and_anatomical_identity_are_separate": True,
|
|
"DPM_retained_as_other_with_explicit_flag": True,
|
|
"missing_ground_truth_is_not_filled_or_guessed": True,
|
|
"other_does_not_mean_known_function": True},
|
|
"provenance": {
|
|
"metadata_path_contract": "context.META; caller must verify its preregistered SHA-256 before loading",
|
|
"metadata_expected_sha256_from_AG_r2": METADATA_SHA256,
|
|
"frozen_rules_path": "/data/digital-drosophila/models/male-cns-lif/transmitter-audit-v1/transmitter_roles_v1.py",
|
|
"frozen_rules_sha256": FROZEN_4X_ROLES_SHA256,
|
|
"frozen_sidecar_path": "/data/digital-drosophila/experiments/phase4x-transmitter-provenance/roles/neuron-evidence-axes.parquet",
|
|
"frozen_sidecar_sha256": FROZEN_4X_SIDECAR_SHA256,
|
|
"frozen_MB_DAN_mapping_path": "/data/digital-drosophila/experiments/phase4h-dopamine-plasticity/mappings/dan-to-mbon-confirmed.parquet",
|
|
"frozen_MB_DAN_mapping_used_to_assign_identity": False,
|
|
"file_hash_verified_by_this_pure_function": False,
|
|
"differences_from_4X_reporting": "ALPN separated from broader olfactory class; DPM grouped other with flag; absent/conflicting identity explicitly unclassified"},
|
|
"source_graph_changed": False, "simulation_executed": False,
|
|
"heldout_dynamic_access": False, "reward_or_valence_inferred": False}
|
|
return labels, report
|