Digital Drosophila Archive
76 lines
5.0 KiB
Python
76 lines
5.0 KiB
Python
"""Register conditional, CPU-only accounting; no new brain trajectories."""
|
|
from datetime import datetime, timezone
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from context import ROOT, MODEL, EXP, AG, AG_MODEL, META, COEFF, read, save, digest, sources, integrity
|
|
from populations import classify, METADATA_SHA256, FROZEN_4X_ROLES_SHA256
|
|
|
|
|
|
def main():
|
|
assert not EXP.exists() or not any(EXP.iterdir()), 'Never reuse a populated audit directory'
|
|
prior = read(AG / 'audit/final-status.json')
|
|
assert prior['status'] == 'FAIL-FIXED-CAL-REPRESENTATION' and prior['numerical_gate_pass']
|
|
for name in ('protocol', 'regression', 'trials', 'audit', 'logs'):
|
|
(EXP / name).mkdir(parents=True, exist_ok=False)
|
|
save(EXP / 'audit/preflight-directory.json', dict(
|
|
first_attempt='Parent experiments directory was not writable; failed before creating any audit directory/file or registration',
|
|
resolution='Administrator creates only the exact new empty audit directory owned by qishizhe; parent/frozen permissions unchanged',
|
|
empty_scaffold_allowed=True, scientific_accounting_started=False))
|
|
command = [sys.executable, '-B', '-m', 'unittest', '-v',
|
|
'test_replay_core', 'test_populations', 'test_accounting']
|
|
completed = subprocess.run(command, cwd=MODEL, capture_output=True, text=True)
|
|
with (EXP / 'regression/unit-tests.log').open('x') as f:
|
|
f.write(completed.stdout + completed.stderr)
|
|
save(EXP / 'regression/unit-tests.json', dict(command=command, exit_code=completed.returncode,
|
|
pass_all=completed.returncode == 0, sha256=digest(EXP / 'regression/unit-tests.log')))
|
|
assert completed.returncode == 0, 'Pure audit tests failed; preserve output and stop'
|
|
paths = [AG / 'manifest.json', AG / 'checksums.sha256', AG_MODEL / 'manifest.json',
|
|
AG_MODEL / 'checksums.sha256', META, COEFF,
|
|
ROOT / 'models/male-cns-lif/author-runtime-lab-v1/author_ext.cu',
|
|
ROOT / 'models/male-cns-lif/transmitter-audit-v1/transmitter_roles_v1.py',
|
|
ROOT / 'datasets/male-cns-v1.0/processed/neuron-only/gpu/csr-weights.int64.npy']
|
|
assert digest(META) == METADATA_SHA256
|
|
assert digest(ROOT / 'models/male-cns-lif/transmitter-audit-v1/transmitter_roles_v1.py') == FROZEN_4X_ROLES_SHA256
|
|
save(EXP / 'protocol/preregistered.json', dict(timestamp_utc=datetime.now(timezone.utc).isoformat(),
|
|
sources=sources(), inputs=[dict(path=str(p.relative_to(ROOT)), sha256=digest(p)) for p in paths],
|
|
parent_commit=subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], text=True).strip(),
|
|
scope='Post-outcome conditional shadow accounting; no new brain simulation, no heldout, no tuning/training',
|
|
before_accounting=True, prior_CAL_result_known=True, output_budget_bytes=2 * 1024 ** 3,
|
|
CPU_only=True, max_threads=4, python=sys.version, numpy=np.__version__, pandas=pd.__version__,
|
|
machine=platform.machine(), longdouble_mantissa_bits=int(np.finfo(np.longdouble).nmant)))
|
|
save(EXP / 'regression/old-integrity-before.json', integrity())
|
|
meta = pd.read_parquet(META).sort_values('neuron_idx').reset_index(drop=True)
|
|
assert len(meta) == 166700 and np.array_equal(meta.neuron_idx, np.arange(166700))
|
|
labels, report = classify(meta)
|
|
assert report['pass_all'], 'Ambiguous/conflicting source identities: audit required'
|
|
for label, file in (('KC', 'kc_globals.npy'), ('APL', 'apl_globals.npy'), ('ALPN', 'pn_globals.npy')):
|
|
assert np.array_equal(np.flatnonzero(labels == label), np.load(AG / 'apl-bundle' / file))
|
|
save(EXP / 'protocol/population-audit.json', report)
|
|
save(EXP / 'protocol/metadata-schema.json', {c: dict(dtype=str(meta[c].dtype),
|
|
nulls=int(meta[c].isna().sum())) for c in meta.columns})
|
|
meta[['neuron_idx', 'body_id', 'annotation_class', 'annotation_type', 'neurotransmitter']].assign(
|
|
audit_source_group=labels).to_parquet(EXP / 'protocol/neuron-source-groups.parquet', index=False)
|
|
inventory = [dict(case=name, repeat=0, kind='fixture') for name in read(AG_MODEL / 'config.json')['fixtures']]
|
|
for row in read(AG / 'protocol/calibration-cases.json'):
|
|
inventory.append(dict(case=row['name'], repeat=0, kind='calibration'))
|
|
if row['seed'] == 51101:
|
|
inventory.append(dict(case=row['name'], repeat=1, kind='calibration'))
|
|
assert len(inventory) == 80 and len({(r['case'], r['repeat']) for r in inventory}) == 80
|
|
for row in inventory:
|
|
source = AG / 'gpu' / (row['case'] + '-r' + str(row['repeat']))
|
|
assert read(source / 'summary.json')['complete']
|
|
row['source'] = str(source.relative_to(ROOT))
|
|
save(EXP / 'protocol/inventory.json', inventory)
|
|
save(EXP / 'protocol/prepared-hashes.json', [dict(path=str(p.relative_to(EXP)), sha256=digest(p))
|
|
for p in sorted((EXP / 'protocol').iterdir()) if p.is_file() and p.name != 'prepared-hashes.json'])
|
|
print('REGISTERED 8 fixtures + 60 CAL + 12 repeats; CPU-only existing-trajectory accounting', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|