Digital Drosophila Archive
70 lines
4.2 KiB
Python
70 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Instrumented 100 ms canonical-backend cost audit."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import resource
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import torch
|
|
import yaml
|
|
|
|
import phase4p
|
|
|
|
|
|
def main() -> None:
|
|
model = Path("/data/digital-drosophila/models/male-cns-lif/deterministic-runtime-v2")
|
|
cfg = yaml.safe_load((model / "phase4p.yaml").read_text()); exp = Path(cfg["paths"]["experiment_root"])
|
|
rt = phase4p.CanonicalRuntime(cfg, model, exp); spec = cfg["reference_trials"][1]
|
|
target_np = np.flatnonzero(rt.pops[spec["population"]]).astype(np.int64)
|
|
targets = torch.as_tensor(target_np, dtype=torch.int64, device=rt.device); steps = 1000
|
|
forced = phase4p.phase4c.generate_forced(steps, len(target_np), float(spec["rate_hz"]), rt.dt, int(spec["seed"]), rt.device)
|
|
rt.reset(); samples = []; stop = threading.Event()
|
|
def sample():
|
|
while not stop.is_set():
|
|
try:
|
|
text = subprocess.check_output(["nvidia-smi", "--id=" + cfg["device"]["visible_gpu_uuid"],
|
|
"--query-gpu=utilization.gpu,memory.used", "--format=csv,noheader,nounits"], text=True).strip()
|
|
u, m = text.split(","); samples.append({"time": time.time(), "gpu_util_percent": float(u), "gpu_memory_mib": float(m)})
|
|
except Exception as e: samples.append({"error": str(e)})
|
|
stop.wait(0.10)
|
|
thread = threading.Thread(target=sample, daemon=True); thread.start()
|
|
use0 = resource.getrusage(resource.RUSAGE_SELF); cpu0 = use0.ru_utime + use0.ru_stime
|
|
torch.cuda.reset_peak_memory_stats(); torch.cuda.synchronize()
|
|
wall0 = time.perf_counter(); state_submit = force_submit = sort_submit = reduce_submit = 0.0; contributions = 0
|
|
for step in range(steps):
|
|
slot = step % rt.qlen; t = time.perf_counter()
|
|
rt.ext.state_step(rt.v, rt.g, rt.ref, rt.queue[slot], rt.spike_u8, rt.counts,
|
|
float(rt.gd), float(rt.alpha), float(rt.rest), float(rt.resetv), float(rt.threshold), rt.ref_steps)
|
|
state_submit += time.perf_counter() - t; t = time.perf_counter()
|
|
rt.ext.force_step(rt.v, rt.ref, rt.spike_u8, rt.counts, targets, forced[step], float(rt.resetv), rt.ref_steps)
|
|
force_submit += time.perf_counter() - t
|
|
sources = torch.nonzero(rt.spike_u8, as_tuple=False).flatten()
|
|
st = rt.canonical_fanout(sources, rt.queue[(step + rt.delay_steps) % rt.qlen])
|
|
sort_submit += st["sort_seconds"]; reduce_submit += st["reduce_seconds"]; contributions += st["contributions"]
|
|
torch.cuda.synchronize(); wall = time.perf_counter() - wall0
|
|
use1 = resource.getrusage(resource.RUSAGE_SELF); cpu1 = use1.ru_utime + use1.ru_stime
|
|
stop.set(); thread.join()
|
|
good = [x for x in samples if "gpu_util_percent" in x]
|
|
out = {"trial": "auditory-100hz", "biological_ms": 100.0, "steps": steps, "wall_seconds": wall,
|
|
"wall_per_100ms_biological": wall, "realtime_factor": wall / 0.1, "steps_per_second": steps / wall,
|
|
"process_CPU_percent_of_one_core": 100 * (cpu1 - cpu0) / wall,
|
|
"GPU_utilization_percent_mean": float(np.mean([x["gpu_util_percent"] for x in good])) if good else None,
|
|
"GPU_utilization_percent_max": float(np.max([x["gpu_util_percent"] for x in good])) if good else None,
|
|
"nvidia_smi_memory_mib_max": float(np.max([x["gpu_memory_mib"] for x in good])) if good else None,
|
|
"torch_peak_VRAM_bytes": int(torch.cuda.max_memory_allocated()), "contribution_count": int(contributions),
|
|
"CPU_submit_seconds": {"state_integration": state_submit, "external_force": force_submit,
|
|
"canonical_sort_and_segment_setup": sort_submit, "segmented_reduction": reduce_submit},
|
|
"sampling_interval_seconds": 0.10, "samples": samples,
|
|
"interpretation": "Submission sub-times overlap asynchronous CUDA work; synchronized total wall time is authoritative."}
|
|
p = exp / "performance/instrumented-100ms.json"; p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(json.dumps(out, indent=2, sort_keys=True) + "\n"); print(json.dumps(out, indent=2), flush=True)
|
|
|
|
|
|
if __name__ == "__main__": main()
|