Profile unchanged original update pipelines with independent full verification

models/male-cns-lif/conditioning-pipeline-acceleration-v1/run_job.py

@@ -6,16 +6,36 @@ import os
 from pathlib import Path
 import re
 import shutil
+import signal
 import subprocess
 import time
 
 ROOT = Path('/home/qishizhe/phase4ar-pipeline-acceleration-20261001')
+
+def output_bytes():
+    total = 0
+    for f in ROOT.rglob('*'):
+        try:
+            if f.is_file() and not f.is_symlink(): total += f.stat().st_size
+        except FileNotFoundError:
+            pass  # Concurrent compiler temporary file removal, not a payload.
+    return total
+
+def stop_group(proc):
+    try: os.killpg(proc.pid, signal.SIGTERM)
+    except ProcessLookupError: pass
+    try: proc.wait(timeout=5)
+    except subprocess.TimeoutExpired: pass
+    try: os.killpg(proc.pid, signal.SIGKILL)
+    except ProcessLookupError: pass
+    return proc.wait()
+
 p = argparse.ArgumentParser()
 p.add_argument('--label', required=True)
 p.add_argument('--timeout', type=int, default=900)
 p.add_argument('command', nargs=argparse.REMAINDER)
 a = p.parse_args()
-assert re.fullmatch(r'[a-z0-9-]+', a.label) and a.command and 0 < a.timeout <= 1800
+assert re.fullmatch(r'[a-z0-9-]+', a.label) and a.command and 0 < a.timeout <= 900
 assert ROOT.is_dir() and not ROOT.is_symlink()
 assert shutil.disk_usage(ROOT).free >= 100 * (1 << 30)
 out = ROOT / 'jobs'
@@ -25,15 +45,36 @@ record = dict(label=a.label, command=a.command, scope='independent engineering r
               started_utc=datetime.now(timezone.utc).isoformat(), scientific_AR_resumed=False)
 with (out / (a.label + '.start.json')).open('x') as f: json.dump(record, f, indent=2)
 with (out / (a.label + '.stdout.log')).open('x') as stdout, (out / (a.label + '.stderr.log')).open('x') as stderr:
+    proc = None
     try:
-        proc = subprocess.run(a.command, stdout=stdout, stderr=stderr, timeout=a.timeout,
-                              env={**os.environ, 'PYTHONDONTWRITEBYTECODE':'1'})
-        record.update(exit_code=proc.returncode, observed_exit=True, timed_out=False)
-    except subprocess.TimeoutExpired:
-        record.update(exit_code=None, observed_exit=False, timed_out=True)
+        # All children belong to a dedicated group created by THIS invocation.
+        # On stop, terminate this group only, never unrelated services/science.
+        proc = subprocess.Popen(a.command, stdout=stdout, stderr=stderr, start_new_session=True,
+                                env={**os.environ, 'PYTHONDONTWRITEBYTECODE':'1'})
+        record.update(pid=proc.pid, process_group=proc.pid)
+        stop_reason = None
+        while True:
+            try:
+                proc.wait(timeout=5)
+                break
+            except subprocess.TimeoutExpired:
+                if time.perf_counter() - start > a.timeout: stop_reason = 'wall_timeout'
+                elif shutil.disk_usage(ROOT).free < 100 * (1 << 30): stop_reason = 'minimum_disk_reserve'
+                elif output_bytes() > 2 * (1 << 30):
+                    stop_reason = 'output_cap'
+                if stop_reason:
+                    break
+        if not stop_reason and output_bytes() > 2 * (1 << 30): stop_reason = 'output_cap'
+        if stop_reason or proc.returncode != 0: stop_group(proc)
+        record.update(exit_code=proc.returncode, observed_exit=True,
+                      timed_out=stop_reason == 'wall_timeout', stop_reason=stop_reason)
     except OSError as error:
-        record.update(exit_code=None, observed_exit=False, timed_out=False, launch_error=str(error))
+        rc = None if proc is None else stop_group(proc)
+        record.update(exit_code=rc, observed_exit=proc is not None, timed_out=False, launch_error=str(error))
+    except BaseException:
+        if proc is not None: stop_group(proc)
+        raise
 record.update(wall_seconds=time.perf_counter()-start, finished_utc=datetime.now(timezone.utc).isoformat())
 with (out / (a.label + '.exit.json')).open('x') as f: json.dump(record, f, indent=2)
 print(json.dumps(record), flush=True)
-raise SystemExit(record['exit_code'] if record['observed_exit'] else 124)
+raise SystemExit(record['exit_code'] if record['observed_exit'] and not record.get('stop_reason') else 124)