Digital Drosophila Archive
371 lines
22 KiB
Python
371 lines
22 KiB
Python
"""Bounded CPU-only residency experiment; never an AR runtime replacement."""
|
|
from __future__ import annotations
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import selectors
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
ROOT = Path('/data/digital-drosophila')
|
|
LAB = ROOT/'models/male-cns-lif/olfactory-conditioning-lab-v1'
|
|
AQ = ROOT/'experiments/phase4aq-conditioning-execution'
|
|
VALIDATOR = ROOT/'models/male-cns-lif/conditioning-execution-v1/validate_update_cpu.py'
|
|
E_SHA = '757935a4815d9f625640df685184ed3fe44a8845323562796da7aa43cbb194e1'
|
|
V_SHA = '44e95eca9e39e9cd25a1d5d2d46b2cea76e839143865a196f865a61fac700b9f'
|
|
MANIFEST_SHA = '2dc0feca75f718f2ee0d7c9240da503221823573fd25f1344d2fd97d97eb2db7'
|
|
FIXTURES = {
|
|
'M': ('mechanical--fresh-parent', '4616a42c255ab839346368aa0e5e4306bb3878e55fcc95b176f058b1993c9b0c'),
|
|
'N': ('performance--natural-light-fresh', '785e3d26d06b47efe7f054883d38fe2e13711195359bf9f4b03aa88887fbfb37'),
|
|
}
|
|
SEQUENCE = ('M', 'N', 'M', 'N_D0', 'N_frozen', 'M_D0', 'M')
|
|
UPDATE_FILES = ('delta-after.npz', 'overlay-manifest.json', 'update-receipt.json', 'chain-receipt.json')
|
|
TIMEOUT = 180
|
|
|
|
|
|
def require(ok, message):
|
|
if not ok:
|
|
raise ValueError(message)
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(8 << 20), b''):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False)
|
|
|
|
|
|
def read(path):
|
|
return json.loads(Path(path).read_text(encoding='utf-8'))
|
|
|
|
|
|
def save(path, value):
|
|
with Path(path).open('x', encoding='utf-8', newline='\n') as stream:
|
|
json.dump(value, stream, sort_keys=True, indent=2, allow_nan=False)
|
|
stream.write('\n')
|
|
|
|
|
|
def safe(path):
|
|
require('..' not in Path(path).parts, 'Parent traversal forbidden')
|
|
path = Path(path).absolute()
|
|
require(not any(p.is_symlink() for p in (path, *path.parents)), 'Symlink path forbidden')
|
|
return path.resolve(strict=False)
|
|
|
|
|
|
def pinned_load(name, path, sha):
|
|
path = safe(path)
|
|
raw = path.read_bytes()
|
|
require(hashlib.sha256(raw).hexdigest() == sha and name not in sys.modules, 'Source/module identity differs')
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
exec(compile(raw, str(path), 'exec'), module.__dict__)
|
|
return module
|
|
|
|
|
|
def worker(role, mode, plan_file, plan_sha):
|
|
require(sys.dont_write_bytecode and os.environ.get('CUDA_VISIBLE_DEVICES') == '', 'CPU-only -B worker required')
|
|
require(digest(plan_file) == plan_sha, 'External plan SHA mismatch')
|
|
plan = read(plan_file)
|
|
out = safe(plan['output_root'])
|
|
pins = plan['prototype_sources']
|
|
require(all(digest(p) == s for p, s in pins.items()), 'Prototype source changed')
|
|
module = None
|
|
seq = 0
|
|
print(canonical({'ready': True, 'pid': os.getpid(), 'role': role}), flush=True)
|
|
for line in sys.stdin:
|
|
job = json.loads(line)
|
|
require(set(job) == {'sequence', 'authority', 'authority_sha256', 'output', 'update_dir', 'chain_sha256'}, 'Wrong request fields')
|
|
require(seq < len(SEQUENCE) and job['sequence'] == seq and
|
|
job['authority'] in plan['authorities'].values() and
|
|
plan['input_sha256'][job['authority']] == job['authority_sha256'] and
|
|
digest(job['authority']) == job['authority_sha256'], 'Sequence/authority differs')
|
|
output = safe(job['output'])
|
|
require(output.is_relative_to(out) and not output.exists() and output.parent.is_dir(), 'Output must be exclusive and within experiment')
|
|
started = time.perf_counter()
|
|
with contextlib.redirect_stdout(sys.stderr):
|
|
if module is None:
|
|
if role == 'update':
|
|
module = pinned_load('_perf_E', LAB/'conditioning_epoch.py', E_SHA)
|
|
elif role == 'verify-reference':
|
|
module = pinned_load('_perf_original_V', VALIDATOR, V_SHA)
|
|
else:
|
|
source = Path(__file__).parent/'resident_validator.py'
|
|
module = pinned_load('_perf_resident_V', source, pins[str(source)])
|
|
if role == 'update':
|
|
module.postprocess_epoch(job['authority'], job['authority_sha256'], output)
|
|
hashes = {name: digest(output/name) for name in UPDATE_FILES}
|
|
else:
|
|
module.validate(job['authority'], job['authority_sha256'], job['update_dir'], job['chain_sha256'], output)
|
|
hashes = {'receipt': digest(output)}
|
|
import torch
|
|
require(os.environ['CUDA_VISIBLE_DEVICES'] == '' and not torch.cuda.is_initialized(), 'CUDA use forbidden')
|
|
require('runtime_light' not in sys.modules, 'Ordinary runtime module collision')
|
|
require(all(digest(p) == s for p, s in pins.items()), 'Prototype source changed during task')
|
|
print(canonical({'ack': True, 'sequence': seq, 'pid': os.getpid(), 'role': role,
|
|
'request_sha256': hashlib.sha256(canonical(job).encode()).hexdigest(),
|
|
'output_sha256': hashes, 'task_seconds': time.perf_counter()-started,
|
|
'CUDA_initialized': False, 'task_ack_is_not_process_exit': True}), flush=True)
|
|
seq += 1
|
|
if mode == 'once':
|
|
break
|
|
|
|
|
|
class Process:
|
|
def __init__(self, role, mode, plan, root, label, deadline):
|
|
self.role, self.mode, self.seq = role, mode, 0
|
|
self.deadline = deadline
|
|
self.label, self.root, self.records = label, root, []
|
|
self.stderr_path = root/'logs'/(label+'.stderr.log')
|
|
self.stderr = self.stderr_path.open('xb')
|
|
env = dict(os.environ, CUDA_VISIBLE_DEVICES='', PYTHONDONTWRITEBYTECODE='1',
|
|
OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1')
|
|
argv = [sys.executable, '-B', str(Path(__file__).resolve()), 'worker', '--role', role,
|
|
'--mode', mode, '--plan', str(plan), '--plan-sha', digest(plan)]
|
|
self.started = time.perf_counter(); self.argv = argv
|
|
self.selector = selectors.DefaultSelector()
|
|
self.p = None
|
|
try:
|
|
self.remaining()
|
|
self.p = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=self.stderr, text=True, bufsize=1, env=env)
|
|
self.selector.register(self.p.stdout, selectors.EVENT_READ)
|
|
self.ready = self.line()
|
|
require(self.ready == {'ready': True, 'pid': self.p.pid, 'role': role}, 'Wrong worker ready record')
|
|
self.startup_seconds = time.perf_counter()-self.started
|
|
except BaseException:
|
|
self.abort('Failed startup/ready')
|
|
raise
|
|
|
|
def remaining(self):
|
|
remaining = self.deadline-time.perf_counter()
|
|
require(remaining > 0, 'Probe absolute wall deadline exceeded')
|
|
return min(TIMEOUT, remaining)
|
|
|
|
def abort(self, reason):
|
|
if self.p is not None:
|
|
if self.p.poll() is None:
|
|
self.p.terminate()
|
|
try: self.p.wait(timeout=10)
|
|
except subprocess.TimeoutExpired: self.p.kill(); self.p.wait(timeout=10)
|
|
for stream in (self.p.stdin, self.p.stdout):
|
|
if stream is not None: stream.close()
|
|
path = self.root/'processes'/(self.label+'.aborted.json')
|
|
if not path.exists():
|
|
save(path, dict(pid=self.p.pid, argv=self.argv, actual_parent_observed_exit=self.p.returncode,
|
|
reason=reason, partial_preserved=True, benchmark_failed=True, no_retry=True))
|
|
self.stderr.close(); self.selector.close()
|
|
|
|
def line(self):
|
|
require(self.selector.select(self.remaining()), 'Worker response timeout; no retry')
|
|
raw = self.p.stdout.readline()
|
|
require(raw, 'Worker EOF before expected response; inspect stderr')
|
|
return json.loads(raw)
|
|
|
|
def request(self, authority, output, update_dir=None):
|
|
request_started = time.perf_counter()
|
|
job = dict(sequence=self.seq, authority=str(authority), authority_sha256=digest(authority),
|
|
output=str(output), update_dir=None if update_dir is None else str(update_dir),
|
|
chain_sha256=None if update_dir is None else digest(update_dir/'chain-receipt.json'))
|
|
save(self.root/'requests'/(self.label+f'-{self.seq:02d}.json'), job)
|
|
started = time.perf_counter()
|
|
self.p.stdin.write(canonical(job)+'\n'); self.p.stdin.flush()
|
|
ack = self.line()
|
|
require(ack.get('ack') is True and ack['sequence'] == self.seq and ack['pid'] == self.p.pid and
|
|
ack['role'] == self.role and ack['request_sha256'] == hashlib.sha256(canonical(job).encode()).hexdigest() and
|
|
ack['CUDA_initialized'] is False and ack['task_ack_is_not_process_exit'] is True, 'Worker acknowledgment differs')
|
|
hashes = ({name: digest(output/name) for name in UPDATE_FILES} if self.role == 'update' else {'receipt': digest(output)})
|
|
require(hashes == ack['output_sha256'], 'Worker output changed or acknowledgment differs')
|
|
record = dict(ack=ack, rpc_parent_seconds=time.perf_counter()-started)
|
|
save(self.root/'responses'/(self.label+f'-{self.seq:02d}.json'), record)
|
|
record['parent_request_seconds'] = time.perf_counter()-request_started
|
|
self.records.append(record); self.seq += 1
|
|
return record
|
|
|
|
def close(self):
|
|
self.p.stdin.close()
|
|
rc = self.p.wait(timeout=self.remaining())
|
|
elapsed = time.perf_counter()-self.started
|
|
trailing = self.p.stdout.read()
|
|
self.p.stdout.close(); self.stderr.close(); self.selector.close()
|
|
receipt = dict(pid=self.p.pid, argv=self.argv, role=self.role, mode=self.mode,
|
|
actual_parent_observed_exit=rc, tasks_acknowledged=self.seq, startup_seconds=self.startup_seconds,
|
|
lifetime_parent_seconds=elapsed, stderr_sha256=digest(self.stderr_path),
|
|
task_ACKs_are_not_individual_OS_exits=True)
|
|
save(self.root/'processes'/(self.label+'.json'), receipt)
|
|
require(rc == 0 and not trailing, 'Worker actual nonzero exit or unsolicited output')
|
|
return receipt
|
|
|
|
|
|
def compare_updates(a, b):
|
|
import numpy as np
|
|
require(set(x.name for x in a.iterdir()) == set(UPDATE_FILES) == set(x.name for x in b.iterdir()), 'Wrong update inventory')
|
|
for name in UPDATE_FILES:
|
|
require(digest(a/name) == digest(b/name), 'Exact output file differs: '+name)
|
|
with np.load(a/'delta-after.npz', allow_pickle=False) as x, np.load(b/'delta-after.npz', allow_pickle=False) as y:
|
|
require(x.files == y.files, 'Array schema differs')
|
|
for key in x.files:
|
|
require(x[key].dtype == y[key].dtype and x[key].shape == y[key].shape and x[key].tobytes() == y[key].tobytes(), 'Raw array differs')
|
|
return {'all_four_files_SHA_identical': True, 'arrays_bitwise_identical': True,
|
|
'full_update_receipt_and_trace_digests_identical': True}
|
|
|
|
|
|
def compare_validation(reference, resident):
|
|
a, b = read(reference), read(resident)
|
|
require(a.pop('schema') == 'conditioning-update-cpu-validation-v1', 'Wrong original verifier schema')
|
|
require(b.pop('schema') == 'conditioning-resident-cpu-validation-prototype-v1', 'Wrong prototype verifier schema')
|
|
original = a.pop('validator_source')
|
|
require(original == {'path': str(VALIDATOR), 'sha256': V_SHA}, 'Wrong original validator identity')
|
|
require(b.pop('original_validator_source') == original and b.pop('resident_prototype') is True, 'Missing prototype original-source provenance')
|
|
require(b.pop('resident_cache_scope') == 'SHA-verified imported modules only; no input/hash/state/delta/checkpoint caching', 'Wrong cache scope')
|
|
identity = b.pop('validator_source')
|
|
require(identity == {'path': str(Path(__file__).resolve().parent/'resident_validator.py'),
|
|
'sha256': digest(Path(__file__).parent/'resident_validator.py')}, 'Wrong resident source identity')
|
|
require(a == b, 'Scientific verifier receipt differs beyond enumerated provenance fields')
|
|
return {'all_scientific_fields_identical': True, 'explicit_differing_fields':
|
|
['schema', 'validator_source', 'original_validator_source', 'resident_prototype', 'resident_cache_scope']}
|
|
|
|
|
|
def prepare(root):
|
|
require(sys.dont_write_bytecode and root.is_dir() and not list(root.iterdir()), 'New empty output root and -B required')
|
|
require(not root.is_relative_to(ROOT/'experiments') and not root.is_relative_to(ROOT/'models'), 'Prototype may not write experimental/model trees')
|
|
require(shutil.disk_usage(root).free > (10 << 30), 'Insufficient temporary free space')
|
|
for name in ('inputs', 'updates', 'validation', 'logs', 'requests', 'responses', 'processes'):
|
|
(root/name).mkdir()
|
|
require(digest(AQ/'manifest.json') == MANIFEST_SHA, 'AQ sealed manifest differs')
|
|
manifest = {x['path']: x for x in read(AQ/'manifest.json')['files']}
|
|
pins = {str(AQ/'manifest.json'): MANIFEST_SHA, str(LAB/'conditioning_epoch.py'): E_SHA, str(VALIDATOR): V_SHA}
|
|
authorities = {}
|
|
for label, (name, expected) in FIXTURES.items():
|
|
authority = AQ/'authorities'/(name+'.epoch.json')
|
|
require(manifest[authority.relative_to(AQ).as_posix()]['sha256'] == expected and digest(authority) == expected, 'AQ fixture authority seal differs')
|
|
authorities[label] = authority
|
|
a = read(authority); pins[str(authority)] = expected
|
|
pins.update(a['frozen_inputs'])
|
|
pins.update({x['path']: x['sha256'] for x in a['files'].values()})
|
|
checkpoint = Path(a['checkpoint_dir']); cp = checkpoint/'checkpoint-manifest.json'
|
|
require(manifest[cp.relative_to(AQ).as_posix()]['sha256'] == a['checkpoint_manifest_sha256'], 'Checkpoint not in fixed AQ seal')
|
|
pins[str(cp)] = a['checkpoint_manifest_sha256']
|
|
for f in checkpoint.iterdir():
|
|
require(f.is_file() and not f.is_symlink(), 'Unexpected checkpoint entry')
|
|
pins[str(f)] = manifest[f.relative_to(AQ).as_posix()]['sha256']
|
|
for filename in UPDATE_FILES:
|
|
f = AQ/'updates'/name/filename
|
|
pins[str(f)] = manifest[f.relative_to(AQ).as_posix()]['sha256']
|
|
for label in ('N_D0', 'N_frozen', 'M_D0'):
|
|
a = read(authorities[label[0]])
|
|
a['epoch_id'] = 'engineering-residency-control-'+label
|
|
if label.endswith('_D0'):
|
|
a['command']['d_command'] = 0.0
|
|
else:
|
|
a['command']['frozen'] = True
|
|
path = root/'inputs'/(label+'.authority.json'); save(path, a)
|
|
authorities[label] = path; pins[str(path)] = digest(path)
|
|
require(all(digest(p) == sha for p, sha in pins.items()), 'Frozen fixture input bytes differ')
|
|
sources = {str(p.resolve()): digest(p) for p in Path(__file__).parent.glob('*.py')}
|
|
plan = dict(schema='conditioning-cpu-residency-probe-v1', output_root=str(root), sequence=list(SEQUENCE),
|
|
authorities={k: str(v) for k, v in authorities.items()}, input_sha256=pins, prototype_sources=sources,
|
|
fixture_selection='Two predeclared complete frozen AQ engineering fixtures; never AR interim results',
|
|
derived_controls='Only D=0/frozen command variants over fixed sealed events; not new brain runs',
|
|
threads=1, CPU_only=True, no_brain_execution=True, no_weight_installation=True,
|
|
current_AR_untouched=True, no_automatic_retry=True, max_wall_seconds=900,
|
|
benchmark_scope='CPU epoch postprocessing and verification only; excludes AP wrapper/ledger and all GPU/brain work',
|
|
acceptance='Exact four update files, bitwise delta, full trace/receipt identity, fresh original verifier exit0, separate resident verifier identity')
|
|
save(root/'benchmark-plan.json', plan)
|
|
return plan
|
|
|
|
|
|
def run(root):
|
|
require(root.is_dir() and not list(root.iterdir()), 'New empty output directory required')
|
|
started = time.perf_counter(); deadline = started+900
|
|
active = []
|
|
try:
|
|
plan = prepare(root); planfile = root/'benchmark-plan.json'
|
|
# Yield to the live experiment; only one worker computes at a time.
|
|
os.nice(19)
|
|
e = Process('update', 'server', planfile, root, 'resident-E', deadline); active.append(e)
|
|
v = Process('verify-resident', 'server', planfile, root, 'resident-V', deadline); active.append(v)
|
|
require(e.p.pid != v.p.pid, 'Independent workers required')
|
|
rows = []
|
|
for number, label in enumerate(SEQUENCE):
|
|
require(time.perf_counter()-started < 900, 'Probe wall cap exceeded')
|
|
authority = Path(plan['authorities'][label]); prefix = f'{number:02d}-{label}'
|
|
refout, resout = (root/'updates'/(prefix+'-'+x) for x in ('fresh', 'resident'))
|
|
reference = Process('update', 'once', planfile, root, prefix+'-fresh-E', deadline); active.append(reference)
|
|
reference.request(authority, refout); ereceipt = reference.close(); active.remove(reference)
|
|
er = e.request(authority, resout)
|
|
equality = compare_updates(refout, resout)
|
|
if label in FIXTURES:
|
|
compare_updates(AQ/'updates'/FIXTURES[label][0], resout)
|
|
freshv, residentv = (root/'validation'/(prefix+'-'+x+'.json') for x in ('fresh', 'resident'))
|
|
validator = Process('verify-reference', 'once', planfile, root, prefix+'-fresh-V', deadline); active.append(validator)
|
|
validator.request(authority, freshv, resout); vreceipt = validator.close(); active.remove(validator)
|
|
vr = v.request(authority, residentv, resout)
|
|
vequality = compare_validation(freshv, residentv)
|
|
if label.endswith('_D0') or label.endswith('_frozen'):
|
|
import numpy as np
|
|
before = Path(read(authority)['files']['overlay_before']['path'])
|
|
with np.load(before, allow_pickle=False) as a, np.load(resout/'delta-after.npz', allow_pickle=False) as b:
|
|
require(a['plastic_delta_mv'].tobytes() == b['plastic_delta_mv'].tobytes(), 'D0/frozen failed bitwise identity')
|
|
row = dict(label=label, position=number, fresh_E_seconds=ereceipt['lifetime_parent_seconds'],
|
|
resident_E_seconds=er['parent_request_seconds'], fresh_V_seconds=vreceipt['lifetime_parent_seconds'],
|
|
resident_V_seconds=vr['parent_request_seconds'], update_comparison=equality, validator_comparison=vequality,
|
|
original_fresh_validator_actual_exit=vreceipt['actual_parent_observed_exit'])
|
|
rows.append(row); save(root/(prefix+'-comparison.json'), row)
|
|
print(canonical({'completed': number+1, 'total': len(SEQUENCE), 'label': label, 'exact_equal': True}), flush=True)
|
|
ep, vp = e.close(), v.close(); active.clear()
|
|
require(all(digest(p) == sha for p, sha in plan['input_sha256'].items()), 'Consumed frozen inputs changed')
|
|
require(all(digest(p) == sha for p, sha in plan['prototype_sources'].items()), 'Prototype changed')
|
|
require(time.perf_counter() < deadline, 'Probe absolute wall deadline exceeded before PASS')
|
|
warm = rows[1:]
|
|
timing = {}
|
|
for field in ('fresh_E_seconds','resident_E_seconds','fresh_V_seconds','resident_V_seconds'):
|
|
timing[field] = dict(all_mean=statistics.mean(r[field] for r in rows),
|
|
warm_mean=statistics.mean(r[field] for r in warm), warm_median=statistics.median(r[field] for r in warm))
|
|
timing['warm_combined_speedup_ratio'] = sum(r['fresh_E_seconds']+r['fresh_V_seconds'] for r in warm)/sum(r['resident_E_seconds']+r['resident_V_seconds'] for r in warm)
|
|
timing['warm_E_only_with_original_fresh_verifier_speedup_ratio'] = sum(r['fresh_E_seconds']+r['fresh_V_seconds'] for r in warm)/sum(r['resident_E_seconds']+r['fresh_V_seconds'] for r in warm)
|
|
report = dict(status='PASS-CPU-RESIDENCY-PROTOTYPE-ONLY', trials=rows, timing=timing,
|
|
resident_process_exits=[ep['actual_parent_observed_exit'], vp['actual_parent_observed_exit']],
|
|
source_and_consumed_input_SHA_unchanged=True, full_history_reaudit=False,
|
|
prototype_plan_sha256=digest(planfile), total_wall_seconds=time.perf_counter()-started,
|
|
timing_boundary='Parent request includes input/chain digest, request+response persistence and output verification. Fresh also includes spawn/ready/exit; resident lifetime/startup separately recorded.',
|
|
AR_modified=False, brain_run=False, GPU_benchmarked=False, static_hash_cache_removed=False,
|
|
learning_test=False, candidate_generated=False, end_to_end_AR_speedup_not_measured=True,
|
|
scope='7 fixed engineering replays; CPU postprocessing only. Not authorized or validated as AR replacement.')
|
|
save(root/'report.json', report)
|
|
files = {str(p.relative_to(root)): {'sha256': digest(p), 'bytes': p.stat().st_size} for p in root.rglob('*') if p.is_file()}
|
|
save(root/'manifest.json', {'files': files, 'manifest_self_excluded': True})
|
|
print(canonical({'report': str(root/'report.json'), 'status': report['status'], 'timing': timing}), flush=True)
|
|
except BaseException as error:
|
|
for process in active:
|
|
process.abort(type(error).__name__+': '+str(error))
|
|
if not (root/'stopped.json').exists():
|
|
save(root/'stopped.json', dict(error_type=type(error).__name__, error=str(error), traceback=traceback.format_exc(),
|
|
partial_preserved=True, no_automatic_retry=True, registered_AR_not_touched=True))
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
sub = parser.add_subparsers(dest='mode', required=True)
|
|
r = sub.add_parser('run'); r.add_argument('--output', required=True)
|
|
w = sub.add_parser('worker'); w.add_argument('--role', choices=('update','verify-reference','verify-resident'), required=True)
|
|
w.add_argument('--mode', dest='worker_mode', choices=('once','server'), required=True)
|
|
w.add_argument('--plan', required=True); w.add_argument('--plan-sha', required=True)
|
|
args = parser.parse_args()
|
|
if args.mode == 'run': run(safe(args.output))
|
|
else: worker(args.role, args.worker_mode, args.plan, args.plan_sha)
|