Digital Drosophila Archive
89 lines
3.2 KiB
Python
89 lines
3.2 KiB
Python
"""Small exclusive file helpers. No neural imports, mutation or retry policy."""
|
|
from __future__ import annotations
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
ROOT = Path('/data/digital-drosophila')
|
|
MODEL = Path(__file__).resolve().parent
|
|
LAB = MODEL.parent/'olfactory-conditioning-lab-v1'
|
|
AQ = MODEL.parent/'conditioning-execution-v1'
|
|
AP = MODEL.parent/'conditioning-interface-v2'
|
|
EXP = ROOT/'experiments/phase4ar-normal-conditioning'
|
|
GPU = 'GPU-79f3a85d-adce-7833-7003-6488140ca596'
|
|
AP_SHA = 'e47651a8ea99d7c44115107ff6b5dc71312183609ae11c891db65bb21ed61e30'
|
|
|
|
def require(ok, message):
|
|
if not ok: raise ValueError(message)
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as f:
|
|
for data in iter(lambda: f.read(8 << 20), b''): h.update(data)
|
|
return h.hexdigest()
|
|
|
|
def canonical(value):
|
|
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False)
|
|
|
|
def content_sha(value):
|
|
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
|
|
|
def read(path):
|
|
def pairs(items):
|
|
result = {}
|
|
for key, value in items:
|
|
require(key not in result, 'Duplicate JSON key'); result[key] = value
|
|
return result
|
|
return json.loads(Path(path).read_text(encoding='utf-8'), object_pairs_hook=pairs,
|
|
parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite JSON')))
|
|
|
|
def save(path, value):
|
|
with unlinked(path).open('x', encoding='utf-8', newline='\n') as f:
|
|
json.dump(value, f, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False); f.write('\n')
|
|
|
|
def unlinked(path):
|
|
p = Path(path)
|
|
require(p.is_absolute() and '..' not in p.parts and not any(x.is_symlink() or
|
|
(hasattr(x, 'is_junction') and x.is_junction()) for x in (p, *p.parents)), 'Unsafe linked path')
|
|
return p
|
|
|
|
def identity(path):
|
|
p = unlinked(path); require(p.is_file(), 'Missing regular file: '+str(p))
|
|
return dict(sha256=digest(p), bytes=p.stat().st_size)
|
|
|
|
def pin(path):
|
|
return dict(path=str(unlinked(path)), sha256=digest(path))
|
|
|
|
def verify(pins):
|
|
for p, sha in pins.items(): require(identity(p)['sha256'] == sha, 'Pinned input changed: '+str(p))
|
|
|
|
def inventory(root):
|
|
result = {}
|
|
for p in sorted(unlinked(root).rglob('*')):
|
|
unlinked(p)
|
|
if p.is_file(): result[str(p)] = identity(p)
|
|
else: require(p.is_dir(), 'Nonregular artifact')
|
|
return result
|
|
|
|
def load_source(name, path, sha):
|
|
path = unlinked(path); raw = path.read_bytes()
|
|
require(hashlib.sha256(raw).hexdigest() == sha and name not in sys.modules, 'Source/module identity differs')
|
|
spec = importlib.util.spec_from_file_location(name, path); module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
exec(compile(raw, str(path), 'exec'), module.__dict__)
|
|
return module
|
|
|
|
def npz(path):
|
|
import numpy as np
|
|
with np.load(path, allow_pickle=False) as f: return {k: f[k].copy() for k in f.files}
|
|
|
|
def save_npz(path, values):
|
|
import numpy as np
|
|
with unlinked(path).open('xb') as f: np.savez_compressed(f, **values)
|
|
|
|
def task_key(task):
|
|
# IDs remain verbatim in receipts; filesystem IDs never incorporate slashes.
|
|
return 'task-'+hashlib.sha256(task['task_id'].encode()).hexdigest()
|