Audit APL PN scope and localize official synaptic contacts without changing frozen dynamics

2026-09-29 23:58:05 +08:00
parent 3981628ff8
commit d6843e675d
62 changed files with 3974 additions and 0 deletions
@@ -0,0 +1,49 @@
+"""User-authorized official additional reference download. Never writes raw."""
+import base64
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import time
+import urllib.parse
+import urllib.request
+from audit import save
+
+ROOT=Path('/data/digital-drosophila')
+OUT=ROOT/'experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners'
+NAME='syn-partners-male-cns-v1.0-minconf-0.5.feather'
+OBJECT='v1.0/connectome-data/flat-connectome/'+NAME
+META='https://storage.googleapis.com/storage/v1/b/flyem-male-cns/o/'+urllib.parse.quote(OBJECT,safe='')
+
+def main():
+    OUT.mkdir(parents=True,exist_ok=True)
+    with urllib.request.urlopen(META,timeout=30) as r:meta=json.load(r)
+    assert meta['name']==OBJECT and int(meta['size'])==6777179098
+    assert meta['generation']=='1780494942562468','Stop on an unexpected upstream revision'
+    md5=base64.b64decode(meta['md5Hash']).hex()
+    if (OUT/'upstream-object.json').exists():
+        prior=json.loads((OUT/'upstream-object.json').read_text())
+        assert all(prior[k]==meta[k] for k in ('name','generation','size','md5Hash'))
+    else:save(OUT/'upstream-object.json',meta)
+    url='https://storage.googleapis.com/flyem-male-cns/'+OBJECT+'?generation='+meta['generation']
+    if not (OUT/'authorization-and-source.json').exists():
+        save(OUT/'authorization-and-source.json',{'user_authorized_additional_download':True,'new_reference_only':True,
+             'source_page':'https://male-cns.janelia.org/download/','url':url,'metadata_url':META,
+             'dataset':'male-cns:v1.0','license':'CC-BY-4.0','EM_downloaded':False,
+             'raw_modified':False,'retrieved_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())})
+    cmd=['aria2c','--continue=true','--max-connection-per-server=8','--split=8','--min-split-size=16M',
+         '--max-tries=8','--retry-wait=5','--connect-timeout=30','--timeout=60','--auto-file-renaming=false',
+         '--allow-overwrite=false','--check-integrity=true','--file-allocation=none','--summary-interval=20',
+         '--enable-color=false','--console-log-level=warn','--checksum=md5='+md5,'--dir='+str(OUT),'--out='+NAME,url]
+    begin=time.perf_counter();subprocess.run(cmd,check=True)
+    file=OUT/NAME;hs=hashlib.sha256();hm=hashlib.md5()
+    with file.open('rb') as f:
+        for b in iter(lambda:f.read(8<<20),b''):hs.update(b);hm.update(b)
+    result={'bytes':file.stat().st_size,'sha256':hs.hexdigest(),'md5':hm.hexdigest(),'official_md5':md5,
+            'generation':meta['generation'],'download_and_verification_seconds':time.perf_counter()-begin,
+            'pass':file.stat().st_size==int(meta['size']) and hm.hexdigest()==md5,'path':str(file)}
+    assert result['pass'],'Official MD5/size mismatch; retain file and stop'
+    if not (OUT/'download-verification.json').exists():save(OUT/'download-verification.json',result)
+    print(json.dumps(result),flush=True)
+
+if __name__=='__main__':main()