Digital Drosophila Archive
50 lines
2.9 KiB
Python
50 lines
2.9 KiB
Python
"""User-authorized official additional reference download. Never writes raw."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from audit import save
|
|
|
|
ROOT=Path('/data/digital-drosophila')
|
|
OUT=ROOT/'experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners'
|
|
NAME='syn-partners-male-cns-v1.0-minconf-0.5.feather'
|
|
OBJECT='v1.0/connectome-data/flat-connectome/'+NAME
|
|
META='https://storage.googleapis.com/storage/v1/b/flyem-male-cns/o/'+urllib.parse.quote(OBJECT,safe='')
|
|
|
|
def main():
|
|
OUT.mkdir(parents=True,exist_ok=True)
|
|
with urllib.request.urlopen(META,timeout=30) as r:meta=json.load(r)
|
|
assert meta['name']==OBJECT and int(meta['size'])==6777179098
|
|
assert meta['generation']=='1780494942562468','Stop on an unexpected upstream revision'
|
|
md5=base64.b64decode(meta['md5Hash']).hex()
|
|
if (OUT/'upstream-object.json').exists():
|
|
prior=json.loads((OUT/'upstream-object.json').read_text())
|
|
assert all(prior[k]==meta[k] for k in ('name','generation','size','md5Hash'))
|
|
else:save(OUT/'upstream-object.json',meta)
|
|
url='https://storage.googleapis.com/flyem-male-cns/'+OBJECT+'?generation='+meta['generation']
|
|
if not (OUT/'authorization-and-source.json').exists():
|
|
save(OUT/'authorization-and-source.json',{'user_authorized_additional_download':True,'new_reference_only':True,
|
|
'source_page':'https://male-cns.janelia.org/download/','url':url,'metadata_url':META,
|
|
'dataset':'male-cns:v1.0','license':'CC-BY-4.0','EM_downloaded':False,
|
|
'raw_modified':False,'retrieved_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())})
|
|
cmd=['aria2c','--continue=true','--max-connection-per-server=8','--split=8','--min-split-size=16M',
|
|
'--max-tries=8','--retry-wait=5','--connect-timeout=30','--timeout=60','--auto-file-renaming=false',
|
|
'--allow-overwrite=false','--check-integrity=true','--file-allocation=none','--summary-interval=20',
|
|
'--enable-color=false','--console-log-level=warn','--checksum=md5='+md5,'--dir='+str(OUT),'--out='+NAME,url]
|
|
begin=time.perf_counter();subprocess.run(cmd,check=True)
|
|
file=OUT/NAME;hs=hashlib.sha256();hm=hashlib.md5()
|
|
with file.open('rb') as f:
|
|
for b in iter(lambda:f.read(8<<20),b''):hs.update(b);hm.update(b)
|
|
result={'bytes':file.stat().st_size,'sha256':hs.hexdigest(),'md5':hm.hexdigest(),'official_md5':md5,
|
|
'generation':meta['generation'],'download_and_verification_seconds':time.perf_counter()-begin,
|
|
'pass':file.stat().st_size==int(meta['size']) and hm.hexdigest()==md5,'path':str(file)}
|
|
assert result['pass'],'Official MD5/size mismatch; retain file and stop'
|
|
if not (OUT/'download-verification.json').exists():save(OUT/'download-verification.json',result)
|
|
print(json.dumps(result),flush=True)
|
|
|
|
if __name__=='__main__':main()
|