Digital Drosophila Archive
122 lines
7.0 KiB
Python
122 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch DOI metadata from Crossref into new files; no paper full-text copying."""
|
|
import argparse
|
|
import concurrent.futures as cf
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import urllib.request
|
|
import urllib.parse
|
|
|
|
CORE = {
|
|
"10.1016/j.cell.2026.08.015": "Formal MaleCNS Cell 2026; distinguish release/schema from publication counts",
|
|
"10.1016/j.cell.2026.08.014": "Official MaleCNS visual pathways",
|
|
"10.1038/s41586-024-07763-9": "Shiu-like LIF reference; not a strict replication claim",
|
|
"10.1038/s41586-024-07939-3": "FlyVis independent visual model",
|
|
"10.1038/srep21841": "DoOR physiology / odor-receptor-glomerular reference",
|
|
"10.1016/j.cell.2024.03.016": "EM neurotransmitter prediction and limitations",
|
|
"10.7554/eLife.03868": "DPM physiology; not a universal edge sign assignment",
|
|
"10.7554/eLife.56954": "Localized APL inhibition",
|
|
"10.7554/eLife.74172": "APL / PN-bouton and KC-dendrite normalization",
|
|
"10.1073/pnas.0914912107": "MB to antennal-lobe feedback physiology",
|
|
"10.1016/j.cell.2006.12.034": "Excitatory antennal-lobe local circuits",
|
|
"10.1016/j.neuron.2010.08.025": "Excitatory local interneuron connectivity",
|
|
"10.1016/j.cell.2019.05.040": "Timing-dependent dopamine plasticity",
|
|
"10.1038/s41467-023-38487-5": "KC subthreshold response and decision timing",
|
|
"10.1038/s41586-025-09029-4": "FlyBody / Google DeepMind; separate physical body/controller research",
|
|
"10.7554/eLife.76712": "KC acetylcholine release plasticity",
|
|
"10.1523/ENEURO.0275-23.2023": "Compartment-specific KC release and bouton heterogeneity",
|
|
"10.1016/j.cell.2015.11.019": "Compartment and timing dependence of transmission",
|
|
"10.7554/eLife.80445": "Postsynaptic cholinergic memory mechanisms",
|
|
"10.1523/JNEUROSCI.1420-19.2019": "Pre/post synaptic odor suppression",
|
|
}
|
|
|
|
def normalize(s):
|
|
# Only known extraction artifacts; do not invent or fuzzy-match DOI.
|
|
return s.rstrip(".,;:)]}").removesuffix("/full").lower()
|
|
|
|
def json_new(path, obj):
|
|
with path.open("x", encoding="utf-8", newline="\n") as f:
|
|
json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True)
|
|
f.write("\n")
|
|
|
|
def escape(s):
|
|
return str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%").replace("_", "\\_")
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--catalog", type=Path, required=True)
|
|
ap.add_argument("--output", type=Path, required=True)
|
|
args = ap.parse_args()
|
|
out = args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
for filename in ("papers.json", "papers.bib", "papers.md", "crossref"):
|
|
if (out / filename).exists():
|
|
raise RuntimeError("Refusing to overwrite " + filename)
|
|
(out / "crossref").mkdir()
|
|
items = {normalize(d): {"doi":d, "use":u, "source_files":[], "extracted_forms":[]} for d,u in CORE.items()}
|
|
for r in json.loads(args.catalog.read_text(encoding="utf-8")):
|
|
for d in r["doi_candidates"]:
|
|
key = normalize(d)
|
|
row = items.setdefault(key, {"doi":key, "use":"Historical citation; consult source file for context", "source_files":[], "extracted_forms":[]})
|
|
row["source_files"].append(r["source_path"])
|
|
row["extracted_forms"].append(d)
|
|
def fetch(pair):
|
|
key, row = pair
|
|
row["source_files"] = sorted(set(row["source_files"]))
|
|
row["extracted_forms"] = sorted(set(row["extracted_forms"]))
|
|
url = "https://api.crossref.org/works/" + urllib.parse.quote(row["doi"], safe="")
|
|
row.update(metadata_url=url, checked_utc=datetime.now(timezone.utc).isoformat(), fulltext_archived=False)
|
|
last = None
|
|
for attempt in range(3):
|
|
try:
|
|
request = urllib.request.Request(url, headers={"User-Agent":"DigitalDrosophilaBibliography/1.0 (research archive)", "Accept":"application/json"})
|
|
with urllib.request.urlopen(request, timeout=45) as r:
|
|
raw = r.read()
|
|
result = json.loads(raw)
|
|
m = result["message"]
|
|
if normalize(m["DOI"]) != key:
|
|
raise ValueError("Returned DOI differs")
|
|
token = hashlib.sha256(key.encode()).hexdigest()[:16]
|
|
with (out / "crossref" / (token + ".json")).open("xb") as f:
|
|
f.write(raw)
|
|
row.update(status="VERIFIED-METADATA", title="; ".join(m.get("title", [])), authors=m.get("author", []),
|
|
journal="; ".join(m.get("container-title", [])), published=m.get("published", {}),
|
|
publisher=m.get("publisher"), volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"),
|
|
doi_url=m.get("URL", "https://doi.org/" + row["doi"]),
|
|
publisher_fulltext_links=m.get("link", []), license_metadata=m.get("license", []),
|
|
metadata_file="crossref/"+token+".json", metadata_sha256=hashlib.sha256(raw).hexdigest())
|
|
print("metadata OK", row["doi"], flush=True)
|
|
return row
|
|
except Exception as e:
|
|
last = type(e).__name__ + ": " + str(e)
|
|
if attempt < 2:
|
|
time.sleep(2 * (attempt+1))
|
|
row.update(status="UNRESOLVED", error=last, doi_url="https://doi.org/"+row["doi"])
|
|
print("metadata UNRESOLVED", row["doi"], flush=True)
|
|
return row
|
|
with cf.ThreadPoolExecutor(max_workers=4) as ex:
|
|
rows = list(ex.map(fetch, sorted(items.items())))
|
|
json_new(out / "papers.json", {"schema_version":1, "metadata_provider":"Crossref / publisher-deposited", "papers":rows})
|
|
with (out / "papers.bib").open("x", encoding="utf-8") as bib, (out / "papers.md").open("x", encoding="utf-8") as md:
|
|
md.write("# 论文索引\n\n出版元数据由 Crossref 获取；未打包论文全文。许可证据见 JSON，不能默认再分发。\n\n")
|
|
for i,r in enumerate(rows,1):
|
|
if r["status"] != "VERIFIED-METADATA":
|
|
md.write(f"- UNRESOLVED [{r['doi']}]({r['doi_url']})；保留原位引用，未补造标题。\n")
|
|
continue
|
|
year = r["published"].get("date-parts", [[None]])[0][0]
|
|
authors = " and ".join(a.get("family",a.get("name","")) + (", " + a["given"] if a.get("given") else "") for a in r["authors"])
|
|
bib.write(f"@article{{malecns_ref_{i:03d},\n")
|
|
for k,v in {"title":r["title"], "author":authors, "year":year, "journal":r["journal"], "doi":r["doi"], "url":r["doi_url"], "volume":r["volume"], "pages":r["pages"]}.items():
|
|
if v is not None:
|
|
bib.write(f" {k} = {{{escape(v)}}},\n")
|
|
bib.write("}\n\n")
|
|
md.write(f"- **{year}** [{r['title']}]({r['doi_url']}) — {r['journal']}。用途：{r['use']}。\n")
|
|
print(json.dumps({"papers":len(rows), "verified":sum(r['status']=='VERIFIED-METADATA' for r in rows), "unresolved":sum(r['status']=='UNRESOLVED' for r in rows)}))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|