#!/usr/bin/env python3 """Fetch DOI metadata from Crossref into new files; no paper full-text copying.""" import argparse import concurrent.futures as cf from datetime import datetime, timezone import hashlib import json from pathlib import Path import re import time import urllib.request import urllib.parse CORE = { "10.1016/j.cell.2026.08.015": "Formal MaleCNS Cell 2026; distinguish release/schema from publication counts", "10.1016/j.cell.2026.08.014": "Official MaleCNS visual pathways", "10.1038/s41586-024-07763-9": "Shiu-like LIF reference; not a strict replication claim", "10.1038/s41586-024-07939-3": "FlyVis independent visual model", "10.1038/srep21841": "DoOR physiology / odor-receptor-glomerular reference", "10.1016/j.cell.2024.03.016": "EM neurotransmitter prediction and limitations", "10.7554/eLife.03868": "DPM physiology; not a universal edge sign assignment", "10.7554/eLife.56954": "Localized APL inhibition", "10.7554/eLife.74172": "APL / PN-bouton and KC-dendrite normalization", "10.1073/pnas.0914912107": "MB to antennal-lobe feedback physiology", "10.1016/j.cell.2006.12.034": "Excitatory antennal-lobe local circuits", "10.1016/j.neuron.2010.08.025": "Excitatory local interneuron connectivity", "10.1016/j.cell.2019.05.040": "Timing-dependent dopamine plasticity", "10.1038/s41467-023-38487-5": "KC subthreshold response and decision timing", "10.1038/s41586-025-09029-4": "FlyBody / Google DeepMind; separate physical body/controller research", "10.7554/eLife.76712": "KC acetylcholine release plasticity", "10.1523/ENEURO.0275-23.2023": "Compartment-specific KC release and bouton heterogeneity", "10.1016/j.cell.2015.11.019": "Compartment and timing dependence of transmission", "10.7554/eLife.80445": "Postsynaptic cholinergic memory mechanisms", "10.1523/JNEUROSCI.1420-19.2019": "Pre/post synaptic odor suppression", } def normalize(s): # Only known extraction artifacts; do not invent or fuzzy-match DOI. return s.rstrip(".,;:)]}").removesuffix("/full").lower() def json_new(path, obj): with path.open("x", encoding="utf-8", newline="\n") as f: json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True) f.write("\n") def escape(s): return str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%").replace("_", "\\_") def main(): ap = argparse.ArgumentParser() ap.add_argument("--catalog", type=Path, required=True) ap.add_argument("--output", type=Path, required=True) args = ap.parse_args() out = args.output out.mkdir(parents=True, exist_ok=True) for filename in ("papers.json", "papers.bib", "papers.md", "crossref"): if (out / filename).exists(): raise RuntimeError("Refusing to overwrite " + filename) (out / "crossref").mkdir() items = {normalize(d): {"doi":d, "use":u, "source_files":[], "extracted_forms":[]} for d,u in CORE.items()} for r in json.loads(args.catalog.read_text(encoding="utf-8")): for d in r["doi_candidates"]: key = normalize(d) row = items.setdefault(key, {"doi":key, "use":"Historical citation; consult source file for context", "source_files":[], "extracted_forms":[]}) row["source_files"].append(r["source_path"]) row["extracted_forms"].append(d) def fetch(pair): key, row = pair row["source_files"] = sorted(set(row["source_files"])) row["extracted_forms"] = sorted(set(row["extracted_forms"])) url = "https://api.crossref.org/works/" + urllib.parse.quote(row["doi"], safe="") row.update(metadata_url=url, checked_utc=datetime.now(timezone.utc).isoformat(), fulltext_archived=False) last = None for attempt in range(3): try: request = urllib.request.Request(url, headers={"User-Agent":"DigitalDrosophilaBibliography/1.0 (research archive)", "Accept":"application/json"}) with urllib.request.urlopen(request, timeout=45) as r: raw = r.read() result = json.loads(raw) m = result["message"] if normalize(m["DOI"]) != key: raise ValueError("Returned DOI differs") token = hashlib.sha256(key.encode()).hexdigest()[:16] with (out / "crossref" / (token + ".json")).open("xb") as f: f.write(raw) row.update(status="VERIFIED-METADATA", title="; ".join(m.get("title", [])), authors=m.get("author", []), journal="; ".join(m.get("container-title", [])), published=m.get("published", {}), publisher=m.get("publisher"), volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"), doi_url=m.get("URL", "https://doi.org/" + row["doi"]), publisher_fulltext_links=m.get("link", []), license_metadata=m.get("license", []), metadata_file="crossref/"+token+".json", metadata_sha256=hashlib.sha256(raw).hexdigest()) print("metadata OK", row["doi"], flush=True) return row except Exception as e: last = type(e).__name__ + ": " + str(e) if attempt < 2: time.sleep(2 * (attempt+1)) row.update(status="UNRESOLVED", error=last, doi_url="https://doi.org/"+row["doi"]) print("metadata UNRESOLVED", row["doi"], flush=True) return row with cf.ThreadPoolExecutor(max_workers=4) as ex: rows = list(ex.map(fetch, sorted(items.items()))) json_new(out / "papers.json", {"schema_version":1, "metadata_provider":"Crossref / publisher-deposited", "papers":rows}) with (out / "papers.bib").open("x", encoding="utf-8") as bib, (out / "papers.md").open("x", encoding="utf-8") as md: md.write("# 论文索引\n\n出版元数据由 Crossref 获取；未打包论文全文。许可证据见 JSON，不能默认再分发。\n\n") for i,r in enumerate(rows,1): if r["status"] != "VERIFIED-METADATA": md.write(f"- UNRESOLVED [{r['doi']}]({r['doi_url']})；保留原位引用，未补造标题。\n") continue year = r["published"].get("date-parts", [[None]])[0][0] authors = " and ".join(a.get("family",a.get("name","")) + (", " + a["given"] if a.get("given") else "") for a in r["authors"]) bib.write(f"@article{{malecns_ref_{i:03d},\n") for k,v in {"title":r["title"], "author":authors, "year":year, "journal":r["journal"], "doi":r["doi"], "url":r["doi_url"], "volume":r["volume"], "pages":r["pages"]}.items(): if v is not None: bib.write(f" {k} = {{{escape(v)}}},\n") bib.write("}\n\n") md.write(f"- **{year}** [{r['title']}]({r['doi_url']}) — {r['journal']}。用途：{r['use']}。\n") print(json.dumps({"papers":len(rows), "verified":sum(r['status']=='VERIFIED-METADATA' for r in rows), "unresolved":sum(r['status']=='UNRESOLVED' for r in rows)})) if __name__ == "__main__": main()