Archive MaleCNS research sources, provenance, bibliography and immutable data inventory

2026-09-29 03:38:32 +08:00
commit 3981628ff8
682 changed files with 161475 additions and 0 deletions
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Fetch DOI metadata from Crossref into new files; no paper full-text copying."""
+import argparse
+import concurrent.futures as cf
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import re
+import time
+import urllib.request
+import urllib.parse
+
+CORE = {
+    "10.1016/j.cell.2026.08.015": "Formal MaleCNS Cell 2026; distinguish release/schema from publication counts",
+    "10.1016/j.cell.2026.08.014": "Official MaleCNS visual pathways",
+    "10.1038/s41586-024-07763-9": "Shiu-like LIF reference; not a strict replication claim",
+    "10.1038/s41586-024-07939-3": "FlyVis independent visual model",
+    "10.1038/srep21841": "DoOR physiology / odor-receptor-glomerular reference",
+    "10.1016/j.cell.2024.03.016": "EM neurotransmitter prediction and limitations",
+    "10.7554/eLife.03868": "DPM physiology; not a universal edge sign assignment",
+    "10.7554/eLife.56954": "Localized APL inhibition",
+    "10.7554/eLife.74172": "APL / PN-bouton and KC-dendrite normalization",
+    "10.1073/pnas.0914912107": "MB to antennal-lobe feedback physiology",
+    "10.1016/j.cell.2006.12.034": "Excitatory antennal-lobe local circuits",
+    "10.1016/j.neuron.2010.08.025": "Excitatory local interneuron connectivity",
+    "10.1016/j.cell.2019.05.040": "Timing-dependent dopamine plasticity",
+    "10.1038/s41467-023-38487-5": "KC subthreshold response and decision timing",
+    "10.1038/s41586-025-09029-4": "FlyBody / Google DeepMind; separate physical body/controller research",
+    "10.7554/eLife.76712": "KC acetylcholine release plasticity",
+    "10.1523/ENEURO.0275-23.2023": "Compartment-specific KC release and bouton heterogeneity",
+    "10.1016/j.cell.2015.11.019": "Compartment and timing dependence of transmission",
+    "10.7554/eLife.80445": "Postsynaptic cholinergic memory mechanisms",
+    "10.1523/JNEUROSCI.1420-19.2019": "Pre/post synaptic odor suppression",
+}
+
+def normalize(s):
+    # Only known extraction artifacts; do not invent or fuzzy-match DOI.
+    return s.rstrip(".,;:)]}").removesuffix("/full").lower()
+
+def json_new(path, obj):
+    with path.open("x", encoding="utf-8", newline="\n") as f:
+        json.dump(obj, f, ensure_ascii=False, indent=2, sort_keys=True)
+        f.write("\n")
+
+def escape(s):
+    return str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%").replace("_", "\\_")
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--catalog", type=Path, required=True)
+    ap.add_argument("--output", type=Path, required=True)
+    args = ap.parse_args()
+    out = args.output
+    out.mkdir(parents=True, exist_ok=True)
+    for filename in ("papers.json", "papers.bib", "papers.md", "crossref"):
+        if (out / filename).exists():
+            raise RuntimeError("Refusing to overwrite " + filename)
+    (out / "crossref").mkdir()
+    items = {normalize(d): {"doi":d, "use":u, "source_files":[], "extracted_forms":[]} for d,u in CORE.items()}
+    for r in json.loads(args.catalog.read_text(encoding="utf-8")):
+        for d in r["doi_candidates"]:
+            key = normalize(d)
+            row = items.setdefault(key, {"doi":key, "use":"Historical citation; consult source file for context", "source_files":[], "extracted_forms":[]})
+            row["source_files"].append(r["source_path"])
+            row["extracted_forms"].append(d)
+    def fetch(pair):
+        key, row = pair
+        row["source_files"] = sorted(set(row["source_files"]))
+        row["extracted_forms"] = sorted(set(row["extracted_forms"]))
+        url = "https://api.crossref.org/works/" + urllib.parse.quote(row["doi"], safe="")
+        row.update(metadata_url=url, checked_utc=datetime.now(timezone.utc).isoformat(), fulltext_archived=False)
+        last = None
+        for attempt in range(3):
+            try:
+                request = urllib.request.Request(url, headers={"User-Agent":"DigitalDrosophilaBibliography/1.0 (research archive)", "Accept":"application/json"})
+                with urllib.request.urlopen(request, timeout=45) as r:
+                    raw = r.read()
+                result = json.loads(raw)
+                m = result["message"]
+                if normalize(m["DOI"]) != key:
+                    raise ValueError("Returned DOI differs")
+                token = hashlib.sha256(key.encode()).hexdigest()[:16]
+                with (out / "crossref" / (token + ".json")).open("xb") as f:
+                    f.write(raw)
+                row.update(status="VERIFIED-METADATA", title="; ".join(m.get("title", [])), authors=m.get("author", []),
+                           journal="; ".join(m.get("container-title", [])), published=m.get("published", {}),
+                           publisher=m.get("publisher"), volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"),
+                           doi_url=m.get("URL", "https://doi.org/" + row["doi"]),
+                           publisher_fulltext_links=m.get("link", []), license_metadata=m.get("license", []),
+                           metadata_file="crossref/"+token+".json", metadata_sha256=hashlib.sha256(raw).hexdigest())
+                print("metadata OK", row["doi"], flush=True)
+                return row
+            except Exception as e:
+                last = type(e).__name__ + ": " + str(e)
+                if attempt < 2:
+                    time.sleep(2 * (attempt+1))
+        row.update(status="UNRESOLVED", error=last, doi_url="https://doi.org/"+row["doi"])
+        print("metadata UNRESOLVED", row["doi"], flush=True)
+        return row
+    with cf.ThreadPoolExecutor(max_workers=4) as ex:
+        rows = list(ex.map(fetch, sorted(items.items())))
+    json_new(out / "papers.json", {"schema_version":1, "metadata_provider":"Crossref / publisher-deposited", "papers":rows})
+    with (out / "papers.bib").open("x", encoding="utf-8") as bib, (out / "papers.md").open("x", encoding="utf-8") as md:
+        md.write("# 论文索引\n\n出版元数据由 Crossref 获取；未打包论文全文。许可证据见 JSON，不能默认再分发。\n\n")
+        for i,r in enumerate(rows,1):
+            if r["status"] != "VERIFIED-METADATA":
+                md.write(f"- UNRESOLVED [{r['doi']}]({r['doi_url']})；保留原位引用，未补造标题。\n")
+                continue
+            year = r["published"].get("date-parts", [[None]])[0][0]
+            authors = " and ".join(a.get("family",a.get("name","")) + (", " + a["given"] if a.get("given") else "") for a in r["authors"])
+            bib.write(f"@article{{malecns_ref_{i:03d},\n")
+            for k,v in {"title":r["title"], "author":authors, "year":year, "journal":r["journal"], "doi":r["doi"], "url":r["doi_url"], "volume":r["volume"], "pages":r["pages"]}.items():
+                if v is not None:
+                    bib.write(f"  {k} = {{{escape(v)}}},\n")
+            bib.write("}\n\n")
+            md.write(f"- **{year}** [{r['title']}]({r['doi_url']}) — {r['journal']}。用途：{r['use']}。\n")
+    print(json.dumps({"papers":len(rows), "verified":sum(r['status']=='VERIFIED-METADATA' for r in rows), "unresolved":sum(r['status']=='UNRESOLVED' for r in rows)}))
+
+if __name__ == "__main__":
+    main()