merge: R-B (Crossref references as third citation fallback)

codex/ingest.py

@@ -130,6 +130,29 @@ def _s2_reference_supplement(paper: Paper) -> list[Citation]:
     ]
 
 
+def _crossref_reference_supplement(paper: Paper) -> list[Citation]:
+    """Fetch a paper's references from Crossref (roadmap R-B supplement).
+
+    Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref).
+    Crossref carries publisher-deposited reference lists keyed on the citing
+    work's DOI, so only DOI papers can be looked up (arXiv-only ids → []).
+    ``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are
+    case-normalised. Network/parse failures degrade to [] rather than aborting.
+    """
+    if not paper.id.startswith("10."):
+        return []
+    try:
+        raw = crossref.fetch_references(paper.id)
+    except Exception:
+        logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True)
+        return []
+    return [
+        Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
+        for c in raw
+        if c.cited_id
+    ]
+
+
 def ingest_paper(
     paper_id: str,
     source_path: str | None = None,
@@ -329,7 +352,7 @@ def ingest_paper(
                 chunks_upserted = len(chunk_rows)
 
         # ---------------------------------------------------------------
-        # 5. Fetch citations  (OpenAlex if openalex_id, else S2)
+        # 5. Fetch citations  (fallback chain: OpenAlex → S2 → Crossref)
         #    Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING
         # ---------------------------------------------------------------
         api_citations: list[Citation]
@@ -347,6 +370,10 @@ def ingest_paper(
                 api_citations = _s2_reference_supplement(paper)
         else:
             api_citations = _s2_reference_supplement(paper)
+        # R-B: if OpenAlex and S2 both came back empty, try Crossref's
+        # publisher-deposited references (DOI papers only) as a third leg.
+        if not api_citations:
+            api_citations = _crossref_reference_supplement(paper)
 
         # Merge API citations and GROBID PDF citations; dedup via set
         all_citations_set: set[tuple[str, str]] = set()