merge: R-B (Crossref references as third citation fallback)

codex/ingest.py

@@ -130,6 +130,29 @@ def _s2_reference_supplement(paper: Paper) -> list[Citation]:
     ]
 
 
+def _crossref_reference_supplement(paper: Paper) -> list[Citation]:
+    """Fetch a paper's references from Crossref (roadmap R-B supplement).
+
+    Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref).
+    Crossref carries publisher-deposited reference lists keyed on the citing
+    work's DOI, so only DOI papers can be looked up (arXiv-only ids → []).
+    ``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are
+    case-normalised. Network/parse failures degrade to [] rather than aborting.
+    """
+    if not paper.id.startswith("10."):
+        return []
+    try:
+        raw = crossref.fetch_references(paper.id)
+    except Exception:
+        logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True)
+        return []
+    return [
+        Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
+        for c in raw
+        if c.cited_id
+    ]
+
+
 def ingest_paper(
     paper_id: str,
     source_path: str | None = None,
@@ -329,7 +352,7 @@ def ingest_paper(
                 chunks_upserted = len(chunk_rows)
 
         # ---------------------------------------------------------------
-        # 5. Fetch citations  (OpenAlex if openalex_id, else S2)
+        # 5. Fetch citations  (fallback chain: OpenAlex → S2 → Crossref)
         #    Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING
         # ---------------------------------------------------------------
         api_citations: list[Citation]
@@ -347,6 +370,10 @@ def ingest_paper(
                 api_citations = _s2_reference_supplement(paper)
         else:
             api_citations = _s2_reference_supplement(paper)
+        # R-B: if OpenAlex and S2 both came back empty, try Crossref's
+        # publisher-deposited references (DOI papers only) as a third leg.
+        if not api_citations:
+            api_citations = _crossref_reference_supplement(paper)
 
         # Merge API citations and GROBID PDF citations; dedup via set
         all_citations_set: set[tuple[str, str]] = set()

codex/sources/crossref.py

@@ -2,10 +2,11 @@
 
 Provides:
 - fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI.
+- fetch_references: retrieve a work's reference list (DOI edges) by DOI.
 
 Crossref is a *third* metadata source after OpenAlex and Semantic Scholar
-(DQ-2 / roadmap R-B). Requests use the Polite Pool (mailto query parameter)
-and are retried on 429/5xx with exponential back-off via tenacity.
+(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto
+query parameter) and are retried on 429/5xx with exponential back-off via tenacity.
 """
 
 from __future__ import annotations
@@ -18,6 +19,7 @@ import httpx
 from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
 
 from codex.config import get_settings
+from codex.models import Citation
 
 logger = logging.getLogger(__name__)
 
@@ -89,3 +91,36 @@ def fetch_abstract(doi: str) -> str | None:
         raise
     message: dict[str, Any] = response.json().get("message", {})
     return _strip_jats(message.get("abstract"))
+
+
+def fetch_references(doi: str) -> list[Citation]:
+    """Fetch a work's reference list from Crossref by DOI (roadmap R-B).
+
+    Crossref carries publisher-deposited reference lists in ``message.reference``.
+    Each entry that cites a DOI-registered work exposes a bare ``DOI`` field;
+    only those become citation-graph edges (book / older references carry just
+    ``unstructured`` text with no resolvable id and are skipped). ``citing_id``
+    is the queried *doi* — the caller rewrites it to the canonical ``papers.id``
+    and normalises the cited DOIs (mirrors the Semantic Scholar reference path).
+
+    Parameters
+    ----------
+    doi:
+        A bare DOI (``"10.1007/s00454-019-00132-8"``).
+
+    Returns
+    -------
+    list[Citation]
+        One Citation per reference that carries a DOI. Empty on 404 or when no
+        references are deposited for the work.
+    """
+    url = f"{_BASE}/works/{doi}"
+    try:
+        response = _get(url)
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return []
+        raise
+    message: dict[str, Any] = response.json().get("message", {})
+    references: list[dict[str, Any]] = message.get("reference") or []
+    return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")]