feat(crossref): reference extraction as third citation fallback (R-B)

Add crossref.fetch_references(doi), which parses message.reference[].DOI into Citations (DOI-less book/unstructured refs are skipped — they cannot join the graph). Wire it as the third leg of the ingest citation fallback chain: OpenAlex-empty -> S2 -> Crossref, for DOI papers only (Crossref's works endpoint is DOI-keyed). Cited DOIs are normalized to the canonical bare lower-case form via _norm_cited_id.

As a fallback it fires only when OpenAlex and S2 both return no references, so it adds a forward-looking third source without changing the already-covered corpus.

Tests: fetch_references unit tests (DOI edges / unstructured skipped / no refs / 404) and ingest tests for both fallback directions. Full suite 361 passed; ruff + mypy clean. Live-validated: 33/40/24 refs for Springer/ACM/Wiley DOIs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/ingest/test_ingest.py

@@ -107,6 +107,60 @@ def test_ingest_paper_basic() -> None:
     )
 
 
+def test_ingest_citation_fallback_to_crossref() -> None:
+    """R-B: OpenAlex + S2 both empty → Crossref references supply the edges.
+
+    Crossref is the third leg of the fallback chain. Its cited DOIs are
+    case-normalised and citing_id is rewritten to the canonical paper.id.
+    """
+    paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    # Upper-case suffix proves the cited DOI is normalised to bare lower-case.
+    crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")]
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),  # OpenAlex empty
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),  # S2 empty
+        patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr,
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper(paper.id)
+
+    # Crossref consulted with the bare DOI; one edge upserted.
+    mock_cr.assert_called_once_with(paper.id)
+    assert result.citations_upserted == 1
+    mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
+    cit_rows = mock_cursor.executemany.call_args_list[0][0][1]
+    assert cit_rows[0][0] == paper.id
+    assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7"
+
+
+def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
+    """R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback)."""
+    paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
+    mock_conn = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch(
+            "codex.ingest.openalex.fetch_citations",
+            return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")],
+        ),
+        patch("codex.ingest.crossref.fetch_references") as mock_cr,
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    mock_cr.assert_not_called()
+
+
 # ---------------------------------------------------------------------------
 # 2. test_ingest_paper_source_tex
 # ---------------------------------------------------------------------------

tests/sources/test_crossref.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import httpx
 import pytest
 
+from codex.models import Citation
 from codex.sources import crossref
 
 
@@ -48,3 +49,56 @@ def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> Non
 
     monkeypatch.setattr(crossref, "_get", mock_get)
     assert crossref.fetch_abstract("10.0000/nonexistent") is None
+
+
+# ---------------------------------------------------------------------------
+# fetch_references (roadmap R-B)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_references_returns_doi_edges(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Only references carrying a DOI become Citations; citing_id is the queried DOI."""
+    body = {
+        "message": {
+            "reference": [
+                {"key": "ref1", "DOI": "10.1090/s0002-9947-04-03545-7"},
+                {"key": "ref2", "unstructured": "A book with no DOI, 1992."},  # skipped
+                {"key": "ref3", "DOI": "10.1007/bf02392449", "article-title": "X"},
+            ]
+        }
+    }
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=body)
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    refs = crossref.fetch_references("10.1007/s00454-019-00132-8")
+
+    assert refs == [
+        Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1090/s0002-9947-04-03545-7"),
+        Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1007/bf02392449"),
+    ]
+
+
+def test_fetch_references_no_reference_key_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A work with no deposited reference list returns []."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"message": {"title": ["No refs deposited"]}})
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_references("10.1007/978-3-642-17413-1_7") == []
+
+
+def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 returns [] rather than raising."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "404",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_references("10.0000/nonexistent") == []