feat(crossref): reference extraction as third citation fallback (R-B)

Add crossref.fetch_references(doi), which parses message.reference[].DOI into Citations (DOI-less book/unstructured refs are skipped — they cannot join the graph). Wire it as the third leg of the ingest citation fallback chain: OpenAlex-empty -> S2 -> Crossref, for DOI papers only (Crossref's works endpoint is DOI-keyed). Cited DOIs are normalized to the canonical bare lower-case form via _norm_cited_id.

As a fallback it fires only when OpenAlex and S2 both return no references, so it adds a forward-looking third source without changing the already-covered corpus.

Tests: fetch_references unit tests (DOI edges / unstructured skipped / no refs / 404) and ingest tests for both fallback directions. Full suite 361 passed; ruff + mypy clean. Live-validated: 33/40/24 refs for Springer/ACM/Wiley DOIs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-17 21:32:46 +02:00
parent 1516684bbb
commit 82cea2a33b
5 changed files with 202 additions and 4 deletions

View File

@@ -107,6 +107,60 @@ def test_ingest_paper_basic() -> None:
)
def test_ingest_citation_fallback_to_crossref() -> None:
"""R-B: OpenAlex + S2 both empty → Crossref references supply the edges.
Crossref is the third leg of the fallback chain. Its cited DOIs are
case-normalised and citing_id is rewritten to the canonical paper.id.
"""
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
mock_conn = MagicMock()
mock_conn.execute = MagicMock()
mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock()
# Upper-case suffix proves the cited DOI is normalised to bare lower-case.
crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")]
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty
patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr,
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
):
result = ingest_paper(paper.id)
# Crossref consulted with the bare DOI; one edge upserted.
mock_cr.assert_called_once_with(paper.id)
assert result.citations_upserted == 1
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
cit_rows = mock_cursor.executemany.call_args_list[0][0][1]
assert cit_rows[0][0] == paper.id
assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7"
def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
"""R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback)."""
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
mock_conn = MagicMock()
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch(
"codex.ingest.openalex.fetch_citations",
return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")],
),
patch("codex.ingest.crossref.fetch_references") as mock_cr,
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
):
ingest_paper(paper.id)
mock_cr.assert_not_called()
# ---------------------------------------------------------------------------
# 2. test_ingest_paper_source_tex
# ---------------------------------------------------------------------------