feat(crossref): reference extraction as third citation fallback (R-B)
Add crossref.fetch_references(doi), which parses message.reference[].DOI into Citations (DOI-less book/unstructured refs are skipped — they cannot join the graph). Wire it as the third leg of the ingest citation fallback chain: OpenAlex-empty -> S2 -> Crossref, for DOI papers only (Crossref's works endpoint is DOI-keyed). Cited DOIs are normalized to the canonical bare lower-case form via _norm_cited_id. As a fallback it fires only when OpenAlex and S2 both return no references, so it adds a forward-looking third source without changing the already-covered corpus. Tests: fetch_references unit tests (DOI edges / unstructured skipped / no refs / 404) and ingest tests for both fallback directions. Full suite 361 passed; ruff + mypy clean. Live-validated: 33/40/24 refs for Springer/ACM/Wiley DOIs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -107,6 +107,60 @@ def test_ingest_paper_basic() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_citation_fallback_to_crossref() -> None:
|
||||
"""R-B: OpenAlex + S2 both empty → Crossref references supply the edges.
|
||||
|
||||
Crossref is the third leg of the fallback chain. Its cited DOIs are
|
||||
case-normalised and citing_id is rewritten to the canonical paper.id.
|
||||
"""
|
||||
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
# Upper-case suffix proves the cited DOI is normalised to bare lower-case.
|
||||
crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty
|
||||
patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper(paper.id)
|
||||
|
||||
# Crossref consulted with the bare DOI; one edge upserted.
|
||||
mock_cr.assert_called_once_with(paper.id)
|
||||
assert result.citations_upserted == 1
|
||||
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
||||
cit_rows = mock_cursor.executemany.call_args_list[0][0][1]
|
||||
assert cit_rows[0][0] == paper.id
|
||||
assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7"
|
||||
|
||||
|
||||
def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
|
||||
"""R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback)."""
|
||||
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
|
||||
mock_conn = MagicMock()
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch(
|
||||
"codex.ingest.openalex.fetch_citations",
|
||||
return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")],
|
||||
),
|
||||
patch("codex.ingest.crossref.fetch_references") as mock_cr,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
ingest_paper(paper.id)
|
||||
|
||||
mock_cr.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. test_ingest_paper_source_tex
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user