feat(ingest): supplement citations from Semantic Scholar when OpenAlex is empty (DQ-1)
12/29 papers had zero citations: OpenAlex indexes arXiv preprints and theses
without a parsed reference list (verified referenced_works_count=0 server-side
for all 11 with an openalex_id). Not an ingest bug — a source-coverage limit.
- ingest.py: when OpenAlex returns no references, fall back to a Semantic
Scholar reference supplement (_s2_reference_supplement); citing_id rewritten
to canonical papers.id, DOI cited-ids lowercased. Removed a now-redundant
discarded S2 probe in the OpenAlex-404 arXiv branch.
- semanticscholar.py: fix TypeError on S2's HTTP-200 {"data": null} no-refs
responses (.get("data", []) returns None when the key is present-but-null).
- tests: regression test for the OpenAlex-empty -> S2 path.
- Live DB backfilled idempotently: +330 citations (590->920), zero-out-edge
papers 12->2, citing coverage 17->27/29. Only the 2 TU-Berlin depositonce
theses remain (no references in any source).
- docs: DATA-QUALITY-2026-06-15.md — DQ-1 resolution, DQ-4 result, and a
free-source acquisition roadmap (R-A..R-E).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -174,6 +174,9 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
# OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1);
|
||||
# mock it empty so only the GROBID refs contribute to the count.
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
||||
@@ -267,6 +270,43 @@ def test_ingest_paper_arxiv_s2_fallback() -> None:
|
||||
assert result.paper_id == paper_id
|
||||
|
||||
|
||||
def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
|
||||
"""DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
|
||||
rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
|
||||
paper = _make_paper() # openalex_id="W123", id="2301.07041"
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
# S2 keys citing_id by the id form passed in; ingest must canonicalise it.
|
||||
s2_refs = [
|
||||
Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"),
|
||||
Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"),
|
||||
]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper(paper.id)
|
||||
|
||||
# S2 consulted with a namespaced id (a bare arXiv id 404s on S2).
|
||||
mock_s2.assert_called_once_with("arXiv:2301.07041")
|
||||
assert result.citations_upserted == 2
|
||||
|
||||
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
||||
inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1]
|
||||
# citing_id rewritten to paper.id, not the S2 id form.
|
||||
assert all(row[0] == paper.id for row in inserted_rows)
|
||||
cited = {row[1] for row in inserted_rows}
|
||||
assert "10.1/upper" in cited # DOI lowercased
|
||||
assert "2202.00002" in cited # arXiv id untouched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. test_ingest_paper_no_abstract_uses_zero_vector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user