feat(ingest): recover missing metadata + abstracts from arXiv/S2/Crossref (DQ-2)

Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404
-> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in:

- arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API
  (authoritative for preprints). Used as an ingest fallback on OpenAlex 404,
  replacing the empty stub.
- crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool.
- semanticscholar.fetch_abstract: fetch just the abstract field.
- ingest.py: _recover_abstract() supplements an empty abstract from S2 then
  Crossref *before* embedding, so the paper gets a real (non-zero) vector.

Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey
PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk
search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter
10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented
limit. Corpus now has 1 paper without an abstract.

Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers
(openalex returns full-URL id vs the bare canonical id) -> re-ingest trips
papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix.

Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/ingest/test_ingest.py

@@ -250,7 +250,7 @@ def test_ingest_paper_idempotent() -> None:
 
 
 def test_ingest_paper_arxiv_s2_fallback() -> None:
-    """If OpenAlex returns None for an arXiv ID, S2 fallback produces a stub Paper."""
+    """OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted."""
     paper_id = "2301.07041"
     mock_conn = MagicMock()
     mock_conn.execute = MagicMock()
@@ -259,17 +259,84 @@ def test_ingest_paper_arxiv_s2_fallback() -> None:
 
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=None),
+        # arXiv + abstract sources also empty → genuine stub fallback path.
+        patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv,
+        patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
+        patch("codex.ingest.crossref.fetch_abstract", return_value=None),
         patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2,
         patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
     ):
         result = ingest_paper(paper_id)
 
-    # S2 was consulted as fallback
+    # arXiv metadata recovery attempted; S2 references consulted as fallback.
+    mock_arxiv.assert_called_once_with(paper_id)
     mock_s2.assert_called()
     assert result.paper_id == paper_id
 
 
+def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None:
+    """DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API,
+    bibkey auto-generated, abstract embedded (no zero vector)."""
+    recovered = Paper(
+        id="1911.00966",
+        title="A discrete version of Liouville's theorem on conformal maps",
+        authors=["Ulrich Pinkall", "Boris Springborn"],
+        year=2019,
+        abstract="Liouville's theorem says that in dimension greater than two...",
+    )
+    fake_emb = _fake_embedder()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=None),
+        patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv,
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=fake_emb),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper("1911.00966")
+
+    mock_arxiv.assert_called_once_with("1911.00966")
+    assert result.paper_id == "1911.00966"
+    params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert params["title"].startswith("A discrete version")
+    assert params["bibkey"] == "PinkallSpringborn2019"  # generated from recovered authors+year
+    # abstract present → real embedding computed, not a zero vector
+    fake_emb.encode_dense.assert_called_once()
+
+
+def test_ingest_paper_empty_abstract_recovered_from_s2() -> None:
+    """DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding."""
+    paper = _make_paper(abstract=None)  # has openalex_id, no abstract
+    fake_emb = _fake_embedder()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch(
+            "codex.ingest.semanticscholar.fetch_abstract",
+            return_value="Recovered abstract text.",
+        ) as mock_abs,
+        patch("codex.ingest.get_embedder", return_value=fake_emb),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    mock_abs.assert_called_once()
+    params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert params["abstract"] == "Recovered abstract text."
+    fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."])
+
+
 def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
     """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
     rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
@@ -325,6 +392,9 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=paper),
         patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # No abstract recoverable from any source (DQ-2) → genuine zero-vector case.
+        patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
+        patch("codex.ingest.crossref.fetch_abstract", return_value=None),
         patch("codex.ingest.get_embedder", return_value=fake_emb),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
     ):

tests/sources/test_arxiv.py

@@ -138,3 +138,63 @@ def test_fetch_pdf_url_pure_computation() -> None:
 
     url2 = arxiv.fetch_pdf_url("1234.56789")
     assert url2 == "https://arxiv.org/pdf/1234.56789.pdf"
+
+
+# ---------------------------------------------------------------------------
+# fetch_metadata — parse the Atom feed into a Paper (DQ-2)
+# ---------------------------------------------------------------------------
+
+_ATOM_FEED = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <entry>
+    <title>A discrete version of Liouville's theorem on conformal maps</title>
+    <summary>Liouville's theorem says that in dimension greater than two,
+    all conformal maps are Moebius transformations.</summary>
+    <published>2019-11-03T00:00:00Z</published>
+    <author><name>Ulrich Pinkall</name></author>
+    <author><name>Boris Springborn</name></author>
+  </entry>
+</feed>"""
+
+_ATOM_EMPTY = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"><title>ArXiv Query</title></feed>"""
+
+
+def test_fetch_metadata_parses_entry(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A valid Atom feed yields a populated Paper."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    paper = arxiv.fetch_metadata("1911.00966")
+    assert paper is not None
+    assert paper.id == "1911.00966"
+    assert paper.title.startswith("A discrete version of Liouville")
+    assert paper.authors == ["Ulrich Pinkall", "Boris Springborn"]
+    assert paper.year == 2019
+    assert paper.abstract and "conformal maps" in paper.abstract
+    assert paper.openalex_id is None
+
+
+def test_fetch_metadata_no_entry_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A feed with no <entry> (id not found) returns None."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_EMPTY, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    assert arxiv.fetch_metadata("0000.00000") is None
+
+
+def test_fetch_metadata_strips_arxiv_prefix(monkeypatch: pytest.MonkeyPatch) -> None:
+    """An ``arXiv:`` prefix is stripped so Paper.id is the bare id."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    paper = arxiv.fetch_metadata("arXiv:1911.00966")
+    assert paper is not None
+    assert paper.id == "1911.00966"

tests/sources/test_crossref.py

@@ -0,0 +1,50 @@
+"""Tests for codex.sources.crossref."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from codex.sources import crossref
+
+
+def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A JATS-tagged abstract is returned as plain text with tags removed."""
+    body = {
+        "message": {
+            "abstract": "<jats:p>We prove a <jats:italic>theorem</jats:italic> about "
+            "polyhedra.</jats:p>"
+        }
+    }
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=body)
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+
+    result = crossref.fetch_abstract("10.1007/s00454-019-00132-8")
+    assert result == "We prove a theorem about polyhedra."
+
+
+def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A work with no abstract field (e.g. a book chapter) returns None."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}})
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None
+
+
+def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 returns None rather than raising."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "404",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_abstract("10.0000/nonexistent") is None

tests/sources/test_semanticscholar.py

@@ -124,3 +124,46 @@ def test_fetch_recommendations_404_returns_empty(
 
     result = semanticscholar.fetch_recommendations("nonexistent")
     assert result == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_references — {"data": null} guard (DQ-1 regression)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_references_null_data_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
+    parsed references; this must yield [] rather than raising TypeError."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"data": None, "citingPaperInfo": {}})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_references("DOI:10.14279/depositonce-5415") == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_abstract (DQ-2)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_abstract_returns_text(monkeypatch: pytest.MonkeyPatch) -> None:
+    """fetch_abstract returns the abstract string when present."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"abstract": "We study ideal hyperbolic polyhedra."})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_abstract("DOI:10.1007/s00454-019-00132-8") == (
+        "We study ideal hyperbolic polyhedra."
+    )
+
+
+def test_fetch_abstract_null_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A null/absent abstract yields None."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"abstract": None})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_abstract("arXiv:1234.5678") is None