feat(ingest): recover missing metadata + abstracts from arXiv/S2/Crossref (DQ-2)

Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404
-> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in:

- arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API
  (authoritative for preprints). Used as an ingest fallback on OpenAlex 404,
  replacing the empty stub.
- crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool.
- semanticscholar.fetch_abstract: fetch just the abstract field.
- ingest.py: _recover_abstract() supplements an empty abstract from S2 then
  Crossref *before* embedding, so the paper gets a real (non-zero) vector.

Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey
PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk
search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter
10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented
limit. Corpus now has 1 paper without an abstract.

Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers
(openalex returns full-URL id vs the bare canonical id) -> re-ingest trips
papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix.

Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/ingest/test_ingest.py

@@ -250,7 +250,7 @@ def test_ingest_paper_idempotent() -> None:
 
 
 def test_ingest_paper_arxiv_s2_fallback() -> None:
-    """If OpenAlex returns None for an arXiv ID, S2 fallback produces a stub Paper."""
+    """OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted."""
     paper_id = "2301.07041"
     mock_conn = MagicMock()
     mock_conn.execute = MagicMock()
@@ -259,17 +259,84 @@ def test_ingest_paper_arxiv_s2_fallback() -> None:
 
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=None),
+        # arXiv + abstract sources also empty → genuine stub fallback path.
+        patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv,
+        patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
+        patch("codex.ingest.crossref.fetch_abstract", return_value=None),
         patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2,
         patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
     ):
         result = ingest_paper(paper_id)
 
-    # S2 was consulted as fallback
+    # arXiv metadata recovery attempted; S2 references consulted as fallback.
+    mock_arxiv.assert_called_once_with(paper_id)
     mock_s2.assert_called()
     assert result.paper_id == paper_id
 
 
+def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None:
+    """DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API,
+    bibkey auto-generated, abstract embedded (no zero vector)."""
+    recovered = Paper(
+        id="1911.00966",
+        title="A discrete version of Liouville's theorem on conformal maps",
+        authors=["Ulrich Pinkall", "Boris Springborn"],
+        year=2019,
+        abstract="Liouville's theorem says that in dimension greater than two...",
+    )
+    fake_emb = _fake_embedder()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=None),
+        patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv,
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=fake_emb),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper("1911.00966")
+
+    mock_arxiv.assert_called_once_with("1911.00966")
+    assert result.paper_id == "1911.00966"
+    params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert params["title"].startswith("A discrete version")
+    assert params["bibkey"] == "PinkallSpringborn2019"  # generated from recovered authors+year
+    # abstract present → real embedding computed, not a zero vector
+    fake_emb.encode_dense.assert_called_once()
+
+
+def test_ingest_paper_empty_abstract_recovered_from_s2() -> None:
+    """DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding."""
+    paper = _make_paper(abstract=None)  # has openalex_id, no abstract
+    fake_emb = _fake_embedder()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch(
+            "codex.ingest.semanticscholar.fetch_abstract",
+            return_value="Recovered abstract text.",
+        ) as mock_abs,
+        patch("codex.ingest.get_embedder", return_value=fake_emb),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    mock_abs.assert_called_once()
+    params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert params["abstract"] == "Recovered abstract text."
+    fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."])
+
+
 def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
     """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
     rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
@@ -325,6 +392,9 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=paper),
         patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # No abstract recoverable from any source (DQ-2) → genuine zero-vector case.
+        patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
+        patch("codex.ingest.crossref.fetch_abstract", return_value=None),
         patch("codex.ingest.get_embedder", return_value=fake_emb),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
     ):