feat(ingest): recover missing metadata + abstracts from arXiv/S2/Crossref (DQ-2)

Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404
-> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in:

- arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API
  (authoritative for preprints). Used as an ingest fallback on OpenAlex 404,
  replacing the empty stub.
- crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool.
- semanticscholar.fetch_abstract: fetch just the abstract field.
- ingest.py: _recover_abstract() supplements an empty abstract from S2 then
  Crossref *before* embedding, so the paper gets a real (non-zero) vector.

Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey
PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk
search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter
10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented
limit. Corpus now has 1 paper without an abstract.

Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers
(openalex returns full-URL id vs the bare canonical id) -> re-ingest trips
papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix.

Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/sources/test_arxiv.py

@@ -138,3 +138,63 @@ def test_fetch_pdf_url_pure_computation() -> None:
 
     url2 = arxiv.fetch_pdf_url("1234.56789")
     assert url2 == "https://arxiv.org/pdf/1234.56789.pdf"
+
+
+# ---------------------------------------------------------------------------
+# fetch_metadata — parse the Atom feed into a Paper (DQ-2)
+# ---------------------------------------------------------------------------
+
+_ATOM_FEED = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <entry>
+    <title>A discrete version of Liouville's theorem on conformal maps</title>
+    <summary>Liouville's theorem says that in dimension greater than two,
+    all conformal maps are Moebius transformations.</summary>
+    <published>2019-11-03T00:00:00Z</published>
+    <author><name>Ulrich Pinkall</name></author>
+    <author><name>Boris Springborn</name></author>
+  </entry>
+</feed>"""
+
+_ATOM_EMPTY = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"><title>ArXiv Query</title></feed>"""
+
+
+def test_fetch_metadata_parses_entry(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A valid Atom feed yields a populated Paper."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    paper = arxiv.fetch_metadata("1911.00966")
+    assert paper is not None
+    assert paper.id == "1911.00966"
+    assert paper.title.startswith("A discrete version of Liouville")
+    assert paper.authors == ["Ulrich Pinkall", "Boris Springborn"]
+    assert paper.year == 2019
+    assert paper.abstract and "conformal maps" in paper.abstract
+    assert paper.openalex_id is None
+
+
+def test_fetch_metadata_no_entry_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A feed with no <entry> (id not found) returns None."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_EMPTY, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    assert arxiv.fetch_metadata("0000.00000") is None
+
+
+def test_fetch_metadata_strips_arxiv_prefix(monkeypatch: pytest.MonkeyPatch) -> None:
+    """An ``arXiv:`` prefix is stripped so Paper.id is the bare id."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    paper = arxiv.fetch_metadata("arXiv:1911.00966")
+    assert paper is not None
+    assert paper.id == "1911.00966"

tests/sources/test_crossref.py

@@ -0,0 +1,50 @@
+"""Tests for codex.sources.crossref."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from codex.sources import crossref
+
+
+def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A JATS-tagged abstract is returned as plain text with tags removed."""
+    body = {
+        "message": {
+            "abstract": "<jats:p>We prove a <jats:italic>theorem</jats:italic> about "
+            "polyhedra.</jats:p>"
+        }
+    }
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=body)
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+
+    result = crossref.fetch_abstract("10.1007/s00454-019-00132-8")
+    assert result == "We prove a theorem about polyhedra."
+
+
+def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A work with no abstract field (e.g. a book chapter) returns None."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}})
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None
+
+
+def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 returns None rather than raising."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "404",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_abstract("10.0000/nonexistent") is None

tests/sources/test_semanticscholar.py

@@ -124,3 +124,46 @@ def test_fetch_recommendations_404_returns_empty(
 
     result = semanticscholar.fetch_recommendations("nonexistent")
     assert result == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_references — {"data": null} guard (DQ-1 regression)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_references_null_data_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
+    parsed references; this must yield [] rather than raising TypeError."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"data": None, "citingPaperInfo": {}})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_references("DOI:10.14279/depositonce-5415") == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_abstract (DQ-2)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_abstract_returns_text(monkeypatch: pytest.MonkeyPatch) -> None:
+    """fetch_abstract returns the abstract string when present."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"abstract": "We study ideal hyperbolic polyhedra."})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_abstract("DOI:10.1007/s00454-019-00132-8") == (
+        "We study ideal hyperbolic polyhedra."
+    )
+
+
+def test_fetch_abstract_null_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A null/absent abstract yields None."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"abstract": None})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_abstract("arXiv:1234.5678") is None