feat(ingest): recover missing metadata + abstracts from arXiv/S2/Crossref (DQ-2)

Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404
-> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in:

- arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API
  (authoritative for preprints). Used as an ingest fallback on OpenAlex 404,
  replacing the empty stub.
- crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool.
- semanticscholar.fetch_abstract: fetch just the abstract field.
- ingest.py: _recover_abstract() supplements an empty abstract from S2 then
  Crossref *before* embedding, so the paper gets a real (non-zero) vector.

Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey
PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk
search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter
10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented
limit. Corpus now has 1 paper without an abstract.

Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers
(openalex returns full-URL id vs the bare canonical id) -> re-ingest trips
papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix.

Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codex/ingest.py

@@ -13,7 +13,7 @@ from codex.db import get_conn
 from codex.embed import get_embedder
 from codex.models import Citation, Paper
 from codex.quality import classify_section, filter_chunks
-from codex.sources import openalex, semanticscholar
+from codex.sources import arxiv, crossref, openalex, semanticscholar
 
 logger = logging.getLogger(__name__)
 
@@ -71,6 +71,31 @@ def _norm_cited_id(cited_id: str) -> str:
     return cited_id.lower() if cited_id.startswith("10.") else cited_id
 
 
+def _recover_abstract(paper: Paper) -> str | None:
+    """Recover a missing abstract from S2, then Crossref (DQ-2 supplement).
+
+    Tried in order of coverage for this corpus: Semantic Scholar (indexes most
+    arXiv + journal abstracts), then Crossref (publisher-deposited, DOI only).
+    Network/parse failures degrade to None rather than aborting the ingest.
+    """
+    s2_id = _s2_id_for(paper.id)
+    if s2_id is not None:
+        try:
+            abstract = semanticscholar.fetch_abstract(s2_id)
+            if abstract and abstract.strip():
+                return abstract.strip()
+        except Exception:
+            logger.warning("S2 abstract recovery failed for %s", paper.id, exc_info=True)
+    if paper.id.startswith("10."):
+        try:
+            abstract = crossref.fetch_abstract(paper.id)
+            if abstract and abstract.strip():
+                return abstract.strip()
+        except Exception:
+            logger.warning("Crossref abstract recovery failed for %s", paper.id, exc_info=True)
+    return None
+
+
 def _s2_reference_supplement(paper: Paper) -> list[Citation]:
     """Fetch a paper's references from Semantic Scholar (DQ-1 supplement).
 
@@ -136,16 +161,25 @@ def ingest_paper(
             len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit()
         )
         if looks_like_arxiv:
-            # OpenAlex 404 on an arXiv id → keep a minimal stub. Citation
-            # recovery from S2 happens in the citation block below (DQ-1
-            # supplement), so no probe fetch is needed here.
-            paper = Paper(id=paper_id, title="")
+            # OpenAlex 404 on an arXiv id → recover authoritative metadata from
+            # the arXiv API (DQ-2); fall back to a minimal stub if arXiv also
+            # has nothing. Citation recovery from S2 happens in the citation
+            # block below (DQ-1 supplement).
+            paper = arxiv.fetch_metadata(paper_id) or Paper(id=paper_id, title="")
         else:
             raise ValueError(f"Paper not found: {paper_id}")
 
     if paper is None:
         raise ValueError(f"Paper not found: {paper_id}")
 
+    # DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute
+    # some publishers' abstracts). Supplement from S2, then Crossref, so the
+    # paper gets a real (non-zero) abstract embedding instead of a zero vector.
+    if not (paper.abstract or "").strip():
+        recovered = _recover_abstract(paper)
+        if recovered:
+            paper.abstract = recovered
+
     # Auto-generate bibkey when source has none (D-04).
     if paper.bibkey is None:
         paper.bibkey = _make_bibkey(paper)

codex/sources/arxiv.py

@@ -1,6 +1,7 @@
 """arXiv API client.
 
 Provides:
+- fetch_metadata: resolve an arXiv ID to a Paper via the Atom export API.
 - fetch_source: download the .tar.gz source of a paper and extract the primary .tex file.
 - fetch_pdf_url: return the canonical PDF URL for a given arXiv ID.
 """
@@ -10,12 +11,89 @@ from __future__ import annotations
 import io
 import logging
 import tarfile
+import xml.etree.ElementTree as ET
 
 import httpx
+from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
+
+from codex.models import Paper
 
 logger = logging.getLogger(__name__)
 
 _BASE = "https://arxiv.org"
+_EXPORT = "https://export.arxiv.org"
+_ATOM = "{http://www.w3.org/2005/Atom}"
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code == 429 or exc.response.status_code >= 500
+    return False
+
+
+@retry(
+    retry=retry_if_exception(_is_retryable),
+    stop=stop_after_attempt(4),
+    wait=wait_exponential(min=1, max=20),
+    before_sleep=lambda rs: logger.warning("arXiv retry %d", rs.attempt_number),
+)
+def _query(arxiv_id: str) -> httpx.Response:
+    response = httpx.get(
+        f"{_EXPORT}/api/query",
+        params={"id_list": arxiv_id},
+        timeout=30,
+        follow_redirects=True,
+    )
+    response.raise_for_status()
+    return response
+
+
+def fetch_metadata(arxiv_id: str) -> Paper | None:
+    """Resolve an arXiv ID to a Paper via the Atom export API.
+
+    Authoritative metadata source for arXiv preprints — used as an ingest
+    fallback when OpenAlex 404s on an arXiv id (DQ-2). The arXiv id (bare,
+    e.g. ``"1911.00966"`` or legacy ``"math/0603097"``) becomes ``Paper.id``;
+    ``openalex_id`` and ``bibkey`` are left for the caller to populate.
+
+    Returns
+    -------
+    Paper | None
+        Populated Paper (title, authors, year, abstract), or None if arXiv
+        has no entry for the id.
+    """
+    bare = arxiv_id[len("arxiv:") :] if arxiv_id.lower().startswith("arxiv:") else arxiv_id
+    try:
+        response = _query(bare)
+    except httpx.HTTPError:
+        logger.warning("arXiv metadata fetch failed for %s", bare, exc_info=True)
+        return None
+    try:
+        root = ET.fromstring(response.text)
+    except ET.ParseError:
+        return None
+    entry = root.find(f"{_ATOM}entry")
+    if entry is None:
+        return None
+    # An id-not-found query still returns a feed but with no <entry>.
+    title = (entry.findtext(f"{_ATOM}title") or "").strip()
+    if not title:
+        return None
+    summary = (entry.findtext(f"{_ATOM}summary") or "").strip()
+    published = entry.findtext(f"{_ATOM}published") or ""
+    year = int(published[:4]) if published[:4].isdigit() else None
+    authors = [
+        name.strip()
+        for a in entry.findall(f"{_ATOM}author")
+        if (name := a.findtext(f"{_ATOM}name")) and name.strip()
+    ]
+    return Paper(
+        id=bare,
+        title=" ".join(title.split()),
+        authors=authors,
+        year=year,
+        abstract=" ".join(summary.split()) or None,
+    )
 
 
 def fetch_source(arxiv_id: str) -> str | None:

codex/sources/crossref.py

@@ -0,0 +1,91 @@
+"""Crossref API client.
+
+Provides:
+- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI.
+
+Crossref is a *third* metadata source after OpenAlex and Semantic Scholar
+(DQ-2 / roadmap R-B). Requests use the Polite Pool (mailto query parameter)
+and are retried on 429/5xx with exponential back-off via tenacity.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+import httpx
+from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
+
+from codex.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+_BASE = "https://api.crossref.org"
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code == 429 or exc.response.status_code >= 500
+    return False
+
+
+def _polite_params() -> dict[str, str]:
+    # Reuse the OpenAlex mailto for Crossref's Polite Pool (same address).
+    mailto = get_settings().openalex_mailto
+    return {"mailto": mailto} if mailto else {}
+
+
+@retry(
+    retry=retry_if_exception(_is_retryable),
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(min=1, max=30),
+    before_sleep=lambda rs: logger.warning(
+        "Crossref retry %d after %s",
+        rs.attempt_number,
+        rs.outcome.exception(),  # type: ignore[union-attr]
+    ),
+)
+def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+    merged: dict[str, str] = {**_polite_params(), **(params or {})}
+    response = httpx.get(url, params=merged, timeout=30, follow_redirects=True)
+    response.raise_for_status()
+    return response
+
+
+def _strip_jats(abstract: str | None) -> str | None:
+    """Strip JATS/XML tags from a Crossref abstract and collapse whitespace."""
+    if not abstract:
+        return None
+    text = _TAG_RE.sub("", abstract)
+    text = " ".join(text.split()).strip()
+    return text or None
+
+
+def fetch_abstract(doi: str) -> str | None:
+    """Fetch a work's abstract from Crossref by DOI.
+
+    Crossref stores abstracts as JATS-tagged markup (``<jats:p>…</jats:p>``);
+    tags are stripped before returning. Many works (notably book chapters)
+    have no abstract deposited — returns None in that case and on 404.
+
+    Parameters
+    ----------
+    doi:
+        A bare DOI (``"10.1007/s00454-019-00132-8"``).
+
+    Returns
+    -------
+    str | None
+        The plain-text abstract, or None when absent.
+    """
+    url = f"{_BASE}/works/{doi}"
+    try:
+        response = _get(url)
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return None
+        raise
+    message: dict[str, Any] = response.json().get("message", {})
+    return _strip_jats(message.get("abstract"))

codex/sources/semanticscholar.py

@@ -107,6 +107,24 @@ def fetch_references(paper_id: str) -> list[Citation]:
     return citations
 
 
+def fetch_abstract(paper_id: str) -> str | None:
+    """Fetch just the abstract for a paper from Semantic Scholar.
+
+    Used as an ingest abstract supplement (DQ-2) when OpenAlex has the paper
+    but no abstract. ``paper_id`` should be namespaced (``arXiv:…`` / ``DOI:…``).
+    Returns None on 404 or when S2 has no abstract.
+    """
+    url = f"{_BASE_GRAPH}/paper/{paper_id}"
+    try:
+        response = _get(url, params={"fields": "abstract"})
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return None
+        raise
+    abstract = response.json().get("abstract")
+    return abstract or None
+
+
 def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]:
     """Fetch recommended paper IDs from Semantic Scholar.