ff03443af4
Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404 -> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in: - arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API (authoritative for preprints). Used as an ingest fallback on OpenAlex 404, replacing the empty stub. - crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool. - semanticscholar.fetch_abstract: fetch just the abstract field. - ingest.py: _recover_abstract() supplements an empty abstract from S2 then Crossref *before* embedding, so the paper gets a real (non-zero) vector. Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter 10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented limit. Corpus now has 1 paper without an abstract. Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers (openalex returns full-URL id vs the bare canonical id) -> re-ingest trips papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix. Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
92 lines
2.7 KiB
Python
92 lines
2.7 KiB
Python
"""Crossref API client.
|
|
|
|
Provides:
|
|
- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI.
|
|
|
|
Crossref is a *third* metadata source after OpenAlex and Semantic Scholar
|
|
(DQ-2 / roadmap R-B). Requests use the Polite Pool (mailto query parameter)
|
|
and are retried on 429/5xx with exponential back-off via tenacity.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
|
|
|
from codex.config import get_settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BASE = "https://api.crossref.org"
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
|
|
|
|
def _is_retryable(exc: BaseException) -> bool:
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return exc.response.status_code == 429 or exc.response.status_code >= 500
|
|
return False
|
|
|
|
|
|
def _polite_params() -> dict[str, str]:
|
|
# Reuse the OpenAlex mailto for Crossref's Polite Pool (same address).
|
|
mailto = get_settings().openalex_mailto
|
|
return {"mailto": mailto} if mailto else {}
|
|
|
|
|
|
@retry(
|
|
retry=retry_if_exception(_is_retryable),
|
|
stop=stop_after_attempt(5),
|
|
wait=wait_exponential(min=1, max=30),
|
|
before_sleep=lambda rs: logger.warning(
|
|
"Crossref retry %d after %s",
|
|
rs.attempt_number,
|
|
rs.outcome.exception(), # type: ignore[union-attr]
|
|
),
|
|
)
|
|
def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
|
merged: dict[str, str] = {**_polite_params(), **(params or {})}
|
|
response = httpx.get(url, params=merged, timeout=30, follow_redirects=True)
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
|
|
def _strip_jats(abstract: str | None) -> str | None:
|
|
"""Strip JATS/XML tags from a Crossref abstract and collapse whitespace."""
|
|
if not abstract:
|
|
return None
|
|
text = _TAG_RE.sub("", abstract)
|
|
text = " ".join(text.split()).strip()
|
|
return text or None
|
|
|
|
|
|
def fetch_abstract(doi: str) -> str | None:
|
|
"""Fetch a work's abstract from Crossref by DOI.
|
|
|
|
Crossref stores abstracts as JATS-tagged markup (``<jats:p>…</jats:p>``);
|
|
tags are stripped before returning. Many works (notably book chapters)
|
|
have no abstract deposited — returns None in that case and on 404.
|
|
|
|
Parameters
|
|
----------
|
|
doi:
|
|
A bare DOI (``"10.1007/s00454-019-00132-8"``).
|
|
|
|
Returns
|
|
-------
|
|
str | None
|
|
The plain-text abstract, or None when absent.
|
|
"""
|
|
url = f"{_BASE}/works/{doi}"
|
|
try:
|
|
response = _get(url)
|
|
except httpx.HTTPStatusError as exc:
|
|
if exc.response.status_code == 404:
|
|
return None
|
|
raise
|
|
message: dict[str, Any] = response.json().get("message", {})
|
|
return _strip_jats(message.get("abstract"))
|