"""Crossref API client. Provides: - fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI. - fetch_references: retrieve a work's reference list (DOI edges) by DOI. Crossref is a *third* metadata source after OpenAlex and Semantic Scholar (DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto query parameter) and are retried on 429/5xx with exponential back-off via tenacity. """ from __future__ import annotations import logging import re from typing import Any import httpx from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential from codex.config import get_settings from codex.models import Citation logger = logging.getLogger(__name__) _BASE = "https://api.crossref.org" _TAG_RE = re.compile(r"<[^>]+>") def _is_retryable(exc: BaseException) -> bool: if isinstance(exc, httpx.HTTPStatusError): return exc.response.status_code == 429 or exc.response.status_code >= 500 return False def _polite_params() -> dict[str, str]: # Reuse the OpenAlex mailto for Crossref's Polite Pool (same address). mailto = get_settings().openalex_mailto return {"mailto": mailto} if mailto else {} @retry( retry=retry_if_exception(_is_retryable), stop=stop_after_attempt(5), wait=wait_exponential(min=1, max=30), before_sleep=lambda rs: logger.warning( "Crossref retry %d after %s", rs.attempt_number, rs.outcome.exception(), # type: ignore[union-attr] ), ) def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response: merged: dict[str, str] = {**_polite_params(), **(params or {})} response = httpx.get(url, params=merged, timeout=30, follow_redirects=True) response.raise_for_status() return response def _strip_jats(abstract: str | None) -> str | None: """Strip JATS/XML tags from a Crossref abstract and collapse whitespace.""" if not abstract: return None text = _TAG_RE.sub("", abstract) text = " ".join(text.split()).strip() return text or None def fetch_abstract(doi: str) -> str | None: """Fetch a work's abstract from Crossref by DOI. Crossref stores abstracts as JATS-tagged markup (````); tags are stripped before returning. Many works (notably book chapters) have no abstract deposited — returns None in that case and on 404. Parameters ---------- doi: A bare DOI (``"10.1007/s00454-019-00132-8"``). Returns ------- str | None The plain-text abstract, or None when absent. """ url = f"{_BASE}/works/{doi}" try: response = _get(url) except httpx.HTTPStatusError as exc: if exc.response.status_code == 404: return None raise message: dict[str, Any] = response.json().get("message", {}) return _strip_jats(message.get("abstract")) def fetch_references(doi: str) -> list[Citation]: """Fetch a work's reference list from Crossref by DOI (roadmap R-B). Crossref carries publisher-deposited reference lists in ``message.reference``. Each entry that cites a DOI-registered work exposes a bare ``DOI`` field; only those become citation-graph edges (book / older references carry just ``unstructured`` text with no resolvable id and are skipped). ``citing_id`` is the queried *doi* — the caller rewrites it to the canonical ``papers.id`` and normalises the cited DOIs (mirrors the Semantic Scholar reference path). Parameters ---------- doi: A bare DOI (``"10.1007/s00454-019-00132-8"``). Returns ------- list[Citation] One Citation per reference that carries a DOI. Empty on 404 or when no references are deposited for the work. """ url = f"{_BASE}/works/{doi}" try: response = _get(url) except httpx.HTTPStatusError as exc: if exc.response.status_code == 404: return [] raise message: dict[str, Any] = response.json().get("message", {}) references: list[dict[str, Any]] = message.get("reference") or [] return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")]