82cea2a33b
Add crossref.fetch_references(doi), which parses message.reference[].DOI into Citations (DOI-less book/unstructured refs are skipped — they cannot join the graph). Wire it as the third leg of the ingest citation fallback chain: OpenAlex-empty -> S2 -> Crossref, for DOI papers only (Crossref's works endpoint is DOI-keyed). Cited DOIs are normalized to the canonical bare lower-case form via _norm_cited_id. As a fallback it fires only when OpenAlex and S2 both return no references, so it adds a forward-looking third source without changing the already-covered corpus. Tests: fetch_references unit tests (DOI edges / unstructured skipped / no refs / 404) and ingest tests for both fallback directions. Full suite 361 passed; ruff + mypy clean. Live-validated: 33/40/24 refs for Springer/ACM/Wiley DOIs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
127 lines
4.1 KiB
Python
127 lines
4.1 KiB
Python
"""Crossref API client.
|
|
|
|
Provides:
|
|
- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI.
|
|
- fetch_references: retrieve a work's reference list (DOI edges) by DOI.
|
|
|
|
Crossref is a *third* metadata source after OpenAlex and Semantic Scholar
|
|
(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto
|
|
query parameter) and are retried on 429/5xx with exponential back-off via tenacity.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
|
|
|
from codex.config import get_settings
|
|
from codex.models import Citation
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BASE = "https://api.crossref.org"
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
|
|
|
|
def _is_retryable(exc: BaseException) -> bool:
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return exc.response.status_code == 429 or exc.response.status_code >= 500
|
|
return False
|
|
|
|
|
|
def _polite_params() -> dict[str, str]:
|
|
# Reuse the OpenAlex mailto for Crossref's Polite Pool (same address).
|
|
mailto = get_settings().openalex_mailto
|
|
return {"mailto": mailto} if mailto else {}
|
|
|
|
|
|
@retry(
|
|
retry=retry_if_exception(_is_retryable),
|
|
stop=stop_after_attempt(5),
|
|
wait=wait_exponential(min=1, max=30),
|
|
before_sleep=lambda rs: logger.warning(
|
|
"Crossref retry %d after %s",
|
|
rs.attempt_number,
|
|
rs.outcome.exception(), # type: ignore[union-attr]
|
|
),
|
|
)
|
|
def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
|
merged: dict[str, str] = {**_polite_params(), **(params or {})}
|
|
response = httpx.get(url, params=merged, timeout=30, follow_redirects=True)
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
|
|
def _strip_jats(abstract: str | None) -> str | None:
|
|
"""Strip JATS/XML tags from a Crossref abstract and collapse whitespace."""
|
|
if not abstract:
|
|
return None
|
|
text = _TAG_RE.sub("", abstract)
|
|
text = " ".join(text.split()).strip()
|
|
return text or None
|
|
|
|
|
|
def fetch_abstract(doi: str) -> str | None:
|
|
"""Fetch a work's abstract from Crossref by DOI.
|
|
|
|
Crossref stores abstracts as JATS-tagged markup (``<jats:p>…</jats:p>``);
|
|
tags are stripped before returning. Many works (notably book chapters)
|
|
have no abstract deposited — returns None in that case and on 404.
|
|
|
|
Parameters
|
|
----------
|
|
doi:
|
|
A bare DOI (``"10.1007/s00454-019-00132-8"``).
|
|
|
|
Returns
|
|
-------
|
|
str | None
|
|
The plain-text abstract, or None when absent.
|
|
"""
|
|
url = f"{_BASE}/works/{doi}"
|
|
try:
|
|
response = _get(url)
|
|
except httpx.HTTPStatusError as exc:
|
|
if exc.response.status_code == 404:
|
|
return None
|
|
raise
|
|
message: dict[str, Any] = response.json().get("message", {})
|
|
return _strip_jats(message.get("abstract"))
|
|
|
|
|
|
def fetch_references(doi: str) -> list[Citation]:
|
|
"""Fetch a work's reference list from Crossref by DOI (roadmap R-B).
|
|
|
|
Crossref carries publisher-deposited reference lists in ``message.reference``.
|
|
Each entry that cites a DOI-registered work exposes a bare ``DOI`` field;
|
|
only those become citation-graph edges (book / older references carry just
|
|
``unstructured`` text with no resolvable id and are skipped). ``citing_id``
|
|
is the queried *doi* — the caller rewrites it to the canonical ``papers.id``
|
|
and normalises the cited DOIs (mirrors the Semantic Scholar reference path).
|
|
|
|
Parameters
|
|
----------
|
|
doi:
|
|
A bare DOI (``"10.1007/s00454-019-00132-8"``).
|
|
|
|
Returns
|
|
-------
|
|
list[Citation]
|
|
One Citation per reference that carries a DOI. Empty on 404 or when no
|
|
references are deposited for the work.
|
|
"""
|
|
url = f"{_BASE}/works/{doi}"
|
|
try:
|
|
response = _get(url)
|
|
except httpx.HTTPStatusError as exc:
|
|
if exc.response.status_code == 404:
|
|
return []
|
|
raise
|
|
message: dict[str, Any] = response.json().get("message", {})
|
|
references: list[dict[str, Any]] = message.get("reference") or []
|
|
return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")]
|