fix(sources): correct OpenAlex ID prefixing, citations endpoint, S2 rate-limit

Review-Gate findings:
- openalex: bare DOIs/arXiv IDs need doi:/arxiv: prefix (bare IDs 404);
  add _resolve_id(); fix fetch_citations to use referenced_works field
  instead of non-existent /works/{id}/references endpoint.
- semanticscholar: wait_fixed(1) was inter-retry only; add per-request
  monotonic rate-limiter (_rate_limit()) before each httpx call.
  Add _is_retryable() filter so 404s don't burn 5 retry slots.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-04 23:49:35 +02:00
parent b7f19b6e2c
commit 52737abe3d
3 changed files with 125 additions and 30 deletions

View File

@@ -24,6 +24,20 @@ logger = logging.getLogger(__name__)
_BASE = "https://api.openalex.org"
def _resolve_id(id: str) -> str:
"""Prefix a bare DOI or arXiv ID for the OpenAlex /works/ endpoint.
OpenAlex accepts: doi:<doi>, arxiv:<id>, W-IDs, or full https:// URLs.
Bare DOIs (starting with "10.") and bare arXiv IDs require a prefix.
"""
s = id.strip()
if s.startswith(("W", "https://", "doi:", "arxiv:")):
return s
if s.startswith("10."):
return f"doi:{s}"
return f"arxiv:{s}"
def _is_retryable(exc: BaseException) -> bool:
"""Return True for HTTP 429 / 5xx responses wrapped in httpx.HTTPStatusError."""
if isinstance(exc, httpx.HTTPStatusError):
@@ -88,14 +102,15 @@ def fetch_paper(id: str) -> Paper | None:
Parameters
----------
id:
An arXiv ID (``"2301.07041"``) or a DOI (``"10.1145/…"``).
An arXiv ID (``"2301.07041"``), a DOI (``"10.1145/…"``),
or an OpenAlex W-ID (``"W2741809807"``).
Returns
-------
Paper | None
A populated Paper instance, or None on 404.
"""
url = f"{_BASE}/works/{id}"
url = f"{_BASE}/works/{_resolve_id(id)}"
try:
response = _get(url)
except httpx.HTTPStatusError as exc:
@@ -109,17 +124,20 @@ def fetch_paper(id: str) -> Paper | None:
def fetch_citations(openalex_id: str) -> list[Citation]:
"""Fetch the reference list for a paper as Citation objects.
Uses the ``referenced_works`` field from the work object — the
``/works/{id}/references`` path is not a real OpenAlex endpoint.
Parameters
----------
openalex_id:
The OpenAlex work ID (``"https://openalex.org/W…"`` or short ``"W…"``).
The OpenAlex work ID (``"https://openalex.org/W…"`` or ``"W…"``).
Returns
-------
list[Citation]
One Citation per referenced work.
One Citation per referenced work (OpenAlex W-ID as cited_id).
"""
url = f"{_BASE}/works/{openalex_id}/references"
url = f"{_BASE}/works/{_resolve_id(openalex_id)}"
try:
response = _get(url)
except httpx.HTTPStatusError as exc:
@@ -127,10 +145,10 @@ def fetch_citations(openalex_id: str) -> list[Citation]:
return []
raise
results: list[dict[str, Any]] = response.json().get("results", [])
citations: list[Citation] = []
for ref in results:
cited_id: str = ref.get("doi") or ref.get("id") or ""
if cited_id:
citations.append(Citation(citing_id=openalex_id, cited_id=cited_id))
return citations
data: dict[str, Any] = response.json()
referenced_works: list[str] = data.get("referenced_works", [])
return [
Citation(citing_id=openalex_id, cited_id=cited_id)
for cited_id in referenced_works
if cited_id
]