fix(sources): correct OpenAlex ID prefixing, citations endpoint, S2 rate-limit
Review-Gate findings:
- openalex: bare DOIs/arXiv IDs need doi:/arxiv: prefix (bare IDs 404);
add _resolve_id(); fix fetch_citations to use referenced_works field
instead of non-existent /works/{id}/references endpoint.
- semanticscholar: wait_fixed(1) was inter-retry only; add per-request
monotonic rate-limiter (_rate_limit()) before each httpx call.
Add _is_retryable() filter so 404s don't burn 5 retry slots.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -24,6 +24,20 @@ logger = logging.getLogger(__name__)
|
||||
_BASE = "https://api.openalex.org"
|
||||
|
||||
|
||||
def _resolve_id(id: str) -> str:
|
||||
"""Prefix a bare DOI or arXiv ID for the OpenAlex /works/ endpoint.
|
||||
|
||||
OpenAlex accepts: doi:<doi>, arxiv:<id>, W-IDs, or full https:// URLs.
|
||||
Bare DOIs (starting with "10.") and bare arXiv IDs require a prefix.
|
||||
"""
|
||||
s = id.strip()
|
||||
if s.startswith(("W", "https://", "doi:", "arxiv:")):
|
||||
return s
|
||||
if s.startswith("10."):
|
||||
return f"doi:{s}"
|
||||
return f"arxiv:{s}"
|
||||
|
||||
|
||||
def _is_retryable(exc: BaseException) -> bool:
|
||||
"""Return True for HTTP 429 / 5xx responses wrapped in httpx.HTTPStatusError."""
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
@@ -88,14 +102,15 @@ def fetch_paper(id: str) -> Paper | None:
|
||||
Parameters
|
||||
----------
|
||||
id:
|
||||
An arXiv ID (``"2301.07041"``) or a DOI (``"10.1145/…"``).
|
||||
An arXiv ID (``"2301.07041"``), a DOI (``"10.1145/…"``),
|
||||
or an OpenAlex W-ID (``"W2741809807"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
Paper | None
|
||||
A populated Paper instance, or None on 404.
|
||||
"""
|
||||
url = f"{_BASE}/works/{id}"
|
||||
url = f"{_BASE}/works/{_resolve_id(id)}"
|
||||
try:
|
||||
response = _get(url)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
@@ -109,17 +124,20 @@ def fetch_paper(id: str) -> Paper | None:
|
||||
def fetch_citations(openalex_id: str) -> list[Citation]:
|
||||
"""Fetch the reference list for a paper as Citation objects.
|
||||
|
||||
Uses the ``referenced_works`` field from the work object — the
|
||||
``/works/{id}/references`` path is not a real OpenAlex endpoint.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
openalex_id:
|
||||
The OpenAlex work ID (``"https://openalex.org/W…"`` or short ``"W…"``).
|
||||
The OpenAlex work ID (``"https://openalex.org/W…"`` or ``"W…"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Citation]
|
||||
One Citation per referenced work.
|
||||
One Citation per referenced work (OpenAlex W-ID as cited_id).
|
||||
"""
|
||||
url = f"{_BASE}/works/{openalex_id}/references"
|
||||
url = f"{_BASE}/works/{_resolve_id(openalex_id)}"
|
||||
try:
|
||||
response = _get(url)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
@@ -127,10 +145,10 @@ def fetch_citations(openalex_id: str) -> list[Citation]:
|
||||
return []
|
||||
raise
|
||||
|
||||
results: list[dict[str, Any]] = response.json().get("results", [])
|
||||
citations: list[Citation] = []
|
||||
for ref in results:
|
||||
cited_id: str = ref.get("doi") or ref.get("id") or ""
|
||||
if cited_id:
|
||||
citations.append(Citation(citing_id=openalex_id, cited_id=cited_id))
|
||||
return citations
|
||||
data: dict[str, Any] = response.json()
|
||||
referenced_works: list[str] = data.get("referenced_works", [])
|
||||
return [
|
||||
Citation(citing_id=openalex_id, cited_id=cited_id)
|
||||
for cited_id in referenced_works
|
||||
if cited_id
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user