fix(sources): correct OpenAlex ID prefixing, citations endpoint, S2 rate-limit

Review-Gate findings:
- openalex: bare DOIs/arXiv IDs need doi:/arxiv: prefix (bare IDs 404);
  add _resolve_id(); fix fetch_citations to use referenced_works field
  instead of non-existent /works/{id}/references endpoint.
- semanticscholar: wait_fixed(1) was inter-retry only; add per-request
  monotonic rate-limiter (_rate_limit()) before each httpx call.
  Add _is_retryable() filter so 404s don't burn 5 retry slots.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-04 23:49:35 +02:00
parent b7f19b6e2c
commit 52737abe3d
3 changed files with 125 additions and 30 deletions

View File

@@ -4,16 +4,19 @@ Provides:
- fetch_references: retrieve references for a paper as Citation dataclasses.
- fetch_recommendations: retrieve recommended paper IDs.
Requests are rate-limited to 1 per second via tenacity wait_fixed.
Rate-limited to 1 req/s (per-request floor via monotonic clock).
Retried on 429/5xx with exponential back-off via tenacity.
"""
from __future__ import annotations
import logging
import threading
import time
from typing import Any
import httpx
from tenacity import retry, stop_after_attempt, wait_fixed
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
from codex.models import Citation
@@ -22,10 +25,32 @@ logger = logging.getLogger(__name__)
_BASE_GRAPH = "https://api.semanticscholar.org/graph/v1"
_BASE_RECS = "https://api.semanticscholar.org/recommendations/v1"
# Per-request rate limit: ≤1 req/s without an API key.
_rate_lock = threading.Lock()
_last_request_time: float = 0.0
_MIN_INTERVAL = 1.0
def _rate_limit() -> None:
global _last_request_time
with _rate_lock:
now = time.monotonic()
wait = _MIN_INTERVAL - (now - _last_request_time)
if wait > 0:
time.sleep(wait)
_last_request_time = time.monotonic()
def _is_retryable(exc: BaseException) -> bool:
if isinstance(exc, httpx.HTTPStatusError):
return exc.response.status_code == 429 or exc.response.status_code >= 500
return False
@retry(
retry=retry_if_exception(_is_retryable),
stop=stop_after_attempt(5),
wait=wait_fixed(1),
wait=wait_exponential(min=1, max=30),
before_sleep=lambda rs: logger.warning(
"SemanticScholar retry %d after %s",
rs.attempt_number,
@@ -33,6 +58,7 @@ _BASE_RECS = "https://api.semanticscholar.org/recommendations/v1"
),
)
def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response:
_rate_limit()
response = httpx.get(url, params=params, timeout=30)
response.raise_for_status()
return response
@@ -70,7 +96,10 @@ def fetch_references(paper_id: str) -> list[Citation]:
context: str | None = contexts[0] if contexts else None
cited_id: str = (
external_ids.get("DOI") or external_ids.get("ArXiv") or cited_paper.get("paperId") or ""
external_ids.get("DOI")
or external_ids.get("ArXiv")
or cited_paper.get("paperId")
or ""
)
if cited_id:
citations.append(Citation(citing_id=paper_id, cited_id=cited_id, context=context))