fix(sources): correct OpenAlex ID prefixing, citations endpoint, S2 rate-limit
Review-Gate findings:
- openalex: bare DOIs/arXiv IDs need doi:/arxiv: prefix (bare IDs 404);
add _resolve_id(); fix fetch_citations to use referenced_works field
instead of non-existent /works/{id}/references endpoint.
- semanticscholar: wait_fixed(1) was inter-retry only; add per-request
monotonic rate-limiter (_rate_limit()) before each httpx call.
Add _is_retryable() filter so 404s don't burn 5 retry slots.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,16 +4,19 @@ Provides:
|
||||
- fetch_references: retrieve references for a paper as Citation dataclasses.
|
||||
- fetch_recommendations: retrieve recommended paper IDs.
|
||||
|
||||
Requests are rate-limited to 1 per second via tenacity wait_fixed.
|
||||
Rate-limited to ≤1 req/s (per-request floor via monotonic clock).
|
||||
Retried on 429/5xx with exponential back-off via tenacity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
||||
|
||||
from codex.models import Citation
|
||||
|
||||
@@ -22,10 +25,32 @@ logger = logging.getLogger(__name__)
|
||||
_BASE_GRAPH = "https://api.semanticscholar.org/graph/v1"
|
||||
_BASE_RECS = "https://api.semanticscholar.org/recommendations/v1"
|
||||
|
||||
# Per-request rate limit: ≤1 req/s without an API key.
|
||||
_rate_lock = threading.Lock()
|
||||
_last_request_time: float = 0.0
|
||||
_MIN_INTERVAL = 1.0
|
||||
|
||||
|
||||
def _rate_limit() -> None:
|
||||
global _last_request_time
|
||||
with _rate_lock:
|
||||
now = time.monotonic()
|
||||
wait = _MIN_INTERVAL - (now - _last_request_time)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
_last_request_time = time.monotonic()
|
||||
|
||||
|
||||
def _is_retryable(exc: BaseException) -> bool:
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return exc.response.status_code == 429 or exc.response.status_code >= 500
|
||||
return False
|
||||
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception(_is_retryable),
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_fixed(1),
|
||||
wait=wait_exponential(min=1, max=30),
|
||||
before_sleep=lambda rs: logger.warning(
|
||||
"SemanticScholar retry %d after %s",
|
||||
rs.attempt_number,
|
||||
@@ -33,6 +58,7 @@ _BASE_RECS = "https://api.semanticscholar.org/recommendations/v1"
|
||||
),
|
||||
)
|
||||
def _get(url: str, params: dict[str, Any] | None = None) -> httpx.Response:
|
||||
_rate_limit()
|
||||
response = httpx.get(url, params=params, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
@@ -70,7 +96,10 @@ def fetch_references(paper_id: str) -> list[Citation]:
|
||||
context: str | None = contexts[0] if contexts else None
|
||||
|
||||
cited_id: str = (
|
||||
external_ids.get("DOI") or external_ids.get("ArXiv") or cited_paper.get("paperId") or ""
|
||||
external_ids.get("DOI")
|
||||
or external_ids.get("ArXiv")
|
||||
or cited_paper.get("paperId")
|
||||
or ""
|
||||
)
|
||||
if cited_id:
|
||||
citations.append(Citation(citing_id=paper_id, cited_id=cited_id, context=context))
|
||||
|
||||
Reference in New Issue
Block a user