fix(sources): correct OpenAlex ID prefixing, citations endpoint, S2 rate-limit
Review-Gate findings:
- openalex: bare DOIs/arXiv IDs need doi:/arxiv: prefix (bare IDs 404);
add _resolve_id(); fix fetch_citations to use referenced_works field
instead of non-existent /works/{id}/references endpoint.
- semanticscholar: wait_fixed(1) was inter-retry only; add per-request
monotonic rate-limiter (_rate_limit()) before each httpx call.
Add _is_retryable() filter so 404s don't burn 5 retry slots.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import pytest
|
||||
|
||||
from codex.models import Citation, Paper
|
||||
from codex.sources import openalex
|
||||
from codex.sources.openalex import _resolve_id
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -27,14 +28,33 @@ _SAMPLE_WORK = {
|
||||
"is": [2],
|
||||
"great": [3],
|
||||
},
|
||||
"referenced_works": [
|
||||
"https://openalex.org/W1",
|
||||
"https://openalex.org/W2",
|
||||
],
|
||||
}
|
||||
|
||||
_SAMPLE_REFS = {
|
||||
"results": [
|
||||
{"doi": "10.1000/ref1", "id": "https://openalex.org/W1"},
|
||||
{"doi": "10.1000/ref2", "id": "https://openalex.org/W2"},
|
||||
]
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _resolve_id
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_doi() -> None:
|
||||
assert _resolve_id("10.1145/3592430") == "doi:10.1145/3592430"
|
||||
|
||||
|
||||
def test_resolve_arxiv() -> None:
|
||||
assert _resolve_id("2301.07041") == "arxiv:2301.07041"
|
||||
|
||||
|
||||
def test_resolve_w_id_unchanged() -> None:
|
||||
assert _resolve_id("W2741809807") == "W2741809807"
|
||||
|
||||
|
||||
def test_resolve_prefixed_unchanged() -> None:
|
||||
assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x"
|
||||
assert _resolve_id("arxiv:2301.07041") == "arxiv:2301.07041"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -46,6 +66,7 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Successful fetch maps all fields to Paper correctly."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
assert "doi:10.1145/3592430" in url
|
||||
return httpx.Response(200, json=_SAMPLE_WORK)
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", mock_get)
|
||||
@@ -61,6 +82,20 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert paper.openalex_id == "https://openalex.org/W2741809807"
|
||||
|
||||
|
||||
def test_fetch_paper_arxiv_id_prefixed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Bare arXiv IDs are prefixed with 'arxiv:' in the URL."""
|
||||
called_url: list[str] = []
|
||||
|
||||
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
called_url.append(url)
|
||||
return httpx.Response(200, json=_SAMPLE_WORK)
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", mock_get)
|
||||
openalex.fetch_paper("2301.07041")
|
||||
|
||||
assert called_url and "arxiv:2301.07041" in called_url[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_paper — 404 → None
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -88,7 +123,7 @@ def test_fetch_paper_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
|
||||
def test_fetch_paper_retry_on_429(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""On 429 x3 then 200, should return 1 Paper after 3 failed attempts."""
|
||||
"""On 429×3 then 200, returns 1 Paper after 3 failed attempts."""
|
||||
attempt = [0]
|
||||
|
||||
def controlled_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
@@ -104,7 +139,6 @@ def test_fetch_paper_retry_on_429(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", controlled_get)
|
||||
|
||||
# Call fetch_paper in a loop to simulate retry behaviour
|
||||
result: Paper | None = None
|
||||
errors = 0
|
||||
for _ in range(5):
|
||||
@@ -116,19 +150,19 @@ def test_fetch_paper_retry_on_429(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
assert result is not None
|
||||
assert isinstance(result, Paper)
|
||||
assert errors == 3 # 3 failures before success
|
||||
assert errors == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_citations
|
||||
# fetch_citations — uses referenced_works field
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_fetch_citations(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""fetch_citations returns one Citation per reference entry."""
|
||||
"""fetch_citations reads referenced_works from the work object."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json=_SAMPLE_REFS)
|
||||
return httpx.Response(200, json=_SAMPLE_WORK)
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", mock_get)
|
||||
|
||||
@@ -138,5 +172,19 @@ def test_fetch_citations(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert len(citations) == 2
|
||||
assert all(isinstance(c, Citation) for c in citations)
|
||||
assert citations[0].citing_id == oa_id
|
||||
assert citations[0].cited_id == "10.1000/ref1"
|
||||
assert citations[1].cited_id == "10.1000/ref2"
|
||||
assert citations[0].cited_id == "https://openalex.org/W1"
|
||||
assert citations[1].cited_id == "https://openalex.org/W2"
|
||||
|
||||
|
||||
def test_fetch_citations_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""404 on the work fetch returns an empty list."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
raise httpx.HTTPStatusError(
|
||||
"Not Found",
|
||||
request=httpx.Request("GET", url),
|
||||
response=httpx.Response(404, request=httpx.Request("GET", url)),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", mock_get)
|
||||
assert openalex.fetch_citations("W999") == []
|
||||
|
||||
Reference in New Issue
Block a user