feat(sources): OpenAlex, SemanticScholar, arXiv API clients

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tests/sources/test_arxiv.py

@@ -0,0 +1,140 @@
+"""Tests for codex.sources.arxiv."""
+
+from __future__ import annotations
+
+import io
+import tarfile
+
+import httpx
+import pytest
+
+from codex.sources import arxiv
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_tar_gz(files: dict[str, bytes]) -> bytes:
+    """Build an in-memory .tar.gz archive from a dict of {name: content}."""
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w:gz") as tf:
+        for name, content in files.items():
+            info = tarfile.TarInfo(name=name)
+            info.size = len(content)
+            tf.addfile(info, io.BytesIO(content))
+    return buf.getvalue()
+
+
+# ---------------------------------------------------------------------------
+# fetch_source — success: in-memory tar.gz with one .tex
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_source_returns_latex(monkeypatch: pytest.MonkeyPatch) -> None:
+    """fetch_source extracts and returns the LaTeX content from a .tar.gz."""
+    latex_content = b"\\documentclass{article}\n\\begin{document}\nHello world\n\\end{document}"
+    tar_bytes = _make_tar_gz({"main.tex": latex_content})
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("2301.07041")
+
+    assert result is not None
+    assert "\\documentclass" in result
+    assert "Hello world" in result
+
+
+# ---------------------------------------------------------------------------
+# fetch_source — 404 → None
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_source_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 response should return None."""
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(404)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("nonexistent-id")
+    assert result is None
+
+
+# ---------------------------------------------------------------------------
+# fetch_source — no .tex in archive → None
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_source_no_tex_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """If the archive contains no .tex files, return None."""
+    tar_bytes = _make_tar_gz({"README.md": b"# Paper\nNo LaTeX here."})
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("2301.07041")
+    assert result is None
+
+
+# ---------------------------------------------------------------------------
+# fetch_source — fallback to largest .tex when no \documentclass
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_source_fallback_to_largest_tex(monkeypatch: pytest.MonkeyPatch) -> None:
+    """When no file has \\documentclass, the largest .tex is returned."""
+    small_tex = b"% small file\n\\section{Intro}"
+    large_tex = b"% large file\n" + b"x" * 500
+
+    tar_bytes = _make_tar_gz({"small.tex": small_tex, "large.tex": large_tex})
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("2301.07041")
+
+    assert result is not None
+    assert "large file" in result
+
+
+# ---------------------------------------------------------------------------
+# fetch_pdf_url — pure computation, no mock needed
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_pdf_url_pure_computation() -> None:
+    """fetch_pdf_url returns the correct URL without making any HTTP request."""
+    url = arxiv.fetch_pdf_url("2301.07041")
+    assert url == "https://arxiv.org/pdf/2301.07041.pdf"
+
+    url2 = arxiv.fetch_pdf_url("1234.56789")
+    assert url2 == "https://arxiv.org/pdf/1234.56789.pdf"

tests/sources/test_openalex.py

@@ -0,0 +1,142 @@
+"""Tests for codex.sources.openalex."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from codex.models import Citation, Paper
+from codex.sources import openalex
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_SAMPLE_WORK = {
+    "id": "https://openalex.org/W2741809807",
+    "doi": "10.1145/3592430",
+    "title": "Conformal Prediction: A Review",
+    "publication_year": 2023,
+    "authorships": [
+        {"author": {"display_name": "Alice Smith"}},
+        {"author": {"display_name": "Bob Jones"}},
+    ],
+    "abstract_inverted_index": {
+        "Conformal": [0],
+        "prediction": [1],
+        "is": [2],
+        "great": [3],
+    },
+}
+
+_SAMPLE_REFS = {
+    "results": [
+        {"doi": "10.1000/ref1", "id": "https://openalex.org/W1"},
+        {"doi": "10.1000/ref2", "id": "https://openalex.org/W2"},
+    ]
+}
+
+
+# ---------------------------------------------------------------------------
+# fetch_paper — success
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Successful fetch maps all fields to Paper correctly."""
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=_SAMPLE_WORK)
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+
+    paper = openalex.fetch_paper("10.1145/3592430")
+
+    assert paper is not None
+    assert isinstance(paper, Paper)
+    assert paper.title == "Conformal Prediction: A Review"
+    assert paper.year == 2023
+    assert paper.authors == ["Alice Smith", "Bob Jones"]
+    assert paper.abstract == "Conformal prediction is great"
+    assert paper.openalex_id == "https://openalex.org/W2741809807"
+
+
+# ---------------------------------------------------------------------------
+# fetch_paper — 404 → None
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_paper_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 response should return None without raising."""
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "Not Found",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+
+    result = openalex.fetch_paper("nonexistent-id")
+    assert result is None
+
+
+# ---------------------------------------------------------------------------
+# fetch_paper — 429 retry then success
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_paper_retry_on_429(monkeypatch: pytest.MonkeyPatch) -> None:
+    """On 429 x3 then 200, should return 1 Paper after 3 failed attempts."""
+    attempt = [0]
+
+    def controlled_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        attempt[0] += 1
+        if attempt[0] <= 3:
+            resp = httpx.Response(429, request=httpx.Request("GET", url))
+            raise httpx.HTTPStatusError(
+                "Too Many Requests",
+                request=httpx.Request("GET", url),
+                response=resp,
+            )
+        return httpx.Response(200, json=_SAMPLE_WORK)
+
+    monkeypatch.setattr(openalex, "_get", controlled_get)
+
+    # Call fetch_paper in a loop to simulate retry behaviour
+    result: Paper | None = None
+    errors = 0
+    for _ in range(5):
+        try:
+            result = openalex.fetch_paper("10.1145/3592430")
+            break
+        except httpx.HTTPStatusError:
+            errors += 1
+
+    assert result is not None
+    assert isinstance(result, Paper)
+    assert errors == 3  # 3 failures before success
+
+
+# ---------------------------------------------------------------------------
+# fetch_citations
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_citations(monkeypatch: pytest.MonkeyPatch) -> None:
+    """fetch_citations returns one Citation per reference entry."""
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=_SAMPLE_REFS)
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+
+    oa_id = "https://openalex.org/W2741809807"
+    citations = openalex.fetch_citations(oa_id)
+
+    assert len(citations) == 2
+    assert all(isinstance(c, Citation) for c in citations)
+    assert citations[0].citing_id == oa_id
+    assert citations[0].cited_id == "10.1000/ref1"
+    assert citations[1].cited_id == "10.1000/ref2"

tests/sources/test_semanticscholar.py

@@ -0,0 +1,126 @@
+"""Tests for codex.sources.semanticscholar."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from codex.models import Citation
+from codex.sources import semanticscholar
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_SAMPLE_REFS_RESPONSE = {
+    "data": [
+        {
+            "citedPaper": {
+                "paperId": "abc123",
+                "externalIds": {"DOI": "10.1000/xyz1"},
+            },
+            "contexts": ["This approach was first described in [1]."],
+        },
+        {
+            "citedPaper": {
+                "paperId": "def456",
+                "externalIds": {"ArXiv": "2301.07041"},
+            },
+            "contexts": [],
+        },
+    ]
+}
+
+_SAMPLE_RECS_RESPONSE = {
+    "recommendedPapers": [
+        {"paperId": "rec_id_1"},
+        {"paperId": "rec_id_2"},
+        {"paperId": "rec_id_3"},
+    ]
+}
+
+
+# ---------------------------------------------------------------------------
+# fetch_references
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_references_returns_citations(monkeypatch: pytest.MonkeyPatch) -> None:
+    """fetch_references maps API response to Citation objects correctly."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=_SAMPLE_REFS_RESPONSE)
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+
+    citations = semanticscholar.fetch_references("abc123")
+
+    assert len(citations) == 2
+    assert all(isinstance(c, Citation) for c in citations)
+
+    # First citation uses DOI and has context
+    assert citations[0].citing_id == "abc123"
+    assert citations[0].cited_id == "10.1000/xyz1"
+    assert citations[0].context == "This approach was first described in [1]."
+
+    # Second citation falls back to ArXiv ID, no context
+    assert citations[1].citing_id == "abc123"
+    assert citations[1].cited_id == "2301.07041"
+    assert citations[1].context is None
+
+
+def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 response should return an empty list."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "Not Found",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+
+    result = semanticscholar.fetch_references("nonexistent")
+    assert result == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_recommendations
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_recommendations_returns_list_of_str(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """fetch_recommendations returns a flat list of paperId strings."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=_SAMPLE_RECS_RESPONSE)
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+
+    result = semanticscholar.fetch_recommendations("abc123", limit=3)
+
+    assert isinstance(result, list)
+    assert len(result) == 3
+    assert all(isinstance(r, str) for r in result)
+    assert result == ["rec_id_1", "rec_id_2", "rec_id_3"]
+
+
+def test_fetch_recommendations_404_returns_empty(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A 404 response should return an empty list."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "Not Found",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+
+    result = semanticscholar.fetch_recommendations("nonexistent")
+    assert result == []