From f504ca62dba916fb0f1a93cbce488ef82d56cf82 Mon Sep 17 00:00:00 2001
From: Tarik Moussa <tarikmoussa@MacBookAir.fritz.box>
Date: Thu, 18 Jun 2026 04:25:22 +0200
Subject: [PATCH] fix(openalex): canonicalize arXiv DOI to bare arXiv id (DQ-5
 arXiv half)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

OpenAlex returns arXiv works under the arXiv DOI (10.48550/arXiv.<id>); _normalize_doi (DQ-5) left it as 10.48550/arxiv.<id>, but the corpus keys arXiv papers on the BARE id (2305.10988 / math/0603097). So re-ingesting an arXiv paper by its bare id missed INSERT ... ON CONFLICT (id) and tripped papers_openalex_id_key — the arXiv half of DQ-5, explicitly deferred there and surfaced again driving the R-C .tex re-ingest.

Add _canonical_id() (normalize DOI, then strip the 10.48550/arxiv. prefix) and use it in _map_paper. Regression tests (modern + legacy arXiv DOI, regular DOI unchanged, fetch_paper bare id). Confirmed live: re-ingesting 2305.10988 now UPDATEs in place. Full suite 370 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 codex/sources/openalex.py      | 23 +++++++++++++++++++-
 tests/sources/test_openalex.py | 38 +++++++++++++++++++++++++++++++++-
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/codex/sources/openalex.py b/codex/sources/openalex.py
index f9be139..2ad1073 100644
--- a/codex/sources/openalex.py
+++ b/codex/sources/openalex.py
@@ -105,6 +105,27 @@ def _normalize_doi(doi: str) -> str:
     return s
 
 
+# arXiv works are registered under a DOI of the form 10.48550/arXiv.<id>.
+_ARXIV_DOI_PREFIX = "10.48550/arxiv."
+
+
+def _canonical_id(doi: str) -> str:
+    """Return the canonical ``papers.id`` for an OpenAlex ``doi`` value.
+
+    Builds on :func:`_normalize_doi` (bare, lower-cased DOI), then strips the
+    arXiv DOI prefix ``10.48550/arxiv.`` so an arXiv work's id is the BARE arXiv
+    id (``2305.10988`` / ``math/0603097``) — the form the corpus is keyed on —
+    not the arXiv DOI. Without this, re-ingesting an arXiv paper by its bare id
+    misses ``INSERT … ON CONFLICT (id)`` and trips ``papers_openalex_id_key``.
+    This is the arXiv half of DQ-5 (the DOI half was fixed there; this was
+    explicitly deferred and surfaced again driving the R-C .tex re-ingest).
+    """
+    normalized = _normalize_doi(doi)
+    if normalized.startswith(_ARXIV_DOI_PREFIX):
+        return normalized[len(_ARXIV_DOI_PREFIX) :]
+    return normalized
+
+
 def _map_paper(data: dict[str, Any]) -> Paper:
     authors: list[str] = [
         a.get("author", {}).get("display_name") or "" for a in data.get("authorships", [])
@@ -112,7 +133,7 @@ def _map_paper(data: dict[str, Any]) -> Paper:
     abstract = _reconstruct_abstract(data.get("abstract_inverted_index"))
     doi = data.get("doi")
     return Paper(
-        id=_normalize_doi(doi) if doi else (data.get("id") or ""),
+        id=_canonical_id(doi) if doi else (data.get("id") or ""),
         title=data.get("title") or "",
         openalex_id=data.get("id"),
         authors=authors,
diff --git a/tests/sources/test_openalex.py b/tests/sources/test_openalex.py
index 6b1e9a2..82c329b 100644
--- a/tests/sources/test_openalex.py
+++ b/tests/sources/test_openalex.py
@@ -7,7 +7,7 @@ import pytest
 
 from codex.models import Citation, Paper
 from codex.sources import openalex
-from codex.sources.openalex import _normalize_doi, _resolve_id
+from codex.sources.openalex import _canonical_id, _normalize_doi, _resolve_id
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -97,6 +97,42 @@ def test_normalize_doi_already_bare_is_idempotent() -> None:
     assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8"
 
 
+# ---------------------------------------------------------------------------
+# _canonical_id — arXiv DOI → bare arXiv id (arXiv half of DQ-5)
+# ---------------------------------------------------------------------------
+
+
+def test_canonical_id_strips_arxiv_doi_modern() -> None:
+    # arXiv works carry the arXiv DOI; the canonical id is the bare arXiv id.
+    assert _canonical_id("https://doi.org/10.48550/arXiv.2305.10988") == "2305.10988"
+
+
+def test_canonical_id_strips_arxiv_doi_legacy() -> None:
+    assert _canonical_id("https://doi.org/10.48550/arXiv.math/0603097") == "math/0603097"
+
+
+def test_canonical_id_leaves_regular_doi_bare() -> None:
+    # A normal journal DOI is only normalized (bare + lower-case), not stripped.
+    assert _canonical_id("https://doi.org/10.1007/S00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_fetch_paper_arxiv_doi_yields_bare_id(monkeypatch: pytest.MonkeyPatch) -> None:
+    """An arXiv work maps to the BARE arXiv id (not the arXiv DOI), so re-ingesting
+    by arXiv id hits ON CONFLICT (id) instead of tripping papers_openalex_id_key."""
+    work = {**_SAMPLE_WORK, "doi": "https://doi.org/10.48550/arXiv.2305.10988"}
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=work)
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+    paper = openalex.fetch_paper("2305.10988")
+
+    assert paper is not None
+    assert paper.id == "2305.10988"
+
+
 def test_map_paper_falls_back_to_openalex_id_without_doi(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None: