fix(openalex): normalize DOI to bare canonical id so re-ingest is idempotent (DQ-5)

openalex._map_paper set Paper.id from the raw `doi` field, which OpenAlex
returns as a full URL (https://doi.org/10.x). The canonical papers.id (M-1
migration) is the bare, lower-cased DOI, so re-ingesting a DOI paper missed
ON CONFLICT (id) and tripped the separate papers_openalex_id_key constraint
(UniqueViolation).

Fix: _normalize_doi() strips https://doi.org//http://doi.org//doi: and
lower-cases; applied in the DOI branch of _map_paper only (W-id fallback
untouched). Also un-breaks _s2_id_for/Crossref recovery, which the URL form
silently defeated. Corrected the _SAMPLE_WORK fixture to the URL form (the bare
fixture hid the bug) + added _normalize_doi and re-ingest idempotency regression
tests.

Live verification: re-ingest of 10.1007/s00454-019-00132-8 (previously crashed)
now UPDATEs in place — single row, added_at unchanged, no stray row, citations
45->45. Unblocks roadmap R-A/R-C. Full suite 348 green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/ingest/test_ingest.py

@@ -10,6 +10,7 @@ from contextlib import contextmanager
 from typing import Any
 from unittest.mock import MagicMock, patch
 
+import httpx
 import numpy as np
 import pytest
 
@@ -592,6 +593,63 @@ def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None:
     assert upsert_params["bibkey"] == "AliceBob2023"
 
 
+def test_ingest_doi_paper_upserts_bare_canonical_id() -> None:
+    """DQ-5 regression: re-ingesting an existing DOI paper must be idempotent.
+
+    OpenAlex returns the ``doi`` as a full URL (and possibly mixed-case), but the
+    stored canonical ``papers.id`` is the BARE, lower-cased DOI (M-1 migration).
+    The papers upsert must therefore carry the BARE id into ``%(id)s`` so
+    ``ON CONFLICT (id)`` matches the existing row and UPDATEs it — rather than
+    attempting a fresh INSERT that trips the separate ``papers_openalex_id_key``
+    unique constraint (the UniqueViolation crash).
+
+    The real ``_map_paper``/``fetch_paper`` run here (only the HTTP ``_get``, the
+    embedder and the DB are mocked), so this exercises the actual normalization
+    through the ingest path. The DB is a MagicMock, so it cannot raise the real
+    constraint error; the live re-ingest of 10.1007/s00454-019-00132-8 is the
+    end-to-end UniqueViolation proof (see DATA-QUALITY-2026-06-15.md, DQ-5).
+    """
+    bare = "10.1007/s00454-019-00132-8"
+    work = {
+        "id": "https://openalex.org/W2899262408",
+        # URL form + an upper-case suffix char — the worst case for ON CONFLICT.
+        "doi": "https://doi.org/10.1007/S00454-019-00132-8",
+        "title": "A Test Paper",
+        "publication_year": 2019,
+        "authorships": [{"author": {"display_name": "Alice Smith"}}],
+        "abstract_inverted_index": {"Hello": [0], "world": [1]},
+        # Non-empty so fetch_citations yields edges and the S2 fallback is skipped.
+        "referenced_works": ["https://openalex.org/W1"],
+    }
+
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch(
+            "codex.ingest.openalex._get",
+            return_value=httpx.Response(200, json=work),
+        ),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper(bare)
+
+    # The IngestResult and the upsert must use the bare canonical id, not the URL.
+    assert result.paper_id == bare
+
+    upsert_sql: str = mock_conn.execute.call_args_list[0][0][0]
+    upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert "INSERT INTO papers" in upsert_sql
+    assert "ON CONFLICT (id)" in upsert_sql
+    assert upsert_params["id"] == bare, (
+        f"upsert id must be the bare canonical DOI '{bare}', "
+        f"not the OpenAlex URL form '{upsert_params['id']}'"
+    )
+
+
 # ---------------------------------------------------------------------------
 
 

tests/sources/test_openalex.py

@@ -7,7 +7,7 @@ import pytest
 
 from codex.models import Citation, Paper
 from codex.sources import openalex
-from codex.sources.openalex import _resolve_id
+from codex.sources.openalex import _normalize_doi, _resolve_id
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -15,7 +15,9 @@ from codex.sources.openalex import _resolve_id
 
 _SAMPLE_WORK = {
     "id": "https://openalex.org/W2741809807",
-    "doi": "10.1145/3592430",
+    # Real OpenAlex returns the DOI as a full URL (T-3: the old bare fixture hid
+    # the URL-form behaviour that DQ-5 is about).
+    "doi": "https://doi.org/10.1145/3592430",
     "title": "Conformal Prediction: A Review",
     "publication_year": 2023,
     "authorships": [
@@ -67,6 +69,51 @@ def test_resolve_prefixed_unchanged() -> None:
     assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x"
 
 
+# ---------------------------------------------------------------------------
+# _normalize_doi — canonical bare, lower-cased form (DQ-5)
+# ---------------------------------------------------------------------------
+
+
+def test_normalize_doi_strips_https_url() -> None:
+    # The exact form OpenAlex returns in the `doi` field.
+    assert _normalize_doi("https://doi.org/10.1007/s00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_normalize_doi_strips_http_and_doi_prefix() -> None:
+    assert _normalize_doi("http://doi.org/10.1145/3592430") == "10.1145/3592430"
+    assert _normalize_doi("doi:10.1145/3592430") == "10.1145/3592430"
+
+
+def test_normalize_doi_lowercases() -> None:
+    # DOIs are case-insensitive; the canonical id (and _norm_cited_id) is lower-case.
+    assert _normalize_doi("https://doi.org/10.1007/S00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_normalize_doi_already_bare_is_idempotent() -> None:
+    assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8"
+
+
+def test_map_paper_falls_back_to_openalex_id_without_doi(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A work with no DOI keeps the OpenAlex W-id (URL form) as id, unchanged —
+    normalization only rewrites the DOI branch."""
+    work = {k: v for k, v in _SAMPLE_WORK.items() if k != "doi"}
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=work)
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+    paper = openalex.fetch_paper("W2741809807")
+
+    assert paper is not None
+    assert paper.id == "https://openalex.org/W2741809807"
+
+
 # ---------------------------------------------------------------------------
 # fetch_paper — success
 # ---------------------------------------------------------------------------
@@ -85,6 +132,9 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
 
     assert paper is not None
     assert isinstance(paper, Paper)
+    # DQ-5 / C-7: paper.id is the canonical BARE DOI, not the full URL OpenAlex
+    # returns. This is what makes the ingest ON CONFLICT (id) upsert idempotent.
+    assert paper.id == "10.1145/3592430"
     assert paper.title == "Conformal Prediction: A Review"
     assert paper.year == 2023
     assert paper.authors == ["Alice Smith", "Bob Jones"]