fix(openalex): normalize DOI to bare canonical id so re-ingest is idempotent (DQ-5)

openalex._map_paper set Paper.id from the raw `doi` field, which OpenAlex
returns as a full URL (https://doi.org/10.x). The canonical papers.id (M-1
migration) is the bare, lower-cased DOI, so re-ingesting a DOI paper missed
ON CONFLICT (id) and tripped the separate papers_openalex_id_key constraint
(UniqueViolation).

Fix: _normalize_doi() strips https://doi.org//http://doi.org//doi: and
lower-cases; applied in the DOI branch of _map_paper only (W-id fallback
untouched). Also un-breaks _s2_id_for/Crossref recovery, which the URL form
silently defeated. Corrected the _SAMPLE_WORK fixture to the URL form (the bare
fixture hid the bug) + added _normalize_doi and re-ingest idempotency regression
tests.

Live verification: re-ingest of 10.1007/s00454-019-00132-8 (previously crashed)
now UPDATEs in place — single row, added_at unchanged, no stray row, citations
45->45. Unblocks roadmap R-A/R-C. Full suite 348 green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-17 00:31:01 +02:00
parent fa43f69dfd
commit ea73b1475c
4 changed files with 181 additions and 15 deletions

View File

@@ -7,7 +7,7 @@ import pytest
from codex.models import Citation, Paper
from codex.sources import openalex
from codex.sources.openalex import _resolve_id
from codex.sources.openalex import _normalize_doi, _resolve_id
# ---------------------------------------------------------------------------
# Helpers
@@ -15,7 +15,9 @@ from codex.sources.openalex import _resolve_id
_SAMPLE_WORK = {
"id": "https://openalex.org/W2741809807",
"doi": "10.1145/3592430",
# Real OpenAlex returns the DOI as a full URL (T-3: the old bare fixture hid
# the URL-form behaviour that DQ-5 is about).
"doi": "https://doi.org/10.1145/3592430",
"title": "Conformal Prediction: A Review",
"publication_year": 2023,
"authorships": [
@@ -67,6 +69,51 @@ def test_resolve_prefixed_unchanged() -> None:
assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x"
# ---------------------------------------------------------------------------
# _normalize_doi — canonical bare, lower-cased form (DQ-5)
# ---------------------------------------------------------------------------
def test_normalize_doi_strips_https_url() -> None:
# The exact form OpenAlex returns in the `doi` field.
assert _normalize_doi("https://doi.org/10.1007/s00454-019-00132-8") == (
"10.1007/s00454-019-00132-8"
)
def test_normalize_doi_strips_http_and_doi_prefix() -> None:
assert _normalize_doi("http://doi.org/10.1145/3592430") == "10.1145/3592430"
assert _normalize_doi("doi:10.1145/3592430") == "10.1145/3592430"
def test_normalize_doi_lowercases() -> None:
# DOIs are case-insensitive; the canonical id (and _norm_cited_id) is lower-case.
assert _normalize_doi("https://doi.org/10.1007/S00454-019-00132-8") == (
"10.1007/s00454-019-00132-8"
)
def test_normalize_doi_already_bare_is_idempotent() -> None:
assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8"
def test_map_paper_falls_back_to_openalex_id_without_doi(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A work with no DOI keeps the OpenAlex W-id (URL form) as id, unchanged —
normalization only rewrites the DOI branch."""
work = {k: v for k, v in _SAMPLE_WORK.items() if k != "doi"}
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
return httpx.Response(200, json=work)
monkeypatch.setattr(openalex, "_get", mock_get)
paper = openalex.fetch_paper("W2741809807")
assert paper is not None
assert paper.id == "https://openalex.org/W2741809807"
# ---------------------------------------------------------------------------
# fetch_paper — success
# ---------------------------------------------------------------------------
@@ -85,6 +132,9 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
assert paper is not None
assert isinstance(paper, Paper)
# DQ-5 / C-7: paper.id is the canonical BARE DOI, not the full URL OpenAlex
# returns. This is what makes the ingest ON CONFLICT (id) upsert idempotent.
assert paper.id == "10.1145/3592430"
assert paper.title == "Conformal Prediction: A Review"
assert paper.year == 2023
assert paper.authors == ["Alice Smith", "Bob Jones"]