fix(ingest): normalize GROBID-derived citation DOIs to bare lower-case

GROBID extracts a cited DOI verbatim from PDF reference text, so it can be mixed-case, a https://doi.org/ URL, or doi:-prefixed. The PDF citation branch inserted it unchanged, while every other path (S2 supplement, OpenAlex, papers.id itself) uses the bare lower-cased DOI. Once R-A runs GROBID reference extraction over arXiv PDFs, those un-normalized cited-ids would never equal a bare-lowercase papers.id/cited_id — dangling the cross-citations and splitting one reference into case-/URL-variant graph nodes.

Extend _norm_cited_id to strip https://doi.org//http://doi.org//doi: prefixes and lower-case (mirroring openalex._normalize_doi), making it the single canonical cited-id normalizer, and route the GROBID branch through it. arXiv ids and S2 paperIds still pass through untouched.

Add test_ingest_pdf_grobid_citations_normalize_doi covering mixed-case, URL-form, and doi:-prefixed DOIs plus an arXiv-only ref.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codex/ingest.py

@@ -65,10 +65,20 @@ def _s2_id_for(pid: str) -> str | None:
 
 
 def _norm_cited_id(cited_id: str) -> str:
-    """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join
+    """Normalize a DOI cited-id to its canonical bare, lower-cased form.
-    does not fragment the same reference into distinct case-variant nodes.
+
-    arXiv ids and S2 paperIds are left untouched."""
+    DOIs are case-insensitive and may arrive bare (``10.…``), as a
-    return cited_id.lower() if cited_id.startswith("10.") else cited_id
+    ``https://doi.org/…`` URL, or ``doi:…``-prefixed: GROBID emits whichever
+    form the PDF reference text used, while S2/OpenAlex hand back bare DOIs.
+    Stripping the prefix and lower-casing keeps the citation-graph join from
+    fragmenting the same reference into distinct case-/URL-variant nodes. arXiv
+    ids and S2 paperIds are left untouched. Mirrors ``openalex._normalize_doi``."""
+    s = cited_id.strip()
+    lowered = s.lower()
+    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
+        if lowered.startswith(prefix):
+            return lowered[len(prefix) :]
+    return lowered if lowered.startswith("10.") else s
 
 
 def _recover_abstract(paper: Paper) -> str | None:
@@ -268,7 +278,12 @@ def ingest_paper(
                 for ref in grobid_refs:
                     cited_id = ref.get("doi") or ref.get("arxiv_id")
                     if cited_id:
-                        pdf_citations.append(Citation(citing_id=paper.id, cited_id=cited_id))
+                        # Normalize the GROBID DOI (bare/URL/``doi:`` → bare lower-case)
+                        # so PDF-derived edges join the same graph nodes as the
+                        # S2/OpenAlex paths; arXiv ids pass through untouched.
+                        pdf_citations.append(
+                            Citation(citing_id=paper.id, cited_id=_norm_cited_id(cited_id))
+                        )
             else:
                 logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
 

tests/ingest/test_ingest.py

@@ -199,6 +199,73 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
     assert mock_cursor.executemany.call_count >= 2
 
 
+def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
+    """R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so
+    it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each
+    must be normalized to the bare, lower-cased canonical form before the Citation
+    is built — otherwise the same reference splits into case-/URL-variant graph
+    nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no
+    DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1)."""
+    pdf_file = tmp_path / "paper.pdf"
+    pdf_file.write_bytes(b"%PDF-1.4 fake content")
+
+    paper = _make_paper()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
+        # Faithful to GROBID's output shape: all five keys always present.
+        return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id}
+
+    grobid_refs = [
+        _ref(doi="10.1007/S00454-019-00132-8"),  # mixed-case bare DOI
+        _ref(doi="https://doi.org/10.1145/AbC.123"),  # URL-form DOI
+        _ref(doi="doi:10.1/MixedCase"),  # doi:-prefixed DOI
+        _ref(arxiv_id="2101.00001"),  # arXiv-only ref → untouched
+    ]
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID
+        # refs contribute to the citation rows under inspection.
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
+        patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
+        patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
+        patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
+    ):
+        result = ingest_paper(paper.id, source_path=str(pdf_file))
+
+    assert result.citations_upserted == 4
+
+    # Locate the citations INSERT among the cursor's executemany calls (chunks
+    # are also inserted via executemany, so filter by SQL rather than by index).
+    mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
+    citation_calls = [
+        c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0]
+    ]
+    assert len(citation_calls) == 1
+    inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1]
+
+    # Every edge hangs off the canonical paper.id (citing side).
+    assert all(row[0] == paper.id for row in inserted_rows)
+
+    cited_ids = {row[1] for row in inserted_rows}
+    assert cited_ids == {
+        "10.1007/s00454-019-00132-8",  # mixed-case → lower-cased
+        "10.1145/abc.123",  # URL prefix stripped + lower-cased
+        "10.1/mixedcase",  # doi: prefix stripped + lower-cased
+        "2101.00001",  # arXiv id untouched
+    }
+    # No raw URL-form / prefixed DOI leaked into the citation graph.
+    assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids)
+
+
 # ---------------------------------------------------------------------------
 # 4. test_ingest_paper_not_found
 # ---------------------------------------------------------------------------