From 0021859094b38ed04c666b696e37255c90d18261 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:39:04 +0200 Subject: [PATCH] fix(ingest): normalize GROBID-derived citation DOIs to bare lower-case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GROBID extracts a cited DOI verbatim from PDF reference text, so it can be mixed-case, a https://doi.org/ URL, or doi:-prefixed. The PDF citation branch inserted it unchanged, while every other path (S2 supplement, OpenAlex, papers.id itself) uses the bare lower-cased DOI. Once R-A runs GROBID reference extraction over arXiv PDFs, those un-normalized cited-ids would never equal a bare-lowercase papers.id/cited_id — dangling the cross-citations and splitting one reference into case-/URL-variant graph nodes. Extend _norm_cited_id to strip https://doi.org//http://doi.org//doi: prefixes and lower-case (mirroring openalex._normalize_doi), making it the single canonical cited-id normalizer, and route the GROBID branch through it. arXiv ids and S2 paperIds still pass through untouched. Add test_ingest_pdf_grobid_citations_normalize_doi covering mixed-case, URL-form, and doi:-prefixed DOIs plus an arXiv-only ref. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 25 +++++++++++--- tests/ingest/test_ingest.py | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index cd950e7..c0f768f 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -65,10 +65,20 @@ def _s2_id_for(pid: str) -> str | None: def _norm_cited_id(cited_id: str) -> str: - """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join - does not fragment the same reference into distinct case-variant nodes. - arXiv ids and S2 paperIds are left untouched.""" - return cited_id.lower() if cited_id.startswith("10.") else cited_id + """Normalize a DOI cited-id to its canonical bare, lower-cased form. + + DOIs are case-insensitive and may arrive bare (``10.…``), as a + ``https://doi.org/…`` URL, or ``doi:…``-prefixed: GROBID emits whichever + form the PDF reference text used, while S2/OpenAlex hand back bare DOIs. + Stripping the prefix and lower-casing keeps the citation-graph join from + fragmenting the same reference into distinct case-/URL-variant nodes. arXiv + ids and S2 paperIds are left untouched. Mirrors ``openalex._normalize_doi``.""" + s = cited_id.strip() + lowered = s.lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if lowered.startswith(prefix): + return lowered[len(prefix) :] + return lowered if lowered.startswith("10.") else s def _recover_abstract(paper: Paper) -> str | None: @@ -268,7 +278,12 @@ def ingest_paper( for ref in grobid_refs: cited_id = ref.get("doi") or ref.get("arxiv_id") if cited_id: - pdf_citations.append(Citation(citing_id=paper.id, cited_id=cited_id)) + # Normalize the GROBID DOI (bare/URL/``doi:`` → bare lower-case) + # so PDF-derived edges join the same graph nodes as the + # S2/OpenAlex paths; arXiv ids pass through untouched. + pdf_citations.append( + Citation(citing_id=paper.id, cited_id=_norm_cited_id(cited_id)) + ) else: logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path) diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 5011fbf..936e6cb 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -199,6 +199,73 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: assert mock_cursor.executemany.call_count >= 2 +def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None: + """R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so + it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each + must be normalized to the bare, lower-cased canonical form before the Citation + is built — otherwise the same reference splits into case-/URL-variant graph + nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no + DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1).""" + pdf_file = tmp_path / "paper.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake content") + + paper = _make_paper() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]: + # Faithful to GROBID's output shape: all five keys always present. + return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id} + + grobid_refs = [ + _ref(doi="10.1007/S00454-019-00132-8"), # mixed-case bare DOI + _ref(doi="https://doi.org/10.1145/AbC.123"), # URL-form DOI + _ref(doi="doi:10.1/MixedCase"), # doi:-prefixed DOI + _ref(arxiv_id="2101.00001"), # arXiv-only ref → untouched + ] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID + # refs contribute to the citation rows under inspection. + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), + patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]), + patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), + patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + ): + result = ingest_paper(paper.id, source_path=str(pdf_file)) + + assert result.citations_upserted == 4 + + # Locate the citations INSERT among the cursor's executemany calls (chunks + # are also inserted via executemany, so filter by SQL rather than by index). + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + citation_calls = [ + c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0] + ] + assert len(citation_calls) == 1 + inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1] + + # Every edge hangs off the canonical paper.id (citing side). + assert all(row[0] == paper.id for row in inserted_rows) + + cited_ids = {row[1] for row in inserted_rows} + assert cited_ids == { + "10.1007/s00454-019-00132-8", # mixed-case → lower-cased + "10.1145/abc.123", # URL prefix stripped + lower-cased + "10.1/mixedcase", # doi: prefix stripped + lower-cased + "2101.00001", # arXiv id untouched + } + # No raw URL-form / prefixed DOI leaked into the citation graph. + assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids) + + # --------------------------------------------------------------------------- # 4. test_ingest_paper_not_found # ---------------------------------------------------------------------------