802 lines
33 KiB
Python
802 lines
33 KiB
Python
"""Tests for codex.ingest — end-to-end idempotent ingest pipeline.
|
|
|
|
All external calls (DB, OpenAlex, S2, embed, parsing) are mocked so
|
|
the suite runs offline in milliseconds.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from contextlib import contextmanager
|
|
from typing import Any
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import httpx
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from codex.ingest import IngestResult, _make_bibkey, ingest_paper
|
|
from codex.models import Citation, FigureChunk, FormulaChunk, Paper
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_conn_cm(mock_conn: MagicMock) -> Any:
|
|
"""Return a context-manager factory that yields *mock_conn*."""
|
|
|
|
@contextmanager # type: ignore[arg-type]
|
|
def _cm() -> Any:
|
|
yield mock_conn
|
|
|
|
return _cm
|
|
|
|
|
|
def _make_paper(
|
|
paper_id: str = "2301.07041",
|
|
openalex_id: str | None = "W123",
|
|
abstract: str | None = "Test abstract.",
|
|
) -> Paper:
|
|
return Paper(
|
|
id=paper_id,
|
|
title="A Test Paper",
|
|
openalex_id=openalex_id,
|
|
authors=["Alice", "Bob"],
|
|
year=2023,
|
|
abstract=abstract,
|
|
)
|
|
|
|
|
|
def _fake_embedder(dim: int = 1024) -> MagicMock:
|
|
"""Return a mock Embedder whose encode_dense returns deterministic output."""
|
|
embedder = MagicMock()
|
|
embedder.dim = dim
|
|
|
|
def encode_dense(texts: list[str]) -> np.ndarray:
|
|
return np.ones((len(texts), dim), dtype=np.float32)
|
|
|
|
embedder.encode_dense.side_effect = encode_dense
|
|
return embedder
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. test_ingest_paper_basic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_basic() -> None:
|
|
"""fetch_paper returns a Paper; DB upsert is called; IngestResult has correct counts."""
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
# OpenAlex returns citing_id=openalex_id; ingest.py must rewrite to paper.id (D-05c).
|
|
raw_api_citations = [
|
|
Citation(citing_id=paper.openalex_id or "", cited_id="W999"),
|
|
]
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper) as mock_fetch,
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=raw_api_citations),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result = ingest_paper(paper.id)
|
|
|
|
mock_fetch.assert_called_once_with(paper.id)
|
|
|
|
# Paper upsert must have been issued
|
|
assert mock_conn.execute.call_count >= 1
|
|
first_sql: str = mock_conn.execute.call_args_list[0][0][0]
|
|
assert "INSERT INTO papers" in first_sql
|
|
assert "ON CONFLICT" in first_sql
|
|
|
|
assert isinstance(result, IngestResult)
|
|
assert result.paper_id == paper.id
|
|
assert result.chunks_upserted == 0
|
|
assert result.citations_upserted == len(raw_api_citations)
|
|
|
|
# D-05c regression: citing_id must be rewritten to paper.id (DOI), not openalex_id.
|
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
|
cit_call = mock_cursor.executemany.call_args_list[0]
|
|
inserted_rows: list[tuple[str, str, Any]] = cit_call[0][1]
|
|
assert inserted_rows[0][0] == paper.id, (
|
|
f"citing_id must be paper.id='{paper.id}', not openalex_id='{paper.openalex_id}'"
|
|
)
|
|
|
|
|
|
def test_ingest_citation_fallback_to_crossref() -> None:
|
|
"""R-B: OpenAlex + S2 both empty → Crossref references supply the edges.
|
|
|
|
Crossref is the third leg of the fallback chain. Its cited DOIs are
|
|
case-normalised and citing_id is rewritten to the canonical paper.id.
|
|
"""
|
|
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
# Upper-case suffix proves the cited DOI is normalised to bare lower-case.
|
|
crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")]
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty
|
|
patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr,
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result = ingest_paper(paper.id)
|
|
|
|
# Crossref consulted with the bare DOI; one edge upserted.
|
|
mock_cr.assert_called_once_with(paper.id)
|
|
assert result.citations_upserted == 1
|
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
|
cit_rows = mock_cursor.executemany.call_args_list[0][0][1]
|
|
assert cit_rows[0][0] == paper.id
|
|
assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7"
|
|
|
|
|
|
def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
|
|
"""R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback)."""
|
|
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
|
|
mock_conn = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch(
|
|
"codex.ingest.openalex.fetch_citations",
|
|
return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")],
|
|
),
|
|
patch("codex.ingest.crossref.fetch_references") as mock_cr,
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
ingest_paper(paper.id)
|
|
|
|
mock_cr.assert_not_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. test_ingest_paper_source_tex
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
|
"""``.tex`` source → section-aware chunking; the stored ``section`` column is
|
|
labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12)."""
|
|
tex_file = tmp_path / "paper.tex"
|
|
tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.")
|
|
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
# (section title, chunk content) pairs from the section-aware chunker.
|
|
section_pairs = [
|
|
("Introduction", "Body of the introduction goes here."),
|
|
("Proof of the Main Theorem", "Body of the proof goes here."),
|
|
("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."),
|
|
]
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
patch("codex.parsing.tex.chunk_sections", return_value=section_pairs),
|
|
patch("codex.ingest.is_quality_chunk", return_value=True),
|
|
):
|
|
result = ingest_paper(paper.id, source_path=str(tex_file))
|
|
|
|
# DELETE + INSERT for chunks
|
|
sql_calls = [str(c[0][0]) for c in mock_conn.execute.call_args_list]
|
|
assert any("DELETE FROM chunks" in sql for sql in sql_calls)
|
|
|
|
# executemany is called on the cursor, not on the connection directly
|
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
|
assert mock_cursor.executemany.call_count >= 1
|
|
chunk_insert_call = mock_cursor.executemany.call_args_list[0]
|
|
chunk_sql: str = chunk_insert_call[0][0]
|
|
assert "INSERT INTO chunks" in chunk_sql
|
|
|
|
# R-F: heading → controlled bucket where it maps ("Introduction"→intro,
|
|
# "Proof of…"→proof), else the cleaned real title ("discrete conformal maps")
|
|
# instead of collapsing to "body".
|
|
chunk_rows = chunk_insert_call[0][1]
|
|
assert [row[4] for row in chunk_rows] == ["intro", "proof", "discrete conformal maps"]
|
|
assert result.chunks_upserted == len(section_pairs)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. test_ingest_paper_source_pdf
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
|
|
"""``.pdf`` source → pdf_to_markdown, chunk_text, embed, grobid refs, chunks + citations."""
|
|
pdf_file = tmp_path / "paper.pdf"
|
|
pdf_file.write_bytes(b"%PDF-1.4 fake content")
|
|
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
fake_chunks = ["pdf chunk one.", "pdf chunk two."]
|
|
grobid_refs = [
|
|
{"title": "Ref A", "authors": "X", "year": "2020", "doi": "10.1/ref_a", "arxiv_id": ""},
|
|
{"title": "Ref B", "authors": "Y", "year": "2021", "doi": "", "arxiv_id": "2101.00001"},
|
|
]
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
# OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1);
|
|
# mock it empty so only the GROBID refs contribute to the count.
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
|
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
|
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
|
patch("codex.ingest.is_quality_chunk", return_value=True),
|
|
):
|
|
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
|
|
|
# Chunks were inserted
|
|
assert result.chunks_upserted == len(fake_chunks)
|
|
|
|
# GROBID refs were merged into citations
|
|
# Two grobid refs → both have either doi or arxiv_id → 2 citations
|
|
assert result.citations_upserted == 2
|
|
|
|
# executemany is called on the cursor for chunks + citations (at least 2 calls total)
|
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
|
assert mock_cursor.executemany.call_count >= 2
|
|
|
|
|
|
def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
|
|
"""R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so
|
|
it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each
|
|
must be normalized to the bare, lower-cased canonical form before the Citation
|
|
is built — otherwise the same reference splits into case-/URL-variant graph
|
|
nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no
|
|
DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1)."""
|
|
pdf_file = tmp_path / "paper.pdf"
|
|
pdf_file.write_bytes(b"%PDF-1.4 fake content")
|
|
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
|
|
# Faithful to GROBID's output shape: all five keys always present.
|
|
return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id}
|
|
|
|
grobid_refs = [
|
|
_ref(doi="10.1007/S00454-019-00132-8"), # mixed-case bare DOI
|
|
_ref(doi="https://doi.org/10.1145/AbC.123"), # URL-form DOI
|
|
_ref(doi="doi:10.1/MixedCase"), # doi:-prefixed DOI
|
|
_ref(arxiv_id="2101.00001"), # arXiv-only ref → untouched
|
|
]
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
# OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID
|
|
# refs contribute to the citation rows under inspection.
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
|
patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
|
|
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
|
patch("codex.ingest.is_quality_chunk", return_value=True),
|
|
):
|
|
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
|
|
|
assert result.citations_upserted == 4
|
|
|
|
# Locate the citations INSERT among the cursor's executemany calls (chunks
|
|
# are also inserted via executemany, so filter by SQL rather than by index).
|
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
|
citation_calls = [
|
|
c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0]
|
|
]
|
|
assert len(citation_calls) == 1
|
|
inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1]
|
|
|
|
# Every edge hangs off the canonical paper.id (citing side).
|
|
assert all(row[0] == paper.id for row in inserted_rows)
|
|
|
|
cited_ids = {row[1] for row in inserted_rows}
|
|
assert cited_ids == {
|
|
"10.1007/s00454-019-00132-8", # mixed-case → lower-cased
|
|
"10.1145/abc.123", # URL prefix stripped + lower-cased
|
|
"10.1/mixedcase", # doi: prefix stripped + lower-cased
|
|
"2101.00001", # arXiv id untouched
|
|
}
|
|
# No raw URL-form / prefixed DOI leaked into the citation graph.
|
|
assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. test_ingest_paper_not_found
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_not_found() -> None:
|
|
"""OpenAlex returns None for unknown ID → ValueError raised."""
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=None),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
pytest.raises(ValueError, match="Paper not found"),
|
|
):
|
|
ingest_paper("10.9999/unknown-doi")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. test_ingest_paper_idempotent
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_idempotent() -> None:
|
|
"""Second call with the same paper_id overwrites without error (no unique violations)."""
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result1 = ingest_paper(paper.id)
|
|
result2 = ingest_paper(paper.id)
|
|
|
|
assert result1.paper_id == result2.paper_id == paper.id
|
|
|
|
# ON CONFLICT … DO UPDATE means no error on second call
|
|
all_execute_calls = mock_conn.execute.call_args_list
|
|
paper_upserts = [c for c in all_execute_calls if "INSERT INTO papers" in str(c[0][0])]
|
|
# Two calls → two paper upserts
|
|
assert len(paper_upserts) == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. test_ingest_paper_arxiv_s2_fallback
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_arxiv_s2_fallback() -> None:
|
|
"""OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted."""
|
|
paper_id = "2301.07041"
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=None),
|
|
# arXiv + abstract sources also empty → genuine stub fallback path.
|
|
patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv,
|
|
patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
|
|
patch("codex.ingest.crossref.fetch_abstract", return_value=None),
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2,
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result = ingest_paper(paper_id)
|
|
|
|
# arXiv metadata recovery attempted; S2 references consulted as fallback.
|
|
mock_arxiv.assert_called_once_with(paper_id)
|
|
mock_s2.assert_called()
|
|
assert result.paper_id == paper_id
|
|
|
|
|
|
def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None:
|
|
"""DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API,
|
|
bibkey auto-generated, abstract embedded (no zero vector)."""
|
|
recovered = Paper(
|
|
id="1911.00966",
|
|
title="A discrete version of Liouville's theorem on conformal maps",
|
|
authors=["Ulrich Pinkall", "Boris Springborn"],
|
|
year=2019,
|
|
abstract="Liouville's theorem says that in dimension greater than two...",
|
|
)
|
|
fake_emb = _fake_embedder()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=None),
|
|
patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv,
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=fake_emb),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result = ingest_paper("1911.00966")
|
|
|
|
mock_arxiv.assert_called_once_with("1911.00966")
|
|
assert result.paper_id == "1911.00966"
|
|
params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
|
assert params["title"].startswith("A discrete version")
|
|
assert params["bibkey"] == "PinkallSpringborn2019" # generated from recovered authors+year
|
|
# abstract present → real embedding computed, not a zero vector
|
|
fake_emb.encode_dense.assert_called_once()
|
|
|
|
|
|
def test_ingest_paper_empty_abstract_recovered_from_s2() -> None:
|
|
"""DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding."""
|
|
paper = _make_paper(abstract=None) # has openalex_id, no abstract
|
|
fake_emb = _fake_embedder()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
|
patch(
|
|
"codex.ingest.semanticscholar.fetch_abstract",
|
|
return_value="Recovered abstract text.",
|
|
) as mock_abs,
|
|
patch("codex.ingest.get_embedder", return_value=fake_emb),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
ingest_paper(paper.id)
|
|
|
|
mock_abs.assert_called_once()
|
|
params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
|
assert params["abstract"] == "Recovered abstract text."
|
|
fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."])
|
|
|
|
|
|
def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
|
|
"""DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
|
|
rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
|
|
paper = _make_paper() # openalex_id="W123", id="2301.07041"
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
# S2 keys citing_id by the id form passed in; ingest must canonicalise it.
|
|
s2_refs = [
|
|
Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"),
|
|
Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"),
|
|
]
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2,
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result = ingest_paper(paper.id)
|
|
|
|
# S2 consulted with a namespaced id (a bare arXiv id 404s on S2).
|
|
mock_s2.assert_called_once_with("arXiv:2301.07041")
|
|
assert result.citations_upserted == 2
|
|
|
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
|
inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1]
|
|
# citing_id rewritten to paper.id, not the S2 id form.
|
|
assert all(row[0] == paper.id for row in inserted_rows)
|
|
cited = {row[1] for row in inserted_rows}
|
|
assert "10.1/upper" in cited # DOI lowercased
|
|
assert "2202.00002" in cited # arXiv id untouched
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 7. test_ingest_paper_no_abstract_uses_zero_vector
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
|
|
"""A paper without an abstract gets a zero embedding vector (no model call)."""
|
|
paper = _make_paper(abstract=None)
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
fake_emb = _fake_embedder(dim=1024)
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
# No abstract recoverable from any source (DQ-2) → genuine zero-vector case.
|
|
patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
|
|
patch("codex.ingest.crossref.fetch_abstract", return_value=None),
|
|
patch("codex.ingest.get_embedder", return_value=fake_emb),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
ingest_paper(paper.id)
|
|
|
|
# encode_dense must NOT have been called (no abstract → zero vector shortcut)
|
|
fake_emb.encode_dense.assert_not_called()
|
|
|
|
# Verify zero vector was passed in the paper upsert
|
|
upsert_call = mock_conn.execute.call_args_list[0]
|
|
params: dict[str, Any] = upsert_call[0][1]
|
|
emb: list[float] = params["abstract_emb"]
|
|
assert len(emb) == 1024
|
|
assert all(v == 0.0 for v in emb)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 8. F-09 Rich Parsing tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> None:
|
|
"""When rich=True and source is PDF, formula + figure parsers are invoked."""
|
|
import codex.parsing.figures as _figures_mod
|
|
import codex.parsing.mathpix as _mathpix_mod
|
|
|
|
pdf_file = tmp_path / "paper.pdf"
|
|
pdf_file.write_bytes(b"%PDF-1.4 fake content")
|
|
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
fake_formula = FormulaChunk(paper_id=paper.id, page=1, raw_latex=r"\alpha", context="test")
|
|
fake_figure = FigureChunk(
|
|
paper_id=paper.id, page=1, image_path="/tmp/fig.png", caption="Figure 1."
|
|
)
|
|
|
|
mock_settings = MagicMock()
|
|
mock_settings.figures_dir = str(tmp_path / "figures")
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
patch("codex.parsing.nougat.pdf_to_markdown", return_value=""),
|
|
patch("codex.parsing.tex.chunk_text", return_value=[]),
|
|
patch("codex.parsing.grobid.extract_references", return_value=[]),
|
|
patch.object(
|
|
_mathpix_mod, "extract_formulas", return_value=[fake_formula]
|
|
) as mock_formulas,
|
|
patch.object(_figures_mod, "extract_figures", return_value=[fake_figure]) as mock_figures,
|
|
patch("codex.ingest.get_settings", return_value=mock_settings),
|
|
):
|
|
result = ingest_paper(paper.id, source_path=str(pdf_file), rich=True)
|
|
|
|
mock_formulas.assert_called_once()
|
|
mock_figures.assert_called_once()
|
|
assert result.formulas_upserted == 1
|
|
assert result.figures_upserted == 1
|
|
|
|
|
|
def test_ingest_paper_no_rich_skips_parsers(tmp_path: Any) -> None:
|
|
"""When rich=False (default), formula and figure parsers are NOT called."""
|
|
import codex.parsing.figures as _figures_mod
|
|
import codex.parsing.mathpix as _mathpix_mod
|
|
|
|
pdf_file = tmp_path / "paper.pdf"
|
|
pdf_file.write_bytes(b"%PDF-1.4 fake content")
|
|
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
patch("codex.parsing.nougat.pdf_to_markdown", return_value=""),
|
|
patch("codex.parsing.tex.chunk_text", return_value=[]),
|
|
patch("codex.parsing.grobid.extract_references", return_value=[]),
|
|
patch.object(_mathpix_mod, "extract_formulas", return_value=[]) as mock_formulas,
|
|
patch.object(_figures_mod, "extract_figures", return_value=[]) as mock_figures,
|
|
):
|
|
result = ingest_paper(paper.id, source_path=str(pdf_file), rich=False)
|
|
|
|
mock_formulas.assert_not_called()
|
|
mock_figures.assert_not_called()
|
|
assert result.formulas_upserted == 0
|
|
assert result.figures_upserted == 0
|
|
|
|
|
|
def test_ingest_paper_rich_requires_pdf_source(tmp_path: Any) -> None:
|
|
"""When rich=True but source is a .tex file, formula/figure parsers are NOT called."""
|
|
import codex.parsing.figures as _figures_mod
|
|
import codex.parsing.mathpix as _mathpix_mod
|
|
|
|
tex_file = tmp_path / "paper.tex"
|
|
tex_file.write_text(r"\section{Intro}")
|
|
|
|
paper = _make_paper()
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
patch("codex.parsing.tex.latex_to_text", return_value=""),
|
|
patch("codex.parsing.tex.chunk_text", return_value=[]),
|
|
patch.object(_mathpix_mod, "extract_formulas", return_value=[]) as mock_formulas,
|
|
patch.object(_figures_mod, "extract_figures", return_value=[]) as mock_figures,
|
|
):
|
|
result = ingest_paper(paper.id, source_path=str(tex_file), rich=True)
|
|
|
|
mock_formulas.assert_not_called()
|
|
mock_figures.assert_not_called()
|
|
assert result.formulas_upserted == 0
|
|
assert result.figures_upserted == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# D-04 — bibkey auto-generation heuristic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_make_bibkey_single_author() -> None:
|
|
paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=2023)
|
|
assert _make_bibkey(paper) == "Smith2023"
|
|
|
|
|
|
def test_make_bibkey_two_authors() -> None:
|
|
paper = Paper(id="doi:10.1/x", title="T", authors=["Alice Foo", "Bob Bar"], year=2015)
|
|
assert _make_bibkey(paper) == "FooBar2015"
|
|
|
|
|
|
def test_make_bibkey_three_authors() -> None:
|
|
paper = Paper(
|
|
id="doi:10.1/x",
|
|
title="T",
|
|
authors=["Alexander Bobenko", "Ulrich Pinkall", "Boris Springborn"],
|
|
year=2015,
|
|
)
|
|
assert _make_bibkey(paper) == "BobenkoPinkallSpringborn2015"
|
|
|
|
|
|
def test_make_bibkey_many_authors_uses_etal() -> None:
|
|
paper = Paper(
|
|
id="doi:10.1/x",
|
|
title="T",
|
|
authors=["A One", "B Two", "C Three", "D Four"],
|
|
year=2020,
|
|
)
|
|
assert _make_bibkey(paper) == "OneEtAl2020"
|
|
|
|
|
|
def test_make_bibkey_no_authors_returns_none() -> None:
|
|
paper = Paper(id="doi:10.1/x", title="T", authors=[], year=2023)
|
|
assert _make_bibkey(paper) is None
|
|
|
|
|
|
def test_make_bibkey_no_year_returns_none() -> None:
|
|
paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=None)
|
|
assert _make_bibkey(paper) is None
|
|
|
|
|
|
def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None:
|
|
"""ingest_paper auto-generates bibkey when paper has none (D-04)."""
|
|
paper = _make_paper() # bibkey=None (default from _make_paper)
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
ingest_paper(paper.id)
|
|
|
|
upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
|
assert upsert_params["bibkey"] is not None, "bibkey must be auto-generated when paper has none"
|
|
# authors=["Alice", "Bob"], year=2023 → 2 authors ≤ 3 → concat last tokens
|
|
assert upsert_params["bibkey"] == "AliceBob2023"
|
|
|
|
|
|
def test_ingest_doi_paper_upserts_bare_canonical_id() -> None:
|
|
"""DQ-5 regression: re-ingesting an existing DOI paper must be idempotent.
|
|
|
|
OpenAlex returns the ``doi`` as a full URL (and possibly mixed-case), but the
|
|
stored canonical ``papers.id`` is the BARE, lower-cased DOI (M-1 migration).
|
|
The papers upsert must therefore carry the BARE id into ``%(id)s`` so
|
|
``ON CONFLICT (id)`` matches the existing row and UPDATEs it — rather than
|
|
attempting a fresh INSERT that trips the separate ``papers_openalex_id_key``
|
|
unique constraint (the UniqueViolation crash).
|
|
|
|
The real ``_map_paper``/``fetch_paper`` run here (only the HTTP ``_get``, the
|
|
embedder and the DB are mocked), so this exercises the actual normalization
|
|
through the ingest path. The DB is a MagicMock, so it cannot raise the real
|
|
constraint error; the live re-ingest of 10.1007/s00454-019-00132-8 is the
|
|
end-to-end UniqueViolation proof (see DATA-QUALITY-2026-06-15.md, DQ-5).
|
|
"""
|
|
bare = "10.1007/s00454-019-00132-8"
|
|
work = {
|
|
"id": "https://openalex.org/W2899262408",
|
|
# URL form + an upper-case suffix char — the worst case for ON CONFLICT.
|
|
"doi": "https://doi.org/10.1007/S00454-019-00132-8",
|
|
"title": "A Test Paper",
|
|
"publication_year": 2019,
|
|
"authorships": [{"author": {"display_name": "Alice Smith"}}],
|
|
"abstract_inverted_index": {"Hello": [0], "world": [1]},
|
|
# Non-empty so fetch_citations yields edges and the S2 fallback is skipped.
|
|
"referenced_works": ["https://openalex.org/W1"],
|
|
}
|
|
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute = MagicMock()
|
|
mock_conn.executemany = MagicMock()
|
|
mock_conn.commit = MagicMock()
|
|
|
|
with (
|
|
patch(
|
|
"codex.ingest.openalex._get",
|
|
return_value=httpx.Response(200, json=work),
|
|
),
|
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
|
):
|
|
result = ingest_paper(bare)
|
|
|
|
# The IngestResult and the upsert must use the bare canonical id, not the URL.
|
|
assert result.paper_id == bare
|
|
|
|
upsert_sql: str = mock_conn.execute.call_args_list[0][0][0]
|
|
upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
|
assert "INSERT INTO papers" in upsert_sql
|
|
assert "ON CONFLICT (id)" in upsert_sql
|
|
assert upsert_params["id"] == bare, (
|
|
f"upsert id must be the bare canonical DOI '{bare}', "
|
|
f"not the OpenAlex URL form '{upsert_params['id']}'"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_result_has_formula_figure_counts() -> None:
|
|
"""IngestResult has formulas_upserted and figures_upserted fields with default 0."""
|
|
result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0)
|
|
assert result.formulas_upserted == 0
|
|
assert result.figures_upserted == 0
|
|
|
|
result2 = IngestResult(
|
|
paper_id="test",
|
|
chunks_upserted=5,
|
|
citations_upserted=3,
|
|
formulas_upserted=10,
|
|
figures_upserted=2,
|
|
)
|
|
assert result2.formulas_upserted == 10
|
|
assert result2.figures_upserted == 2
|