user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(F-09): rich parsing — formula + figure extraction

Browse Source 
- codex/parsing/mathpix.py: pix2tex (local, CPU) primary + MathPix API
  optional; bbox heuristic h>15px, math-char-count>5; singleton model cache
- codex/parsing/figures.py: pymupdf embedded-image extraction → PNG;
  caption detection via proximity + "Figure/Fig./Abbildung" prefix
- codex/models.py: FormulaChunk + FigureChunk dataclasses (R-10/R-11)
- codex/ingest.py: --rich flag wires formula+figure extraction post-ingest
- codex/cli.py: search_app sub-typer (paper + formula subcommands),
  --rich flag on ingest; wiki_app from F-12 preserved intact
- codex/config.py: mathpix_app_id/key, pix2tex_fallback, figures_dir
- infra/schema.sql: formulas + figures tables with HNSW pgvector indexes
- pyproject.toml: pymupdf>=1.24, pix2tex>=0.1.4
- tests/parsing/test_mathpix.py + test_figures.py: 31 tests (mock pix2tex
  + MathPix HTTP, real pymupdf on synthetic PDF)

Gate: 158 passed, ruff clean, mypy clean (20 files)
Requirements: R-10 R-11 R-12 R-13 R-14 → done

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 01:16:24 +02:00
parent d2f9141a5c
commit 1df9be6563
13 changed files with 1930 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
tests/cli/test_cli.py
View File

@@ -39,12 +39,36 @@ class TestIngest:
assert "5 chunks" in result.stdout
assert "3 citations" in result.stdout
def test_ingest_with_rich_flag(self) -> None:
"""Test `ingest --rich` passes rich=True and reports formulas/figures."""
with patch("codex.ingest.ingest_paper") as mock_ingest:
mock_ingest.return_value = IngestResult(
paper_id="2301.07041",
chunks_upserted=5,
citations_upserted=3,
formulas_upserted=12,
figures_upserted=4,
)
result = runner.invoke(app, ["ingest", "2301.07041", "--source", "paper.pdf", "--rich"])
assert result.exit_code == 0
assert "12 formulas" in result.stdout
assert "4 figures" in result.stdout
# Verify rich=True was passed
_, kwargs = mock_ingest.call_args
assert kwargs.get("rich") is True or mock_ingest.call_args[1].get("rich") is True
class TestSearch:
"""Tests for `codex search` command."""
"""Tests for `codex search` subcommands (paper + formula)."""
def test_search_command(self) -> None:
"""Test search command with mocked embedder and DB."""
"""Alias: delegates to test_search_paper_command for backwards compat."""
self.test_search_paper_command()
def test_search_paper_command(self) -> None:
"""Test `search paper` command with mocked embedder and DB."""
with (
patch("codex.embed.get_embedder") as mock_embedder,
patch("codex.db.get_conn") as mock_get_conn,
@@ -74,7 +98,7 @@ class TestSearch:
]
mock_get_conn.return_value = mock_conn
result = runner.invoke(app, ["search", "machine learning", "--limit", "2"])
result = runner.invoke(app, ["search", "paper", "machine learning", "--limit", "2"])
assert result.exit_code == 0
assert "2301.07041" in result.stdout
@@ -82,6 +106,43 @@ class TestSearch:
assert "Test Paper" in result.stdout
assert "0.123" in result.stdout
def test_search_formula_command(self) -> None:
"""Test `search formula` command with mocked DB."""
with patch("codex.db.get_conn") as mock_get_conn:
mock_conn = MagicMock()
mock_conn.__enter__.return_value = mock_conn
mock_conn.__exit__.return_value = None
mock_conn.execute.return_value.fetchall.return_value = [
{
"paper_id": "2301.07041",
"page": 3,
"raw_latex": r"\sum_{i=1}^{n} x_i",
"context": "summation formula",
"rank": 0.856,
},
]
mock_get_conn.return_value = mock_conn
result = runner.invoke(app, ["search", "formula", "summation", "--limit", "5"])
assert result.exit_code == 0
assert "2301.07041" in result.stdout
assert "p.3" in result.stdout
def test_search_formula_no_results(self) -> None:
"""`search formula` with no DB matches prints a 'no results' message."""
with patch("codex.db.get_conn") as mock_get_conn:
mock_conn = MagicMock()
mock_conn.__enter__.return_value = mock_conn
mock_conn.__exit__.return_value = None
mock_conn.execute.return_value.fetchall.return_value = []
mock_get_conn.return_value = mock_conn
result = runner.invoke(app, ["search", "formula", "nonexistent"])
assert result.exit_code == 0
assert "No matching" in result.stdout
class TestDiscoverLeads:
"""Tests for `codex discover leads` command."""

133
tests/ingest/test_ingest.py
View File

@@ -14,7 +14,7 @@ import numpy as np
import pytest
from codex.ingest import IngestResult, ingest_paper
from codex.models import Citation, Paper
from codex.models import Citation, FigureChunk, FormulaChunk, Paper
# ---------------------------------------------------------------------------
# Helpers
@@ -288,3 +288,134 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
emb: list[float] = params["abstract_emb"]
assert len(emb) == 1024
assert all(v == 0.0 for v in emb)
# ---------------------------------------------------------------------------
# 8. F-09 Rich Parsing tests
# ---------------------------------------------------------------------------
def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> None:
"""When rich=True and source is PDF, formula + figure parsers are invoked."""
import codex.parsing.figures as _figures_mod
import codex.parsing.mathpix as _mathpix_mod
pdf_file = tmp_path / "paper.pdf"
pdf_file.write_bytes(b"%PDF-1.4 fake content")
paper = _make_paper()
mock_conn = MagicMock()
mock_conn.execute = MagicMock()
mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock()
fake_formula = FormulaChunk(paper_id=paper.id, page=1, raw_latex=r"\alpha", context="test")
fake_figure = FigureChunk(
paper_id=paper.id, page=1, image_path="/tmp/fig.png", caption="Figure 1."
)
mock_settings = MagicMock()
mock_settings.figures_dir = str(tmp_path / "figures")
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
patch("codex.parsing.nougat.pdf_to_markdown", return_value=""),
patch("codex.parsing.tex.chunk_text", return_value=[]),
patch("codex.parsing.grobid.extract_references", return_value=[]),
patch.object(
_mathpix_mod, "extract_formulas", return_value=[fake_formula]
) as mock_formulas,
patch.object(_figures_mod, "extract_figures", return_value=[fake_figure]) as mock_figures,
patch("codex.ingest.get_settings", return_value=mock_settings),
):
result = ingest_paper(paper.id, source_path=str(pdf_file), rich=True)
mock_formulas.assert_called_once()
mock_figures.assert_called_once()
assert result.formulas_upserted == 1
assert result.figures_upserted == 1
def test_ingest_paper_no_rich_skips_parsers(tmp_path: Any) -> None:
"""When rich=False (default), formula and figure parsers are NOT called."""
import codex.parsing.figures as _figures_mod
import codex.parsing.mathpix as _mathpix_mod
pdf_file = tmp_path / "paper.pdf"
pdf_file.write_bytes(b"%PDF-1.4 fake content")
paper = _make_paper()
mock_conn = MagicMock()
mock_conn.execute = MagicMock()
mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock()
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
patch("codex.parsing.nougat.pdf_to_markdown", return_value=""),
patch("codex.parsing.tex.chunk_text", return_value=[]),
patch("codex.parsing.grobid.extract_references", return_value=[]),
patch.object(_mathpix_mod, "extract_formulas", return_value=[]) as mock_formulas,
patch.object(_figures_mod, "extract_figures", return_value=[]) as mock_figures,
):
result = ingest_paper(paper.id, source_path=str(pdf_file), rich=False)
mock_formulas.assert_not_called()
mock_figures.assert_not_called()
assert result.formulas_upserted == 0
assert result.figures_upserted == 0
def test_ingest_paper_rich_requires_pdf_source(tmp_path: Any) -> None:
"""When rich=True but source is a .tex file, formula/figure parsers are NOT called."""
import codex.parsing.figures as _figures_mod
import codex.parsing.mathpix as _mathpix_mod
tex_file = tmp_path / "paper.tex"
tex_file.write_text(r"\section{Intro}")
paper = _make_paper()
mock_conn = MagicMock()
mock_conn.execute = MagicMock()
mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock()
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
patch("codex.parsing.tex.latex_to_text", return_value=""),
patch("codex.parsing.tex.chunk_text", return_value=[]),
patch.object(_mathpix_mod, "extract_formulas", return_value=[]) as mock_formulas,
patch.object(_figures_mod, "extract_figures", return_value=[]) as mock_figures,
):
result = ingest_paper(paper.id, source_path=str(tex_file), rich=True)
mock_formulas.assert_not_called()
mock_figures.assert_not_called()
assert result.formulas_upserted == 0
assert result.figures_upserted == 0
def test_ingest_result_has_formula_figure_counts() -> None:
"""IngestResult has formulas_upserted and figures_upserted fields with default 0."""
result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0)
assert result.formulas_upserted == 0
assert result.figures_upserted == 0
result2 = IngestResult(
paper_id="test",
chunks_upserted=5,
citations_upserted=3,
formulas_upserted=10,
figures_upserted=2,
)
assert result2.formulas_upserted == 10
assert result2.figures_upserted == 2

245
tests/parsing/test_figures.py Normal file
View File

@@ -0,0 +1,245 @@
"""Tests for codex.parsing.figures — figure extraction module.
All external dependencies (fitz, PIL) are mocked so the suite runs offline.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
def _make_mock_rect(
x0: float = 10.0, y0: float = 10.0, x1: float = 200.0, y1: float = 150.0
) -> MagicMock:
"""Build a mock fitz.Rect-like object."""
rect = MagicMock()
rect.x0 = x0
rect.y0 = y0
rect.x1 = x1
rect.y1 = y1
return rect
def _make_mock_doc(
images: list[tuple[int, ...]], # list of (xref, ...)
text_blocks: list[Any],
img_rects: list[Any],
page_count: int = 1,
) -> tuple[MagicMock, MagicMock]:
"""Build a mock fitz document with one page."""
mock_pix = MagicMock()
mock_pix.n = 3 # RGB — no conversion needed
mock_pix.save = MagicMock()
mock_page = MagicMock()
mock_page.get_images.return_value = images
mock_page.get_text.return_value = text_blocks
mock_page.get_image_rects.return_value = img_rects
mock_doc = MagicMock()
mock_doc.__len__.return_value = page_count
mock_doc.__getitem__ = MagicMock(return_value=mock_page)
return mock_doc, mock_pix
class TestExtractFigures:
"""Tests for extract_figures()."""
def test_no_caption_returns_empty_string(self, tmp_path: Any) -> None:
"""Figure with no nearby caption gets empty caption string."""
img_rect = _make_mock_rect(10, 10, 200, 150)
text_blocks = [
(300.0, 300.0, 400.0, 320.0, "Unrelated text here.", 0, 0),
]
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=text_blocks,
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert len(results) == 1
assert results[0].caption == ""
def test_figure_png_is_saved(self, tmp_path: Any) -> None:
"""PNG save is called with the correct output path."""
img_rect = _make_mock_rect()
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[],
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert len(results) == 1
mock_pix.save.assert_called_once()
saved_path: str = mock_pix.save.call_args[0][0]
assert saved_path.endswith(".png")
def test_caption_detected_below_image(self, tmp_path: Any) -> None:
"""Caption block immediately below the image is detected."""
img_rect = _make_mock_rect(10, 10, 200, 150)
caption_block = (10.0, 155.0, 200.0, 170.0, "Figure 1: Architecture overview.", 0, 0)
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[caption_block],
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert results[0].caption == "Figure 1: Architecture overview."
def test_caption_detected_above_image(self, tmp_path: Any) -> None:
"""Caption block immediately above the image is also detected."""
img_rect = _make_mock_rect(10, 80, 200, 200)
caption_block = (10.0, 50.0, 200.0, 75.0, "Fig. 2: Loss curve.", 0, 0)
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[caption_block],
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert results[0].caption == "Fig. 2: Loss curve."
def test_german_caption_prefix(self, tmp_path: Any) -> None:
"""'Abbildung' prefix is recognized as a valid German caption."""
img_rect = _make_mock_rect(10, 10, 200, 150)
caption_block = (10.0, 155.0, 200.0, 170.0, "Abbildung 3: Ergebnisse.", 0, 0)
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[caption_block],
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert "Abbildung" in results[0].caption
def test_unrelated_text_not_used_as_caption(self, tmp_path: Any) -> None:
"""Text blocks not starting with a caption prefix are ignored."""
img_rect = _make_mock_rect(10, 10, 200, 150)
non_caption = (10.0, 155.0, 200.0, 170.0, "This is a random sentence.", 0, 0)
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[non_caption],
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert results[0].caption == ""
def test_no_images_returns_empty_list(self, tmp_path: Any) -> None:
"""PDF with no embedded images returns empty list."""
mock_doc, mock_pix = _make_mock_doc(
images=[],
text_blocks=[],
img_rects=[],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert results == []
def test_image_without_rect_is_skipped(self, tmp_path: Any) -> None:
"""Images for which get_image_rects returns empty list are skipped."""
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[],
img_rects=[], # no rect
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("paper.pdf", output_dir=str(tmp_path))
assert results == []
def test_output_dir_created_if_absent(self, tmp_path: Any) -> None:
"""output_dir is created if it does not exist."""
new_dir = tmp_path / "new_subdir" / "figures"
assert not new_dir.exists()
mock_doc, mock_pix = _make_mock_doc(images=[], text_blocks=[], img_rects=[])
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
extract_figures("paper.pdf", output_dir=str(new_dir))
assert new_dir.exists()
def test_paper_id_from_pdf_stem(self, tmp_path: Any) -> None:
"""paper_id in FigureChunk is the stem of the PDF filename."""
img_rect = _make_mock_rect()
mock_doc, mock_pix = _make_mock_doc(
images=[(5, 0, 0, 0, 0, 0, 0, 0, 8, "png")],
text_blocks=[],
img_rects=[img_rect],
)
with (
patch("fitz.open", return_value=mock_doc),
patch("fitz.Pixmap", return_value=mock_pix),
):
from codex.parsing.figures import extract_figures
results = extract_figures("/tmp/2301.07041.pdf", output_dir=str(tmp_path))
assert results[0].paper_id == "2301.07041"

386
tests/parsing/test_mathpix.py Normal file
View File

@@ -0,0 +1,386 @@
"""Tests for codex.parsing.mathpix — formula extraction module.
All external dependencies (fitz, pix2tex, httpx, get_settings) are mocked
so the suite runs offline without real API calls or model downloads.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from codex.models import FormulaChunk
def _mock_open_pdf() -> MagicMock:
"""Return a mock that satisfies ``open(path, 'rb')`` as a context manager."""
mock_file = MagicMock()
mock_file.__enter__ = MagicMock(return_value=mock_file)
mock_file.__exit__ = MagicMock(return_value=False)
mock_open = MagicMock(return_value=mock_file)
return mock_open
class TestMathPixAPIPath:
"""Tests for the MathPix cloud API backend."""
def test_mathpix_200_returns_formulas(self) -> None:
"""extract_formulas routes to MathPix when creds are provided and parses result."""
mmd_content = "\\[\n\\alpha + \\beta\n\\]\n\\[\nE = mc^2\n\\]"
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = True
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch("builtins.open", _mock_open_pdf()),
patch("httpx.post") as mock_post,
patch("httpx.get") as mock_get,
patch("time.sleep"),
):
mock_post.return_value = MagicMock(
status_code=200,
json=lambda: {"pdf_id": "abc123"},
)
mock_post.return_value.raise_for_status = MagicMock()
mock_get.return_value = MagicMock(
status_code=200,
text=mmd_content,
)
from codex.parsing.mathpix import _extract_formulas_mathpix
results = _extract_formulas_mathpix("paper.pdf", "app_id", "app_key")
assert len(results) == 2
assert all(isinstance(r, FormulaChunk) for r in results)
assert results[0].raw_latex == "\\[\n\\alpha + \\beta\n\\]"
def test_mathpix_empty_pdf_id_returns_empty(self) -> None:
"""When MathPix returns no pdf_id, result is empty list."""
with (
patch("builtins.open", _mock_open_pdf()),
patch("httpx.post") as mock_post,
patch("time.sleep"),
):
mock_post.return_value = MagicMock(
status_code=200,
json=lambda: {}, # No pdf_id
)
mock_post.return_value.raise_for_status = MagicMock()
from codex.parsing.mathpix import _extract_formulas_mathpix
results = _extract_formulas_mathpix("paper.pdf", "app_id", "app_key")
assert results == []
def test_mathpix_paper_id_from_stem(self) -> None:
"""paper_id in FormulaChunk is derived from the PDF filename stem."""
mmd_content = "\\[\nx^2\n\\]"
with (
patch("builtins.open", _mock_open_pdf()),
patch("httpx.post") as mock_post,
patch("httpx.get") as mock_get,
patch("time.sleep"),
):
mock_post.return_value = MagicMock(
status_code=200,
json=lambda: {"pdf_id": "xyz"},
)
mock_post.return_value.raise_for_status = MagicMock()
mock_get.return_value = MagicMock(status_code=200, text=mmd_content)
from codex.parsing.mathpix import _extract_formulas_mathpix
results = _extract_formulas_mathpix("/tmp/2301.07041.pdf", "id", "key")
assert results[0].paper_id == "2301.07041"
def test_mathpix_page_counter_increments(self) -> None:
"""Page counter increments on MathPix page markers in MMD."""
mmd_content = "\\[\nA\n\\]\n<!-- Page 2 -->\n\\[\nB\n\\]"
with (
patch("builtins.open", _mock_open_pdf()),
patch("httpx.post") as mock_post,
patch("httpx.get") as mock_get,
patch("time.sleep"),
):
mock_post.return_value = MagicMock(
status_code=200,
json=lambda: {"pdf_id": "p1"},
)
mock_post.return_value.raise_for_status = MagicMock()
mock_get.return_value = MagicMock(status_code=200, text=mmd_content)
from codex.parsing.mathpix import _extract_formulas_mathpix
results = _extract_formulas_mathpix("p.pdf", "id", "key")
assert results[0].page == 1
assert results[1].page == 2
def test_mathpix_http_error_propagates(self) -> None:
"""HTTP errors from MathPix are raised (tenacity reraises after retries)."""
import httpx
with (
patch("builtins.open", _mock_open_pdf()),
patch("httpx.post") as mock_post,
):
mock_post.return_value = MagicMock(status_code=401)
mock_post.return_value.raise_for_status.side_effect = httpx.HTTPStatusError(
"401", request=MagicMock(), response=MagicMock()
)
from codex.parsing.mathpix import _extract_formulas_mathpix
with pytest.raises(httpx.HTTPStatusError):
_extract_formulas_mathpix("p.pdf", "id", "key")
class TestPix2TexFallbackPath:
"""Tests for the local pix2tex backend."""
def _make_fitz_page(
self,
blocks: list[Any],
pixmap_samples: bytes = b"\xff" * (30 * 100 * 3),
pixmap_w: int = 100,
pixmap_h: int = 30,
) -> MagicMock:
"""Build a mock fitz page with given text blocks and a fake pixmap."""
mock_pix = MagicMock()
mock_pix.width = pixmap_w
mock_pix.height = pixmap_h
mock_pix.samples = pixmap_samples
mock_page = MagicMock()
mock_page.get_text.return_value = blocks
mock_page.get_pixmap.return_value = mock_pix
return mock_page
def test_pix2tex_fallback_invoked_without_creds(self) -> None:
"""When no MathPix creds, pix2tex fallback is called."""
math_block = (0.0, 0.0, 200.0, 50.0, "∑ α∫β∇γ∂δ∞ε", 0, 0)
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = True
mock_settings.figures_dir = "/tmp/figs"
mock_model = MagicMock(return_value=r"\sum \alpha")
mock_page = self._make_fitz_page([math_block])
mock_doc = MagicMock()
mock_doc.__len__.return_value = 1
mock_doc.__getitem__ = MagicMock(return_value=mock_page)
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch("codex.parsing.mathpix._get_pix2tex_model", return_value=mock_model),
patch("fitz.open", return_value=mock_doc),
patch("fitz.Matrix"),
patch("fitz.Rect"),
):
from PIL import Image
with patch.object(Image, "frombytes", return_value=MagicMock()):
from codex.parsing.mathpix import extract_formulas
results = extract_formulas("paper.pdf")
assert len(results) == 1
assert results[0].raw_latex == r"\sum \alpha"
def test_pix2tex_bbox_size_filter(self) -> None:
"""Blocks smaller than MIN_HEIGHT_PT or MIN_WIDTH_PT are skipped."""
tiny_block = (0.0, 0.0, 10.0, 5.0, "∑∫∂∞∇ε", 0, 0) # 10pt wide, 5pt tall
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = True
mock_model = MagicMock(return_value=r"\sum")
mock_page = self._make_fitz_page([tiny_block])
mock_doc = MagicMock()
mock_doc.__len__.return_value = 1
mock_doc.__getitem__ = MagicMock(return_value=mock_page)
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch("codex.parsing.mathpix._get_pix2tex_model", return_value=mock_model),
patch("fitz.open", return_value=mock_doc),
patch("fitz.Matrix"),
patch("fitz.Rect"),
):
from codex.parsing.mathpix import extract_formulas
results = extract_formulas("paper.pdf")
assert results == []
def test_pix2tex_math_ratio_filter(self) -> None:
"""Blocks with low math-character ratio are skipped."""
plain_text_block = (0.0, 0.0, 200.0, 50.0, "This is just regular text with no math.", 0, 0)
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = True
mock_model = MagicMock(return_value=r"\alpha")
mock_page = self._make_fitz_page([plain_text_block])
mock_doc = MagicMock()
mock_doc.__len__.return_value = 1
mock_doc.__getitem__ = MagicMock(return_value=mock_page)
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch("codex.parsing.mathpix._get_pix2tex_model", return_value=mock_model),
patch("fitz.open", return_value=mock_doc),
patch("fitz.Matrix"),
patch("fitz.Rect"),
):
from codex.parsing.mathpix import extract_formulas
results = extract_formulas("paper.pdf")
assert results == []
def test_pix2tex_exception_swallowed(self) -> None:
"""If pix2tex raises, the block is skipped (no exception propagates)."""
math_block = (0.0, 0.0, 200.0, 50.0, "∑ α∫β∇γ∂δ∞ε", 0, 0)
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = True
mock_model = MagicMock(side_effect=RuntimeError("GPU OOM"))
mock_page = self._make_fitz_page([math_block])
mock_doc = MagicMock()
mock_doc.__len__.return_value = 1
mock_doc.__getitem__ = MagicMock(return_value=mock_page)
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch("codex.parsing.mathpix._get_pix2tex_model", return_value=mock_model),
patch("fitz.open", return_value=mock_doc),
patch("fitz.Matrix"),
patch("fitz.Rect"),
):
from PIL import Image
with patch.object(Image, "frombytes", return_value=MagicMock()):
from codex.parsing.mathpix import extract_formulas
results = extract_formulas("paper.pdf") # Must not raise
assert results == []
class TestRouteSelection:
"""Tests for extract_formulas route selection logic."""
def test_mathpix_selected_when_creds_present(self) -> None:
"""extract_formulas calls MathPix backend when both creds are set."""
mock_settings = MagicMock()
mock_settings.mathpix_app_id = "my_id"
mock_settings.mathpix_app_key = "my_key"
mock_settings.pix2tex_fallback = True
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch(
"codex.parsing.mathpix._extract_formulas_mathpix", return_value=[]
) as mock_mathpix,
patch(
"codex.parsing.mathpix._extract_formulas_pix2tex", return_value=[]
) as mock_pix2tex,
):
from codex.parsing.mathpix import extract_formulas
extract_formulas("paper.pdf")
mock_mathpix.assert_called_once_with("paper.pdf", "my_id", "my_key")
mock_pix2tex.assert_not_called()
def test_pix2tex_selected_without_creds(self) -> None:
"""extract_formulas calls pix2tex when no MathPix creds are set."""
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = True
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch(
"codex.parsing.mathpix._extract_formulas_mathpix", return_value=[]
) as mock_mathpix,
patch(
"codex.parsing.mathpix._extract_formulas_pix2tex", return_value=[]
) as mock_pix2tex,
):
from codex.parsing.mathpix import extract_formulas
extract_formulas("paper.pdf")
mock_pix2tex.assert_called_once_with("paper.pdf")
mock_mathpix.assert_not_called()
def test_empty_string_creds_treated_as_absent(self) -> None:
"""Empty string credentials fall through to pix2tex (not MathPix)."""
mock_settings = MagicMock()
mock_settings.mathpix_app_id = ""
mock_settings.mathpix_app_key = ""
mock_settings.pix2tex_fallback = True
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch(
"codex.parsing.mathpix._extract_formulas_mathpix", return_value=[]
) as mock_mathpix,
patch(
"codex.parsing.mathpix._extract_formulas_pix2tex", return_value=[]
) as mock_pix2tex,
):
from codex.parsing.mathpix import extract_formulas
extract_formulas("paper.pdf")
mock_pix2tex.assert_called_once()
mock_mathpix.assert_not_called()
def test_formula_extraction_disabled_when_fallback_false(self) -> None:
"""No extraction when pix2tex_fallback=False and no creds."""
mock_settings = MagicMock()
mock_settings.mathpix_app_id = None
mock_settings.mathpix_app_key = None
mock_settings.pix2tex_fallback = False
with (
patch("codex.parsing.mathpix.get_settings", return_value=mock_settings),
patch(
"codex.parsing.mathpix._extract_formulas_mathpix", return_value=[]
) as mock_mathpix,
patch(
"codex.parsing.mathpix._extract_formulas_pix2tex", return_value=[]
) as mock_pix2tex,
):
from codex.parsing.mathpix import extract_formulas
results = extract_formulas("paper.pdf")
assert results == []
mock_mathpix.assert_not_called()
mock_pix2tex.assert_not_called()