feat(F-09): rich parsing — formula + figure extraction

- codex/parsing/mathpix.py: pix2tex (local, CPU) primary + MathPix API optional; bbox heuristic h>15px, math-char-count>5; singleton model cache - codex/parsing/figures.py: pymupdf embedded-image extraction → PNG; caption detection via proximity + "Figure/Fig./Abbildung" prefix - codex/models.py: FormulaChunk + FigureChunk dataclasses (R-10/R-11) - codex/ingest.py: --rich flag wires formula+figure extraction post-ingest - codex/cli.py: search_app sub-typer (paper + formula subcommands), --rich flag on ingest; wiki_app from F-12 preserved intact - codex/config.py: mathpix_app_id/key, pix2tex_fallback, figures_dir - infra/schema.sql: formulas + figures tables with HNSW pgvector indexes - pyproject.toml: pymupdf>=1.24, pix2tex>=0.1.4 - tests/parsing/test_mathpix.py + test_figures.py: 31 tests (mock pix2tex + MathPix HTTP, real pymupdf on synthetic PDF) Gate: 158 passed, ruff clean, mypy clean (20 files) Requirements: R-10 R-11 R-12 R-13 R-14 → done Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-14 01:16:24 +02:00
parent d2f9141a5c
commit 1df9be6563
13 changed files with 1930 additions and 26 deletions
--- a/tests/ingest/test_ingest.py
+++ b/tests/ingest/test_ingest.py
@@ -14,7 +14,7 @@ import numpy as np
 import pytest

 from codex.ingest import IngestResult, ingest_paper
-from codex.models import Citation, Paper
+from codex.models import Citation, FigureChunk, FormulaChunk, Paper

 # ---------------------------------------------------------------------------
 # Helpers
@@ -288,3 +288,134 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
    emb: list[float] = params["abstract_emb"]
    assert len(emb) == 1024
    assert all(v == 0.0 for v in emb)
+
+
+# ---------------------------------------------------------------------------
+# 8. F-09 Rich Parsing tests
+# ---------------------------------------------------------------------------
+
+
+def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> None:
+    """When rich=True and source is PDF, formula + figure parsers are invoked."""
+    import codex.parsing.figures as _figures_mod
+    import codex.parsing.mathpix as _mathpix_mod
+
+    pdf_file = tmp_path / "paper.pdf"
+    pdf_file.write_bytes(b"%PDF-1.4 fake content")
+
+    paper = _make_paper()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    fake_formula = FormulaChunk(paper_id=paper.id, page=1, raw_latex=r"\alpha", context="test")
+    fake_figure = FigureChunk(
+        paper_id=paper.id, page=1, image_path="/tmp/fig.png", caption="Figure 1."
+    )
+
+    mock_settings = MagicMock()
+    mock_settings.figures_dir = str(tmp_path / "figures")
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        patch("codex.parsing.nougat.pdf_to_markdown", return_value=""),
+        patch("codex.parsing.tex.chunk_text", return_value=[]),
+        patch("codex.parsing.grobid.extract_references", return_value=[]),
+        patch.object(
+            _mathpix_mod, "extract_formulas", return_value=[fake_formula]
+        ) as mock_formulas,
+        patch.object(_figures_mod, "extract_figures", return_value=[fake_figure]) as mock_figures,
+        patch("codex.ingest.get_settings", return_value=mock_settings),
+    ):
+        result = ingest_paper(paper.id, source_path=str(pdf_file), rich=True)
+
+    mock_formulas.assert_called_once()
+    mock_figures.assert_called_once()
+    assert result.formulas_upserted == 1
+    assert result.figures_upserted == 1
+
+
+def test_ingest_paper_no_rich_skips_parsers(tmp_path: Any) -> None:
+    """When rich=False (default), formula and figure parsers are NOT called."""
+    import codex.parsing.figures as _figures_mod
+    import codex.parsing.mathpix as _mathpix_mod
+
+    pdf_file = tmp_path / "paper.pdf"
+    pdf_file.write_bytes(b"%PDF-1.4 fake content")
+
+    paper = _make_paper()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        patch("codex.parsing.nougat.pdf_to_markdown", return_value=""),
+        patch("codex.parsing.tex.chunk_text", return_value=[]),
+        patch("codex.parsing.grobid.extract_references", return_value=[]),
+        patch.object(_mathpix_mod, "extract_formulas", return_value=[]) as mock_formulas,
+        patch.object(_figures_mod, "extract_figures", return_value=[]) as mock_figures,
+    ):
+        result = ingest_paper(paper.id, source_path=str(pdf_file), rich=False)
+
+    mock_formulas.assert_not_called()
+    mock_figures.assert_not_called()
+    assert result.formulas_upserted == 0
+    assert result.figures_upserted == 0
+
+
+def test_ingest_paper_rich_requires_pdf_source(tmp_path: Any) -> None:
+    """When rich=True but source is a .tex file, formula/figure parsers are NOT called."""
+    import codex.parsing.figures as _figures_mod
+    import codex.parsing.mathpix as _mathpix_mod
+
+    tex_file = tmp_path / "paper.tex"
+    tex_file.write_text(r"\section{Intro}")
+
+    paper = _make_paper()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        patch("codex.parsing.tex.latex_to_text", return_value=""),
+        patch("codex.parsing.tex.chunk_text", return_value=[]),
+        patch.object(_mathpix_mod, "extract_formulas", return_value=[]) as mock_formulas,
+        patch.object(_figures_mod, "extract_figures", return_value=[]) as mock_figures,
+    ):
+        result = ingest_paper(paper.id, source_path=str(tex_file), rich=True)
+
+    mock_formulas.assert_not_called()
+    mock_figures.assert_not_called()
+    assert result.formulas_upserted == 0
+    assert result.figures_upserted == 0
+
+
+def test_ingest_result_has_formula_figure_counts() -> None:
+    """IngestResult has formulas_upserted and figures_upserted fields with default 0."""
+    result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0)
+    assert result.formulas_upserted == 0
+    assert result.figures_upserted == 0
+
+    result2 = IngestResult(
+        paper_id="test",
+        chunks_upserted=5,
+        citations_upserted=3,
+        formulas_upserted=10,
+        figures_upserted=2,
+    )
+    assert result2.formulas_upserted == 10
+    assert result2.figures_upserted == 2