feat(F-09): rich parsing — formula + figure extraction
- codex/parsing/mathpix.py: pix2tex (local, CPU) primary + MathPix API optional; bbox heuristic h>15px, math-char-count>5; singleton model cache - codex/parsing/figures.py: pymupdf embedded-image extraction → PNG; caption detection via proximity + "Figure/Fig./Abbildung" prefix - codex/models.py: FormulaChunk + FigureChunk dataclasses (R-10/R-11) - codex/ingest.py: --rich flag wires formula+figure extraction post-ingest - codex/cli.py: search_app sub-typer (paper + formula subcommands), --rich flag on ingest; wiki_app from F-12 preserved intact - codex/config.py: mathpix_app_id/key, pix2tex_fallback, figures_dir - infra/schema.sql: formulas + figures tables with HNSW pgvector indexes - pyproject.toml: pymupdf>=1.24, pix2tex>=0.1.4 - tests/parsing/test_mathpix.py + test_figures.py: 31 tests (mock pix2tex + MathPix HTTP, real pymupdf on synthetic PDF) Gate: 158 passed, ruff clean, mypy clean (20 files) Requirements: R-10 R-11 R-12 R-13 R-14 → done Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -39,12 +39,36 @@ class TestIngest:
|
||||
assert "5 chunks" in result.stdout
|
||||
assert "3 citations" in result.stdout
|
||||
|
||||
def test_ingest_with_rich_flag(self) -> None:
|
||||
"""Test `ingest --rich` passes rich=True and reports formulas/figures."""
|
||||
with patch("codex.ingest.ingest_paper") as mock_ingest:
|
||||
mock_ingest.return_value = IngestResult(
|
||||
paper_id="2301.07041",
|
||||
chunks_upserted=5,
|
||||
citations_upserted=3,
|
||||
formulas_upserted=12,
|
||||
figures_upserted=4,
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["ingest", "2301.07041", "--source", "paper.pdf", "--rich"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "12 formulas" in result.stdout
|
||||
assert "4 figures" in result.stdout
|
||||
# Verify rich=True was passed
|
||||
_, kwargs = mock_ingest.call_args
|
||||
assert kwargs.get("rich") is True or mock_ingest.call_args[1].get("rich") is True
|
||||
|
||||
|
||||
class TestSearch:
|
||||
"""Tests for `codex search` command."""
|
||||
"""Tests for `codex search` subcommands (paper + formula)."""
|
||||
|
||||
def test_search_command(self) -> None:
|
||||
"""Test search command with mocked embedder and DB."""
|
||||
"""Alias: delegates to test_search_paper_command for backwards compat."""
|
||||
self.test_search_paper_command()
|
||||
|
||||
def test_search_paper_command(self) -> None:
|
||||
"""Test `search paper` command with mocked embedder and DB."""
|
||||
with (
|
||||
patch("codex.embed.get_embedder") as mock_embedder,
|
||||
patch("codex.db.get_conn") as mock_get_conn,
|
||||
@@ -74,7 +98,7 @@ class TestSearch:
|
||||
]
|
||||
mock_get_conn.return_value = mock_conn
|
||||
|
||||
result = runner.invoke(app, ["search", "machine learning", "--limit", "2"])
|
||||
result = runner.invoke(app, ["search", "paper", "machine learning", "--limit", "2"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "2301.07041" in result.stdout
|
||||
@@ -82,6 +106,43 @@ class TestSearch:
|
||||
assert "Test Paper" in result.stdout
|
||||
assert "0.123" in result.stdout
|
||||
|
||||
def test_search_formula_command(self) -> None:
|
||||
"""Test `search formula` command with mocked DB."""
|
||||
with patch("codex.db.get_conn") as mock_get_conn:
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.__enter__.return_value = mock_conn
|
||||
mock_conn.__exit__.return_value = None
|
||||
mock_conn.execute.return_value.fetchall.return_value = [
|
||||
{
|
||||
"paper_id": "2301.07041",
|
||||
"page": 3,
|
||||
"raw_latex": r"\sum_{i=1}^{n} x_i",
|
||||
"context": "summation formula",
|
||||
"rank": 0.856,
|
||||
},
|
||||
]
|
||||
mock_get_conn.return_value = mock_conn
|
||||
|
||||
result = runner.invoke(app, ["search", "formula", "summation", "--limit", "5"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "2301.07041" in result.stdout
|
||||
assert "p.3" in result.stdout
|
||||
|
||||
def test_search_formula_no_results(self) -> None:
|
||||
"""`search formula` with no DB matches prints a 'no results' message."""
|
||||
with patch("codex.db.get_conn") as mock_get_conn:
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.__enter__.return_value = mock_conn
|
||||
mock_conn.__exit__.return_value = None
|
||||
mock_conn.execute.return_value.fetchall.return_value = []
|
||||
mock_get_conn.return_value = mock_conn
|
||||
|
||||
result = runner.invoke(app, ["search", "formula", "nonexistent"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "No matching" in result.stdout
|
||||
|
||||
|
||||
class TestDiscoverLeads:
|
||||
"""Tests for `codex discover leads` command."""
|
||||
|
||||
Reference in New Issue
Block a user