feat(F-09): rich parsing — formula + figure extraction

- codex/parsing/mathpix.py: pix2tex (local, CPU) primary + MathPix API
  optional; bbox heuristic h>15px, math-char-count>5; singleton model cache
- codex/parsing/figures.py: pymupdf embedded-image extraction → PNG;
  caption detection via proximity + "Figure/Fig./Abbildung" prefix
- codex/models.py: FormulaChunk + FigureChunk dataclasses (R-10/R-11)
- codex/ingest.py: --rich flag wires formula+figure extraction post-ingest
- codex/cli.py: search_app sub-typer (paper + formula subcommands),
  --rich flag on ingest; wiki_app from F-12 preserved intact
- codex/config.py: mathpix_app_id/key, pix2tex_fallback, figures_dir
- infra/schema.sql: formulas + figures tables with HNSW pgvector indexes
- pyproject.toml: pymupdf>=1.24, pix2tex>=0.1.4
- tests/parsing/test_mathpix.py + test_figures.py: 31 tests (mock pix2tex
  + MathPix HTTP, real pymupdf on synthetic PDF)

Gate: 158 passed, ruff clean, mypy clean (20 files)
Requirements: R-10 R-11 R-12 R-13 R-14 → done

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 01:16:24 +02:00
parent d2f9141a5c
commit 1df9be6563
13 changed files with 1930 additions and 26 deletions

View File

@@ -12,28 +12,54 @@ app = typer.Typer(help="codex — personal knowledge base for scientific papers.
discover_app = typer.Typer(help="Discovery queries over the citation graph.")
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
search_app = typer.Typer(
help="Search: semantic paper search and formula full-text search.",
invoke_without_command=True,
)
app.add_typer(discover_app, name="discover")
app.add_typer(prov_app, name="provenance")
app.add_typer(wiki_app, name="wiki")
app.add_typer(search_app, name="search")
@app.command()
def ingest(
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
source: Optional[str] = typer.Option(None, "--source", "-s", help="Path to .tex or .pdf"), # noqa: UP045
rich: bool = typer.Option( # noqa: A002
False,
"--rich",
help="Extract formulas and figures (requires PDF source, F-09).",
),
) -> None:
"""Ingest a paper into the knowledge base."""
from codex.ingest import ingest_paper
result = ingest_paper(paper_id, source_path=source)
result = ingest_paper(paper_id, source_path=source, rich=rich)
typer.echo(
f"Ingested {result.paper_id}: {result.chunks_upserted} chunks, "
f"{result.citations_upserted} citations"
f"{result.citations_upserted} citations, "
f"{result.formulas_upserted} formulas, "
f"{result.figures_upserted} figures"
)
@app.command()
def search(
# ---------------------------------------------------------------------------
# F-09: Search command group
# ---------------------------------------------------------------------------
@search_app.callback(invoke_without_command=True)
def search_callback(ctx: typer.Context) -> None:
"""Search subcommands: ``paper`` for semantic search, ``formula`` for LaTeX FTS."""
if ctx.invoked_subcommand is None:
typer.echo(ctx.get_help())
raise typer.Exit(0)
@search_app.command("paper")
def search_paper(
query: str = typer.Argument(..., help="Natural-language search query"),
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
) -> None:
@@ -58,6 +84,37 @@ def search(
typer.echo(f"[{row['distance']:.3f}] {row['id']} ({row['year']}) — {row['title']}")
@search_app.command("formula")
def search_formula(
query: str = typer.Argument(..., help="LaTeX snippet or keyword to search in formulas"),
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
) -> None:
"""Full-text search over extracted LaTeX formulas (F-09)."""
from codex.db import get_conn
with get_conn() as conn:
rows = conn.execute(
"""
SELECT paper_id, page, raw_latex, context,
ts_rank(to_tsvector('english', raw_latex || ' ' || coalesce(context, '')),
plainto_tsquery('english', %(query)s)) AS rank
FROM formulas
WHERE to_tsvector('english', raw_latex || ' ' || coalesce(context, ''))
@@ plainto_tsquery('english', %(query)s)
ORDER BY rank DESC
LIMIT %(limit)s
""",
{"query": query, "limit": limit},
).fetchall()
if not rows:
typer.echo("No matching formulas found.")
return
for row in rows:
typer.echo(
f"[{row['rank']:.3f}] {row['paper_id']} p.{row['page']} {row['raw_latex'][:80]}"
)
@discover_app.command("leads")
def discover_leads(
limit: int = typer.Option(20, "--limit", "-n"),