feat(F-09): rich parsing — formula + figure extraction
- codex/parsing/mathpix.py: pix2tex (local, CPU) primary + MathPix API optional; bbox heuristic h>15px, math-char-count>5; singleton model cache - codex/parsing/figures.py: pymupdf embedded-image extraction → PNG; caption detection via proximity + "Figure/Fig./Abbildung" prefix - codex/models.py: FormulaChunk + FigureChunk dataclasses (R-10/R-11) - codex/ingest.py: --rich flag wires formula+figure extraction post-ingest - codex/cli.py: search_app sub-typer (paper + formula subcommands), --rich flag on ingest; wiki_app from F-12 preserved intact - codex/config.py: mathpix_app_id/key, pix2tex_fallback, figures_dir - infra/schema.sql: formulas + figures tables with HNSW pgvector indexes - pyproject.toml: pymupdf>=1.24, pix2tex>=0.1.4 - tests/parsing/test_mathpix.py + test_figures.py: 31 tests (mock pix2tex + MathPix HTTP, real pymupdf on synthetic PDF) Gate: 158 passed, ruff clean, mypy clean (20 files) Requirements: R-10 R-11 R-12 R-13 R-14 → done Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
65
codex/cli.py
65
codex/cli.py
@@ -12,28 +12,54 @@ app = typer.Typer(help="codex — personal knowledge base for scientific papers.
|
||||
discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
||||
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||||
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
|
||||
search_app = typer.Typer(
|
||||
help="Search: semantic paper search and formula full-text search.",
|
||||
invoke_without_command=True,
|
||||
)
|
||||
app.add_typer(discover_app, name="discover")
|
||||
app.add_typer(prov_app, name="provenance")
|
||||
app.add_typer(wiki_app, name="wiki")
|
||||
app.add_typer(search_app, name="search")
|
||||
|
||||
|
||||
@app.command()
|
||||
def ingest(
|
||||
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
|
||||
source: Optional[str] = typer.Option(None, "--source", "-s", help="Path to .tex or .pdf"), # noqa: UP045
|
||||
rich: bool = typer.Option( # noqa: A002
|
||||
False,
|
||||
"--rich",
|
||||
help="Extract formulas and figures (requires PDF source, F-09).",
|
||||
),
|
||||
) -> None:
|
||||
"""Ingest a paper into the knowledge base."""
|
||||
from codex.ingest import ingest_paper
|
||||
|
||||
result = ingest_paper(paper_id, source_path=source)
|
||||
result = ingest_paper(paper_id, source_path=source, rich=rich)
|
||||
typer.echo(
|
||||
f"Ingested {result.paper_id}: {result.chunks_upserted} chunks, "
|
||||
f"{result.citations_upserted} citations"
|
||||
f"{result.citations_upserted} citations, "
|
||||
f"{result.formulas_upserted} formulas, "
|
||||
f"{result.figures_upserted} figures"
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def search(
|
||||
# ---------------------------------------------------------------------------
|
||||
# F-09: Search command group
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@search_app.callback(invoke_without_command=True)
|
||||
def search_callback(ctx: typer.Context) -> None:
|
||||
"""Search subcommands: ``paper`` for semantic search, ``formula`` for LaTeX FTS."""
|
||||
if ctx.invoked_subcommand is None:
|
||||
typer.echo(ctx.get_help())
|
||||
raise typer.Exit(0)
|
||||
|
||||
|
||||
@search_app.command("paper")
|
||||
def search_paper(
|
||||
query: str = typer.Argument(..., help="Natural-language search query"),
|
||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||||
) -> None:
|
||||
@@ -58,6 +84,37 @@ def search(
|
||||
typer.echo(f"[{row['distance']:.3f}] {row['id']} ({row['year']}) — {row['title']}")
|
||||
|
||||
|
||||
@search_app.command("formula")
|
||||
def search_formula(
|
||||
query: str = typer.Argument(..., help="LaTeX snippet or keyword to search in formulas"),
|
||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||||
) -> None:
|
||||
"""Full-text search over extracted LaTeX formulas (F-09)."""
|
||||
from codex.db import get_conn
|
||||
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT paper_id, page, raw_latex, context,
|
||||
ts_rank(to_tsvector('english', raw_latex || ' ' || coalesce(context, '')),
|
||||
plainto_tsquery('english', %(query)s)) AS rank
|
||||
FROM formulas
|
||||
WHERE to_tsvector('english', raw_latex || ' ' || coalesce(context, ''))
|
||||
@@ plainto_tsquery('english', %(query)s)
|
||||
ORDER BY rank DESC
|
||||
LIMIT %(limit)s
|
||||
""",
|
||||
{"query": query, "limit": limit},
|
||||
).fetchall()
|
||||
if not rows:
|
||||
typer.echo("No matching formulas found.")
|
||||
return
|
||||
for row in rows:
|
||||
typer.echo(
|
||||
f"[{row['rank']:.3f}] {row['paper_id']} p.{row['page']} {row['raw_latex'][:80]}"
|
||||
)
|
||||
|
||||
|
||||
@discover_app.command("leads")
|
||||
def discover_leads(
|
||||
limit: int = typer.Option(20, "--limit", "-n"),
|
||||
|
||||
Reference in New Issue
Block a user