feat(F-09): rich parsing — formula + figure extraction
- codex/parsing/mathpix.py: pix2tex (local, CPU) primary + MathPix API optional; bbox heuristic h>15px, math-char-count>5; singleton model cache - codex/parsing/figures.py: pymupdf embedded-image extraction → PNG; caption detection via proximity + "Figure/Fig./Abbildung" prefix - codex/models.py: FormulaChunk + FigureChunk dataclasses (R-10/R-11) - codex/ingest.py: --rich flag wires formula+figure extraction post-ingest - codex/cli.py: search_app sub-typer (paper + formula subcommands), --rich flag on ingest; wiki_app from F-12 preserved intact - codex/config.py: mathpix_app_id/key, pix2tex_fallback, figures_dir - infra/schema.sql: formulas + figures tables with HNSW pgvector indexes - pyproject.toml: pymupdf>=1.24, pix2tex>=0.1.4 - tests/parsing/test_mathpix.py + test_figures.py: 31 tests (mock pix2tex + MathPix HTTP, real pymupdf on synthetic PDF) Gate: 158 passed, ruff clean, mypy clean (20 files) Requirements: R-10 R-11 R-12 R-13 R-14 → done Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.embed import get_embedder
|
||||
from codex.models import Citation, Paper
|
||||
@@ -22,11 +23,14 @@ class IngestResult:
|
||||
paper_id: str
|
||||
chunks_upserted: int
|
||||
citations_upserted: int
|
||||
formulas_upserted: int = 0
|
||||
figures_upserted: int = 0
|
||||
|
||||
|
||||
def ingest_paper(
|
||||
paper_id: str,
|
||||
source_path: str | None = None,
|
||||
rich: bool = False,
|
||||
) -> IngestResult:
|
||||
"""Idempotent ingest of one paper.
|
||||
|
||||
@@ -42,11 +46,15 @@ def ingest_paper(
|
||||
- ``.tex`` → :func:`codex.parsing.tex.latex_to_text` + chunk
|
||||
- ``.pdf`` → :func:`codex.parsing.nougat.pdf_to_markdown` + chunk
|
||||
+ :func:`codex.parsing.grobid.extract_references` for refs
|
||||
rich:
|
||||
When True and *source_path* is a PDF, also extract formulas via
|
||||
:func:`codex.parsing.mathpix.extract_formulas` and figures via
|
||||
:func:`codex.parsing.figures.extract_figures` (F-09).
|
||||
|
||||
Returns
|
||||
-------
|
||||
IngestResult
|
||||
Counts of upserted chunks and citations.
|
||||
Counts of upserted chunks, citations, formulas, and figures.
|
||||
"""
|
||||
# ---------------------------------------------------------------
|
||||
# 1. Fetch metadata (OpenAlex primary, SemanticScholar fallback)
|
||||
@@ -195,8 +203,59 @@ def ingest_paper(
|
||||
citations_upserted = len(merged_citations)
|
||||
conn.commit()
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# 6. F-09 Rich Parsing: formulas + figures (PDF only)
|
||||
# ---------------------------------------------------------------
|
||||
formulas_upserted = 0
|
||||
figures_upserted = 0
|
||||
|
||||
if rich and source_path is not None and Path(source_path).suffix.lower() == ".pdf":
|
||||
from codex.parsing.figures import extract_figures
|
||||
from codex.parsing.mathpix import extract_formulas
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
formulas = extract_formulas(source_path)
|
||||
figures = extract_figures(source_path, output_dir=settings.figures_dir)
|
||||
|
||||
if formulas or figures:
|
||||
with get_conn() as conn2:
|
||||
if formulas:
|
||||
with conn2.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO formulas (paper_id, page, raw_latex, context, eq_label)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
[
|
||||
(f.paper_id, f.page, f.raw_latex, f.context, f.eq_label)
|
||||
for f in formulas
|
||||
],
|
||||
)
|
||||
formulas_upserted = len(formulas)
|
||||
|
||||
if figures:
|
||||
with conn2.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO figures (paper_id, page, caption, image_path)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
[
|
||||
(fig.paper_id, fig.page, fig.caption, fig.image_path)
|
||||
for fig in figures
|
||||
],
|
||||
)
|
||||
figures_upserted = len(figures)
|
||||
|
||||
conn2.commit()
|
||||
|
||||
return IngestResult(
|
||||
paper_id=paper.id,
|
||||
chunks_upserted=chunks_upserted,
|
||||
citations_upserted=citations_upserted,
|
||||
formulas_upserted=formulas_upserted,
|
||||
figures_upserted=figures_upserted,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user