feat(F-09): rich parsing — formula + figure extraction #1
81
codex/cli.py
81
codex/cli.py
@@ -11,14 +11,9 @@ import typer
|
||||
app = typer.Typer(help="codex — personal knowledge base for scientific papers.")
|
||||
discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
||||
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||||
search_app = typer.Typer(
|
||||
help="Search commands: paper abstracts (default) or formulas (formula subcommand).",
|
||||
invoke_without_command=True,
|
||||
)
|
||||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||||
app.add_typer(discover_app, name="discover")
|
||||
app.add_typer(prov_app, name="provenance")
|
||||
app.add_typer(search_app, name="search")
|
||||
app.add_typer(wiki_app, name="wiki")
|
||||
|
||||
|
||||
@@ -26,33 +21,19 @@ app.add_typer(wiki_app, name="wiki")
|
||||
def ingest(
|
||||
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
|
||||
source: Optional[str] = typer.Option(None, "--source", "-s", help="Path to .tex or .pdf"), # noqa: UP045
|
||||
rich: bool = typer.Option( # noqa: FBT002
|
||||
False, "--rich", help="Also extract formulas and figures (PDF only)."
|
||||
),
|
||||
) -> None:
|
||||
"""Ingest a paper into the knowledge base."""
|
||||
from codex.ingest import ingest_paper
|
||||
|
||||
result = ingest_paper(paper_id, source_path=source, rich=rich)
|
||||
msg = (
|
||||
result = ingest_paper(paper_id, source_path=source)
|
||||
typer.echo(
|
||||
f"Ingested {result.paper_id}: {result.chunks_upserted} chunks, "
|
||||
f"{result.citations_upserted} citations"
|
||||
)
|
||||
if rich:
|
||||
msg += f", {result.formulas_upserted} formulas, {result.figures_upserted} figures"
|
||||
typer.echo(msg)
|
||||
|
||||
|
||||
@search_app.callback(invoke_without_command=True)
|
||||
def search_callback(ctx: typer.Context) -> None:
|
||||
"""Search commands for papers and formulas."""
|
||||
if ctx.invoked_subcommand is None:
|
||||
typer.echo(ctx.get_help())
|
||||
raise typer.Exit(0)
|
||||
|
||||
|
||||
@search_app.command("paper")
|
||||
def search_paper(
|
||||
@app.command()
|
||||
def search(
|
||||
query: str = typer.Argument(..., help="Natural-language search query"),
|
||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||||
) -> None:
|
||||
@@ -179,50 +160,6 @@ def ask(question: str = typer.Argument(..., help="Question to answer")) -> None:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F-09: Specialised search commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@search_app.command("formula")
|
||||
def search_formula(
|
||||
query: str = typer.Argument(..., help="LaTeX snippet or natural-language description"),
|
||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||||
) -> None:
|
||||
"""Search for mathematical formulas by LaTeX content or surrounding context.
|
||||
|
||||
Performs a full-text search over ``formulas.raw_latex`` and
|
||||
``formulas.context``, ranked by relevance.
|
||||
"""
|
||||
from codex.db import get_conn
|
||||
|
||||
_fts = "to_tsvector('english', coalesce(f.raw_latex, '') || ' ' || coalesce(f.context, ''))"
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT f.id, f.paper_id, f.page, f.raw_latex, f.context, f.eq_label,
|
||||
ts_rank({_fts}, plainto_tsquery('english', %(query)s)) AS rank
|
||||
FROM formulas f
|
||||
WHERE {_fts} @@ plainto_tsquery('english', %(query)s)
|
||||
ORDER BY rank DESC
|
||||
LIMIT %(limit)s
|
||||
""",
|
||||
{"query": query, "limit": limit},
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
typer.echo("No formula results found.")
|
||||
return
|
||||
|
||||
for row in rows:
|
||||
label = f" [{row['eq_label']}]" if row["eq_label"] else ""
|
||||
typer.echo(
|
||||
f"[rank={row['rank']:.3f}] {row['paper_id']} p.{row['page']}{label}\n"
|
||||
f" LaTeX: {row['raw_latex']}\n"
|
||||
f" Context: {row['context'][:120] if row['context'] else ''}\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F-12: Wiki command group
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -272,6 +209,7 @@ def wiki_list(
|
||||
),
|
||||
) -> None:
|
||||
"""List compiled concept pages with freshness information."""
|
||||
import json as _json
|
||||
from pathlib import Path
|
||||
|
||||
from codex.config import get_settings
|
||||
@@ -283,8 +221,8 @@ def wiki_list(
|
||||
state: dict[str, str] = {}
|
||||
if state_path.exists():
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
state = _json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except (_json.JSONDecodeError, OSError):
|
||||
state = {}
|
||||
|
||||
pages = sorted(wiki_dir.glob("*.md"))
|
||||
@@ -297,13 +235,10 @@ def wiki_list(
|
||||
continue
|
||||
slug = page.stem
|
||||
hash_val = state.get(slug, "—")[:8]
|
||||
# Count claims and ungrounded in the page markdown
|
||||
content = page.read_text(encoding="utf-8")
|
||||
n_claims = content.count("[")
|
||||
n_ungrounded = content.count("⚠")
|
||||
mtime = datetime.fromtimestamp(page.stat().st_mtime, tz=UTC).strftime(
|
||||
"%Y-%m-%d %H:%M"
|
||||
)
|
||||
mtime = datetime.fromtimestamp(page.stat().st_mtime, tz=UTC).strftime("%Y-%m-%d %H:%M")
|
||||
typer.echo(
|
||||
f"{slug:40s} last={mtime} hash={hash_val} claims≈{n_claims} ⚠={n_ungrounded}"
|
||||
)
|
||||
|
||||
@@ -84,41 +84,6 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# F-09 Rich Parsing — MathPix + pix2tex + figures
|
||||
# ------------------------------------------------------------------
|
||||
mathpix_app_id: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"MathPix application ID for the MathPix API "
|
||||
"(env var: MATHPIX_APP_ID). Optional — pix2tex is used as fallback."
|
||||
),
|
||||
)
|
||||
|
||||
mathpix_app_key: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"MathPix application key for the MathPix API "
|
||||
"(env var: MATHPIX_APP_KEY). Optional — pix2tex is used as fallback."
|
||||
),
|
||||
)
|
||||
|
||||
pix2tex_fallback: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Enable pix2tex LatexOCR as the local fallback for formula extraction "
|
||||
"when MathPix credentials are not set. Disable only for testing."
|
||||
),
|
||||
)
|
||||
|
||||
figures_dir: str = Field(
|
||||
default="figures/",
|
||||
description=(
|
||||
"Directory where extracted figure images (PNG) are written. "
|
||||
"Relative paths are resolved from the current working directory."
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# F-12 Wiki-Compile
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user