feat(F-09): rich parsing — formula + figure extraction #1
81
codex/cli.py
81
codex/cli.py
@@ -11,14 +11,9 @@ import typer
|
|||||||
app = typer.Typer(help="codex — personal knowledge base for scientific papers.")
|
app = typer.Typer(help="codex — personal knowledge base for scientific papers.")
|
||||||
discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
||||||
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||||||
search_app = typer.Typer(
|
|
||||||
help="Search commands: paper abstracts (default) or formulas (formula subcommand).",
|
|
||||||
invoke_without_command=True,
|
|
||||||
)
|
|
||||||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||||||
app.add_typer(discover_app, name="discover")
|
app.add_typer(discover_app, name="discover")
|
||||||
app.add_typer(prov_app, name="provenance")
|
app.add_typer(prov_app, name="provenance")
|
||||||
app.add_typer(search_app, name="search")
|
|
||||||
app.add_typer(wiki_app, name="wiki")
|
app.add_typer(wiki_app, name="wiki")
|
||||||
|
|
||||||
|
|
||||||
@@ -26,33 +21,19 @@ app.add_typer(wiki_app, name="wiki")
|
|||||||
def ingest(
|
def ingest(
|
||||||
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
|
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
|
||||||
source: Optional[str] = typer.Option(None, "--source", "-s", help="Path to .tex or .pdf"), # noqa: UP045
|
source: Optional[str] = typer.Option(None, "--source", "-s", help="Path to .tex or .pdf"), # noqa: UP045
|
||||||
rich: bool = typer.Option( # noqa: FBT002
|
|
||||||
False, "--rich", help="Also extract formulas and figures (PDF only)."
|
|
||||||
),
|
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Ingest a paper into the knowledge base."""
|
"""Ingest a paper into the knowledge base."""
|
||||||
from codex.ingest import ingest_paper
|
from codex.ingest import ingest_paper
|
||||||
|
|
||||||
result = ingest_paper(paper_id, source_path=source, rich=rich)
|
result = ingest_paper(paper_id, source_path=source)
|
||||||
msg = (
|
typer.echo(
|
||||||
f"Ingested {result.paper_id}: {result.chunks_upserted} chunks, "
|
f"Ingested {result.paper_id}: {result.chunks_upserted} chunks, "
|
||||||
f"{result.citations_upserted} citations"
|
f"{result.citations_upserted} citations"
|
||||||
)
|
)
|
||||||
if rich:
|
|
||||||
msg += f", {result.formulas_upserted} formulas, {result.figures_upserted} figures"
|
|
||||||
typer.echo(msg)
|
|
||||||
|
|
||||||
|
|
||||||
@search_app.callback(invoke_without_command=True)
|
@app.command()
|
||||||
def search_callback(ctx: typer.Context) -> None:
|
def search(
|
||||||
"""Search commands for papers and formulas."""
|
|
||||||
if ctx.invoked_subcommand is None:
|
|
||||||
typer.echo(ctx.get_help())
|
|
||||||
raise typer.Exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
@search_app.command("paper")
|
|
||||||
def search_paper(
|
|
||||||
query: str = typer.Argument(..., help="Natural-language search query"),
|
query: str = typer.Argument(..., help="Natural-language search query"),
|
||||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -179,50 +160,6 @@ def ask(question: str = typer.Argument(..., help="Question to answer")) -> None:
|
|||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# F-09: Specialised search commands
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@search_app.command("formula")
|
|
||||||
def search_formula(
|
|
||||||
query: str = typer.Argument(..., help="LaTeX snippet or natural-language description"),
|
|
||||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
|
||||||
) -> None:
|
|
||||||
"""Search for mathematical formulas by LaTeX content or surrounding context.
|
|
||||||
|
|
||||||
Performs a full-text search over ``formulas.raw_latex`` and
|
|
||||||
``formulas.context``, ranked by relevance.
|
|
||||||
"""
|
|
||||||
from codex.db import get_conn
|
|
||||||
|
|
||||||
_fts = "to_tsvector('english', coalesce(f.raw_latex, '') || ' ' || coalesce(f.context, ''))"
|
|
||||||
with get_conn() as conn:
|
|
||||||
rows = conn.execute(
|
|
||||||
f"""
|
|
||||||
SELECT f.id, f.paper_id, f.page, f.raw_latex, f.context, f.eq_label,
|
|
||||||
ts_rank({_fts}, plainto_tsquery('english', %(query)s)) AS rank
|
|
||||||
FROM formulas f
|
|
||||||
WHERE {_fts} @@ plainto_tsquery('english', %(query)s)
|
|
||||||
ORDER BY rank DESC
|
|
||||||
LIMIT %(limit)s
|
|
||||||
""",
|
|
||||||
{"query": query, "limit": limit},
|
|
||||||
).fetchall()
|
|
||||||
|
|
||||||
if not rows:
|
|
||||||
typer.echo("No formula results found.")
|
|
||||||
return
|
|
||||||
|
|
||||||
for row in rows:
|
|
||||||
label = f" [{row['eq_label']}]" if row["eq_label"] else ""
|
|
||||||
typer.echo(
|
|
||||||
f"[rank={row['rank']:.3f}] {row['paper_id']} p.{row['page']}{label}\n"
|
|
||||||
f" LaTeX: {row['raw_latex']}\n"
|
|
||||||
f" Context: {row['context'][:120] if row['context'] else ''}\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# F-12: Wiki command group
|
# F-12: Wiki command group
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -272,6 +209,7 @@ def wiki_list(
|
|||||||
),
|
),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""List compiled concept pages with freshness information."""
|
"""List compiled concept pages with freshness information."""
|
||||||
|
import json as _json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from codex.config import get_settings
|
from codex.config import get_settings
|
||||||
@@ -283,8 +221,8 @@ def wiki_list(
|
|||||||
state: dict[str, str] = {}
|
state: dict[str, str] = {}
|
||||||
if state_path.exists():
|
if state_path.exists():
|
||||||
try:
|
try:
|
||||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
state = _json.loads(state_path.read_text(encoding="utf-8"))
|
||||||
except (json.JSONDecodeError, OSError):
|
except (_json.JSONDecodeError, OSError):
|
||||||
state = {}
|
state = {}
|
||||||
|
|
||||||
pages = sorted(wiki_dir.glob("*.md"))
|
pages = sorted(wiki_dir.glob("*.md"))
|
||||||
@@ -297,13 +235,10 @@ def wiki_list(
|
|||||||
continue
|
continue
|
||||||
slug = page.stem
|
slug = page.stem
|
||||||
hash_val = state.get(slug, "—")[:8]
|
hash_val = state.get(slug, "—")[:8]
|
||||||
# Count claims and ungrounded in the page markdown
|
|
||||||
content = page.read_text(encoding="utf-8")
|
content = page.read_text(encoding="utf-8")
|
||||||
n_claims = content.count("[")
|
n_claims = content.count("[")
|
||||||
n_ungrounded = content.count("⚠")
|
n_ungrounded = content.count("⚠")
|
||||||
mtime = datetime.fromtimestamp(page.stat().st_mtime, tz=UTC).strftime(
|
mtime = datetime.fromtimestamp(page.stat().st_mtime, tz=UTC).strftime("%Y-%m-%d %H:%M")
|
||||||
"%Y-%m-%d %H:%M"
|
|
||||||
)
|
|
||||||
typer.echo(
|
typer.echo(
|
||||||
f"{slug:40s} last={mtime} hash={hash_val} claims≈{n_claims} ⚠={n_ungrounded}"
|
f"{slug:40s} last={mtime} hash={hash_val} claims≈{n_claims} ⚠={n_ungrounded}"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -84,41 +84,6 @@ class Settings(BaseSettings):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
# F-09 Rich Parsing — MathPix + pix2tex + figures
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
mathpix_app_id: str | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"MathPix application ID for the MathPix API "
|
|
||||||
"(env var: MATHPIX_APP_ID). Optional — pix2tex is used as fallback."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
mathpix_app_key: str | None = Field(
|
|
||||||
default=None,
|
|
||||||
description=(
|
|
||||||
"MathPix application key for the MathPix API "
|
|
||||||
"(env var: MATHPIX_APP_KEY). Optional — pix2tex is used as fallback."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
pix2tex_fallback: bool = Field(
|
|
||||||
default=True,
|
|
||||||
description=(
|
|
||||||
"Enable pix2tex LatexOCR as the local fallback for formula extraction "
|
|
||||||
"when MathPix credentials are not set. Disable only for testing."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
figures_dir: str = Field(
|
|
||||||
default="figures/",
|
|
||||||
description=(
|
|
||||||
"Directory where extracted figure images (PNG) are written. "
|
|
||||||
"Relative paths are resolved from the current working directory."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# F-12 Wiki-Compile
|
# F-12 Wiki-Compile
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user