feat(F-16): chunk quality gate + section classification
3-signal quality filter (length/alpha-ratio/bib-score) + rule-based section classifier + retroactive CLI pass. 35 new tests, 287 total green.
This commit is contained in:
46
codex/cli.py
46
codex/cli.py
@@ -13,6 +13,7 @@ discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
||||
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||||
synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
|
||||
quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
|
||||
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
|
||||
search_app = typer.Typer(
|
||||
help="Search: semantic paper search and formula full-text search.",
|
||||
@@ -22,6 +23,7 @@ app.add_typer(discover_app, name="discover")
|
||||
app.add_typer(prov_app, name="provenance")
|
||||
app.add_typer(wiki_app, name="wiki")
|
||||
app.add_typer(synth_app, name="synthesis")
|
||||
app.add_typer(quality_app, name="quality")
|
||||
app.add_typer(search_app, name="search")
|
||||
|
||||
|
||||
@@ -419,9 +421,7 @@ def synthesis_conjectures(
|
||||
conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm)
|
||||
write_leads(conjectures, leads_dir)
|
||||
|
||||
typer.echo(
|
||||
f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}"
|
||||
)
|
||||
typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}")
|
||||
|
||||
|
||||
@synth_app.command("report")
|
||||
@@ -454,13 +454,45 @@ def synthesis_report(
|
||||
|
||||
typer.echo("")
|
||||
typer.echo("=" * 72)
|
||||
typer.echo(
|
||||
f"CONJECTURES ({len(conj_files)}) → {conj_dir} "
|
||||
"[UNVERIFIED — never in wiki/]"
|
||||
)
|
||||
typer.echo(f"CONJECTURES ({len(conj_files)}) → {conj_dir} [UNVERIFIED — never in wiki/]")
|
||||
typer.echo("=" * 72)
|
||||
if not conj_files:
|
||||
typer.echo("(none)")
|
||||
for f in conj_files:
|
||||
first = f.read_text(encoding="utf-8").splitlines()[0]
|
||||
typer.echo(f" {first}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F-16 — quality sub-commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@quality_app.command("run")
|
||||
def quality_run(
|
||||
paper_id: Optional[str] = typer.Option( # noqa: UP045
|
||||
None,
|
||||
"--paper-id",
|
||||
help="Restrict pass to one paper (omit to process all papers).",
|
||||
),
|
||||
) -> None:
|
||||
"""Apply quality gate + section classification to existing chunks.
|
||||
|
||||
Iterates over chunks in the database, removes those that fail the
|
||||
quality thresholds, and back-fills the ``section`` column for the rest.
|
||||
"""
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.quality import run_quality_pass
|
||||
|
||||
settings = get_settings()
|
||||
with get_conn() as conn:
|
||||
stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings)
|
||||
|
||||
scope = f"paper {paper_id}" if paper_id else "all papers"
|
||||
typer.echo(
|
||||
f"Quality pass ({scope}): "
|
||||
f"{stats['kept']} kept, "
|
||||
f"{stats['removed']} removed, "
|
||||
f"{stats['tagged']} section-tagged"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user