feat(F-16): chunk quality gate + section classification

3-signal quality filter (length/alpha-ratio/bib-score) + rule-based section classifier + retroactive CLI pass. 35 new tests, 287 total green.
This commit is contained in:
2026-06-15 01:22:44 +00:00
parent 1a9afb4433
commit 9647897173
9 changed files with 529 additions and 17 deletions

View File

@@ -13,6 +13,7 @@ discover_app = typer.Typer(help="Discovery queries over the citation graph.")
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
search_app = typer.Typer(
help="Search: semantic paper search and formula full-text search.",
@@ -22,6 +23,7 @@ app.add_typer(discover_app, name="discover")
app.add_typer(prov_app, name="provenance")
app.add_typer(wiki_app, name="wiki")
app.add_typer(synth_app, name="synthesis")
app.add_typer(quality_app, name="quality")
app.add_typer(search_app, name="search")
@@ -419,9 +421,7 @@ def synthesis_conjectures(
conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm)
write_leads(conjectures, leads_dir)
typer.echo(
f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}"
)
typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}")
@synth_app.command("report")
@@ -454,13 +454,45 @@ def synthesis_report(
typer.echo("")
typer.echo("=" * 72)
typer.echo(
f"CONJECTURES ({len(conj_files)}) → {conj_dir} "
"[UNVERIFIED — never in wiki/]"
)
typer.echo(f"CONJECTURES ({len(conj_files)}) → {conj_dir} [UNVERIFIED — never in wiki/]")
typer.echo("=" * 72)
if not conj_files:
typer.echo("(none)")
for f in conj_files:
first = f.read_text(encoding="utf-8").splitlines()[0]
typer.echo(f" {first}")
# ---------------------------------------------------------------------------
# F-16 — quality sub-commands
# ---------------------------------------------------------------------------
@quality_app.command("run")
def quality_run(
paper_id: Optional[str] = typer.Option( # noqa: UP045
None,
"--paper-id",
help="Restrict pass to one paper (omit to process all papers).",
),
) -> None:
"""Apply quality gate + section classification to existing chunks.
Iterates over chunks in the database, removes those that fail the
quality thresholds, and back-fills the ``section`` column for the rest.
"""
from codex.config import get_settings
from codex.db import get_conn
from codex.quality import run_quality_pass
settings = get_settings()
with get_conn() as conn:
stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings)
scope = f"paper {paper_id}" if paper_id else "all papers"
typer.echo(
f"Quality pass ({scope}): "
f"{stats['kept']} kept, "
f"{stats['removed']} removed, "
f"{stats['tagged']} section-tagged"
)