feat(F-16): chunk quality gate + section classification
- codex/quality.py: 3-signal filter (length / alpha-ratio / bib-score) + rule-based section classifier + run_quality_pass retroactive DB pass - codex/ingest.py: promote quality imports to module level; apply filter_chunks before embedding; store section column in chunks INSERT - codex/config.py: CHUNK_MIN_CHARS / CHUNK_MIN_ALPHA_RATIO / CHUNK_MAX_BIB_SCORE - infra/schema.sql: ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT - .env.example: document F-16 quality thresholds - codex/cli.py: quality run sub-command (scope by --paper-id or all papers) - tests/quality/: 35 new tests covering all quality functions - tests/ingest/: patch filter_chunks in source-path tests to isolate ingest Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
46
codex/cli.py
46
codex/cli.py
@@ -13,6 +13,7 @@ discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
||||
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||||
synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
|
||||
quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
|
||||
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
|
||||
search_app = typer.Typer(
|
||||
help="Search: semantic paper search and formula full-text search.",
|
||||
@@ -22,6 +23,7 @@ app.add_typer(discover_app, name="discover")
|
||||
app.add_typer(prov_app, name="provenance")
|
||||
app.add_typer(wiki_app, name="wiki")
|
||||
app.add_typer(synth_app, name="synthesis")
|
||||
app.add_typer(quality_app, name="quality")
|
||||
app.add_typer(search_app, name="search")
|
||||
|
||||
|
||||
@@ -419,9 +421,7 @@ def synthesis_conjectures(
|
||||
conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm)
|
||||
write_leads(conjectures, leads_dir)
|
||||
|
||||
typer.echo(
|
||||
f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}"
|
||||
)
|
||||
typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}")
|
||||
|
||||
|
||||
@synth_app.command("report")
|
||||
@@ -454,13 +454,45 @@ def synthesis_report(
|
||||
|
||||
typer.echo("")
|
||||
typer.echo("=" * 72)
|
||||
typer.echo(
|
||||
f"CONJECTURES ({len(conj_files)}) → {conj_dir} "
|
||||
"[UNVERIFIED — never in wiki/]"
|
||||
)
|
||||
typer.echo(f"CONJECTURES ({len(conj_files)}) → {conj_dir} [UNVERIFIED — never in wiki/]")
|
||||
typer.echo("=" * 72)
|
||||
if not conj_files:
|
||||
typer.echo("(none)")
|
||||
for f in conj_files:
|
||||
first = f.read_text(encoding="utf-8").splitlines()[0]
|
||||
typer.echo(f" {first}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F-16 — quality sub-commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@quality_app.command("run")
|
||||
def quality_run(
|
||||
paper_id: Optional[str] = typer.Option( # noqa: UP045
|
||||
None,
|
||||
"--paper-id",
|
||||
help="Restrict pass to one paper (omit to process all papers).",
|
||||
),
|
||||
) -> None:
|
||||
"""Apply quality gate + section classification to existing chunks.
|
||||
|
||||
Iterates over chunks in the database, removes those that fail the
|
||||
quality thresholds, and back-fills the ``section`` column for the rest.
|
||||
"""
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.quality import run_quality_pass
|
||||
|
||||
settings = get_settings()
|
||||
with get_conn() as conn:
|
||||
stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings)
|
||||
|
||||
scope = f"paper {paper_id}" if paper_id else "all papers"
|
||||
typer.echo(
|
||||
f"Quality pass ({scope}): "
|
||||
f"{stats['kept']} kept, "
|
||||
f"{stats['removed']} removed, "
|
||||
f"{stats['tagged']} section-tagged"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user