user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(F-16): chunk quality gate + section classification

Browse Source 
- codex/quality.py: 3-signal filter (length / alpha-ratio / bib-score)
  + rule-based section classifier + run_quality_pass retroactive DB pass
- codex/ingest.py: promote quality imports to module level; apply
  filter_chunks before embedding; store section column in chunks INSERT
- codex/config.py: CHUNK_MIN_CHARS / CHUNK_MIN_ALPHA_RATIO / CHUNK_MAX_BIB_SCORE
- infra/schema.sql: ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT
- .env.example: document F-16 quality thresholds
- codex/cli.py: quality run sub-command (scope by --paper-id or all papers)
- tests/quality/: 35 new tests covering all quality functions
- tests/ingest/: patch filter_chunks in source-path tests to isolate ingest

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-15 03:22:11 +02:00
parent 1a9afb4433
commit b61d84d4bd
9 changed files with 529 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
codex/cli.py
View File

@@ -13,6 +13,7 @@ discover_app = typer.Typer(help="Discovery queries over the citation graph.")
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
search_app = typer.Typer(
help="Search: semantic paper search and formula full-text search.",
@@ -22,6 +23,7 @@ app.add_typer(discover_app, name="discover")
app.add_typer(prov_app, name="provenance")
app.add_typer(wiki_app, name="wiki")
app.add_typer(synth_app, name="synthesis")
app.add_typer(quality_app, name="quality")
app.add_typer(search_app, name="search")
@@ -419,9 +421,7 @@ def synthesis_conjectures(
conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm)
write_leads(conjectures, leads_dir)
typer.echo(
f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}"
)
typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}")
@synth_app.command("report")
@@ -454,13 +454,45 @@ def synthesis_report(
typer.echo("")
typer.echo("=" * 72)
typer.echo(
f"CONJECTURES ({len(conj_files)}) → {conj_dir} "
"[UNVERIFIED — never in wiki/]"
)
typer.echo(f"CONJECTURES ({len(conj_files)}) → {conj_dir} [UNVERIFIED — never in wiki/]")
typer.echo("=" * 72)
if not conj_files:
typer.echo("(none)")
for f in conj_files:
first = f.read_text(encoding="utf-8").splitlines()[0]
typer.echo(f" {first}")
# ---------------------------------------------------------------------------
# F-16 — quality sub-commands
# ---------------------------------------------------------------------------
@quality_app.command("run")
def quality_run(
paper_id: Optional[str] = typer.Option( # noqa: UP045
None,
"--paper-id",
help="Restrict pass to one paper (omit to process all papers).",
),
) -> None:
"""Apply quality gate + section classification to existing chunks.
Iterates over chunks in the database, removes those that fail the
quality thresholds, and back-fills the ``section`` column for the rest.
"""
from codex.config import get_settings
from codex.db import get_conn
from codex.quality import run_quality_pass
settings = get_settings()
with get_conn() as conn:
stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings)
scope = f"paper {paper_id}" if paper_id else "all papers"
typer.echo(
f"Quality pass ({scope}): "
f"{stats['kept']} kept, "
f"{stats['removed']} removed, "
f"{stats['tagged']} section-tagged"
)

36
codex/config.py
View File

@@ -191,9 +191,7 @@ class Settings(BaseSettings):
synthesis_llm_url: str | None = Field(
default=None,
description=(
"Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."
),
description=("Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."),
)
synthesis_top_k: int = Field(
@@ -256,6 +254,38 @@ class Settings(BaseSettings):
),
)
# ------------------------------------------------------------------
# F-16 Chunk Quality Gate
# ------------------------------------------------------------------
chunk_min_chars: int = Field(
default=60,
gt=0,
description=(
"Minimum character count for a chunk to pass the quality gate. "
"Chunks shorter than this value are discarded before embedding."
),
)
chunk_min_alpha_ratio: float = Field(
default=0.40,
ge=0.0,
le=1.0,
description=(
"Minimum fraction of alphabetic characters in a chunk. "
"Chunks with a lower ratio are treated as OCR artefacts and discarded."
),
)
chunk_max_bib_score: float = Field(
default=0.70,
ge=0.0,
le=1.0,
description=(
"Maximum bibliography heuristic score before a chunk is rejected. "
"Score is based on DOI density, 'et al.' and year-in-brackets patterns."
),
)
@lru_cache(maxsize=1)
def get_settings() -> Settings:

20
codex/ingest.py
View File

@@ -13,6 +13,7 @@ from codex.config import get_settings
from codex.db import get_conn
from codex.embed import get_embedder
from codex.models import Citation, Paper
from codex.quality import classify_section, filter_chunks
from codex.sources import openalex, semanticscholar
logger = logging.getLogger(__name__)
@@ -80,7 +81,6 @@ def ingest_paper(
# ---------------------------------------------------------------
paper: Paper | None = openalex.fetch_paper(paper_id)
if paper is None:
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
pid_lower = paper_id.lower()
@@ -177,7 +177,8 @@ def ingest_paper(
else:
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
chunks_text = chunk_text(text) if text else []
raw_chunks = chunk_text(text) if text else []
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
# Delete existing chunks for this paper
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
@@ -186,13 +187,19 @@ def ingest_paper(
# Embed all chunks in one batch
chunk_embeddings = embedder.encode_dense(chunks_text)
chunk_rows = [
(paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist())
(
paper.id,
ord_idx,
content,
chunk_embeddings[ord_idx].tolist(),
classify_section(content),
)
for ord_idx, content in enumerate(chunks_text)
]
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chunks (paper_id, ord, content, embedding)"
" VALUES (%s, %s, %s, %s)",
"INSERT INTO chunks (paper_id, ord, content, embedding, section)"
" VALUES (%s, %s, %s, %s, %s)",
chunk_rows,
)
chunks_upserted = len(chunk_rows)
@@ -207,8 +214,7 @@ def ingest_paper(
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
api_citations = [
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
for c in raw
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
]
else:
api_citations = semanticscholar.fetch_references(paper_id)

189
codex/quality.py Normal file
View File

@@ -0,0 +1,189 @@
"""Chunk quality filtering and section classification (F-16).
Three independent quality signals are applied at ingest time:
1. **Length** — structural; no content required.
2. **Alpha-ratio** — OCR artefacts have high non-alpha character density.
3. **Bib-score** — DOI + "et al." + (YYYY) patterns co-occur almost only
in reference list entries.
Section classification is rule-based (no LLM) and runs on the first 200
characters of each chunk. The retroactive ``run_quality_pass`` function
can be called from the CLI to back-fill existing chunks.
"""
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from codex.config import Settings
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Section classification patterns (checked in order; first match wins)
# ---------------------------------------------------------------------------
_SECTION_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
("abstract", re.compile(r"^\s*(abstract|zusammenfassung|r[eé]sum[eé])\b", re.I)),
("intro", re.compile(r"^\s*(introduction|einleitung|1\.\s)", re.I)),
("theorem", re.compile(r"^\s*(theorem|lemma|proposition|corollary|definition)\b", re.I)),
("proof", re.compile(r"^\s*(proof\b|beweis\b|proof\s+of\b)", re.I)),
("bibliography", re.compile(r"^\s*(references|bibliography|bibliographie|literatur)\b", re.I)),
]
_DOI_RE = re.compile(r"10\.\d{4,}/\S+")
_ET_AL_RE = re.compile(r"\bet\s+al\b", re.I)
_YEAR_BRACKETS_RE = re.compile(r"\(\d{4}\)")
# ---------------------------------------------------------------------------
# Bibliography heuristic score
# ---------------------------------------------------------------------------
def _bib_score(text: str) -> float:
"""Return a [0..1] heuristic for how bibliography-like a chunk is.
Three signals contribute:
* DOI occurrences (weight 3)
* "et al." occurrences (weight 2)
* year-in-brackets occurrences (weight 1)
The raw signal is normalised against a rough word-count proxy so that
long chunks with occasional references don't get flagged.
"""
if not text:
return 0.0
word_count = max(len(text.split()), 1)
doi_hits = len(_DOI_RE.findall(text))
etal_hits = len(_ET_AL_RE.findall(text))
year_hits = len(_YEAR_BRACKETS_RE.findall(text))
raw = (doi_hits * 3 + etal_hits * 2 + year_hits) / max(word_count / 10, 1)
return min(raw, 1.0)
# ---------------------------------------------------------------------------
# Quality predicate
# ---------------------------------------------------------------------------
def is_quality_chunk(text: str, *, settings: Settings) -> bool:
"""Return True when *text* passes all three quality thresholds.
Checks (all configurable via :class:`codex.config.Settings`):
1. ``chunk_min_chars`` — character count floor.
2. ``chunk_min_alpha_ratio`` — minimum fraction of alphabetic chars.
3. ``chunk_max_bib_score`` — bibliography heuristic ceiling.
"""
if len(text) < settings.chunk_min_chars:
return False
alpha_ratio = sum(c.isalpha() for c in text) / max(len(text), 1)
if alpha_ratio < settings.chunk_min_alpha_ratio:
return False
return _bib_score(text) <= settings.chunk_max_bib_score
# ---------------------------------------------------------------------------
# Section classification
# ---------------------------------------------------------------------------
def classify_section(text: str) -> str:
"""Classify a chunk's section using rule-based regex matching.
Inspects only the first 200 characters. Returns one of:
``abstract``, ``intro``, ``theorem``, ``proof``, ``bibliography``, ``body``.
"""
snippet = text[:200]
for section_name, pattern in _SECTION_PATTERNS:
if pattern.search(snippet):
return section_name
# Fallback: bibliography by DOI density even without a header
if _bib_score(text) > 0.5:
return "bibliography"
return "body"
# ---------------------------------------------------------------------------
# Batch filter
# ---------------------------------------------------------------------------
def filter_chunks(chunks: list[str], *, settings: Settings) -> list[str]:
"""Return only the chunks that pass all quality filters.
Logs the keep ratio at DEBUG level.
"""
kept = [c for c in chunks if is_quality_chunk(c, settings=settings)]
logger.debug(
"Quality filter: %d/%d chunks kept (%.0f%%)",
len(kept),
len(chunks),
100 * len(kept) / max(len(chunks), 1),
)
return kept
# ---------------------------------------------------------------------------
# Retroactive DB pass
# ---------------------------------------------------------------------------
def run_quality_pass(
*,
paper_id: str | None = None,
conn: Any,
settings: Settings,
) -> dict[str, int]:
"""Apply quality filter + section classification to existing chunks in DB.
For each chunk:
* Fails quality → DELETE.
* Passes quality → UPDATE ``section`` with :func:`classify_section`.
Parameters
----------
paper_id:
When provided, restrict the pass to chunks for this paper only.
conn:
Open psycopg connection (dict-row factory assumed).
settings:
Application settings for quality thresholds.
Returns
-------
dict with keys ``kept``, ``removed``, ``tagged`` (= kept).
"""
if paper_id is not None:
rows = conn.execute(
"SELECT id, content FROM chunks WHERE paper_id = %(pid)s",
{"pid": paper_id},
).fetchall()
else:
rows = conn.execute("SELECT id, content FROM chunks").fetchall()
kept = 0
removed = 0
for row in rows:
chunk_id = row["id"]
content = row["content"]
if not is_quality_chunk(content, settings=settings):
conn.execute("DELETE FROM chunks WHERE id = %(id)s", {"id": chunk_id})
removed += 1
else:
section = classify_section(content)
conn.execute(
"UPDATE chunks SET section = %(section)s WHERE id = %(id)s",
{"section": section, "id": chunk_id},
)
kept += 1
conn.commit()
logger.info("Quality pass done: %d kept, %d removed, %d section-tagged", kept, removed, kept)
return {"kept": kept, "removed": removed, "tagged": kept}