diff --git a/.env.example b/.env.example index 483ba9b..f7b60ed 100644 --- a/.env.example +++ b/.env.example @@ -27,3 +27,8 @@ OPENALEX_MAILTO=you@example.com # Nougat HTTP-Server (Jetson: http://192.168.178.103:8080 | lokal: http://localhost:8080) # Used by `codex ingest --rich` for full-PDF Mathpix Markdown output. NOUGAT_URL=http://localhost:8080 + +# F-16 Chunk Quality Gate thresholds (all optional — defaults shown) +CHUNK_MIN_CHARS=60 # Discard chunks shorter than this many characters +CHUNK_MIN_ALPHA_RATIO=0.40 # Discard chunks with < 40% alphabetic characters +CHUNK_MAX_BIB_SCORE=0.70 # Discard chunks scoring above this bibliography threshold diff --git a/codex/cli.py b/codex/cli.py index 78c0739..0ddaefc 100644 --- a/codex/cli.py +++ b/codex/cli.py @@ -13,6 +13,7 @@ discover_app = typer.Typer(help="Discovery queries over the citation graph.") prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.") wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.") synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).") +quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).") # F-09: search sub-app with paper (semantic) and formula (FTS) subcommands search_app = typer.Typer( help="Search: semantic paper search and formula full-text search.", @@ -22,6 +23,7 @@ app.add_typer(discover_app, name="discover") app.add_typer(prov_app, name="provenance") app.add_typer(wiki_app, name="wiki") app.add_typer(synth_app, name="synthesis") +app.add_typer(quality_app, name="quality") app.add_typer(search_app, name="search") @@ -419,9 +421,7 @@ def synthesis_conjectures( conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm) write_leads(conjectures, leads_dir) - typer.echo( - f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}" - ) + typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}") @synth_app.command("report") @@ -454,13 +454,45 @@ def synthesis_report( typer.echo("") typer.echo("=" * 72) - typer.echo( - f"CONJECTURES ({len(conj_files)}) → {conj_dir} " - "[UNVERIFIED — never in wiki/]" - ) + typer.echo(f"CONJECTURES ({len(conj_files)}) → {conj_dir} [UNVERIFIED — never in wiki/]") typer.echo("=" * 72) if not conj_files: typer.echo("(none)") for f in conj_files: first = f.read_text(encoding="utf-8").splitlines()[0] typer.echo(f" {first}") + + +# --------------------------------------------------------------------------- +# F-16 — quality sub-commands +# --------------------------------------------------------------------------- + + +@quality_app.command("run") +def quality_run( + paper_id: Optional[str] = typer.Option( # noqa: UP045 + None, + "--paper-id", + help="Restrict pass to one paper (omit to process all papers).", + ), +) -> None: + """Apply quality gate + section classification to existing chunks. + + Iterates over chunks in the database, removes those that fail the + quality thresholds, and back-fills the ``section`` column for the rest. + """ + from codex.config import get_settings + from codex.db import get_conn + from codex.quality import run_quality_pass + + settings = get_settings() + with get_conn() as conn: + stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings) + + scope = f"paper {paper_id}" if paper_id else "all papers" + typer.echo( + f"Quality pass ({scope}): " + f"{stats['kept']} kept, " + f"{stats['removed']} removed, " + f"{stats['tagged']} section-tagged" + ) diff --git a/codex/config.py b/codex/config.py index ede21c8..91698d2 100644 --- a/codex/config.py +++ b/codex/config.py @@ -191,9 +191,7 @@ class Settings(BaseSettings): synthesis_llm_url: str | None = Field( default=None, - description=( - "Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL." - ), + description=("Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."), ) synthesis_top_k: int = Field( @@ -256,6 +254,38 @@ class Settings(BaseSettings): ), ) + # ------------------------------------------------------------------ + # F-16 Chunk Quality Gate + # ------------------------------------------------------------------ + chunk_min_chars: int = Field( + default=60, + gt=0, + description=( + "Minimum character count for a chunk to pass the quality gate. " + "Chunks shorter than this value are discarded before embedding." + ), + ) + + chunk_min_alpha_ratio: float = Field( + default=0.40, + ge=0.0, + le=1.0, + description=( + "Minimum fraction of alphabetic characters in a chunk. " + "Chunks with a lower ratio are treated as OCR artefacts and discarded." + ), + ) + + chunk_max_bib_score: float = Field( + default=0.70, + ge=0.0, + le=1.0, + description=( + "Maximum bibliography heuristic score before a chunk is rejected. " + "Score is based on DOI density, 'et al.' and year-in-brackets patterns." + ), + ) + @lru_cache(maxsize=1) def get_settings() -> Settings: diff --git a/codex/ingest.py b/codex/ingest.py index 2c2e912..87c9d73 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -13,6 +13,7 @@ from codex.config import get_settings from codex.db import get_conn from codex.embed import get_embedder from codex.models import Citation, Paper +from codex.quality import classify_section, filter_chunks from codex.sources import openalex, semanticscholar logger = logging.getLogger(__name__) @@ -80,7 +81,6 @@ def ingest_paper( # --------------------------------------------------------------- paper: Paper | None = openalex.fetch_paper(paper_id) - if paper is None: # Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix pid_lower = paper_id.lower() @@ -177,7 +177,8 @@ def ingest_paper( else: logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path) - chunks_text = chunk_text(text) if text else [] + raw_chunks = chunk_text(text) if text else [] + chunks_text = filter_chunks(raw_chunks, settings=get_settings()) # Delete existing chunks for this paper conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,)) @@ -186,13 +187,19 @@ def ingest_paper( # Embed all chunks in one batch chunk_embeddings = embedder.encode_dense(chunks_text) chunk_rows = [ - (paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist()) + ( + paper.id, + ord_idx, + content, + chunk_embeddings[ord_idx].tolist(), + classify_section(content), + ) for ord_idx, content in enumerate(chunks_text) ] with conn.cursor() as cur: cur.executemany( - "INSERT INTO chunks (paper_id, ord, content, embedding)" - " VALUES (%s, %s, %s, %s)", + "INSERT INTO chunks (paper_id, ord, content, embedding, section)" + " VALUES (%s, %s, %s, %s, %s)", chunk_rows, ) chunks_upserted = len(chunk_rows) @@ -207,8 +214,7 @@ def ingest_paper( # OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI. # Rewrite citing_id to paper.id so the FK constraint is satisfied. api_citations = [ - Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) - for c in raw + Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw ] else: api_citations = semanticscholar.fetch_references(paper_id) diff --git a/codex/quality.py b/codex/quality.py new file mode 100644 index 0000000..ef6ee72 --- /dev/null +++ b/codex/quality.py @@ -0,0 +1,189 @@ +"""Chunk quality filtering and section classification (F-16). + +Three independent quality signals are applied at ingest time: + +1. **Length** — structural; no content required. +2. **Alpha-ratio** — OCR artefacts have high non-alpha character density. +3. **Bib-score** — DOI + "et al." + (YYYY) patterns co-occur almost only + in reference list entries. + +Section classification is rule-based (no LLM) and runs on the first 200 +characters of each chunk. The retroactive ``run_quality_pass`` function +can be called from the CLI to back-fill existing chunks. +""" + +from __future__ import annotations + +import logging +import re +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from codex.config import Settings + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Section classification patterns (checked in order; first match wins) +# --------------------------------------------------------------------------- + +_SECTION_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ + ("abstract", re.compile(r"^\s*(abstract|zusammenfassung|r[eé]sum[eé])\b", re.I)), + ("intro", re.compile(r"^\s*(introduction|einleitung|1\.\s)", re.I)), + ("theorem", re.compile(r"^\s*(theorem|lemma|proposition|corollary|definition)\b", re.I)), + ("proof", re.compile(r"^\s*(proof\b|beweis\b|proof\s+of\b)", re.I)), + ("bibliography", re.compile(r"^\s*(references|bibliography|bibliographie|literatur)\b", re.I)), +] + +_DOI_RE = re.compile(r"10\.\d{4,}/\S+") +_ET_AL_RE = re.compile(r"\bet\s+al\b", re.I) +_YEAR_BRACKETS_RE = re.compile(r"\(\d{4}\)") + + +# --------------------------------------------------------------------------- +# Bibliography heuristic score +# --------------------------------------------------------------------------- + + +def _bib_score(text: str) -> float: + """Return a [0..1] heuristic for how bibliography-like a chunk is. + + Three signals contribute: + * DOI occurrences (weight 3) + * "et al." occurrences (weight 2) + * year-in-brackets occurrences (weight 1) + + The raw signal is normalised against a rough word-count proxy so that + long chunks with occasional references don't get flagged. + """ + if not text: + return 0.0 + word_count = max(len(text.split()), 1) + doi_hits = len(_DOI_RE.findall(text)) + etal_hits = len(_ET_AL_RE.findall(text)) + year_hits = len(_YEAR_BRACKETS_RE.findall(text)) + raw = (doi_hits * 3 + etal_hits * 2 + year_hits) / max(word_count / 10, 1) + return min(raw, 1.0) + + +# --------------------------------------------------------------------------- +# Quality predicate +# --------------------------------------------------------------------------- + + +def is_quality_chunk(text: str, *, settings: Settings) -> bool: + """Return True when *text* passes all three quality thresholds. + + Checks (all configurable via :class:`codex.config.Settings`): + + 1. ``chunk_min_chars`` — character count floor. + 2. ``chunk_min_alpha_ratio`` — minimum fraction of alphabetic chars. + 3. ``chunk_max_bib_score`` — bibliography heuristic ceiling. + """ + if len(text) < settings.chunk_min_chars: + return False + alpha_ratio = sum(c.isalpha() for c in text) / max(len(text), 1) + if alpha_ratio < settings.chunk_min_alpha_ratio: + return False + return _bib_score(text) <= settings.chunk_max_bib_score + + +# --------------------------------------------------------------------------- +# Section classification +# --------------------------------------------------------------------------- + + +def classify_section(text: str) -> str: + """Classify a chunk's section using rule-based regex matching. + + Inspects only the first 200 characters. Returns one of: + ``abstract``, ``intro``, ``theorem``, ``proof``, ``bibliography``, ``body``. + """ + snippet = text[:200] + for section_name, pattern in _SECTION_PATTERNS: + if pattern.search(snippet): + return section_name + # Fallback: bibliography by DOI density even without a header + if _bib_score(text) > 0.5: + return "bibliography" + return "body" + + +# --------------------------------------------------------------------------- +# Batch filter +# --------------------------------------------------------------------------- + + +def filter_chunks(chunks: list[str], *, settings: Settings) -> list[str]: + """Return only the chunks that pass all quality filters. + + Logs the keep ratio at DEBUG level. + """ + kept = [c for c in chunks if is_quality_chunk(c, settings=settings)] + logger.debug( + "Quality filter: %d/%d chunks kept (%.0f%%)", + len(kept), + len(chunks), + 100 * len(kept) / max(len(chunks), 1), + ) + return kept + + +# --------------------------------------------------------------------------- +# Retroactive DB pass +# --------------------------------------------------------------------------- + + +def run_quality_pass( + *, + paper_id: str | None = None, + conn: Any, + settings: Settings, +) -> dict[str, int]: + """Apply quality filter + section classification to existing chunks in DB. + + For each chunk: + * Fails quality → DELETE. + * Passes quality → UPDATE ``section`` with :func:`classify_section`. + + Parameters + ---------- + paper_id: + When provided, restrict the pass to chunks for this paper only. + conn: + Open psycopg connection (dict-row factory assumed). + settings: + Application settings for quality thresholds. + + Returns + ------- + dict with keys ``kept``, ``removed``, ``tagged`` (= kept). + """ + if paper_id is not None: + rows = conn.execute( + "SELECT id, content FROM chunks WHERE paper_id = %(pid)s", + {"pid": paper_id}, + ).fetchall() + else: + rows = conn.execute("SELECT id, content FROM chunks").fetchall() + + kept = 0 + removed = 0 + for row in rows: + chunk_id = row["id"] + content = row["content"] + if not is_quality_chunk(content, settings=settings): + conn.execute("DELETE FROM chunks WHERE id = %(id)s", {"id": chunk_id}) + removed += 1 + else: + section = classify_section(content) + conn.execute( + "UPDATE chunks SET section = %(section)s WHERE id = %(id)s", + {"section": section, "id": chunk_id}, + ) + kept += 1 + + conn.commit() + logger.info("Quality pass done: %d kept, %d removed, %d section-tagged", kept, removed, kept) + return {"kept": kept, "removed": removed, "tagged": kept} diff --git a/infra/schema.sql b/infra/schema.sql index 4e77c41..9259506 100644 --- a/infra/schema.sql +++ b/infra/schema.sql @@ -122,3 +122,6 @@ CREATE TABLE IF NOT EXISTS figures ( CREATE INDEX IF NOT EXISTS figures_emb_idx ON figures USING hnsw (embedding vector_cosine_ops); CREATE INDEX IF NOT EXISTS figures_paper_idx ON figures (paper_id); + +-- F-16 Chunk Quality Gate: section classification +ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT; diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index d5b012c..80e4d23 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -131,6 +131,7 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."), patch("codex.parsing.tex.chunk_text", return_value=fake_chunks), + patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), ): result = ingest_paper(paper.id, source_path=str(tex_file)) @@ -178,6 +179,7 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), patch("codex.parsing.tex.chunk_text", return_value=fake_chunks), patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), + patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), ): result = ingest_paper(paper.id, source_path=str(pdf_file)) diff --git a/tests/quality/__init__.py b/tests/quality/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/quality/test_quality.py b/tests/quality/test_quality.py new file mode 100644 index 0000000..b131d2f --- /dev/null +++ b/tests/quality/test_quality.py @@ -0,0 +1,245 @@ +"""Tests for codex.quality — chunk quality gate and section classification (F-16).""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from codex.quality import ( + _bib_score, + classify_section, + filter_chunks, + is_quality_chunk, + run_quality_pass, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _settings(min_chars: int = 60, min_alpha: float = 0.40, max_bib: float = 0.70): + s = MagicMock() + s.chunk_min_chars = min_chars + s.chunk_min_alpha_ratio = min_alpha + s.chunk_max_bib_score = max_bib + return s + + +GOOD_TEXT = ( + "In differential geometry, the discrete Laplace–Beltrami operator " + "generalises the smooth operator to polyhedral surfaces by using cotangent " + "weights. This discretisation preserves key spectral properties." +) + +BIB_TEXT = ( + "Bobenko, A. and Pinkall, U. (2015) Discrete differential geometry. " + "doi:10.1145/2010324.1964978 Smith et al. (2018) Surface remeshing. " + "doi:10.1007/s00211-019-01082-w Jones et al. (2020) doi:10.1145/1234567.890123" +) + + +# --------------------------------------------------------------------------- +# _bib_score +# --------------------------------------------------------------------------- + + +class TestBibScore: + def test_empty_string(self): + assert _bib_score("") == 0.0 + + def test_clean_prose_is_low(self): + assert _bib_score(GOOD_TEXT) < 0.10 + + def test_bib_like_text_is_high(self): + assert _bib_score(BIB_TEXT) > 0.70 + + def test_single_doi_short_text_spikes(self): + text = "See doi:10.1145/2010324.1964978 for details." + # 8 words, 1 DOI (weight 3) → raw = 3 / max(0.8, 1) = 3.0 → capped 1.0 + assert _bib_score(text) == 1.0 + + def test_score_capped_at_one(self): + # _DOI_RE requires \d{4,} — use a valid registrant length + heavy = "10.1145/abcdef " * 20 + assert _bib_score(heavy) == 1.0 + + +# --------------------------------------------------------------------------- +# is_quality_chunk +# --------------------------------------------------------------------------- + + +class TestIsQualityChunk: + def test_passes_good_text(self): + assert is_quality_chunk(GOOD_TEXT, settings=_settings()) is True + + def test_fails_too_short(self): + assert is_quality_chunk("Short.", settings=_settings(min_chars=60)) is False + + def test_fails_low_alpha_ratio(self): + # Mostly digits and symbols + junk = "1234 5678 9012 3456 7890 |||| ==== ---- ____ .... ;;;; 0000 9999 8888 7777" + assert is_quality_chunk(junk, settings=_settings(min_alpha=0.40)) is False + + def test_fails_high_bib_score(self): + assert is_quality_chunk(BIB_TEXT, settings=_settings(max_bib=0.70)) is False + + def test_bib_threshold_is_strict_gt(self): + # A text that scores exactly at the threshold should still pass (> not >=) + s = _settings(max_bib=1.0) + assert is_quality_chunk(GOOD_TEXT, settings=s) is True + + def test_exactly_min_chars_passes(self): + # check is `len < min_chars`, so exactly at the threshold passes + text = "a" * 60 + assert is_quality_chunk(text, settings=_settings(min_chars=60)) is True + + def test_one_below_min_chars_fails(self): + text = "a" * 59 + assert is_quality_chunk(text, settings=_settings(min_chars=60)) is False + + +# --------------------------------------------------------------------------- +# classify_section +# --------------------------------------------------------------------------- + + +class TestClassifySection: + def test_abstract_header(self): + assert classify_section("Abstract\n\nThis paper presents...") == "abstract" + + def test_abstract_german(self): + assert classify_section("Zusammenfassung\nDiese Arbeit...") == "abstract" + + def test_intro_header(self): + assert classify_section("Introduction\nOver the past decade...") == "intro" + + def test_intro_numbered(self): + assert classify_section("1. Introduction\nWe study...") == "intro" + + def test_theorem_header(self): + assert classify_section("Theorem 3.1 (Existence). Let M be...") == "theorem" + + def test_lemma_header(self): + assert classify_section("Lemma 2. If f is smooth then...") == "theorem" + + def test_proof_header(self): + assert classify_section("Proof. We proceed by induction...") == "proof" + + def test_bibliography_header(self): + assert classify_section("References\n[1] Bobenko, A....") == "bibliography" + + def test_bibliography_german(self): + assert classify_section("Literatur\n[1] Müller, H....") == "bibliography" + + def test_body_fallback(self): + assert classify_section("The cotangent weight for edge (i,j) is...") == "body" + + def test_doi_density_fallback_to_bibliography(self): + # No "References" header but heavy DOI density → bibliography + dense = BIB_TEXT + assert classify_section(dense) == "bibliography" + + def test_only_first_200_chars_checked(self): + # Pattern in position >200 should NOT be matched as section header + prefix = "x" * 201 + text = prefix + "\nAbstract" + assert classify_section(text) == "body" + + +# --------------------------------------------------------------------------- +# filter_chunks +# --------------------------------------------------------------------------- + + +class TestFilterChunks: + def test_empty_list(self): + assert filter_chunks([], settings=_settings()) == [] + + def test_all_good(self): + chunks = [GOOD_TEXT, GOOD_TEXT + " More content here."] + assert filter_chunks(chunks, settings=_settings()) == chunks + + def test_all_bad(self): + chunks = ["x", "y"] + assert filter_chunks(chunks, settings=_settings()) == [] + + def test_mixed(self): + chunks = [GOOD_TEXT, "Short.", GOOD_TEXT] + result = filter_chunks(chunks, settings=_settings()) + assert len(result) == 2 + assert all(c == GOOD_TEXT for c in result) + + +# --------------------------------------------------------------------------- +# run_quality_pass +# --------------------------------------------------------------------------- + + +def _make_mock_conn(rows: list[dict]) -> MagicMock: + """Return a mock psycopg connection whose execute().fetchall() returns rows.""" + conn = MagicMock() + conn.execute.return_value.fetchall.return_value = rows + return conn + + +class TestRunQualityPass: + def test_removes_bad_chunk(self): + conn = _make_mock_conn([{"id": 1, "content": "x"}]) + stats = run_quality_pass(conn=conn, settings=_settings()) + assert stats["removed"] == 1 + assert stats["kept"] == 0 + # The DELETE must have been called with chunk id=1 + delete_calls = [c for c in conn.execute.call_args_list if "DELETE" in str(c)] + assert len(delete_calls) == 1 + + def test_tags_good_chunk(self): + conn = _make_mock_conn([{"id": 2, "content": GOOD_TEXT}]) + stats = run_quality_pass(conn=conn, settings=_settings()) + assert stats["kept"] == 1 + assert stats["tagged"] == 1 + assert stats["removed"] == 0 + # UPDATE must have been called + update_calls = [c for c in conn.execute.call_args_list if "UPDATE" in str(c)] + assert len(update_calls) == 1 + + def test_mixed_batch(self): + rows = [ + {"id": 1, "content": "x"}, + {"id": 2, "content": GOOD_TEXT}, + {"id": 3, "content": "y"}, + {"id": 4, "content": GOOD_TEXT + " extra words here."}, + ] + conn = _make_mock_conn(rows) + stats = run_quality_pass(conn=conn, settings=_settings()) + assert stats["removed"] == 2 + assert stats["kept"] == 2 + assert stats["tagged"] == 2 + + def test_paper_id_scopes_query(self): + conn = _make_mock_conn([]) + run_quality_pass(paper_id="2301.07041", conn=conn, settings=_settings()) + first_call = conn.execute.call_args_list[0] + sql = first_call[0][0] + assert "paper_id" in sql + + def test_all_papers_query_has_no_filter(self): + conn = _make_mock_conn([]) + run_quality_pass(paper_id=None, conn=conn, settings=_settings()) + first_call = conn.execute.call_args_list[0] + sql = first_call[0][0] + assert "paper_id" not in sql + + def test_commit_is_called(self): + conn = _make_mock_conn([]) + run_quality_pass(conn=conn, settings=_settings()) + conn.commit.assert_called_once() + + def test_section_label_stored_correctly(self): + conn = _make_mock_conn([{"id": 5, "content": "Abstract\n\n" + GOOD_TEXT}]) + run_quality_pass(conn=conn, settings=_settings()) + update_calls = [c for c in conn.execute.call_args_list if "UPDATE" in str(c)] + assert len(update_calls) == 1 + kwargs = update_calls[0][0][1] + assert kwargs["section"] == "abstract" + assert kwargs["id"] == 5