feat(F-16): chunk quality gate + section classification

- codex/quality.py: 3-signal filter (length / alpha-ratio / bib-score)
  + rule-based section classifier + run_quality_pass retroactive DB pass
- codex/ingest.py: promote quality imports to module level; apply
  filter_chunks before embedding; store section column in chunks INSERT
- codex/config.py: CHUNK_MIN_CHARS / CHUNK_MIN_ALPHA_RATIO / CHUNK_MAX_BIB_SCORE
- infra/schema.sql: ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT
- .env.example: document F-16 quality thresholds
- codex/cli.py: quality run sub-command (scope by --paper-id or all papers)
- tests/quality/: 35 new tests covering all quality functions
- tests/ingest/: patch filter_chunks in source-path tests to isolate ingest

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-15 03:22:11 +02:00
parent 1a9afb4433
commit b61d84d4bd
9 changed files with 529 additions and 17 deletions

View File

@@ -13,6 +13,7 @@ from codex.config import get_settings
from codex.db import get_conn
from codex.embed import get_embedder
from codex.models import Citation, Paper
from codex.quality import classify_section, filter_chunks
from codex.sources import openalex, semanticscholar
logger = logging.getLogger(__name__)
@@ -80,7 +81,6 @@ def ingest_paper(
# ---------------------------------------------------------------
paper: Paper | None = openalex.fetch_paper(paper_id)
if paper is None:
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
pid_lower = paper_id.lower()
@@ -177,7 +177,8 @@ def ingest_paper(
else:
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
chunks_text = chunk_text(text) if text else []
raw_chunks = chunk_text(text) if text else []
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
# Delete existing chunks for this paper
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
@@ -186,13 +187,19 @@ def ingest_paper(
# Embed all chunks in one batch
chunk_embeddings = embedder.encode_dense(chunks_text)
chunk_rows = [
(paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist())
(
paper.id,
ord_idx,
content,
chunk_embeddings[ord_idx].tolist(),
classify_section(content),
)
for ord_idx, content in enumerate(chunks_text)
]
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chunks (paper_id, ord, content, embedding)"
" VALUES (%s, %s, %s, %s)",
"INSERT INTO chunks (paper_id, ord, content, embedding, section)"
" VALUES (%s, %s, %s, %s, %s)",
chunk_rows,
)
chunks_upserted = len(chunk_rows)
@@ -207,8 +214,7 @@ def ingest_paper(
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
api_citations = [
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
for c in raw
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
]
else:
api_citations = semanticscholar.fetch_references(paper_id)