feat(F-16): chunk quality gate + section classification

3-signal quality filter (length/alpha-ratio/bib-score) + rule-based section classifier + retroactive CLI pass. 35 new tests, 287 total green.
This commit is contained in:
2026-06-15 01:22:44 +00:00
parent 1a9afb4433
commit 9647897173
9 changed files with 529 additions and 17 deletions

View File

@@ -13,6 +13,7 @@ from codex.config import get_settings
from codex.db import get_conn
from codex.embed import get_embedder
from codex.models import Citation, Paper
from codex.quality import classify_section, filter_chunks
from codex.sources import openalex, semanticscholar
logger = logging.getLogger(__name__)
@@ -80,7 +81,6 @@ def ingest_paper(
# ---------------------------------------------------------------
paper: Paper | None = openalex.fetch_paper(paper_id)
if paper is None:
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
pid_lower = paper_id.lower()
@@ -177,7 +177,8 @@ def ingest_paper(
else:
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
chunks_text = chunk_text(text) if text else []
raw_chunks = chunk_text(text) if text else []
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
# Delete existing chunks for this paper
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
@@ -186,13 +187,19 @@ def ingest_paper(
# Embed all chunks in one batch
chunk_embeddings = embedder.encode_dense(chunks_text)
chunk_rows = [
(paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist())
(
paper.id,
ord_idx,
content,
chunk_embeddings[ord_idx].tolist(),
classify_section(content),
)
for ord_idx, content in enumerate(chunks_text)
]
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chunks (paper_id, ord, content, embedding)"
" VALUES (%s, %s, %s, %s)",
"INSERT INTO chunks (paper_id, ord, content, embedding, section)"
" VALUES (%s, %s, %s, %s, %s)",
chunk_rows,
)
chunks_upserted = len(chunk_rows)
@@ -207,8 +214,7 @@ def ingest_paper(
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
api_citations = [
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
for c in raw
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
]
else:
api_citations = semanticscholar.fetch_references(paper_id)