feat(F-16): chunk quality gate + section classification
3-signal quality filter (length/alpha-ratio/bib-score) + rule-based section classifier + retroactive CLI pass. 35 new tests, 287 total green.
This commit is contained in:
@@ -13,6 +13,7 @@ from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.embed import get_embedder
|
||||
from codex.models import Citation, Paper
|
||||
from codex.quality import classify_section, filter_chunks
|
||||
from codex.sources import openalex, semanticscholar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -80,7 +81,6 @@ def ingest_paper(
|
||||
# ---------------------------------------------------------------
|
||||
paper: Paper | None = openalex.fetch_paper(paper_id)
|
||||
|
||||
|
||||
if paper is None:
|
||||
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
|
||||
pid_lower = paper_id.lower()
|
||||
@@ -177,7 +177,8 @@ def ingest_paper(
|
||||
else:
|
||||
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
|
||||
|
||||
chunks_text = chunk_text(text) if text else []
|
||||
raw_chunks = chunk_text(text) if text else []
|
||||
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
|
||||
|
||||
# Delete existing chunks for this paper
|
||||
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
|
||||
@@ -186,13 +187,19 @@ def ingest_paper(
|
||||
# Embed all chunks in one batch
|
||||
chunk_embeddings = embedder.encode_dense(chunks_text)
|
||||
chunk_rows = [
|
||||
(paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist())
|
||||
(
|
||||
paper.id,
|
||||
ord_idx,
|
||||
content,
|
||||
chunk_embeddings[ord_idx].tolist(),
|
||||
classify_section(content),
|
||||
)
|
||||
for ord_idx, content in enumerate(chunks_text)
|
||||
]
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chunks (paper_id, ord, content, embedding)"
|
||||
" VALUES (%s, %s, %s, %s)",
|
||||
"INSERT INTO chunks (paper_id, ord, content, embedding, section)"
|
||||
" VALUES (%s, %s, %s, %s, %s)",
|
||||
chunk_rows,
|
||||
)
|
||||
chunks_upserted = len(chunk_rows)
|
||||
@@ -207,8 +214,7 @@ def ingest_paper(
|
||||
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
|
||||
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
|
||||
api_citations = [
|
||||
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
|
||||
for c in raw
|
||||
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
|
||||
]
|
||||
else:
|
||||
api_citations = semanticscholar.fetch_references(paper_id)
|
||||
|
||||
Reference in New Issue
Block a user