feat(quality): store real section titles for .tex chunks (R-F)
Add quality.section_label(title, content): for section-aware .tex ingest, keep the controlled bucket (intro/theorem/proof/abstract/bibliography) when the real \section heading maps to one, else store the cleaned title verbatim (e.g. 'preliminaries', 'rigidity') instead of collapsing to 'body'. Math papers title most sections descriptively, so R-C's vocab-only mapping left ~90% as 'body'; this lifts the section signal toward near-full. .pdf/.txt/run_quality_pass keep content-based classify_section unchanged. Safe because the section column is write-only (no consumer filters on the vocab). New _clean_title (strip LaTeX/numbering, lower-case, truncate). Tests: section_label bucket/verbatim/fallback paths + _clean_title; .tex ingest stores a descriptive heading as its title. Full suite 377 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -12,7 +12,7 @@ from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.embed import get_embedder
|
||||
from codex.models import Citation, Paper
|
||||
from codex.quality import classify_section, is_quality_chunk
|
||||
from codex.quality import is_quality_chunk, section_label
|
||||
from codex.sources import arxiv, crossref, openalex, semanticscholar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -312,10 +312,11 @@ def ingest_paper(
|
||||
ord_idx,
|
||||
content,
|
||||
chunk_embeddings[ord_idx].tolist(),
|
||||
# .tex: classify from the section heading (prepended to the
|
||||
# snippet) so the label reflects the real section; otherwise
|
||||
# fall back to content-only classification.
|
||||
classify_section(f"{title}. {content}" if title else content),
|
||||
# .tex: label from the real \section heading — keeps the
|
||||
# controlled bucket when it matches, else stores the cleaned
|
||||
# title (R-F); .txt/.pdf (title=None) classify by content.
|
||||
# NB: `section` semantics now differ by source (write-only col).
|
||||
section_label(title, content),
|
||||
)
|
||||
for ord_idx, (title, content) in enumerate(kept_pairs)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user