feat(quality): store real section titles for .tex chunks (R-F)

Add quality.section_label(title, content): for section-aware .tex ingest, keep the controlled bucket (intro/theorem/proof/abstract/bibliography) when the real \section heading maps to one, else store the cleaned title verbatim (e.g. 'preliminaries', 'rigidity') instead of collapsing to 'body'. Math papers title most sections descriptively, so R-C's vocab-only mapping left ~90% as 'body'; this lifts the section signal toward near-full. .pdf/.txt/run_quality_pass keep content-based classify_section unchanged. Safe because the section column is write-only (no consumer filters on the vocab).

New _clean_title (strip LaTeX/numbering, lower-case, truncate). Tests: section_label bucket/verbatim/fallback paths + _clean_title; .tex ingest stores a descriptive heading as its title. Full suite 377 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-18 09:03:46 +02:00
parent c22c0db3d0
commit c70ff9aa1e
4 changed files with 82 additions and 7 deletions

View File

@@ -128,6 +128,7 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
section_pairs = [
("Introduction", "Body of the introduction goes here."),
("Proof of the Main Theorem", "Body of the proof goes here."),
("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."),
]
with (
@@ -151,9 +152,11 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
chunk_sql: str = chunk_insert_call[0][0]
assert "INSERT INTO chunks" in chunk_sql
# Section label comes from the heading: "Introduction"intro, "Proof of…" → proof.
# R-F: heading → controlled bucket where it maps ("Introduction"intro,
# "Proof of…"→proof), else the cleaned real title ("discrete conformal maps")
# instead of collapsing to "body".
chunk_rows = chunk_insert_call[0][1]
assert [row[4] for row in chunk_rows] == ["intro", "proof"]
assert [row[4] for row in chunk_rows] == ["intro", "proof", "discrete conformal maps"]
assert result.chunks_upserted == len(section_pairs)

View File

@@ -6,10 +6,12 @@ from unittest.mock import MagicMock
from codex.quality import (
_bib_score,
_clean_title,
classify_section,
filter_chunks,
is_quality_chunk,
run_quality_pass,
section_label,
)
# ---------------------------------------------------------------------------
@@ -243,3 +245,36 @@ class TestRunQualityPass:
kwargs = update_calls[0][0][1]
assert kwargs["section"] == "abstract"
assert kwargs["id"] == 5
class TestCleanTitle:
def test_strips_leading_numbering(self):
assert _clean_title("3.2 Variational Principles") == "variational principles"
def test_strips_latex_and_braces(self):
assert _clean_title(r"The \emph{Discrete} Laplacian") == "the discrete laplacian"
def test_collapses_and_lowercases(self):
assert _clean_title(" Rigidity Results ") == "rigidity results"
class TestSectionLabel:
def test_title_maps_to_controlled_bucket(self):
# Headings matching the controlled vocab keep the canonical bucket.
assert section_label("Introduction", "body text") == "intro"
assert section_label("Proof of Theorem 3", "body text") == "proof"
assert section_label("References", "body text") == "bibliography"
def test_descriptive_title_stored_verbatim(self):
# R-F win: a non-vocab heading is stored as its cleaned title, not "body".
assert section_label("Preliminaries", "body text") == "preliminaries"
assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps"
def test_none_or_blank_title_falls_back_to_content(self):
# .pdf/.txt path: unchanged content-based classification.
assert section_label(None, "Theorem 1. Let X be ...") == "theorem"
assert section_label("", "ordinary body prose here") == "body"
def test_title_cleaning_to_empty_falls_back_to_content(self):
# A pure-numbering heading cleans to "" → classify by content instead.
assert section_label("3.2", "Lemma 2. Suppose ...") == "theorem"