feat(quality): store real section titles for .tex chunks (R-F)
Add quality.section_label(title, content): for section-aware .tex ingest, keep the controlled bucket (intro/theorem/proof/abstract/bibliography) when the real \section heading maps to one, else store the cleaned title verbatim (e.g. 'preliminaries', 'rigidity') instead of collapsing to 'body'. Math papers title most sections descriptively, so R-C's vocab-only mapping left ~90% as 'body'; this lifts the section signal toward near-full. .pdf/.txt/run_quality_pass keep content-based classify_section unchanged. Safe because the section column is write-only (no consumer filters on the vocab). New _clean_title (strip LaTeX/numbering, lower-case, truncate). Tests: section_label bucket/verbatim/fallback paths + _clean_title; .tex ingest stores a descriptive heading as its title. Full suite 377 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -128,6 +128,7 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
section_pairs = [
|
||||
("Introduction", "Body of the introduction goes here."),
|
||||
("Proof of the Main Theorem", "Body of the proof goes here."),
|
||||
("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."),
|
||||
]
|
||||
|
||||
with (
|
||||
@@ -151,9 +152,11 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
chunk_sql: str = chunk_insert_call[0][0]
|
||||
assert "INSERT INTO chunks" in chunk_sql
|
||||
|
||||
# Section label comes from the heading: "Introduction" → intro, "Proof of…" → proof.
|
||||
# R-F: heading → controlled bucket where it maps ("Introduction"→intro,
|
||||
# "Proof of…"→proof), else the cleaned real title ("discrete conformal maps")
|
||||
# instead of collapsing to "body".
|
||||
chunk_rows = chunk_insert_call[0][1]
|
||||
assert [row[4] for row in chunk_rows] == ["intro", "proof"]
|
||||
assert [row[4] for row in chunk_rows] == ["intro", "proof", "discrete conformal maps"]
|
||||
assert result.chunks_upserted == len(section_pairs)
|
||||
|
||||
|
||||
|
||||
@@ -6,10 +6,12 @@ from unittest.mock import MagicMock
|
||||
|
||||
from codex.quality import (
|
||||
_bib_score,
|
||||
_clean_title,
|
||||
classify_section,
|
||||
filter_chunks,
|
||||
is_quality_chunk,
|
||||
run_quality_pass,
|
||||
section_label,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -243,3 +245,36 @@ class TestRunQualityPass:
|
||||
kwargs = update_calls[0][0][1]
|
||||
assert kwargs["section"] == "abstract"
|
||||
assert kwargs["id"] == 5
|
||||
|
||||
|
||||
class TestCleanTitle:
|
||||
def test_strips_leading_numbering(self):
|
||||
assert _clean_title("3.2 Variational Principles") == "variational principles"
|
||||
|
||||
def test_strips_latex_and_braces(self):
|
||||
assert _clean_title(r"The \emph{Discrete} Laplacian") == "the discrete laplacian"
|
||||
|
||||
def test_collapses_and_lowercases(self):
|
||||
assert _clean_title(" Rigidity Results ") == "rigidity results"
|
||||
|
||||
|
||||
class TestSectionLabel:
|
||||
def test_title_maps_to_controlled_bucket(self):
|
||||
# Headings matching the controlled vocab keep the canonical bucket.
|
||||
assert section_label("Introduction", "body text") == "intro"
|
||||
assert section_label("Proof of Theorem 3", "body text") == "proof"
|
||||
assert section_label("References", "body text") == "bibliography"
|
||||
|
||||
def test_descriptive_title_stored_verbatim(self):
|
||||
# R-F win: a non-vocab heading is stored as its cleaned title, not "body".
|
||||
assert section_label("Preliminaries", "body text") == "preliminaries"
|
||||
assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps"
|
||||
|
||||
def test_none_or_blank_title_falls_back_to_content(self):
|
||||
# .pdf/.txt path: unchanged content-based classification.
|
||||
assert section_label(None, "Theorem 1. Let X be ...") == "theorem"
|
||||
assert section_label("", "ordinary body prose here") == "body"
|
||||
|
||||
def test_title_cleaning_to_empty_falls_back_to_content(self):
|
||||
# A pure-numbering heading cleans to "" → classify by content instead.
|
||||
assert section_label("3.2", "Lemma 2. Suppose ...") == "theorem"
|
||||
|
||||
Reference in New Issue
Block a user