Data-quality roadmap: R-A–R-F + DQ-5 fixes (citations 590→1075, section signal 10%→95%) #16
@@ -113,14 +113,18 @@ def classify_section(text: str) -> str:
|
||||
def _clean_title(title: str) -> str:
|
||||
"""Normalize a LaTeX ``\\section`` title for use as a section label.
|
||||
|
||||
Strips LaTeX commands (``\\emph`` …) and ``{}$``, drops leading section
|
||||
numbering (``3.2 ``), collapses whitespace, lower-cases, and truncates to
|
||||
60 chars so the stored ``section`` value is a clean, comparable string.
|
||||
Strips LaTeX word-commands (``\\emph`` …), accent/escape macros (``\\"o`` →
|
||||
``o``, ``\\&``), ``~`` ties and ``{}$``, drops leading section numbering
|
||||
(``3.2 ``), collapses whitespace, lower-cases, and truncates to 60 chars at a
|
||||
word boundary so the stored ``section`` value is a clean, comparable string.
|
||||
"""
|
||||
t = re.sub(r"\\[a-zA-Z]+\*?", " ", title) # LaTeX commands
|
||||
t = title.replace("~", " ") # LaTeX non-breaking tie → space
|
||||
t = re.sub(r"\\[a-zA-Z]+\*?", " ", t) # word-commands: \emph, \mathcal …
|
||||
t = re.sub(r"\\[^a-zA-Z]", "", t) # accent/escape macros: \"o → o, \', \`, \&
|
||||
t = re.sub(r"[{}$]", "", t) # braces / math delimiters
|
||||
t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 "
|
||||
return " ".join(t.split()).strip().lower()[:60]
|
||||
t = " ".join(t.split()).strip().lower()
|
||||
return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t
|
||||
|
||||
|
||||
def section_label(title: str | None, content: str) -> str:
|
||||
|
||||
@@ -257,6 +257,19 @@ class TestCleanTitle:
|
||||
def test_collapses_and_lowercases(self):
|
||||
assert _clean_title(" Rigidity Results ") == "rigidity results"
|
||||
|
||||
def test_strips_accent_macros(self):
|
||||
assert _clean_title(r"M\"obius Invariance") == "mobius invariance"
|
||||
|
||||
def test_strips_ties_and_accents(self):
|
||||
assert _clean_title(r"Colin~de~Verdi\`ere") == "colin de verdiere"
|
||||
|
||||
def test_truncates_at_word_boundary(self):
|
||||
long = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu omega"
|
||||
out = _clean_title(long)
|
||||
assert len(out) <= 60
|
||||
assert not out.endswith(" ")
|
||||
assert long.lower().startswith(out) # whole words from the start, no partial tail
|
||||
|
||||
|
||||
class TestSectionLabel:
|
||||
def test_title_maps_to_controlled_bucket(self):
|
||||
|
||||
Reference in New Issue
Block a user