Data-quality roadmap: R-A–R-F + DQ-5 fixes (citations 590→1075, section signal 10%→95%) #16

Merged
user2595 merged 31 commits from integration/roadmap into main 2026-06-27 07:12:31 +00:00
2 changed files with 22 additions and 5 deletions
Showing only changes of commit 1a879d4827 - Show all commits

View File

@@ -113,14 +113,18 @@ def classify_section(text: str) -> str:
def _clean_title(title: str) -> str: def _clean_title(title: str) -> str:
"""Normalize a LaTeX ``\\section`` title for use as a section label. """Normalize a LaTeX ``\\section`` title for use as a section label.
Strips LaTeX commands (``\\emph`` …) and ``{}$``, drops leading section Strips LaTeX word-commands (``\\emph`` …), accent/escape macros (``\\"o`` →
numbering (``3.2 ``), collapses whitespace, lower-cases, and truncates to ``o``, ``\\&``), ``~`` ties and ``{}$``, drops leading section numbering
60 chars so the stored ``section`` value is a clean, comparable string. (``3.2 ``), collapses whitespace, lower-cases, and truncates to 60 chars at a
word boundary so the stored ``section`` value is a clean, comparable string.
""" """
t = re.sub(r"\\[a-zA-Z]+\*?", " ", title) # LaTeX commands t = title.replace("~", " ") # LaTeX non-breaking tie → space
t = re.sub(r"\\[a-zA-Z]+\*?", " ", t) # word-commands: \emph, \mathcal …
t = re.sub(r"\\[^a-zA-Z]", "", t) # accent/escape macros: \"o → o, \', \`, \&
t = re.sub(r"[{}$]", "", t) # braces / math delimiters t = re.sub(r"[{}$]", "", t) # braces / math delimiters
t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 " t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 "
return " ".join(t.split()).strip().lower()[:60] t = " ".join(t.split()).strip().lower()
return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t
def section_label(title: str | None, content: str) -> str: def section_label(title: str | None, content: str) -> str:

View File

@@ -257,6 +257,19 @@ class TestCleanTitle:
def test_collapses_and_lowercases(self): def test_collapses_and_lowercases(self):
assert _clean_title(" Rigidity Results ") == "rigidity results" assert _clean_title(" Rigidity Results ") == "rigidity results"
def test_strips_accent_macros(self):
assert _clean_title(r"M\"obius Invariance") == "mobius invariance"
def test_strips_ties_and_accents(self):
assert _clean_title(r"Colin~de~Verdi\`ere") == "colin de verdiere"
def test_truncates_at_word_boundary(self):
long = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu omega"
out = _clean_title(long)
assert len(out) <= 60
assert not out.endswith(" ")
assert long.lower().startswith(out) # whole words from the start, no partial tail
class TestSectionLabel: class TestSectionLabel:
def test_title_maps_to_controlled_bucket(self): def test_title_maps_to_controlled_bucket(self):