diff --git a/codex/wiki.py b/codex/wiki.py index e66024f..7a0ec8b 100644 --- a/codex/wiki.py +++ b/codex/wiki.py @@ -273,7 +273,13 @@ _STOPWORDS: frozenset[str] = frozenset( } ) -_SENTENCE_SPLIT_RE = re.compile(r"[.!?]") +# Split on sentence-ending punctuation only when followed by whitespace/end, so a +# decimal like "1.5" does not create a spurious sentence boundary (audit R-1). +_SENTENCE_SPLIT_RE = re.compile(r"[.!?]+(?=\s|$)") + +# Punctuation stripped from token edges before grounding comparison — sentence/ +# clause marks and quotes, NOT math delimiters like ()=+ (audit R-1). +_EDGE_PUNCT = ".,;:!?\"'" def _last_sentence(text: str) -> str: @@ -292,8 +298,16 @@ def _last_sentence(text: str) -> str: def _content_words(text: str) -> list[str]: - """Return lowercased tokens with stopwords removed.""" - return [w for w in text.lower().split() if w not in _STOPWORDS] + """Return lowercased content tokens, edge-punctuation-stripped, stopwords removed. + + Stripping leading/trailing sentence punctuation makes grounding robust to + surface variation ("volume," vs "volume") so a faithful paraphrase is not + marked ungrounded over a comma (audit R-1). Math delimiters (``()=+``) are + preserved. Claim and chunk tokens are normalised identically, so matching + stays consistent. + """ + tokens = (w.strip(_EDGE_PUNCT) for w in text.lower().split()) + return [w for w in tokens if w and w not in _STOPWORDS] def _run_grounding_guard( diff --git a/tests/wiki/test_compile.py b/tests/wiki/test_compile.py index 632905c..c0db957 100644 --- a/tests/wiki/test_compile.py +++ b/tests/wiki/test_compile.py @@ -213,6 +213,23 @@ def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored() ) +def test_grounding_guard_robust_to_trailing_comma() -> None: + """A faithful 5-word claim grounds despite a trailing comma (audit R-1). + + The chunk says "... the volume function V0 is strictly concave on the angle + domain." Before edge-punctuation stripping, the claim token "concave," != the + chunk token "concave", so the only content-5-gram failed to match and the + claim was wrongly marked ungrounded. + """ + claim = Claim( + text="the volume function V0 is strictly concave,", + bibkey="Springborn2008", + locator="chunk 16", + ) + result = _run_grounding_guard([claim], FIXTURE_CHUNKS) + assert result[0].grounded is True + + # --------------------------------------------------------------------------- # _inject_cross_refs # ---------------------------------------------------------------------------