fix(wiki): sentence-level claim granularity + content-5-gram grounding guard

Replace 3-gram substring match with:
- _last_sentence(): only the last sentence before a citation is checked
  (prevents a hallucinated paragraph grounding via a phrase at its end)
- _content_words(): stopwords removed from both claim and chunk before
  n-gram comparison (prevents bypass via "the discrete conformal map")
- Content-5-gram: require 5 consecutive non-stopword tokens from last
  sentence to appear in the chunk's content-word stream
- Claims with <5 content words in last sentence → ungrounded by default

Add adversarial tests: stopword-3-gram bypass → grounded=False;
legitimate content-5-gram → grounded=True.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

codex/wiki.py

@@ -238,25 +238,66 @@ def _parse_claims(markdown: str) -> list[Claim]:
     return claims
 
 
+_STOPWORDS: frozenset[str] = frozenset(
+    {
+        "the", "a", "an", "in", "on", "of", "to", "is", "are", "was", "were",
+        "and", "or", "but", "for", "with", "this", "that", "it", "we", "they",
+        "be", "as", "at", "by", "from", "has", "have", "not", "which",
+    }
+)
+
+_SENTENCE_SPLIT_RE = re.compile(r"[.!?]")
+
+
+def _last_sentence(text: str) -> str:
+    """Return the last non-empty sentence from *text*.
+
+    Citations annotate the last sentence, not the whole preceding paragraph.
+    Splitting on ``.``, ``!``, ``?`` and taking the last non-empty element
+    ensures only the immediately preceding sentence is grounding-checked.
+    """
+    parts = _SENTENCE_SPLIT_RE.split(text)
+    for part in reversed(parts):
+        stripped = part.strip()
+        if stripped:
+            return stripped
+    return text.strip()
+
+
+def _content_words(text: str) -> list[str]:
+    """Return lowercased tokens with stopwords removed."""
+    return [w for w in text.lower().split() if w not in _STOPWORDS]
+
+
 def _run_grounding_guard(
     claims: list[Claim],
     chunks: list[dict[str, Any]],
 ) -> list[Claim]:
-    """Check each claim against its cited chunk via substring match (MVP).
+    """Check each claim against its cited chunk via content-5-gram match.
 
-    A claim is *grounded* if at least one phrase from its text (> 4 words)
-    appears as a substring in a chunk attributed to the same bibkey,
-    OR if the claim text shares ≥ 3 consecutive words with any chunk of
-    that bibkey.
+    A claim is *grounded* if the **last sentence** before its citation
+    contains at least one sequence of 5 consecutive non-stopword tokens
+    (a "content-5-gram") that also appears as 5 consecutive non-stopword
+    tokens in any chunk attributed to the same bibkey.
+
+    Matching is done in content-word space (stopwords removed from BOTH
+    claim and chunk), which prevents trivial bypass via common scientific
+    phrases like "the discrete conformal map" (only 2 content words).
+
+    Using sentence-level granularity prevents a hallucinated paragraph from
+    being grounded by a single matching phrase at its end.
 
     Sets ``claim.grounded = False`` for any claim that fails this check.
     """
-    # Build a bibkey → [content] index
+    # Build a bibkey → [content-word sequence] index
+    # We store the content-word list (joined as string for fast substring search)
     bib_index: dict[str, list[str]] = {}
     for chunk in chunks:
         bk = str(chunk.get("bibkey") or "")
         if bk:
-            bib_index.setdefault(bk, []).append(chunk["content"].lower())
+            # Content words of the chunk (stopwords removed, lowercased)
+            cw = " ".join(_content_words(chunk["content"]))
+            bib_index.setdefault(bk, []).append(cw)
 
     for claim in claims:
         sources = bib_index.get(claim.bibkey)
@@ -264,16 +305,22 @@ def _run_grounding_guard(
             claim.grounded = False
             continue
 
-        # Try to find any n-gram overlap (n ≥ 3 words)
-        words = claim.text.lower().split()
+        # Grounding operates on the LAST SENTENCE only
+        sentence = _last_sentence(claim.text)
+        content = _content_words(sentence)
+
+        if len(content) < 5:
+            # Too short for a content-5-gram → cannot be grounded.
+            # A claim sentence with fewer than 5 non-stopword tokens provides
+            # insufficient signal and is marked ungrounded by default.
+            claim.grounded = False
+            continue
+
         found = False
-        for n in range(min(len(words), 6), 2, -1):  # try 6-grams down to 3-grams
-            for i in range(len(words) - n + 1):
-                phrase = " ".join(words[i : i + n])
-                if any(phrase in src for src in sources):
-                    found = True
-                    break
-            if found:
+        for i in range(len(content) - 4):  # content-5-grams
+            gram = " ".join(content[i : i + 5])
+            if any(gram in src for src in sources):
+                found = True
                 break
         claim.grounded = found