codex/wiki.py

@@ -238,25 +238,66 @@ def _parse_claims(markdown: str) -> list[Claim]:
     return claims
 
 
+_STOPWORDS: frozenset[str] = frozenset(
+    {
+        "the", "a", "an", "in", "on", "of", "to", "is", "are", "was", "were",
+        "and", "or", "but", "for", "with", "this", "that", "it", "we", "they",
+        "be", "as", "at", "by", "from", "has", "have", "not", "which",
+    }
+)
+
+_SENTENCE_SPLIT_RE = re.compile(r"[.!?]")
+
+
+def _last_sentence(text: str) -> str:
+    """Return the last non-empty sentence from *text*.
+
+    Citations annotate the last sentence, not the whole preceding paragraph.
+    Splitting on ``.``, ``!``, ``?`` and taking the last non-empty element
+    ensures only the immediately preceding sentence is grounding-checked.
+    """
+    parts = _SENTENCE_SPLIT_RE.split(text)
+    for part in reversed(parts):
+        stripped = part.strip()
+        if stripped:
+            return stripped
+    return text.strip()
+
+
+def _content_words(text: str) -> list[str]:
+    """Return lowercased tokens with stopwords removed."""
+    return [w for w in text.lower().split() if w not in _STOPWORDS]
+
+
 def _run_grounding_guard(
     claims: list[Claim],
     chunks: list[dict[str, Any]],
 ) -> list[Claim]:
-    """Check each claim against its cited chunk via substring match (MVP).
+    """Check each claim against its cited chunk via content-5-gram match.
 
-    A claim is *grounded* if at least one phrase from its text (> 4 words)
+    A claim is *grounded* if the **last sentence** before its citation
-    appears as a substring in a chunk attributed to the same bibkey,
+    contains at least one sequence of 5 consecutive non-stopword tokens
-    OR if the claim text shares ≥ 3 consecutive words with any chunk of
+    (a "content-5-gram") that also appears as 5 consecutive non-stopword
-    that bibkey.
+    tokens in any chunk attributed to the same bibkey.
+
+    Matching is done in content-word space (stopwords removed from BOTH
+    claim and chunk), which prevents trivial bypass via common scientific
+    phrases like "the discrete conformal map" (only 2 content words).
+
+    Using sentence-level granularity prevents a hallucinated paragraph from
+    being grounded by a single matching phrase at its end.
 
     Sets ``claim.grounded = False`` for any claim that fails this check.
     """
-    # Build a bibkey → [content] index
+    # Build a bibkey → [content-word sequence] index
+    # We store the content-word list (joined as string for fast substring search)
     bib_index: dict[str, list[str]] = {}
     for chunk in chunks:
         bk = str(chunk.get("bibkey") or "")
         if bk:
-            bib_index.setdefault(bk, []).append(chunk["content"].lower())
+            # Content words of the chunk (stopwords removed, lowercased)
+            cw = " ".join(_content_words(chunk["content"]))
+            bib_index.setdefault(bk, []).append(cw)
 
     for claim in claims:
         sources = bib_index.get(claim.bibkey)
@@ -264,17 +305,23 @@ def _run_grounding_guard(
             claim.grounded = False
             continue
 
-        # Try to find any n-gram overlap (n ≥ 3 words)
+        # Grounding operates on the LAST SENTENCE only
-        words = claim.text.lower().split()
+        sentence = _last_sentence(claim.text)
+        content = _content_words(sentence)
+
+        if len(content) < 5:
+            # Too short for a content-5-gram → cannot be grounded.
+            # A claim sentence with fewer than 5 non-stopword tokens provides
+            # insufficient signal and is marked ungrounded by default.
+            claim.grounded = False
+            continue
+
         found = False
-        for n in range(min(len(words), 6), 2, -1):  # try 6-grams down to 3-grams
+        for i in range(len(content) - 4):  # content-5-grams
-            for i in range(len(words) - n + 1):
+            gram = " ".join(content[i : i + 5])
-                phrase = " ".join(words[i : i + n])
+            if any(gram in src for src in sources):
-                if any(phrase in src for src in sources):
                 found = True
                 break
-            if found:
-                break
         claim.grounded = found
 
     return claims

tests/wiki/test_compile.py

@@ -170,6 +170,51 @@ def test_grounding_guard_ungrounded_when_bibkey_missing() -> None:
     assert result[0].grounded is False
 
 
+def test_grounding_guard_adversarial_stopword_bypass() -> None:
+    """Adversarial: stopword-3-gram matches are NOT sufficient for grounding.
+
+    'the discrete conformal map' appears in the chunk, but this is a stopword
+    phrase. The preceding hallucinated sentences contain NO content-5-gram
+    present in the chunk → grounded=False.
+    """
+    # The adversarial text from the session-prompt review finding:
+    adversarial_text = (
+        "The Riemann hypothesis is proven by combining Hodge theory with quantum entanglement. "
+        "Furthermore P=NP holds for hyperbolic tetrahedra. The discrete conformal map"
+    )
+    claim = Claim(
+        text=adversarial_text,
+        bibkey="Springborn2008",
+        locator="chunk 0",
+    )
+    result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
+    assert result[0].grounded is False, (
+        "Stopword 3-gram 'the discrete conformal map' must NOT ground a hallucinated claim"
+    )
+
+
+def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored() -> None:
+    """Only the last sentence is checked for grounding.
+
+    If the last sentence IS grounded, the earlier hallucinated sentence is
+    irrelevant (it has no citation → it won't be parsed as a Claim at all).
+    This test verifies that a claim whose text = the last sentence grounds correctly.
+    """
+    # The last sentence of the claim text is directly grounded in the fixture chunk.
+    grounded_sentence = (
+        "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)"
+    )
+    claim = Claim(
+        text=grounded_sentence,
+        bibkey="Springborn2008",
+        locator="chunk 16",
+    )
+    result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
+    assert result[0].grounded is True, (
+        "Last sentence with content-5-gram present in chunk must be grounded"
+    )
+
+
 # ---------------------------------------------------------------------------
 # _inject_cross_refs
 # ---------------------------------------------------------------------------