fix(wiki): sentence-level claim granularity + content-5-gram grounding guard
Replace 3-gram substring match with: - _last_sentence(): only the last sentence before a citation is checked (prevents a hallucinated paragraph grounding via a phrase at its end) - _content_words(): stopwords removed from both claim and chunk before n-gram comparison (prevents bypass via "the discrete conformal map") - Content-5-gram: require 5 consecutive non-stopword tokens from last sentence to appear in the chunk's content-word stream - Claims with <5 content words in last sentence → ungrounded by default Add adversarial tests: stopword-3-gram bypass → grounded=False; legitimate content-5-gram → grounded=True. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -170,6 +170,51 @@ def test_grounding_guard_ungrounded_when_bibkey_missing() -> None:
|
||||
assert result[0].grounded is False
|
||||
|
||||
|
||||
def test_grounding_guard_adversarial_stopword_bypass() -> None:
|
||||
"""Adversarial: stopword-3-gram matches are NOT sufficient for grounding.
|
||||
|
||||
'the discrete conformal map' appears in the chunk, but this is a stopword
|
||||
phrase. The preceding hallucinated sentences contain NO content-5-gram
|
||||
present in the chunk → grounded=False.
|
||||
"""
|
||||
# The adversarial text from the session-prompt review finding:
|
||||
adversarial_text = (
|
||||
"The Riemann hypothesis is proven by combining Hodge theory with quantum entanglement. "
|
||||
"Furthermore P=NP holds for hyperbolic tetrahedra. The discrete conformal map"
|
||||
)
|
||||
claim = Claim(
|
||||
text=adversarial_text,
|
||||
bibkey="Springborn2008",
|
||||
locator="chunk 0",
|
||||
)
|
||||
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
||||
assert result[0].grounded is False, (
|
||||
"Stopword 3-gram 'the discrete conformal map' must NOT ground a hallucinated claim"
|
||||
)
|
||||
|
||||
|
||||
def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored() -> None:
|
||||
"""Only the last sentence is checked for grounding.
|
||||
|
||||
If the last sentence IS grounded, the earlier hallucinated sentence is
|
||||
irrelevant (it has no citation → it won't be parsed as a Claim at all).
|
||||
This test verifies that a claim whose text = the last sentence grounds correctly.
|
||||
"""
|
||||
# The last sentence of the claim text is directly grounded in the fixture chunk.
|
||||
grounded_sentence = (
|
||||
"the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)"
|
||||
)
|
||||
claim = Claim(
|
||||
text=grounded_sentence,
|
||||
bibkey="Springborn2008",
|
||||
locator="chunk 16",
|
||||
)
|
||||
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
||||
assert result[0].grounded is True, (
|
||||
"Last sentence with content-5-gram present in chunk must be grounded"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _inject_cross_refs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user