From f422b0fb84b20a4fcb6191909932dabfea2b82cc Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 15 Jun 2026 21:46:02 +0200 Subject: [PATCH] fix(wiki): token-boundary grounding match + sharper ref-list filter (C-6, R-2) - C-6: the content-5-gram check used 'gram in src' on space-joined strings, so the first/last gram token could match a prefix/suffix of a longer chunk word ('set' inside 'subset'). Both gram and chunk are now space-padded, so a match requires whole-token alignment. Regression test added. - R-2: the reference-list filter's author pattern ('Surname, I.') also matched 'Theorem A.'/'Lemma B.', so theorem-dense chunks could be dropped as a bibliography. The pattern now excludes common math-structure words. Co-Authored-By: Claude Fable 5 --- codex/wiki.py | 18 +++++++++++++----- tests/wiki/test_compile.py | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/codex/wiki.py b/codex/wiki.py index 7a0ec8b..59e17df 100644 --- a/codex/wiki.py +++ b/codex/wiki.py @@ -186,8 +186,15 @@ def _retrieve_chunks( ).fetchall() # Filter reference-list chunks: skip chunks whose content looks like a bibliography - # (heuristic: > 60 % of lines match "^\[\d+\]" or "^[A-Z][a-z]+,?\s+[A-Z]\."). - ref_pattern = re.compile(r"^\s*(\[\d+\]|[A-Z][a-z]+,?\s+[A-Z]\.)", re.MULTILINE) + # (heuristic: > 60 % of lines look like "[12]" refs or "Surname, I." authors). + # The author alternative excludes common math-structure words ("Theorem A.", + # "Lemma B.") so theorem-dense chunks aren't mistaken for a reference list (R-2). + ref_pattern = re.compile( + r"^\s*(\[\d+\]|(?!(?:Theorem|Lemma|Proposition|Corollary|Definition|Proof|" + r"Section|Figure|Fig|Table|Equation|Eq|Remark|Example|Chapter)\b)" + r"[A-Z][a-z]+,?\s+[A-Z]\.)", + re.MULTILINE, + ) filtered: list[dict[str, Any]] = [] for row in rows: content: str = row["content"] @@ -336,8 +343,9 @@ def _run_grounding_guard( for chunk in chunks: bk = str(chunk.get("bibkey") or "") if bk: - # Content words of the chunk (stopwords removed, lowercased) - cw = " ".join(_content_words(chunk["content"])) + # Content words of the chunk, space-padded so a 5-gram match is bounded + # by whole tokens, not a substring bleeding across words (audit C-6). + cw = " " + " ".join(_content_words(chunk["content"])) + " " bib_index.setdefault(bk, []).append(cw) for claim in claims: @@ -359,7 +367,7 @@ def _run_grounding_guard( found = False for i in range(len(content) - 4): # content-5-grams - gram = " ".join(content[i : i + 5]) + gram = " " + " ".join(content[i : i + 5]) + " " # padded: token-boundary match if any(gram in src for src in sources): found = True break diff --git a/tests/wiki/test_compile.py b/tests/wiki/test_compile.py index c0db957..8afcd5b 100644 --- a/tests/wiki/test_compile.py +++ b/tests/wiki/test_compile.py @@ -230,6 +230,30 @@ def test_grounding_guard_robust_to_trailing_comma() -> None: assert result[0].grounded is True +def test_grounding_guard_no_substring_bleed_across_tokens() -> None: + """A 5-gram must match whole tokens, not bleed into a longer chunk word (audit C-6). + + The chunk contains 'subset'; a claim whose first token is 'set' must NOT ground + by substring-matching the suffix of 'subset'. + """ + chunks = [ + { + "id": 99, + "paper_id": "p", + "ord": 0, + "bibkey": "TestBib", + "content": "the subset conformal map energy functional is minimized here", + } + ] + claim = Claim( + text="the set conformal map energy functional", + bibkey="TestBib", + locator="chunk 0", + ) + result = _run_grounding_guard([claim], chunks) + assert result[0].grounded is False + + # --------------------------------------------------------------------------- # _inject_cross_refs # ---------------------------------------------------------------------------