fix(wiki): token-boundary grounding match + sharper ref-list filter (C-6, R-2)
- C-6: the content-5-gram check used 'gram in src' on space-joined strings, so the
first/last gram token could match a prefix/suffix of a longer chunk word
('set' inside 'subset'). Both gram and chunk are now space-padded, so a match
requires whole-token alignment. Regression test added.
- R-2: the reference-list filter's author pattern ('Surname, I.') also matched
'Theorem A.'/'Lemma B.', so theorem-dense chunks could be dropped as a
bibliography. The pattern now excludes common math-structure words.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -186,8 +186,15 @@ def _retrieve_chunks(
|
||||
).fetchall()
|
||||
|
||||
# Filter reference-list chunks: skip chunks whose content looks like a bibliography
|
||||
# (heuristic: > 60 % of lines match "^\[\d+\]" or "^[A-Z][a-z]+,?\s+[A-Z]\.").
|
||||
ref_pattern = re.compile(r"^\s*(\[\d+\]|[A-Z][a-z]+,?\s+[A-Z]\.)", re.MULTILINE)
|
||||
# (heuristic: > 60 % of lines look like "[12]" refs or "Surname, I." authors).
|
||||
# The author alternative excludes common math-structure words ("Theorem A.",
|
||||
# "Lemma B.") so theorem-dense chunks aren't mistaken for a reference list (R-2).
|
||||
ref_pattern = re.compile(
|
||||
r"^\s*(\[\d+\]|(?!(?:Theorem|Lemma|Proposition|Corollary|Definition|Proof|"
|
||||
r"Section|Figure|Fig|Table|Equation|Eq|Remark|Example|Chapter)\b)"
|
||||
r"[A-Z][a-z]+,?\s+[A-Z]\.)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
filtered: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
content: str = row["content"]
|
||||
@@ -336,8 +343,9 @@ def _run_grounding_guard(
|
||||
for chunk in chunks:
|
||||
bk = str(chunk.get("bibkey") or "")
|
||||
if bk:
|
||||
# Content words of the chunk (stopwords removed, lowercased)
|
||||
cw = " ".join(_content_words(chunk["content"]))
|
||||
# Content words of the chunk, space-padded so a 5-gram match is bounded
|
||||
# by whole tokens, not a substring bleeding across words (audit C-6).
|
||||
cw = " " + " ".join(_content_words(chunk["content"])) + " "
|
||||
bib_index.setdefault(bk, []).append(cw)
|
||||
|
||||
for claim in claims:
|
||||
@@ -359,7 +367,7 @@ def _run_grounding_guard(
|
||||
|
||||
found = False
|
||||
for i in range(len(content) - 4): # content-5-grams
|
||||
gram = " ".join(content[i : i + 5])
|
||||
gram = " " + " ".join(content[i : i + 5]) + " " # padded: token-boundary match
|
||||
if any(gram in src for src in sources):
|
||||
found = True
|
||||
break
|
||||
|
||||
Reference in New Issue
Block a user