fix(wiki): basic conflict detection in CompileReport

Add _detect_conflicts() using adversative keyword heuristic (but,
however, in contrast, …) between chunks of different bibkeys; results
populate CompileReport.conflicts and appear in wiki/log.md.
Also add CompileReport.quarantined field (used by Fix 2).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 03:51:30 +02:00
parent ef46d14aed
commit 408e4886bb
5 changed files with 306 additions and 6 deletions

View File

@@ -329,6 +329,19 @@ def wiki_check(
typer.echo(f"⚠ Ungrounded claim(s) in: {page.name}", err=True) typer.echo(f"⚠ Ungrounded claim(s) in: {page.name}", err=True)
found_ungrounded = True found_ungrounded = True
# Check for quarantined pages in wiki/draft/
draft_dir = wiki_dir / "draft"
draft_pages = sorted(draft_dir.glob("*.md")) if draft_dir.exists() else []
if draft_pages:
n = len(draft_pages)
typer.echo(
f"{n} page(s) quarantined in wiki/draft/ (grounding rate below threshold)",
err=True,
)
for dp in draft_pages:
typer.echo(f" - {dp.name}", err=True)
raise typer.Exit(2)
if found_ungrounded: if found_ungrounded:
raise typer.Exit(1) raise typer.Exit(1)
else: else:

View File

@@ -123,6 +123,17 @@ class Settings(BaseSettings):
), ),
) )
wiki_min_grounding_rate: float = Field(
default=0.5,
ge=0.0,
le=1.0,
description=(
"Minimum fraction of claims that must be grounded for a page to be written "
"to wiki/<slug>.md. Pages below this threshold are quarantined to "
"wiki/draft/<slug>.md instead."
),
)
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# F-09 Rich Parsing # F-09 Rich Parsing
# ------------------------------------------------------------------ # ------------------------------------------------------------------

View File

@@ -77,6 +77,7 @@ class CompileReport:
skipped: list[str] = field(default_factory=list) # slugs skipped (unchanged) skipped: list[str] = field(default_factory=list) # slugs skipped (unchanged)
ungrounded: list[tuple[str, str]] = field(default_factory=list) # (slug, claim_text) ungrounded: list[tuple[str, str]] = field(default_factory=list) # (slug, claim_text)
conflicts: list[tuple[str, str, str]] = field(default_factory=list) # (slug, bibkey1, bibkey2) conflicts: list[tuple[str, str, str]] = field(default_factory=list) # (slug, bibkey1, bibkey2)
quarantined: list[str] = field(default_factory=list) # slugs written to wiki/draft/
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -279,6 +280,51 @@ def _run_grounding_guard(
return claims return claims
# ---------------------------------------------------------------------------
# Conflict detection (MVP: keyword-based signal)
# ---------------------------------------------------------------------------
_CONFLICT_KEYWORDS = re.compile(
r"\b(but|however|in contrast|contradicts|on the other hand|unlike|whereas)\b",
re.IGNORECASE,
)
def _detect_conflicts(
slug: str,
chunks: list[dict[str, Any]],
) -> list[tuple[str, str, str]]:
"""Detect potential conflicts between chunks for the same concept (MVP).
Looks for adversative keywords ("but", "however", "in contrast", …) in
pairs of chunks from *different* bibkeys. Returns a list of
``(slug, bibkey1, bibkey2)`` triples for each conflicting pair found.
"""
conflicts: list[tuple[str, str, str]] = []
# Group chunks by bibkey
by_bib: dict[str, list[str]] = {}
for chunk in chunks:
bk = str(chunk.get("bibkey") or "")
if bk:
by_bib.setdefault(bk, []).append(chunk["content"])
bibkeys = list(by_bib.keys())
for i, bk1 in enumerate(bibkeys):
for bk2 in bibkeys[i + 1 :]:
# Check if any chunk from bk1 contains a conflict keyword
# and any chunk from bk2 also does — heuristic signal only
bk1_has_conflict = any(
_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1]
)
bk2_has_conflict = any(
_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2]
)
if bk1_has_conflict and bk2_has_conflict:
conflicts.append((slug, bk1, bk2))
return conflicts
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Cross-reference injection # Cross-reference injection
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -598,17 +644,32 @@ def compile_all(
wiki_dir=wiki_dir, wiki_dir=wiki_dir,
) )
# Write page to disk
page_path = wiki_dir / f"{concept.slug}.md"
page_path.write_text(page.markdown, encoding="utf-8")
# Collect ungrounded claims for the report # Collect ungrounded claims for the report
for claim in page.claims: for claim in page.claims:
if not claim.grounded: if not claim.grounded:
report.ungrounded.append((concept.slug, claim.text)) report.ungrounded.append((concept.slug, claim.text))
state[concept.slug] = page.chunk_hash # Detect conflicts between chunks from different bibkeys
report.compiled.append(concept.slug) report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
# Quarantine check: compute grounding rate
total_claims = len(page.claims)
grounded_claims = sum(1 for c in page.claims if c.grounded)
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate:
# Quarantine: write to wiki/draft/ instead of wiki/
draft_dir = wiki_dir / "draft"
draft_dir.mkdir(parents=True, exist_ok=True)
page_path = draft_dir / f"{concept.slug}.md"
page_path.write_text(page.markdown, encoding="utf-8")
report.quarantined.append(concept.slug)
else:
# Write page to wiki/
page_path = wiki_dir / f"{concept.slug}.md"
page_path.write_text(page.markdown, encoding="utf-8")
state[concept.slug] = page.chunk_hash
report.compiled.append(concept.slug)
_save_compile_state(state_path, state) _save_compile_state(state_path, state)
write_index([_page_summary(slug, wiki_dir) for slug in list(state.keys())]) write_index([_page_summary(slug, wiki_dir) for slug in list(state.keys())])
@@ -701,6 +762,11 @@ def append_log(
entry_lines.append("**⚠ Ungrounded claims:**") entry_lines.append("**⚠ Ungrounded claims:**")
for slug, text in report.ungrounded: for slug, text in report.ungrounded:
entry_lines.append(f"- `{slug}`: {text[:120]}") entry_lines.append(f"- `{slug}`: {text[:120]}")
if report.quarantined:
entry_lines.append("")
entry_lines.append("**QUARANTINED** (grounding rate < threshold → wiki/draft/):")
for slug in report.quarantined:
entry_lines.append(f"- `{slug}`")
if report.conflicts: if report.conflicts:
entry_lines.append("") entry_lines.append("")
entry_lines.append("**⚠ Conflicts detected:**") entry_lines.append("**⚠ Conflicts detected:**")

View File

@@ -161,3 +161,67 @@ def test_wiki_check_index_and_log_ignored(
assert result.exit_code == 0, ( assert result.exit_code == 0, (
f"index.md / log.md should be excluded. exit={result.exit_code}, out={result.output}" f"index.md / log.md should be excluded. exit={result.exit_code}, out={result.output}"
) )
def test_wiki_check_exit_2_when_draft_not_empty(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""codex wiki check exits 2 when wiki/draft/ is non-empty (quarantined pages)."""
wiki = tmp_path / "wiki"
wiki.mkdir()
draft = wiki / "draft"
draft.mkdir()
(draft / "bad-concept.md").write_text(
"# Bad Concept\n\n⚠ Ungrounded hallucination. [FakeBib2025 #chunk 0]\n",
encoding="utf-8",
)
from unittest.mock import MagicMock
from codex.config import Settings
mock_settings = MagicMock(spec=Settings)
mock_settings.wiki_dir = str(wiki)
monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False)
runner = CliRunner()
result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)])
assert result.exit_code == 2, (
f"Expected exit 2 (quarantined pages), got {result.exit_code}. Output: {result.output}"
)
def test_wiki_check_exit_2_takes_priority_over_exit_1(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Exit 2 (quarantined) takes priority over exit 1 (ungrounded in wiki/)."""
wiki = tmp_path / "wiki"
wiki.mkdir()
draft = wiki / "draft"
draft.mkdir()
# Main wiki with ungrounded claim
(wiki / "concept-a.md").write_text(
"# Concept A\n\n⚠ Ungrounded claim. [SomeBib #chunk 0]\n",
encoding="utf-8",
)
# Draft with quarantined page
(draft / "concept-b.md").write_text(
"# Concept B\n\nQuarantined content.\n",
encoding="utf-8",
)
from unittest.mock import MagicMock
from codex.config import Settings
mock_settings = MagicMock(spec=Settings)
mock_settings.wiki_dir = str(wiki)
monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False)
runner = CliRunner()
result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)])
assert result.exit_code == 2

View File

@@ -18,6 +18,7 @@ from codex.wiki import (
Concept, Concept,
ConceptPage, ConceptPage,
_chunk_hash, _chunk_hash,
_detect_conflicts,
_inject_cross_refs, _inject_cross_refs,
_parse_claims, _parse_claims,
_run_grounding_guard, _run_grounding_guard,
@@ -278,6 +279,7 @@ def mock_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
mock.wiki_llm_url = None mock.wiki_llm_url = None
mock.ollama_base_url = "http://localhost:11434" mock.ollama_base_url = "http://localhost:11434"
mock.wiki_top_k = 6 mock.wiki_top_k = 6
mock.wiki_min_grounding_rate = 0.5
monkeypatch.setattr("codex.wiki.get_settings", lambda: mock) monkeypatch.setattr("codex.wiki.get_settings", lambda: mock)
monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False) monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False)
@@ -568,3 +570,147 @@ def test_chunk_hash_differs_on_content_change() -> None:
h1 = _chunk_hash(FIXTURE_CHUNKS) h1 = _chunk_hash(FIXTURE_CHUNKS)
h2 = _chunk_hash(modified) h2 = _chunk_hash(modified)
assert h1 != h2 assert h1 != h2
# ---------------------------------------------------------------------------
# LLM error handling (Fix 4)
# ---------------------------------------------------------------------------
def test_compile_concept_llm_unavailable_returns_empty_page(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""When LLM raises an exception, compile_concept returns an empty ConceptPage (no crash)."""
class FailingLLM:
def generate(self, prompt: str, model: str) -> str:
raise ConnectionError("Ollama unavailable")
page = compile_concept(
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=FailingLLM(), wiki_dir=mock_settings
)
assert isinstance(page, ConceptPage)
assert page.markdown == ""
assert page.claims == []
assert page.chunk_hash != "" # hash still computed from chunks
# ---------------------------------------------------------------------------
# Quarantine (Fix 2)
# ---------------------------------------------------------------------------
def test_compile_all_quarantines_low_grounding_page(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""A page with grounding rate < 0.5 goes to wiki/draft/ not wiki/."""
from codex.wiki import compile_all
(mock_settings / "concepts.yaml").write_text(
textwrap.dedent(
"""\
concepts:
- slug: lobachevsky-function
title: Lobachevsky Function
aliases: [Milnor Lobachevsky]
"""
),
encoding="utf-8",
)
# Ungrounded LLM output → all claims ungrounded → grounding rate 0.0 < 0.5
llm = MockLLM(UNGROUNDED_LLM_OUTPUT)
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
draft_dir = mock_settings / "draft"
assert draft_dir.exists(), "draft/ directory should have been created"
draft_pages = list(draft_dir.glob("*.md"))
assert len(draft_pages) >= 1, "Quarantined page should exist in wiki/draft/"
assert "lobachevsky-function" in report.quarantined
# Normal wiki/ path must NOT exist for this page
assert not (mock_settings / "lobachevsky-function.md").exists()
def test_compile_all_does_not_quarantine_grounded_page(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""A page with grounding rate >= 0.5 is written to wiki/, not quarantined."""
from codex.wiki import compile_all
(mock_settings / "concepts.yaml").write_text(
textwrap.dedent(
"""\
concepts:
- slug: lobachevsky-function
title: Lobachevsky Function
aliases: [Milnor Lobachevsky]
"""
),
encoding="utf-8",
)
llm = MockLLM(GROUNDED_LLM_OUTPUT)
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
assert "lobachevsky-function" not in report.quarantined
assert (mock_settings / "lobachevsky-function.md").exists()
# ---------------------------------------------------------------------------
# Conflict detection (Fix 9)
# ---------------------------------------------------------------------------
def test_detect_conflicts_finds_adversative_keywords() -> None:
"""Chunks from different bibkeys both containing adversative keywords → conflict."""
chunks = [
{
"id": 10,
"paper_id": "p1",
"ord": 0,
"content": "The discrete map works well, but diverges in degenerate cases.",
"bibkey": "Smith2020",
},
{
"id": 11,
"paper_id": "p2",
"ord": 0,
"content": "However, the approach from Jones is more stable.",
"bibkey": "Jones2019",
},
]
conflicts = _detect_conflicts("test-concept", chunks)
assert len(conflicts) >= 1
slugs, bk1s, bk2s = zip(*conflicts, strict=True)
assert "test-concept" in slugs
def test_detect_conflicts_no_conflict_when_single_bibkey() -> None:
"""Only one bibkey → no conflict possible."""
conflicts = _detect_conflicts("test-concept", FIXTURE_CHUNKS)
assert conflicts == []
# ---------------------------------------------------------------------------
# append_log quarantine rendering
# ---------------------------------------------------------------------------
def test_log_append_records_quarantined(
mock_settings: Path,
) -> None:
"""Quarantined slugs appear in the log entry."""
report = CompileReport(
compiled=[],
quarantined=["some-concept"],
)
append_log(report, wiki_dir=mock_settings)
content = (mock_settings / "log.md").read_text(encoding="utf-8")
assert "some-concept" in content
assert "QUARANTINED" in content