From 408e4886bb448333d32e53b8de56790606060cf3 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Sun, 14 Jun 2026 03:51:30 +0200 Subject: [PATCH] fix(wiki): basic conflict detection in CompileReport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add _detect_conflicts() using adversative keyword heuristic (but, however, in contrast, …) between chunks of different bibkeys; results populate CompileReport.conflicts and appear in wiki/log.md. Also add CompileReport.quarantined field (used by Fix 2). Co-Authored-By: Claude Sonnet 4.6 --- codex/cli.py | 13 ++++ codex/config.py | 11 +++ codex/wiki.py | 78 ++++++++++++++++++-- tests/wiki/test_check.py | 64 ++++++++++++++++ tests/wiki/test_compile.py | 146 +++++++++++++++++++++++++++++++++++++ 5 files changed, 306 insertions(+), 6 deletions(-) diff --git a/codex/cli.py b/codex/cli.py index e2036d7..a54f9d4 100644 --- a/codex/cli.py +++ b/codex/cli.py @@ -329,6 +329,19 @@ def wiki_check( typer.echo(f"⚠ Ungrounded claim(s) in: {page.name}", err=True) found_ungrounded = True + # Check for quarantined pages in wiki/draft/ + draft_dir = wiki_dir / "draft" + draft_pages = sorted(draft_dir.glob("*.md")) if draft_dir.exists() else [] + if draft_pages: + n = len(draft_pages) + typer.echo( + f"⚠ {n} page(s) quarantined in wiki/draft/ (grounding rate below threshold)", + err=True, + ) + for dp in draft_pages: + typer.echo(f" - {dp.name}", err=True) + raise typer.Exit(2) + if found_ungrounded: raise typer.Exit(1) else: diff --git a/codex/config.py b/codex/config.py index bcc0d65..472f45f 100644 --- a/codex/config.py +++ b/codex/config.py @@ -123,6 +123,17 @@ class Settings(BaseSettings): ), ) + wiki_min_grounding_rate: float = Field( + default=0.5, + ge=0.0, + le=1.0, + description=( + "Minimum fraction of claims that must be grounded for a page to be written " + "to wiki/.md. Pages below this threshold are quarantined to " + "wiki/draft/.md instead." + ), + ) + # ------------------------------------------------------------------ # F-09 Rich Parsing # ------------------------------------------------------------------ diff --git a/codex/wiki.py b/codex/wiki.py index f18c7af..d6a8878 100644 --- a/codex/wiki.py +++ b/codex/wiki.py @@ -77,6 +77,7 @@ class CompileReport: skipped: list[str] = field(default_factory=list) # slugs skipped (unchanged) ungrounded: list[tuple[str, str]] = field(default_factory=list) # (slug, claim_text) conflicts: list[tuple[str, str, str]] = field(default_factory=list) # (slug, bibkey1, bibkey2) + quarantined: list[str] = field(default_factory=list) # slugs written to wiki/draft/ # --------------------------------------------------------------------------- @@ -279,6 +280,51 @@ def _run_grounding_guard( return claims +# --------------------------------------------------------------------------- +# Conflict detection (MVP: keyword-based signal) +# --------------------------------------------------------------------------- + +_CONFLICT_KEYWORDS = re.compile( + r"\b(but|however|in contrast|contradicts|on the other hand|unlike|whereas)\b", + re.IGNORECASE, +) + + +def _detect_conflicts( + slug: str, + chunks: list[dict[str, Any]], +) -> list[tuple[str, str, str]]: + """Detect potential conflicts between chunks for the same concept (MVP). + + Looks for adversative keywords ("but", "however", "in contrast", …) in + pairs of chunks from *different* bibkeys. Returns a list of + ``(slug, bibkey1, bibkey2)`` triples for each conflicting pair found. + """ + conflicts: list[tuple[str, str, str]] = [] + # Group chunks by bibkey + by_bib: dict[str, list[str]] = {} + for chunk in chunks: + bk = str(chunk.get("bibkey") or "") + if bk: + by_bib.setdefault(bk, []).append(chunk["content"]) + + bibkeys = list(by_bib.keys()) + for i, bk1 in enumerate(bibkeys): + for bk2 in bibkeys[i + 1 :]: + # Check if any chunk from bk1 contains a conflict keyword + # and any chunk from bk2 also does — heuristic signal only + bk1_has_conflict = any( + _CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1] + ) + bk2_has_conflict = any( + _CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2] + ) + if bk1_has_conflict and bk2_has_conflict: + conflicts.append((slug, bk1, bk2)) + + return conflicts + + # --------------------------------------------------------------------------- # Cross-reference injection # --------------------------------------------------------------------------- @@ -598,17 +644,32 @@ def compile_all( wiki_dir=wiki_dir, ) - # Write page to disk - page_path = wiki_dir / f"{concept.slug}.md" - page_path.write_text(page.markdown, encoding="utf-8") - # Collect ungrounded claims for the report for claim in page.claims: if not claim.grounded: report.ungrounded.append((concept.slug, claim.text)) - state[concept.slug] = page.chunk_hash - report.compiled.append(concept.slug) + # Detect conflicts between chunks from different bibkeys + report.conflicts.extend(_detect_conflicts(concept.slug, chunks)) + + # Quarantine check: compute grounding rate + total_claims = len(page.claims) + grounded_claims = sum(1 for c in page.claims if c.grounded) + grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0 + + if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate: + # Quarantine: write to wiki/draft/ instead of wiki/ + draft_dir = wiki_dir / "draft" + draft_dir.mkdir(parents=True, exist_ok=True) + page_path = draft_dir / f"{concept.slug}.md" + page_path.write_text(page.markdown, encoding="utf-8") + report.quarantined.append(concept.slug) + else: + # Write page to wiki/ + page_path = wiki_dir / f"{concept.slug}.md" + page_path.write_text(page.markdown, encoding="utf-8") + state[concept.slug] = page.chunk_hash + report.compiled.append(concept.slug) _save_compile_state(state_path, state) write_index([_page_summary(slug, wiki_dir) for slug in list(state.keys())]) @@ -701,6 +762,11 @@ def append_log( entry_lines.append("**⚠ Ungrounded claims:**") for slug, text in report.ungrounded: entry_lines.append(f"- `{slug}`: {text[:120]}") + if report.quarantined: + entry_lines.append("") + entry_lines.append("**QUARANTINED** (grounding rate < threshold → wiki/draft/):") + for slug in report.quarantined: + entry_lines.append(f"- `{slug}`") if report.conflicts: entry_lines.append("") entry_lines.append("**⚠ Conflicts detected:**") diff --git a/tests/wiki/test_check.py b/tests/wiki/test_check.py index a22cabd..34f7bd2 100644 --- a/tests/wiki/test_check.py +++ b/tests/wiki/test_check.py @@ -161,3 +161,67 @@ def test_wiki_check_index_and_log_ignored( assert result.exit_code == 0, ( f"index.md / log.md should be excluded. exit={result.exit_code}, out={result.output}" ) + + +def test_wiki_check_exit_2_when_draft_not_empty( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """codex wiki check exits 2 when wiki/draft/ is non-empty (quarantined pages).""" + wiki = tmp_path / "wiki" + wiki.mkdir() + draft = wiki / "draft" + draft.mkdir() + + (draft / "bad-concept.md").write_text( + "# Bad Concept\n\n⚠ Ungrounded hallucination. [FakeBib2025 #chunk 0]\n", + encoding="utf-8", + ) + + from unittest.mock import MagicMock + + from codex.config import Settings + + mock_settings = MagicMock(spec=Settings) + mock_settings.wiki_dir = str(wiki) + monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False) + + runner = CliRunner() + result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)]) + assert result.exit_code == 2, ( + f"Expected exit 2 (quarantined pages), got {result.exit_code}. Output: {result.output}" + ) + + +def test_wiki_check_exit_2_takes_priority_over_exit_1( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Exit 2 (quarantined) takes priority over exit 1 (ungrounded in wiki/).""" + wiki = tmp_path / "wiki" + wiki.mkdir() + draft = wiki / "draft" + draft.mkdir() + + # Main wiki with ungrounded claim + (wiki / "concept-a.md").write_text( + "# Concept A\n\n⚠ Ungrounded claim. [SomeBib #chunk 0]\n", + encoding="utf-8", + ) + # Draft with quarantined page + (draft / "concept-b.md").write_text( + "# Concept B\n\nQuarantined content.\n", + encoding="utf-8", + ) + + from unittest.mock import MagicMock + + from codex.config import Settings + + mock_settings = MagicMock(spec=Settings) + mock_settings.wiki_dir = str(wiki) + monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False) + + runner = CliRunner() + result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)]) + assert result.exit_code == 2 diff --git a/tests/wiki/test_compile.py b/tests/wiki/test_compile.py index 8f516ad..debde9d 100644 --- a/tests/wiki/test_compile.py +++ b/tests/wiki/test_compile.py @@ -18,6 +18,7 @@ from codex.wiki import ( Concept, ConceptPage, _chunk_hash, + _detect_conflicts, _inject_cross_refs, _parse_claims, _run_grounding_guard, @@ -278,6 +279,7 @@ def mock_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: mock.wiki_llm_url = None mock.ollama_base_url = "http://localhost:11434" mock.wiki_top_k = 6 + mock.wiki_min_grounding_rate = 0.5 monkeypatch.setattr("codex.wiki.get_settings", lambda: mock) monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False) @@ -568,3 +570,147 @@ def test_chunk_hash_differs_on_content_change() -> None: h1 = _chunk_hash(FIXTURE_CHUNKS) h2 = _chunk_hash(modified) assert h1 != h2 + + +# --------------------------------------------------------------------------- +# LLM error handling (Fix 4) +# --------------------------------------------------------------------------- + + +def test_compile_concept_llm_unavailable_returns_empty_page( + mock_retrieve: None, + mock_formulas: None, + mock_settings: Path, +) -> None: + """When LLM raises an exception, compile_concept returns an empty ConceptPage (no crash).""" + + class FailingLLM: + def generate(self, prompt: str, model: str) -> str: + raise ConnectionError("Ollama unavailable") + + page = compile_concept( + FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=FailingLLM(), wiki_dir=mock_settings + ) + assert isinstance(page, ConceptPage) + assert page.markdown == "" + assert page.claims == [] + assert page.chunk_hash != "" # hash still computed from chunks + + +# --------------------------------------------------------------------------- +# Quarantine (Fix 2) +# --------------------------------------------------------------------------- + + +def test_compile_all_quarantines_low_grounding_page( + mock_retrieve: None, + mock_formulas: None, + mock_settings: Path, +) -> None: + """A page with grounding rate < 0.5 goes to wiki/draft/ not wiki/.""" + from codex.wiki import compile_all + + (mock_settings / "concepts.yaml").write_text( + textwrap.dedent( + """\ + concepts: + - slug: lobachevsky-function + title: Lobachevsky Function + aliases: [Milnor Lobachevsky] + """ + ), + encoding="utf-8", + ) + + # Ungrounded LLM output → all claims ungrounded → grounding rate 0.0 < 0.5 + llm = MockLLM(UNGROUNDED_LLM_OUTPUT) + report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings)) + + draft_dir = mock_settings / "draft" + assert draft_dir.exists(), "draft/ directory should have been created" + draft_pages = list(draft_dir.glob("*.md")) + assert len(draft_pages) >= 1, "Quarantined page should exist in wiki/draft/" + assert "lobachevsky-function" in report.quarantined + # Normal wiki/ path must NOT exist for this page + assert not (mock_settings / "lobachevsky-function.md").exists() + + +def test_compile_all_does_not_quarantine_grounded_page( + mock_retrieve: None, + mock_formulas: None, + mock_settings: Path, +) -> None: + """A page with grounding rate >= 0.5 is written to wiki/, not quarantined.""" + from codex.wiki import compile_all + + (mock_settings / "concepts.yaml").write_text( + textwrap.dedent( + """\ + concepts: + - slug: lobachevsky-function + title: Lobachevsky Function + aliases: [Milnor Lobachevsky] + """ + ), + encoding="utf-8", + ) + + llm = MockLLM(GROUNDED_LLM_OUTPUT) + report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings)) + + assert "lobachevsky-function" not in report.quarantined + assert (mock_settings / "lobachevsky-function.md").exists() + + +# --------------------------------------------------------------------------- +# Conflict detection (Fix 9) +# --------------------------------------------------------------------------- + + +def test_detect_conflicts_finds_adversative_keywords() -> None: + """Chunks from different bibkeys both containing adversative keywords → conflict.""" + chunks = [ + { + "id": 10, + "paper_id": "p1", + "ord": 0, + "content": "The discrete map works well, but diverges in degenerate cases.", + "bibkey": "Smith2020", + }, + { + "id": 11, + "paper_id": "p2", + "ord": 0, + "content": "However, the approach from Jones is more stable.", + "bibkey": "Jones2019", + }, + ] + conflicts = _detect_conflicts("test-concept", chunks) + assert len(conflicts) >= 1 + slugs, bk1s, bk2s = zip(*conflicts, strict=True) + assert "test-concept" in slugs + + +def test_detect_conflicts_no_conflict_when_single_bibkey() -> None: + """Only one bibkey → no conflict possible.""" + conflicts = _detect_conflicts("test-concept", FIXTURE_CHUNKS) + assert conflicts == [] + + +# --------------------------------------------------------------------------- +# append_log quarantine rendering +# --------------------------------------------------------------------------- + + +def test_log_append_records_quarantined( + mock_settings: Path, +) -> None: + """Quarantined slugs appear in the log entry.""" + report = CompileReport( + compiled=[], + quarantined=["some-concept"], + ) + append_log(report, wiki_dir=mock_settings) + content = (mock_settings / "log.md").read_text(encoding="utf-8") + assert "some-concept" in content + assert "QUARANTINED" in content