fix(wiki): basic conflict detection in CompileReport

Add _detect_conflicts() using adversative keyword heuristic (but,
however, in contrast, …) between chunks of different bibkeys; results
populate CompileReport.conflicts and appear in wiki/log.md.
Also add CompileReport.quarantined field (used by Fix 2).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 03:51:30 +02:00
parent ef46d14aed
commit 408e4886bb
5 changed files with 306 additions and 6 deletions

View File

@@ -161,3 +161,67 @@ def test_wiki_check_index_and_log_ignored(
assert result.exit_code == 0, (
f"index.md / log.md should be excluded. exit={result.exit_code}, out={result.output}"
)
def test_wiki_check_exit_2_when_draft_not_empty(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""codex wiki check exits 2 when wiki/draft/ is non-empty (quarantined pages)."""
wiki = tmp_path / "wiki"
wiki.mkdir()
draft = wiki / "draft"
draft.mkdir()
(draft / "bad-concept.md").write_text(
"# Bad Concept\n\n⚠ Ungrounded hallucination. [FakeBib2025 #chunk 0]\n",
encoding="utf-8",
)
from unittest.mock import MagicMock
from codex.config import Settings
mock_settings = MagicMock(spec=Settings)
mock_settings.wiki_dir = str(wiki)
monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False)
runner = CliRunner()
result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)])
assert result.exit_code == 2, (
f"Expected exit 2 (quarantined pages), got {result.exit_code}. Output: {result.output}"
)
def test_wiki_check_exit_2_takes_priority_over_exit_1(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Exit 2 (quarantined) takes priority over exit 1 (ungrounded in wiki/)."""
wiki = tmp_path / "wiki"
wiki.mkdir()
draft = wiki / "draft"
draft.mkdir()
# Main wiki with ungrounded claim
(wiki / "concept-a.md").write_text(
"# Concept A\n\n⚠ Ungrounded claim. [SomeBib #chunk 0]\n",
encoding="utf-8",
)
# Draft with quarantined page
(draft / "concept-b.md").write_text(
"# Concept B\n\nQuarantined content.\n",
encoding="utf-8",
)
from unittest.mock import MagicMock
from codex.config import Settings
mock_settings = MagicMock(spec=Settings)
mock_settings.wiki_dir = str(wiki)
monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False)
runner = CliRunner()
result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)])
assert result.exit_code == 2

View File

@@ -18,6 +18,7 @@ from codex.wiki import (
Concept,
ConceptPage,
_chunk_hash,
_detect_conflicts,
_inject_cross_refs,
_parse_claims,
_run_grounding_guard,
@@ -278,6 +279,7 @@ def mock_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
mock.wiki_llm_url = None
mock.ollama_base_url = "http://localhost:11434"
mock.wiki_top_k = 6
mock.wiki_min_grounding_rate = 0.5
monkeypatch.setattr("codex.wiki.get_settings", lambda: mock)
monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False)
@@ -568,3 +570,147 @@ def test_chunk_hash_differs_on_content_change() -> None:
h1 = _chunk_hash(FIXTURE_CHUNKS)
h2 = _chunk_hash(modified)
assert h1 != h2
# ---------------------------------------------------------------------------
# LLM error handling (Fix 4)
# ---------------------------------------------------------------------------
def test_compile_concept_llm_unavailable_returns_empty_page(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""When LLM raises an exception, compile_concept returns an empty ConceptPage (no crash)."""
class FailingLLM:
def generate(self, prompt: str, model: str) -> str:
raise ConnectionError("Ollama unavailable")
page = compile_concept(
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=FailingLLM(), wiki_dir=mock_settings
)
assert isinstance(page, ConceptPage)
assert page.markdown == ""
assert page.claims == []
assert page.chunk_hash != "" # hash still computed from chunks
# ---------------------------------------------------------------------------
# Quarantine (Fix 2)
# ---------------------------------------------------------------------------
def test_compile_all_quarantines_low_grounding_page(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""A page with grounding rate < 0.5 goes to wiki/draft/ not wiki/."""
from codex.wiki import compile_all
(mock_settings / "concepts.yaml").write_text(
textwrap.dedent(
"""\
concepts:
- slug: lobachevsky-function
title: Lobachevsky Function
aliases: [Milnor Lobachevsky]
"""
),
encoding="utf-8",
)
# Ungrounded LLM output → all claims ungrounded → grounding rate 0.0 < 0.5
llm = MockLLM(UNGROUNDED_LLM_OUTPUT)
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
draft_dir = mock_settings / "draft"
assert draft_dir.exists(), "draft/ directory should have been created"
draft_pages = list(draft_dir.glob("*.md"))
assert len(draft_pages) >= 1, "Quarantined page should exist in wiki/draft/"
assert "lobachevsky-function" in report.quarantined
# Normal wiki/ path must NOT exist for this page
assert not (mock_settings / "lobachevsky-function.md").exists()
def test_compile_all_does_not_quarantine_grounded_page(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""A page with grounding rate >= 0.5 is written to wiki/, not quarantined."""
from codex.wiki import compile_all
(mock_settings / "concepts.yaml").write_text(
textwrap.dedent(
"""\
concepts:
- slug: lobachevsky-function
title: Lobachevsky Function
aliases: [Milnor Lobachevsky]
"""
),
encoding="utf-8",
)
llm = MockLLM(GROUNDED_LLM_OUTPUT)
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
assert "lobachevsky-function" not in report.quarantined
assert (mock_settings / "lobachevsky-function.md").exists()
# ---------------------------------------------------------------------------
# Conflict detection (Fix 9)
# ---------------------------------------------------------------------------
def test_detect_conflicts_finds_adversative_keywords() -> None:
"""Chunks from different bibkeys both containing adversative keywords → conflict."""
chunks = [
{
"id": 10,
"paper_id": "p1",
"ord": 0,
"content": "The discrete map works well, but diverges in degenerate cases.",
"bibkey": "Smith2020",
},
{
"id": 11,
"paper_id": "p2",
"ord": 0,
"content": "However, the approach from Jones is more stable.",
"bibkey": "Jones2019",
},
]
conflicts = _detect_conflicts("test-concept", chunks)
assert len(conflicts) >= 1
slugs, bk1s, bk2s = zip(*conflicts, strict=True)
assert "test-concept" in slugs
def test_detect_conflicts_no_conflict_when_single_bibkey() -> None:
"""Only one bibkey → no conflict possible."""
conflicts = _detect_conflicts("test-concept", FIXTURE_CHUNKS)
assert conflicts == []
# ---------------------------------------------------------------------------
# append_log quarantine rendering
# ---------------------------------------------------------------------------
def test_log_append_records_quarantined(
mock_settings: Path,
) -> None:
"""Quarantined slugs appear in the log entry."""
report = CompileReport(
compiled=[],
quarantined=["some-concept"],
)
append_log(report, wiki_dir=mock_settings)
content = (mock_settings / "log.md").read_text(encoding="utf-8")
assert "some-concept" in content
assert "QUARANTINED" in content