fix(wiki): basic conflict detection in CompileReport
Add _detect_conflicts() using adversative keyword heuristic (but, however, in contrast, …) between chunks of different bibkeys; results populate CompileReport.conflicts and appear in wiki/log.md. Also add CompileReport.quarantined field (used by Fix 2). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
13
codex/cli.py
13
codex/cli.py
@@ -329,6 +329,19 @@ def wiki_check(
|
|||||||
typer.echo(f"⚠ Ungrounded claim(s) in: {page.name}", err=True)
|
typer.echo(f"⚠ Ungrounded claim(s) in: {page.name}", err=True)
|
||||||
found_ungrounded = True
|
found_ungrounded = True
|
||||||
|
|
||||||
|
# Check for quarantined pages in wiki/draft/
|
||||||
|
draft_dir = wiki_dir / "draft"
|
||||||
|
draft_pages = sorted(draft_dir.glob("*.md")) if draft_dir.exists() else []
|
||||||
|
if draft_pages:
|
||||||
|
n = len(draft_pages)
|
||||||
|
typer.echo(
|
||||||
|
f"⚠ {n} page(s) quarantined in wiki/draft/ (grounding rate below threshold)",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
for dp in draft_pages:
|
||||||
|
typer.echo(f" - {dp.name}", err=True)
|
||||||
|
raise typer.Exit(2)
|
||||||
|
|
||||||
if found_ungrounded:
|
if found_ungrounded:
|
||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -123,6 +123,17 @@ class Settings(BaseSettings):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
wiki_min_grounding_rate: float = Field(
|
||||||
|
default=0.5,
|
||||||
|
ge=0.0,
|
||||||
|
le=1.0,
|
||||||
|
description=(
|
||||||
|
"Minimum fraction of claims that must be grounded for a page to be written "
|
||||||
|
"to wiki/<slug>.md. Pages below this threshold are quarantined to "
|
||||||
|
"wiki/draft/<slug>.md instead."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# F-09 Rich Parsing
|
# F-09 Rich Parsing
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ class CompileReport:
|
|||||||
skipped: list[str] = field(default_factory=list) # slugs skipped (unchanged)
|
skipped: list[str] = field(default_factory=list) # slugs skipped (unchanged)
|
||||||
ungrounded: list[tuple[str, str]] = field(default_factory=list) # (slug, claim_text)
|
ungrounded: list[tuple[str, str]] = field(default_factory=list) # (slug, claim_text)
|
||||||
conflicts: list[tuple[str, str, str]] = field(default_factory=list) # (slug, bibkey1, bibkey2)
|
conflicts: list[tuple[str, str, str]] = field(default_factory=list) # (slug, bibkey1, bibkey2)
|
||||||
|
quarantined: list[str] = field(default_factory=list) # slugs written to wiki/draft/
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -279,6 +280,51 @@ def _run_grounding_guard(
|
|||||||
return claims
|
return claims
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Conflict detection (MVP: keyword-based signal)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_CONFLICT_KEYWORDS = re.compile(
|
||||||
|
r"\b(but|however|in contrast|contradicts|on the other hand|unlike|whereas)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_conflicts(
|
||||||
|
slug: str,
|
||||||
|
chunks: list[dict[str, Any]],
|
||||||
|
) -> list[tuple[str, str, str]]:
|
||||||
|
"""Detect potential conflicts between chunks for the same concept (MVP).
|
||||||
|
|
||||||
|
Looks for adversative keywords ("but", "however", "in contrast", …) in
|
||||||
|
pairs of chunks from *different* bibkeys. Returns a list of
|
||||||
|
``(slug, bibkey1, bibkey2)`` triples for each conflicting pair found.
|
||||||
|
"""
|
||||||
|
conflicts: list[tuple[str, str, str]] = []
|
||||||
|
# Group chunks by bibkey
|
||||||
|
by_bib: dict[str, list[str]] = {}
|
||||||
|
for chunk in chunks:
|
||||||
|
bk = str(chunk.get("bibkey") or "")
|
||||||
|
if bk:
|
||||||
|
by_bib.setdefault(bk, []).append(chunk["content"])
|
||||||
|
|
||||||
|
bibkeys = list(by_bib.keys())
|
||||||
|
for i, bk1 in enumerate(bibkeys):
|
||||||
|
for bk2 in bibkeys[i + 1 :]:
|
||||||
|
# Check if any chunk from bk1 contains a conflict keyword
|
||||||
|
# and any chunk from bk2 also does — heuristic signal only
|
||||||
|
bk1_has_conflict = any(
|
||||||
|
_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1]
|
||||||
|
)
|
||||||
|
bk2_has_conflict = any(
|
||||||
|
_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2]
|
||||||
|
)
|
||||||
|
if bk1_has_conflict and bk2_has_conflict:
|
||||||
|
conflicts.append((slug, bk1, bk2))
|
||||||
|
|
||||||
|
return conflicts
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Cross-reference injection
|
# Cross-reference injection
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -598,17 +644,32 @@ def compile_all(
|
|||||||
wiki_dir=wiki_dir,
|
wiki_dir=wiki_dir,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Write page to disk
|
|
||||||
page_path = wiki_dir / f"{concept.slug}.md"
|
|
||||||
page_path.write_text(page.markdown, encoding="utf-8")
|
|
||||||
|
|
||||||
# Collect ungrounded claims for the report
|
# Collect ungrounded claims for the report
|
||||||
for claim in page.claims:
|
for claim in page.claims:
|
||||||
if not claim.grounded:
|
if not claim.grounded:
|
||||||
report.ungrounded.append((concept.slug, claim.text))
|
report.ungrounded.append((concept.slug, claim.text))
|
||||||
|
|
||||||
state[concept.slug] = page.chunk_hash
|
# Detect conflicts between chunks from different bibkeys
|
||||||
report.compiled.append(concept.slug)
|
report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
|
||||||
|
|
||||||
|
# Quarantine check: compute grounding rate
|
||||||
|
total_claims = len(page.claims)
|
||||||
|
grounded_claims = sum(1 for c in page.claims if c.grounded)
|
||||||
|
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
|
||||||
|
|
||||||
|
if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate:
|
||||||
|
# Quarantine: write to wiki/draft/ instead of wiki/
|
||||||
|
draft_dir = wiki_dir / "draft"
|
||||||
|
draft_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
page_path = draft_dir / f"{concept.slug}.md"
|
||||||
|
page_path.write_text(page.markdown, encoding="utf-8")
|
||||||
|
report.quarantined.append(concept.slug)
|
||||||
|
else:
|
||||||
|
# Write page to wiki/
|
||||||
|
page_path = wiki_dir / f"{concept.slug}.md"
|
||||||
|
page_path.write_text(page.markdown, encoding="utf-8")
|
||||||
|
state[concept.slug] = page.chunk_hash
|
||||||
|
report.compiled.append(concept.slug)
|
||||||
|
|
||||||
_save_compile_state(state_path, state)
|
_save_compile_state(state_path, state)
|
||||||
write_index([_page_summary(slug, wiki_dir) for slug in list(state.keys())])
|
write_index([_page_summary(slug, wiki_dir) for slug in list(state.keys())])
|
||||||
@@ -701,6 +762,11 @@ def append_log(
|
|||||||
entry_lines.append("**⚠ Ungrounded claims:**")
|
entry_lines.append("**⚠ Ungrounded claims:**")
|
||||||
for slug, text in report.ungrounded:
|
for slug, text in report.ungrounded:
|
||||||
entry_lines.append(f"- `{slug}`: {text[:120]}")
|
entry_lines.append(f"- `{slug}`: {text[:120]}")
|
||||||
|
if report.quarantined:
|
||||||
|
entry_lines.append("")
|
||||||
|
entry_lines.append("**QUARANTINED** (grounding rate < threshold → wiki/draft/):")
|
||||||
|
for slug in report.quarantined:
|
||||||
|
entry_lines.append(f"- `{slug}`")
|
||||||
if report.conflicts:
|
if report.conflicts:
|
||||||
entry_lines.append("")
|
entry_lines.append("")
|
||||||
entry_lines.append("**⚠ Conflicts detected:**")
|
entry_lines.append("**⚠ Conflicts detected:**")
|
||||||
|
|||||||
@@ -161,3 +161,67 @@ def test_wiki_check_index_and_log_ignored(
|
|||||||
assert result.exit_code == 0, (
|
assert result.exit_code == 0, (
|
||||||
f"index.md / log.md should be excluded. exit={result.exit_code}, out={result.output}"
|
f"index.md / log.md should be excluded. exit={result.exit_code}, out={result.output}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_wiki_check_exit_2_when_draft_not_empty(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
"""codex wiki check exits 2 when wiki/draft/ is non-empty (quarantined pages)."""
|
||||||
|
wiki = tmp_path / "wiki"
|
||||||
|
wiki.mkdir()
|
||||||
|
draft = wiki / "draft"
|
||||||
|
draft.mkdir()
|
||||||
|
|
||||||
|
(draft / "bad-concept.md").write_text(
|
||||||
|
"# Bad Concept\n\n⚠ Ungrounded hallucination. [FakeBib2025 #chunk 0]\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from codex.config import Settings
|
||||||
|
|
||||||
|
mock_settings = MagicMock(spec=Settings)
|
||||||
|
mock_settings.wiki_dir = str(wiki)
|
||||||
|
monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False)
|
||||||
|
|
||||||
|
runner = CliRunner()
|
||||||
|
result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)])
|
||||||
|
assert result.exit_code == 2, (
|
||||||
|
f"Expected exit 2 (quarantined pages), got {result.exit_code}. Output: {result.output}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_wiki_check_exit_2_takes_priority_over_exit_1(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
"""Exit 2 (quarantined) takes priority over exit 1 (ungrounded in wiki/)."""
|
||||||
|
wiki = tmp_path / "wiki"
|
||||||
|
wiki.mkdir()
|
||||||
|
draft = wiki / "draft"
|
||||||
|
draft.mkdir()
|
||||||
|
|
||||||
|
# Main wiki with ungrounded claim
|
||||||
|
(wiki / "concept-a.md").write_text(
|
||||||
|
"# Concept A\n\n⚠ Ungrounded claim. [SomeBib #chunk 0]\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
# Draft with quarantined page
|
||||||
|
(draft / "concept-b.md").write_text(
|
||||||
|
"# Concept B\n\nQuarantined content.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from codex.config import Settings
|
||||||
|
|
||||||
|
mock_settings = MagicMock(spec=Settings)
|
||||||
|
mock_settings.wiki_dir = str(wiki)
|
||||||
|
monkeypatch.setattr("codex.cli.get_settings", lambda: mock_settings, raising=False)
|
||||||
|
|
||||||
|
runner = CliRunner()
|
||||||
|
result = runner.invoke(app, ["wiki", "check", "--output-dir", str(wiki)])
|
||||||
|
assert result.exit_code == 2
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from codex.wiki import (
|
|||||||
Concept,
|
Concept,
|
||||||
ConceptPage,
|
ConceptPage,
|
||||||
_chunk_hash,
|
_chunk_hash,
|
||||||
|
_detect_conflicts,
|
||||||
_inject_cross_refs,
|
_inject_cross_refs,
|
||||||
_parse_claims,
|
_parse_claims,
|
||||||
_run_grounding_guard,
|
_run_grounding_guard,
|
||||||
@@ -278,6 +279,7 @@ def mock_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
|||||||
mock.wiki_llm_url = None
|
mock.wiki_llm_url = None
|
||||||
mock.ollama_base_url = "http://localhost:11434"
|
mock.ollama_base_url = "http://localhost:11434"
|
||||||
mock.wiki_top_k = 6
|
mock.wiki_top_k = 6
|
||||||
|
mock.wiki_min_grounding_rate = 0.5
|
||||||
|
|
||||||
monkeypatch.setattr("codex.wiki.get_settings", lambda: mock)
|
monkeypatch.setattr("codex.wiki.get_settings", lambda: mock)
|
||||||
monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False)
|
monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False)
|
||||||
@@ -568,3 +570,147 @@ def test_chunk_hash_differs_on_content_change() -> None:
|
|||||||
h1 = _chunk_hash(FIXTURE_CHUNKS)
|
h1 = _chunk_hash(FIXTURE_CHUNKS)
|
||||||
h2 = _chunk_hash(modified)
|
h2 = _chunk_hash(modified)
|
||||||
assert h1 != h2
|
assert h1 != h2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# LLM error handling (Fix 4)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_concept_llm_unavailable_returns_empty_page(
|
||||||
|
mock_retrieve: None,
|
||||||
|
mock_formulas: None,
|
||||||
|
mock_settings: Path,
|
||||||
|
) -> None:
|
||||||
|
"""When LLM raises an exception, compile_concept returns an empty ConceptPage (no crash)."""
|
||||||
|
|
||||||
|
class FailingLLM:
|
||||||
|
def generate(self, prompt: str, model: str) -> str:
|
||||||
|
raise ConnectionError("Ollama unavailable")
|
||||||
|
|
||||||
|
page = compile_concept(
|
||||||
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=FailingLLM(), wiki_dir=mock_settings
|
||||||
|
)
|
||||||
|
assert isinstance(page, ConceptPage)
|
||||||
|
assert page.markdown == ""
|
||||||
|
assert page.claims == []
|
||||||
|
assert page.chunk_hash != "" # hash still computed from chunks
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Quarantine (Fix 2)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_all_quarantines_low_grounding_page(
|
||||||
|
mock_retrieve: None,
|
||||||
|
mock_formulas: None,
|
||||||
|
mock_settings: Path,
|
||||||
|
) -> None:
|
||||||
|
"""A page with grounding rate < 0.5 goes to wiki/draft/ not wiki/."""
|
||||||
|
from codex.wiki import compile_all
|
||||||
|
|
||||||
|
(mock_settings / "concepts.yaml").write_text(
|
||||||
|
textwrap.dedent(
|
||||||
|
"""\
|
||||||
|
concepts:
|
||||||
|
- slug: lobachevsky-function
|
||||||
|
title: Lobachevsky Function
|
||||||
|
aliases: [Milnor Lobachevsky]
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ungrounded LLM output → all claims ungrounded → grounding rate 0.0 < 0.5
|
||||||
|
llm = MockLLM(UNGROUNDED_LLM_OUTPUT)
|
||||||
|
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
|
||||||
|
|
||||||
|
draft_dir = mock_settings / "draft"
|
||||||
|
assert draft_dir.exists(), "draft/ directory should have been created"
|
||||||
|
draft_pages = list(draft_dir.glob("*.md"))
|
||||||
|
assert len(draft_pages) >= 1, "Quarantined page should exist in wiki/draft/"
|
||||||
|
assert "lobachevsky-function" in report.quarantined
|
||||||
|
# Normal wiki/ path must NOT exist for this page
|
||||||
|
assert not (mock_settings / "lobachevsky-function.md").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_compile_all_does_not_quarantine_grounded_page(
|
||||||
|
mock_retrieve: None,
|
||||||
|
mock_formulas: None,
|
||||||
|
mock_settings: Path,
|
||||||
|
) -> None:
|
||||||
|
"""A page with grounding rate >= 0.5 is written to wiki/, not quarantined."""
|
||||||
|
from codex.wiki import compile_all
|
||||||
|
|
||||||
|
(mock_settings / "concepts.yaml").write_text(
|
||||||
|
textwrap.dedent(
|
||||||
|
"""\
|
||||||
|
concepts:
|
||||||
|
- slug: lobachevsky-function
|
||||||
|
title: Lobachevsky Function
|
||||||
|
aliases: [Milnor Lobachevsky]
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
||||||
|
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
|
||||||
|
|
||||||
|
assert "lobachevsky-function" not in report.quarantined
|
||||||
|
assert (mock_settings / "lobachevsky-function.md").exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Conflict detection (Fix 9)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_detect_conflicts_finds_adversative_keywords() -> None:
|
||||||
|
"""Chunks from different bibkeys both containing adversative keywords → conflict."""
|
||||||
|
chunks = [
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"paper_id": "p1",
|
||||||
|
"ord": 0,
|
||||||
|
"content": "The discrete map works well, but diverges in degenerate cases.",
|
||||||
|
"bibkey": "Smith2020",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"paper_id": "p2",
|
||||||
|
"ord": 0,
|
||||||
|
"content": "However, the approach from Jones is more stable.",
|
||||||
|
"bibkey": "Jones2019",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
conflicts = _detect_conflicts("test-concept", chunks)
|
||||||
|
assert len(conflicts) >= 1
|
||||||
|
slugs, bk1s, bk2s = zip(*conflicts, strict=True)
|
||||||
|
assert "test-concept" in slugs
|
||||||
|
|
||||||
|
|
||||||
|
def test_detect_conflicts_no_conflict_when_single_bibkey() -> None:
|
||||||
|
"""Only one bibkey → no conflict possible."""
|
||||||
|
conflicts = _detect_conflicts("test-concept", FIXTURE_CHUNKS)
|
||||||
|
assert conflicts == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# append_log quarantine rendering
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_append_records_quarantined(
|
||||||
|
mock_settings: Path,
|
||||||
|
) -> None:
|
||||||
|
"""Quarantined slugs appear in the log entry."""
|
||||||
|
report = CompileReport(
|
||||||
|
compiled=[],
|
||||||
|
quarantined=["some-concept"],
|
||||||
|
)
|
||||||
|
append_log(report, wiki_dir=mock_settings)
|
||||||
|
content = (mock_settings / "log.md").read_text(encoding="utf-8")
|
||||||
|
assert "some-concept" in content
|
||||||
|
assert "QUARANTINED" in content
|
||||||
|
|||||||
Reference in New Issue
Block a user