f422b0fb84
- C-6: the content-5-gram check used 'gram in src' on space-joined strings, so the
first/last gram token could match a prefix/suffix of a longer chunk word
('set' inside 'subset'). Both gram and chunk are now space-padded, so a match
requires whole-token alignment. Regression test added.
- R-2: the reference-list filter's author pattern ('Surname, I.') also matched
'Theorem A.'/'Lemma B.', so theorem-dense chunks could be dropped as a
bibliography. The pattern now excludes common math-structure words.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
851 lines
29 KiB
Python
851 lines
29 KiB
Python
"""Tests for compile_concept, compile_all, write_index, append_log.
|
|
|
|
DB and LLM are fully mocked — no network or database access required.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import textwrap
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from codex.wiki import (
|
|
Claim,
|
|
CompileReport,
|
|
Concept,
|
|
ConceptPage,
|
|
_chunk_hash,
|
|
_detect_conflicts,
|
|
_inject_cross_refs,
|
|
_parse_claims,
|
|
_run_grounding_guard,
|
|
append_log,
|
|
compile_concept,
|
|
write_index,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
FIXTURE_CHUNKS: list[dict[str, Any]] = [
|
|
{
|
|
"id": 1,
|
|
"paper_id": "springborn-2008",
|
|
"ord": 16,
|
|
"content": (
|
|
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
|
|
"V = L(gamma1) + L(gamma2) + L(gamma3). "
|
|
"The volume function V0 is strictly concave on the angle domain."
|
|
),
|
|
"bibkey": "Springborn2008",
|
|
"dist": 0.31,
|
|
},
|
|
{
|
|
"id": 2,
|
|
"paper_id": "springborn-2008",
|
|
"ord": 0,
|
|
"content": (
|
|
"A discrete conformal map is defined by logarithmic scale factors u_i "
|
|
"at each vertex such that the edge lengths satisfy a compatibility condition."
|
|
),
|
|
"bibkey": "Springborn2008",
|
|
"dist": 0.35,
|
|
},
|
|
]
|
|
|
|
FIXTURE_CONCEPT = Concept(
|
|
slug="lobachevsky-function",
|
|
title="Lobachevsky Function",
|
|
aliases=["Milnor Lobachevsky", "Clausen function"],
|
|
emphasis="Verwendung in hyperbolischen Volumenformeln (Springborn 2008).",
|
|
)
|
|
|
|
# Grounded LLM output: claim text is a substring of the fixture chunk
|
|
_GROUNDED_CLAIM = (
|
|
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is"
|
|
" V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]"
|
|
)
|
|
GROUNDED_LLM_OUTPUT = (
|
|
"The Lobachevsky function L appears as the building block of hyperbolic volume.\n"
|
|
+ _GROUNDED_CLAIM
|
|
+ "\nThe volume function V0 is strictly concave on the angle domain."
|
|
" [Springborn2008 #chunk 16]\n"
|
|
)
|
|
|
|
# Ungrounded LLM output: claim text not present in any chunk
|
|
UNGROUNDED_LLM_OUTPUT = textwrap.dedent(
|
|
"""\
|
|
The closed form is L(x) = -integral from 0 to x of log|2 sin t| dt. [Springborn2008 #chunk 99]
|
|
"""
|
|
)
|
|
|
|
|
|
class MockLLM:
|
|
"""Deterministic mock LLM that returns a preset response."""
|
|
|
|
def __init__(self, response: str) -> None:
|
|
self._response = response
|
|
|
|
def generate(self, prompt: str, model: str) -> str:
|
|
return self._response
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _parse_claims
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_parse_claims_finds_citation() -> None:
|
|
"""Claims with [BibKey #locator] format are parsed correctly."""
|
|
md = "Some result. [Springborn2008 #chunk 16]"
|
|
claims = _parse_claims(md)
|
|
assert len(claims) == 1
|
|
assert claims[0].bibkey == "Springborn2008"
|
|
assert claims[0].locator == "chunk 16"
|
|
|
|
|
|
def test_parse_claims_no_citations() -> None:
|
|
"""Text without citation markers returns empty list."""
|
|
md = "A standalone sentence with no citation."
|
|
claims = _parse_claims(md)
|
|
assert claims == []
|
|
|
|
|
|
def test_parse_claims_multiple() -> None:
|
|
"""Multiple citations in same text are all parsed."""
|
|
md = "First claim. [Smith2020 #page 5]\nSecond claim. [Jones2019 #eq 3]\n"
|
|
claims = _parse_claims(md)
|
|
assert len(claims) == 2
|
|
|
|
|
|
def test_parse_claims_ignores_markdown_links() -> None:
|
|
"""Markdown hyperlinks like [text](https://example.com) are NOT parsed as claims."""
|
|
md = "See the [related work](https://example.com) for details."
|
|
claims = _parse_claims(md)
|
|
assert claims == []
|
|
|
|
|
|
def test_parse_claims_ignores_multi_word_bibkey() -> None:
|
|
"""Multi-word 'bibkeys' (spaces inside) are not treated as real citations."""
|
|
md = "Some text [not a bibkey here] and more."
|
|
claims = _parse_claims(md)
|
|
assert claims == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _run_grounding_guard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_grounding_guard_marks_grounded_claim() -> None:
|
|
"""A claim whose text appears in the cited chunk is marked grounded."""
|
|
claim = Claim(
|
|
text="the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)",
|
|
bibkey="Springborn2008",
|
|
locator="chunk 16",
|
|
)
|
|
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
|
assert result[0].grounded is True
|
|
|
|
|
|
def test_grounding_guard_marks_ungrounded_claim() -> None:
|
|
"""A claim not present in the cited chunk is marked ungrounded."""
|
|
claim = Claim(
|
|
text="integral from 0 to x of log 2 sin t dt closed form definition",
|
|
bibkey="Springborn2008",
|
|
locator="chunk 99",
|
|
)
|
|
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
|
assert result[0].grounded is False
|
|
|
|
|
|
def test_grounding_guard_ungrounded_when_bibkey_missing() -> None:
|
|
"""A claim citing a bibkey not in any chunk is ungrounded."""
|
|
claim = Claim(text="some statement", bibkey="UnknownBib2000", locator="page 1")
|
|
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
|
assert result[0].grounded is False
|
|
|
|
|
|
def test_grounding_guard_adversarial_stopword_bypass() -> None:
|
|
"""Adversarial: stopword-3-gram matches are NOT sufficient for grounding.
|
|
|
|
'the discrete conformal map' appears in the chunk, but this is a stopword
|
|
phrase. The preceding hallucinated sentences contain NO content-5-gram
|
|
present in the chunk → grounded=False.
|
|
"""
|
|
# The adversarial text from the session-prompt review finding:
|
|
adversarial_text = (
|
|
"The Riemann hypothesis is proven by combining Hodge theory with quantum entanglement. "
|
|
"Furthermore P=NP holds for hyperbolic tetrahedra. The discrete conformal map"
|
|
)
|
|
claim = Claim(
|
|
text=adversarial_text,
|
|
bibkey="Springborn2008",
|
|
locator="chunk 0",
|
|
)
|
|
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
|
assert result[0].grounded is False, (
|
|
"Stopword 3-gram 'the discrete conformal map' must NOT ground a hallucinated claim"
|
|
)
|
|
|
|
|
|
def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored() -> None:
|
|
"""Only the last sentence is checked for grounding.
|
|
|
|
If the last sentence IS grounded, the earlier hallucinated sentence is
|
|
irrelevant (it has no citation → it won't be parsed as a Claim at all).
|
|
This test verifies that a claim whose text = the last sentence grounds correctly.
|
|
"""
|
|
# The last sentence of the claim text is directly grounded in the fixture chunk.
|
|
grounded_sentence = "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)"
|
|
claim = Claim(
|
|
text=grounded_sentence,
|
|
bibkey="Springborn2008",
|
|
locator="chunk 16",
|
|
)
|
|
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
|
assert result[0].grounded is True, (
|
|
"Last sentence with content-5-gram present in chunk must be grounded"
|
|
)
|
|
|
|
|
|
def test_grounding_guard_robust_to_trailing_comma() -> None:
|
|
"""A faithful 5-word claim grounds despite a trailing comma (audit R-1).
|
|
|
|
The chunk says "... the volume function V0 is strictly concave on the angle
|
|
domain." Before edge-punctuation stripping, the claim token "concave," != the
|
|
chunk token "concave", so the only content-5-gram failed to match and the
|
|
claim was wrongly marked ungrounded.
|
|
"""
|
|
claim = Claim(
|
|
text="the volume function V0 is strictly concave,",
|
|
bibkey="Springborn2008",
|
|
locator="chunk 16",
|
|
)
|
|
result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
|
|
assert result[0].grounded is True
|
|
|
|
|
|
def test_grounding_guard_no_substring_bleed_across_tokens() -> None:
|
|
"""A 5-gram must match whole tokens, not bleed into a longer chunk word (audit C-6).
|
|
|
|
The chunk contains 'subset'; a claim whose first token is 'set' must NOT ground
|
|
by substring-matching the suffix of 'subset'.
|
|
"""
|
|
chunks = [
|
|
{
|
|
"id": 99,
|
|
"paper_id": "p",
|
|
"ord": 0,
|
|
"bibkey": "TestBib",
|
|
"content": "the subset conformal map energy functional is minimized here",
|
|
}
|
|
]
|
|
claim = Claim(
|
|
text="the set conformal map energy functional",
|
|
bibkey="TestBib",
|
|
locator="chunk 0",
|
|
)
|
|
result = _run_grounding_guard([claim], chunks)
|
|
assert result[0].grounded is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _inject_cross_refs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_inject_cross_refs_replaces_title() -> None:
|
|
"""Occurrences of another concept's title are replaced with [[slug]]."""
|
|
concepts = [
|
|
Concept(
|
|
slug="discrete-conformal-map",
|
|
title="Discrete Conformal Map",
|
|
aliases=[],
|
|
),
|
|
FIXTURE_CONCEPT,
|
|
]
|
|
text = "This is related to Discrete Conformal Map theory."
|
|
result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
|
|
assert "[[discrete-conformal-map]]" in result
|
|
|
|
|
|
def test_inject_cross_refs_skips_current_slug() -> None:
|
|
"""The current concept's own title is not replaced."""
|
|
concepts = [FIXTURE_CONCEPT]
|
|
text = "The Lobachevsky Function is important."
|
|
result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
|
|
assert "[[lobachevsky-function]]" not in result
|
|
|
|
|
|
def test_inject_cross_refs_replaces_alias() -> None:
|
|
"""Aliases are also replaced with [[slug]]."""
|
|
concepts = [
|
|
Concept(
|
|
slug="circle-packing",
|
|
title="Circle Packing",
|
|
aliases=["Koebe-Andreev-Thurston", "circle pattern"],
|
|
),
|
|
FIXTURE_CONCEPT,
|
|
]
|
|
text = "The Koebe-Andreev-Thurston theorem gives a packing."
|
|
result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
|
|
assert "[[circle-packing]]" in result
|
|
|
|
|
|
def test_inject_cross_refs_skips_inline_code() -> None:
|
|
"""Concept titles inside backtick inline-code spans are NOT replaced."""
|
|
concepts = [
|
|
Concept(
|
|
slug="circle-packing",
|
|
title="circle-packing",
|
|
aliases=[],
|
|
),
|
|
FIXTURE_CONCEPT,
|
|
]
|
|
# The term "circle-packing" appears both bare (should be replaced)
|
|
# and inside backticks (should NOT be replaced).
|
|
text = "Use `circle-packing` for this. See also circle-packing theory."
|
|
result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
|
|
assert "`circle-packing`" in result # inline-code preserved
|
|
assert "[[circle-packing]]" in result # bare occurrence replaced
|
|
|
|
|
|
def test_inject_cross_refs_skips_math_spans() -> None:
|
|
"""Concept names inside LaTeX math are NOT rewritten to [[slug]] links (audit R-9)."""
|
|
concepts = [
|
|
Concept(slug="energy", title="Energy", aliases=[]),
|
|
FIXTURE_CONCEPT,
|
|
]
|
|
text = "Prose Energy here. Math $x = Energy^2$ and inline \\(Energy\\) stay literal."
|
|
result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
|
|
assert "$x = Energy^2$" in result # display/inline $...$ math untouched
|
|
assert "\\(Energy\\)" in result # \(...\) math untouched
|
|
assert "Prose [[energy]] here." in result # prose occurrence still linked
|
|
|
|
|
|
def test_inject_cross_refs_full_list_even_when_single_concept() -> None:
|
|
"""Cross-ref injection uses the full concept list, not just the compiled concept."""
|
|
concepts = [
|
|
Concept(slug="circle-packing", title="Circle Packing", aliases=[]),
|
|
Concept(slug="discrete-conformal-map", title="Discrete Conformal Map", aliases=[]),
|
|
FIXTURE_CONCEPT,
|
|
]
|
|
# Compiling lobachevsky-function only; text mentions the other two concepts
|
|
text = "Discrete Conformal Map is used with Circle Packing."
|
|
result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
|
|
assert "[[circle-packing]]" in result
|
|
assert "[[discrete-conformal-map]]" in result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# compile_concept (full mock)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_retrieve(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Patch _retrieve_chunks to return deterministic fixture chunks."""
|
|
monkeypatch.setattr(
|
|
"codex.wiki._retrieve_chunks",
|
|
lambda queries, top_k: FIXTURE_CHUNKS,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_formulas(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Patch _try_embed_formulas to be a no-op (F-09 not present)."""
|
|
monkeypatch.setattr("codex.wiki._try_embed_formulas", lambda concept, chunks: None)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
|
"""Patch get_settings to return a settings pointing at tmp_path."""
|
|
from codex.config import Settings
|
|
|
|
wiki_path = tmp_path / "wiki"
|
|
wiki_path.mkdir()
|
|
|
|
mock = MagicMock(spec=Settings)
|
|
mock.wiki_dir = str(wiki_path)
|
|
mock.wiki_llm_model = "test-model"
|
|
mock.wiki_llm_url = None
|
|
mock.ollama_base_url = "http://localhost:11434"
|
|
mock.wiki_top_k = 6
|
|
mock.wiki_min_grounding_rate = 0.5
|
|
|
|
monkeypatch.setattr("codex.wiki.get_settings", lambda: mock)
|
|
monkeypatch.setattr("codex.cli.get_settings", lambda: mock, raising=False)
|
|
return wiki_path
|
|
|
|
|
|
def test_compile_concept_returns_concept_page(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""compile_concept returns a ConceptPage with non-empty markdown."""
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
page = compile_concept(
|
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings
|
|
)
|
|
assert isinstance(page, ConceptPage)
|
|
assert page.concept.slug == "lobachevsky-function"
|
|
assert len(page.markdown) > 0
|
|
|
|
|
|
def test_compile_concept_has_chunk_hash(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""Compiled page has a non-empty chunk_hash."""
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
page = compile_concept(
|
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings
|
|
)
|
|
assert len(page.chunk_hash) == 64 # SHA-256 hex
|
|
|
|
|
|
def test_compile_concept_grounded_claim_not_flagged(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""A grounded claim does NOT receive the ⚠ marker in the output markdown."""
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
page = compile_concept(
|
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings
|
|
)
|
|
# When all claims are grounded, no ⚠ in markdown
|
|
# (may still have 0 ⚠ if claims found; just check no false positive for grounded)
|
|
grounded = [c for c in page.claims if c.grounded]
|
|
for claim in grounded:
|
|
assert claim.grounded is True
|
|
|
|
|
|
def test_compile_concept_ungrounded_claim_marked(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""An ungrounded claim is marked ⚠ in the page markdown."""
|
|
llm = MockLLM(UNGROUNDED_LLM_OUTPUT)
|
|
page = compile_concept(
|
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings
|
|
)
|
|
ungrounded = [c for c in page.claims if not c.grounded]
|
|
if ungrounded:
|
|
assert "⚠" in page.markdown
|
|
|
|
|
|
def test_compile_concept_header_present(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""Page markdown starts with an H1 header for the concept title."""
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
page = compile_concept(
|
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings
|
|
)
|
|
assert page.markdown.startswith("# Lobachevsky Function")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# write_index
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_index_creates_index_md(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""write_index creates wiki/index.md."""
|
|
pages = [
|
|
("discrete-conformal-map", "Discrete Conformal Map"),
|
|
("circle-packing", "Circle Packing"),
|
|
]
|
|
write_index(pages, wiki_dir=mock_settings)
|
|
index_path = mock_settings / "index.md"
|
|
assert index_path.exists()
|
|
|
|
|
|
def test_index_contains_links(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""wiki/index.md contains [[slug]] links for each page."""
|
|
pages = [
|
|
("discrete-conformal-map", "Discrete Conformal Map"),
|
|
("circle-packing", "Circle Packing"),
|
|
]
|
|
write_index(pages, wiki_dir=mock_settings)
|
|
content = (mock_settings / "index.md").read_text(encoding="utf-8")
|
|
assert "[[discrete-conformal-map]]" in content
|
|
assert "[[circle-packing]]" in content
|
|
|
|
|
|
def test_index_contains_header(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""wiki/index.md starts with a # Wiki Index header."""
|
|
write_index([], wiki_dir=mock_settings)
|
|
content = (mock_settings / "index.md").read_text(encoding="utf-8")
|
|
assert content.startswith("# Wiki Index")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# append_log
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_log_append_creates_log_md(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""append_log creates wiki/log.md if it does not exist."""
|
|
report = CompileReport(
|
|
compiled=["lobachevsky-function"],
|
|
skipped=[],
|
|
ungrounded=[],
|
|
)
|
|
append_log(report, wiki_dir=mock_settings)
|
|
log_path = mock_settings / "log.md"
|
|
assert log_path.exists()
|
|
|
|
|
|
def test_log_append_does_not_overwrite(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""Two calls to append_log accumulate entries — older entry is NOT overwritten."""
|
|
report1 = CompileReport(compiled=["first-concept"])
|
|
report2 = CompileReport(compiled=["second-concept"])
|
|
|
|
append_log(report1, wiki_dir=mock_settings)
|
|
append_log(report2, wiki_dir=mock_settings)
|
|
|
|
content = (mock_settings / "log.md").read_text(encoding="utf-8")
|
|
assert "first-concept" in content
|
|
assert "second-concept" in content
|
|
|
|
|
|
def test_log_append_records_ungrounded(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""Ungrounded claims appear in the log entry."""
|
|
report = CompileReport(
|
|
compiled=["lobachevsky-function"],
|
|
ungrounded=[("lobachevsky-function", "some ungrounded claim text")],
|
|
)
|
|
append_log(report, wiki_dir=mock_settings)
|
|
content = (mock_settings / "log.md").read_text(encoding="utf-8")
|
|
assert "some ungrounded claim text" in content
|
|
assert "⚠" in content
|
|
|
|
|
|
def test_log_append_records_skipped(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""Skipped slugs appear in the log entry."""
|
|
report = CompileReport(
|
|
compiled=[],
|
|
skipped=["circle-packing"],
|
|
)
|
|
append_log(report, wiki_dir=mock_settings)
|
|
content = (mock_settings / "log.md").read_text(encoding="utf-8")
|
|
assert "circle-packing" in content
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audit C-4: zero-citation pages must NOT bypass the quarantine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_zero_citation_page_is_quarantined_not_published(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""A confident-but-uncited LLM response yields zero parseable claims
|
|
(grounding_rate 0.0). It must be quarantined — not written to wiki/ and
|
|
marked compiled. Regression for audit C-4: the old ``total_claims > 0``
|
|
guard let zero-claim (and LLM-outage empty) pages fall through to wiki/.
|
|
"""
|
|
from codex.wiki import compile_all
|
|
|
|
(mock_settings / "concepts.yaml").write_text(
|
|
textwrap.dedent(
|
|
"""\
|
|
concepts:
|
|
- slug: lobachevsky-function
|
|
title: Lobachevsky Function
|
|
aliases: [Milnor Lobachevsky]
|
|
"""
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
# No [BibKey] citations anywhere → zero claims → grounding_rate 0.0.
|
|
uncited = MockLLM(
|
|
"The Lobachevsky function is obviously central to all of geometry. "
|
|
"Every result follows from it. This paragraph cites nothing at all."
|
|
)
|
|
|
|
report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings))
|
|
|
|
assert "lobachevsky-function" in report.quarantined
|
|
assert "lobachevsky-function" not in report.compiled
|
|
assert not (mock_settings / "lobachevsky-function.md").exists()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Idempotency: second compile without chunk change → no rewrite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_idempotent_no_rewrite_when_unchanged(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Second compile_all with unchanged chunks skips the concept (changed_only=True)."""
|
|
from codex.wiki import compile_all
|
|
|
|
call_count = 0
|
|
original_compile = compile_concept
|
|
|
|
def counting_compile(concept: Concept, chunks: list[Any], **kwargs: Any) -> ConceptPage:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return original_compile(concept, chunks, **kwargs)
|
|
|
|
monkeypatch.setattr("codex.wiki.compile_concept", counting_compile)
|
|
|
|
# Write concepts.yaml into wiki_dir
|
|
yaml_content = textwrap.dedent(
|
|
"""\
|
|
concepts:
|
|
- slug: lobachevsky-function
|
|
title: Lobachevsky Function
|
|
aliases: [Milnor Lobachevsky]
|
|
"""
|
|
)
|
|
(mock_settings / "concepts.yaml").write_text(yaml_content, encoding="utf-8")
|
|
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
|
|
# First compile: should call compile_concept
|
|
compile_all(changed_only=True, llm=llm, output_dir=str(mock_settings))
|
|
first_count = call_count
|
|
|
|
# Second compile with same chunks: should skip
|
|
compile_all(changed_only=True, llm=llm, output_dir=str(mock_settings))
|
|
second_count = call_count
|
|
|
|
assert first_count >= 1, "First run should compile at least once"
|
|
assert second_count == first_count, "Second run should not recompile (unchanged chunks)"
|
|
|
|
|
|
def test_force_all_recompiles_even_when_unchanged(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""compile_all with changed_only=False recompiles regardless of hash."""
|
|
from codex.wiki import compile_all
|
|
|
|
call_count = 0
|
|
original_compile = compile_concept
|
|
|
|
def counting_compile(concept: Concept, chunks: list[Any], **kwargs: Any) -> ConceptPage:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return original_compile(concept, chunks, **kwargs)
|
|
|
|
monkeypatch.setattr("codex.wiki.compile_concept", counting_compile)
|
|
|
|
yaml_content = textwrap.dedent(
|
|
"""\
|
|
concepts:
|
|
- slug: lobachevsky-function
|
|
title: Lobachevsky Function
|
|
aliases: [Milnor Lobachevsky]
|
|
"""
|
|
)
|
|
(mock_settings / "concepts.yaml").write_text(yaml_content, encoding="utf-8")
|
|
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
|
|
compile_all(changed_only=True, llm=llm, output_dir=str(mock_settings))
|
|
compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
|
|
|
|
assert call_count >= 2, "Force --all should recompile even when unchanged"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _chunk_hash
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_chunk_hash_is_stable() -> None:
|
|
"""Same chunks produce the same hash each time."""
|
|
h1 = _chunk_hash(FIXTURE_CHUNKS)
|
|
h2 = _chunk_hash(FIXTURE_CHUNKS)
|
|
assert h1 == h2
|
|
assert len(h1) == 64
|
|
|
|
|
|
def test_chunk_hash_differs_on_content_change() -> None:
|
|
"""Different content produces different hash."""
|
|
modified = [dict(FIXTURE_CHUNKS[0], content="completely different content"), FIXTURE_CHUNKS[1]]
|
|
h1 = _chunk_hash(FIXTURE_CHUNKS)
|
|
h2 = _chunk_hash(modified)
|
|
assert h1 != h2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LLM error handling (Fix 4)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_compile_concept_llm_unavailable_returns_empty_page(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""When LLM raises an exception, compile_concept returns an empty ConceptPage (no crash)."""
|
|
|
|
class FailingLLM:
|
|
def generate(self, prompt: str, model: str) -> str:
|
|
raise ConnectionError("Ollama unavailable")
|
|
|
|
page = compile_concept(
|
|
FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=FailingLLM(), wiki_dir=mock_settings
|
|
)
|
|
assert isinstance(page, ConceptPage)
|
|
assert page.markdown == ""
|
|
assert page.claims == []
|
|
assert page.chunk_hash != "" # hash still computed from chunks
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Quarantine (Fix 2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_compile_all_quarantines_low_grounding_page(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""A page with grounding rate < 0.5 goes to wiki/draft/ not wiki/."""
|
|
from codex.wiki import compile_all
|
|
|
|
(mock_settings / "concepts.yaml").write_text(
|
|
textwrap.dedent(
|
|
"""\
|
|
concepts:
|
|
- slug: lobachevsky-function
|
|
title: Lobachevsky Function
|
|
aliases: [Milnor Lobachevsky]
|
|
"""
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
# Ungrounded LLM output → all claims ungrounded → grounding rate 0.0 < 0.5
|
|
llm = MockLLM(UNGROUNDED_LLM_OUTPUT)
|
|
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
|
|
|
|
draft_dir = mock_settings / "draft"
|
|
assert draft_dir.exists(), "draft/ directory should have been created"
|
|
draft_pages = list(draft_dir.glob("*.md"))
|
|
assert len(draft_pages) >= 1, "Quarantined page should exist in wiki/draft/"
|
|
assert "lobachevsky-function" in report.quarantined
|
|
# Normal wiki/ path must NOT exist for this page
|
|
assert not (mock_settings / "lobachevsky-function.md").exists()
|
|
|
|
|
|
def test_compile_all_does_not_quarantine_grounded_page(
|
|
mock_retrieve: None,
|
|
mock_formulas: None,
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""A page with grounding rate >= 0.5 is written to wiki/, not quarantined."""
|
|
from codex.wiki import compile_all
|
|
|
|
(mock_settings / "concepts.yaml").write_text(
|
|
textwrap.dedent(
|
|
"""\
|
|
concepts:
|
|
- slug: lobachevsky-function
|
|
title: Lobachevsky Function
|
|
aliases: [Milnor Lobachevsky]
|
|
"""
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
llm = MockLLM(GROUNDED_LLM_OUTPUT)
|
|
report = compile_all(changed_only=False, llm=llm, output_dir=str(mock_settings))
|
|
|
|
assert "lobachevsky-function" not in report.quarantined
|
|
assert (mock_settings / "lobachevsky-function.md").exists()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Conflict detection (Fix 9)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_detect_conflicts_finds_adversative_keywords() -> None:
|
|
"""Chunks from different bibkeys both containing adversative keywords → conflict."""
|
|
chunks = [
|
|
{
|
|
"id": 10,
|
|
"paper_id": "p1",
|
|
"ord": 0,
|
|
"content": "The discrete map works well, but diverges in degenerate cases.",
|
|
"bibkey": "Smith2020",
|
|
},
|
|
{
|
|
"id": 11,
|
|
"paper_id": "p2",
|
|
"ord": 0,
|
|
"content": "However, the approach from Jones is more stable.",
|
|
"bibkey": "Jones2019",
|
|
},
|
|
]
|
|
conflicts = _detect_conflicts("test-concept", chunks)
|
|
assert len(conflicts) >= 1
|
|
slugs, bk1s, bk2s = zip(*conflicts, strict=True)
|
|
assert "test-concept" in slugs
|
|
|
|
|
|
def test_detect_conflicts_no_conflict_when_single_bibkey() -> None:
|
|
"""Only one bibkey → no conflict possible."""
|
|
conflicts = _detect_conflicts("test-concept", FIXTURE_CHUNKS)
|
|
assert conflicts == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# append_log quarantine rendering
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_log_append_records_quarantined(
|
|
mock_settings: Path,
|
|
) -> None:
|
|
"""Quarantined slugs appear in the log entry."""
|
|
report = CompileReport(
|
|
compiled=[],
|
|
quarantined=["some-concept"],
|
|
)
|
|
append_log(report, wiki_dir=mock_settings)
|
|
content = (mock_settings / "log.md").read_text(encoding="utf-8")
|
|
assert "some-concept" in content
|
|
assert "QUARANTINED" in content
|