fix: audit remediation Wave 1 — correctness fixes C-1, C-4, C-11 (+ doc fixes) #12

Merged
user2595 merged 4 commits from fix/audit-wave-1 into main 2026-06-16 04:53:20 +00:00
2 changed files with 54 additions and 7 deletions
Showing only changes of commit 115bb63f2d - Show all commits

View File

@@ -722,17 +722,23 @@ def compile_all(
# Detect conflicts between chunks from different bibkeys
report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
# Quarantine check: compute grounding rate
# Quarantine check: compute grounding rate.
# A page with NO citations at all (total_claims == 0) carries zero grounding
# evidence — this also covers the LLM-outage case where compile_concept
# returns an empty page — and must NOT be published (audit C-4). Previously
# the `total_claims > 0` guard let such pages fall through to wiki/.
total_claims = len(page.claims)
grounded_claims = sum(1 for c in page.claims if c.grounded)
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate:
# Quarantine: write to wiki/draft/ instead of wiki/
draft_dir = wiki_dir / "draft"
draft_dir.mkdir(parents=True, exist_ok=True)
page_path = draft_dir / f"{concept.slug}.md"
page_path.write_text(page.markdown, encoding="utf-8")
if total_claims == 0 or grounding_rate < settings.wiki_min_grounding_rate:
# Quarantine and do NOT update the compile-state, so a transient failure
# (e.g. LLM outage) is retried on the next run instead of being frozen as
# "compiled". An empty body is not worth a draft file — just record it.
if page.markdown.strip():
draft_dir = wiki_dir / "draft"
draft_dir.mkdir(parents=True, exist_ok=True)
(draft_dir / f"{concept.slug}.md").write_text(page.markdown, encoding="utf-8")
report.quarantined.append(concept.slug)
else:
# Write page to wiki/

View File

@@ -504,6 +504,47 @@ def test_log_append_records_skipped(
assert "circle-packing" in content
# ---------------------------------------------------------------------------
# Audit C-4: zero-citation pages must NOT bypass the quarantine
# ---------------------------------------------------------------------------
def test_zero_citation_page_is_quarantined_not_published(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""A confident-but-uncited LLM response yields zero parseable claims
(grounding_rate 0.0). It must be quarantined — not written to wiki/ and
marked compiled. Regression for audit C-4: the old ``total_claims > 0``
guard let zero-claim (and LLM-outage empty) pages fall through to wiki/.
"""
from codex.wiki import compile_all
(mock_settings / "concepts.yaml").write_text(
textwrap.dedent(
"""\
concepts:
- slug: lobachevsky-function
title: Lobachevsky Function
aliases: [Milnor Lobachevsky]
"""
),
encoding="utf-8",
)
# No [BibKey] citations anywhere → zero claims → grounding_rate 0.0.
uncited = MockLLM(
"The Lobachevsky function is obviously central to all of geometry. "
"Every result follows from it. This paragraph cites nothing at all."
)
report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings))
assert "lobachevsky-function" in report.quarantined
assert "lobachevsky-function" not in report.compiled
assert not (mock_settings / "lobachevsky-function.md").exists()
# ---------------------------------------------------------------------------
# Idempotency: second compile without chunk change → no rewrite
# ---------------------------------------------------------------------------