fix(wiki): quarantine zero-citation pages instead of publishing (audit C-4)

compile_all's quarantine gate was guarded by `total_claims > 0`, so a page
with no parseable citations — including the LLM-outage case where
compile_concept returns an empty page — fell through to the publish branch and
was written to wiki/ + marked compiled. Such a page carries zero grounding
evidence.

Now: total_claims == 0 (or grounding_rate below threshold) quarantines and does
NOT update the compile-state, so a transient LLM failure retries next run
instead of freezing an empty/uncited page as 'compiled'. Empty bodies are
recorded but not written as draft files.

Regression test reproduces the exact bypass (uncited LLM output -> quarantined,
not published).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-15 11:06:12 +02:00
parent 8af9f9ad0a
commit 115bb63f2d
2 changed files with 54 additions and 7 deletions

View File

@@ -722,17 +722,23 @@ def compile_all(
# Detect conflicts between chunks from different bibkeys # Detect conflicts between chunks from different bibkeys
report.conflicts.extend(_detect_conflicts(concept.slug, chunks)) report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
# Quarantine check: compute grounding rate # Quarantine check: compute grounding rate.
# A page with NO citations at all (total_claims == 0) carries zero grounding
# evidence — this also covers the LLM-outage case where compile_concept
# returns an empty page — and must NOT be published (audit C-4). Previously
# the `total_claims > 0` guard let such pages fall through to wiki/.
total_claims = len(page.claims) total_claims = len(page.claims)
grounded_claims = sum(1 for c in page.claims if c.grounded) grounded_claims = sum(1 for c in page.claims if c.grounded)
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0 grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate: if total_claims == 0 or grounding_rate < settings.wiki_min_grounding_rate:
# Quarantine: write to wiki/draft/ instead of wiki/ # Quarantine and do NOT update the compile-state, so a transient failure
draft_dir = wiki_dir / "draft" # (e.g. LLM outage) is retried on the next run instead of being frozen as
draft_dir.mkdir(parents=True, exist_ok=True) # "compiled". An empty body is not worth a draft file — just record it.
page_path = draft_dir / f"{concept.slug}.md" if page.markdown.strip():
page_path.write_text(page.markdown, encoding="utf-8") draft_dir = wiki_dir / "draft"
draft_dir.mkdir(parents=True, exist_ok=True)
(draft_dir / f"{concept.slug}.md").write_text(page.markdown, encoding="utf-8")
report.quarantined.append(concept.slug) report.quarantined.append(concept.slug)
else: else:
# Write page to wiki/ # Write page to wiki/

View File

@@ -504,6 +504,47 @@ def test_log_append_records_skipped(
assert "circle-packing" in content assert "circle-packing" in content
# ---------------------------------------------------------------------------
# Audit C-4: zero-citation pages must NOT bypass the quarantine
# ---------------------------------------------------------------------------
def test_zero_citation_page_is_quarantined_not_published(
mock_retrieve: None,
mock_formulas: None,
mock_settings: Path,
) -> None:
"""A confident-but-uncited LLM response yields zero parseable claims
(grounding_rate 0.0). It must be quarantined — not written to wiki/ and
marked compiled. Regression for audit C-4: the old ``total_claims > 0``
guard let zero-claim (and LLM-outage empty) pages fall through to wiki/.
"""
from codex.wiki import compile_all
(mock_settings / "concepts.yaml").write_text(
textwrap.dedent(
"""\
concepts:
- slug: lobachevsky-function
title: Lobachevsky Function
aliases: [Milnor Lobachevsky]
"""
),
encoding="utf-8",
)
# No [BibKey] citations anywhere → zero claims → grounding_rate 0.0.
uncited = MockLLM(
"The Lobachevsky function is obviously central to all of geometry. "
"Every result follows from it. This paragraph cites nothing at all."
)
report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings))
assert "lobachevsky-function" in report.quarantined
assert "lobachevsky-function" not in report.compiled
assert not (mock_settings / "lobachevsky-function.md").exists()
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Idempotency: second compile without chunk change → no rewrite # Idempotency: second compile without chunk change → no rewrite
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------