From 115bb63f2d865f64cfb26a7664b8084a8e3a904e Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 15 Jun 2026 11:06:12 +0200 Subject: [PATCH] fix(wiki): quarantine zero-citation pages instead of publishing (audit C-4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit compile_all's quarantine gate was guarded by `total_claims > 0`, so a page with no parseable citations — including the LLM-outage case where compile_concept returns an empty page — fell through to the publish branch and was written to wiki/ + marked compiled. Such a page carries zero grounding evidence. Now: total_claims == 0 (or grounding_rate below threshold) quarantines and does NOT update the compile-state, so a transient LLM failure retries next run instead of freezing an empty/uncited page as 'compiled'. Empty bodies are recorded but not written as draft files. Regression test reproduces the exact bypass (uncited LLM output -> quarantined, not published). Co-Authored-By: Claude Fable 5 --- codex/wiki.py | 20 ++++++++++++------- tests/wiki/test_compile.py | 41 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/codex/wiki.py b/codex/wiki.py index b47a98a..b913258 100644 --- a/codex/wiki.py +++ b/codex/wiki.py @@ -722,17 +722,23 @@ def compile_all( # Detect conflicts between chunks from different bibkeys report.conflicts.extend(_detect_conflicts(concept.slug, chunks)) - # Quarantine check: compute grounding rate + # Quarantine check: compute grounding rate. + # A page with NO citations at all (total_claims == 0) carries zero grounding + # evidence — this also covers the LLM-outage case where compile_concept + # returns an empty page — and must NOT be published (audit C-4). Previously + # the `total_claims > 0` guard let such pages fall through to wiki/. total_claims = len(page.claims) grounded_claims = sum(1 for c in page.claims if c.grounded) grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0 - if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate: - # Quarantine: write to wiki/draft/ instead of wiki/ - draft_dir = wiki_dir / "draft" - draft_dir.mkdir(parents=True, exist_ok=True) - page_path = draft_dir / f"{concept.slug}.md" - page_path.write_text(page.markdown, encoding="utf-8") + if total_claims == 0 or grounding_rate < settings.wiki_min_grounding_rate: + # Quarantine and do NOT update the compile-state, so a transient failure + # (e.g. LLM outage) is retried on the next run instead of being frozen as + # "compiled". An empty body is not worth a draft file — just record it. + if page.markdown.strip(): + draft_dir = wiki_dir / "draft" + draft_dir.mkdir(parents=True, exist_ok=True) + (draft_dir / f"{concept.slug}.md").write_text(page.markdown, encoding="utf-8") report.quarantined.append(concept.slug) else: # Write page to wiki/ diff --git a/tests/wiki/test_compile.py b/tests/wiki/test_compile.py index 53ff8fc..6835ec0 100644 --- a/tests/wiki/test_compile.py +++ b/tests/wiki/test_compile.py @@ -504,6 +504,47 @@ def test_log_append_records_skipped( assert "circle-packing" in content +# --------------------------------------------------------------------------- +# Audit C-4: zero-citation pages must NOT bypass the quarantine +# --------------------------------------------------------------------------- + + +def test_zero_citation_page_is_quarantined_not_published( + mock_retrieve: None, + mock_formulas: None, + mock_settings: Path, +) -> None: + """A confident-but-uncited LLM response yields zero parseable claims + (grounding_rate 0.0). It must be quarantined — not written to wiki/ and + marked compiled. Regression for audit C-4: the old ``total_claims > 0`` + guard let zero-claim (and LLM-outage empty) pages fall through to wiki/. + """ + from codex.wiki import compile_all + + (mock_settings / "concepts.yaml").write_text( + textwrap.dedent( + """\ + concepts: + - slug: lobachevsky-function + title: Lobachevsky Function + aliases: [Milnor Lobachevsky] + """ + ), + encoding="utf-8", + ) + # No [BibKey] citations anywhere → zero claims → grounding_rate 0.0. + uncited = MockLLM( + "The Lobachevsky function is obviously central to all of geometry. " + "Every result follows from it. This paragraph cites nothing at all." + ) + + report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings)) + + assert "lobachevsky-function" in report.quarantined + assert "lobachevsky-function" not in report.compiled + assert not (mock_settings / "lobachevsky-function.md").exists() + + # --------------------------------------------------------------------------- # Idempotency: second compile without chunk change → no rewrite # ---------------------------------------------------------------------------