fix: audit remediation Wave 1 — correctness fixes C-1, C-4, C-11 (+ doc fixes) #12
@@ -722,17 +722,23 @@ def compile_all(
|
|||||||
# Detect conflicts between chunks from different bibkeys
|
# Detect conflicts between chunks from different bibkeys
|
||||||
report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
|
report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
|
||||||
|
|
||||||
# Quarantine check: compute grounding rate
|
# Quarantine check: compute grounding rate.
|
||||||
|
# A page with NO citations at all (total_claims == 0) carries zero grounding
|
||||||
|
# evidence — this also covers the LLM-outage case where compile_concept
|
||||||
|
# returns an empty page — and must NOT be published (audit C-4). Previously
|
||||||
|
# the `total_claims > 0` guard let such pages fall through to wiki/.
|
||||||
total_claims = len(page.claims)
|
total_claims = len(page.claims)
|
||||||
grounded_claims = sum(1 for c in page.claims if c.grounded)
|
grounded_claims = sum(1 for c in page.claims if c.grounded)
|
||||||
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
|
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
|
||||||
|
|
||||||
if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate:
|
if total_claims == 0 or grounding_rate < settings.wiki_min_grounding_rate:
|
||||||
# Quarantine: write to wiki/draft/ instead of wiki/
|
# Quarantine and do NOT update the compile-state, so a transient failure
|
||||||
draft_dir = wiki_dir / "draft"
|
# (e.g. LLM outage) is retried on the next run instead of being frozen as
|
||||||
draft_dir.mkdir(parents=True, exist_ok=True)
|
# "compiled". An empty body is not worth a draft file — just record it.
|
||||||
page_path = draft_dir / f"{concept.slug}.md"
|
if page.markdown.strip():
|
||||||
page_path.write_text(page.markdown, encoding="utf-8")
|
draft_dir = wiki_dir / "draft"
|
||||||
|
draft_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(draft_dir / f"{concept.slug}.md").write_text(page.markdown, encoding="utf-8")
|
||||||
report.quarantined.append(concept.slug)
|
report.quarantined.append(concept.slug)
|
||||||
else:
|
else:
|
||||||
# Write page to wiki/
|
# Write page to wiki/
|
||||||
|
|||||||
@@ -504,6 +504,47 @@ def test_log_append_records_skipped(
|
|||||||
assert "circle-packing" in content
|
assert "circle-packing" in content
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Audit C-4: zero-citation pages must NOT bypass the quarantine
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_zero_citation_page_is_quarantined_not_published(
|
||||||
|
mock_retrieve: None,
|
||||||
|
mock_formulas: None,
|
||||||
|
mock_settings: Path,
|
||||||
|
) -> None:
|
||||||
|
"""A confident-but-uncited LLM response yields zero parseable claims
|
||||||
|
(grounding_rate 0.0). It must be quarantined — not written to wiki/ and
|
||||||
|
marked compiled. Regression for audit C-4: the old ``total_claims > 0``
|
||||||
|
guard let zero-claim (and LLM-outage empty) pages fall through to wiki/.
|
||||||
|
"""
|
||||||
|
from codex.wiki import compile_all
|
||||||
|
|
||||||
|
(mock_settings / "concepts.yaml").write_text(
|
||||||
|
textwrap.dedent(
|
||||||
|
"""\
|
||||||
|
concepts:
|
||||||
|
- slug: lobachevsky-function
|
||||||
|
title: Lobachevsky Function
|
||||||
|
aliases: [Milnor Lobachevsky]
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
# No [BibKey] citations anywhere → zero claims → grounding_rate 0.0.
|
||||||
|
uncited = MockLLM(
|
||||||
|
"The Lobachevsky function is obviously central to all of geometry. "
|
||||||
|
"Every result follows from it. This paragraph cites nothing at all."
|
||||||
|
)
|
||||||
|
|
||||||
|
report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings))
|
||||||
|
|
||||||
|
assert "lobachevsky-function" in report.quarantined
|
||||||
|
assert "lobachevsky-function" not in report.compiled
|
||||||
|
assert not (mock_settings / "lobachevsky-function.md").exists()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Idempotency: second compile without chunk change → no rewrite
|
# Idempotency: second compile without chunk change → no rewrite
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user