From 0202a2266e190d10d033c7922b6d88cd6c22cd7f Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Sun, 14 Jun 2026 03:48:33 +0200 Subject: [PATCH] fix(wiki): pass chunks explicitly to compile_concept (single retrieve) compile_concept now accepts chunks as a positional parameter; compile_all retrieves chunks once and passes them directly, avoiding double-retrieve. Also separates all_concepts (full list) from compile_concepts (filtered), so cross-ref injection always uses the complete concept list regardless of --concept filter (Fix 6 + Fix 8). Co-Authored-By: Claude Sonnet 4.6 --- codex/wiki.py | 40 ++++++++++++++++++++++++-------------- tests/wiki/test_compile.py | 32 +++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/codex/wiki.py b/codex/wiki.py index 8a386e5..44577a6 100644 --- a/codex/wiki.py +++ b/codex/wiki.py @@ -426,6 +426,7 @@ def _render_page_markdown( def compile_concept( concept: Concept, + chunks: list[dict[str, Any]], *, top_k: int, llm: LLMClient, @@ -434,19 +435,24 @@ def compile_concept( ) -> ConceptPage: """Compile a single concept page. - 1. Retrieve Top-K chunks (hybrid dense+FTS) for concept title + aliases. - 2. Synthesise via LLM (Ollama) with per-claim citation format. - 3. Run Grounding-Guard: mark ungrounded claims as ⚠. - 4. Inject cross-references to other concepts as [[slug]] links. - 5. Embed formula chunks if ``formulas`` table is present (graceful). + Parameters + ---------- + concept: + The concept to compile. + chunks: + Pre-retrieved source chunks for this concept (from :func:`_retrieve_chunks`). + Passing chunks explicitly avoids a second retrieve and ensures the stored + hash matches the chunks actually used for synthesis. + + 1. Synthesise via LLM (Ollama) with per-claim citation format. + 2. Run Grounding-Guard: mark ungrounded claims as ⚠. + 3. Inject cross-references to other concepts as [[slug]] links. + 4. Embed formula chunks if ``formulas`` table is present (graceful). Returns a :class:`ConceptPage` with full markdown and claim list. """ settings = get_settings() - _wiki_dir = wiki_dir or Path(settings.wiki_dir) - - queries = [concept.title] + concept.aliases - chunks = _retrieve_chunks(queries, top_k=top_k) + _wiki_dir = wiki_dir or Path(settings.wiki_dir) # noqa: F841 — kept for future use h = _chunk_hash(chunks) prompt = _build_synthesis_prompt(concept, chunks) @@ -532,10 +538,13 @@ def compile_all( if not concepts_path.exists(): # Fall back to sibling concepts.yaml next to wiki/ dir concepts_path = wiki_dir.parent / "wiki" / "concepts.yaml" - concepts = load_concepts(str(concepts_path)) + # Load the FULL concept list — used for cross-reference injection regardless of filter + all_concepts = load_concepts(str(concepts_path)) - if concept_filter: - concepts = [c for c in concepts if c.slug == concept_filter] + # Apply filter only to the set of concepts that will be (re-)compiled + compile_concepts = ( + [c for c in all_concepts if c.slug == concept_filter] if concept_filter else all_concepts + ) state_path = wiki_dir / ".compile-state.json" state = _load_compile_state(state_path) @@ -549,8 +558,8 @@ def compile_all( report = CompileReport() - for concept in concepts: - # Fast check: retrieve chunks and compare hash + for concept in compile_concepts: + # Retrieve chunks once — reuse for hash check and synthesis queries = [concept.title] + concept.aliases chunks = _retrieve_chunks(queries, top_k=k) h = _chunk_hash(chunks) @@ -561,9 +570,10 @@ def compile_all( page = compile_concept( concept, + chunks, top_k=k, llm=_llm, - all_concepts=concepts, + all_concepts=all_concepts, # always the full list for cross-refs wiki_dir=wiki_dir, ) diff --git a/tests/wiki/test_compile.py b/tests/wiki/test_compile.py index 89498be..8996f7d 100644 --- a/tests/wiki/test_compile.py +++ b/tests/wiki/test_compile.py @@ -259,7 +259,9 @@ def test_compile_concept_returns_concept_page( ) -> None: """compile_concept returns a ConceptPage with non-empty markdown.""" llm = MockLLM(GROUNDED_LLM_OUTPUT) - page = compile_concept(FIXTURE_CONCEPT, top_k=6, llm=llm, wiki_dir=mock_settings) + page = compile_concept( + FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings + ) assert isinstance(page, ConceptPage) assert page.concept.slug == "lobachevsky-function" assert len(page.markdown) > 0 @@ -272,7 +274,9 @@ def test_compile_concept_has_chunk_hash( ) -> None: """Compiled page has a non-empty chunk_hash.""" llm = MockLLM(GROUNDED_LLM_OUTPUT) - page = compile_concept(FIXTURE_CONCEPT, top_k=6, llm=llm, wiki_dir=mock_settings) + page = compile_concept( + FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings + ) assert len(page.chunk_hash) == 64 # SHA-256 hex @@ -283,7 +287,9 @@ def test_compile_concept_grounded_claim_not_flagged( ) -> None: """A grounded claim does NOT receive the ⚠ marker in the output markdown.""" llm = MockLLM(GROUNDED_LLM_OUTPUT) - page = compile_concept(FIXTURE_CONCEPT, top_k=6, llm=llm, wiki_dir=mock_settings) + page = compile_concept( + FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings + ) # When all claims are grounded, no ⚠ in markdown # (may still have 0 ⚠ if claims found; just check no false positive for grounded) grounded = [c for c in page.claims if c.grounded] @@ -298,7 +304,9 @@ def test_compile_concept_ungrounded_claim_marked( ) -> None: """An ungrounded claim is marked ⚠ in the page markdown.""" llm = MockLLM(UNGROUNDED_LLM_OUTPUT) - page = compile_concept(FIXTURE_CONCEPT, top_k=6, llm=llm, wiki_dir=mock_settings) + page = compile_concept( + FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings + ) ungrounded = [c for c in page.claims if not c.grounded] if ungrounded: assert "⚠" in page.markdown @@ -311,7 +319,9 @@ def test_compile_concept_header_present( ) -> None: """Page markdown starts with an H1 header for the concept title.""" llm = MockLLM(GROUNDED_LLM_OUTPUT) - page = compile_concept(FIXTURE_CONCEPT, top_k=6, llm=llm, wiki_dir=mock_settings) + page = compile_concept( + FIXTURE_CONCEPT, FIXTURE_CHUNKS, top_k=6, llm=llm, wiki_dir=mock_settings + ) assert page.markdown.startswith("# Lobachevsky Function") @@ -434,10 +444,12 @@ def test_idempotent_no_rewrite_when_unchanged( call_count = 0 original_compile = compile_concept - def counting_compile(concept: Concept, **kwargs: Any) -> ConceptPage: + def counting_compile( + concept: Concept, chunks: list[Any], **kwargs: Any + ) -> ConceptPage: nonlocal call_count call_count += 1 - return original_compile(concept, **kwargs) + return original_compile(concept, chunks, **kwargs) monkeypatch.setattr("codex.wiki.compile_concept", counting_compile) @@ -478,10 +490,12 @@ def test_force_all_recompiles_even_when_unchanged( call_count = 0 original_compile = compile_concept - def counting_compile(concept: Concept, **kwargs: Any) -> ConceptPage: + def counting_compile( + concept: Concept, chunks: list[Any], **kwargs: Any + ) -> ConceptPage: nonlocal call_count call_count += 1 - return original_compile(concept, **kwargs) + return original_compile(concept, chunks, **kwargs) monkeypatch.setattr("codex.wiki.compile_concept", counting_compile)