From c4106b0f51ceabdfc1913a5fc5ccb15c6d40f835 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 15 Jun 2026 16:32:01 +0200 Subject: [PATCH] fix(synthesis): make coverage-map gaps opt-in (audit R-8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit find_gaps defaulted its probe topics to paper titles, so a title — which retrieves mostly its own single bibkey — fell below synthesis_gap_min_coverage and flagged almost every paper as a 'gap'. Coverage-map gaps now run only for explicitly-requested topics; the CLI gains a repeatable --topic option. The code-vs-corpus gap path (precise) is unchanged and still default. Tests: the two cases that relied on default-title coverage gaps now pass explicit topics; added a regression that no coverage gaps appear without --topic. Co-Authored-By: Claude Fable 5 --- codex/cli.py | 12 ++++++++++-- codex/synthesis.py | 15 ++++++++------- tests/synthesis/test_gaps.py | 27 ++++++++++++++++++++++++++- tests/synthesis/test_grounded.py | 3 ++- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/codex/cli.py b/codex/cli.py index 3a46872..4ba4152 100644 --- a/codex/cli.py +++ b/codex/cli.py @@ -422,7 +422,15 @@ def synthesis_leads( "--lib-path", help=( "Path to a C++ source tree. Enables improvement leads (@cite-aware) and " - "code-vs-corpus gap detection. Without it only connection + topic gaps run." + "code-vs-corpus gap detection." + ), + ), + topic: list[str] | None = typer.Option( # noqa: B008 + None, + "--topic", + help=( + "Topic to check for coverage gaps (repeatable). Opt-in: without --topic " + "no coverage-map gaps run (avoids per-paper false positives, audit R-8)." ), ), output_dir: Optional[str] = typer.Option( # noqa: UP045 @@ -444,7 +452,7 @@ def synthesis_leads( llm = default_llm_client() connections = find_connections(top_k=settings.synthesis_top_k, llm=llm) - gaps = find_gaps(lib_path=lib_path, llm=llm) + gaps = find_gaps(lib_path=lib_path, llm=llm, topics=topic or None) improvements = ( find_improvements(lib_path, top_k=settings.synthesis_top_k, llm=llm) if lib_path else [] ) diff --git a/codex/synthesis.py b/codex/synthesis.py index ff86e89..d04481d 100644 --- a/codex/synthesis.py +++ b/codex/synthesis.py @@ -416,11 +416,12 @@ def find_gaps( Two complementary signals are used: - * **Topic coverage map** — for each topic in ``topics`` (or, if - ``None``, the union of paper titles), retrieve chunks and check - bibkey coverage. A topic covered by < ``synthesis_gap_min_coverage`` - bibkeys is reported as a gap candidate and the LLM is asked to - articulate it. + * **Topic coverage map** — for each **explicitly-requested** topic in + ``topics``, retrieve chunks and check bibkey coverage. A topic covered by + < ``synthesis_gap_min_coverage`` bibkeys is a gap candidate and the LLM is + asked to articulate it. Defaulting to paper titles flagged almost every + paper as a gap (a title retrieves mostly its own single bibkey), so the + coverage map is opt-in now; pass ``topics`` / CLI ``--topic`` (audit R-8). * **Code-vs-corpus gap** — when ``lib_path`` is given, any @cite bibkey scanned from code that is NOT present in the corpus is flagged. This catches the "cited in code, never ingested" case. @@ -429,9 +430,9 @@ def find_gaps( seq = 1 leads: list[Lead] = [] + # Coverage-map probes are the explicit topics only — no default-to-titles, + # which turned every paper into a spurious gap (audit R-8). probe_topics: list[str] = list(topics or []) - if not probe_topics: - probe_topics = [title for _, title in _list_paper_titles(db_conn)] known_bibkeys = {bk for bk, _ in _list_paper_titles(db_conn)} diff --git a/tests/synthesis/test_gaps.py b/tests/synthesis/test_gaps.py index 61e54bf..aa4007a 100644 --- a/tests/synthesis/test_gaps.py +++ b/tests/synthesis/test_gaps.py @@ -112,10 +112,35 @@ def test_find_gaps_coverage_map_low_coverage_triggers_llm( "For an ideal tetrahedron with dihedral angles the hyperbolic volume is " "V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n" ) - leads = find_gaps(llm=StubLLM(grounded_body)) + # Coverage gaps are opt-in (audit R-8): pass the topic explicitly. + leads = find_gaps(llm=StubLLM(grounded_body), topics=["hyperbolic volume formula"]) assert any(lead.kind == "gap" for lead in leads) +def test_find_gaps_no_coverage_gaps_without_explicit_topics( + monkeypatch: pytest.MonkeyPatch, + mock_paper_titles: None, + mock_settings: Path, +) -> None: + """Without explicit topics, paper titles do NOT auto-generate coverage gaps (R-8).""" + sparse_chunks: list[dict[str, Any]] = [ + { + "id": 10, + "paper_id": "springborn-2008", + "ord": 16, + "content": "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)", + "bibkey": "Springborn2008", + }, + ] + monkeypatch.setattr( + "codex.synthesis._retrieve_chunks", + lambda queries, top_k: [dict(c) for c in sparse_chunks], + ) + # No lib_path, no topics → the coverage map must not run despite sparse coverage. + leads = find_gaps(llm=StubLLM("a gap. [Springborn2008 #chunk 16]")) + assert leads == [] + + def test_find_gaps_explicit_topics_override_default( monkeypatch: pytest.MonkeyPatch, mock_paper_titles: None, diff --git a/tests/synthesis/test_grounded.py b/tests/synthesis/test_grounded.py index 7d27ba7..59ac7ef 100644 --- a/tests/synthesis/test_grounded.py +++ b/tests/synthesis/test_grounded.py @@ -148,7 +148,8 @@ def test_find_gaps_topic_with_no_chunks( ) -> None: """When retrieval returns no chunks, a gap lead is emitted directly.""" monkeypatch.setattr("codex.synthesis._retrieve_chunks", lambda queries, top_k: []) - leads = find_gaps(llm=StubLLM("")) + # Coverage gaps are opt-in now (audit R-8): pass an explicit topic. + leads = find_gaps(llm=StubLLM(""), topics=["nonexistent topic"]) assert len(leads) >= 1 assert all(lead.kind == "gap" for lead in leads) # Direct gap leads carry a validation hint (re-ingest)