fix(synthesis): make coverage-map gaps opt-in (audit R-8)
find_gaps defaulted its probe topics to paper titles, so a title — which retrieves mostly its own single bibkey — fell below synthesis_gap_min_coverage and flagged almost every paper as a 'gap'. Coverage-map gaps now run only for explicitly-requested topics; the CLI gains a repeatable --topic option. The code-vs-corpus gap path (precise) is unchanged and still default. Tests: the two cases that relied on default-title coverage gaps now pass explicit topics; added a regression that no coverage gaps appear without --topic. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
12
codex/cli.py
12
codex/cli.py
@@ -422,7 +422,15 @@ def synthesis_leads(
|
||||
"--lib-path",
|
||||
help=(
|
||||
"Path to a C++ source tree. Enables improvement leads (@cite-aware) and "
|
||||
"code-vs-corpus gap detection. Without it only connection + topic gaps run."
|
||||
"code-vs-corpus gap detection."
|
||||
),
|
||||
),
|
||||
topic: list[str] | None = typer.Option( # noqa: B008
|
||||
None,
|
||||
"--topic",
|
||||
help=(
|
||||
"Topic to check for coverage gaps (repeatable). Opt-in: without --topic "
|
||||
"no coverage-map gaps run (avoids per-paper false positives, audit R-8)."
|
||||
),
|
||||
),
|
||||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||||
@@ -444,7 +452,7 @@ def synthesis_leads(
|
||||
llm = default_llm_client()
|
||||
|
||||
connections = find_connections(top_k=settings.synthesis_top_k, llm=llm)
|
||||
gaps = find_gaps(lib_path=lib_path, llm=llm)
|
||||
gaps = find_gaps(lib_path=lib_path, llm=llm, topics=topic or None)
|
||||
improvements = (
|
||||
find_improvements(lib_path, top_k=settings.synthesis_top_k, llm=llm) if lib_path else []
|
||||
)
|
||||
|
||||
@@ -416,11 +416,12 @@ def find_gaps(
|
||||
|
||||
Two complementary signals are used:
|
||||
|
||||
* **Topic coverage map** — for each topic in ``topics`` (or, if
|
||||
``None``, the union of paper titles), retrieve chunks and check
|
||||
bibkey coverage. A topic covered by < ``synthesis_gap_min_coverage``
|
||||
bibkeys is reported as a gap candidate and the LLM is asked to
|
||||
articulate it.
|
||||
* **Topic coverage map** — for each **explicitly-requested** topic in
|
||||
``topics``, retrieve chunks and check bibkey coverage. A topic covered by
|
||||
< ``synthesis_gap_min_coverage`` bibkeys is a gap candidate and the LLM is
|
||||
asked to articulate it. Defaulting to paper titles flagged almost every
|
||||
paper as a gap (a title retrieves mostly its own single bibkey), so the
|
||||
coverage map is opt-in now; pass ``topics`` / CLI ``--topic`` (audit R-8).
|
||||
* **Code-vs-corpus gap** — when ``lib_path`` is given, any @cite
|
||||
bibkey scanned from code that is NOT present in the corpus is
|
||||
flagged. This catches the "cited in code, never ingested" case.
|
||||
@@ -429,9 +430,9 @@ def find_gaps(
|
||||
seq = 1
|
||||
leads: list[Lead] = []
|
||||
|
||||
# Coverage-map probes are the explicit topics only — no default-to-titles,
|
||||
# which turned every paper into a spurious gap (audit R-8).
|
||||
probe_topics: list[str] = list(topics or [])
|
||||
if not probe_topics:
|
||||
probe_topics = [title for _, title in _list_paper_titles(db_conn)]
|
||||
|
||||
known_bibkeys = {bk for bk, _ in _list_paper_titles(db_conn)}
|
||||
|
||||
|
||||
@@ -112,10 +112,35 @@ def test_find_gaps_coverage_map_low_coverage_triggers_llm(
|
||||
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
|
||||
"V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
|
||||
)
|
||||
leads = find_gaps(llm=StubLLM(grounded_body))
|
||||
# Coverage gaps are opt-in (audit R-8): pass the topic explicitly.
|
||||
leads = find_gaps(llm=StubLLM(grounded_body), topics=["hyperbolic volume formula"])
|
||||
assert any(lead.kind == "gap" for lead in leads)
|
||||
|
||||
|
||||
def test_find_gaps_no_coverage_gaps_without_explicit_topics(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""Without explicit topics, paper titles do NOT auto-generate coverage gaps (R-8)."""
|
||||
sparse_chunks: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": 10,
|
||||
"paper_id": "springborn-2008",
|
||||
"ord": 16,
|
||||
"content": "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)",
|
||||
"bibkey": "Springborn2008",
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis._retrieve_chunks",
|
||||
lambda queries, top_k: [dict(c) for c in sparse_chunks],
|
||||
)
|
||||
# No lib_path, no topics → the coverage map must not run despite sparse coverage.
|
||||
leads = find_gaps(llm=StubLLM("a gap. [Springborn2008 #chunk 16]"))
|
||||
assert leads == []
|
||||
|
||||
|
||||
def test_find_gaps_explicit_topics_override_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
|
||||
@@ -148,7 +148,8 @@ def test_find_gaps_topic_with_no_chunks(
|
||||
) -> None:
|
||||
"""When retrieval returns no chunks, a gap lead is emitted directly."""
|
||||
monkeypatch.setattr("codex.synthesis._retrieve_chunks", lambda queries, top_k: [])
|
||||
leads = find_gaps(llm=StubLLM(""))
|
||||
# Coverage gaps are opt-in now (audit R-8): pass an explicit topic.
|
||||
leads = find_gaps(llm=StubLLM(""), topics=["nonexistent topic"])
|
||||
assert len(leads) >= 1
|
||||
assert all(lead.kind == "gap" for lead in leads)
|
||||
# Direct gap leads carry a validation hint (re-ingest)
|
||||
|
||||
Reference in New Issue
Block a user