feat(synthesis): Lead/Provenance model + grounded leads + conjecture generator
- codex/synthesis.py: Lead/Provenance dataclasses, find_connections/gaps/improvements, propose_conjectures (status=unverified, quarantined in leads/conjectures/), write_leads (grounded→leads/grounded/, conjectures→leads/conjectures/ HARD invariant) - codex/cli.py: synthesis leads/conjectures/report command group - codex/config.py: F-13 settings (leads_dir, synthesis_llm_*, synthesis_top_k, synthesis_min_grounded_ratio, synthesis_gap_min_coverage) - tests/synthesis/: 42 tests covering model, grounding, conjecture invariant, quarantine, CLI - spike/: F-13 spike script + output (live run attempted) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
136
tests/synthesis/test_gaps.py
Normal file
136
tests/synthesis/test_gaps.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Tests for find_gaps — coverage map + code-vs-corpus signals."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from codex.synthesis import find_gaps
|
||||
from tests.synthesis.conftest import StubLLM
|
||||
|
||||
|
||||
def test_find_gaps_code_cites_missing_bibkey(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""A bibkey referenced in code but absent from corpus → direct gap lead."""
|
||||
# Build a small C++ source tree with an @cite pointing at a missing bibkey
|
||||
src_dir = tmp_path / "src"
|
||||
src_dir.mkdir()
|
||||
(src_dir / "main.cpp").write_text(
|
||||
"// @cite UnknownPaper2099\nvoid foo() {}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# All probes return the fixture chunks (so coverage map yields nothing)
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis._retrieve_chunks",
|
||||
lambda queries, top_k: [dict(c) for c in fixture_chunks],
|
||||
)
|
||||
|
||||
leads = find_gaps(lib_path=str(src_dir), llm=StubLLM(""))
|
||||
bibkeys_in_gaps = {p.bibkey for lead in leads for p in lead.provenance}
|
||||
# The missing bibkey shows up as a gap
|
||||
assert any("UnknownPaper2099" in lead.title for lead in leads)
|
||||
assert "UnknownPaper2099" in bibkeys_in_gaps
|
||||
|
||||
|
||||
def test_find_gaps_code_known_bibkey_not_reported(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""A bibkey already in the corpus is NOT flagged by the code-vs-corpus path."""
|
||||
src_dir = tmp_path / "src"
|
||||
src_dir.mkdir()
|
||||
(src_dir / "main.cpp").write_text(
|
||||
"// @cite Springborn2008\nvoid foo() {}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis._retrieve_chunks",
|
||||
lambda queries, top_k: [dict(c) for c in fixture_chunks],
|
||||
)
|
||||
leads = find_gaps(lib_path=str(src_dir), llm=StubLLM(""))
|
||||
# Springborn2008 IS in the corpus → not a gap
|
||||
assert not any("Springborn2008" in lead.title for lead in leads)
|
||||
|
||||
|
||||
def test_find_gaps_missing_lib_path_ok(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
) -> None:
|
||||
"""Without lib_path the code-vs-corpus path is skipped (no crash)."""
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis._retrieve_chunks",
|
||||
lambda queries, top_k: [dict(c) for c in fixture_chunks],
|
||||
)
|
||||
leads = find_gaps(lib_path=None, llm=StubLLM(""))
|
||||
# No crash; result list may be empty depending on coverage map.
|
||||
assert isinstance(leads, list)
|
||||
|
||||
|
||||
def test_find_gaps_coverage_map_low_coverage_triggers_llm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""A topic covered by only 1 bibkey (< 2) triggers the LLM gap prompt.
|
||||
|
||||
The grounded body must reference Springborn2008 [...] in a content-5-gram
|
||||
that exists in the chunk, otherwise the lead is dropped.
|
||||
"""
|
||||
sparse_chunks: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": 10,
|
||||
"paper_id": "springborn-2008",
|
||||
"ord": 16,
|
||||
"content": (
|
||||
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
|
||||
"V = L(gamma1) + L(gamma2) + L(gamma3). "
|
||||
"The volume function V0 is strictly concave on the angle domain."
|
||||
),
|
||||
"bibkey": "Springborn2008",
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis._retrieve_chunks",
|
||||
lambda queries, top_k: [dict(c) for c in sparse_chunks],
|
||||
)
|
||||
grounded_body = (
|
||||
"Only Springborn2008 covers the volume formula at depth, while comparison "
|
||||
"papers are missing.\n"
|
||||
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
|
||||
"V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
|
||||
)
|
||||
leads = find_gaps(llm=StubLLM(grounded_body))
|
||||
assert any(lead.kind == "gap" for lead in leads)
|
||||
|
||||
|
||||
def test_find_gaps_explicit_topics_override_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""Passing topics= explicitly overrides the default (paper titles)."""
|
||||
captured_queries: list[list[str]] = []
|
||||
|
||||
def fake_retrieve(queries: list[str], top_k: int) -> list[dict[str, Any]]:
|
||||
captured_queries.append(queries)
|
||||
return [] # forces direct-gap-no-chunks path
|
||||
|
||||
monkeypatch.setattr("codex.synthesis._retrieve_chunks", fake_retrieve)
|
||||
find_gaps(llm=StubLLM(""), topics=["custom topic A", "custom topic B"])
|
||||
# Both custom topics should have been probed
|
||||
flat = [q for qs in captured_queries for q in qs]
|
||||
assert "custom topic A" in flat
|
||||
assert "custom topic B" in flat
|
||||
Reference in New Issue
Block a user