user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(synthesis): Lead/Provenance model + grounded leads + conjecture generator

Browse Source 
- codex/synthesis.py: Lead/Provenance dataclasses, find_connections/gaps/improvements,
  propose_conjectures (status=unverified, quarantined in leads/conjectures/),
  write_leads (grounded→leads/grounded/, conjectures→leads/conjectures/ HARD invariant)
- codex/cli.py: synthesis leads/conjectures/report command group
- codex/config.py: F-13 settings (leads_dir, synthesis_llm_*, synthesis_top_k,
  synthesis_min_grounded_ratio, synthesis_gap_min_coverage)
- tests/synthesis/: 42 tests covering model, grounding, conjecture invariant, quarantine, CLI
- spike/: F-13 spike script + output (live run attempted)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 08:37:42 +02:00
parent 967e6baa59
commit 8cf0cc7e01
14 changed files with 2139 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
tests/synthesis/__init__.py Normal file
View File

124
tests/synthesis/conftest.py Normal file
View File

@@ -0,0 +1,124 @@
"""Shared synthesis fixtures — keep DB/LLM/embed fully mocked."""
from __future__ import annotations
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock
import pytest
# Shared corpus fixture used across the test suite.
FIXTURE_CHUNKS: list[dict[str, Any]] = [
{
"id": 1,
"paper_id": "springborn-2008",
"ord": 16,
"content": (
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
"V = L(gamma1) + L(gamma2) + L(gamma3). "
"The volume function V0 is strictly concave on the angle domain."
),
"bibkey": "Springborn2008",
},
{
"id": 2,
"paper_id": "bps-2015",
"ord": 3,
"content": (
"A discrete conformal map is defined by logarithmic scale factors u_i "
"at each vertex such that the edge lengths satisfy a compatibility condition."
),
"bibkey": "BobenkoPinkallSpringborn2015",
},
{
"id": 3,
"paper_id": "luo-2004",
"ord": 0,
"content": (
"The combinatorial Yamabe flow drives the edge lengths towards a target "
"curvature on a triangulated surface."
),
"bibkey": "Luo2004",
},
]
class StubLLM:
"""Deterministic mock LLM that returns a preset response."""
def __init__(self, response: str | list[str]) -> None:
if isinstance(response, str):
self._responses = [response]
else:
self._responses = list(response)
self._calls: list[tuple[str, str]] = []
def generate(self, prompt: str, model: str) -> str:
self._calls.append((prompt, model))
if not self._responses:
return ""
if len(self._responses) == 1:
return self._responses[0]
return self._responses.pop(0)
@property
def calls(self) -> list[tuple[str, str]]:
return list(self._calls)
@pytest.fixture
def stub_llm() -> StubLLM:
"""Default stub returning empty string (override per test as needed)."""
return StubLLM("")
@pytest.fixture
def fixture_chunks() -> list[dict[str, Any]]:
return [dict(c) for c in FIXTURE_CHUNKS]
@pytest.fixture
def mock_retrieve(monkeypatch: pytest.MonkeyPatch) -> None:
"""Patch _retrieve_chunks to return deterministic fixture chunks (no DB)."""
monkeypatch.setattr(
"codex.synthesis._retrieve_chunks",
lambda queries, top_k: [dict(c) for c in FIXTURE_CHUNKS],
)
@pytest.fixture
def mock_paper_titles(monkeypatch: pytest.MonkeyPatch) -> None:
"""Patch _list_paper_titles to return a deterministic three-paper corpus."""
monkeypatch.setattr(
"codex.synthesis._list_paper_titles",
lambda db_conn: [
("Springborn2008", "A variational principle for weighted Delaunay triangulations"),
(
"BobenkoPinkallSpringborn2015",
"Discrete conformal maps and ideal hyperbolic polyhedra",
),
("Luo2004", "Combinatorial Yamabe Flow on Surfaces"),
],
)
@pytest.fixture
def mock_settings(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Settings stub pointing leads_dir at tmp_path/leads."""
from codex.config import Settings
leads_path = tmp_path / "leads"
mock = MagicMock(spec=Settings)
mock.leads_dir = str(leads_path)
mock.synthesis_llm_model = "test-model"
mock.synthesis_llm_url = None
mock.ollama_base_url = "http://localhost:11434"
mock.synthesis_top_k = 6
mock.synthesis_min_grounded_ratio = 0.5
mock.synthesis_gap_min_coverage = 2
mock.wiki_dir = str(tmp_path / "wiki")
monkeypatch.setattr("codex.synthesis.get_settings", lambda: mock)
return leads_path

179
tests/synthesis/test_cli.py Normal file
View File

@@ -0,0 +1,179 @@
"""Tests for the ``codex synthesis`` CLI command group."""
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from typer.testing import CliRunner
from codex.cli import app
from codex.synthesis import Lead, Provenance
runner = CliRunner()
def _grounded_lead(lead_id: str = "L-0001") -> Lead:
return Lead(
id=lead_id,
kind="connection",
title="Connection — X ⇄ Y",
body="Some grounded body. [X2020 #chunk 1]",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.8,
)
def _conjecture(lead_id: str = "L-0002") -> Lead:
return Lead(
id=lead_id,
kind="conjecture",
title="Bold guess",
body="Hypothetical.",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.0,
suggested_validation="Run a falsification spike.",
)
@pytest.fixture
def isolated_leads_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Point Settings.leads_dir at tmp_path/leads for CLI tests."""
from codex.config import Settings
leads_path = tmp_path / "leads"
settings = MagicMock(spec=Settings)
settings.leads_dir = str(leads_path)
settings.synthesis_llm_model = "test-model"
settings.synthesis_llm_url = None
settings.ollama_base_url = "http://localhost:11434"
settings.synthesis_top_k = 6
settings.synthesis_min_grounded_ratio = 0.5
settings.synthesis_gap_min_coverage = 2
monkeypatch.setattr("codex.cli.get_settings", lambda: settings, raising=False)
monkeypatch.setattr("codex.synthesis.get_settings", lambda: settings)
monkeypatch.setattr(
"codex.config.get_settings", lambda: settings, raising=False
)
return leads_path
def test_synthesis_leads_invokes_three_finders(
monkeypatch: pytest.MonkeyPatch,
isolated_leads_dir: Path,
) -> None:
"""`codex synthesis leads --lib-path` calls connection + gap + improvement."""
calls: list[str] = []
def fake_connections(*, top_k: int, llm: object) -> list[Lead]:
calls.append("connection")
return [_grounded_lead("L-0001")]
def fake_gaps(
*,
lib_path: str | None = None,
llm: object,
topics: list[str] | None = None,
) -> list[Lead]:
calls.append("gap")
return []
def fake_improvements(lib_path: str, *, top_k: int, llm: object) -> list[Lead]:
calls.append("improvement")
return []
monkeypatch.setattr("codex.synthesis.find_connections", fake_connections)
monkeypatch.setattr("codex.synthesis.find_gaps", fake_gaps)
monkeypatch.setattr("codex.synthesis.find_improvements", fake_improvements)
monkeypatch.setattr("codex.synthesis.default_llm_client", lambda: object())
result = runner.invoke(app, ["synthesis", "leads", "--lib-path", "/tmp/fake"])
assert result.exit_code == 0, result.stdout
assert calls == ["connection", "gap", "improvement"]
assert (isolated_leads_dir / "grounded" / "L-0001.md").exists()
def test_synthesis_leads_without_lib_path_skips_improvement(
monkeypatch: pytest.MonkeyPatch,
isolated_leads_dir: Path,
) -> None:
"""Without --lib-path, the improvement finder must be skipped (graceful)."""
calls: list[str] = []
monkeypatch.setattr(
"codex.synthesis.find_connections",
lambda *, top_k, llm: (calls.append("connection"), [])[1],
)
monkeypatch.setattr(
"codex.synthesis.find_gaps",
lambda *, lib_path=None, llm, topics=None: (calls.append("gap"), [])[1],
)
monkeypatch.setattr(
"codex.synthesis.find_improvements",
lambda *args, **kwargs: (calls.append("improvement"), [])[1],
)
monkeypatch.setattr("codex.synthesis.default_llm_client", lambda: object())
result = runner.invoke(app, ["synthesis", "leads"])
assert result.exit_code == 0, result.stdout
assert "improvement" not in calls
assert "connection" in calls
assert "gap" in calls
def test_synthesis_conjectures_writes_to_conjectures_dir(
monkeypatch: pytest.MonkeyPatch,
isolated_leads_dir: Path,
) -> None:
"""`codex synthesis conjectures` lands files under leads/conjectures/."""
monkeypatch.setattr(
"codex.synthesis.propose_conjectures",
lambda *, top_k, llm: [_conjecture("L-0042")],
)
monkeypatch.setattr("codex.synthesis.default_llm_client", lambda: object())
result = runner.invoke(app, ["synthesis", "conjectures"])
assert result.exit_code == 0, result.stdout
assert (isolated_leads_dir / "conjectures" / "L-0042.md").exists()
assert not (isolated_leads_dir / "grounded" / "L-0042.md").exists()
def test_synthesis_report_separates_sections(
monkeypatch: pytest.MonkeyPatch,
isolated_leads_dir: Path,
) -> None:
"""`codex synthesis report` shows grounded and conjectures in distinct headers."""
from codex.synthesis import write_leads
write_leads(
[_grounded_lead("L-0001"), _conjecture("L-0002")],
str(isolated_leads_dir),
)
result = runner.invoke(app, ["synthesis", "report"])
assert result.exit_code == 0, result.stdout
assert "GROUNDED LEADS" in result.stdout
assert "CONJECTURES" in result.stdout
# Visible separation: GROUNDED header strictly precedes CONJECTURES header
assert result.stdout.index("GROUNDED LEADS") < result.stdout.index("CONJECTURES")
# UNVERIFIED tag in the conjectures header
assert "UNVERIFIED" in result.stdout
def test_synthesis_report_empty(
isolated_leads_dir: Path,
) -> None:
"""`codex synthesis report` with no leads still prints both sections."""
result = runner.invoke(app, ["synthesis", "report"])
assert result.exit_code == 0
assert "GROUNDED LEADS" in result.stdout
assert "CONJECTURES" in result.stdout
assert "(none)" in result.stdout

113
tests/synthesis/test_conjecture.py Normal file
View File

@@ -0,0 +1,113 @@
"""Tests for stage-5 conjecture generation.
HARD invariant: every conjecture MUST carry status='unverified', a
non-empty suggested_validation, and non-empty provenance.
"""
from __future__ import annotations
from pathlib import Path
from codex.synthesis import _parse_conjecture_output, propose_conjectures
from tests.synthesis.conftest import StubLLM
_VALID_CONJECTURE = (
"Title: A discrete cosmetic-surgery analogue exists for Yamabe flows.\n"
"Body: The combinatorial Yamabe flow [Luo2004 #chunk 0] is a discrete "
"analogue of conformal change. One could hope the cosmetic surgery "
"conjecture has a discrete formulation in this setting.\n"
"Validation: Search for a counter-example among simplicial 3-manifolds "
"with at most 12 tetrahedra using a CSP solver."
)
_MISSING_VALIDATION = (
"Title: A naive idea.\n"
"Body: Some thoughts that follow [Luo2004 #chunk 0]. No falsification path "
"is described."
)
def test_parse_valid_conjecture_output() -> None:
"""_parse_conjecture_output extracts title / body / validation."""
parsed = _parse_conjecture_output(_VALID_CONJECTURE)
assert parsed is not None
title, body, validation = parsed
assert title.startswith("A discrete cosmetic-surgery")
assert "Luo2004" in body
assert "counter-example" in validation
def test_parse_conjecture_missing_validation_returns_none() -> None:
"""Output without a Validation: line is rejected."""
assert _parse_conjecture_output(_MISSING_VALIDATION) is None
def test_parse_conjecture_empty_input() -> None:
"""Empty LLM output → None."""
assert _parse_conjecture_output("") is None
def test_propose_conjectures_marks_status_unverified(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Every emitted conjecture has status='unverified'."""
llm = StubLLM(_VALID_CONJECTURE)
leads = propose_conjectures(top_k=6, llm=llm)
assert len(leads) >= 1
for lead in leads:
assert lead.kind == "conjecture"
assert lead.status == "unverified"
def test_propose_conjectures_requires_validation(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Every emitted conjecture has a non-empty suggested_validation."""
llm = StubLLM(_VALID_CONJECTURE)
leads = propose_conjectures(top_k=6, llm=llm)
assert len(leads) >= 1
for lead in leads:
assert lead.suggested_validation.strip() != ""
def test_propose_conjectures_requires_provenance(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Every conjecture has at least one Provenance entry (seed chunks)."""
llm = StubLLM(_VALID_CONJECTURE)
leads = propose_conjectures(top_k=6, llm=llm)
assert len(leads) >= 1
for lead in leads:
assert len(lead.provenance) >= 1
def test_propose_conjectures_drops_when_validation_missing(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""LLM output without a Validation: line is rejected (zero leads)."""
llm = StubLLM(_MISSING_VALIDATION)
leads = propose_conjectures(top_k=6, llm=llm)
assert leads == []
def test_propose_conjectures_handles_llm_failure(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""LLM raising → returns [] (mirrors F-12 graceful degradation)."""
class FailingLLM:
def generate(self, prompt: str, model: str) -> str:
raise ConnectionError("ollama unreachable")
leads = propose_conjectures(top_k=6, llm=FailingLLM())
assert leads == []

136
tests/synthesis/test_gaps.py Normal file
View File

@@ -0,0 +1,136 @@
"""Tests for find_gaps — coverage map + code-vs-corpus signals."""
from __future__ import annotations
from pathlib import Path
from typing import Any
import pytest
from codex.synthesis import find_gaps
from tests.synthesis.conftest import StubLLM
def test_find_gaps_code_cites_missing_bibkey(
monkeypatch: pytest.MonkeyPatch,
mock_paper_titles: None,
mock_settings: Path,
fixture_chunks: list[dict[str, object]],
tmp_path: Path,
) -> None:
"""A bibkey referenced in code but absent from corpus → direct gap lead."""
# Build a small C++ source tree with an @cite pointing at a missing bibkey
src_dir = tmp_path / "src"
src_dir.mkdir()
(src_dir / "main.cpp").write_text(
"// @cite UnknownPaper2099\nvoid foo() {}\n",
encoding="utf-8",
)
# All probes return the fixture chunks (so coverage map yields nothing)
monkeypatch.setattr(
"codex.synthesis._retrieve_chunks",
lambda queries, top_k: [dict(c) for c in fixture_chunks],
)
leads = find_gaps(lib_path=str(src_dir), llm=StubLLM(""))
bibkeys_in_gaps = {p.bibkey for lead in leads for p in lead.provenance}
# The missing bibkey shows up as a gap
assert any("UnknownPaper2099" in lead.title for lead in leads)
assert "UnknownPaper2099" in bibkeys_in_gaps
def test_find_gaps_code_known_bibkey_not_reported(
monkeypatch: pytest.MonkeyPatch,
mock_paper_titles: None,
mock_settings: Path,
fixture_chunks: list[dict[str, object]],
tmp_path: Path,
) -> None:
"""A bibkey already in the corpus is NOT flagged by the code-vs-corpus path."""
src_dir = tmp_path / "src"
src_dir.mkdir()
(src_dir / "main.cpp").write_text(
"// @cite Springborn2008\nvoid foo() {}\n",
encoding="utf-8",
)
monkeypatch.setattr(
"codex.synthesis._retrieve_chunks",
lambda queries, top_k: [dict(c) for c in fixture_chunks],
)
leads = find_gaps(lib_path=str(src_dir), llm=StubLLM(""))
# Springborn2008 IS in the corpus → not a gap
assert not any("Springborn2008" in lead.title for lead in leads)
def test_find_gaps_missing_lib_path_ok(
monkeypatch: pytest.MonkeyPatch,
mock_paper_titles: None,
mock_settings: Path,
fixture_chunks: list[dict[str, object]],
) -> None:
"""Without lib_path the code-vs-corpus path is skipped (no crash)."""
monkeypatch.setattr(
"codex.synthesis._retrieve_chunks",
lambda queries, top_k: [dict(c) for c in fixture_chunks],
)
leads = find_gaps(lib_path=None, llm=StubLLM(""))
# No crash; result list may be empty depending on coverage map.
assert isinstance(leads, list)
def test_find_gaps_coverage_map_low_coverage_triggers_llm(
monkeypatch: pytest.MonkeyPatch,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""A topic covered by only 1 bibkey (< 2) triggers the LLM gap prompt.
The grounded body must reference Springborn2008 [...] in a content-5-gram
that exists in the chunk, otherwise the lead is dropped.
"""
sparse_chunks: list[dict[str, Any]] = [
{
"id": 10,
"paper_id": "springborn-2008",
"ord": 16,
"content": (
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
"V = L(gamma1) + L(gamma2) + L(gamma3). "
"The volume function V0 is strictly concave on the angle domain."
),
"bibkey": "Springborn2008",
},
]
monkeypatch.setattr(
"codex.synthesis._retrieve_chunks",
lambda queries, top_k: [dict(c) for c in sparse_chunks],
)
grounded_body = (
"Only Springborn2008 covers the volume formula at depth, while comparison "
"papers are missing.\n"
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
"V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
)
leads = find_gaps(llm=StubLLM(grounded_body))
assert any(lead.kind == "gap" for lead in leads)
def test_find_gaps_explicit_topics_override_default(
monkeypatch: pytest.MonkeyPatch,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Passing topics= explicitly overrides the default (paper titles)."""
captured_queries: list[list[str]] = []
def fake_retrieve(queries: list[str], top_k: int) -> list[dict[str, Any]]:
captured_queries.append(queries)
return [] # forces direct-gap-no-chunks path
monkeypatch.setattr("codex.synthesis._retrieve_chunks", fake_retrieve)
find_gaps(llm=StubLLM(""), topics=["custom topic A", "custom topic B"])
# Both custom topics should have been probed
flat = [q for qs in captured_queries for q in qs]
assert "custom topic A" in flat
assert "custom topic B" in flat

175
tests/synthesis/test_grounded.py Normal file
View File

@@ -0,0 +1,175 @@
"""Tests for grounded leads (connection / gap / improvement).
The Grounding-Guard is re-used from :mod:`codex.wiki` (F-12). These tests
verify that ungrounded claims are either dropped or annotated with ⚠,
and that grounded statements survive intact.
"""
from __future__ import annotations
from pathlib import Path
from codex.synthesis import (
_finalise_grounded_lead,
_mark_ungrounded,
find_connections,
find_gaps,
)
from tests.synthesis.conftest import StubLLM
_GROUNDED_CONNECTION = (
"Both papers build hyperbolic volumes from the Lobachevsky function.\n"
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
"V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
"A discrete conformal map is defined by logarithmic scale factors u_i at "
"each vertex such that the edge lengths satisfy a compatibility condition. "
"[BobenkoPinkallSpringborn2015 #chunk 3]\n"
)
_UNGROUNDED_CONNECTION = (
"Both papers prove the Riemann hypothesis via the Lobachevsky function. "
"[Springborn2008 #chunk 99]\n"
)
def test_finalise_grounded_lead_returns_none_without_claims(
fixture_chunks: list[dict[str, object]],
) -> None:
"""Body with no inline citations yields no grounded lead."""
lead = _finalise_grounded_lead(
seq=1,
kind="connection",
title="t",
body="A plain statement with no citation marker.",
chunks=fixture_chunks,
min_grounded_ratio=0.5,
)
assert lead is None
def test_finalise_grounded_lead_drops_when_ratio_below_floor(
fixture_chunks: list[dict[str, object]],
) -> None:
"""All-ungrounded body → None (not silently promoted)."""
lead = _finalise_grounded_lead(
seq=1,
kind="connection",
title="t",
body=_UNGROUNDED_CONNECTION,
chunks=fixture_chunks,
min_grounded_ratio=0.5,
)
assert lead is None
def test_finalise_grounded_lead_succeeds_when_grounded(
fixture_chunks: list[dict[str, object]],
) -> None:
"""Grounded body passes the floor and returns a populated Lead."""
lead = _finalise_grounded_lead(
seq=1,
kind="connection",
title="Both papers share a building block",
body=_GROUNDED_CONNECTION,
chunks=fixture_chunks,
min_grounded_ratio=0.5,
)
assert lead is not None
assert lead.kind == "connection"
assert lead.id == "L-0001"
assert lead.confidence >= 0.5
assert lead.status == "unverified"
assert lead.suggested_validation == ""
# All claims should be grounded
assert all(c.grounded for c in lead.claims)
def test_mark_ungrounded_prefixes_warning() -> None:
"""_mark_ungrounded prefixes ⚠ to the matching claim text once."""
from codex.wiki import Claim
body = "Some fact. [X2020 #c 1]\nOther fact. [Y2021 #c 2]\n"
claims = [
Claim(text="Some fact", bibkey="X2020", locator="c 1", grounded=False),
Claim(text="Other fact", bibkey="Y2021", locator="c 2", grounded=True),
]
out = _mark_ungrounded(body, claims)
assert "⚠ Some fact" in out
# Grounded claim is not annotated
assert "⚠ Other fact" not in out
def test_find_connections_emits_grounded_lead(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""End-to-end: with a grounded LLM response, a connection lead is produced."""
llm = StubLLM(_GROUNDED_CONNECTION)
leads = find_connections(top_k=6, llm=llm)
assert len(leads) >= 1
lead = leads[0]
assert lead.kind == "connection"
assert lead.provenance, "grounded lead must carry provenance"
# No ⚠ should appear because all claims grounded
assert "⚠" not in lead.body
def test_find_connections_drops_ungrounded_response(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Ungrounded LLM responses are dropped, not emitted with ⚠."""
llm = StubLLM(_UNGROUNDED_CONNECTION)
leads = find_connections(top_k=6, llm=llm)
assert leads == []
def test_find_connections_handles_llm_failure(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""LLM raising → returns [] (mirrors F-12 graceful degradation)."""
class FailingLLM:
def generate(self, prompt: str, model: str) -> str:
raise ConnectionError("ollama unreachable")
leads = find_connections(top_k=6, llm=FailingLLM())
assert leads == []
def test_find_gaps_topic_with_no_chunks(
monkeypatch,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""When retrieval returns no chunks, a gap lead is emitted directly."""
monkeypatch.setattr("codex.synthesis._retrieve_chunks", lambda queries, top_k: [])
leads = find_gaps(llm=StubLLM(""))
assert len(leads) >= 1
assert all(lead.kind == "gap" for lead in leads)
# Direct gap leads carry a validation hint (re-ingest)
assert all(lead.suggested_validation for lead in leads)
def test_find_gaps_well_covered_topic_skipped(
monkeypatch,
mock_paper_titles: None,
mock_settings: Path,
fixture_chunks: list[dict[str, object]],
) -> None:
"""Topics covered by enough bibkeys are NOT emitted as gaps.
Three distinct bibkeys in the fixture > synthesis_gap_min_coverage=2,
so every topic is well-covered and gets skipped.
"""
monkeypatch.setattr(
"codex.synthesis._retrieve_chunks",
lambda queries, top_k: [dict(c) for c in fixture_chunks],
)
leads = find_gaps(llm=StubLLM(""))
# All probes well-covered → no gap leads
assert leads == []

87
tests/synthesis/test_model.py Normal file
View File

@@ -0,0 +1,87 @@
"""Tests for the Lead / Provenance datamodel (F-13)."""
from __future__ import annotations
from typing import get_args
import pytest
from codex.synthesis import Lead, LeadKind, LeadStatus, Provenance
def test_provenance_requires_bibkey_and_locator() -> None:
"""Provenance has two required positional fields."""
p = Provenance(bibkey="Smith2020", locator="chunk 7")
assert p.bibkey == "Smith2020"
assert p.locator == "chunk 7"
def test_provenance_missing_field_raises() -> None:
"""Provenance without locator must fail to construct (TypeError)."""
with pytest.raises(TypeError):
Provenance(bibkey="Smith2020") # type: ignore[call-arg]
def test_lead_required_fields() -> None:
"""Lead requires id, kind, title, body, provenance, confidence."""
lead = Lead(
id="L-0001",
kind="connection",
title="A connection",
body="Some body text.",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.8,
)
assert lead.id == "L-0001"
assert lead.kind == "connection"
assert lead.status == "unverified" # default
assert lead.suggested_validation == "" # default
assert lead.claims == [] # default factory
def test_lead_kind_literals_exact() -> None:
"""Lead kinds are exactly: connection, gap, improvement, conjecture."""
assert set(get_args(LeadKind)) == {"connection", "gap", "improvement", "conjecture"}
def test_lead_status_literals_exact() -> None:
"""Lead statuses are exactly: unverified, spiking, go, no-go."""
assert set(get_args(LeadStatus)) == {"unverified", "spiking", "go", "no-go"}
def test_lead_provenance_can_be_multiple() -> None:
"""A Lead may carry multiple provenance pointers."""
provs = [
Provenance(bibkey="A2020", locator="chunk 1"),
Provenance(bibkey="B2021", locator="chunk 7"),
]
lead = Lead(
id="L-0002",
kind="gap",
title="t",
body="b",
provenance=provs,
confidence=0.5,
)
assert len(lead.provenance) == 2
def test_lead_missing_provenance_field_raises() -> None:
"""Constructing Lead without provenance is a TypeError."""
with pytest.raises(TypeError):
Lead( # type: ignore[call-arg]
id="L-0003",
kind="connection",
title="x",
body="y",
confidence=0.5,
)
def test_lead_id_format_helper() -> None:
"""The internal _make_lead_id helper produces zero-padded L-XXXX strings."""
from codex.synthesis import _make_lead_id
assert _make_lead_id(1) == "L-0001"
assert _make_lead_id(42) == "L-0042"
assert _make_lead_id(9999) == "L-9999"

132
tests/synthesis/test_quarantine.py Normal file
View File

@@ -0,0 +1,132 @@
"""INVARIANT tests — conjectures live in leads/conjectures/, never in wiki/.
This is the hard F-13 invariant. Violations would corrupt the wiki layer
and must be unrepresentable in the API.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from codex.synthesis import Lead, Provenance, write_leads
def _grounded_lead(lead_id: str = "L-0001") -> Lead:
return Lead(
id=lead_id,
kind="connection",
title="Connection title",
body="Some grounded body. [X2020 #chunk 1]",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.8,
status="unverified",
)
def _conjecture(lead_id: str = "L-0002") -> Lead:
return Lead(
id=lead_id,
kind="conjecture",
title="A bold conjecture",
body="Hypothetical statement seeded by [X2020 #chunk 1].",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.0,
status="unverified",
suggested_validation="Search for a counter-example with N<=8.",
)
def test_conjecture_lands_in_conjectures_dir(tmp_path: Path) -> None:
"""A conjecture lead writes to leads/conjectures/<id>.md."""
write_leads([_conjecture("L-0002")], str(tmp_path))
assert (tmp_path / "conjectures" / "L-0002.md").exists()
# And NOT in grounded/
assert not (tmp_path / "grounded" / "L-0002.md").exists()
def test_conjecture_never_in_grounded_dir(tmp_path: Path) -> None:
"""No conjecture file may appear under leads/grounded/."""
write_leads([_grounded_lead("L-0001"), _conjecture("L-0002")], str(tmp_path))
grounded_files = list((tmp_path / "grounded").glob("*.md"))
conj_files = list((tmp_path / "conjectures").glob("*.md"))
assert len(grounded_files) == 1
assert grounded_files[0].name == "L-0001.md"
assert len(conj_files) == 1
assert conj_files[0].name == "L-0002.md"
def test_conjecture_never_in_wiki_dir(tmp_path: Path) -> None:
"""The wiki/ directory is never touched by write_leads."""
wiki_dir = tmp_path / "wiki"
wiki_dir.mkdir()
write_leads([_conjecture("L-0002")], str(tmp_path / "leads"))
# No file landed in wiki/
assert list(wiki_dir.glob("*.md")) == []
def test_conjecture_markdown_contains_unverified_marker(tmp_path: Path) -> None:
"""The on-disk conjecture file announces UNVERIFIED status."""
write_leads([_conjecture("L-0002")], str(tmp_path))
md = (tmp_path / "conjectures" / "L-0002.md").read_text(encoding="utf-8")
assert "UNVERIFIED" in md
assert "Suggested validation" in md
assert "Search for a counter-example" in md
def test_invariant_violation_raises(tmp_path: Path) -> None:
"""Constructing a Lead with kind='conjecture' but routed by code into
grounded/ would be a programmer error — guarded by the kind→dir check.
We simulate it by monkey-patching the kind set, which is the only way
a writer could attempt this. The expectation is that no path exists in
the public API to write a conjecture into grounded/.
"""
# Hard guarantee: the only sanctioned writer routes by lead.kind.
# Verify the dataclass kind field cannot trivially be spoofed: it is
# a Literal-typed string so any value other than the four kinds is a
# type error at static-checking time. At runtime, write_leads still
# rejects unknown kinds.
bogus = Lead(
id="L-0003",
kind="conjecture", # legitimate kind …
title="x",
body="y",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.0,
suggested_validation="run a spike",
)
write_leads([bogus], str(tmp_path))
# Conjecture must land in conjectures/, not grounded/
assert (tmp_path / "conjectures" / "L-0003.md").exists()
assert not (tmp_path / "grounded" / "L-0003.md").exists()
def test_unknown_kind_rejected(tmp_path: Path) -> None:
"""A Lead with an unrecognised kind raises at write time."""
lead = Lead(
id="L-0009",
kind="bogus-kind", # type: ignore[arg-type]
title="t",
body="b",
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
confidence=0.5,
)
with pytest.raises(RuntimeError):
write_leads([lead], str(tmp_path))
def test_index_separates_grounded_and_conjectures(tmp_path: Path) -> None:
"""INDEX.md visibly separates grounded leads from conjectures."""
write_leads([_grounded_lead("L-0001"), _conjecture("L-0002")], str(tmp_path))
index = (tmp_path / "INDEX.md").read_text(encoding="utf-8")
# Two distinct, visibly separated sections
assert "## Grounded leads" in index
assert "## Conjectures" in index
# Ordering: grounded section appears before conjectures
assert index.index("Grounded leads") < index.index("Conjectures")
# Both ids are listed
assert "L-0001" in index
assert "L-0002" in index