feat(synthesis): Lead/Provenance model + grounded leads + conjecture generator
- codex/synthesis.py: Lead/Provenance dataclasses, find_connections/gaps/improvements, propose_conjectures (status=unverified, quarantined in leads/conjectures/), write_leads (grounded→leads/grounded/, conjectures→leads/conjectures/ HARD invariant) - codex/cli.py: synthesis leads/conjectures/report command group - codex/config.py: F-13 settings (leads_dir, synthesis_llm_*, synthesis_top_k, synthesis_min_grounded_ratio, synthesis_gap_min_coverage) - tests/synthesis/: 42 tests covering model, grounding, conjecture invariant, quarantine, CLI - spike/: F-13 spike script + output (live run attempted) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
179
tests/synthesis/test_cli.py
Normal file
179
tests/synthesis/test_cli.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""Tests for the ``codex synthesis`` CLI command group."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from codex.cli import app
|
||||
from codex.synthesis import Lead, Provenance
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def _grounded_lead(lead_id: str = "L-0001") -> Lead:
|
||||
return Lead(
|
||||
id=lead_id,
|
||||
kind="connection",
|
||||
title="Connection — X ⇄ Y",
|
||||
body="Some grounded body. [X2020 #chunk 1]",
|
||||
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
|
||||
def _conjecture(lead_id: str = "L-0002") -> Lead:
|
||||
return Lead(
|
||||
id=lead_id,
|
||||
kind="conjecture",
|
||||
title="Bold guess",
|
||||
body="Hypothetical.",
|
||||
provenance=[Provenance(bibkey="X2020", locator="chunk 1")],
|
||||
confidence=0.0,
|
||||
suggested_validation="Run a falsification spike.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_leads_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
"""Point Settings.leads_dir at tmp_path/leads for CLI tests."""
|
||||
from codex.config import Settings
|
||||
|
||||
leads_path = tmp_path / "leads"
|
||||
|
||||
settings = MagicMock(spec=Settings)
|
||||
settings.leads_dir = str(leads_path)
|
||||
settings.synthesis_llm_model = "test-model"
|
||||
settings.synthesis_llm_url = None
|
||||
settings.ollama_base_url = "http://localhost:11434"
|
||||
settings.synthesis_top_k = 6
|
||||
settings.synthesis_min_grounded_ratio = 0.5
|
||||
settings.synthesis_gap_min_coverage = 2
|
||||
|
||||
monkeypatch.setattr("codex.cli.get_settings", lambda: settings, raising=False)
|
||||
monkeypatch.setattr("codex.synthesis.get_settings", lambda: settings)
|
||||
monkeypatch.setattr(
|
||||
"codex.config.get_settings", lambda: settings, raising=False
|
||||
)
|
||||
return leads_path
|
||||
|
||||
|
||||
def test_synthesis_leads_invokes_three_finders(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_leads_dir: Path,
|
||||
) -> None:
|
||||
"""`codex synthesis leads --lib-path` calls connection + gap + improvement."""
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_connections(*, top_k: int, llm: object) -> list[Lead]:
|
||||
calls.append("connection")
|
||||
return [_grounded_lead("L-0001")]
|
||||
|
||||
def fake_gaps(
|
||||
*,
|
||||
lib_path: str | None = None,
|
||||
llm: object,
|
||||
topics: list[str] | None = None,
|
||||
) -> list[Lead]:
|
||||
calls.append("gap")
|
||||
return []
|
||||
|
||||
def fake_improvements(lib_path: str, *, top_k: int, llm: object) -> list[Lead]:
|
||||
calls.append("improvement")
|
||||
return []
|
||||
|
||||
monkeypatch.setattr("codex.synthesis.find_connections", fake_connections)
|
||||
monkeypatch.setattr("codex.synthesis.find_gaps", fake_gaps)
|
||||
monkeypatch.setattr("codex.synthesis.find_improvements", fake_improvements)
|
||||
monkeypatch.setattr("codex.synthesis.default_llm_client", lambda: object())
|
||||
|
||||
result = runner.invoke(app, ["synthesis", "leads", "--lib-path", "/tmp/fake"])
|
||||
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert calls == ["connection", "gap", "improvement"]
|
||||
assert (isolated_leads_dir / "grounded" / "L-0001.md").exists()
|
||||
|
||||
|
||||
def test_synthesis_leads_without_lib_path_skips_improvement(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_leads_dir: Path,
|
||||
) -> None:
|
||||
"""Without --lib-path, the improvement finder must be skipped (graceful)."""
|
||||
calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis.find_connections",
|
||||
lambda *, top_k, llm: (calls.append("connection"), [])[1],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis.find_gaps",
|
||||
lambda *, lib_path=None, llm, topics=None: (calls.append("gap"), [])[1],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis.find_improvements",
|
||||
lambda *args, **kwargs: (calls.append("improvement"), [])[1],
|
||||
)
|
||||
monkeypatch.setattr("codex.synthesis.default_llm_client", lambda: object())
|
||||
|
||||
result = runner.invoke(app, ["synthesis", "leads"])
|
||||
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert "improvement" not in calls
|
||||
assert "connection" in calls
|
||||
assert "gap" in calls
|
||||
|
||||
|
||||
def test_synthesis_conjectures_writes_to_conjectures_dir(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_leads_dir: Path,
|
||||
) -> None:
|
||||
"""`codex synthesis conjectures` lands files under leads/conjectures/."""
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis.propose_conjectures",
|
||||
lambda *, top_k, llm: [_conjecture("L-0042")],
|
||||
)
|
||||
monkeypatch.setattr("codex.synthesis.default_llm_client", lambda: object())
|
||||
|
||||
result = runner.invoke(app, ["synthesis", "conjectures"])
|
||||
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert (isolated_leads_dir / "conjectures" / "L-0042.md").exists()
|
||||
assert not (isolated_leads_dir / "grounded" / "L-0042.md").exists()
|
||||
|
||||
|
||||
def test_synthesis_report_separates_sections(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_leads_dir: Path,
|
||||
) -> None:
|
||||
"""`codex synthesis report` shows grounded and conjectures in distinct headers."""
|
||||
from codex.synthesis import write_leads
|
||||
|
||||
write_leads(
|
||||
[_grounded_lead("L-0001"), _conjecture("L-0002")],
|
||||
str(isolated_leads_dir),
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["synthesis", "report"])
|
||||
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert "GROUNDED LEADS" in result.stdout
|
||||
assert "CONJECTURES" in result.stdout
|
||||
# Visible separation: GROUNDED header strictly precedes CONJECTURES header
|
||||
assert result.stdout.index("GROUNDED LEADS") < result.stdout.index("CONJECTURES")
|
||||
# UNVERIFIED tag in the conjectures header
|
||||
assert "UNVERIFIED" in result.stdout
|
||||
|
||||
|
||||
def test_synthesis_report_empty(
|
||||
isolated_leads_dir: Path,
|
||||
) -> None:
|
||||
"""`codex synthesis report` with no leads still prints both sections."""
|
||||
result = runner.invoke(app, ["synthesis", "report"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "GROUNDED LEADS" in result.stdout
|
||||
assert "CONJECTURES" in result.stdout
|
||||
assert "(none)" in result.stdout
|
||||
Reference in New Issue
Block a user