feat(synthesis): Lead/Provenance model + grounded leads + conjecture generator
- codex/synthesis.py: Lead/Provenance dataclasses, find_connections/gaps/improvements, propose_conjectures (status=unverified, quarantined in leads/conjectures/), write_leads (grounded→leads/grounded/, conjectures→leads/conjectures/ HARD invariant) - codex/cli.py: synthesis leads/conjectures/report command group - codex/config.py: F-13 settings (leads_dir, synthesis_llm_*, synthesis_top_k, synthesis_min_grounded_ratio, synthesis_gap_min_coverage) - tests/synthesis/: 42 tests covering model, grounding, conjecture invariant, quarantine, CLI - spike/: F-13 spike script + output (live run attempted) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
113
tests/synthesis/test_conjecture.py
Normal file
113
tests/synthesis/test_conjecture.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Tests for stage-5 conjecture generation.
|
||||
|
||||
HARD invariant: every conjecture MUST carry status='unverified', a
|
||||
non-empty suggested_validation, and non-empty provenance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from codex.synthesis import _parse_conjecture_output, propose_conjectures
|
||||
from tests.synthesis.conftest import StubLLM
|
||||
|
||||
_VALID_CONJECTURE = (
|
||||
"Title: A discrete cosmetic-surgery analogue exists for Yamabe flows.\n"
|
||||
"Body: The combinatorial Yamabe flow [Luo2004 #chunk 0] is a discrete "
|
||||
"analogue of conformal change. One could hope the cosmetic surgery "
|
||||
"conjecture has a discrete formulation in this setting.\n"
|
||||
"Validation: Search for a counter-example among simplicial 3-manifolds "
|
||||
"with at most 12 tetrahedra using a CSP solver."
|
||||
)
|
||||
|
||||
_MISSING_VALIDATION = (
|
||||
"Title: A naive idea.\n"
|
||||
"Body: Some thoughts that follow [Luo2004 #chunk 0]. No falsification path "
|
||||
"is described."
|
||||
)
|
||||
|
||||
|
||||
def test_parse_valid_conjecture_output() -> None:
|
||||
"""_parse_conjecture_output extracts title / body / validation."""
|
||||
parsed = _parse_conjecture_output(_VALID_CONJECTURE)
|
||||
assert parsed is not None
|
||||
title, body, validation = parsed
|
||||
assert title.startswith("A discrete cosmetic-surgery")
|
||||
assert "Luo2004" in body
|
||||
assert "counter-example" in validation
|
||||
|
||||
|
||||
def test_parse_conjecture_missing_validation_returns_none() -> None:
|
||||
"""Output without a Validation: line is rejected."""
|
||||
assert _parse_conjecture_output(_MISSING_VALIDATION) is None
|
||||
|
||||
|
||||
def test_parse_conjecture_empty_input() -> None:
|
||||
"""Empty LLM output → None."""
|
||||
assert _parse_conjecture_output("") is None
|
||||
|
||||
|
||||
def test_propose_conjectures_marks_status_unverified(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""Every emitted conjecture has status='unverified'."""
|
||||
llm = StubLLM(_VALID_CONJECTURE)
|
||||
leads = propose_conjectures(top_k=6, llm=llm)
|
||||
assert len(leads) >= 1
|
||||
for lead in leads:
|
||||
assert lead.kind == "conjecture"
|
||||
assert lead.status == "unverified"
|
||||
|
||||
|
||||
def test_propose_conjectures_requires_validation(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""Every emitted conjecture has a non-empty suggested_validation."""
|
||||
llm = StubLLM(_VALID_CONJECTURE)
|
||||
leads = propose_conjectures(top_k=6, llm=llm)
|
||||
assert len(leads) >= 1
|
||||
for lead in leads:
|
||||
assert lead.suggested_validation.strip() != ""
|
||||
|
||||
|
||||
def test_propose_conjectures_requires_provenance(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""Every conjecture has at least one Provenance entry (seed chunks)."""
|
||||
llm = StubLLM(_VALID_CONJECTURE)
|
||||
leads = propose_conjectures(top_k=6, llm=llm)
|
||||
assert len(leads) >= 1
|
||||
for lead in leads:
|
||||
assert len(lead.provenance) >= 1
|
||||
|
||||
|
||||
def test_propose_conjectures_drops_when_validation_missing(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""LLM output without a Validation: line is rejected (zero leads)."""
|
||||
llm = StubLLM(_MISSING_VALIDATION)
|
||||
leads = propose_conjectures(top_k=6, llm=llm)
|
||||
assert leads == []
|
||||
|
||||
|
||||
def test_propose_conjectures_handles_llm_failure(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""LLM raising → returns [] (mirrors F-12 graceful degradation)."""
|
||||
|
||||
class FailingLLM:
|
||||
def generate(self, prompt: str, model: str) -> str:
|
||||
raise ConnectionError("ollama unreachable")
|
||||
|
||||
leads = propose_conjectures(top_k=6, llm=FailingLLM())
|
||||
assert leads == []
|
||||
Reference in New Issue
Block a user