feat(synthesis): Lead/Provenance model + grounded leads + conjecture generator

- codex/synthesis.py: Lead/Provenance dataclasses, find_connections/gaps/improvements,
  propose_conjectures (status=unverified, quarantined in leads/conjectures/),
  write_leads (grounded→leads/grounded/, conjectures→leads/conjectures/ HARD invariant)
- codex/cli.py: synthesis leads/conjectures/report command group
- codex/config.py: F-13 settings (leads_dir, synthesis_llm_*, synthesis_top_k,
  synthesis_min_grounded_ratio, synthesis_gap_min_coverage)
- tests/synthesis/: 42 tests covering model, grounding, conjecture invariant, quarantine, CLI
- spike/: F-13 spike script + output (live run attempted)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 08:37:42 +02:00
parent 967e6baa59
commit 8cf0cc7e01
14 changed files with 2139 additions and 0 deletions

View File

@@ -0,0 +1,113 @@
"""Tests for stage-5 conjecture generation.
HARD invariant: every conjecture MUST carry status='unverified', a
non-empty suggested_validation, and non-empty provenance.
"""
from __future__ import annotations
from pathlib import Path
from codex.synthesis import _parse_conjecture_output, propose_conjectures
from tests.synthesis.conftest import StubLLM
_VALID_CONJECTURE = (
"Title: A discrete cosmetic-surgery analogue exists for Yamabe flows.\n"
"Body: The combinatorial Yamabe flow [Luo2004 #chunk 0] is a discrete "
"analogue of conformal change. One could hope the cosmetic surgery "
"conjecture has a discrete formulation in this setting.\n"
"Validation: Search for a counter-example among simplicial 3-manifolds "
"with at most 12 tetrahedra using a CSP solver."
)
_MISSING_VALIDATION = (
"Title: A naive idea.\n"
"Body: Some thoughts that follow [Luo2004 #chunk 0]. No falsification path "
"is described."
)
def test_parse_valid_conjecture_output() -> None:
"""_parse_conjecture_output extracts title / body / validation."""
parsed = _parse_conjecture_output(_VALID_CONJECTURE)
assert parsed is not None
title, body, validation = parsed
assert title.startswith("A discrete cosmetic-surgery")
assert "Luo2004" in body
assert "counter-example" in validation
def test_parse_conjecture_missing_validation_returns_none() -> None:
"""Output without a Validation: line is rejected."""
assert _parse_conjecture_output(_MISSING_VALIDATION) is None
def test_parse_conjecture_empty_input() -> None:
"""Empty LLM output → None."""
assert _parse_conjecture_output("") is None
def test_propose_conjectures_marks_status_unverified(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Every emitted conjecture has status='unverified'."""
llm = StubLLM(_VALID_CONJECTURE)
leads = propose_conjectures(top_k=6, llm=llm)
assert len(leads) >= 1
for lead in leads:
assert lead.kind == "conjecture"
assert lead.status == "unverified"
def test_propose_conjectures_requires_validation(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Every emitted conjecture has a non-empty suggested_validation."""
llm = StubLLM(_VALID_CONJECTURE)
leads = propose_conjectures(top_k=6, llm=llm)
assert len(leads) >= 1
for lead in leads:
assert lead.suggested_validation.strip() != ""
def test_propose_conjectures_requires_provenance(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""Every conjecture has at least one Provenance entry (seed chunks)."""
llm = StubLLM(_VALID_CONJECTURE)
leads = propose_conjectures(top_k=6, llm=llm)
assert len(leads) >= 1
for lead in leads:
assert len(lead.provenance) >= 1
def test_propose_conjectures_drops_when_validation_missing(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""LLM output without a Validation: line is rejected (zero leads)."""
llm = StubLLM(_MISSING_VALIDATION)
leads = propose_conjectures(top_k=6, llm=llm)
assert leads == []
def test_propose_conjectures_handles_llm_failure(
mock_retrieve: None,
mock_paper_titles: None,
mock_settings: Path,
) -> None:
"""LLM raising → returns [] (mirrors F-12 graceful degradation)."""
class FailingLLM:
def generate(self, prompt: str, model: str) -> str:
raise ConnectionError("ollama unreachable")
leads = propose_conjectures(top_k=6, llm=FailingLLM())
assert leads == []