feat(synthesis): Lead/Provenance model + grounded leads + conjecture generator
- codex/synthesis.py: Lead/Provenance dataclasses, find_connections/gaps/improvements, propose_conjectures (status=unverified, quarantined in leads/conjectures/), write_leads (grounded→leads/grounded/, conjectures→leads/conjectures/ HARD invariant) - codex/cli.py: synthesis leads/conjectures/report command group - codex/config.py: F-13 settings (leads_dir, synthesis_llm_*, synthesis_top_k, synthesis_min_grounded_ratio, synthesis_gap_min_coverage) - tests/synthesis/: 42 tests covering model, grounding, conjecture invariant, quarantine, CLI - spike/: F-13 spike script + output (live run attempted) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
175
tests/synthesis/test_grounded.py
Normal file
175
tests/synthesis/test_grounded.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""Tests for grounded leads (connection / gap / improvement).
|
||||
|
||||
The Grounding-Guard is re-used from :mod:`codex.wiki` (F-12). These tests
|
||||
verify that ungrounded claims are either dropped or annotated with ⚠,
|
||||
and that grounded statements survive intact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from codex.synthesis import (
|
||||
_finalise_grounded_lead,
|
||||
_mark_ungrounded,
|
||||
find_connections,
|
||||
find_gaps,
|
||||
)
|
||||
from tests.synthesis.conftest import StubLLM
|
||||
|
||||
_GROUNDED_CONNECTION = (
|
||||
"Both papers build hyperbolic volumes from the Lobachevsky function.\n"
|
||||
"For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
|
||||
"V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
|
||||
"A discrete conformal map is defined by logarithmic scale factors u_i at "
|
||||
"each vertex such that the edge lengths satisfy a compatibility condition. "
|
||||
"[BobenkoPinkallSpringborn2015 #chunk 3]\n"
|
||||
)
|
||||
|
||||
_UNGROUNDED_CONNECTION = (
|
||||
"Both papers prove the Riemann hypothesis via the Lobachevsky function. "
|
||||
"[Springborn2008 #chunk 99]\n"
|
||||
)
|
||||
|
||||
|
||||
def test_finalise_grounded_lead_returns_none_without_claims(
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
) -> None:
|
||||
"""Body with no inline citations yields no grounded lead."""
|
||||
lead = _finalise_grounded_lead(
|
||||
seq=1,
|
||||
kind="connection",
|
||||
title="t",
|
||||
body="A plain statement with no citation marker.",
|
||||
chunks=fixture_chunks,
|
||||
min_grounded_ratio=0.5,
|
||||
)
|
||||
assert lead is None
|
||||
|
||||
|
||||
def test_finalise_grounded_lead_drops_when_ratio_below_floor(
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
) -> None:
|
||||
"""All-ungrounded body → None (not silently promoted)."""
|
||||
lead = _finalise_grounded_lead(
|
||||
seq=1,
|
||||
kind="connection",
|
||||
title="t",
|
||||
body=_UNGROUNDED_CONNECTION,
|
||||
chunks=fixture_chunks,
|
||||
min_grounded_ratio=0.5,
|
||||
)
|
||||
assert lead is None
|
||||
|
||||
|
||||
def test_finalise_grounded_lead_succeeds_when_grounded(
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
) -> None:
|
||||
"""Grounded body passes the floor and returns a populated Lead."""
|
||||
lead = _finalise_grounded_lead(
|
||||
seq=1,
|
||||
kind="connection",
|
||||
title="Both papers share a building block",
|
||||
body=_GROUNDED_CONNECTION,
|
||||
chunks=fixture_chunks,
|
||||
min_grounded_ratio=0.5,
|
||||
)
|
||||
assert lead is not None
|
||||
assert lead.kind == "connection"
|
||||
assert lead.id == "L-0001"
|
||||
assert lead.confidence >= 0.5
|
||||
assert lead.status == "unverified"
|
||||
assert lead.suggested_validation == ""
|
||||
# All claims should be grounded
|
||||
assert all(c.grounded for c in lead.claims)
|
||||
|
||||
|
||||
def test_mark_ungrounded_prefixes_warning() -> None:
|
||||
"""_mark_ungrounded prefixes ⚠ to the matching claim text once."""
|
||||
from codex.wiki import Claim
|
||||
|
||||
body = "Some fact. [X2020 #c 1]\nOther fact. [Y2021 #c 2]\n"
|
||||
claims = [
|
||||
Claim(text="Some fact", bibkey="X2020", locator="c 1", grounded=False),
|
||||
Claim(text="Other fact", bibkey="Y2021", locator="c 2", grounded=True),
|
||||
]
|
||||
out = _mark_ungrounded(body, claims)
|
||||
assert "⚠ Some fact" in out
|
||||
# Grounded claim is not annotated
|
||||
assert "⚠ Other fact" not in out
|
||||
|
||||
|
||||
def test_find_connections_emits_grounded_lead(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""End-to-end: with a grounded LLM response, a connection lead is produced."""
|
||||
llm = StubLLM(_GROUNDED_CONNECTION)
|
||||
leads = find_connections(top_k=6, llm=llm)
|
||||
assert len(leads) >= 1
|
||||
lead = leads[0]
|
||||
assert lead.kind == "connection"
|
||||
assert lead.provenance, "grounded lead must carry provenance"
|
||||
# No ⚠ should appear because all claims grounded
|
||||
assert "⚠" not in lead.body
|
||||
|
||||
|
||||
def test_find_connections_drops_ungrounded_response(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""Ungrounded LLM responses are dropped, not emitted with ⚠."""
|
||||
llm = StubLLM(_UNGROUNDED_CONNECTION)
|
||||
leads = find_connections(top_k=6, llm=llm)
|
||||
assert leads == []
|
||||
|
||||
|
||||
def test_find_connections_handles_llm_failure(
|
||||
mock_retrieve: None,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""LLM raising → returns [] (mirrors F-12 graceful degradation)."""
|
||||
|
||||
class FailingLLM:
|
||||
def generate(self, prompt: str, model: str) -> str:
|
||||
raise ConnectionError("ollama unreachable")
|
||||
|
||||
leads = find_connections(top_k=6, llm=FailingLLM())
|
||||
assert leads == []
|
||||
|
||||
|
||||
def test_find_gaps_topic_with_no_chunks(
|
||||
monkeypatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
) -> None:
|
||||
"""When retrieval returns no chunks, a gap lead is emitted directly."""
|
||||
monkeypatch.setattr("codex.synthesis._retrieve_chunks", lambda queries, top_k: [])
|
||||
leads = find_gaps(llm=StubLLM(""))
|
||||
assert len(leads) >= 1
|
||||
assert all(lead.kind == "gap" for lead in leads)
|
||||
# Direct gap leads carry a validation hint (re-ingest)
|
||||
assert all(lead.suggested_validation for lead in leads)
|
||||
|
||||
|
||||
def test_find_gaps_well_covered_topic_skipped(
|
||||
monkeypatch,
|
||||
mock_paper_titles: None,
|
||||
mock_settings: Path,
|
||||
fixture_chunks: list[dict[str, object]],
|
||||
) -> None:
|
||||
"""Topics covered by enough bibkeys are NOT emitted as gaps.
|
||||
|
||||
Three distinct bibkeys in the fixture > synthesis_gap_min_coverage=2,
|
||||
so every topic is well-covered and gets skipped.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"codex.synthesis._retrieve_chunks",
|
||||
lambda queries, top_k: [dict(c) for c in fixture_chunks],
|
||||
)
|
||||
leads = find_gaps(llm=StubLLM(""))
|
||||
# All probes well-covered → no gap leads
|
||||
assert leads == []
|
||||
Reference in New Issue
Block a user