merge: reconcile data-quality roadmap with audit-remediation main (#12-15)

Resolved 4 conflicts: cli.py (R-D citing-coverage warning + main's small-corpus JSON/hub-title changes both kept); ingest.py (C-7 paper.id=caller-id pin first, then DQ-2 abstract recovery); test_ingest.py + test_openalex.py (kept both branches' tests).

C-7/DQ-5 overlap (both canonicalise papers.id): kept both layers — openalex._canonical_id normalizes fetch_paper's id (DQ-5); ingest pins papers.id to the caller's bare id (C-7); they converge on the bare canonical id. Fixed the auto-merged test_fetch_paper_success that had contradictory bare/URL assertions. 402 tests pass; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/graph/test_cli.py

@@ -28,9 +28,11 @@ def _make_conn_cm(conn: MagicMock):
 
 
 def _make_conn_with_paper_ids(*paper_ids: str) -> MagicMock:
-    """Mock conn that returns paper rows for SELECT id FROM papers."""
+    """Mock conn that returns paper rows for SELECT id, title FROM papers."""
     conn = MagicMock()
-    conn.execute.return_value.fetchall.return_value = [{"id": pid} for pid in paper_ids]
+    conn.execute.return_value.fetchall.return_value = [
+        {"id": pid, "title": f"Title of {pid}"} for pid in paper_ids
+    ]
     return conn
 
 

tests/ingest/test_ingest.py

@@ -12,6 +12,7 @@ from unittest.mock import MagicMock, patch
 
 import httpx
 import numpy as np
+import psycopg
 import pytest
 
 from codex.ingest import IngestResult, _make_bibkey, ingest_paper
@@ -161,6 +162,37 @@ def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
     mock_cr.assert_not_called()
 
 
+def test_ingest_canonicalises_paper_id_to_caller_arg() -> None:
+    """papers.id is the caller-supplied id, not OpenAlex's URL-form id (audit C-7).
+
+    Real OpenAlex returns the doi/id as a full URL; ingest must key the paper on
+    the bare id the caller passed (matching ingest scripts and CLI lookups), and
+    keep the OpenAlex work id only as openalex_id.
+    """
+    oa_paper = Paper(
+        id="https://doi.org/10.48550/arxiv.math/0603097",  # URL form, as OpenAlex emits
+        title="A Test Paper",
+        openalex_id="https://openalex.org/W4301005794",
+        authors=["Alice", "Bob"],
+        year=2008,
+        abstract="Test abstract.",
+    )
+    mock_conn = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=oa_paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper("math/0603097")
+
+    papers_params = mock_conn.execute.call_args_list[0][0][1]
+    assert papers_params["id"] == "math/0603097", "papers.id must be the caller's bare id"
+    assert papers_params["openalex_id"] == "https://openalex.org/W4301005794"
+    assert result.paper_id == "math/0603097"
+
+
 # ---------------------------------------------------------------------------
 # 2. test_ingest_paper_source_tex
 # ---------------------------------------------------------------------------
@@ -562,9 +594,11 @@ def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> No
     mock_conn.executemany = MagicMock()
     mock_conn.commit = MagicMock()
 
-    fake_formula = FormulaChunk(paper_id=paper.id, page=1, raw_latex=r"\alpha", context="test")
+    # Extractors derive paper_id from the PDF filename stem ("paper"), which differs
+    # from papers.id — ingest must insert papers.id, not this stem (audit C-10).
+    fake_formula = FormulaChunk(paper_id="paper", page=1, raw_latex=r"\alpha", context="test")
     fake_figure = FigureChunk(
-        paper_id=paper.id, page=1, image_path="/tmp/fig.png", caption="Figure 1."
+        paper_id="paper", page=1, image_path="/tmp/fig.png", caption="Figure 1."
     )
 
     mock_settings = MagicMock()
@@ -591,6 +625,16 @@ def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> No
     assert result.formulas_upserted == 1
     assert result.figures_upserted == 1
 
+    # C-10: formula/figure rows must be keyed on papers.id, not the extractor's
+    # filename-stem paper_id, or the FK to papers(id) is violated.
+    cursor = mock_conn.cursor.return_value.__enter__.return_value
+    inserted_paper_ids = {
+        row[0] for call in cursor.executemany.call_args_list for row in call[0][1]
+    }
+    assert inserted_paper_ids == {paper.id}, (
+        f"formula/figure inserts must use papers.id, got {inserted_paper_ids}"
+    )
+
 
 def test_ingest_paper_no_rich_skips_parsers(tmp_path: Any) -> None:
     """When rich=False (default), formula and figure parsers are NOT called."""
@@ -781,6 +825,69 @@ def test_ingest_doi_paper_upserts_bare_canonical_id() -> None:
     )
 
 
+def test_ingest_disambiguates_bibkey_on_unique_violation() -> None:
+    """A papers.bibkey UNIQUE collision is retried with an a/b/c suffix (audit C-15)."""
+    paper = _make_paper()  # bibkey auto-generates to "AliceBob2023"
+    mock_conn = MagicMock()
+    state = {"first": True}
+
+    def execute_side_effect(sql: str, params: Any = None) -> MagicMock:
+        # The first papers INSERT collides on the bibkey; the retry must succeed.
+        if "INSERT INTO papers" in sql and state["first"]:
+            state["first"] = False
+            raise psycopg.errors.UniqueViolation(
+                'duplicate key value violates unique constraint "papers_bibkey_key"'
+            )
+        return MagicMock()
+
+    mock_conn.execute.side_effect = execute_side_effect
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    papers_inserts = [
+        c for c in mock_conn.execute.call_args_list if "INSERT INTO papers" in c[0][0]
+    ]
+    assert len(papers_inserts) == 2, "expected one retry after the bibkey collision"
+    assert papers_inserts[0][0][1]["bibkey"] == "AliceBob2023"
+    assert papers_inserts[1][0][1]["bibkey"] == "AliceBob2023a"
+    mock_conn.rollback.assert_called_once()
+
+
+def test_ingest_openalex_collision_raises_clear_error() -> None:
+    """An openalex_id UNIQUE collision raises a clear, actionable error (audit C-15).
+
+    Unlike a bibkey clash (auto-suffixed), a shared openalex_id means two papers
+    claim the same OpenAlex work — a real data conflict that must surface, not the
+    raw psycopg error.
+    """
+    paper = _make_paper()
+    mock_conn = MagicMock()
+
+    def execute_side_effect(sql: str, params: Any = None) -> MagicMock:
+        if "INSERT INTO papers" in sql:
+            raise psycopg.errors.UniqueViolation(
+                'duplicate key value violates unique constraint "papers_openalex_id_key"'
+            )
+        return MagicMock()
+
+    mock_conn.execute.side_effect = execute_side_effect
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        pytest.raises(ValueError, match="openalex_id"),
+    ):
+        ingest_paper(paper.id)
+
+
 # ---------------------------------------------------------------------------
 
 

tests/mcp/test_http_auth.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 from unittest.mock import MagicMock, patch
 
 import pytest
+from pydantic import SecretStr
 
 # ---------------------------------------------------------------------------
 # _require_http_token
@@ -37,7 +38,7 @@ def test_require_http_token_returns_token_when_set() -> None:
     from codex.mcp_server import _require_http_token
 
     mock_settings = MagicMock()
-    mock_settings.mcp_auth_token = "super-secret-token"
+    mock_settings.mcp_auth_token = SecretStr("super-secret-token")
 
     with patch("codex.config.get_settings", return_value=mock_settings):
         result = _require_http_token()
@@ -97,7 +98,7 @@ def test_main_http_with_token_calls_uvicorn() -> None:
 
     mock_settings = MagicMock()
     mock_settings.mcp_transport = "http"
-    mock_settings.mcp_auth_token = "test-token"
+    mock_settings.mcp_auth_token = SecretStr("test-token")
     mock_settings.mcp_host = "127.0.0.1"
     mock_settings.mcp_port = 8765
 

tests/parsing/test_mathpix.py

@@ -10,6 +10,7 @@ from typing import Any
 from unittest.mock import MagicMock, patch
 
 import pytest
+from pydantic import SecretStr
 
 from codex.models import FormulaChunk
 
@@ -295,8 +296,8 @@ class TestRouteSelection:
     def test_mathpix_selected_when_creds_present(self) -> None:
         """extract_formulas calls MathPix backend when both creds are set."""
         mock_settings = MagicMock()
-        mock_settings.mathpix_app_id = "my_id"
-        mock_settings.mathpix_app_key = "my_key"
+        mock_settings.mathpix_app_id = SecretStr("my_id")
+        mock_settings.mathpix_app_key = SecretStr("my_key")
         mock_settings.pix2tex_fallback = True
 
         with (

tests/scaffold/test_config.py

@@ -8,7 +8,10 @@ from codex.config import Settings, get_settings
 def test_settings_defaults() -> None:
     """Settings() is constructable without env vars; expected defaults are set."""
     s = Settings()
-    assert s.database_url == "postgresql://researcher:change_me@localhost:5432/papers"
+    assert (
+        s.database_url.get_secret_value()
+        == "postgresql://researcher:change_me@localhost:5432/papers"
+    )
     assert s.embedding_model == "BAAI/bge-m3"
     assert s.embedding_dim == 1024
 
@@ -20,7 +23,7 @@ def test_settings_env_override(monkeypatch: object) -> None:
     mp: pytest.MonkeyPatch = monkeypatch  # type: ignore[assignment]
     mp.setenv("DATABASE_URL", "postgresql://x:y@h:5432/db")
     s = Settings()
-    assert s.database_url == "postgresql://x:y@h:5432/db"
+    assert s.database_url.get_secret_value() == "postgresql://x:y@h:5432/db"
 
 
 def test_get_settings_is_singleton() -> None:

tests/scaffold/test_db.py

@@ -33,3 +33,40 @@ def test_apply_schema_executes_sql() -> None:
 
     mock_conn.execute.assert_called_once()
     mock_conn.commit.assert_called_once()
+
+
+def test_schema_sql_is_idempotent() -> None:
+    """Every CREATE TABLE/INDEX in infra/schema.sql uses IF NOT EXISTS (audit M-1, T-2).
+
+    Re-applying schema.sql must be a safe no-op so the migration can run
+    repeatedly against an existing database.
+    """
+    import re
+    from pathlib import Path
+
+    schema = (Path(__file__).resolve().parents[2] / "infra" / "schema.sql").read_text()
+    # Drop line comments so commented-out example queries don't count.
+    code = "\n".join(line.split("--", 1)[0] for line in schema.splitlines())
+    non_idempotent = re.findall(
+        r"CREATE\s+(?:UNIQUE\s+)?(?:TABLE|INDEX)\s+(?!IF\s+NOT\s+EXISTS)(\w+)",
+        code,
+        re.IGNORECASE,
+    )
+    assert not non_idempotent, f"schema.sql has non-idempotent CREATE statements: {non_idempotent}"
+
+
+def test_migrate_command_reports_privilege_error() -> None:
+    """`codex migrate` surfaces InsufficientPrivilege with guidance and exits 1 (M-1)."""
+    import psycopg
+    from typer.testing import CliRunner
+
+    from codex.cli import app
+
+    def boom(*args: object, **kwargs: object) -> object:
+        raise psycopg.errors.InsufficientPrivilege("must be owner of table chunks")
+
+    with patch("psycopg.connect", boom):
+        result = CliRunner().invoke(app, ["migrate"])
+
+    assert result.exit_code == 1
+    assert "MIGRATION_DATABASE_URL" in result.output

tests/sources/test_openalex.py

@@ -15,8 +15,8 @@ from codex.sources.openalex import _canonical_id, _normalize_doi, _resolve_id
 
 _SAMPLE_WORK = {
     "id": "https://openalex.org/W2741809807",
-    # Real OpenAlex returns the DOI as a full URL (T-3: the old bare fixture hid
-    # the URL-form behaviour that DQ-5 is about).
+    # Real OpenAlex returns the DOI as a full URL, not a bare DOI (audit T-3 / DQ-5
+    # — the old bare fixture hid the URL-form behaviour both fixes address).
     "doi": "https://doi.org/10.1145/3592430",
     "title": "Conformal Prediction: A Review",
     "publication_year": 2023,
@@ -168,8 +168,11 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
 
     assert paper is not None
     assert isinstance(paper, Paper)
-    # DQ-5 / C-7: paper.id is the canonical BARE DOI, not the full URL OpenAlex
-    # returns. This is what makes the ingest ON CONFLICT (id) upsert idempotent.
+    # DQ-5: openalex._canonical_id normalizes OpenAlex's URL-form doi to the BARE
+    # lower-cased DOI at the source layer; ingest additionally pins papers.id to the
+    # caller's id (audit C-7). Both converge on the bare canonical id — which is what
+    # makes the ingest ON CONFLICT (id) upsert idempotent. (T-3: the fixture is
+    # URL-form so this mapping can't silently regress.)
     assert paper.id == "10.1145/3592430"
     assert paper.title == "Conformal Prediction: A Review"
     assert paper.year == 2023

tests/synthesis/test_gaps.py

@@ -112,10 +112,35 @@ def test_find_gaps_coverage_map_low_coverage_triggers_llm(
         "For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
         "V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
     )
-    leads = find_gaps(llm=StubLLM(grounded_body))
+    # Coverage gaps are opt-in (audit R-8): pass the topic explicitly.
+    leads = find_gaps(llm=StubLLM(grounded_body), topics=["hyperbolic volume formula"])
     assert any(lead.kind == "gap" for lead in leads)
 
 
+def test_find_gaps_no_coverage_gaps_without_explicit_topics(
+    monkeypatch: pytest.MonkeyPatch,
+    mock_paper_titles: None,
+    mock_settings: Path,
+) -> None:
+    """Without explicit topics, paper titles do NOT auto-generate coverage gaps (R-8)."""
+    sparse_chunks: list[dict[str, Any]] = [
+        {
+            "id": 10,
+            "paper_id": "springborn-2008",
+            "ord": 16,
+            "content": "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)",
+            "bibkey": "Springborn2008",
+        },
+    ]
+    monkeypatch.setattr(
+        "codex.synthesis._retrieve_chunks",
+        lambda queries, top_k: [dict(c) for c in sparse_chunks],
+    )
+    # No lib_path, no topics → the coverage map must not run despite sparse coverage.
+    leads = find_gaps(llm=StubLLM("a gap. [Springborn2008 #chunk 16]"))
+    assert leads == []
+
+
 def test_find_gaps_explicit_topics_override_default(
     monkeypatch: pytest.MonkeyPatch,
     mock_paper_titles: None,

tests/synthesis/test_grounded.py

@@ -76,7 +76,7 @@ def test_finalise_grounded_lead_succeeds_when_grounded(
     )
     assert lead is not None
     assert lead.kind == "connection"
-    assert lead.id == "L-0001"
+    assert lead.id == "L-C-0001"  # kind-prefixed id (audit C-11)
     assert lead.confidence >= 0.5
     assert lead.status == "unverified"
     assert lead.suggested_validation == ""
@@ -148,7 +148,8 @@ def test_find_gaps_topic_with_no_chunks(
 ) -> None:
     """When retrieval returns no chunks, a gap lead is emitted directly."""
     monkeypatch.setattr("codex.synthesis._retrieve_chunks", lambda queries, top_k: [])
-    leads = find_gaps(llm=StubLLM(""))
+    # Coverage gaps are opt-in now (audit R-8): pass an explicit topic.
+    leads = find_gaps(llm=StubLLM(""), topics=["nonexistent topic"])
     assert len(leads) >= 1
     assert all(lead.kind == "gap" for lead in leads)
     # Direct gap leads carry a validation hint (re-ingest)

tests/synthesis/test_model.py

@@ -92,9 +92,21 @@ def test_lead_empty_provenance_raises() -> None:
 
 
 def test_lead_id_format_helper() -> None:
-    """The internal _make_lead_id helper produces zero-padded L-XXXX strings."""
+    """_make_lead_id produces a kind-prefixed, zero-padded L-<K>-XXXX string."""
     from codex.synthesis import _make_lead_id
 
-    assert _make_lead_id(1) == "L-0001"
-    assert _make_lead_id(42) == "L-0042"
-    assert _make_lead_id(9999) == "L-9999"
+    assert _make_lead_id(1, "connection") == "L-C-0001"
+    assert _make_lead_id(42, "gap") == "L-G-0042"
+    assert _make_lead_id(9999, "improvement") == "L-I-9999"
+    assert _make_lead_id(1, "conjecture") == "L-X-0001"
+
+
+def test_lead_id_namespaced_per_kind_no_collision() -> None:
+    """Regression for audit C-11: each stage numbers from 1, so without a kind
+    prefix connection/gap/improvement all collide on L-0001 and overwrite one
+    another in grounded/. The prefix must keep same-seq ids distinct.
+    """
+    from codex.synthesis import _make_lead_id
+
+    ids = {_make_lead_id(1, k) for k in ("connection", "gap", "improvement", "conjecture")}
+    assert len(ids) == 4, f"same-seq ids must stay distinct across kinds, got {ids}"

tests/wiki/test_compile.py

@@ -213,6 +213,47 @@ def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored()
     )
 
 
+def test_grounding_guard_robust_to_trailing_comma() -> None:
+    """A faithful 5-word claim grounds despite a trailing comma (audit R-1).
+
+    The chunk says "... the volume function V0 is strictly concave on the angle
+    domain." Before edge-punctuation stripping, the claim token "concave," != the
+    chunk token "concave", so the only content-5-gram failed to match and the
+    claim was wrongly marked ungrounded.
+    """
+    claim = Claim(
+        text="the volume function V0 is strictly concave,",
+        bibkey="Springborn2008",
+        locator="chunk 16",
+    )
+    result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
+    assert result[0].grounded is True
+
+
+def test_grounding_guard_no_substring_bleed_across_tokens() -> None:
+    """A 5-gram must match whole tokens, not bleed into a longer chunk word (audit C-6).
+
+    The chunk contains 'subset'; a claim whose first token is 'set' must NOT ground
+    by substring-matching the suffix of 'subset'.
+    """
+    chunks = [
+        {
+            "id": 99,
+            "paper_id": "p",
+            "ord": 0,
+            "bibkey": "TestBib",
+            "content": "the subset conformal map energy functional is minimized here",
+        }
+    ]
+    claim = Claim(
+        text="the set conformal map energy functional",
+        bibkey="TestBib",
+        locator="chunk 0",
+    )
+    result = _run_grounding_guard([claim], chunks)
+    assert result[0].grounded is False
+
+
 # ---------------------------------------------------------------------------
 # _inject_cross_refs
 # ---------------------------------------------------------------------------
@@ -274,6 +315,19 @@ def test_inject_cross_refs_skips_inline_code() -> None:
     assert "[[circle-packing]]" in result  # bare occurrence replaced
 
 
+def test_inject_cross_refs_skips_math_spans() -> None:
+    """Concept names inside LaTeX math are NOT rewritten to [[slug]] links (audit R-9)."""
+    concepts = [
+        Concept(slug="energy", title="Energy", aliases=[]),
+        FIXTURE_CONCEPT,
+    ]
+    text = "Prose Energy here. Math $x = Energy^2$ and inline \\(Energy\\) stay literal."
+    result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
+    assert "$x = Energy^2$" in result  # display/inline $...$ math untouched
+    assert "\\(Energy\\)" in result  # \(...\) math untouched
+    assert "Prose [[energy]] here." in result  # prose occurrence still linked
+
+
 def test_inject_cross_refs_full_list_even_when_single_concept() -> None:
     """Cross-ref injection uses the full concept list, not just the compiled concept."""
     concepts = [
@@ -504,6 +558,47 @@ def test_log_append_records_skipped(
     assert "circle-packing" in content
 
 
+# ---------------------------------------------------------------------------
+# Audit C-4: zero-citation pages must NOT bypass the quarantine
+# ---------------------------------------------------------------------------
+
+
+def test_zero_citation_page_is_quarantined_not_published(
+    mock_retrieve: None,
+    mock_formulas: None,
+    mock_settings: Path,
+) -> None:
+    """A confident-but-uncited LLM response yields zero parseable claims
+    (grounding_rate 0.0). It must be quarantined — not written to wiki/ and
+    marked compiled. Regression for audit C-4: the old ``total_claims > 0``
+    guard let zero-claim (and LLM-outage empty) pages fall through to wiki/.
+    """
+    from codex.wiki import compile_all
+
+    (mock_settings / "concepts.yaml").write_text(
+        textwrap.dedent(
+            """\
+            concepts:
+              - slug: lobachevsky-function
+                title: Lobachevsky Function
+                aliases: [Milnor Lobachevsky]
+            """
+        ),
+        encoding="utf-8",
+    )
+    # No [BibKey] citations anywhere → zero claims → grounding_rate 0.0.
+    uncited = MockLLM(
+        "The Lobachevsky function is obviously central to all of geometry. "
+        "Every result follows from it. This paragraph cites nothing at all."
+    )
+
+    report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings))
+
+    assert "lobachevsky-function" in report.quarantined
+    assert "lobachevsky-function" not in report.compiled
+    assert not (mock_settings / "lobachevsky-function.md").exists()
+
+
 # ---------------------------------------------------------------------------
 # Idempotency: second compile without chunk change → no rewrite
 # ---------------------------------------------------------------------------