docs: Cold-Start-Prompt fuer Evaluate/Review/Release ergaenzt

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -5,6 +5,12 @@
 # Example: postgresql://researcher:change_me@localhost:5432/papers
 DATABASE_URL=postgresql://researcher:change_me@localhost:5432/papers
 
+# Optional: privileged connection used by `codex migrate` to apply schema DDL.
+# The app DATABASE_URL is often a least-privilege DML role that cannot CREATE/
+# ALTER owner-held tables; point this at an owner/superuser connection.
+# Falls back to DATABASE_URL when unset.
+# MIGRATION_DATABASE_URL=postgresql://postgres:change_me@localhost:5432/papers
+
 # GROBID service base URL (containerised — see infra/docker-compose.yml)
 GROBID_URL=http://localhost:8070
 
@@ -24,6 +30,11 @@ EMBEDDING_DIM=1024
 # Required by OpenAlex ToS when making automated requests.
 OPENALEX_MAILTO=you@example.com
 
-# Nougat HTTP-Server (Jetson: http://192.168.178.103:8080 | lokal: http://localhost:8080)
+# Nougat HTTP-Server (Jetson: http://jetson.local:8080 | lokal: http://localhost:8080)
 # Used by `codex ingest <id> --rich` for full-PDF Mathpix Markdown output.
 NOUGAT_URL=http://localhost:8080
+
+# F-16 Chunk Quality Gate thresholds (all optional — defaults shown)
+CHUNK_MIN_CHARS=60          # Discard chunks shorter than this many characters
+CHUNK_MIN_ALPHA_RATIO=0.40  # Discard chunks with < 40% alphabetic characters
+CHUNK_MAX_BIB_SCORE=0.70    # Discard chunks scoring above this bibliography threshold

.gitignore

@@ -18,6 +18,8 @@ build/
 
 # Environment secrets — NEVER commit
 .env
+.env.*
+!.env.example
 
 # Type-checker and linter caches
 .mypy_cache/
@@ -36,3 +38,6 @@ htmlcov/
 
 # Claude Code — settings.json IS committed (team-wide); only personal overrides ignored
 .claude/settings.local.json
+
+# Generated wiki index (regenerated by `codex wiki compile`; concepts.yaml is the source)
+wiki/index.md

README.md

@@ -19,13 +19,11 @@ $EDITOR .env
 # 3. Install Python dependencies (requires uv)
 uv sync
 
-# 4. Apply the database schema (first run only)
+# 4. Apply the database schema (idempotent — safe to re-run after upgrades)
-uv run python -c "
+#    Needs a role that can run DDL. If DATABASE_URL is a least-privilege app
-from codex.db import get_conn, apply_schema
+#    role, point MIGRATION_DATABASE_URL at an owner/superuser connection first:
-with get_conn() as conn:
+#      MIGRATION_DATABASE_URL=postgresql://postgres:...@host:5432/papers
-    apply_schema(conn)
+uv run codex migrate
-print('Schema applied.')
-"
 ```
 
 ---
@@ -34,7 +32,8 @@ print('Schema applied.')
 
 | Variable | Default | Description |
 |---|---|---|
-| `DATABASE_URL` | `postgresql://researcher:change_me@localhost:5432/papers` | libpq connection string |
+| `DATABASE_URL` | `postgresql://researcher:change_me@localhost:5432/papers` | libpq connection string (app role; DML) |
+| `MIGRATION_DATABASE_URL` | *(falls back to `DATABASE_URL`)* | Privileged connection used by `codex migrate` for schema DDL (owner/superuser) |
 | `GROBID_URL` | `http://localhost:8070` | GROBID HTTP API base URL |
 | `OLLAMA_BASE_URL` | `http://localhost:11434` | Local Ollama endpoint (optional) |
 | `EMBEDDING_MODEL` | `BAAI/bge-m3` | sentence-transformers model name |

codex/cli.py

@@ -13,6 +13,8 @@ discover_app = typer.Typer(help="Discovery queries over the citation graph.")
 prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
 wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
 synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
+quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
+graph_app = typer.Typer(help="Citation graph analytics: PageRank, coupling, dangling leads (F-15).")
 # F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
 search_app = typer.Typer(
     help="Search: semantic paper search and formula full-text search.",
@@ -22,9 +24,42 @@ app.add_typer(discover_app, name="discover")
 app.add_typer(prov_app, name="provenance")
 app.add_typer(wiki_app, name="wiki")
 app.add_typer(synth_app, name="synthesis")
+app.add_typer(quality_app, name="quality")
+app.add_typer(graph_app, name="graph")
 app.add_typer(search_app, name="search")
 
 
+@app.command()
+def migrate() -> None:
+    """Apply infra/schema.sql to bring the database schema up to date (idempotent).
+
+    Must connect as a role that can run DDL (owns / can CREATE + ALTER the tables).
+    The application's DATABASE_URL is usually a least-privilege DML role and will
+    fail with InsufficientPrivilege; set MIGRATION_DATABASE_URL to a privileged
+    (owner/superuser) connection (audit M-1).
+    """
+    import psycopg
+    import psycopg.rows
+
+    from codex.config import get_settings
+    from codex.db import apply_schema
+
+    settings = get_settings()
+    url = (settings.migration_database_url or settings.database_url).get_secret_value()
+    try:
+        with psycopg.connect(url, row_factory=psycopg.rows.dict_row) as conn:
+            apply_schema(conn)
+    except psycopg.errors.InsufficientPrivilege as exc:
+        typer.echo(
+            "ERROR: schema migration needs a role that owns the tables "
+            "(CREATE/ALTER). Set MIGRATION_DATABASE_URL to a privileged connection "
+            f"and retry. ({exc})",
+            err=True,
+        )
+        raise typer.Exit(1) from exc
+    typer.echo("Schema applied — database is up to date.")
+
+
 @app.command()
 def ingest(
     paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
@@ -64,8 +99,15 @@ def search_callback(ctx: typer.Context) -> None:
 def search_paper(
     query: str = typer.Argument(..., help="Natural-language search query"),
     limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
+    cite_boost: bool = typer.Option(
+        False,
+        "--cite-boost",
+        help="Re-rank the top results by citation PageRank — a within-page "
+        "tie-breaker, not a hard re-ranking (F-15). Graceful when corpus < 5 papers.",
+    ),
 ) -> None:
     """Semantic similarity search over paper abstracts."""
+    from codex.config import get_settings
     from codex.db import get_conn
     from codex.embed import get_embedder
 
@@ -74,14 +116,33 @@ def search_paper(
         rows = conn.execute(
             """
             SELECT id, title, year,
-                   abstract_emb <-> %(emb)s AS distance
+                   abstract_emb <-> %(emb)s::vector AS distance
             FROM   papers
             WHERE  abstract_emb IS NOT NULL
-            ORDER  BY abstract_emb <-> %(emb)s
+            ORDER  BY abstract_emb <-> %(emb)s::vector
             LIMIT  %(limit)s
             """,
             {"emb": emb, "limit": limit},
         ).fetchall()
+
+        if cite_boost and rows:
+            from codex.graph import build_citation_graph, citation_pagerank
+
+            settings = get_settings()
+            graph = build_citation_graph(conn)
+            pr = citation_pagerank(graph)
+            alpha = settings.graph_cite_boost_alpha
+            results = []
+            for row in rows:
+                pr_score = pr.get(row["id"], 0.0)
+                # distance is lower=better; divide to reduce distance for high-PR papers
+                boosted = row["distance"] / (1.0 + alpha * pr_score)
+                results.append((boosted, row))
+            results.sort(key=lambda x: x[0])
+            for boosted_dist, row in results:
+                typer.echo(f"[{boosted_dist:.3f}] {row['id']} ({row['year']}) — {row['title']}")
+            return
+
     for row in rows:
         typer.echo(f"[{row['distance']:.3f}] {row['id']} ({row['year']}) — {row['title']}")
 
@@ -128,6 +189,24 @@ def discover_leads(
         typer.echo(f"{item['pull']:>4}×  {item['cited_id']}")
 
 
+@discover_app.command("recommend")
+def discover_recommend(
+    paper_id: str = typer.Argument(
+        ..., help="Semantic Scholar paper ID (or arXiv:/DOI: prefixed)."
+    ),
+    limit: int = typer.Option(20, "--limit", "-n", help="Number of recommendations."),
+) -> None:
+    """Recommended papers for PAPER_ID via Semantic Scholar."""
+    from codex.sources.semanticscholar import fetch_recommendations
+
+    rec = fetch_recommendations(paper_id, limit=limit)
+    if not rec:
+        typer.echo(f"No recommendations found for {paper_id}.")
+        return
+    for pid in rec:
+        typer.echo(pid)
+
+
 @discover_app.command("citing")
 def discover_citing(paper_id: str = typer.Argument(...)) -> None:
     """List papers that cite PAPER_ID."""
@@ -362,7 +441,15 @@ def synthesis_leads(
         "--lib-path",
         help=(
             "Path to a C++ source tree. Enables improvement leads (@cite-aware) and "
-            "code-vs-corpus gap detection. Without it only connection + topic gaps run."
+            "code-vs-corpus gap detection."
+        ),
+    ),
+    topic: list[str] | None = typer.Option(  # noqa: B008
+        None,
+        "--topic",
+        help=(
+            "Topic to check for coverage gaps (repeatable). Opt-in: without --topic "
+            "no coverage-map gaps run (avoids per-paper false positives, audit R-8)."
         ),
     ),
     output_dir: Optional[str] = typer.Option(  # noqa: UP045
@@ -384,7 +471,7 @@ def synthesis_leads(
     llm = default_llm_client()
 
     connections = find_connections(top_k=settings.synthesis_top_k, llm=llm)
-    gaps = find_gaps(lib_path=lib_path, llm=llm)
+    gaps = find_gaps(lib_path=lib_path, llm=llm, topics=topic or None)
     improvements = (
         find_improvements(lib_path, top_k=settings.synthesis_top_k, llm=llm) if lib_path else []
     )
@@ -419,9 +506,7 @@ def synthesis_conjectures(
     conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm)
     write_leads(conjectures, leads_dir)
 
-    typer.echo(
+    typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}")
-        f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}"
-    )
 
 
 @synth_app.command("report")
@@ -454,13 +539,188 @@ def synthesis_report(
 
     typer.echo("")
     typer.echo("=" * 72)
-    typer.echo(
+    typer.echo(f"CONJECTURES  ({len(conj_files)})  → {conj_dir}  [UNVERIFIED — never in wiki/]")
-        f"CONJECTURES  ({len(conj_files)})  → {conj_dir}  "
-        "[UNVERIFIED — never in wiki/]"
-    )
     typer.echo("=" * 72)
     if not conj_files:
         typer.echo("(none)")
     for f in conj_files:
         first = f.read_text(encoding="utf-8").splitlines()[0]
         typer.echo(f"  {first}")
+
+
+# ---------------------------------------------------------------------------
+# F-16 — quality sub-commands
+# ---------------------------------------------------------------------------
+
+
+@quality_app.command("run")
+def quality_run(
+    paper_id: Optional[str] = typer.Option(  # noqa: UP045
+        None,
+        "--paper-id",
+        help="Restrict pass to one paper (omit to process all papers).",
+    ),
+) -> None:
+    """Apply quality gate + section classification to existing chunks.
+
+    Iterates over chunks in the database, removes those that fail the
+    quality thresholds, and back-fills the ``section`` column for the rest.
+    """
+    from codex.config import get_settings
+    from codex.db import get_conn
+    from codex.quality import run_quality_pass
+
+    settings = get_settings()
+    with get_conn() as conn:
+        stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings)
+
+    scope = f"paper {paper_id}" if paper_id else "all papers"
+    typer.echo(
+        f"Quality pass ({scope}): "
+        f"{stats['kept']} kept, "
+        f"{stats['removed']} removed, "
+        f"{stats['tagged']} section-tagged"
+    )
+
+
+# ---------------------------------------------------------------------------
+# F-15 — graph sub-commands
+# ---------------------------------------------------------------------------
+
+
+@graph_app.command("report")
+def graph_report(
+    top_n: int = typer.Option(10, "--top-n", "-n", help="Number of hub papers to show."),
+    output_json: bool = typer.Option(False, "--json", help="Output as JSON."),
+) -> None:
+    """Show citation graph report: top hub papers, dangling citations, cluster summary."""
+    from codex.config import get_settings
+    from codex.db import get_conn
+    from codex.graph import (
+        build_citation_graph,
+        citation_pagerank,
+        citing_coverage,
+        dangling_citations,
+    )
+
+    settings = get_settings()
+    with get_conn() as conn:
+        graph = build_citation_graph(conn)
+        paper_rows = conn.execute("SELECT id, title FROM papers").fetchall()
+    known_ids = {row["id"] for row in paper_rows}
+    titles = {row["id"]: row["title"] for row in paper_rows}
+
+    if graph.number_of_nodes() == 0:
+        typer.echo("Citation graph is empty — ingest some papers first.")
+        return
+
+    n_papers = len(known_ids)
+    pr = citation_pagerank(graph)
+    hubs = sorted(pr.items(), key=lambda x: -x[1])[:top_n]
+    dangling = sorted(dangling_citations(graph, known_ids))
+    n_citing, _ = citing_coverage(graph, known_ids)
+    coverage = n_citing / n_papers if n_papers else 0.0
+    small_corpus = n_papers < settings.graph_min_corpus_size
+    warning = (
+        f"only {n_papers} ingested papers; recommended >= {settings.graph_min_corpus_size} "
+        "for meaningful ranking"
+        if small_corpus
+        else None
+    )
+
+    if output_json:
+        typer.echo(
+            json.dumps(
+                {
+                    "nodes": graph.number_of_nodes(),
+                    "edges": graph.number_of_edges(),
+                    "citing_coverage": {
+                        "citing": n_citing,
+                        "papers": n_papers,
+                        "fraction": round(coverage, 4),
+                    },
+                    "small_corpus_warning": warning,  # null unless below threshold (audit U-3)
+                    "hubs": [
+                        {"paper_id": pid, "title": titles.get(pid), "pagerank": score}
+                        for pid, score in hubs
+                    ],
+                    "dangling_count": len(dangling),
+                    "dangling": dangling,
+                },
+                indent=2,
+            )
+        )
+        return
+
+    if warning:
+        typer.echo(f"Warning: {warning}.", err=True)
+    # R-D: low citing-paper coverage starves PageRank/coupling even at a healthy
+    # paper count — warn on the share of papers that actually have out-edges.
+    if coverage < settings.graph_min_citing_coverage:
+        typer.echo(
+            f"Warning: only {n_citing}/{n_papers} papers ({coverage:.0%}) have out-edges "
+            f"(recommended ≥ {settings.graph_min_citing_coverage:.0%}); "
+            f"low citing coverage weakens PageRank/coupling.",
+            err=True,
+        )
+    typer.echo(f"Graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges")
+    typer.echo(f"Citing coverage: {n_citing}/{n_papers} papers with out-edges ({coverage:.0%})")
+    typer.echo("")
+    typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)")
+    typer.echo("-" * 48)
+    for pid, score in hubs:
+        # in-KB hubs show their title; dangling (OpenAlex-id) hubs are flagged (U-1)
+        label = titles.get(pid) or "(dangling — not ingested)"
+        typer.echo(f"  {score:.4f}  {pid}  {label}")
+    typer.echo("")
+    typer.echo(f"DANGLING CITATIONS ({len(dangling)} cited but not ingested)")
+    typer.echo("-" * 48)
+    for pid in dangling[:top_n]:
+        typer.echo(f"  {pid}")
+    if len(dangling) > top_n:
+        typer.echo(f"  … and {len(dangling) - top_n} more")
+
+
+@graph_app.command("related")
+def graph_related(
+    paper_id: str = typer.Argument(..., help="Paper ID to find related papers for."),
+    min_shared: int = typer.Option(
+        2, "--min-shared", help="Minimum shared references for bibliographic coupling."
+    ),
+) -> None:
+    """List bibliographically related papers (shared references ≥ min-shared)."""
+    from codex.db import get_conn
+    from codex.graph import build_citation_graph, find_related
+
+    with get_conn() as conn:
+        graph = build_citation_graph(conn)
+
+    related = find_related(paper_id, graph, min_shared=min_shared)
+    if not related:
+        typer.echo(f"No papers with ≥ {min_shared} shared references found for {paper_id}.")
+        return
+    typer.echo(
+        f"Papers related to {paper_id} (bibliographic coupling, ≥ {min_shared} shared refs):"
+    )
+    for pid in related:
+        typer.echo(f"  {pid}")
+
+
+@graph_app.command("cocited")
+def graph_cocited(
+    paper_id: str = typer.Argument(..., help="Paper ID to find co-cited papers for."),
+) -> None:
+    """List papers frequently co-cited with PAPER_ID (graph co-citation)."""
+    from codex.db import get_conn
+    from codex.graph import build_citation_graph, find_co_cited
+
+    with get_conn() as conn:
+        graph = build_citation_graph(conn)
+
+    results = find_co_cited(paper_id, graph)
+    if not results:
+        typer.echo(f"No papers co-cited with {paper_id}.")
+        return
+    typer.echo(f"Papers co-cited with {paper_id} (count desc):")
+    for pid, count in results:
+        typer.echo(f"  {count:>3}×  {pid}")

codex/config.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from functools import lru_cache
 
-from pydantic import AliasChoices, Field
+from pydantic import AliasChoices, Field, SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -26,14 +26,25 @@ class Settings(BaseSettings):
     # ------------------------------------------------------------------
     # Database
     # ------------------------------------------------------------------
-    database_url: str = Field(
+    database_url: SecretStr = Field(
-        default="postgresql://researcher:change_me@localhost:5432/papers",
+        default=SecretStr("postgresql://researcher:change_me@localhost:5432/papers"),
         description=(
             "libpq-compatible connection string consumed by psycopg. "
             "Example: postgresql://user:pass@host:5432/dbname"
         ),
     )
 
+    migration_database_url: SecretStr | None = Field(
+        default=None,
+        description=(
+            "Optional privileged connection string used by `codex migrate` to apply "
+            "schema DDL. The app's database_url is typically a least-privilege DML "
+            "role that cannot CREATE/ALTER owner-held tables (raises "
+            "InsufficientPrivilege); point this at an owner/superuser connection. "
+            "Falls back to database_url when unset."
+        ),
+    )
+
     # ------------------------------------------------------------------
     # External services
     # ------------------------------------------------------------------
@@ -109,7 +120,7 @@ class Settings(BaseSettings):
         description=(
             "Ollama base URL for wiki synthesis. "
             "When None, falls back to OLLAMA_BASE_URL "
-            "(http://192.168.178.103:11434 for the Jetson). "
+            "(http://jetson.local:11434 for the Jetson). "
             "Set WIKI_LLM_URL to override."
         ),
     )
@@ -137,7 +148,7 @@ class Settings(BaseSettings):
     # ------------------------------------------------------------------
     # F-09 Rich Parsing
     # ------------------------------------------------------------------
-    mathpix_app_id: str | None = Field(
+    mathpix_app_id: SecretStr | None = Field(
         default=None,
         description=(
             "MathPix App ID for cloud formula extraction. "
@@ -145,7 +156,7 @@ class Settings(BaseSettings):
         ),
     )
 
-    mathpix_app_key: str | None = Field(
+    mathpix_app_key: SecretStr | None = Field(
         default=None,
         description=(
             "MathPix App Key for cloud formula extraction. "
@@ -191,9 +202,7 @@ class Settings(BaseSettings):
 
     synthesis_llm_url: str | None = Field(
         default=None,
-        description=(
+        description=("Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."),
-            "Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."
-        ),
     )
 
     synthesis_top_k: int = Field(
@@ -248,7 +257,7 @@ class Settings(BaseSettings):
         description="Bind port for HTTP transport (ignored for stdio).",
     )
 
-    mcp_auth_token: str | None = Field(
+    mcp_auth_token: SecretStr | None = Field(
         default=None,
         description=(
             "Bearer token required when mcp_transport='http'. "
@@ -256,6 +265,75 @@ class Settings(BaseSettings):
         ),
     )
 
+    # ------------------------------------------------------------------
+    # F-16 Chunk Quality Gate
+    # ------------------------------------------------------------------
+    chunk_min_chars: int = Field(
+        default=60,
+        gt=0,
+        description=(
+            "Minimum character count for a chunk to pass the quality gate. "
+            "Chunks shorter than this value are discarded before embedding."
+        ),
+    )
+
+    chunk_min_alpha_ratio: float = Field(
+        default=0.40,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Minimum fraction of alphabetic characters in a chunk. "
+            "Chunks with a lower ratio are treated as OCR artefacts and discarded."
+        ),
+    )
+
+    chunk_max_bib_score: float = Field(
+        default=0.70,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Maximum bibliography heuristic score before a chunk is rejected. "
+            "Score is based on DOI density, 'et al.' and year-in-brackets patterns."
+        ),
+    )
+
+    # ------------------------------------------------------------------
+    # F-15 Literature Graph
+    # ------------------------------------------------------------------
+    graph_cite_boost_alpha: float = Field(
+        default=0.3,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Weight of the PageRank citation-boost in hybrid search. "
+            "boosted_distance = distance / (1 + alpha * pagerank_score) — dividing "
+            "lowers the cosine distance of high-PageRank papers (lower = closer). "
+            "Set to 0.0 to disable the boost without removing the flag."
+        ),
+    )
+
+    graph_min_corpus_size: int = Field(
+        default=15,
+        gt=0,
+        description=(
+            "Minimum number of ingested papers for citation_pagerank to yield "
+            "meaningful rankings. Below this threshold a warning is logged and "
+            "uniform scores are returned."
+        ),
+    )
+
+    graph_min_citing_coverage: float = Field(
+        default=0.8,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Minimum share of ingested papers that must have ≥1 out-edge (i.e. "
+            "cite something) before `codex graph report` warns. Low citing "
+            "coverage starves PageRank/coupling even when the paper count looks "
+            "healthy — the share of *citing* papers is what matters (R-D)."
+        ),
+    )
+
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:

codex/db.py

@@ -36,7 +36,7 @@ def get_conn() -> Generator[psycopg.Connection[psycopg.rows.DictRow], None, None
     """
     settings = get_settings()
     with psycopg.connect(
-        settings.database_url,
+        settings.database_url.get_secret_value(),
         row_factory=psycopg.rows.dict_row,
     ) as conn:
         yield conn
@@ -45,11 +45,16 @@ def get_conn() -> Generator[psycopg.Connection[psycopg.rows.DictRow], None, None
 def apply_schema(conn: psycopg.Connection[psycopg.rows.DictRow]) -> None:
     """Idempotently apply ``infra/schema.sql`` to the connected database.
 
-    Safe to call on every startup — all DDL statements use
+    Safe to call repeatedly — every statement is ``CREATE … IF NOT EXISTS`` or
-    ``CREATE … IF NOT EXISTS``.
+    ``ADD COLUMN IF NOT EXISTS``, so re-application is a no-op.
+
+    Requires a connection whose role can run DDL (owns / can CREATE + ALTER the
+    tables). The least-privilege application role is typically DML-only and will
+    raise ``InsufficientPrivilege``; use a privileged connection — see
+    ``codex migrate`` and ``MIGRATION_DATABASE_URL`` (audit M-1).
 
     Args:
-        conn: An open psycopg connection (obtained via :func:`get_conn`).
+        conn: An open psycopg connection with DDL privileges.
     """
     schema_path = Path(__file__).parent.parent / "infra" / "schema.sql"
     sql = schema_path.read_text(encoding="utf-8")

codex/discover.py

@@ -1,22 +1,32 @@
-"""Discovery queries over the citation graph stored in PostgreSQL."""
+"""Discovery queries over the citation graph stored in PostgreSQL.
+
+Every query resolves ``citations.cited_id`` to a canonical ``papers.id`` through
+the shared :data:`codex.graph.RESOLVED_CITATIONS_SQL`, so this module and the
+citation-graph layer agree on what counts as "ingested". Previously this module
+compared the raw OpenAlex ``cited_id`` against ``papers.id`` (a DOI/arXiv id),
+which mis-reported already-ingested papers as discovery leads (audit C-1).
+"""
 
 from __future__ import annotations
 
 from codex.db import get_conn
+from codex.graph import RESOLVED_CITATIONS_SQL
 
 
 def discovery_leads(limit: int = 20) -> list[dict[str, int | str]]:
     """Return papers referenced by already-ingested papers but not yet collected.
 
     Returns list of dicts with keys:
-        cited_id (str)   — arXiv ID, DOI, or OpenAlex W-ID of the target
+        cited_id (str)   — canonical id of the referenced (not-yet-ingested) work
         pull     (int)   — how many already-ingested papers cite this target
 
-    Ordered by pull DESC, limited to `limit` rows.
+    Ordered by pull DESC, limited to `limit` rows. References that resolve to a
+    paper already in the corpus are excluded — they are not leads.
     """
-    sql = """
+    sql = f"""
+        WITH resolved AS ({RESOLVED_CITATIONS_SQL})
         SELECT cited_id, count(*) AS pull
-        FROM   citations
+        FROM   resolved
         WHERE  cited_id NOT IN (SELECT id FROM papers)
         GROUP  BY cited_id
         ORDER  BY pull DESC
@@ -29,7 +39,10 @@ def discovery_leads(limit: int = 20) -> list[dict[str, int | str]]:
 
 def citing_papers(paper_id: str) -> list[str]:
     """Return IDs of all papers that cite `paper_id` (in-graph reverse citations)."""
-    sql = "SELECT citing_id FROM citations WHERE cited_id = %(paper_id)s"
+    sql = f"""
+        WITH resolved AS ({RESOLVED_CITATIONS_SQL})
+        SELECT citing_id FROM resolved WHERE cited_id = %(paper_id)s
+    """
     with get_conn() as conn:
         rows = conn.execute(sql, {"paper_id": paper_id}).fetchall()
     return [row["citing_id"] for row in rows]
@@ -37,7 +50,10 @@ def citing_papers(paper_id: str) -> list[str]:
 
 def cited_by(paper_id: str) -> list[str]:
     """Return IDs of all papers that `paper_id` cites (forward citations)."""
-    sql = "SELECT cited_id FROM citations WHERE citing_id = %(paper_id)s"
+    sql = f"""
+        WITH resolved AS ({RESOLVED_CITATIONS_SQL})
+        SELECT cited_id FROM resolved WHERE citing_id = %(paper_id)s
+    """
     with get_conn() as conn:
         rows = conn.execute(sql, {"paper_id": paper_id}).fetchall()
     return [row["cited_id"] for row in rows]
@@ -49,10 +65,11 @@ def cocited_papers(paper_id: str, limit: int = 10) -> list[dict[str, int | str]]
     A paper X is co-cited with `paper_id` if some paper cites both.
     Returns list of dicts: {paper_id: str, co_citations: int}, ordered DESC.
     """
-    sql = """
+    sql = f"""
+        WITH resolved AS ({RESOLVED_CITATIONS_SQL})
         SELECT c2.cited_id AS paper_id, count(*) AS co_citations
-        FROM   citations c1
+        FROM   resolved c1
-        JOIN   citations c2 ON c1.citing_id = c2.citing_id
+        JOIN   resolved c2 ON c1.citing_id = c2.citing_id
         WHERE  c1.cited_id = %(paper_id)s
           AND  c2.cited_id != %(paper_id)s
         GROUP  BY c2.cited_id

codex/embed.py

@@ -1,4 +1,4 @@
-"""Hybrid dense + sparse embeddings via BGE-M3 (ADR-0002).
+"""Dense (+ optional sparse) embeddings via BGE-M3 (ADR-0002).
 
 Uses :class:`FlagEmbedding.BGEM3FlagModel` for encoding because the
 ``return_dense`` / ``return_sparse`` kwargs are part of the FlagEmbedding
@@ -7,6 +7,11 @@ vanilla ``SentenceTransformer`` load of ``BAAI/bge-m3`` (same weights,
 same model); sparse output is a list of ``{token_id: weight}`` dicts per
 text.
 
+Status: only the **dense** path is wired into ingest/search today; the search
+"hybrid" is dense + Postgres FTS. :meth:`Embedder.encode_sparse` / :meth:`encode`
+are provided for a future sparse-retrieval layer but are not yet consumed by the
+pipeline (audit C-14).
+
 Notes for callers
 -----------------
 * Empty input is handled explicitly — no model call is issued.

codex/graph.py

@@ -0,0 +1,209 @@
+"""Citation graph analytics (F-15).
+
+Builds an in-memory NetworkX DiGraph from the ``citations`` table and
+provides PageRank, bibliographic coupling, co-citation, and dangling-
+citation queries.  The graph is ephemeral — rebuilt per call, < 1 s for
+≤ 100 papers.
+
+Graceful degradation
+--------------------
+* ``citation_pagerank`` returns uniform scores when the graph has fewer
+  than 5 nodes, and logs a warning.
+* All functions accept an empty graph without raising.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, cast
+
+import networkx as nx
+
+logger = logging.getLogger(__name__)
+
+_MIN_PAGERANK_NODES = 5
+
+# Single source of truth for resolving ``citations.cited_id`` to a canonical
+# paper id. ``cited_id`` stores OpenAlex IDs while ``papers.id`` stores DOIs /
+# arXiv ids, so an in-KB reference cited by its OpenAlex id does not match
+# ``papers.id`` directly. This resolves it to the canonical ``papers.id`` (via
+# ``papers.openalex_id``, then the ``paper_identifiers`` alias table); dangling
+# references keep their raw id. Both :func:`build_citation_graph` and
+# :mod:`codex.discover` query through this so the "what counts as ingested" rule
+# cannot drift between them again (audit C-1). Trusted constant — no user input.
+RESOLVED_CITATIONS_SQL = """
+    SELECT c.citing_id,
+           COALESCE(p.id, pi.paper_id, c.cited_id) AS cited_id
+    FROM citations c
+    LEFT JOIN papers p ON p.openalex_id = c.cited_id
+    LEFT JOIN paper_identifiers pi
+           ON pi.openalex_id = c.cited_id AND p.id IS NULL
+"""
+
+
+def build_citation_graph(conn: Any) -> nx.DiGraph:
+    """Load the ``citations`` table into a directed graph.
+
+    Nodes are paper IDs (strings).  An edge ``citing → cited`` means
+    the citing paper references the cited paper.  Both ingested papers
+    and dangling targets (cited but not yet ingested) appear as nodes.
+
+    ``cited_id`` in the citations table stores OpenAlex IDs; ``papers.id``
+    stores DOIs.  A LEFT JOIN resolves cited papers that are in the KB to
+    their canonical DOI key via ``papers.openalex_id``.  Dangling references
+    (not in KB) keep their raw OpenAlex ID.
+
+    Parameters
+    ----------
+    conn:
+        Open psycopg connection (dict-row factory assumed).
+
+    Returns
+    -------
+    nx.DiGraph with every (citing_id, cited_id) pair as an edge.
+    """
+    rows = conn.execute(RESOLVED_CITATIONS_SQL).fetchall()
+    g: nx.DiGraph = nx.DiGraph()
+    for row in rows:
+        g.add_edge(row["citing_id"], row["cited_id"])
+    logger.debug(
+        "Built citation graph: %d nodes, %d edges", g.number_of_nodes(), g.number_of_edges()
+    )
+    return g
+
+
+def citation_pagerank(graph: nx.DiGraph, *, damping: float = 0.85) -> dict[str, float]:
+    """Compute PageRank over the citation graph.
+
+    Parameters
+    ----------
+    graph:
+        DiGraph from :func:`build_citation_graph`.
+    damping:
+        PageRank damping factor (default 0.85).
+
+    Returns
+    -------
+    ``{paper_id: score}`` dict.  Scores sum to approximately 1.0.
+
+    When ``graph`` has fewer than :data:`_MIN_PAGERANK_NODES` (5) nodes,
+    returns a uniform distribution and logs a warning — the graph is too
+    sparse for the random-walk model to converge meaningfully.  The
+    ``damping`` parameter is ignored in this branch.  The user-facing
+    corpus-size threshold (default 15) lives in
+    :attr:`codex.config.Settings.graph_min_corpus_size` and is surfaced
+    by ``codex graph report``.
+    """
+    n = graph.number_of_nodes()
+    if n == 0:
+        return {}
+    if n < _MIN_PAGERANK_NODES:
+        logger.warning(
+            "citation_pagerank: only %d nodes — corpus too small for meaningful ranking "
+            "(need ≥ %d). Returning uniform scores.",
+            n,
+            _MIN_PAGERANK_NODES,
+        )
+        uniform = 1.0 / n
+        return {node: uniform for node in graph.nodes()}
+    return cast(dict[str, float], nx.pagerank(graph, alpha=damping))
+
+
+def citing_coverage(graph: nx.DiGraph, known_ids: set[str]) -> tuple[int, int]:
+    """Return ``(papers_with_out_edges, total_papers)`` — the citing-paper coverage.
+
+    A paper "covers" the citation graph when it has ≥1 out-edge (it cites at least
+    one work). Low citing coverage starves the F-15 layer (PageRank / coupling /
+    co-citation) even when the total paper count looks healthy, because those
+    metrics are driven by out-edges, not by node count. Surfaced by
+    ``codex graph report`` (R-D); the warning threshold is
+    :attr:`codex.config.Settings.graph_min_citing_coverage`.
+    """
+    n_citing = sum(1 for pid in known_ids if pid in graph and graph.out_degree(pid) > 0)
+    return n_citing, len(known_ids)
+
+
+def find_related(paper_id: str, graph: nx.DiGraph, *, min_shared: int = 2) -> list[str]:
+    """Bibliographic coupling: papers sharing ≥ ``min_shared`` references with ``paper_id``.
+
+    Two papers are bibliographically coupled when they cite the same sources.
+    The more shared references, the more likely they treat related topics.
+
+    Parameters
+    ----------
+    paper_id:
+        Source paper whose references form the seed set.
+    graph:
+        DiGraph from :func:`build_citation_graph`.
+    min_shared:
+        Minimum number of shared references to be included.
+
+    Returns
+    -------
+    List of paper IDs ordered by shared-reference count DESC.
+    """
+    if paper_id not in graph:
+        return []
+    # papers cited by paper_id
+    references: set[str] = set(graph.successors(paper_id))
+    if not references:
+        return []
+
+    shared: dict[str, int] = {}
+    for ref in references:
+        # other papers that also cite this reference
+        for co_citer in graph.predecessors(ref):
+            if co_citer == paper_id:
+                continue
+            shared[co_citer] = shared.get(co_citer, 0) + 1
+
+    return [
+        pid for pid, count in sorted(shared.items(), key=lambda x: -x[1]) if count >= min_shared
+    ]
+
+
+def find_co_cited(paper_id: str, graph: nx.DiGraph) -> list[tuple[str, int]]:
+    """Co-citation: papers frequently cited alongside ``paper_id``.
+
+    A paper X is co-cited with ``paper_id`` when some third paper cites both.
+
+    Returns
+    -------
+    ``[(paper_id, count)]`` ordered by count DESC.
+    """
+    if paper_id not in graph:
+        return []
+    # papers that cite paper_id
+    citers: set[str] = set(graph.predecessors(paper_id))
+    if not citers:
+        return []
+
+    co_cited: dict[str, int] = {}
+    for citer in citers:
+        for other in graph.successors(citer):
+            if other == paper_id:
+                continue
+            co_cited[other] = co_cited.get(other, 0) + 1
+
+    return sorted(co_cited.items(), key=lambda x: -x[1])
+
+
+def dangling_citations(graph: nx.DiGraph, known_ids: set[str]) -> list[str]:
+    """Return graph nodes not present in ``known_ids`` (cited but not yet ingested).
+
+    Complements :func:`codex.discover.discovery_leads` with graph context:
+    the returned IDs are reachable in the citation graph but have no paper
+    record in the DB.
+
+    Parameters
+    ----------
+    graph:
+        DiGraph from :func:`build_citation_graph`.
+    known_ids:
+        Set of paper IDs that exist in the ``papers`` table.
+
+    Returns
+    -------
+    List of dangling paper IDs (unordered).
+    """
+    return [node for node in graph.nodes() if node not in known_ids]

codex/ingest.py

@@ -2,18 +2,19 @@
 
 from __future__ import annotations
 
-import contextlib
 import logging
 from dataclasses import dataclass
 from pathlib import Path
 
 import numpy as np
+import psycopg
 
 from codex.config import get_settings
 from codex.db import get_conn
 from codex.embed import get_embedder
 from codex.models import Citation, Paper
-from codex.sources import openalex, semanticscholar
+from codex.quality import is_quality_chunk, section_label
+from codex.sources import arxiv, crossref, openalex, semanticscholar
 
 logger = logging.getLogger(__name__)
 
@@ -46,6 +47,113 @@ def _make_bibkey(paper: Paper) -> str | None:
     return surnames[0] + "EtAl" + year
 
 
+def _s2_id_for(pid: str) -> str | None:
+    """Format a canonical paper id for the Semantic Scholar paper endpoint.
+
+    S2 needs a namespaced id (``arXiv:<id>``, ``DOI:<doi>``) or a bare 40-hex
+    S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by
+    the ``10.`` prefix; everything else is assumed to be an arXiv id (modern
+    ``2305.10988`` or legacy ``math/0603097``).
+    """
+    p = (pid or "").strip()
+    if not p:
+        return None
+    if p.lower().startswith(("arxiv:", "doi:")):
+        return p
+    if p.startswith("10."):
+        return f"DOI:{p}"
+    return f"arXiv:{p}"
+
+
+def _norm_cited_id(cited_id: str) -> str:
+    """Normalize a DOI cited-id to its canonical bare, lower-cased form.
+
+    DOIs are case-insensitive and may arrive bare (``10.…``), as a
+    ``https://doi.org/…`` URL, or ``doi:…``-prefixed: GROBID emits whichever
+    form the PDF reference text used, while S2/OpenAlex hand back bare DOIs.
+    Stripping the prefix and lower-casing keeps the citation-graph join from
+    fragmenting the same reference into distinct case-/URL-variant nodes. arXiv
+    ids and S2 paperIds are left untouched. Mirrors ``openalex._normalize_doi``."""
+    s = cited_id.strip()
+    lowered = s.lower()
+    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
+        if lowered.startswith(prefix):
+            return lowered[len(prefix) :]
+    return lowered if lowered.startswith("10.") else s
+
+
+def _recover_abstract(paper: Paper) -> str | None:
+    """Recover a missing abstract from S2, then Crossref (DQ-2 supplement).
+
+    Tried in order of coverage for this corpus: Semantic Scholar (indexes most
+    arXiv + journal abstracts), then Crossref (publisher-deposited, DOI only).
+    Network/parse failures degrade to None rather than aborting the ingest.
+    """
+    s2_id = _s2_id_for(paper.id)
+    if s2_id is not None:
+        try:
+            abstract = semanticscholar.fetch_abstract(s2_id)
+            if abstract and abstract.strip():
+                return abstract.strip()
+        except Exception:
+            logger.warning("S2 abstract recovery failed for %s", paper.id, exc_info=True)
+    if paper.id.startswith("10."):
+        try:
+            abstract = crossref.fetch_abstract(paper.id)
+            if abstract and abstract.strip():
+                return abstract.strip()
+        except Exception:
+            logger.warning("Crossref abstract recovery failed for %s", paper.id, exc_info=True)
+    return None
+
+
+def _s2_reference_supplement(paper: Paper) -> list[Citation]:
+    """Fetch a paper's references from Semantic Scholar (DQ-1 supplement).
+
+    Used when OpenAlex returns an empty ``referenced_works`` list — common for
+    arXiv preprints and institutional theses that OpenAlex indexes without a
+    parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id``
+    so the FK holds, and cited DOIs are case-normalised. Network/parse failures
+    degrade to an empty list rather than aborting the ingest.
+    """
+    s2_id = _s2_id_for(paper.id)
+    if s2_id is None:
+        return []
+    try:
+        raw = semanticscholar.fetch_references(s2_id)
+    except Exception:
+        logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True)
+        return []
+    return [
+        Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
+        for c in raw
+        if c.cited_id
+    ]
+
+
+def _crossref_reference_supplement(paper: Paper) -> list[Citation]:
+    """Fetch a paper's references from Crossref (roadmap R-B supplement).
+
+    Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref).
+    Crossref carries publisher-deposited reference lists keyed on the citing
+    work's DOI, so only DOI papers can be looked up (arXiv-only ids → []).
+    ``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are
+    case-normalised. Network/parse failures degrade to [] rather than aborting.
+    """
+    if not paper.id.startswith("10."):
+        return []
+    try:
+        raw = crossref.fetch_references(paper.id)
+    except Exception:
+        logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True)
+        return []
+    return [
+        Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
+        for c in raw
+        if c.cited_id
+    ]
+
+
 def ingest_paper(
     paper_id: str,
     source_path: str | None = None,
@@ -80,7 +188,6 @@ def ingest_paper(
     # ---------------------------------------------------------------
     paper: Paper | None = openalex.fetch_paper(paper_id)
 
-
     if paper is None:
         # Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
         pid_lower = paper_id.lower()
@@ -88,16 +195,36 @@ def ingest_paper(
             len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit()
         )
         if looks_like_arxiv:
-            arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}"
+            # OpenAlex 404 on an arXiv id → recover authoritative metadata from
-            with contextlib.suppress(Exception):
+            # the arXiv API (DQ-2); fall back to a minimal stub if arXiv also
-                semanticscholar.fetch_references(arxiv_key)
+            # has nothing. Citation recovery from S2 happens in the citation
-            paper = Paper(id=paper_id, title="")
+            # block below (DQ-1 supplement).
+            paper = arxiv.fetch_metadata(paper_id) or Paper(id=paper_id, title="")
         else:
             raise ValueError(f"Paper not found: {paper_id}")
 
     if paper is None:
         raise ValueError(f"Paper not found: {paper_id}")
 
+    # Canonical identity is the caller-supplied id — what the CLI / ingest scripts
+    # use and what papers.id is contracted to hold (arXiv id or DOI). Pin it to the
+    # caller's id (audit C-7), but normalize it the same way cited-ids are (strip a
+    # https://doi.org/ or doi: prefix, lower-case DOIs) so a non-bare caller cannot
+    # store a URL-form papers.id that would defeat DQ-5's ON CONFLICT (id) idempotency
+    # or the `paper.id.startswith("10.")` recovery gates below. arXiv/W ids pass
+    # through unchanged. The OpenAlex work id is retained as openalex_id / in
+    # paper_identifiers.
+    paper.id = _norm_cited_id(paper_id.strip())
+
+    # DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute
+    # some publishers' abstracts). Supplement from S2, then Crossref (uses the
+    # canonical paper.id set above), so the paper gets a real (non-zero) abstract
+    # embedding instead of a zero vector.
+    if not (paper.abstract or "").strip():
+        recovered = _recover_abstract(paper)
+        if recovered:
+            paper.abstract = recovered
+
     # Auto-generate bibkey when source has none (D-04).
     if paper.bibkey is None:
         paper.bibkey = _make_bibkey(paper)
@@ -117,35 +244,77 @@ def ingest_paper(
     # ---------------------------------------------------------------
     # 3. Upsert paper  (INSERT … ON CONFLICT (id) DO UPDATE SET …)
     # ---------------------------------------------------------------
+    upsert_sql = """
+        INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
+                            source_path, abstract_emb)
+        VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
+                %(abstract)s, %(source_path)s, %(abstract_emb)s)
+        ON CONFLICT (id) DO UPDATE SET
+            openalex_id  = EXCLUDED.openalex_id,
+            bibkey       = COALESCE(papers.bibkey, EXCLUDED.bibkey),
+            title        = EXCLUDED.title,
+            authors      = EXCLUDED.authors,
+            year         = EXCLUDED.year,
+            abstract     = EXCLUDED.abstract,
+            source_path  = COALESCE(EXCLUDED.source_path, papers.source_path),
+            abstract_emb = EXCLUDED.abstract_emb
+    """
     with get_conn() as conn:
-        conn.execute(
+        # The id-keyed upsert does not catch a papers.bibkey UNIQUE collision (two
-            """
+        # same-author/year papers generate the same key). On such a collision retry
-            INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
+        # with an a/b/c… suffix instead of failing the whole ingest (audit C-15).
-                                source_path, abstract_emb)
+        base_bibkey = paper.bibkey
-            VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
+        for attempt in range(27):
-                    %(abstract)s, %(source_path)s, %(abstract_emb)s)
+            try:
-            ON CONFLICT (id) DO UPDATE SET
+                conn.execute(
-                openalex_id  = EXCLUDED.openalex_id,
+                    upsert_sql,
-                bibkey       = COALESCE(papers.bibkey, EXCLUDED.bibkey),
+                    {
-                title        = EXCLUDED.title,
+                        "id": paper.id,
-                authors      = EXCLUDED.authors,
+                        "openalex_id": paper.openalex_id,
-                year         = EXCLUDED.year,
+                        "bibkey": paper.bibkey,
-                abstract     = EXCLUDED.abstract,
+                        "title": paper.title,
-                source_path  = COALESCE(EXCLUDED.source_path, papers.source_path),
+                        "authors": paper.authors,
-                abstract_emb = EXCLUDED.abstract_emb
+                        "year": paper.year,
-            """,
+                        "abstract": paper.abstract,
-            {
+                        "source_path": source_path,
-                "id": paper.id,
+                        "abstract_emb": abstract_emb,
-                "openalex_id": paper.openalex_id,
+                    },
-                "bibkey": paper.bibkey,
+                )
-                "title": paper.title,
+                break
-                "authors": paper.authors,
+            except psycopg.errors.UniqueViolation as exc:
-                "year": paper.year,
+                msg = str(exc).lower()
-                "abstract": paper.abstract,
+                conn.rollback()
-                "source_path": source_path,
+                if "openalex" in msg:
-                "abstract_emb": abstract_emb,
+                    # Two different papers.id claim the same OpenAlex work — a real data
-            },
+                    # conflict, not something to auto-rename. Surface it clearly instead
-        )
+                    # of the raw psycopg error (companion gap to C-15's bibkey case).
+                    raise ValueError(
+                        f"openalex_id {paper.openalex_id!r} is already attached to a "
+                        f"different paper; ingesting '{paper.id}' would duplicate it — "
+                        "resolve the conflicting paper first."
+                    ) from exc
+                if base_bibkey is None or "bibkey" not in msg or attempt == 26:
+                    raise
+                paper.bibkey = f"{base_bibkey}{chr(ord('a') + attempt)}"
+                logger.warning(
+                    "bibkey %r already exists; ingesting '%s' as %r instead",
+                    base_bibkey,
+                    paper.id,
+                    paper.bibkey,
+                )
+
+        # Register openalex_id in paper_identifiers (alias table).
+        # Every ingest records the primary openalex_id so the graph JOIN
+        # resolves it even if papers.openalex_id is later updated.
+        if paper.openalex_id:
+            conn.execute(
+                """
+                INSERT INTO paper_identifiers (paper_id, openalex_id)
+                VALUES (%s, %s)
+                ON CONFLICT DO NOTHING
+                """,
+                (paper.id, paper.openalex_id),
+            )
 
         # ---------------------------------------------------------------
         # 4. Parse + chunk + embed source file (if provided)
@@ -157,48 +326,74 @@ def ingest_paper(
         if source_path is not None:
             from codex.parsing.grobid import extract_references
             from codex.parsing.nougat import pdf_to_markdown
-            from codex.parsing.tex import chunk_text, latex_to_text
+            from codex.parsing.tex import chunk_sections, chunk_text
 
             suffix = Path(source_path).suffix.lower()
-            text = ""
+            # (section_title | None, chunk_text) pairs. `.tex` is section-aware so
+            # the stored `section` column reflects the real heading (R-C / R-12);
+            # `.txt`/`.pdf` carry no section structure, so the title is None.
+            raw_pairs: list[tuple[str | None, str]] = []
 
             if suffix == ".tex":
-                text = latex_to_text(Path(source_path).read_text())
+                raw_pairs = chunk_sections(Path(source_path).read_text())
             elif suffix == ".txt":
                 text = Path(source_path).read_text(encoding="utf-8", errors="replace")
+                raw_pairs = [(None, c) for c in chunk_text(text)]
             elif suffix == ".pdf":
-                text = pdf_to_markdown(source_path)
+                raw_pairs = [(None, c) for c in chunk_text(pdf_to_markdown(source_path))]
                 # Also extract GROBID refs
                 grobid_refs = extract_references(source_path)
                 for ref in grobid_refs:
                     cited_id = ref.get("doi") or ref.get("arxiv_id")
                     if cited_id:
-                        pdf_citations.append(Citation(citing_id=paper.id, cited_id=cited_id))
+                        # Normalize the GROBID DOI (bare/URL/``doi:`` → bare lower-case)
+                        # so PDF-derived edges join the same graph nodes as the
+                        # S2/OpenAlex paths; arXiv ids pass through untouched.
+                        pdf_citations.append(
+                            Citation(citing_id=paper.id, cited_id=_norm_cited_id(cited_id))
+                        )
             else:
                 logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
 
-            chunks_text = chunk_text(text) if text else []
+            # Quality-gate the pairs (keeps each chunk aligned with its section title).
+            settings = get_settings()
+            kept_pairs = [
+                (title, content)
+                for title, content in raw_pairs
+                if is_quality_chunk(content, settings=settings)
+            ]
 
             # Delete existing chunks for this paper
             conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
 
-            if chunks_text:
+            if kept_pairs:
                 # Embed all chunks in one batch
-                chunk_embeddings = embedder.encode_dense(chunks_text)
+                chunk_contents = [content for _, content in kept_pairs]
+                chunk_embeddings = embedder.encode_dense(chunk_contents)
                 chunk_rows = [
-                    (paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist())
+                    (
-                    for ord_idx, content in enumerate(chunks_text)
+                        paper.id,
+                        ord_idx,
+                        content,
+                        chunk_embeddings[ord_idx].tolist(),
+                        # .tex: label from the real \section heading — keeps the
+                        # controlled bucket when it matches, else stores the cleaned
+                        # title (R-F); .txt/.pdf (title=None) classify by content.
+                        # NB: `section` semantics now differ by source (write-only col).
+                        section_label(title, content),
+                    )
+                    for ord_idx, (title, content) in enumerate(kept_pairs)
                 ]
                 with conn.cursor() as cur:
                     cur.executemany(
-                        "INSERT INTO chunks (paper_id, ord, content, embedding)"
+                        "INSERT INTO chunks (paper_id, ord, content, embedding, section)"
-                        " VALUES (%s, %s, %s, %s)",
+                        " VALUES (%s, %s, %s, %s, %s)",
                         chunk_rows,
                     )
                 chunks_upserted = len(chunk_rows)
 
         # ---------------------------------------------------------------
-        # 5. Fetch citations  (OpenAlex if openalex_id, else S2)
+        # 5. Fetch citations  (fallback chain: OpenAlex → S2 → Crossref)
         #    Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING
         # ---------------------------------------------------------------
         api_citations: list[Citation]
@@ -207,11 +402,19 @@ def ingest_paper(
             # OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
             # Rewrite citing_id to paper.id so the FK constraint is satisfied.
             api_citations = [
-                Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
+                Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
-                for c in raw
             ]
+            # DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a
+            # parsed reference list (referenced_works == []). Fall back to the
+            # Semantic Scholar references, which often has them.
+            if not api_citations:
+                api_citations = _s2_reference_supplement(paper)
         else:
-            api_citations = semanticscholar.fetch_references(paper_id)
+            api_citations = _s2_reference_supplement(paper)
+        # R-B: if OpenAlex and S2 both came back empty, try Crossref's
+        # publisher-deposited references (DOI papers only) as a third leg.
+        if not api_citations:
+            api_citations = _crossref_reference_supplement(paper)
 
         # Merge API citations and GROBID PDF citations; dedup via set
         all_citations_set: set[tuple[str, str]] = set()
@@ -267,7 +470,10 @@ def ingest_paper(
                             VALUES (%s, %s, %s, %s, %s)
                             """,
                             [
-                                (f.paper_id, f.page, f.raw_latex, f.context, f.eq_label)
+                                # paper.id, not f.paper_id: the extractor derives its
+                                # paper_id from the PDF filename stem, which need not match
+                                # papers.id and would violate the FK (audit C-10).
+                                (paper.id, f.page, f.raw_latex, f.context, f.eq_label)
                                 for f in formulas
                             ],
                         )
@@ -281,7 +487,8 @@ def ingest_paper(
                             VALUES (%s, %s, %s, %s)
                             """,
                             [
-                                (fig.paper_id, fig.page, fig.caption, fig.image_path)
+                                # paper.id, not fig.paper_id (filename stem) — see C-10 above.
+                                (paper.id, fig.page, fig.caption, fig.image_path)
                                 for fig in figures
                             ],
                         )

codex/mcp_server.py

@@ -24,6 +24,7 @@ Transport
 from __future__ import annotations
 
 import logging
+import secrets
 
 from mcp.server.fastmcp import FastMCP
 from mcp.server.transport_security import TransportSecuritySettings
@@ -121,7 +122,6 @@ def wiki_read(concept_slug: str) -> dict[str, object]:
     directory or the requested page does not exist.
     """
     try:
-        import json
         from pathlib import Path
 
         from codex.config import get_settings
@@ -135,16 +135,12 @@ def wiki_read(concept_slug: str) -> dict[str, object]:
 
         markdown = page_path.read_text(encoding="utf-8")
 
-        # Collect cited bibkeys from the compile state
+        # Collect the actual cited bibkeys from the page's inline [BibKey #loc]
-        state_path = wiki_dir / ".compile-state.json"
+        # citations (audit C-3: this previously returned the concept slug itself,
-        sources: list[str] = []
+        # not the cited sources).
-        if state_path.exists():
+        from codex.wiki import _parse_claims
-            try:
+
-                state: dict[str, str] = json.loads(state_path.read_text(encoding="utf-8"))
+        sources = sorted({claim.bibkey for claim in _parse_claims(markdown)})
-                if concept_slug in state:
-                    sources = [concept_slug]
-            except (json.JSONDecodeError, OSError):
-                pass
 
         return {"markdown": markdown, "sources": sources}
     except Exception as exc:  # noqa: BLE001
@@ -298,12 +294,13 @@ def _require_http_token() -> str:
     from codex.config import get_settings
 
     settings = get_settings()
-    if not settings.mcp_auth_token:
+    token = settings.mcp_auth_token.get_secret_value() if settings.mcp_auth_token else ""
+    if not token:
         raise RuntimeError(
             "HTTP transport requires MCP_AUTH_TOKEN to be set. "
             "Set the environment variable or add it to .env."
         )
-    return settings.mcp_auth_token
+    return token
 
 
 # ---------------------------------------------------------------------------
@@ -337,7 +334,9 @@ def main() -> None:
                 self, request: Request, call_next: RequestResponseEndpoint
             ) -> Response:
                 auth = request.headers.get("Authorization", "")
-                if auth != f"Bearer {token}":
+                # Constant-time comparison to avoid a token-length/equality timing
+                # side-channel (audit S-2).
+                if not secrets.compare_digest(auth, f"Bearer {token}"):
                     return Response("Unauthorized", status_code=401)
                 return await call_next(request)
 

codex/parsing/grobid.py

@@ -26,6 +26,7 @@ def _text(element: ET.Element | None) -> str:
 def extract_references(
     pdf_path: str,
     grobid_url: str | None = None,
+    timeout: float = 60.0,
 ) -> list[dict[str, str]]:
     """Extract a structured reference list from a PDF via GROBID.
 
@@ -35,6 +36,9 @@ def extract_references(
         Path to the PDF file on disk.
     grobid_url:
         Base URL of the GROBID server.  Defaults to ``Settings().grobid_url``.
+    timeout:
+        HTTP client timeout in seconds.  Large PDFs (e.g. theses) can exceed the
+        60s default, especially against a slower/emulated GROBID; raise it then.
 
     Returns
     -------
@@ -46,7 +50,7 @@ def extract_references(
     if grobid_url is None:
         grobid_url = Settings().grobid_url
 
-    with open(pdf_path, "rb") as fh, httpx.Client(timeout=60.0) as client:
+    with open(pdf_path, "rb") as fh, httpx.Client(timeout=timeout) as client:
         response = client.post(
             f"{grobid_url}/api/processReferences",
             files={"input": (pdf_path, fh, "application/pdf")},

codex/parsing/mathpix.py

@@ -302,8 +302,10 @@ def extract_formulas(
     """
     settings = get_settings()
 
-    app_id = mathpix_app_id or settings.mathpix_app_id or ""
+    settings_id = settings.mathpix_app_id.get_secret_value() if settings.mathpix_app_id else ""
-    app_key = mathpix_app_key or settings.mathpix_app_key or ""
+    settings_key = settings.mathpix_app_key.get_secret_value() if settings.mathpix_app_key else ""
+    app_id = mathpix_app_id or settings_id
+    app_key = mathpix_app_key or settings_key
 
     if app_id and app_key:
         logger.info("Using MathPix backend for %s", pdf_path)

codex/parsing/tex.py

@@ -17,6 +17,10 @@ _SECTION_RE = re.compile(
     re.DOTALL,
 )
 
+# \input{file} / \include{file} — arXiv papers routinely split the body across
+# files, so these must be inlined before parsing (R-C multi-file flattening).
+_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}")
+
 # Patterns for LaTeX noise removal (applied in order).
 _COMMENT_RE = re.compile(r"%[^\n]*")
 _CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}")
@@ -157,3 +161,63 @@ def latex_to_text(latex: str) -> str:
         return "\n\n".join(body for _, body in sections)
     # Fallback: no section structure — clean the whole string.
     return _clean_latex(latex)
+
+
+def _norm_texkey(name: str) -> str:
+    """Normalize a tex file name / ``\\input`` argument to a comparable key.
+
+    Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing
+    ``.tex`` so ``\\input{sections/intro}`` matches the archive member
+    ``sections/intro.tex``.
+    """
+    n = name.strip()
+    if n.startswith("./"):  # a single leading "./" only — don't eat "../" or ".hidden"
+        n = n[2:]
+    return n[:-4] if n.endswith(".tex") else n
+
+
+def flatten_inputs(primary: str, files: dict[str, str]) -> str:
+    """Recursively inline ``\\input``/``\\include`` directives (R-C).
+
+    *files* maps each source file's archive name to its contents (keys may carry
+    a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv
+    multi-file projects leave the ``\\documentclass`` file as little more than a
+    skeleton of ``\\input`` lines, so the body must be assembled before parsing.
+    Unresolved references are dropped; recursion is capped at depth 10 to guard
+    against include cycles.
+    """
+    norm = {_norm_texkey(k): v for k, v in files.items()}
+
+    def _expand(text: str, depth: int) -> str:
+        if depth > 10:
+            # Cap reached (likely an \input cycle): strip any still-unresolved
+            # directives rather than leak a literal "\input{…}" into the parsed text.
+            return _INPUT_RE.sub("", text)
+
+        def _repl(match: re.Match[str]) -> str:
+            content = norm.get(_norm_texkey(match.group(1)))
+            return _expand(content, depth + 1) if content is not None else ""
+
+        return _INPUT_RE.sub(_repl, text)
+
+    return _expand(primary, 0)
+
+
+def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]:
+    """Chunk LaTeX *within* each section, pairing every chunk with its title.
+
+    Unlike :func:`latex_to_text` (which discards titles and flattens the whole
+    document into one block before word-window chunking), this keeps chunks from
+    spanning section boundaries and remembers which ``\\section`` each came from.
+    The caller can then label the stored ``section`` column from the real heading
+    instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less
+    chunks of the whole cleaned document when there are no section commands.
+    """
+    sections = extract_sections(latex)
+    if not sections:
+        return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)]
+    pairs: list[tuple[str | None, str]] = []
+    for title, body in sections:
+        for chunk in chunk_text(body, size, overlap):
+            pairs.append((title or None, chunk))
+    return pairs

codex/quality.py

@@ -0,0 +1,248 @@
+"""Chunk quality filtering and section classification (F-16).
+
+Three independent quality signals are applied at ingest time:
+
+1. **Length** — structural; no content required.
+2. **Alpha-ratio** — OCR artefacts have high non-alpha character density.
+3. **Bib-score** — DOI + "et al." + (YYYY) patterns co-occur almost only
+   in reference list entries.
+
+Section classification is rule-based (no LLM) and runs on the first 200
+characters of each chunk.  The retroactive ``run_quality_pass`` function
+can be called from the CLI to back-fill existing chunks.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from codex.config import Settings
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Section classification patterns (checked in order; first match wins)
+# ---------------------------------------------------------------------------
+
+_SECTION_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    ("abstract", re.compile(r"^\s*(abstract|zusammenfassung|r[eé]sum[eé])\b", re.I)),
+    ("intro", re.compile(r"^\s*(introduction|einleitung|1\.\s)", re.I)),
+    ("theorem", re.compile(r"^\s*(theorem|lemma|proposition|corollary|definition)\b", re.I)),
+    ("proof", re.compile(r"^\s*(proof\b|beweis\b|proof\s+of\b)", re.I)),
+    ("bibliography", re.compile(r"^\s*(references|bibliography|bibliographie|literatur)\b", re.I)),
+]
+
+_DOI_RE = re.compile(r"10\.\d{4,}/\S+")
+_ET_AL_RE = re.compile(r"\bet\s+al\b", re.I)
+_YEAR_BRACKETS_RE = re.compile(r"\(\d{4}\)")
+
+
+# ---------------------------------------------------------------------------
+# Bibliography heuristic score
+# ---------------------------------------------------------------------------
+
+
+def _bib_score(text: str) -> float:
+    """Return a [0..1] heuristic for how bibliography-like a chunk is.
+
+    Three signals contribute:
+    * DOI occurrences (weight 3)
+    * "et al." occurrences (weight 2)
+    * year-in-brackets occurrences (weight 1)
+
+    The raw signal is normalised against a rough word-count proxy so that
+    long chunks with occasional references don't get flagged.
+    """
+    if not text:
+        return 0.0
+    word_count = max(len(text.split()), 1)
+    doi_hits = len(_DOI_RE.findall(text))
+    etal_hits = len(_ET_AL_RE.findall(text))
+    year_hits = len(_YEAR_BRACKETS_RE.findall(text))
+    raw = (doi_hits * 3 + etal_hits * 2 + year_hits) / max(word_count / 10, 1)
+    return min(raw, 1.0)
+
+
+# ---------------------------------------------------------------------------
+# Quality predicate
+# ---------------------------------------------------------------------------
+
+
+def is_quality_chunk(text: str, *, settings: Settings) -> bool:
+    """Return True when *text* passes all three quality thresholds.
+
+    Checks (all configurable via :class:`codex.config.Settings`):
+
+    1. ``chunk_min_chars`` — character count floor.
+    2. ``chunk_min_alpha_ratio`` — minimum fraction of alphabetic chars.
+    3. ``chunk_max_bib_score`` — bibliography heuristic ceiling.
+    """
+    if len(text) < settings.chunk_min_chars:
+        return False
+    alpha_ratio = sum(c.isalpha() for c in text) / max(len(text), 1)
+    if alpha_ratio < settings.chunk_min_alpha_ratio:
+        return False
+    return _bib_score(text) <= settings.chunk_max_bib_score
+
+
+# ---------------------------------------------------------------------------
+# Section classification
+# ---------------------------------------------------------------------------
+
+
+def classify_section(text: str) -> str:
+    """Classify a chunk's section using rule-based regex matching.
+
+    Inspects only the first 200 characters.  Returns one of:
+    ``abstract``, ``intro``, ``theorem``, ``proof``, ``bibliography``, ``body``.
+    """
+    snippet = text[:200]
+    for section_name, pattern in _SECTION_PATTERNS:
+        if pattern.search(snippet):
+            return section_name
+    # Fallback: bibliography by DOI density even without a header
+    if _bib_score(text) > 0.5:
+        return "bibliography"
+    return "body"
+
+
+def _clean_title(title: str) -> str:
+    """Normalize a LaTeX ``\\section`` title for use as a section label.
+
+    Strips LaTeX word-commands (``\\emph`` …), accent/escape macros (``\\"o`` →
+    ``o``, ``\\&``), ``~`` ties and ``{}$``, drops leading section numbering
+    (``3.2 ``), collapses whitespace, lower-cases, and truncates to 60 chars at a
+    word boundary so the stored ``section`` value is a clean, comparable string.
+    """
+    t = title.replace("~", " ")  # LaTeX non-breaking tie → space
+    t = re.sub(r"\\[a-zA-Z]+\*?", " ", t)  # word-commands: \emph, \mathcal …
+    t = re.sub(r"\\[^a-zA-Z]", "", t)  # accent/escape macros: \"o → o, \', \`, \&
+    t = re.sub(r"[{}$]", "", t)  # braces / math delimiters
+    t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t)  # leading numbering "3.2 "
+    t = " ".join(t.split()).strip().lower()
+    return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t
+
+
+# A heading collapses to a controlled bucket only when it IS one of these canonical
+# section words — NOT when it merely starts with one. Prefix-matching via
+# classify_section mislabels real sections ("Abstract Nonsense and Categories" →
+# abstract, "References Architecture" → bibliography); an exact map avoids that while
+# keeping cross-source consistency for the bare headings.
+_SECTION_TITLE_BUCKETS = {
+    "abstract": "abstract",
+    "zusammenfassung": "abstract",
+    "introduction": "intro",
+    "einleitung": "intro",
+    "proof": "proof",
+    "beweis": "proof",
+    "references": "bibliography",
+    "bibliography": "bibliography",
+    "bibliographie": "bibliography",
+    "literatur": "bibliography",
+}
+
+
+def section_label(title: str | None, content: str) -> str:
+    """Label a chunk's section, preferring the real ``\\section`` heading (R-F).
+
+    For section-aware ``.tex`` ingest *title* is the real heading: a bare canonical
+    heading ("Introduction", "References", "Proof") maps to its controlled bucket
+    (cross-source consistency); any other heading is stored as its cleaned real
+    title (e.g. ``"preliminaries"``, ``"introduction to operator algebras"``) — far
+    more signal than collapsing to ``body`` AND without the prefix-match mislabels
+    that ``classify_section`` would produce on descriptive titles.
+
+    When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass``
+    paths) this falls back to content-based :func:`classify_section` — unchanged
+    behaviour. Safe to store free-text titles because the ``section`` column is
+    write-only (no consumer filters on the controlled vocabulary).
+    """
+    if not title or not title.strip():
+        return classify_section(content)
+    cleaned = _clean_title(title)
+    if not cleaned:
+        return classify_section(content)
+    return _SECTION_TITLE_BUCKETS.get(cleaned, cleaned)
+
+
+# ---------------------------------------------------------------------------
+# Batch filter
+# ---------------------------------------------------------------------------
+
+
+def filter_chunks(chunks: list[str], *, settings: Settings) -> list[str]:
+    """Return only the chunks that pass all quality filters.
+
+    Logs the keep ratio at DEBUG level.
+    """
+    kept = [c for c in chunks if is_quality_chunk(c, settings=settings)]
+    logger.debug(
+        "Quality filter: %d/%d chunks kept (%.0f%%)",
+        len(kept),
+        len(chunks),
+        100 * len(kept) / max(len(chunks), 1),
+    )
+    return kept
+
+
+# ---------------------------------------------------------------------------
+# Retroactive DB pass
+# ---------------------------------------------------------------------------
+
+
+def run_quality_pass(
+    *,
+    paper_id: str | None = None,
+    conn: Any,
+    settings: Settings,
+) -> dict[str, int]:
+    """Apply quality filter + section classification to existing chunks in DB.
+
+    For each chunk:
+    * Fails quality → DELETE.
+    * Passes quality → UPDATE ``section`` with :func:`classify_section`.
+
+    Parameters
+    ----------
+    paper_id:
+        When provided, restrict the pass to chunks for this paper only.
+    conn:
+        Open psycopg connection (dict-row factory assumed).
+    settings:
+        Application settings for quality thresholds.
+
+    Returns
+    -------
+    dict with keys ``kept``, ``removed``, ``tagged`` (= kept).
+    """
+    if paper_id is not None:
+        rows = conn.execute(
+            "SELECT id, content FROM chunks WHERE paper_id = %(pid)s",
+            {"pid": paper_id},
+        ).fetchall()
+    else:
+        rows = conn.execute("SELECT id, content FROM chunks").fetchall()
+
+    kept = 0
+    removed = 0
+    for row in rows:
+        chunk_id = row["id"]
+        content = row["content"]
+        if not is_quality_chunk(content, settings=settings):
+            conn.execute("DELETE FROM chunks WHERE id = %(id)s", {"id": chunk_id})
+            removed += 1
+        else:
+            section = classify_section(content)
+            conn.execute(
+                "UPDATE chunks SET section = %(section)s WHERE id = %(id)s",
+                {"section": section, "id": chunk_id},
+            )
+            kept += 1
+
+    conn.commit()
+    logger.info("Quality pass done: %d kept, %d removed, %d section-tagged", kept, removed, kept)
+    return {"kept": kept, "removed": removed, "tagged": kept}

codex/sources/arxiv.py

@@ -1,21 +1,110 @@
 """arXiv API client.
 
 Provides:
+- fetch_metadata: resolve an arXiv ID to a Paper via the Atom export API.
 - fetch_source: download the .tar.gz source of a paper and extract the primary .tex file.
 - fetch_pdf_url: return the canonical PDF URL for a given arXiv ID.
 """
 
 from __future__ import annotations
 
+import gzip
 import io
 import logging
 import tarfile
+import xml.etree.ElementTree as ET
 
 import httpx
+from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
+
+from codex.models import Paper
 
 logger = logging.getLogger(__name__)
 
 _BASE = "https://arxiv.org"
+_EXPORT = "https://export.arxiv.org"
+_ATOM = "{http://www.w3.org/2005/Atom}"
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code == 429 or exc.response.status_code >= 500
+    return False
+
+
+@retry(
+    retry=retry_if_exception(_is_retryable),
+    stop=stop_after_attempt(4),
+    wait=wait_exponential(min=1, max=20),
+    before_sleep=lambda rs: logger.warning("arXiv retry %d", rs.attempt_number),
+)
+def _query(arxiv_id: str) -> httpx.Response:
+    response = httpx.get(
+        f"{_EXPORT}/api/query",
+        params={"id_list": arxiv_id},
+        timeout=30,
+        follow_redirects=True,
+    )
+    response.raise_for_status()
+    return response
+
+
+def fetch_metadata(arxiv_id: str) -> Paper | None:
+    """Resolve an arXiv ID to a Paper via the Atom export API.
+
+    Authoritative metadata source for arXiv preprints — used as an ingest
+    fallback when OpenAlex 404s on an arXiv id (DQ-2). The arXiv id (bare,
+    e.g. ``"1911.00966"`` or legacy ``"math/0603097"``) becomes ``Paper.id``;
+    ``openalex_id`` and ``bibkey`` are left for the caller to populate.
+
+    Returns
+    -------
+    Paper | None
+        Populated Paper (title, authors, year, abstract), or None if arXiv
+        has no entry for the id.
+    """
+    bare = arxiv_id[len("arxiv:") :] if arxiv_id.lower().startswith("arxiv:") else arxiv_id
+    try:
+        response = _query(bare)
+    except httpx.HTTPError:
+        logger.warning("arXiv metadata fetch failed for %s", bare, exc_info=True)
+        return None
+    try:
+        root = ET.fromstring(response.text)
+    except ET.ParseError:
+        return None
+    entry = root.find(f"{_ATOM}entry")
+    if entry is None:
+        return None
+    # An id-not-found query still returns a feed but with no <entry>.
+    title = (entry.findtext(f"{_ATOM}title") or "").strip()
+    if not title:
+        return None
+    summary = (entry.findtext(f"{_ATOM}summary") or "").strip()
+    published = entry.findtext(f"{_ATOM}published") or ""
+    year = int(published[:4]) if published[:4].isdigit() else None
+    authors = [
+        name.strip()
+        for a in entry.findall(f"{_ATOM}author")
+        if (name := a.findtext(f"{_ATOM}name")) and name.strip()
+    ]
+    return Paper(
+        id=bare,
+        title=" ".join(title.split()),
+        authors=authors,
+        year=year,
+        abstract=" ".join(summary.split()) or None,
+    )
+
+
+def _has_documentclass(content: str) -> bool:
+    """True if *content* has an UN-commented ``\\documentclass`` line.
+
+    A plain ``"\\documentclass" in content`` substring test also matches a
+    commented-out ``%\\documentclass`` (common in arXiv preambles); requiring the
+    command to start its line avoids selecting the wrong primary file.
+    """
+    return any(line.lstrip().startswith("\\documentclass") for line in content.splitlines())
 
 
 def fetch_source(arxiv_id: str) -> str | None:
@@ -23,8 +112,9 @@ def fetch_source(arxiv_id: str) -> str | None:
 
     Downloads the .tar.gz source bundle from ``https://arxiv.org/src/{arxiv_id}``,
     locates the primary .tex file (preferring any file containing ``\\documentclass``,
-    falling back to the largest .tex by size), and returns its contents as a UTF-8
+    falling back to the largest .tex by size), and inlines its ``\\input``/
-    string.
+    ``\\include`` directives from the other archive members so multi-file projects
+    return the full document body, not just the primary file's include skeleton.
 
     Parameters
     ----------
@@ -38,49 +128,65 @@ def fetch_source(arxiv_id: str) -> str | None:
         file is present (signals Nougat fallback).
     """
     url = f"{_BASE}/src/{arxiv_id}"
-    try:
+    response = httpx.get(url, timeout=60, follow_redirects=True)
-        response = httpx.get(url, timeout=60, follow_redirects=True)
-    except httpx.RequestError:
-        raise
     if response.status_code == 404:
         logger.debug("arXiv 404 for source id=%s", arxiv_id)
         return None
     if response.status_code != 200:
         response.raise_for_status()
 
+    from codex.parsing.tex import flatten_inputs
+
     raw = response.content
+    # Image/font/binary members are never \input'd as text; skip them so the source
+    # map stays text-only (decode-with-replace would otherwise store garbled bytes).
+    skip_ext = (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".pdf", ".eps", ".ps", ".ttf", ".otf")
     try:
         with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf:
-            tex_members = [m for m in tf.getmembers() if m.name.endswith(".tex")]
+            files: dict[str, str] = {}
-            if not tex_members:
+            primary_name: str | None = None
-                logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
+            for member in tf.getmembers():
-                return None
+                if not member.isfile() or member.name.lower().endswith(skip_ext):
-
+                    continue
-            # Prefer the file containing \documentclass (primary document)
-            primary: tarfile.TarInfo | None = None
-            for member in tex_members:
                 f = tf.extractfile(member)
                 if f is None:
                     continue
-                content_bytes = f.read()
+                # Keep ALL text members resolvable so \input of a non-.tex include
-                if b"\\documentclass" in content_bytes:
+                # (e.g. a .def or extension-less body file) is not silently dropped (R-C).
-                    primary = member
+                files[member.name] = f.read().decode("utf-8", errors="replace")
-                    # Decode and return immediately — first match wins
+                # Primary doc = a .tex with an UN-commented \documentclass.
-                    return content_bytes.decode("utf-8", errors="replace")
+                if (
+                    primary_name is None
+                    and member.name.endswith(".tex")
+                    and _has_documentclass(files[member.name])
+                ):
+                    primary_name = member.name
 
-            if primary is None:
+            tex_files = {k: v for k, v in files.items() if k.endswith(".tex")}
-                # Fallback: largest .tex by size
+            if not tex_files:
-                largest = max(tex_members, key=lambda m: m.size)
+                logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
-                f = tf.extractfile(largest)
+                return None
-                if f is None:
+
-                    return None
+            # No \documentclass anywhere → fall back to the largest .tex.
-                return f.read().decode("utf-8", errors="replace")
+            if primary_name is None:
-    except tarfile.TarError as exc:
+                primary_name = max(tex_files, key=lambda k: len(tex_files[k]))
-        logger.warning("Failed to open tar archive for %s: %s", arxiv_id, exc)
+
+            # Inline \input/\include so multi-file projects yield the full body,
+            # not just the primary file's skeleton of include directives (R-C).
+            return flatten_inputs(files[primary_name], files)
+    except tarfile.TarError:
+        # Old arXiv submissions are served as a single bare gzipped .tex with no
+        # tar wrapper (e.g. legacy math/* papers) — gunzip the body directly.
+        try:
+            single = gzip.decompress(raw).decode("utf-8", errors="replace")
+        except (OSError, EOFError) as exc:
+            logger.warning("Failed to read arXiv source for %s: %s", arxiv_id, exc)
+            return None
+        if "\\" in single[:5000]:  # looks like LaTeX (has backslash commands)
+            return single
+        logger.debug("arXiv source for %s is not LaTeX", arxiv_id)
         return None
 
-    return None  # unreachable but satisfies type checker
-
 
 def fetch_pdf_url(arxiv_id: str) -> str:
     """Return the canonical PDF URL for an arXiv paper.

codex/sources/crossref.py

@@ -0,0 +1,126 @@
+"""Crossref API client.
+
+Provides:
+- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI.
+- fetch_references: retrieve a work's reference list (DOI edges) by DOI.
+
+Crossref is a *third* metadata source after OpenAlex and Semantic Scholar
+(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto
+query parameter) and are retried on 429/5xx with exponential back-off via tenacity.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+import httpx
+from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
+
+from codex.config import get_settings
+from codex.models import Citation
+
+logger = logging.getLogger(__name__)
+
+_BASE = "https://api.crossref.org"
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code == 429 or exc.response.status_code >= 500
+    return False
+
+
+def _polite_params() -> dict[str, str]:
+    # Reuse the OpenAlex mailto for Crossref's Polite Pool (same address).
+    mailto = get_settings().openalex_mailto
+    return {"mailto": mailto} if mailto else {}
+
+
+@retry(
+    retry=retry_if_exception(_is_retryable),
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(min=1, max=30),
+    before_sleep=lambda rs: logger.warning(
+        "Crossref retry %d after %s",
+        rs.attempt_number,
+        rs.outcome.exception(),  # type: ignore[union-attr]
+    ),
+)
+def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+    merged: dict[str, str] = {**_polite_params(), **(params or {})}
+    response = httpx.get(url, params=merged, timeout=30, follow_redirects=True)
+    response.raise_for_status()
+    return response
+
+
+def _strip_jats(abstract: str | None) -> str | None:
+    """Strip JATS/XML tags from a Crossref abstract and collapse whitespace."""
+    if not abstract:
+        return None
+    text = _TAG_RE.sub("", abstract)
+    text = " ".join(text.split()).strip()
+    return text or None
+
+
+def fetch_abstract(doi: str) -> str | None:
+    """Fetch a work's abstract from Crossref by DOI.
+
+    Crossref stores abstracts as JATS-tagged markup (``<jats:p>…</jats:p>``);
+    tags are stripped before returning. Many works (notably book chapters)
+    have no abstract deposited — returns None in that case and on 404.
+
+    Parameters
+    ----------
+    doi:
+        A bare DOI (``"10.1007/s00454-019-00132-8"``).
+
+    Returns
+    -------
+    str | None
+        The plain-text abstract, or None when absent.
+    """
+    url = f"{_BASE}/works/{doi}"
+    try:
+        response = _get(url)
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return None
+        raise
+    message: dict[str, Any] = response.json().get("message", {})
+    return _strip_jats(message.get("abstract"))
+
+
+def fetch_references(doi: str) -> list[Citation]:
+    """Fetch a work's reference list from Crossref by DOI (roadmap R-B).
+
+    Crossref carries publisher-deposited reference lists in ``message.reference``.
+    Each entry that cites a DOI-registered work exposes a bare ``DOI`` field;
+    only those become citation-graph edges (book / older references carry just
+    ``unstructured`` text with no resolvable id and are skipped). ``citing_id``
+    is the queried *doi* — the caller rewrites it to the canonical ``papers.id``
+    and normalises the cited DOIs (mirrors the Semantic Scholar reference path).
+
+    Parameters
+    ----------
+    doi:
+        A bare DOI (``"10.1007/s00454-019-00132-8"``).
+
+    Returns
+    -------
+    list[Citation]
+        One Citation per reference that carries a DOI. Empty on 404 or when no
+        references are deposited for the work.
+    """
+    url = f"{_BASE}/works/{doi}"
+    try:
+        response = _get(url)
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return []
+        raise
+    message: dict[str, Any] = response.json().get("message", {})
+    references: list[dict[str, Any]] = message.get("reference") or []
+    return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")]

codex/sources/openalex.py

@@ -85,13 +85,55 @@ def _reconstruct_abstract(inverted_index: dict[str, list[int]] | None) -> str |
     return " ".join(w for _, w in word_positions)
 
 
+def _normalize_doi(doi: str) -> str:
+    """Normalize an OpenAlex DOI to the canonical bare, lower-cased form.
+
+    OpenAlex returns the ``doi`` field as a full URL
+    (``https://doi.org/10.1007/s00454-019-00132-8``), but the canonical
+    ``papers.id`` produced by the M-1 migration is the BARE, lower-cased DOI
+    (``10.1007/s00454-019-00132-8``). Stripping the ``https://doi.org/`` /
+    ``doi:`` prefix and lower-casing (DOIs are case-insensitive by spec) makes a
+    re-ingested DOI paper match its stored row, so the ingest
+    ``INSERT … ON CONFLICT (id)`` updates in place instead of attempting a fresh
+    INSERT that trips the separate ``papers_openalex_id_key`` unique constraint
+    (DQ-5). Mirrors the cited-id convention in ``ingest._norm_cited_id``.
+    """
+    s = doi.strip().lower()
+    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
+        if s.startswith(prefix):
+            return s[len(prefix) :]
+    return s
+
+
+# arXiv works are registered under a DOI of the form 10.48550/arXiv.<id>.
+_ARXIV_DOI_PREFIX = "10.48550/arxiv."
+
+
+def _canonical_id(doi: str) -> str:
+    """Return the canonical ``papers.id`` for an OpenAlex ``doi`` value.
+
+    Builds on :func:`_normalize_doi` (bare, lower-cased DOI), then strips the
+    arXiv DOI prefix ``10.48550/arxiv.`` so an arXiv work's id is the BARE arXiv
+    id (``2305.10988`` / ``math/0603097``) — the form the corpus is keyed on —
+    not the arXiv DOI. Without this, re-ingesting an arXiv paper by its bare id
+    misses ``INSERT … ON CONFLICT (id)`` and trips ``papers_openalex_id_key``.
+    This is the arXiv half of DQ-5 (the DOI half was fixed there; this was
+    explicitly deferred and surfaced again driving the R-C .tex re-ingest).
+    """
+    normalized = _normalize_doi(doi)
+    if normalized.startswith(_ARXIV_DOI_PREFIX):
+        return normalized[len(_ARXIV_DOI_PREFIX) :]
+    return normalized
+
+
 def _map_paper(data: dict[str, Any]) -> Paper:
     authors: list[str] = [
         a.get("author", {}).get("display_name") or "" for a in data.get("authorships", [])
     ]
     abstract = _reconstruct_abstract(data.get("abstract_inverted_index"))
+    doi = data.get("doi")
     return Paper(
-        id=data.get("doi") or data.get("id") or "",
+        id=_canonical_id(doi) if doi else (data.get("id") or ""),
         title=data.get("title") or "",
         openalex_id=data.get("id"),
         authors=authors,

codex/sources/semanticscholar.py

@@ -14,6 +14,7 @@ import logging
 import threading
 import time
 from typing import Any
+from urllib.parse import quote
 
 import httpx
 from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
@@ -77,7 +78,9 @@ def fetch_references(paper_id: str) -> list[Citation]:
     list[Citation]
         One Citation per reference, with optional context snippet.
     """
-    url = f"{_BASE_GRAPH}/paper/{paper_id}/references"
+    # quote the id so a legacy-arXiv "/" (e.g. arXiv:math/0603097) doesn't break
+    # the URL path; keep ":" for the arXiv:/DOI: scheme prefix (audit R-5).
+    url = f"{_BASE_GRAPH}/paper/{quote(paper_id, safe=':')}/references"
     params: dict[str, Any] = {"fields": "externalIds,contexts"}
     try:
         response = _get(url, params=params)
@@ -87,7 +90,11 @@ def fetch_references(paper_id: str) -> list[Citation]:
         raise
 
     data = response.json()
-    raw_refs: list[dict[str, Any]] = data.get("data", [])
+    # S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
+    # parsed reference list (e.g. some theses). ``.get("data", [])`` would yield
+    # the explicit ``None`` (the default only applies when the key is absent),
+    # so iterate over a guaranteed list instead.
+    raw_refs: list[dict[str, Any]] = data.get("data") or []
     citations: list[Citation] = []
     for entry in raw_refs:
         cited_paper: dict[str, Any] = entry.get("citedPaper", {})
@@ -96,16 +103,31 @@ def fetch_references(paper_id: str) -> list[Citation]:
         context: str | None = contexts[0] if contexts else None
 
         cited_id: str = (
-            external_ids.get("DOI")
+            external_ids.get("DOI") or external_ids.get("ArXiv") or cited_paper.get("paperId") or ""
-            or external_ids.get("ArXiv")
-            or cited_paper.get("paperId")
-            or ""
         )
         if cited_id:
             citations.append(Citation(citing_id=paper_id, cited_id=cited_id, context=context))
     return citations
 
 
+def fetch_abstract(paper_id: str) -> str | None:
+    """Fetch just the abstract for a paper from Semantic Scholar.
+
+    Used as an ingest abstract supplement (DQ-2) when OpenAlex has the paper
+    but no abstract. ``paper_id`` should be namespaced (``arXiv:…`` / ``DOI:…``).
+    Returns None on 404 or when S2 has no abstract.
+    """
+    url = f"{_BASE_GRAPH}/paper/{paper_id}"
+    try:
+        response = _get(url, params={"fields": "abstract"})
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return None
+        raise
+    abstract = response.json().get("abstract")
+    return abstract or None
+
+
 def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]:
     """Fetch recommended paper IDs from Semantic Scholar.
 
@@ -121,7 +143,7 @@ def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]:
     list[str]
         List of recommended paper IDs.
     """
-    url = f"{_BASE_RECS}/papers/forpaper/{paper_id}"
+    url = f"{_BASE_RECS}/papers/forpaper/{quote(paper_id, safe=':')}"
     params: dict[str, Any] = {"limit": limit}
     try:
         response = _get(url, params=params)

codex/synthesis.py

@@ -248,9 +248,23 @@ def _grounded_ratio(claims: list[Claim]) -> float:
     return n_grounded / len(claims)
 
 
-def _make_lead_id(seq: int) -> str:
+_KIND_PREFIX: dict[LeadKind, str] = {
-    """Format a sequential lead id like ``L-0001``."""
+    "connection": "C",
-    return f"L-{seq:04d}"
+    "gap": "G",
+    "improvement": "I",
+    "conjecture": "X",
+}
+
+
+def _make_lead_id(seq: int, kind: LeadKind) -> str:
+    """Format a per-kind sequential lead id like ``L-C-0001``.
+
+    Each synthesis stage numbers its leads from 1, so without a kind prefix a
+    connection, a gap, and an improvement all produce ``L-0001`` and clobber
+    one another in the shared ``grounded/`` directory (audit C-11). The kind
+    prefix namespaces the ids so they stay distinct.
+    """
+    return f"L-{_KIND_PREFIX[kind]}-{seq:04d}"
 
 
 def _safe_llm_generate(llm: LLMClient, prompt: str, model: str) -> str:
@@ -296,7 +310,7 @@ def _finalise_grounded_lead(
         return None
     marked = _mark_ungrounded(body, claims)
     return Lead(
-        id=_make_lead_id(seq),
+        id=_make_lead_id(seq, kind),
         kind=kind,
         title=title,
         body=marked,
@@ -365,11 +379,7 @@ def find_connections(
                 continue
             seen_pairs.add(pair)  # type: ignore[arg-type]
 
-            pair_chunks = [
+            pair_chunks = [c for c in chunks if c.get("bibkey") in {bibkey, other}]
-                c
-                for c in chunks
-                if c.get("bibkey") in {bibkey, other}
-            ]
             prompt = _CONNECTION_PROMPT.format(chunks_block=_format_chunks(pair_chunks))
             raw = _safe_llm_generate(llm, prompt, settings.synthesis_llm_model)
             if not raw.strip():
@@ -406,11 +416,12 @@ def find_gaps(
 
     Two complementary signals are used:
 
-    * **Topic coverage map** — for each topic in ``topics`` (or, if
+    * **Topic coverage map** — for each **explicitly-requested** topic in
-      ``None``, the union of paper titles), retrieve chunks and check
+      ``topics``, retrieve chunks and check bibkey coverage. A topic covered by
-      bibkey coverage. A topic covered by < ``synthesis_gap_min_coverage``
+      < ``synthesis_gap_min_coverage`` bibkeys is a gap candidate and the LLM is
-      bibkeys is reported as a gap candidate and the LLM is asked to
+      asked to articulate it. Defaulting to paper titles flagged almost every
-      articulate it.
+      paper as a gap (a title retrieves mostly its own single bibkey), so the
+      coverage map is opt-in now; pass ``topics`` / CLI ``--topic`` (audit R-8).
     * **Code-vs-corpus gap** — when ``lib_path`` is given, any @cite
       bibkey scanned from code that is NOT present in the corpus is
       flagged. This catches the "cited in code, never ingested" case.
@@ -419,9 +430,9 @@ def find_gaps(
     seq = 1
     leads: list[Lead] = []
 
+    # Coverage-map probes are the explicit topics only — no default-to-titles,
+    # which turned every paper into a spurious gap (audit R-8).
     probe_topics: list[str] = list(topics or [])
-    if not probe_topics:
-        probe_topics = [title for _, title in _list_paper_titles(db_conn)]
 
     known_bibkeys = {bk for bk, _ in _list_paper_titles(db_conn)}
 
@@ -432,7 +443,7 @@ def find_gaps(
             # No coverage at all → record the gap directly (LLM not strictly needed).
             leads.append(
                 Lead(
-                    id=_make_lead_id(seq),
+                    id=_make_lead_id(seq, "gap"),
                     kind="gap",
                     title=f"Gap: corpus has no material on '{topic}'",
                     body=(
@@ -485,7 +496,7 @@ def find_gaps(
         for bibkey in sorted(missing):
             leads.append(
                 Lead(
-                    id=_make_lead_id(seq),
+                    id=_make_lead_id(seq, "gap"),
                     kind="gap",
                     title=f"Gap: code cites '{bibkey}' but corpus does not contain it",
                     body=(
@@ -662,7 +673,7 @@ def propose_conjectures(
             continue
         leads.append(
             Lead(
-                id=_make_lead_id(seq),
+                id=_make_lead_id(seq, "conjecture"),
                 kind="conjecture",
                 title=title,
                 body=body,
@@ -763,10 +774,11 @@ def write_leads(leads: list[Lead], leads_dir: str) -> None:
     * Grounded leads → ``leads_dir/grounded/<id>.md``
     * Conjectures   → ``leads_dir/conjectures/<id>.md`` (HARD invariant)
 
-    The function ALSO maintains an append-only ``leads_dir/INDEX.md``:
+    The function ALSO rebuilds ``leads_dir/INDEX.md`` from the on-disk files on
-    a previously-recorded id is kept on subsequent runs even if it is
+    every run (see :func:`_update_index`): it reflects whatever lead files
-    overwritten with a newer body. The two sections (grounded /
+    currently exist under ``grounded/`` and ``conjectures/``, listed as two
-    conjectures) are visibly separated.
+    visibly separated sections. It is not append-only — a deleted lead file
+    drops out of the index on the next run.
 
     Notes
     -----

codex/wiki.py

@@ -186,8 +186,15 @@ def _retrieve_chunks(
         ).fetchall()
 
     # Filter reference-list chunks: skip chunks whose content looks like a bibliography
-    # (heuristic: > 60 % of lines match "^\[\d+\]" or "^[A-Z][a-z]+,?\s+[A-Z]\.").
+    # (heuristic: > 60 % of lines look like "[12]" refs or "Surname, I." authors).
-    ref_pattern = re.compile(r"^\s*(\[\d+\]|[A-Z][a-z]+,?\s+[A-Z]\.)", re.MULTILINE)
+    # The author alternative excludes common math-structure words ("Theorem A.",
+    # "Lemma B.") so theorem-dense chunks aren't mistaken for a reference list (R-2).
+    ref_pattern = re.compile(
+        r"^\s*(\[\d+\]|(?!(?:Theorem|Lemma|Proposition|Corollary|Definition|Proof|"
+        r"Section|Figure|Fig|Table|Equation|Eq|Remark|Example|Chapter)\b)"
+        r"[A-Z][a-z]+,?\s+[A-Z]\.)",
+        re.MULTILINE,
+    )
     filtered: list[dict[str, Any]] = []
     for row in rows:
         content: str = row["content"]
@@ -240,13 +247,46 @@ def _parse_claims(markdown: str) -> list[Claim]:
 
 _STOPWORDS: frozenset[str] = frozenset(
     {
-        "the", "a", "an", "in", "on", "of", "to", "is", "are", "was", "were",
+        "the",
-        "and", "or", "but", "for", "with", "this", "that", "it", "we", "they",
+        "a",
-        "be", "as", "at", "by", "from", "has", "have", "not", "which",
+        "an",
+        "in",
+        "on",
+        "of",
+        "to",
+        "is",
+        "are",
+        "was",
+        "were",
+        "and",
+        "or",
+        "but",
+        "for",
+        "with",
+        "this",
+        "that",
+        "it",
+        "we",
+        "they",
+        "be",
+        "as",
+        "at",
+        "by",
+        "from",
+        "has",
+        "have",
+        "not",
+        "which",
     }
 )
 
-_SENTENCE_SPLIT_RE = re.compile(r"[.!?]")
+# Split on sentence-ending punctuation only when followed by whitespace/end, so a
+# decimal like "1.5" does not create a spurious sentence boundary (audit R-1).
+_SENTENCE_SPLIT_RE = re.compile(r"[.!?]+(?=\s|$)")
+
+# Punctuation stripped from token edges before grounding comparison — sentence/
+# clause marks and quotes, NOT math delimiters like ()=+ (audit R-1).
+_EDGE_PUNCT = ".,;:!?\"'"
 
 
 def _last_sentence(text: str) -> str:
@@ -265,8 +305,16 @@ def _last_sentence(text: str) -> str:
 
 
 def _content_words(text: str) -> list[str]:
-    """Return lowercased tokens with stopwords removed."""
+    """Return lowercased content tokens, edge-punctuation-stripped, stopwords removed.
-    return [w for w in text.lower().split() if w not in _STOPWORDS]
+
+    Stripping leading/trailing sentence punctuation makes grounding robust to
+    surface variation ("volume," vs "volume") so a faithful paraphrase is not
+    marked ungrounded over a comma (audit R-1). Math delimiters (``()=+``) are
+    preserved. Claim and chunk tokens are normalised identically, so matching
+    stays consistent.
+    """
+    tokens = (w.strip(_EDGE_PUNCT) for w in text.lower().split())
+    return [w for w in tokens if w and w not in _STOPWORDS]
 
 
 def _run_grounding_guard(
@@ -295,8 +343,9 @@ def _run_grounding_guard(
     for chunk in chunks:
         bk = str(chunk.get("bibkey") or "")
         if bk:
-            # Content words of the chunk (stopwords removed, lowercased)
+            # Content words of the chunk, space-padded so a 5-gram match is bounded
-            cw = " ".join(_content_words(chunk["content"]))
+            # by whole tokens, not a substring bleeding across words (audit C-6).
+            cw = " " + " ".join(_content_words(chunk["content"])) + " "
             bib_index.setdefault(bk, []).append(cw)
 
     for claim in claims:
@@ -318,7 +367,7 @@ def _run_grounding_guard(
 
         found = False
         for i in range(len(content) - 4):  # content-5-grams
-            gram = " ".join(content[i : i + 5])
+            gram = " " + " ".join(content[i : i + 5]) + " "  # padded: token-boundary match
             if any(gram in src for src in sources):
                 found = True
                 break
@@ -360,12 +409,8 @@ def _detect_conflicts(
         for bk2 in bibkeys[i + 1 :]:
             # Check if any chunk from bk1 contains a conflict keyword
             # and any chunk from bk2 also does — heuristic signal only
-            bk1_has_conflict = any(
+            bk1_has_conflict = any(_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1])
-                _CONFLICT_KEYWORDS.search(c) for c in by_bib[bk1]
+            bk2_has_conflict = any(_CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2])
-            )
-            bk2_has_conflict = any(
-                _CONFLICT_KEYWORDS.search(c) for c in by_bib[bk2]
-            )
             if bk1_has_conflict and bk2_has_conflict:
                 conflicts.append((slug, bk1, bk2))
 
@@ -379,6 +424,13 @@ def _detect_conflicts(
 
 _INLINE_CODE_RE = re.compile(r"`[^`]+`")
 
+# LaTeX math spans — protected from cross-ref injection so a concept name inside
+# a formula is not rewritten to a [[slug]] link, corrupting the LaTeX (audit R-9).
+_MATH_SPAN_RE = re.compile(
+    r"\$\$.*?\$\$|\$[^$\n]*?\$|\\\(.*?\\\)|\\\[.*?\\\]",
+    re.DOTALL,
+)
+
 
 def _inject_cross_refs(
     markdown: str,
@@ -388,11 +440,12 @@ def _inject_cross_refs(
     """Replace occurrences of other concept titles/aliases with ``[[slug]]`` links.
 
     Only exact case-insensitive whole-word matches outside of existing
-    ``[[…]]`` blocks or inline code spans are replaced.
+    ``[[…]]`` blocks, inline code spans, or LaTeX math spans are replaced.
-    Inline-code spans (`` `…` ``) are temporarily protected by null-byte
+    Inline-code (`` `…` ``) and math (``$…$``, ``$$…$$``, ``\\(…\\)``,
-    placeholders and restored after injection.
+    ``\\[…\\]``) spans are temporarily protected by null-byte placeholders and
+    restored after injection.
     """
-    # Step 1: protect inline-code spans from replacement
+    # Step 1: protect inline-code and math spans from replacement
     placeholders: dict[str, str] = {}
 
     def _protect(m: re.Match[str]) -> str:
@@ -401,6 +454,7 @@ def _inject_cross_refs(
         return key
 
     markdown = _INLINE_CODE_RE.sub(_protect, markdown)
+    markdown = _MATH_SPAN_RE.sub(_protect, markdown)  # don't rewrite names inside LaTeX (R-9)
 
     # Step 2: inject cross-refs on unprotected text
     for concept in all_concepts:
@@ -579,14 +633,14 @@ def compile_concept(
     claims = _parse_claims(raw_output)
     claims = _run_grounding_guard(claims, chunks)
 
-    _all_concepts = all_concepts or []
+    # Graceful: formula-embedding hook (F-09) — currently a no-op (audit C-12).
-    raw_output = _inject_cross_refs(raw_output, _all_concepts, concept.slug)
-
-    # Graceful: try to embed formula chunks (F-09) — skip if table missing
     _try_embed_formulas(concept, chunks)
 
     compiled_at = datetime.now(UTC)
+    # Mark ungrounded claims on the raw body FIRST, then inject cross-refs: a
+    # concept name inside a claim must not stop the ⚠ regex from matching (C-13).
     markdown = _render_page_markdown(concept, raw_output, claims, compiled_at)
+    markdown = _inject_cross_refs(markdown, all_concepts or [], concept.slug)
 
     return ConceptPage(
         concept=concept,
@@ -598,20 +652,13 @@ def compile_concept(
 
 
 def _try_embed_formulas(concept: Concept, chunks: list[dict[str, Any]]) -> None:
-    """Attempt to look up formula chunks for the concept — graceful no-op if F-09 absent."""
+    """Reserved hook for embedding F-09 formula chunks into a page.
-    try:
-        from codex.db import get_conn
 
-        with get_conn() as conn:
+    Currently a no-op — formula embedding is not implemented yet. Kept as a stable
-            # Check if formulas table exists
+    seam for compile_concept's call site. The previous body issued a per-concept
-            row = conn.execute(
+    DB round-trip (information_schema lookup) that did nothing (audit C-12).
-                "SELECT 1 FROM information_schema.tables WHERE table_name = 'formulas'"
+    """
-            ).fetchone()
+    return
-            if row is None:
-                return  # F-09 not present
-            # (Future: embed relevant raw_latex into the page)
-    except Exception:  # noqa: BLE001
-        return  # DB not reachable or other error — degrade gracefully
 
 
 # ---------------------------------------------------------------------------
@@ -699,17 +746,23 @@ def compile_all(
         # Detect conflicts between chunks from different bibkeys
         report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
 
-        # Quarantine check: compute grounding rate
+        # Quarantine check: compute grounding rate.
+        # A page with NO citations at all (total_claims == 0) carries zero grounding
+        # evidence — this also covers the LLM-outage case where compile_concept
+        # returns an empty page — and must NOT be published (audit C-4). Previously
+        # the `total_claims > 0` guard let such pages fall through to wiki/.
         total_claims = len(page.claims)
         grounded_claims = sum(1 for c in page.claims if c.grounded)
         grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
 
-        if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate:
+        if total_claims == 0 or grounding_rate < settings.wiki_min_grounding_rate:
-            # Quarantine: write to wiki/draft/ instead of wiki/
+            # Quarantine and do NOT update the compile-state, so a transient failure
-            draft_dir = wiki_dir / "draft"
+            # (e.g. LLM outage) is retried on the next run instead of being frozen as
-            draft_dir.mkdir(parents=True, exist_ok=True)
+            # "compiled". An empty body is not worth a draft file — just record it.
-            page_path = draft_dir / f"{concept.slug}.md"
+            if page.markdown.strip():
-            page_path.write_text(page.markdown, encoding="utf-8")
+                draft_dir = wiki_dir / "draft"
+                draft_dir.mkdir(parents=True, exist_ok=True)
+                (draft_dir / f"{concept.slug}.md").write_text(page.markdown, encoding="utf-8")
             report.quarantined.append(concept.slug)
         else:
             # Write page to wiki/

docs/SESSION-2026-06-17-open-items.md

@@ -0,0 +1,59 @@
+# Open items — session 2026-06-17 (handoff notice)
+
+Everything still open after this session. Done items are in
+`docs/audit/DATA-QUALITY-2026-06-15.md` (DQ-1..DQ-5 + roadmap R-A,R-B,R-C,R-D,R-F all
+DONE; R-E deferred). This file is only the **loose ends**.
+
+## 1. ✅ RESOLVED — Jetson is online (the tunnel had dropped, not the host)
+- What looked like the Jetson going offline was the **SSH tunnel** breaking, not the
+  host. The Jetson (`jetson.local`) is online. Re-open the tunnel with keepalive:
+  `ssh -o ServerAliveInterval=15 -f -N -L 5433:localhost:5432 alfred@jetson.local`.
+  Better still: run long ops in tmux-on-Jetson (item #4) to avoid the tunnel entirely.
+
+## 2. ✅ DONE — R-F accent re-clean
+- R-F is functionally DONE (section signal 34/329 → **314/329 = 95 %**). The remaining
+  cosmetic accent/tie artifacts in live `chunks.section` labels were re-cleaned in place
+  once the tunnel was back: applied `codex.quality._clean_title` to every label
+  containing `\` or `~` — **8 labels fixed**, no re-embed. Verified **0 artifacts**
+  remain. R-F fully complete.
+
+## 3. ✅ READY TO MERGE — all branches integrated + validated
+All five feature branches are merged (**zero conflicts**) into **`integration/roadmap`**
+(off `audit/loop-1`) and validated green: **391 tests pass, ruff + ruff-format + mypy
+clean, tree clean**. This is the single merge candidate for `main`.
+
+Contents (DQ-1..DQ-5 + R-A..R-F + infra docs):
+| Source branch | Contents |
+|---|---|
+| `audit/loop-1` (base) | DQ-1..DQ-5, **R-A** (GROBID backfill + native arm64 GROBID), this notice |
+| `feat/section-labels` (incl. `feat/tex-ingest`) | **R-C** (.tex ingest) + DQ-5 arXiv-id fix + gzip fallback + **R-F** (section labels) |
+| `feat/crossref-references` | **R-B** (Crossref refs) + GROBID-on-demand tweak |
+| `feat/graph-coverage-warning` | **R-D** (citing-coverage warning) |
+| `chore/infra-ssh-stability` | infra TODO (#4) |
+
+**To land on `main`** (user-triggered):
+```
+git checkout main && git merge --no-ff integration/roadmap
+```
+…or open a PR `integration/roadmap` → `main`. The individual feature branches are
+preserved unchanged for per-feature review if preferred.
+
+**Cleanup — branches prunable AFTER the merge lands** (not deleted here; destructive):
+- Already in `main`: `fix/cli-readme-drift`, `fix/d04-d05-ingest-fixes`.
+- Folded into `integration/roadmap`: the 5 feature branches + `audit/loop-1`.
+- Pre-session / stale (verify before pruning): `feat/F-15-literature-graph`,
+  `feat/F-16-chunk-quality`, `fix/audit-wave-1`/`-2`/`-3`, `scratch/spike-embed`.
+- Working tree is clean — only gitignored `.venv`/caches/`.env`; no stray tracked files.
+
+## 4. Infra TODO — tmux-on-Jetson for stable long ops (documented, not implemented)
+- `docs/infra/jetson-ssh-stability.md` (branch `chore/infra-ssh-stability`). The
+  Mac↔Jetson tunnel dropped repeatedly on multi-minute writes all session. Fix: run long
+  ops **on the Jetson in `tmux`** (DB is localhost there → no tunnel; survives SSH drops);
+  `autossh` for interactive tunnels. Open.
+
+## 5. Known limitations (documented, NOT action items)
+- `math/0306167` (old AMS-TeX, no `\section{}`) stays `section=body` — inherent.
+- The 2 edited-volume books yielded 0 GROBID-parsed DOI/arXiv refs in the R-A sweep —
+  inherent (per-chapter refs, no clean end-bibliography).
+- **DQ-6** (upstream PDF→`.txt` extraction fidelity, done outside codex-py) — flagged
+  earlier, largely **mooted** by R-A/R-C ingesting from PDF/`.tex` directly.

docs/adr/ADR-F15-literature-graph.md

@@ -0,0 +1,204 @@
+# ADR-F15 — Literature Graph / Citation PageRank
+
+**Status:** IMPL — GO after live-corpus spike 2026-06-15. See **Audit Update
+(2026-06-15)** at the end: the ID-format mismatch is now resolved in code
+(audit C-1, C-7) and the Spike Result numbers below predate that resolution.
+**Owners:** F-15 Worker (Sonnet)
+**Last updated:** 2026-06-15 (audit addendum)
+
+---
+
+## Context
+
+F-15 adds a citation graph layer on top of the corpus DB.  The goals are:
+
+1. **Hub detection** — identify which papers are most central in the citation
+   network (PageRank) so that `codex search paper --cite-boost` can
+   up-weight them in dense retrieval.
+2. **Bibliographic coupling / co-citation** — surface related papers that
+   share many references or are frequently cited together.
+3. **Dangling-citation discovery** — enumerate works cited by the corpus but
+   not yet ingested, serving as an acquisition queue.
+
+Primary risk: **degenerate PageRank** on a small, sparse corpus producing
+scores so flat that the boost has no practical effect or — worse — distorts
+results by promoting low-quality dangling works.
+
+---
+
+## Spike Result (2026-06-15)
+
+Run on the live Jetson corpus: **29 ingested papers, 590 citation edges,
+495 total graph nodes, 478 dangling nodes**.
+
+### Hub-Paper Ranking
+
+| Rank | OpenAlex ID | Year | Authors | Title | PR |
+|------|-------------|------|---------|-------|----|
+| 1 | W1974956622 | 1993 | Pinkall, Polthier | Computing Discrete Minimal Surfaces and Their Conjugates | 0.002282 |
+| 2 | W1964119857 | 1962 | Ford, Fulkerson | Flows in Networks | 0.002228 |
+| 3 | W312477465  | 1968 | Trudinger | Remarks on conformal deformation of Riemannian structures | 0.002204 |
+| 4 | W1595775335 | 1960 | Yamabe | On a deformation of Riemannian structures on compact manifolds | 0.002204 |
+| 5 | W2023551584 | 1996 | Cooper, Rivin | Combinatorial scalar curvature and rigidity of ball packings | 0.002204 |
+| 6 | W2159531493 | 1984 | Schoen | Conformal deformation of a Riemannian metric to constant scalar curvature | 0.002204 |
+| 7 | **W2163787581** | **2007** | **Bobenko, Springborn** | **A Discrete Laplace–Beltrami Operator for Simplicial Surfaces** | **0.002195** |
+| 8 | W3099917509 | 2012 | Mercat | Discrete Riemann Surfaces and the Ising Model | 0.002190 |
+| 9 | W2021879624 | 2008 | Mullen, Tong, et al. | Spectral Conformal Parameterization | 0.002164 |
+| 10 | W1976584051 | 2002 | Desbrun, Meyer, et al. | Intrinsic Parameterizations of Surface Meshes | 0.002159 |
+
+### Validation Against Manual Expectation
+
+| Criterion | Target | Result |
+|-----------|--------|--------|
+| Bobenko or Springborn in top-10 hubs | ✓ expected hub | **PASS — Bobenko & Springborn 2007 at rank 7** |
+| Pinkall & Polthier 1993 prominent | ✓ foundational DGP paper | **PASS — rank 1** |
+| Score flatness acceptable | < 2× spread over top-10 | PASS — 5.7 % spread (0.002282 → 0.002159) |
+| Graceful small-corpus warning | shown when < 15 in-KB papers | **PASS — shown at 29 in-KB papers** (< `graph_min_corpus_size=15` not triggered; Springborn 2008 is not in-KB so not visible as "ingested hub") |
+
+**GO** — PageRank produces a plausible field-specific ranking.  The top hits
+(Pinkall/Polthier, Yamabe, Schoen, Bobenko/Springborn) are exactly the
+foundational works a DGP researcher would expect to see elevated.
+
+---
+
+## Decision
+
+### Graph Construction
+
+`build_citation_graph` builds an in-memory `nx.DiGraph` from the `citations`
+table: `citing_id → cited_id`.  No schema migration needed.
+
+- `citing_id` stores DOIs (matches `papers.id`).
+- `cited_id` stores OpenAlex IDs (external references).
+
+**Consequence:** in the current corpus, ingested papers do not appear as
+in-degree targets of each other — inter-KB citations are structurally
+invisible because the `cited_id` column uses OpenAlex IDs while `papers.id`
+uses DOIs.  The graph is therefore bipartite-like (29 DOI-nodes →
+478 OpenAlex-nodes), and all PageRank mass concentrates on the dangling
+OpenAlex-nodes cited by many of our 29 papers.
+
+This is acceptable for the `--cite-boost` use-case: dangling hub papers are
+*foundational works* our corpus cites repeatedly, so boosting search results
+that match them is desirable.
+
+### PageRank Graceful Degradation
+
+`citation_pagerank` returns a **uniform distribution** when the graph has
+fewer than 5 nodes (`_MIN_PAGERANK_NODES`) and logs a warning.
+
+The CLI `graph report` emits a separate user-visible warning when the count
+of in-KB papers is below `graph_min_corpus_size` (default 15) — because at
+that point the *hub ordering* (which uses PageRank over all 495 nodes) is
+still valid, but its *discriminative power* is limited by the few citing
+sources.
+
+At 29 ingested papers, `graph_min_corpus_size=15` is satisfied; the score
+spread is narrow (5.7 %) but the ranking order is meaningful.
+
+### cite-boost Formula
+
+```
+boosted_distance = distance / (1.0 + alpha * pr_score)
+```
+
+`alpha = 0.3` (config: `graph_cite_boost_alpha`).  Dividing reduces cosine
+distance for high-PR papers (lower = closer in pgvector).  Multiplying
+would invert the boost — this was the critical bug corrected during the
+F-15 review gate.
+
+At `alpha=0.3` and the observed PR scores (~0.002), the maximum boost is
+approximately 0.06 % per unit score — effectively a tie-breaker on equally
+relevant papers rather than a hard re-ranking.  This is intentional: we do
+not want citation authority to override semantic relevance for queries that
+genuinely target niche papers.
+
+### Dangling Citations as Acquisition Queue
+
+`dangling_citations(graph, known_ids)` returns the 478 OpenAlex IDs that
+are cited but not ingested.  `codex graph report` surfaces these so the
+researcher can prioritize which papers to add to the corpus next.
+
+The top hubs (rank 1–10) are the highest-value acquisition targets: adding
+Pinkall & Polthier 1993, Yamabe 1960, and Schoen 1984 would improve both
+retrieval quality and PageRank discrimination.
+
+---
+
+## Fallback
+
+If the corpus has < `_MIN_PAGERANK_NODES` (5) nodes: `citation_pagerank`
+returns uniform scores; `--cite-boost` has no effect (each paper is boosted
+equally); no crash.
+
+If `--cite-boost` is omitted: search works as F-03/F-04 baseline.
+
+If the `citations` table is empty: graph has 0 nodes; `graph report` prints
+"Graph is empty" and exits 0; all other commands degrade gracefully.
+
+---
+
+## Known Limitation: ID-Format Mismatch
+
+> **RESOLVED (audit 2026-06-15) — see Audit Update below.** This no longer holds:
+> `build_citation_graph` resolves OpenAlex `cited_id`s to the canonical
+> `papers.id` via the shared `RESOLVED_CITATIONS_SQL` (C-1) and ingest stores
+> canonical ids (C-7), so inter-KB citations *are* now edges. Original text kept
+> for the record.
+
+`cited_id` uses OpenAlex IDs; `papers.id` uses DOIs.  Cross-citations
+between ingested papers are therefore not represented as graph edges.
+This is a D-05-class issue (same root cause as the `citing_id`/`openalex_id`
+FK mismatch fixed in PR #9).  A future migration could add an `openalex_id`
+column to `papers` and populate `cited_id` cross-links; but the benefit is
+small while the corpus is < 100 papers.
+
+---
+
+## Consequences
+
+- R-41: `build_citation_graph` — in-memory DiGraph, no schema change ✓
+- R-42: `citation_pagerank` + `codex search paper --cite-boost` ✓
+- R-43: `find_related` (bibliographic coupling) + `find_co_cited` ✓
+- R-44: `codex graph report [--top-n N] [--json]` ✓
+- R-45: graceful degradation < 5 nodes: uniform scores, warning ✓
+- R-46: **this document** — GO; Bobenko+Springborn 2007 at rank 7 validates hub detection; score flatness expected at corpus size 29; increase to > 100 papers for stronger PageRank differentiation
+
+---
+
+## Audit Update (2026-06-15)
+
+A post-hoc audit of the shipped F-15 code found that the live-corpus spike above
+was measured on a graph the current code no longer produces:
+
+- **ID-format mismatch resolved.** `build_citation_graph` resolves OpenAlex
+  `cited_id`s to the canonical `papers.id` through the shared
+  `RESOLVED_CITATIONS_SQL`, now also used by `codex.discover` so the two cannot
+  diverge (audit C-1). Ingest stores the caller's canonical id as `papers.id`
+  instead of OpenAlex's URL-form doi (audit C-7). Inter-KB citations are
+  therefore represented as edges — the "Known Limitation" above is obsolete.
+- **Spike re-measured after migration (2026-06-15).** The corpus was re-ingested
+  onto canonical bare ids (29 papers, all bare — verified `url_form_ids = 0`), and
+  the graph re-run on the post-migration corpus:
+
+  | Metric | Original spike (pre-fix) | Post-migration (re-measured) |
+  |--------|--------------------------|------------------------------|
+  | Ingested papers | 29 | 29 |
+  | Graph nodes | 495 | 489 |
+  | Graph edges | 590 | 590 |
+  | Dangling nodes | 478 | 472 |
+  | Top-10 PR spread | 5.7 % | 5.4 % |
+  | Rank-1 hub | W1974956622 (dangling) | **Pinkall/Polthier 1993 — in-KB (resolved)** |
+  | Bobenko/Springborn 2007 | rank 7 | rank 7 |
+
+  The resolution fix's signature is visible: Pinkall/Polthier 1993 now resolves to
+  its canonical DOI and rises to **rank 1 as an in-KB hub** (previously a dangling
+  OpenAlex node). Bobenko/Springborn 2007 holds rank 7, and the foundational works
+  (Pinkall/Polthier, Yamabe, Schoen, Bobenko/Springborn) still surface. **GO
+  re-affirmed** on the shipping topology.
+
+  Migration footnotes: two papers initially survived in URL form because the
+  id-keyed upsert did not catch a `papers.openalex_id` collision, and `2305.10988`
+  failed on a `papers.bibkey` collision with `2310.17529` (both →
+  `BobenkoLutz2023`). Same upsert gap — resolved by the bibkey-retry (audit C-15)
+  plus a delete + re-ingest of the two stragglers; corpus is now 29/29 canonical.

docs/audit/AUDIT-2026-06-15-full-repo.md

@@ -0,0 +1,616 @@
+# AUDIT — Full Repository (iterative)
+
+**Auditor:** Audit-Loop (Opus)
+**Started:** 2026-06-15
+**Scope:** entire `main` tree, audited iteratively module-by-module.
+**Companion doc:** [`AUDIT-2026-06-15-loop-1.md`](AUDIT-2026-06-15-loop-1.md) —
+the F-15 / dev-loop-1 findings (S-1…U-4). This document continues the **same
+finding-ID register** (so `C-4` here follows `C-3` there) and covers the rest of
+the repo.
+
+> Severity: **HIGH** fix before relying · **MED** fix soon · **LOW** polish ·
+> **INFO** accepted/no-action. Findings only — remediation planned separately.
+
+## Coverage tracker
+
+| Part | Area | Status |
+|------|------|--------|
+| (loop-1) | F-15 graph, db/schema, ingest, discover, mcp, provenance-scan, cli(search+graph), config(graph) | ✅ documented |
+| **A** | Grounding guard (`wiki.py` `_parse_claims`/`_run_grounding_guard`, quarantine) — shared by `synthesis.py` | ✅ documented |
+| **B** | `sources/` (arxiv, openalex, semanticscholar) | ✅ documented |
+| **C** | `parsing/` (grobid, nougat, mathpix, figures, tex) | ✅ documented |
+| **D** | `synthesis.py` orchestration (find_connections/gaps/improvements, conjectures, write_leads) | ✅ documented |
+| **E** | `wiki.py` rest (retrieve, compile_concept, cross-refs, index/log, embed) | ✅ documented |
+| **F** | `embed.py`, `quality.py`, `models.py` | ✅ documented |
+| **G** | `cli.py` rest (ingest/wiki/synthesis/provenance commands), `config.py` rest | ✅ documented |
+
+---
+
+## Part A — Grounding guard (the "zero-fact-leak" core)
+
+The guard lives in `wiki.py` (`_parse_claims:216`, `_run_grounding_guard:299`)
+and is imported by `synthesis.py:57-59`, so **one implementation gates both**
+the wiki pages and the synthesis leads. A claim is "grounded" iff the **last
+sentence** before its `[BibKey #loc]` citation contains a run of 5 consecutive
+non-stopword tokens that appears as a substring of any chunk of the same bibkey.
+
+### C-4 — Zero-claim pages bypass quarantine and publish as "compiled" · **HIGH**
+- **Evidence:** `wiki.py:730` — `if total_claims > 0 and grounding_rate < threshold:`
+  quarantines; the `else` (`:737-742`) **publishes to `wiki/` and records the
+  hash** as successfully compiled.
+- **Bug:** when the LLM output has **no parseable citations** (`total_claims == 0`,
+  so `grounding_rate == 0.0`), the `total_claims > 0` guard is false → it takes
+  the *publish* branch. The quarantine gate is skipped exactly when grounding
+  evidence is **weakest** (zero citations).
+- **Trigger paths:** (a) the LLM emits uncited prose; (b) **LLM outage** — the
+  module header guarantees "LLM unavailable → empty ConceptPage, no crash"
+  (`wiki.py:13-14`), so a transient Ollama outage writes near-empty stub pages to
+  `wiki/`, marks them compiled, and stores their hash. On the next
+  `changed_only` run the hash matches → the stub is **skipped forever** until a
+  forced recompile. A research wiki silently fills with empty/uncited pages.
+
+### C-5 — "Every claim is grounded" holds only for the last cited sentence · **MED**
+- **Evidence:** `_CLAIM_RE` (`wiki.py:211-213`) captures `text = [^\[]+?` (the
+  span back to the previous `]`/start); `_last_sentence` (`:279-291`) then keeps
+  only the final sentence; `_run_grounding_guard` checks that sentence only.
+- **Two holes in the guarantee:**
+  1. **Paragraph passengers** — any sentence that is *not* the last one before a
+     citation is never grounding-checked and never `⚠`-marked. "Hallucination one.
+     Hallucination two. Grounded tail [Bib]." passes with only the tail verified.
+  2. **Uncited text is invisible** — a sentence with no `[BibKey]` is not a Claim
+     at all (`_parse_claims` only yields cited spans), so it is never grounded
+     nor marked. The LLM can emit fully uncited assertions into a published page.
+- **Impact:** the effective invariant is "*the last sentence immediately before
+  each citation has lexical overlap with the cited paper*" — materially weaker
+  than the N-02 "zero-fact-leak enforcement" framing. The module **header is
+  honest** ("Substring-Match MVP", `wiki.py:4,13`); the commit/ADR language is
+  not. Recommend aligning the claim or strengthening the guard (check every
+  sentence; flag uncited sentences).
+
+### C-6 — Substring match does not enforce the documented "5 consecutive tokens" · **LOW**
+- **Evidence:** `wiki.py:347-349` — `gram = " ".join(content[i:i+5])`;
+  `if any(gram in src for src in sources)` where `src` is the space-joined
+  content-word string of the chunk.
+- **Issue:** a *substring* test on the joined string lets the first/last gram
+  token match as a **prefix/suffix of a longer chunk word** (e.g. gram
+  `"cat dog …"` substring-matches inside `"scat dogma …"`). The docstring claims
+  matching "as 5 consecutive non-stopword **tokens**" — true sequence matching
+  would compare token lists with a sliding window (or add boundary sentinels).
+  Low real-world impact at 5-gram length, but the claimed invariant is not what
+  the code enforces.
+
+### R-1 — Tokenizer brittleness causes false-negative over-quarantine · **MED**
+- **Evidence:** `_content_words` (`wiki.py:294-296`) lowercases and `.split()`s on
+  whitespace only — **no punctuation stripping**. `_last_sentence` splits on
+  `.!?` (`:276`).
+- **Impact:** punctuation stays glued to tokens, so a faithful paraphrase grounds
+  only on *exact* surface form — `"volume,"` ≠ `"volume"`, `"L(gamma1)"` must
+  match character-for-character, and a trailing decimal like `"V = 1.5"` makes
+  `_last_sentence` return the fragment `"5"` (split at the `.`), which then has
+  `< 5` content words → **forced ungrounded** (`:339-344`). Math-heavy and terse
+  claims are systematically marked ungrounded even when verbatim, inflating the
+  ungrounded ratio and pushing real pages into quarantine (and into C-4's
+  blind spot when it drives `total_claims` interactions). The passing test
+  `test_grounding_guard_…` only succeeds because its fixture reproduces the chunk
+  text *exactly*.
+
+### R-2 — Reference-list filter can drop theorem-dense chunks · **LOW**
+- **Evidence:** `_retrieve_chunks` (`wiki.py:188-199`) skips a chunk when
+  `>60 %` of its lines match `^\s*(\[\d+\]|[A-Z][a-z]+,?\s+[A-Z]\.)`.
+- **Issue:** the second alternative (`[A-Z][a-z]+,?\s+[A-Z]\.`) also matches prose
+  like `"Theorem A."`, `"Lemma B."`, `"Section C."`. A chunk formatted as a list
+  of theorems/lemmas can exceed 60 % and be discarded as a "reference list" —
+  silently dropping exactly the mathematical content the wiki wants.
+
+### R-3 — Conflict detection has near-zero precision · **LOW**
+- **Evidence:** `_detect_conflicts` (`wiki.py:367-…`) flags a pair `(bk1, bk2)` if
+  **each** has *any* chunk containing a hedging keyword
+  (`but|however|in contrast|…`, `_CONFLICT_KEYWORDS:361-364`).
+- **Issue:** it never checks the keywords relate to a shared topic or to each
+  other. In a corpus where most papers contain "however"/"but" somewhere, this
+  flags ~O(n²) pairs as "conflicting" — noise surfaced in `CompileReport`.
+  Labeled "heuristic signal only", but as-is it is not actionable.
+
+**Part A net:** 1 HIGH (C-4), 2 MED (C-5, R-1), 3 LOW (C-6, R-2, R-3). The guard
+is a reasonable lexical MVP, but (a) it has a concrete publish-bypass bug for
+zero-citation output, and (b) the "zero-fact-leak" framing overstates what a
+last-sentence substring check can guarantee.
+
+---
+
+## Part B — `sources/` (external API clients)
+
+Solid foundations: all three clients set httpx timeouts; OpenAlex and S2 retry
+429/5xx with exponential back-off (tenacity); S2 enforces a ≥1 req/s floor with a
+monotonic clock under a lock. The findings are about **ID format consistency**
+and a couple of dead/inert paths.
+
+### C-7 — `papers.id` is stored in OpenAlex *URL* form, breaking the bare-ID contract · **HIGH**
+- **Evidence:** `openalex.py:94` — `_map_paper` sets
+  `id = data.get("doi") or data.get("id") or ""`. Real OpenAlex returns `doi` as
+  a **full URL** (`https://doi.org/10.x`) and `id` as `https://openalex.org/W…`
+  (the repo's own test confirms the URL form for `openalex_id`,
+  `test_openalex.py:92`). `ingest.py:82` takes `paper.id` from this unchanged and
+  upserts it as `papers.id` (`ingest.py:138`).
+- **Why it's currently masked:** the test fixture `_SAMPLE_WORK["doi"]` is a
+  **bare** DOI `"10.1145/3592430"` (`test_openalex.py:18`) — not the URL form real
+  OpenAlex emits — and **no test asserts `paper.id`** at all. So the suite is
+  green while production stores `papers.id = "https://doi.org/10.1145/3592430"`.
+- **Impact:**
+  1. **Bare-ID lookups fail.** `ingest_all.sh` ingests `"10.1145/1964921.1964997"`,
+     but `papers.id` becomes the URL; `codex graph related 10.1145/…`,
+     `discover citing 10.1145/…`, search-by-id, etc. find nothing.
+  2. **Second ID-mismatch axis (compounds C-1).** `cited_id` from the S2/GROBID
+     paths is a **bare** DOI/arXiv (`semanticscholar.py:98-99`,
+     `ingest.py:187`), so a bare-DOI reference can never equal a URL-form
+     `papers.id` — bare-DOI cross-citations stay "dangling" regardless of the
+     F-15 W-ID resolution (which only matches `papers.openalex_id`, the URL form).
+- **Action:** confirm with `SELECT id FROM papers LIMIT 5` on the Jetson DB; if
+  URL-form, normalize `papers.id` to a canonical bare form at ingest (and the
+  `cited_id` writers to match), then add a representative fixture + `paper.id`
+  assertion (see T-3).
+
+### T-3 — OpenAlex test fixture is unrepresentative and skips `paper.id` · **MED**
+- **Evidence:** `_SAMPLE_WORK["doi"] = "10.1145/3592430"` (bare) vs the real
+  `https://doi.org/…` URL form; `test_fetch_paper_*` assert title/year/authors/
+  abstract/`openalex_id` but **never** `paper.id`.
+- **Impact:** the id-mapping — the exact surface of C-7 — is untested, and the
+  fixture actively hides the URL-form behaviour. Any normalization fix needs this
+  closed first or it will regress invisibly.
+
+### R-5 — S2 `fetch_references` breaks on legacy arXiv IDs containing `/` · **LOW**
+- **Evidence:** `semanticscholar.py:80` interpolates `paper_id` into the path
+  `…/paper/{paper_id}/references`. The ingest fallback passes `arXiv:math/0603097`
+  (`ingest.py:91`) — the embedded `/` yields a malformed path → 404 →
+  `fetch_references` silently returns `[]`. Legacy-arXiv papers that miss the
+  OpenAlex path get **zero references** with no error.
+
+### R-4 — `arxiv.fetch_source` no-op try/except + no retry · **LOW**
+- **Evidence:** `arxiv.py:41-44` — `try: httpx.get(...) except httpx.RequestError: raise`
+  catches and immediately re-raises, doing nothing. Unlike OpenAlex/S2, the arXiv
+  client has **no** retry/back-off, so a transient network blip aborts ingest.
+  Minor resilience asymmetry.
+
+### C-9 — `semanticscholar.fetch_recommendations` is unreachable app code · **LOW**
+- **Evidence:** referenced only by `tests/sources/test_semanticscholar.py`; no
+  `codex/` module or CLI calls it (`grep fetch_recommendations`). Tested but
+  never wired — dead feature surface (decide: wire a `discover recommend`
+  command, or drop).
+
+**Part B net:** 1 HIGH (C-7, ID-form contract — the big one; confirm on live DB),
+1 MED (T-3, fixture masks it), 3 LOW (R-4, R-5, C-9). The retry/timeout/rate-limit
+hygiene is genuinely good; the weakness is the **un-normalized, heterogeneous ID
+space** (URL vs bare, W-ID vs DOI vs arXiv vs S2-hash) threaded through `papers.id`
+and `citations.cited_id` — the same root cause behind C-1 and the F-15 ID dance.
+
+---
+
+## Part C — `parsing/` (tex, grobid, nougat, mathpix, figures)
+
+> **Scope reality:** the live corpus is ingested entirely as **`.txt`**
+> (`ingest_all.sh` maps every paper to a `*.txt` file), so the `.tex`-clean,
+> GROBID, Nougat, Mathpix, and Figures paths are **latent** — only
+> `tex.chunk_text` + `quality.*` run on the live data. The findings below are
+> real but mostly fire only once `.tex`/`.pdf --rich` ingest is used.
+>
+> **Not a finding:** the broad `except Exception` in `mathpix.py:109,151` and
+> `figures.py:125,152` are *justified* per-item graceful degradation
+> (`fitz.open` fails → return empty; pix2tex/image fails → skip that item), not
+> swallowed bugs.
+
+### C-10 — Formulas/figures key on the PDF *filename*, not `paper.id` · **MED** (latent, `.pdf --rich`)
+- **Evidence:** `figures.py:119` and the mathpix extractor set
+  `paper_id = Path(pdf_path).stem`. `ingest.py:289,303` insert
+  `f.paper_id` / `fig.paper_id` **unchanged** into `formulas`/`figures`, whose
+  schema declares `paper_id TEXT REFERENCES papers(id)` (`schema.sql:97,115`).
+- **Impact:** the filename stem (e.g. `springborn-2008-weighted-delaunay…`) will
+  not equal `papers.id` (an OpenAlex URL/bare id — see C-7), so the FK insert
+  **fails and aborts rich ingest**, or (without FK enforcement) writes orphaned
+  rows. Identity is derived from the *file*, not the *paper*. Fix: pass `paper.id`
+  into the extractors, or have ingest overwrite `f.paper_id = paper.id` before insert.
+
+### R-7 — `_clean_latex` misses `\(…\)` / `\[…\]` math + strips all math from chunks · **LOW** (`.tex` path)
+- **Evidence:** `tex.py:21-56` strips `%comment`, `$…$`, `$$…$$`,
+  `equation`/`align` envs, `\cite/\label/\ref` — but **not** the `\(…\)` / `\[…\]`
+  math delimiters. Papers using those leave raw `\( … \)` markup in the chunk text.
+- **Impact:** (a) leftover math markup pollutes embeddings; (b) *all* math is
+  removed from chunk text, so chunk-level semantic search can never match formula
+  content — it relies entirely on the separate formula FTS (F-09), which is itself
+  latent here. Acceptable as a separation-of-concerns choice, but worth stating.
+
+### R-6 — PDF-path external calls are unwrapped / un-retried · **LOW** (`.pdf` path)
+- **Evidence:** `ingest.py:185` calls `extract_references()` outside any
+  `try`; GROBID (`grobid.py:54,133`) and the mathpix HTTP calls `raise_for_status`
+  with **no** retry/back-off (unlike OpenAlex/S2). A transient GROBID/Mathpix
+  5xx aborts the whole PDF ingest.
+- **INFO:** `grobid.py:56` uses stdlib `ET.fromstring` on the server's XML.
+  Fine for a trusted self-hosted GROBID; switch to `defusedxml` if it is ever
+  pointed at an untrusted endpoint.
+
+**Part C net:** 1 MED (C-10), 2 LOW (R-6, R-7), all **latent** for the current
+`.txt` corpus. `nougat.py` was skimmed only (54 lines, `.pdf`-only wrapper) — no
+deep review yet; flagged in the ledger. Parsing hygiene is otherwise good.
+
+> **Request (a) — second audit pass — is now complete:** §8 of the loop-1 doc
+> (grounding guard ✓ Part A, `sources/` ✓ Part B, `parsing/` ✓ Part C) is closed.
+> Remaining whole-repo parts D–G are the broader sweep.
+
+---
+
+## Part D — `synthesis.py` orchestration
+
+The lead pipeline (connections → gaps → improvements → conjectures) reuses the
+Part-A grounding guard for the three grounded kinds and degrades gracefully on
+LLM/DB failure (`_safe_llm_generate`/`_list_paper_titles` swallow + log). The
+conjecture invariants are genuinely solid (status hard-set `"unverified"`,
+mandatory falsification path or the lead is dropped, written to `conjectures/`
+only). Two real defects and a doc contradiction:
+
+### C-11 — Lead-ID collision silently overwrites synthesis output · **HIGH**
+- **Evidence:** every stage restarts its counter at `seq = 1`
+  (`synthesis.py:341` connections, `:415` gaps, `:530` improvements,
+  `:634` conjectures) and ids are `L-0001, L-0002, …` (`_make_lead_id`). The CLI
+  concatenates the grounded kinds — `all_leads = connections + gaps + improvements`
+  (`cli.py:421`) — and `write_leads` routes **all** grounded kinds to the *same*
+  `grounded/<id>.md` (`synthesis.py:783-796`), writing in list order.
+- **Bug:** connection `L-0001`, gap `L-0001`, improvement `L-0001` all target
+  `grounded/L-0001.md`; later writes overwrite earlier → **improvements clobber
+  gaps clobber connections** at each number. Surviving files =
+  `max(#connections, #gaps, #improvements)`, **not** the sum. With 5/3/4 leads you
+  get 5 files, not 12; connections are lost first. Silent data loss of the
+  synthesis engine's core output, masked because "files do get written".
+- **Fix direction:** namespace ids by kind (`L-C-0001` / `L-G-0001` / `L-I-0001`)
+  or use a single shared counter across stages.
+
+### R-8 — Default-topic gap detection is a false-positive generator · **MED**
+- **Evidence:** `find_gaps` defaults `probe_topics` to paper **titles** when
+  `topics is None` (`synthesis.py:418-420`); a title query retrieves mostly its
+  *own* paper's chunks → `bibkeys_for_topic` ≈ 1, which is `< synthesis_gap_min_coverage`
+  (`:450-453`) → the topic is flagged as a gap and the LLM is asked to articulate
+  one.
+- **Impact:** nearly every ingested paper becomes a spurious "Gap: '<title>'
+  covered by only 1 bibkey(s)" candidate — the LLM invents a gap for material that
+  is actually well covered. The grounding guard catches some, but the default
+  signal is mostly noise. (User-supplied `--topics` avoid this.)
+
+### D-4 — `write_leads` vs `_update_index` docstrings contradict each other · **LOW**
+- **Evidence:** `write_leads` (`synthesis.py:762-765`) claims it "maintains an
+  **append-only** `INDEX.md`: a previously-recorded id is kept on subsequent runs",
+  but `_update_index` (`:802-803`) states — and implements — a "**full rebuild**,
+  not append-only" by globbing the on-disk dirs.
+- **Impact:** there is no append-only / id-retention logic; the index purely
+  reflects current disk state. The guarantee in the `write_leads` docstring is
+  false (and interacts with C-11: overwritten ids simply vanish from the index).
+
+**Part D net:** 1 HIGH (C-11, silent lead overwrite), 1 MED (R-8, gap noise),
+1 LOW (D-4, doc contradiction). The conjecture path is exemplary; the grounded
+path's persistence layer is where it breaks.
+
+---
+
+## Part E — `wiki.py` rest (compile, cross-refs, state, index/log)
+
+Good: `yaml.safe_load`, deterministic `_chunk_hash` (sorted by chunk id) driving a
+sound incremental-skip, append-only `log.md`, inline-code protection during
+cross-ref injection. The issues are math-corruption, a dead step, and a marking
+order bug.
+
+### R-9 — Cross-ref injection corrupts LaTeX math · **MED**
+- **Evidence:** `_inject_cross_refs` (`wiki.py:426`) protects only inline-code
+  spans (`_INLINE_CODE_RE`); it then runs whole-word, case-insensitive title/alias
+  → `[[slug]]` replacement over everything else (`:429-438`). **Math spans
+  (`$…$`, `$$…$$`, `\(…\)`) are not protected.**
+- **Impact:** the synthesis prompt explicitly asks for LaTeX `$ … $`
+  (`wiki.py:495`), so bodies contain math. A concept title/alias that appears
+  inside a formula (e.g. a concept named "Energy" inside `$E_{\text{Energy}}$`,
+  or a single-word alias) is rewritten to `[[slug]]`, corrupting the LaTeX. The
+  risk scales with how common the concept titles/aliases are as math tokens.
+
+### C-12 — `_try_embed_formulas` is a no-op that still costs a DB round-trip per concept · **LOW**
+- **Evidence:** `wiki.py:623-637` opens a connection, queries
+  `information_schema.tables` for `formulas`, and returns — the actual embed is
+  `"(Future: …)"`. Called unconditionally per concept (`compile_concept:609`).
+- **Impact:** step 4 of `compile_concept`'s docstring ("Embed formula chunks")
+  does nothing, and each concept compile pays an extra DB connection + query for
+  no effect. Placeholder masquerading as a feature.
+
+### C-13 — Ungrounded-claim marking can miss claims mutated by cross-ref injection · **LOW**
+- **Evidence:** order in `compile_concept` — claims parsed from `raw_output`
+  (`:602-603`), then `raw_output` is rewritten by `_inject_cross_refs` (`:606`),
+  then `_render_page_markdown` (`:612`) re-finds the *original* `claim.text` by
+  regex in the **injected** body (`:544-552`).
+- **Impact:** if an ungrounded claim's text contained a term that got replaced
+  with `[[slug]]`, the original text no longer matches → the `⚠` mark is silently
+  not applied. (Also note: wiki's marking lacks the `(?<!⚠ )` guard that
+  synthesis's `_mark_ungrounded` has — two divergent mark implementations.)
+
+### R-10 — Non-atomic compile-state write · **LOW**
+- **Evidence:** `_save_compile_state` (`wiki.py:475-477`) writes
+  `.compile-state.json` in place. A crash mid-write corrupts it; `_load`
+  then catches `JSONDecodeError` → `{}` → every concept recompiles. Graceful but
+  loses incrementality. Write to a temp file + `os.replace` for atomicity.
+
+### R-11 — `load_concepts` raises `KeyError` on malformed YAML · **LOW**
+- **Evidence:** `entry["slug"]` / `entry["title"]` (`wiki.py:135-136`) are
+  unguarded; a concepts.yaml entry missing a key crashes the whole compile with a
+  bare `KeyError` rather than a actionable message.
+
+> **Minor:** `compile_all` builds the index from `state.keys()` (`:745`), which
+> accumulates slugs across runs — a concept removed from `concepts.yaml` lingers
+> in `index.md`/state until manually pruned. Low.
+
+**Part E net:** 1 MED (R-9, math corruption), 4 LOW (C-12, C-13, R-10, R-11). The
+incremental-compile machinery is sound; the rendering/cross-ref layer has the rough
+edges. `OllamaClient.generate` (no retry, 120 s timeout, `raise` on error) feeds
+directly into **C-4** (its exception → empty page → published), so C-4's blast
+radius includes every transient Ollama hiccup.
+
+---
+
+## Part F — `embed.py`, `quality.py`, `models.py`
+
+The best-built corner of the repo. `embed.py` L2-normalises correctly (cosine ⇒
+dot product), guards empty input and zero-norm rows, and caches a process
+singleton. `quality.py` thresholds are all config-driven, `run_quality_pass` is
+CLI-wired (`cli.py:513-517`) and atomic (single commit). `models.py` dataclasses
+mirror the schema cleanly. Two dead/low-signal items and one corroboration:
+
+### C-14 — The sparse-embedding ("hybrid") path is dead code · **LOW**
+- **Evidence:** `Embedder.encode_sparse` / `encode` / `_coerce_sparse` are called
+  nowhere in `codex/` (grep for `encode_sparse` / `.encode(` finds only
+  `str.encode` in `wiki.py:455`). Only `encode_dense` is wired (ingest, search,
+  wiki). There is no sparse column in the schema.
+- **Impact:** the `embed.py` header and ADR-0002 advertise "**Hybrid dense +
+  sparse** embeddings via BGE-M3" as a headline, but the sparse half is
+  unreachable — the actual "hybrid" elsewhere is dense + Postgres FTS. Either wire
+  sparse retrieval or drop the capability + correct the docs.
+
+### R-12 — `classify_section` is low-signal as fed · **LOW**
+- **Evidence:** `classify_section` matches section headers in the **first 200
+  chars** (`quality.py:103-106`), but `tex.chunk_text` produces overlapping
+  *word-window* chunks that almost never begin at a header. So most chunks fall
+  through to `"body"` (or the DOI-density `"bibliography"` fallback).
+- **Impact:** the `chunks.section` column is mostly `"body"` — limited value for
+  any section-aware retrieval that depends on it. Not wrong, just weak signal
+  given the chunker.
+
+### Corroboration of C-7 (not a new finding)
+- `models.Paper.id` docstring (`models.py:20-21`) states the canonical id is "an
+  arXiv ID … **or a DOI (e.g. `10.1145/3592430`)**" — the **bare** form. Production
+  fills it from OpenAlex in **URL** form (C-7), so the code violates its own model
+  contract. This makes C-7 a documented-contract breach, not a judgement call.
+
+**Part F net:** 2 LOW (C-14, R-12) + a C-7 corroboration. These three modules are
+clean; no correctness defects.
+
+---
+
+## Part G — `cli.py` rest + `config.py`
+
+Most CLI commands are thin Typer wrappers delegating to already-audited modules
+(`ingest`, `search`, `discover`, `wiki`, `synthesis`, `provenance`, `quality`,
+`graph`). Settings are pydantic-validated with sensible bounds (`gt=0`, `0..1`).
+Findings are config-coupling and output hygiene.
+
+### C-15 — `embedding_dim` is a footgun decoupled from the hardcoded schema · **MED**
+- **Evidence:** `embedding_dim` is configurable (`config.py:67-74`, default 1024)
+  and `.env.example`/`schema.sql:7` invite other models (Qwen3 1024, **Jina v4
+  2048**). But `schema.sql` hardcodes `vector(1024)` on every embedding column
+  (`:25,38,102,119`), and `Embedder` derives the real output dim from the *model*,
+  using the `dim` setting only for empty-input zero-vectors (`embed.py:50,89`).
+- **Impact:** set `EMBEDDING_DIM=2048` (or pick a non-1024 model) and inserts
+  fail on a pgvector dimension mismatch — silently for empty abstracts
+  (`(1,2048)` zero-vector into `vector(1024)`), structurally for real vectors.
+  Nothing couples the knob to the DDL; the setting reads as supported but is not.
+
+### R-13 — `export_bib` does no BibTeX escaping and forces `@article` · **LOW**
+- **Evidence:** `provenance.py:174-181` interpolates `title`/`author`/`year` raw
+  into `@article{…}`; no escaping/brace-balancing of BibTeX specials
+  (`{ } \ % # $ _ &`).
+- **Impact:** a title with an unbalanced brace or a stray special char produces a
+  malformed `.bib` entry that breaks Doxygen/BibTeX. Also every paper is emitted
+  as `@article` although the corpus includes theses/books/chapters
+  (`ingest_all.sh`) — bibliographically wrong (functional for `@cite` keys though).
+
+### S-4 — Secrets stored as plain `str`, not `SecretStr` · **LOW**
+- **Evidence:** `database_url`, `mathpix_app_key`, `mcp_auth_token`
+  (`config.py:29,148,249`) are plain `str`. Any `repr(Settings())` or debug log of
+  the settings object prints the DB password / API key / bearer token in clear.
+- **Impact:** log/exception-dump leakage surface (complements S-1). Use
+  `pydantic.SecretStr` for credential fields.
+
+### R-14 — `mcp_transport` enum not enforced · **LOW**
+- **Evidence:** `config.py:229-236` documents "stdio or http" but no validator;
+  `main()` (`mcp_server.py:323`) treats anything non-`http` as stdio. A typo
+  (`htttp`) silently starts stdio instead of failing loudly.
+
+### D-5 — Inconsistent `validation_alias` on `nougat_url` only · **LOW**
+- **Evidence:** `nougat_url` adds `AliasChoices("NOUGAT_URL", "nougat_url")`
+  (`config.py:47`) that no other field has — redundant under
+  `case_sensitive=False`, and the inconsistency invites confusion.
+
+> **Corroboration of C-7:** `cli.py:45` ingest does **no** id normalization and
+> echoes `result.paper_id` (the URL-form id) straight back — no normalization
+> layer exists at any tier (source → ingest → CLI).
+
+**Part G net:** 1 MED (C-15, dim/schema footgun), 4 LOW (R-13, S-4, R-14, D-5).
+CLI/config are otherwise solid and well-documented.
+
+---
+
+# Executive Summary — whole-repo audit complete (loop-1 + Parts A–G)
+
+**Tally: 45 findings — 8 HIGH, 11 MED, 26 LOW/INFO** (after live-DB verification:
+**7 HIGH, 12 MED** — M-1 reclassified, see the verification section at the bottom).
+Test suite 330 green, `ruff`/`mypy` clean. The codebase is well-structured and idiomatic; the defects
+cluster into five root themes, not random scatter.
+
+### The 8 HIGH findings
+| ID | Theme | One-liner |
+|----|-------|-----------|
+| S-1 | secrets | live DB password in un-ignored `.env.jetson-ingest` (mitigated this session) |
+| M-1 | migration | `paper_identifiers` has no migration path; `schema.sql` not re-applyable, `apply_schema` never called |
+| C-1 | identity | `discover.py` keeps the ID-mismatch bug F-15 fixed in `graph.py` (over-reports ingested papers as leads) |
+| C-7 | identity | `papers.id` stored in OpenAlex **URL** form, violating the bare-ID contract (breaks lookups + cross-citation match) |
+| C-4 | silent failure | zero-citation/LLM-outage wiki pages **bypass quarantine** and publish as "compiled" |
+| C-11 | silent data loss | synthesis leads collide on `L-000N` across stages → improvements overwrite gaps overwrite connections |
+| D-1 | doc integrity | ADR-F15 GO rests on spike numbers measured **before** the shipped resolution fix |
+| T-1 | verification | the F-15 resolution SQL (the substance of the last 2 commits) has **zero** real-DB coverage |
+
+### Five root-cause themes
+1. **Un-normalized identity (the dominant theme).** C-1, C-7, C-10, M-1, T-3,
+   and most of the F-15 ID dance are one problem: a paper's identity exists in
+   many un-reconciled forms (bare vs URL · DOI vs arXiv vs OpenAlex-W vs S2-hash
+   vs PDF-filename) with **no canonical normalization layer**. One normalization
+   boundary at ingest would retire ~5 HIGH/MED findings at once.
+2. **"Green but unverified."** T-1/T-2/T-3 — the most error-prone code (resolution
+   SQL, migrations, id-mapping) is mocked or tested with unrepresentative fixtures,
+   so real bugs (C-4, C-7, C-11) sit behind 330 passing tests.
+3. **Silent failure / data loss under degradation.** C-4 (publish empty), C-11
+   (overwrite leads), R-1 (over-quarantine) corrupt *outputs* instead of erroring
+   — the worst failure mode for a research tool whose value is trust.
+4. **Overstated guarantees.** D-1 (stale ADR), D-2/D-4 (doc↔code drift), C-5
+   ("zero-fact-leak" ≫ a last-sentence substring check), C-14 ("hybrid" sparse is
+   dead) — the docs/commit-language claim more than the code delivers.
+5. **Latent rich-path debt.** C-10, R-6, R-7 are real but dormant because the live
+   corpus is `.txt`; they detonate on the first `.pdf --rich` ingest.
+
+### Suggested remediation sequencing (themes, not a plan — planning deferred)
+- **First:** S-1 (rotate credential — secret already protected), M-1 (migration
+  path), C-7+C-1 (one identity-normalization layer), T-1 (real-DB test) — these
+  unblock trust in the live corpus.
+- **Then:** C-4 + C-11 (stop silent output corruption), D-1 (re-validate ADR-F15).
+- **Then:** MED batch (R-1, R-8, R-9, C-15, C-10) and the LOW polish.
+
+### Coverage honesty
+Every `codex/` module was read. After the follow-up pass **all are deep-reviewed**
+(`nougat.py` included — see Runtime Validation). Runtime state that was initially
+*inferred from code* (C-7 id-form, M-1 table existence, and the C-4/C-11
+behaviours) has since been **empirically confirmed** — see the *Live DB
+Verification* and *Runtime Validation* sections below.
+
+---
+
+# Live DB Verification — 2026-06-15 (read-only, via existing SSH tunnel)
+
+Ran read-only queries against the live Jetson corpus (`papers=36`,
+`citations=811`). This **confirms the two identity HIGHs with hard numbers** and
+**corrects an over-statement in M-1**.
+
+### C-7 — CONFIRMED ✅ (severity stands: HIGH)
+`papers.id` form distribution: **`doi_url=35`, `bare_arxiv=1`, everything else 0**
+(of 36). Samples: `https://doi.org/10.48550/arxiv.math/0603097`,
+`https://doi.org/10.48550/arxiv.2310.17529`. Even arXiv papers are stored as their
+arXiv-DOI **URL**, never the bare `math/0603097` / `2301.x` the schema, `models.Paper`,
+and `ingest_all.sh` assume. The single bare id is the one paper OpenAlex 404'd
+(fell to `ingest.py:94`). → bare-ID lookups fail for 35/36 papers. Confirmed.
+
+### C-1 — CONFIRMED ✅ with impact (severity stands: HIGH)
+- `citations.cited_id` is uniformly **`oa_url=811`** (all OpenAlex URLs) — the live
+  corpus was ingested via the OpenAlex path, so the heterogeneous bare-DOI/S2 mix
+  theorised in Part B is **not present here** (it would appear only via S2/GROBID,
+  i.e. `.pdf`/fallback ingest).
+- **`discover.py` wrongly flags 13 already-ingested papers as "dangling / not
+  ingested"** (their `cited_id` is an OpenAlex URL whose paper *is* in the corpus
+  via `openalex_id`). Meanwhile **`graph.py` correctly resolves 57 cross-citation
+  edges** via the `openalex_id` JOIN. The two "dangling" implementations
+  measurably disagree — exactly the C-1 divergence, now quantified.
+
+### M-1 — REFINED ⤵ (downgrade acute severity; structural finding stands)
+`paper_identifiers` **exists on the live DB with 39 rows** — the catastrophic
+"graph JOIN crashes with relation-does-not-exist" scenario **does not occur**;
+the table was applied (manually or via DB re-create). The *structural* gap stands:
+`apply_schema` is still non-idempotent and never invoked, so the **next** schema
+change has no migration path — but the acute "live graph is broken" framing was
+**over-stated**. Re-rank M-1 from HIGH → **MED** (latent migration debt, not an
+active outage).
+
+### C-2 — PARTIALLY REFUTED ⤵
+`paper_identifiers` has **39 rows vs ~35 papers-with-openalex_id**, so ~4 papers
+carry a diverging alias → the `pi` JOIN is **not** fully inert (it resolves a
+handful of cases the `p` JOIN can't). Keep as LOW/INFO: the table earns its keep
+for a few papers, though the cost/benefit at this corpus size is still marginal.
+
+### Net adjustment to the tally
+HIGH **8 → 7** (M-1 → MED). C-7 and C-1 are now **confirmed**, not inferred — they
+should lead remediation. The dominant root theme (**un-normalized identity**) is
+empirically validated: 35/36 ids in the "wrong" form, 13 papers mis-classified by
+the older code path, 57 edges rescued by the newer one.
+
+---
+
+# Runtime Validation — 2026-06-15 (C-4 and C-11 reproduced end-to-end)
+
+The two HIGH findings flagged "code-certain, runtime-confirmable" were exercised
+against the **real** code paths (injected fake LLM; no Ollama, no DB needed).
+
+### C-11 — CONFIRMED ✅ (lead-ID collision → silent loss; stays HIGH)
+Reproduced the exact CLI assembly (`cli.py:421` `connections + gaps +
+improvements`, each stage numbered from `L-0001`): **4 distinct leads** (2
+connection, 1 gap, 1 improvement) → `write_leads` produced **2 files**.
+`grounded/L-0001.md` survived as the **improvement** ("IMPROVEMENT-one"); the
+connection and gap `L-0001` were silently overwritten. **2 of 4 leads lost.**
+
+### C-4 — CONFIRMED ✅ (zero-claim page bypasses quarantine; stays HIGH)
+Drove the real `compile_all` with a fake LLM returning confident **uncited** prose
+(→ 0 parseable claims). With `wiki_min_grounding_rate = 0.5` and an actual
+grounding rate of `0.0`, the page was **published to `wiki/` and marked
+compiled** (`report.compiled=['test-concept']`, `report.quarantined=[]`) — not
+quarantined to `wiki/draft/`. The `total_claims > 0` guard (`wiki.py:730`) is the
+exact bypass; the bug fires precisely when grounding evidence is weakest.
+
+### `nougat.py` — now fully reviewed (no new finding)
+Folds into **R-6** (PDF-path resilience). Detail: `pdf_to_markdown` retries **only**
+`httpx.ConnectError` with `wait_fixed(0)` (no 5xx retry, no back-off) and is
+unwrapped at `ingest.py:183` — a Nougat 5xx aborts PDF ingest.
+
+**Closure:** every `codex/` module is deep-reviewed; the two runtime-confirmable
+HIGHs are empirically reproduced. The **audit / finding-identification pass is
+complete**. Remediation planning is the open next phase.
+
+---
+
+# Migration Incident — 2026-06-15 (M-1 confirmed live, new privilege dimension)
+
+While running the C-7 re-ingest migration (`infra/reingest_canonical_ids.sh`),
+M-1 manifested exactly as predicted — and exposed a dimension the static audit
+did **not** capture.
+
+**What happened.** The helper ran `TRUNCATE papers CASCADE` (corpus wiped to 0),
+then `ingest_all.sh` failed on the first chunk insert:
+`UndefinedColumn: column "section" of relation "chunks" does not exist`. The live
+`chunks` table predates the F-16 `section` column, and nothing had ever applied
+that DDL to the live DB (M-1: `apply_schema` is never called; `schema.sql` is not
+re-applyable). The ingest code was ahead of the live schema.
+
+**New finding — privilege separation (extends M-1).** The fix
+`ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT` then failed with
+`InsufficientPrivilege: must be owner of table chunks`. The live ownership:
+
+| table | owner |
+|-------|-------|
+| papers, chunks, citations, code_links, formulas, figures | `postgres` |
+| paper_identifiers | `researcher` (app user) |
+
+The app role `researcher` has DML + `TRUNCATE` but **not** DDL on the
+postgres-owned core tables. So a `codex migrate` / `apply_schema` invoked with the
+application's `DATABASE_URL` would fail on every `ALTER` / `CREATE INDEX` against
+those tables. **Implication for the M-1 fix:** making `schema.sql` idempotent is
+necessary but *not sufficient* — the migration must run under a privileged role
+(owner/superuser), via a separate admin connection, not the least-privilege app
+user. This is now part of M-1's remediation scope.
+
+**Consequences observed.**
+- Corpus left wiped (papers = 0) until the owner adds the column and re-ingest is
+  re-run — a destructive-then-fail sequence.
+- Helper hardened (`fix(infra)` on `fix/audit-wave-2`): a **preflight** verifies
+  `chunks.section` exists *before* the `TRUNCATE`, aborting early with the
+  owner-level `ALTER` to run, so a schema gap can no longer cost the corpus.
+
+**Unblock (owner action):**
+`sudo -u postgres psql -d papers -c "ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;"`
+then re-run `bash ingest_all.sh`.
+
+This incident is the strongest possible validation of M-1 and should anchor its
+Wave-3 fix: **idempotent `schema.sql` + a `codex migrate` command that connects as
+a privileged role**, plus a documented owner/grant model for the app user.

docs/audit/AUDIT-2026-06-15-loop-1.md

@@ -0,0 +1,227 @@
+# AUDIT — Development Loop 1 (main branch)
+
+**Auditor:** Audit-Loop (Opus)
+**Date:** 2026-06-15
+**Scope:** entire `main` branch (HEAD `5dfa6e4`)
+**Method:** static review of source + schema + tests, branch-wide pattern scan
+(`ruff`, `mypy`, defect-class greps), full test run (`330 passed`).
+
+> Status legend — **HIGH** = fix before relying on the feature ·
+> **MED** = fix soon, correctness/maintainability · **LOW** = polish ·
+> **INFO** = accepted risk / no action, documented for the record.
+
+This document **only records findings**. Remediation planning is a separate
+step (next phase of the audit loop).
+
+---
+
+## 0. Headline
+
+The most recent dev-loop work (**F-15 citation graph** + the follow-up fixes
+`cited_id→papers.id` resolution and the `paper_identifiers` alias table) is
+**functionally green but unverified where it matters**: the SQL that *is* the
+fix has zero real-database coverage, the live-corpus validation in the ADR
+predates the fix, and the same ID-mismatch bug the fix solves still lives in a
+parallel code path (`discover.py`). Separately, a **live DB password sits in an
+un-ignored file** and the new table has **no migration path onto the live DB**.
+
+Test suite: **330 passed**, `ruff check` clean, `ruff format --check` clean,
+`mypy` clean on reviewed modules.
+
+---
+
+## 1. Security
+
+### S-1 — Live DB credential in an un-ignored file · **HIGH**
+- **Evidence:** `.env.jetson-ingest` contains
+  `DATABASE_URL=postgresql://researcher:05761b87…@localhost:5433/papers`.
+  `.gitignore` ignores only the literal `.env` (line: `# Environment secrets — NEVER commit` → `.env`).
+  `git check-ignore .env.jetson-ingest` → **NOT IGNORED**.
+- **Impact:** one `git add -A && git commit` leaks a real Postgres password to
+  history. The file is currently untracked (`??`), so the leak has **not**
+  happened yet — this is a near-miss, not an incident.
+- **Note:** the same gap exposes any future `.env.<profile>` file.
+
+### S-2 — Non-constant-time token comparison (MCP HTTP) · **LOW**
+- **Evidence:** `codex/mcp_server.py:340` — `if auth != f"Bearer {token}":`.
+- **Impact:** theoretical timing side-channel on the Bearer token. Low on a
+  trusted LAN, but trivially fixable with `secrets.compare_digest`.
+
+### S-3 — DNS-rebinding protection disabled · **INFO (accepted)**
+- **Evidence:** `codex/mcp_server.py:35-37`,
+  `TransportSecuritySettings(enable_dns_rebinding_protection=False)` (commit `d29dcf2`).
+- **Assessment:** documented and mitigated — HTTP transport requires a Bearer
+  token (`_TokenAuth`) and is firewalled to LAN IPs (UFW). A rebound browser
+  request lacks the token. Residual risk only if the firewall assumption fails.
+  No action; recorded so the trade-off stays visible.
+
+---
+
+## 2. Correctness
+
+### C-1 — `discover.py` still has the ID-format-mismatch bug F-15 fixed · **HIGH**
+- **Evidence:** `codex/discover.py:20` —
+  `WHERE cited_id NOT IN (SELECT id FROM papers)`; same pattern in
+  `cocited_papers` (`discover.py:52-61`), `citing_papers`, `cited_by`.
+- **Root cause:** `citations.cited_id` is a **mix** of OpenAlex W-IDs (OpenAlex
+  API path, `ingest.py:229-231` ← `sources/openalex.py:155`) and DOIs/arXiv
+  (GROBID/S2 paths). `papers.id` is the canonical DOI/arXiv. An ingested paper
+  cited by its **OpenAlex ID** is *not* in `(SELECT id FROM papers)`, so it is
+  wrongly reported as a discovery lead.
+- **Impact:** `codex discover leads`, `codex discover cocited`, and the MCP
+  `discover_leads` tool over-report — foundational papers the user has **already
+  ingested** show up as "cited but not ingested". This is exactly the
+  D-05-class issue the F-15 graph path resolves via the
+  `papers.openalex_id` JOIN (`graph.py:48-56`) — but the fix was **not
+  propagated** to `discover.py`. Two divergent "dangling" implementations now
+  disagree on what "ingested" means.
+
+### C-2 — `paper_identifiers` JOIN is effectively inert · **MED**
+- **Evidence:** `graph.py:50-56` second LEFT JOIN; populated only at
+  `ingest.py:153-161`, which inserts **the same** `paper.openalex_id` that
+  `papers.openalex_id` already holds.
+- **Impact:** because ingest never records an alias that differs from
+  `papers.openalex_id`, the `pi` JOIN can only resolve what the `p` JOIN already
+  resolves. It adds value **only** in the narrow case where a paper's
+  `openalex_id` changes across re-ingests (old alias retained via
+  `ON CONFLICT DO NOTHING`). The table + unique index + seed INSERT + migration
+  burden buy near-zero behaviour today — speculative generality. No test
+  exercises an alias that diverges from `papers.openalex_id`.
+
+### C-3 — MCP `wiki_read` returns the slug as its own "sources" · **MED**
+- **Evidence:** `codex/mcp_server.py:138-149` — comment says "Collect cited
+  bibkeys", code sets `sources = [concept_slug]`.
+- **Impact:** the `sources` field is a placeholder masquerading as data; any MCP
+  client trusting it to list cited bibkeys gets the concept's own slug.
+
+---
+
+## 3. Documentation / Code Drift
+
+### D-1 — ADR-F15 validation predates the shipped graph · **HIGH (integrity)**
+- **Evidence:** ADR-F15 (commit `a1a6f45`) lands **before** the resolution fixes
+  `664906f` (`cited_id→papers.id`) and `5dfa6e4` (`paper_identifiers`).
+- **Stale claims:**
+  - "Spike Result": *478 dangling nodes*, specific PageRank scores,
+    *Bobenko/Springborn rank 7*, *5.7 % spread* — all measured on the
+    **pre-resolution** (bipartite DOI→OpenAlex) graph. Post-fix, in-KB cited
+    papers resolve to DOIs, so dangling count and PageRank distribution change.
+  - "Decision → Graph Construction": "inter-KB citations are structurally
+    invisible" — **now false**.
+  - "Known Limitation: ID-Format Mismatch": "cross-citations… not represented"
+    — **now resolved by the code**, contradicting the section.
+- **Impact:** the **GO** decision rests on numbers the shipping code no longer
+  produces. Validation must be re-run on the post-fix graph (or the ADR marked
+  validation-pending).
+
+### D-2 — `graph_cite_boost_alpha` doc describes the *fixed* bug · **MED**
+- **Evidence:** `config.py:296-300` describes
+  `Final score = dense_score * (1 + alpha * pagerank_score)` — a **multiply**.
+  Actual code (`cli.py:107`) divides distance:
+  `boosted = distance / (1.0 + alpha * pr_score)`. The ADR explicitly states the
+  multiply form was the "critical bug corrected during the F-15 review gate".
+- **Impact:** the config docstring still documents the inverted (buggy) formula.
+
+### D-3 — `paper_identifiers` tagged "F-17" in schema, shipped as F-15 fix · **LOW**
+- **Evidence:** `infra/schema.sql:129` comment "F-17 Paper identifier aliases",
+  but the table is introduced by the F-15 graph follow-up (`5dfa6e4`) and the
+  ADR-F15 work. Feature-tag inconsistency; ADR-F15 doesn't mention the table at all.
+
+---
+
+## 4. Schema / Migration
+
+### M-1 — New table has no migration path onto the live DB · **HIGH (operational)**
+- **Evidence chain:**
+  1. `apply_schema` docstring (`db.py:46-49`) claims "all DDL statements use
+     CREATE … IF NOT EXISTS" — **false**: `papers`, `chunks`, `citations`,
+     `code_links` and their indexes use bare `CREATE TABLE/INDEX`
+     (`schema.sql:16,33,56,68,30,41,…`). Only F-09/F-16/F-17 DDL is idempotent.
+  2. `apply_schema` is **never called** anywhere in the app — no `init`/`migrate`
+     CLI command (`grep apply_schema` → only `db.py` + a mocked unit test).
+  3. `ingest_all.sh` does **not** apply the schema; it only runs `codex ingest`.
+- **Impact:** re-running `infra/schema.sql` against the **existing** Jetson DB
+  fails on the first `CREATE TABLE papers` (already exists) and rolls back the
+  whole script in one implicit transaction — so the new `paper_identifiers`
+  table (and its seed INSERT) is **never created** on a DB that predates it.
+  `build_citation_graph` then fails with `relation "paper_identifiers" does not
+  exist` on the live corpus. The migration step is currently manual and undocumented.
+
+---
+
+## 5. Test Integrity
+
+### T-1 — The F-15 resolution SQL has zero real coverage · **HIGH**
+- **Evidence:** every test in `tests/graph/test_graph.py` and
+  `tests/graph/test_cli.py` builds the graph from a `MagicMock` conn
+  (`_make_conn`, `_make_conn_with_paper_ids`). `test_cross_citation_uses_doi_when_in_kb`
+  (`test_graph.py:83-99`) **pre-resolves** the rows in Python and asserts the
+  graph builds — it tests the mock, not the `COALESCE(p.id, pi.paper_id, …)` +
+  two LEFT JOINs that *are* the fix.
+- **Impact:** the substance of the two most recent commits is unverified. 330
+  green tests give false confidence about the resolution logic, the `p.id IS NULL`
+  correlated guard, and the `paper_identifiers` unique-index behaviour. Needs an
+  integration test against a real (or `pytest`-managed) Postgres.
+
+### T-2 — `apply_schema` idempotency untested · **MED**
+- **Evidence:** `tests/scaffold/test_db.py:24-32` mocks the conn and only checks
+  `execute`/`commit` are called. The (false) idempotency claim in M-1 is never
+  exercised.
+
+---
+
+## 6. Design / Usability
+
+### U-1 — `graph report` prints bare IDs, no titles · **LOW**
+- `cli.py:582-588` emits `score  <id>` and bare dangling IDs — a mix of DOIs and
+  `W…` OpenAlex IDs with no title/year. The ADR's readable author/title table is
+  not what the CLI produces; output is hard to interpret.
+
+### U-2 — `--cite-boost` can only reorder within the top-`limit` page · **LOW**
+- `cli.py:84-94` applies `ORDER BY … LIMIT limit` in SQL **before** the boost
+  (`cli.py:96-109`). A high-PR paper just outside the semantic cut can never be
+  pulled in. With α=0.3 and PR≈0.002 the effect is ~0.06 % — effectively a
+  within-page tie-breaker. Consistent with ADR intent, but worth stating as a
+  limitation (the flag does far less than "weight results by PageRank" implies).
+
+### U-3 — `graph report --json` drops the small-corpus warning · **LOW**
+- `cli.py:557-570` returns before the `graph_min_corpus_size` warning
+  (`cli.py:572-577`). JSON consumers get neither the warning nor a flag for it.
+
+### U-4 — `graph.find_co_cited` implemented + tested but not wired to a CLI · **LOW**
+- `graph.find_co_cited` (`graph.py:142`) is unit-tested but exposed by no `graph`
+  sub-command (only the separate SQL-based `discover cocited` is wired). ADR-F15
+  R-43's co-citation is only half-surfaced.
+
+---
+
+## 7. State of Tree (INFO — no defect)
+
+- The uncommitted diff (`wiki.py`, `synthesis.py`, `semanticscholar.py`,
+  `tests/synthesis/test_cli.py`, `tests/wiki/test_compile.py`) is a **legitimate
+  `ruff format` normalization** to `line-length=100`. `ruff format --check`
+  reports all 25 files formatted *with* these changes applied, meaning the
+  **committed** code was non-compliant. These edits should be **committed**, not
+  reverted. (Not a finding; clarifies the working-tree churn.)
+
+---
+
+## 8. Coverage of this audit (honesty ledger)
+
+**Deep-reviewed:** `graph.py`, `db.py`, `infra/schema.sql`, `ingest.py`,
+`discover.py`, `provenance.py` (scan), `mcp_server.py`, `cli.py` (search +
+graph), `config.py` (graph settings), F-15 tests, ADR-F15; branch-wide pattern
+scan (SQL injection, bare except, eval/exec/subprocess, secrets, timeouts).
+
+**Not yet deep-reviewed (no findings asserted):**
+- `synthesis.py` grounding guard / claim parser (`_run_grounding_guard`,
+  `_parse_claims`) — the "zero-fact-leak (N-02)" core. Prompts reviewed; guard
+  logic not yet traced.
+- `wiki.py` grounding guard + conflict detection (only the format diff seen).
+- `sources/` (arxiv, openalex, semanticscholar) beyond citation shapes.
+- `parsing/` (grobid, nougat, mathpix, figures, tex) — error handling in
+  `mathpix.py`/`figures.py` uses broad `except Exception` without `noqa`
+  (`mathpix.py:109,151`, `figures.py:125,152`) — flagged for a later pass.
+- `embed.py`, `quality.py`, `models.py`.
+
+A second pass should close these before the audit is declared complete.

docs/audit/DATA-QUALITY-2026-06-15.md

@@ -0,0 +1,575 @@
+# DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session)
+
+**Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement
+applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is
+strong after fixing a P0 `codex search paper` crash. **DQ-2 RESOLVED 2026-06-16**
+(1911.00966 fully recovered from arXiv; s00454 abstract from S2; book chapter is a
+documented limit; recovery wired into ingest). **DQ-3 RESOLVED 2026-06-16**
+(content fidelity clean — full coverage, only OCR junk dropped). **All four
+assessment items (DQ-1..DQ-4) DONE. DQ-5 RESOLVED 2026-06-17** (DOI id normalized
+to bare canonical form in `openalex._map_paper`; live re-ingest of
+`10.1007/s00454-019-00132-8` confirmed idempotent → **unblocks R-A/R-C**).
+**Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section).
+**Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory
+of the audit conversation — everything needed to continue is in this file.
+
+---
+
+## Why this exists — the second audit axis
+
+The code/pipeline audit (`AUDIT-2026-06-15-loop-1.md`, `-full-repo.md`) verified the
+**loader is correct**: IDs are canonical, the citation graph is consistent, the
+ingest pipeline does what it claims. It did **not** ask whether the *information
+in the database is actually good*. A correct loader can faithfully load thin,
+incomplete, or unrepresentative data. This document is that second axis — a
+**data-quality** assessment of the live corpus, independent of code correctness.
+
+---
+
+## How to reach the data (self-contained)
+
+- **Live DB:** Jetson PostgreSQL via an SSH tunnel (open it first):
+  ```
+  ssh -f -N -L 5433:localhost:5432 alfred@jetson.local
+  ```
+- **Credentials:** `.env.jetson-ingest` (gitignored) holds `DATABASE_URL`
+  (`postgresql://researcher:…@localhost:5433/papers`). The `researcher` role is
+  DML-only (read freely; no DDL).
+- **Run probes:** `set -a; source .env.jetson-ingest; set +a` then
+  `PYTHONPATH=. .venv/bin/python` with `psycopg` + `psycopg.rows.dict_row`.
+  Reusable domain helpers: `codex.quality` (`is_quality_chunk`, `_bib_score`,
+  `classify_section`), `codex.graph`, `codex.discover`.
+- **Corpus snapshot (2026-06-15, post canonical-id migration):**
+  **29 papers, 1507 chunks, 590 citations.**
+
+---
+
+## Baseline — what is already GOOD (measured, no action needed)
+
+- **Chunk content is clean.** Re-running the F-16 quality gate over all stored
+  chunks: **0 / 1507 fail** `is_quality_chunk` (none too-short, none low-alpha /
+  OCR-ish, none bibliography-like). The gate did its job on the `.txt` ingest.
+- **Chunk sizing healthy.** chars: min 478 / median 2913 / max 4154; **0 exact
+  duplicate** chunk bodies.
+- **`section` column is low-signal** (98 % `body`; 13 `bibliography`, 9 `theorem`,
+  2 `intro`, 1 `proof`). This confirms code-audit **R-12** — word-window chunks
+  rarely start at a section header. Not a data defect; the column is just weak.
+
+---
+
+## Findings (open investigation items)
+
+### DQ-1 — Citation coverage: 12 / 29 papers (41 %) had ZERO citations · **RESOLVED 2026-06-16**
+
+**Resolution (what was done):**
+- **Verified upstream, not a bug.** Queried the OpenAlex API directly for all 11
+  zero-citation papers that have an `openalex_id`: every one returns
+  `referenced_works_count = 0` server-side, W-ids all match what we stored. They
+  are `type: preprint` (arXiv) / `article` / `dissertation` — works OpenAlex
+  indexes without a parsed bibliography. Confirmed source-coverage limit.
+- **Semantic Scholar supplement applied to the live DB** (idempotent backfill,
+  `ON CONFLICT (citing_id,cited_id) DO NOTHING`): **+330 citations (590 → 920)**.
+  Zero-out-edge papers **12 → 2**; in-corpus citing coverage **17/29 → 27/29**;
+  graph 489→704 nodes, 590→920 edges; discovery-lead targets 472→677. 9 of the
+  new edges are in-corpus (e.g. `2305.10988 → 10.1007/s00454-019-00132-8`).
+- **Wired into ingest** so future re-ingests self-heal: when OpenAlex returns an
+  empty `referenced_works`, `ingest.py` now falls back to
+  `_s2_reference_supplement()` (citing_id rewritten to canonical `papers.id`,
+  DOI cited-ids lowercased). Same helper drove the one-off backfill.
+- **Bug fixed (would have crashed the batch):** `semanticscholar.fetch_references`
+  used `data.get("data", [])`, but S2 returns HTTP 200 `{"data": null}` for a
+  known paper with no parsed refs (e.g. `depositonce-5415`) → `TypeError`.
+  Changed to `data.get("data") or []`. Removed a now-redundant discarded S2 probe
+  in the OpenAlex-404 arXiv branch. New regression test added.
+- **Irreducible remainder (documented limit):** the **2 `depositonce` theses**
+  (`10.14279/depositonce-20357`, `-5415`) stay at zero — neither OpenAlex nor S2
+  indexes references for TU-Berlin institutional DOIs. Accept as inherent.
+- **Files touched (uncommitted in working tree):** `codex/sources/semanticscholar.py`,
+  `codex/ingest.py`, `tests/ingest/test_ingest.py`. Full suite: 331 passed.
+
+**Original finding (for context):**
+- **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph
+  comes from only ~17 papers. The whole F-15 layer (PageRank / coupling /
+  co-citation / discovery leads) therefore runs on ~59 % of the corpus.
+- **Sharpening — it is mostly a source-coverage limit, not an ingest bug:**
+  **11 of the 12** zero-citation papers *do* have an `openalex_id`, i.e. ingest
+  called `openalex.fetch_citations(openalex_id)` and OpenAlex returned an **empty
+  `referenced_works`** list. Only `1911.00966` has no `openalex_id` (OpenAlex
+  404 → arXiv/S2 fallback, which also yielded nothing).
+- **Zero-citation ids:** `1005.2698`, `1505.01341`, `1911.00966` (no oa),
+  `2206.13461`, `2305.10988`, `2310.17529`, `2601.22903`, `math/0001176`,
+  `math/0503219`, `math/0603097`, `10.14279/depositonce-20357`,
+  `10.14279/depositonce-5415`. (Pattern: arXiv preprints + the two `depositonce`
+  theses — works OpenAlex indexes without parsed references.)
+- **To investigate next:**
+  1. Confirm it is OpenAlex coverage, not a `fetch_citations` bug: for 2–3 of the
+     ids, hit `GET https://api.openalex.org/works/<openalex_id>` and check whether
+     `referenced_works` is genuinely empty server-side.
+  2. If genuinely empty: enrich via the **Semantic Scholar references** path
+     (`semanticscholar.fetch_references("arXiv:<id>")`) as a *supplement* for
+     OpenAlex-empty papers — S2 often has references where OpenAlex doesn't. Note
+     S2 cited_ids are bare DOI/arXiv, so they flow through the existing
+     `RESOLVED_CITATIONS_SQL` resolver (audit C-1) fine.
+  3. Decide whether the F-15 `graph_min_corpus_size` warning should also flag *low
+     citing-paper coverage*, not just paper count.
+- **Acceptance:** either (a) citation coverage materially improves after an S2
+  supplement, or (b) documented as an inherent OpenAlex-coverage limit with the
+  graph caveated accordingly.
+
+### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **RESOLVED 2026-06-16**
+
+**Resolution (what was done):**
+- **`1911.00966` fully recovered** from the arXiv Atom API (OpenAlex 404'd on it):
+  title "A discrete version of Liouville's theorem on conformal maps",
+  Pinkall & Springborn, 2019, 384-char abstract → bibkey `PinkallSpringborn2019`
+  auto-generated, abstract embedded (non-zero). It was the worst node in the
+  corpus — now `@cite`-able, in wiki grounding, and its **10 chunks are visible
+  to chunk search** (previously filtered out by `bibkey IS NOT NULL`). Citations
+  were already added in DQ-1 (27 S2 refs).
+- **`10.1007/s00454-019-00132-8` abstract recovered from S2** (1186 chars) and
+  re-embedded (targeted `UPDATE`, not full re-ingest — see DQ-5).
+- **`10.1007/978-3-642-17413-1_7` (book chapter): documented inherent limit.**
+  No abstract in OpenAlex, S2, **or Crossref** (verified). Left as the single
+  remaining zero-vector paper; user may add an abstract by hand.
+- **Corpus now: 1 paper without abstract** (the book chapter), down from 3+1.
+  Zero-vector search pollution (DQ-4 secondary finding) reduced from 2 tail
+  fillers to 1.
+- **Wired into ingest** so this self-heals: OpenAlex-404 on an arXiv id now
+  recovers metadata via `arxiv.fetch_metadata` (new); an empty abstract is
+  supplemented via `_recover_abstract` → S2 then Crossref (new
+  `codex/sources/crossref.py`). New `semanticscholar.fetch_abstract`.
+- **Files touched:** `codex/sources/arxiv.py`, `codex/sources/crossref.py` (new),
+  `codex/sources/semanticscholar.py`, `codex/ingest.py`, + tests. Full suite: 342.
+
+**Original finding (for context):**
+- **Measured:**
+  - **No abstract (3):** `10.1007/978-3-642-17413-1_7`,
+    `10.1007/s00454-019-00132-8`, `1911.00966`. These get a **zero-vector**
+    `abstract_emb`, so paper-level semantic search can't place them.
+  - **No bibkey / year (1):** `1911.00966` only — and its authors array is empty
+    too. This paper is **fully degraded** (OpenAlex 404 → `Paper(id, title="")`
+    fallback): no abstract, no bibkey, no year, empty authors, no citations. With
+    no bibkey it cannot be `@cite`d and is invisible to wiki grounding (which keys
+    on bibkey).
+- **To investigate next:**
+  1. `1911.00966` — confirm the arXiv id is correct and whether OpenAlex/S2 has it
+     under a different id (DOI?); if recoverable, re-ingest to populate metadata.
+     If not, decide: keep as a degraded node or drop.
+  2. The 2 no-abstract DOIs — check if OpenAlex has an `abstract_inverted_index`
+     that the mapper missed, or if the abstract is genuinely absent upstream.
+- **Acceptance:** every paper has at least a bibkey + non-zero abstract embedding,
+  or the exceptions are documented with rationale.
+
+### DQ-3 — Content fidelity (chunks vs source `.txt`): VERIFIED CLEAN · **RESOLVED 2026-06-16**
+
+**Resolution (read-only; re-ran `chunk_text` + `is_quality_chunk` on all 29
+source files and compared to stored chunks):**
+- **No paper is silently gutted.** Coverage (stored chunk chars / source chars)
+  is **109–114% for every paper** — the >100% is exactly the 64/512-word overlap
+  inflation, i.e. full retention. `stored == kept` for all 29, so the live DB
+  matches what the current pipeline produces.
+- **Only 18 chunks dropped corpus-wide, all `low_alpha`** (0 short, 0 bib),
+  concentrated in `10.1007/0-387-29555-0_13` (6) and `depositonce-5415` (10).
+  Inspected: they are **garbled OCR tables** from scanned-book sources (alpha
+  0.30–0.36, e.g. `"(u: 1) 7 < u D:= ud3424 — else H®: out.v. Family 1…"`) —
+  exactly the artefacts the alpha-ratio gate targets. **No real math content lost**
+  (the handoff's feared failure mode did not occur; math prose stays > 0.40 alpha).
+- **Head/tail alignment exact** on sampled papers: chunk 0 begins at the source
+  head (title/authors/abstract captured), last chunk ends at the source tail → no
+  front/back truncation.
+- **Note:** 0 bibliography drops across the corpus → the upstream `.txt` extraction
+  had already stripped reference lists, which is why DQ-1's citation graph relies
+  on the API/GROBID sources rather than text-mined refs.
+- **Acceptance met** for all 29 papers (not just a sample). No action needed.
+- **⚠ SCOPE LIMIT — DQ-3 covers `.txt → chunks` ONLY.** It treats each source
+  `.txt` as ground truth and verifies the *ingest pipeline* preserves it. It does
+  **not** measure the layer above — **PDF/source → `.txt`** — which was done by an
+  upstream tool in `ConformalLabpp` (outside codex-py) and is unexamined. There is
+  already evidence that layer is lossy: the 18 dropped chunks were garbled OCR
+  tables *already present in the `.txt`* (i.e. the PDF→txt step garbled the
+  table/formula regions), and the bibliographies were stripped upstream. So
+  "VERIFIED CLEAN" means **the loader faithfully preserves whatever the `.txt`
+  contains, not that no information was lost from the source paper.** See DQ-6.
+
+### DQ-6 — PDF/source → `.txt` extraction fidelity: NOT MEASURED · **MED, open**
+- **Open question:** how much did the upstream PDF→`.txt` extraction (done in
+  `ConformalLabpp`, outside codex-py) drop or garble vs the original papers?
+  Math/equations, tables, multi-column layouts, and figure captions are the usual
+  casualties of PDF text extraction — and we have direct evidence of garble
+  (DQ-3's low_alpha drops were already-garbled tables in the `.txt`).
+- **Checkable:** the 36 original PDFs are present in `…/ConformalLabpp/papers/`
+  (map each paper's `source_path` `.txt` to its sibling `.pdf` by filename stem).
+- **To investigate:** for a sample (incl. the known-bad scanned book
+  `0-387-29555-0_13` and a born-digital arXiv PDF), extract PDF text independently
+  and compare length / page-coverage to the `.txt`; spot-check whether specific
+  theorem statements & equations survive. Flag papers where the `.txt` is missing
+  large spans.
+- **Acceptance:** a per-sample source→txt coverage estimate, or a list of papers
+  whose `.txt` is materially degraded (candidates for re-extraction — note roadmap
+  **R-A/R-C** would re-ingest from PDF/`.tex` directly and largely sidestep this).
+
+**Original open question (for context):**
+- **Open question:** does the stored chunk set faithfully reconstruct each source
+  file, or did the chunker / `filter_chunks` silently drop material (e.g. an
+  abstract, a section, math-heavy passages)? The whole corpus was ingested from
+  `.txt` (`PAPERS_DIR=/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt`), so
+  the `.tex`/`.pdf` paths never ran.
+- **To investigate next:** for a sample of ~5 papers, compare
+  `len("".join(stored chunks))` against `len(source.txt)` (coverage %), and
+  eyeball the first/last chunk vs the file head/tail. Flag papers where coverage
+  is low (content lost) — F-16 dropping >X % of a paper is a signal.
+- **Acceptance:** sampled papers retain ≳ the expected fraction of source text;
+  no paper is silently gutted by the quality gate.
+
+### DQ-4 — Retrieval quality · **MEASURED 2026-06-16 — good, after fixing a P0 crash**
+
+**P0 bug found + fixed: `codex search paper` was crash-broken on the live DB.**
+The CLI passed the query embedding uncast, so pgvector raised
+`operator does not exist: vector <-> double precision[]` on every paper search.
+The working call sites (`mcp_server.py`, `wiki.py`) cast `%(emb)s::vector`; the
+CLI did not. Fixed both `<->` occurrences in `codex/cli.py search_paper`; added a
+regression guard in `tests/cli/test_cli.py` asserting the cast (mocked-DB unit
+tests can't catch the type error — this is why it survived the code audit).
+
+**Relevance verdict (post-fix): strong.** Six domain queries, both surfaces
+(paper-level abstract search + chunk-level). **rank-1 was the exactly-correct
+paper in 6/6 queries** for chunk search and 5/6 for paper-level (Laplace–Beltrami
+put the exact paper at rank-2 behind the closely-related "Polygon Laplacian Made
+Simple" — acceptable). Examples: "combinatorial Yamabe flow" → `math/0306167`
+(exact, all 5 top chunks that paper); "variational principle for Delaunay
+triangulations" → `math/0603097` (exact); "circle packing rigidity" →
+`2601.22903` (exact). Three of the DQ-1-rescued papers (`2305.10988`,
+`1005.2698`, `depositonce-5415`) now surface as top hits. The KB is trustworthy
+for lookups.
+
+**Secondary finding → reinforces DQ-2: zero-vector abstract pollution.** Papers
+with no abstract get a zero `abstract_emb`, which sits at constant L2 distance
+≈1.000 from every query and appears as filler in the paper-level tail whenever
+< 5 strongly-relevant papers exist. `1911.00966` (fully degraded, DQ-2) showed up
+at distance 1.000 in 4/6 queries. Harmless at rank>1 with a visible 1.000 score,
+but noise — and a concrete reason to fix DQ-2. Chunk-level search is immune (it
+filters `bibkey IS NOT NULL`, and 1911.00966 has no bibkey/chunks).
+
+**Note:** the MCP `search` docstring claims "hybrid dense + FTS" but the SQL is
+dense-only; `score = 1.0 - dist` can go negative (bge-m3 L2 ranges 0–2). Ranking
+is fine (monotonic); only the absolute score label is misleading.
+
+**Original open question (for context):**
+- **Open question:** end-to-end, do real queries return *relevant* results? Clean
+  chunks + a working index don't guarantee useful retrieval.
+- **To investigate next:** run a handful of domain queries through the actual
+  search path (`codex search paper "<q>"` and the chunk-level MCP `search`), e.g.
+  "discrete conformal map", "circle packing rigidity", "combinatorial Yamabe
+  flow", "discrete Laplace-Beltrami operator", and judge whether the top-5 hits
+  are on-topic and point at the right papers. Cross-check a couple against
+  `--cite-boost` to see if the boost helps or hurts on this corpus.
+- **Acceptance:** a short relevance table (query → top-5 → on/off-topic) good
+  enough to trust the KB for lookups, or a list of failure modes to fix.
+
+### DQ-5 — `ingest_paper` is NOT idempotent for DOI papers · **RESOLVED 2026-06-17** (found 2026-06-16)
+
+**Resolution (what was done):**
+- **Root-cause fix, minimal + contained:** added `openalex._normalize_doi()` and
+  applied it in `_map_paper` so the DOI branch yields the canonical **bare,
+  lower-cased** form (strips `https://doi.org/` / `http://doi.org/` / `doi:`;
+  lower-cases — DOIs are case-insensitive, matching `ingest._norm_cited_id`). The
+  no-DOI fallback (OpenAlex W-id URL) is left untouched. `_map_paper`→`fetch_paper`
+  →`ingest_paper` is the only consumer chain, so blast radius is one function.
+- **Checked nothing depended on the URL form.** It was the opposite: the full-URL
+  id silently *defeated* `_s2_id_for(paper.id)` and `_recover_abstract`'s
+  `paper.id.startswith("10.")` (a `https://…` string is neither `10.`- nor
+  `doi:`-prefixed), so S2/Crossref recovery never fired for DOI papers. The fix
+  repairs those paths too.
+- **Tests (offline, mocked DB):** closed audit **T-3** — the OpenAlex fixture now
+  emits the real URL-form DOI and asserts `paper.id` is the bare form; added
+  `_normalize_doi` unit tests (https/http/`doi:`/upper-case/already-bare) and a
+  W-id-fallback test; added ingest regression `test_ingest_doi_paper_upserts_bare
+  _canonical_id` (runs the real `_map_paper` through ingest, asserts the papers
+  upsert carries the **bare** id into `%(id)s` and uses `ON CONFLICT (id)`).
+  Verified it has teeth (fails on the pre-fix code). Full suite **348 passed**;
+  ruff + mypy --strict clean.
+- **Live idempotency confirmed (tunnel up):** re-ingested
+  `10.1007/s00454-019-00132-8` against the Jetson DB → no `UniqueViolation`;
+  `IngestResult(paper_id='10.1007/s00454-019-00132-8', citations_upserted=45)`.
+  After-state: **UPDATE in place** — `id` and `openalex_id` unchanged, **`added_at`
+  unchanged** (proves no fresh INSERT), exactly **1** row for the DOI (no URL-form
+  duplicate), citations 45→45, total papers 29→29.
+- **Unblocks R-A (GROBID re-ingest) and R-C (.tex re-ingest)** — both re-ingest
+  existing papers. **Files touched:** `codex/sources/openalex.py`,
+  `tests/sources/test_openalex.py`, `tests/ingest/test_ingest.py`.
+- **Note (out of scope, pre-existing):** arXiv papers stored as bare ids
+  (`2305.10988`) are still not round-trip idempotent if OpenAlex returns an
+  arXiv-DOI/W-id instead of the bare arXiv id — a separate canonicalization
+  concern, not this UniqueViolation. The DOI path (this finding) is fixed.
+
+**Original finding (for context):**
+- **Symptom:** re-ingesting an existing DOI paper crashes with
+  `UniqueViolation: papers_openalex_id_key`. Surfaced trying to re-ingest
+  `10.1007/s00454-019-00132-8` during the DQ-2 backfill.
+- **Root cause:** `openalex._map_paper` sets `Paper.id = data["doi"]`, which is the
+  **full URL** `https://doi.org/10.1007/…`. The stored canonical id is the **bare**
+  `10.1007/…` (post the M-1 canonical-id migration). So `INSERT … ON CONFLICT (id)`
+  finds no id match, attempts a fresh INSERT, and trips the *separate* unique
+  constraint on `openalex_id` (already held by the canonical-id row).
+- **Confirmed:** `openalex.fetch_paper("10.1007/s00454-019-00132-8").id ==
+  "https://doi.org/10.1007/s00454-019-00132-8"` (≠ stored bare id).
+- **Impact:** any re-ingest of a DOI paper aborts. This **blocks roadmap R-A
+  (GROBID on PDFs) and R-C (.tex ingest)** — both re-ingest existing papers. Also
+  implies *new* DOI ingests store full-URL ids, inconsistent with the bare-id
+  corpus (latent split-identity risk).
+- **Proposed fix (handle with care — M-1 incident territory):** normalize the DOI
+  in `_map_paper` to the canonical bare, lower-cased form (strip
+  `https://doi.org/`), matching what the M-1 migration produced. Add a re-ingest
+  idempotency test over a DOI paper. Did **not** fix inline this session — the
+  canonical-id derivation is exactly where the M-1 migration incident occurred, so
+  it needs its own focused change + a live re-ingest check.
+- **Workaround used for DQ-2:** targeted `UPDATE … SET abstract, abstract_emb`
+  instead of full re-ingest.
+
+---
+
+## Roadmap — data-acquisition levers (post-audit, all free sources)
+
+These are *forward-looking acquisition improvements*, distinct from the DQ-1..DQ-4
+audit findings (which assessed existing data). They came out of a strategy
+discussion on 2026-06-16: for this Math/CS corpus the valuable data (references,
+abstracts, full text) is almost entirely **open** (arXiv, OpenAlex, Crossref, S2),
+so the quality lever is *combining more free sources*, not buying paywall access.
+Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item
+is self-contained so a cold session can pick it up.
+
+### R-A — Run GROBID reference extraction on PDFs · **CORE DONE 2026-06-17** (full sweep optional)
+
+**Resolution (what was done) — citing coverage 27/29 → 29/29 (100%); edges 920 → 1022:**
+- **References-only backfill, not full re-ingest.** Added
+  [scripts/ra_grobid_backfill.py](../../scripts/ra_grobid_backfill.py): per paper,
+  run `grobid.extract_references(pdf)`, normalize cited-ids to the canonical bare
+  form (DOIs de-URL'd + lower-cased; `arXiv:` stripped), INSERT into `citations`
+  (`ON CONFLICT DO NOTHING`). Deliberately avoids `ingest_paper(source_path=pdf)`,
+  whose PDF branch re-runs Nougat + `DELETE FROM chunks` and would overwrite the
+  DQ-3-verified-clean `.txt` chunks (and needs Nougat). Idempotent; dry-run by
+  default; 7 tests; ruff/mypy clean.
+- **GROBID provisioning:** the pinned `grobid/grobid:0.8.2` is amd64-only, but the
+  Jetson and the dev Mac are both arm64 (this is why GROBID never ran). Ran the
+  lighter CRF image (`grobid/grobid:0.8.2-crf`) on the Mac under Podman+Rosetta
+  (PDFs are local; citations written over the SSH DB tunnel). Made
+  `extract_references` timeout configurable (theses need >60s emulated). A separate
+  task is provisioning a native arm64 GROBID on the Jetson (`docs/infra/grobid-jetson.md`).
+- **Both `depositonce` theses closed:** `…depositonce-20357` (lutz-2024) **+96**
+  edges (5 in-corpus); `…depositonce-5415` (sechelmann-2016) **+6** edges — its
+  147 MB/173-page scan blew pdfalto's 120s limit, so the 9 bibliography pages were
+  extracted with PyMuPDF first (77 refs, only 6 with a DOI/arXiv). **No paper has
+  zero out-edges now.**
+- **Optional remaining (secondary acceptance):** sweep the other 27 already-covered
+  papers for *extra* GROBID refs via `python scripts/ra_grobid_backfill.py --write`
+  (the 2 book PDFs need the same bib-page extraction as sechelmann).
+
+**Original plan (for context):**
+- **✓ Unblocked (DQ-5 RESOLVED 2026-06-17):** DOI re-ingest is now idempotent, so
+  the GROBID re-ingest no longer aborts on `papers_openalex_id_key`.
+- **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`),
+  so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper
+  so `codex.parsing.grobid.extract_references` parses each bibliography.
+- **Why:** extracts references the APIs lack — a *complement* to DQ-1's S2
+  supplement. Crucially, the **2 `depositonce` theses** that stayed at zero in
+  DQ-1 (no API references anywhere) almost certainly have a references section in
+  their open-access PDFs → GROBID could finally give them out-edges. Closes the
+  last 2/29.
+- **How:** no new code — ingest already merges + dedups `api_citations +
+  pdf_citations` ([codex/ingest.py:288](../../codex/ingest.py)). GROBID server is
+  already configured (`GROBID_URL=http://jetson.local:8070` in
+  `.env.jetson-ingest`). Download free arXiv PDFs, re-ingest with `source_path`.
+- **Acceptance:** the 2 theses gain references; published papers gain
+  GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29.
+
+### R-B — Add Crossref as a third citation + abstract source · **DONE 2026-06-17**
+
+**Resolution (what was done):**
+- **`crossref.fetch_references(doi)`** added (mirrors `semanticscholar.fetch_references`):
+  GETs `/works/{doi}`, returns one `Citation` per `message.reference[]` entry that
+  carries a registered `DOI` (book/`unstructured`-only refs are skipped — they
+  can't join the graph). `citing_id` is the queried DOI; the ingest supplement
+  rewrites it to `paper.id`.
+- **Wired as the third leg of the citation fallback chain** in `ingest.py`:
+  OpenAlex-empty → S2 → **Crossref** (`_crossref_reference_supplement`, DOI papers
+  only — Crossref's works endpoint is DOI-keyed). cited DOIs pass through
+  `_norm_cited_id` (bare, lower-cased). Network/parse failures degrade to [].
+- **Tests:** `fetch_references` unit tests (DOI edges / `unstructured` skipped / no
+  `reference` key / 404) + ingest tests asserting the Crossref leg fires when
+  OpenAlex+S2 are empty (cited DOI normalised) **and** does *not* fire when OpenAlex
+  has citations. Suite **361 green**, ruff/mypy clean. The DQ-2 abstract half
+  (`fetch_abstract`) already shipped earlier.
+- **Live-validated:** `fetch_references` against the real API returned 33 / 40 / 24
+  DOI-bearing refs for `10.1007/s00454-019-00132-8` / `10.1145/3132705` /
+  `10.1111/cgf.13931` (Springer / ACM / Wiley).
+- **No live corpus change (by design):** as a *fallback*, Crossref fires only when
+  OpenAlex **and** S2 are both empty. Post-DQ-1/R-A the only such papers were the 2
+  `depositonce` theses, which Crossref doesn't index either — so R-B is a
+  forward-looking third leg for future DOI ingests, not a backfill. (Harvesting
+  Crossref's *unique* refs for already-covered papers would need an always-merged
+  supplement instead — a deliberate scope choice not taken, per the doc's "fallback
+  chain" spec.) **Files:** `codex/sources/crossref.py`, `codex/ingest.py`, + tests.
+
+**Original plan (for context):**
+- **What:** new `codex.sources.crossref` client hitting Crossref REST
+  `/works/{doi}` for the `reference` array and `abstract`.
+- **Why:** Crossref carries **publisher-deposited references** for many DOIs (free,
+  polite pool via `mailto`) — a third leg after OpenAlex/S2. It also recovers
+  abstracts OpenAlex is contractually barred from redistributing (likely the 2
+  Springer DOIs missing abstracts in DQ-2). cited_ids are DOIs → flow through the
+  existing graph resolver unchanged.
+- **How:** mirror `codex/sources/openalex.py` (tenacity retry, polite pool). Extend
+  the ingest fallback chain to OpenAlex-empty → S2 → Crossref (same
+  `_s2_reference_supplement` shape). Add unit tests with mocked HTTP.
+- **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where
+  OpenAlex+S2 are empty.
+
+### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **DONE 2026-06-17**
+
+**Resolution (what was done) — all 13 arXiv papers re-ingested from `.tex`:**
+- **Multi-file flatten:** `arxiv.fetch_source` now inlines `\input`/`\include`
+  (`tex.flatten_inputs`) so multi-file projects yield the full body, not the
+  primary file's include skeleton (e.g. math/0603097 = 15 files / 12 includes;
+  2305.10988 = 6 / 5). Also handles legacy single-file **bare-gzipped** `.tex`
+  (no tar wrapper) for old math/* papers (commit 121b582).
+- **Section-aware chunking:** new `tex.chunk_sections` returns `(title, chunk)`
+  pairs (chunks never span a `\section`); the ingest `.tex` path labels the stored
+  `section` from the real heading instead of running `classify_section` on a
+  header-less word-window. Quality-gated as pairs so labels stay aligned.
+- **Unblocked by the arXiv half of DQ-5** (commit f504ca6): `_canonical_id`
+  strips the `10.48550/arxiv.` DOI prefix so an arXiv paper's id is the bare
+  arXiv id — without it, re-ingest tripped `papers_openalex_id_key` (confirmed
+  live, then fixed; idempotent re-ingest verified).
+- **Outcome:** `section` column gained signal — ~34/329 arXiv chunks now
+  intro/theorem/proof (was ~all `body`); math/0603097 `{body:31}`→`{body:29,
+  proof:9}`, 2310.17529 gained proof+intro, etc. Math papers with descriptively
+  titled sections stay `body` (classify_section's fixed vocab) — modest but real
+  (R-12). No paper gutted (chunk counts stable/▲); F-16 gate unchanged.
+- **Files:** `codex/parsing/tex.py`, `codex/sources/arxiv.py`, `codex/ingest.py`,
+  `codex/sources/openalex.py`, `scripts/rc_tex_reingest.py` (driver, dry-run
+  default), + tests. Full suite **370 passed**; ruff/mypy clean. Branch
+  `feat/tex-ingest` (commits 1da81c7, 121b582, f504ca6).
+
+**Original plan (for context):**
+- **What:** re-ingest from arXiv LaTeX source through the existing
+  `codex.parsing.tex.latex_to_text` path instead of flattened `.txt`.
+- **Why:** highest-fidelity input — preserves math, structure, and section
+  boundaries. Directly serves **DQ-3** (content fidelity) and audit **R-12** (the
+  `section` column is low-signal because word-window `.txt` chunks rarely start at
+  a real header; `.tex` headers fix this).
+- **How:** download free arXiv source tarballs, feed `.tex` to ingest with
+  `source_path`. Handle multi-file projects (main `.tex` + includes).
+- **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage
+  vs source improves; no regression in the F-16 quality gate.
+
+### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **DONE 2026-06-17**
+
+**Resolution (what was done):**
+- Added `graph.citing_coverage(graph, known_ids)` → `(papers_with_out_edges,
+  total)`, a `graph_min_citing_coverage` setting (default 0.8), and wired both into
+  `codex graph report`: it now prints `Citing coverage: N/M papers with out-edges
+  (P%)` (and in `--json`), and emits a `Warning` when the share is below the
+  threshold ("low citing coverage weakens PageRank/coupling"). Separate from the
+  existing `graph_min_corpus_size` (total-count) warning.
+- Tests: `citing_coverage` unit tests + CLI tests (line shown; warning fires at
+  low coverage; suppressed at 100%; JSON field). Full suite green; ruff/mypy clean.
+- **Live:** `codex graph report` on the corpus prints `Citing coverage: 29/29
+  papers with out-edges (100%)` with no warning (post-R-A). Branch
+  `feat/graph-coverage-warning`. Files: `codex/config.py`, `codex/graph.py`,
+  `codex/cli.py`, + tests.
+
+**Original plan (for context):**
+- **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper
+  count. Add a warning when the share of papers with ≥1 out-edge is low (e.g.
+  < 80%), since that is what actually starves PageRank/coupling.
+- **Why:** at 17/29 the graph silently ran on 59% of the corpus with no signal.
+  Post-DQ-1 it's 27/29 (93%) — but the guard should catch future regressions.
+- **Acceptance:** `codex graph report` surfaces citing-coverage % and warns below
+  threshold.
+
+### R-E — Paywall / institutional access · **DEFERRED — only for a paywall-only expansion**
+- **Decision (2026-06-16):** a uni login does **not** help the current corpus —
+  every DQ-1/DQ-2 gap is in openly-available works (arXiv preprints, OA theses),
+  and all metadata/citation sources are free APIs. Revisit *only* if the corpus
+  expands to papers that exist solely behind a paywall with no preprint; then
+  institutional access → full-text PDF → GROBID refs (R-A) + clean chunks (R-C).
+- **ToS caveat:** manual download of individual papers you have legitimate access
+  to is fine; **automated bulk download through a uni proxy (EZproxy/Shibboleth)
+  violates most publisher terms and can get the whole institution's access
+  revoked.** Do not wire a publisher login into the ingest pipeline.
+
+### R-F — Richer section labels: store real `\section` titles · **DONE 2026-06-17**
+
+**Resolution (what was done):**
+- **Why:** R-C labelled `.tex` chunks by mapping the real `\section` heading through
+  `classify_section`'s fixed vocab, so descriptively-titled Math sections collapsed
+  to `body` (only **34/329 = 10 %** non-`body`). A research pass confirmed the
+  `section` column is **write-only** (nothing filters on the vocab — `mcp_server.py`
+  and `wiki.py` don't even SELECT it), so storing free-text titles is safe.
+- **Change:** new `quality.section_label(title, content)` (+ `_clean_title`) — keep the
+  controlled bucket (intro/theorem/proof/abstract/bibliography) when the heading maps
+  to one, else store the cleaned real title ("the flip algorithm", "main results", …).
+  The `.tex` ingest path uses it; `.pdf`/`.txt`/`run_quality_pass` keep
+  `classify_section` unchanged. `_clean_title` strips LaTeX commands/accents/ties +
+  leading numbering, lower-cases, truncates at a word boundary. ~1 helper + 1 ingest
+  line; no schema change (column already free `TEXT`).
+- **Result (live):** re-applied to the 13 arXiv papers → non-`body` **34/329 (10 %) →
+  314/329 (95 %)**. Only `math/0306167` (old AMS-TeX, no `\section{}`) stays `body`.
+- **Tests:** `section_label` / `_clean_title` units + `.tex` ingest stores a
+  descriptive heading verbatim. Full suite 377 passed; ruff/mypy clean. Branch
+  `feat/section-labels` (off `feat/tex-ingest`).
+- **Accent re-clean — DONE 2026-06-17.** The `_clean_title` polish (`m\"obius`→`mobius`,
+  `~` ties, word-boundary truncation) landed in code, and the existing live labels were
+  re-cleaned in place (apply `_clean_title` to any `chunks.section` containing `\` or
+  `~`; **8 labels** fixed, no re-embed). Verified: **0** labels with LaTeX artifacts
+  remain; non-`body` holds at **314/329 (95 %)**. R-F fully complete.
+
+---
+
+## Priority for the next session
+1. ~~**DQ-1**~~ — **DONE 2026-06-16** (live DB at 920 citations, 27/29 citing
+   coverage; remedy wired into ingest).
+2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance
+   verified strong; surfaced the zero-vector pollution that motivates DQ-2).
+3. ~~**DQ-2**~~ — **DONE 2026-06-16** (1911.00966 fully recovered; s00454 abstract
+   from S2; book chapter documented; recovery wired into ingest).
+4. ~~**DQ-3**~~ — **DONE 2026-06-16** (content fidelity clean; full coverage, only
+   18 OCR-junk chunks dropped, no math lost). **Assessment axis DQ-1..DQ-4 complete.**
+5. ~~**DQ-5**~~ — **DONE 2026-06-17** (DOI id normalized to bare form in
+   `openalex._map_paper`; live re-ingest of `10.1007/s00454-019-00132-8` idempotent
+   — UPDATE in place, `added_at` unchanged; suite 348 green). **Unblocks R-A/R-C.**
+6. **Roadmap levers** (see section above — now all unblocked): **R-A** (GROBID on
+   arXiv PDFs — closes the last 2 zero-citation theses), **R-B** (Crossref refs —
+   abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3),
+   **R-D** (F-15 coverage warning, quick). **R-E** (paywall) deferred.
+
+DQ-1..DQ-4 are *data*/assessment work; DQ-5 + roadmap R-A..R-D add **new code**
+(canonical-id fix, re-ingest scripts, Crossref references, a `.tex` path, a graph
+warning) on top of the already-remediated pipeline (see the AUDIT-* docs and PRs
+#12–#14). The DQ-1/DQ-2 ingest wiring + Crossref/arXiv abstract recovery already
+landed this session.
+
+---
+
+## Appendix — reproduce the baseline scan
+
+```python
+import os, psycopg, statistics
+from collections import Counter, defaultdict
+from psycopg.rows import dict_row
+from codex.config import Settings
+from codex.quality import is_quality_chunk, _bib_score
+
+s = Settings()
+with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as c:
+    papers = c.execute("SELECT id, bibkey, title, abstract, year, authors, openalex_id FROM papers").fetchall()
+    chunks = c.execute("SELECT paper_id, content, section FROM chunks").fetchall()
+    cit = c.execute("SELECT citing_id, count(*) n FROM citations GROUP BY citing_id").fetchall()
+
+# zero-citation papers (DQ-1)
+cited = {r["citing_id"] for r in cit}
+zero = [p["id"] for p in papers if p["id"] not in cited]
+# metadata gaps (DQ-2): p["abstract"] empty / p["bibkey"] None / not p["authors"]
+# F-16 re-gate (baseline): [ch for ch in chunks if not is_quality_chunk(ch["content"], settings=s)]
+# section dist (R-12): Counter(ch["section"] or "(null)" for ch in chunks)
+```

docs/coldstart-evaluate-review-release.md

@@ -0,0 +1,59 @@
+# Cold-start session prompt — evaluate open points · review · release
+
+Paste the block below into a fresh Claude Code session (it assumes no memory of prior work).
+
+---
+
+You are picking up **codex-py** cold. The data-quality roadmap (**DQ-1–DQ-5 + R-A–R-F**)
+was merged to `main` via **PR #16** (merge commit `5aa2f7d`). Your job this session, in
+order: **(1) evaluate the remaining open points, (2) code-review the released state,
+(3) cut a release.** Read the two handoff docs first, then work the tasks. Ask before any
+destructive action (branch deletion) or before publishing the release.
+
+## Orient yourself first
+- Repo: `/Users/tarikmoussa/Desktop/codex-py`, branch `main` (at the PR #16 merge).
+- Read: `docs/SESSION-2026-06-17-open-items.md` (open items + branch-cleanup lists),
+  `docs/audit/DATA-QUALITY-2026-06-15.md` (DQ-1–DQ-5 + R-A–R-F resolution blocks),
+  `docs/infra/jetson-ssh-stability.md` (infra TODO), `docs/infra/grobid-jetson.md`.
+- Live DB: Jetson Postgres via SSH tunnel
+  `ssh -o ServerAliveInterval=15 -f -N -L 5433:localhost:5432 alfred@jetson.local`;
+  creds in `.env.jetson-ingest`. **The tunnel is chronically flaky** — prefer short,
+  idempotent ops; for long live runs use tmux **on the Jetson** (no tunnel needed there).
+- Tests: `PYTHONPATH=. .venv/bin/python -m pytest -q`. **Important:** importing `codex`
+  eagerly loads the embedding libs (~2 min) so the suite is slow, and this sandbox
+  auto-backgrounds + reaps long commands. **Launch ONE pytest run and issue no other
+  commands until it notifies you** — that lets it finish (polling its output file with a
+  read-only Read is fine; a new Bash command reaps it). Baseline after PR #16: 408 green.
+- Remote: `origin` = `git.eulernest.eu/user2595/codex-py`. Use the **gitea MCP** for
+  PRs/tags/releases (tools `mcp__gitea-eulernest__*`).
+
+## Task 1 — Evaluate open points
+- **Branch cleanup:** confirm these are fully merged into `main`, then propose deletion
+  (local + remote): `feat/tex-ingest`, `feat/crossref-references`,
+  `feat/graph-coverage-warning`, `feat/section-labels`, `chore/infra-ssh-stability`,
+  `audit/loop-1`, `integration/roadmap`; plus pre-session stale: `feat/F-15-literature-graph`,
+  `feat/F-16-chunk-quality`, `fix/audit-wave-1/2/3`, `scratch/spike-embed`.
+- **Deferred review findings** (intentionally left in PR #16 — see the `fix(review)`
+  commit): #3 (`chunk_sections` drops pre-`\section` text), #8 (script id-normalizer
+  duplication), #9 (arXiv cited-id case), #11 (gzip heuristic), #12 (title truncation).
+  Re-assess whether any is worth doing now.
+- **Infra TODO:** tmux-on-Jetson for stable long ops — documented, not implemented.
+- **DQ-6** (upstream PDF→`.txt` fidelity) — flagged, believed mooted by R-A/R-C; confirm.
+- If the tunnel is up, a quick corpus scan for any new data-quality gap.
+
+## Task 2 — Review
+- Run `/code-review` on `main` (the released state) and confirm no regressions.
+- Confirm the full suite is green (`pytest -q`, honoring the launch-and-wait rule), plus
+  `ruff check`, `ruff format --check`, `mypy codex/ scripts/`.
+
+## Task 3 — Release
+- Only after the review + suite are green.
+- Bump the version in `pyproject.toml` (currently `0.1.0`; the roadmap is a substantial
+  feature set → `0.2.0`). Commit on a branch (do not commit straight to `main`) and PR it,
+  or tag the merge commit directly if that matches the team's flow.
+- Write release notes summarizing DQ-1–DQ-5 + R-A–R-F + the audit-remediation reconcile;
+  corpus impact: **citations 590→1075, citing coverage 29/29, section signal 10%→95%**.
+- Create the tag + Gitea release via the gitea MCP; confirm it's published.
+
+Deliverable: a short report of (1) the open-point decisions, (2) the review outcome,
+(3) the published release tag/URL.

docs/infra/grobid-jetson.md

@@ -0,0 +1,192 @@
+# GROBID on the Jetson (aarch64) — deploy runbook
+
+**Status:** ACTIVE
+**Host:** `alfred@jetson.local` — NVIDIA Jetson Orin Nano (aarch64, 8 GB RAM, 8 GB swap, 6 cores)
+**Service:** GROBID reference extraction for roadmap **R-A** (`scripts/ra_grobid_backfill.py`)
+**Last updated:** 2026-06-17
+**Verified:** 2026-06-17 — `grobid/grobid:0.9.0-crf` pulled as native `arm64`, `/api/isalive`→`true`,
+end-to-end `extract_references` on `lutz-2024-thesis.pdf` → 116 refs (101 with DOI/arXiv) in ~36s.
+
+---
+
+## Why this image (`grobid/grobid:0.9.0-crf`)
+
+GROBID had never come up on the Jetson because `infra/docker-compose.yml` pinned
+`grobid/grobid:0.8.2`, and **every `0.8.x` tag (and the full `0.9.0`) is published
+amd64-only**. A Docker tag resolves to a multi-arch *manifest list*; on aarch64
+the `0.8.2` list has no `linux/arm64` entry, so there is simply nothing to pull or
+run (hence `curl localhost:8070` → connection refused, nothing listening).
+
+The fix is to select a tag whose manifest list actually contains a `linux/arm64`
+build:
+
+| Candidate                     | arm64? | Size    | Notes                                  |
+|-------------------------------|:------:|--------:|----------------------------------------|
+| `grobid/grobid:0.8.2`         | ✗      | ~9.5 GB | full DL image, amd64-only (old pin)    |
+| `grobid/grobid:0.9.0` / `-full`| ✗     | ~10 GB  | full DL image, amd64-only              |
+| **`grobid/grobid:0.9.0-crf`** | **✓**  | ~500 MB | **CRF-only, official, multi-arch** ←   |
+| `lfoppiano/grobid:0.9.0-crf`  | ✓      | ~500 MB | same build, community namespace (alt.) |
+
+We use **`grobid/grobid:0.9.0-crf`**:
+
+- **CRF-only** is sufficient for reference extraction (`/api/processReferences`)
+  and is ~500 MB vs ~10 GB for the deep-learning image.
+- **Native arm64** — no qemu emulation (too slow / RAM-heavy on a Jetson).
+- **Official namespace**, **pinned version** (not `latest-crf`, which is a moving
+  target). arm64 builds only exist from `0.9.0-crf` onward — there is no arm64
+  `0.8.x`, so this is a forced (but compatible) bump from the old `0.8.2` pin. The
+  TEI that `codex/parsing/grobid.py` parses (`biblStruct`, `persName`,
+  `idno[@type='DOI']`, …) is unchanged across the bump.
+
+> Caveat from upstream: the arm64 image is documented as "tested only on macOS,
+> not linux/arm64". The container is the *same* `linux/arm64` ELF either way, so
+> bare-metal Jetson works; the end-to-end check below is what de-risks it.
+
+---
+
+## One-time prerequisites
+
+### 1. Docker permission for `alfred`
+
+The SSH user `alfred` is in the `sudo` group but **not** the `docker` group, so
+`docker …` fails with `permission denied … /var/run/docker.sock`. Grant access
+once (requires the sudo password):
+
+```bash
+ssh alfred@jetson.local
+sudo usermod -aG docker alfred      # add to docker group (persistent)
+```
+
+Then **start a fresh login** so the new group takes effect (group membership is
+only re-evaluated at login — an existing session/`ssh` won't see it; a *new*
+`ssh` connection will):
+
+```bash
+exit && ssh alfred@jetson.local
+docker ps                            # should now work without sudo
+```
+
+(Alternatively run all `docker` commands below under `sudo`, but group membership
+is cleaner and lets `ra_grobid_backfill.py` / curl checks be driven over plain SSH.)
+
+### 2. Docker Compose v2 plugin (arm64)
+
+This host has Docker Engine but **not** the Compose v2 CLI plugin (on Linux they
+are separate packages; only Docker Desktop bundles Compose). Without it
+`docker compose …` fails with `docker: unknown command: docker compose`. Install
+the arm64 plugin into the user-local plugin dir (no sudo needed):
+
+```bash
+mkdir -p ~/.docker/cli-plugins
+curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-aarch64 \
+  -o ~/.docker/cli-plugins/docker-compose
+chmod +x ~/.docker/cli-plugins/docker-compose
+docker compose version               # confirm it resolves
+```
+
+---
+
+## Deploy
+
+GROBID is **service-scoped** on purpose: Postgres (`papers-db`) is already running
+on this host, so we bring up only the `grobid` service and never `docker compose up`
+the whole stack (that would try to start a second `papers-db`).
+
+```bash
+# copy this repo's compose file to the Jetson (first time only)
+scp infra/docker-compose.yml infra/schema.sql alfred@jetson.local:~/codex-infra/
+
+ssh alfred@jetson.local
+cd ~/codex-infra
+docker compose up -d grobid          # pulls grobid/grobid:0.9.0-crf (arm64), starts papers-grobid
+docker compose logs -f grobid        # wait for "Started ... in N seconds" (model load ~20–40s)
+```
+
+The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by
+Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves
+headroom alongside Postgres. The `restart: "no"` policy means GROBID does **not**
+auto-start on boot — see "Run on-demand" below.
+
+---
+
+## Run on-demand (stop when idle)
+
+GROBID is only needed **during reference extraction** (`scripts/ra_grobid_backfill.py`);
+nothing else in the stack uses it. Idle it still holds ~3.5 GB, so on the 8 GB Jetson
+keep it stopped and start it only for an R-A run:
+
+```bash
+ssh alfred@jetson.local 'docker start papers-grobid'        # ~30s model load
+curl -s --retry 30 --retry-delay 2 --retry-connrefused \
+  http://jetson.local:8070/api/isalive                      # wait for: true
+
+# ... run R-A from the Mac ...
+PYTHONPATH=. python scripts/ra_grobid_backfill.py --write
+
+ssh alfred@jetson.local 'docker stop papers-grobid'         # reclaim ~3.5 GB
+```
+
+First-time deploy uses `docker compose up -d grobid` (above); afterwards the
+container persists, so `docker start/stop papers-grobid` is all you need.
+
+> Why on-demand rather than a lighter tool: refextract was evaluated head-to-head
+> on the Jetson and rejected — far worse DOI recall on this math corpus (lutz
+> thesis: GROBID **101** usable IDs vs refextract **2**), and it was *slower* per
+> PDF despite a smaller footprint. GROBID stays; running it on-demand removes its
+> only real downside (idle RAM).
+
+---
+
+## Verify `GET /api/isalive` → `true`
+
+```bash
+# on the Jetson
+curl -s http://localhost:8070/api/isalive          # => true
+
+# from the Mac (LAN) — same value the corpus' GROBID_URL points at
+curl -s http://jetson.local:8070/api/isalive     # => true
+```
+
+Anything other than `true` (timeout / connection refused / `false`) means the
+container isn't up — check `docker compose logs grobid` and that nothing else
+holds port 8070.
+
+---
+
+## End-to-end reference-extraction check + running R-A
+
+With `GROBID_URL=http://jetson.local:8070` (already set in `.env.jetson-ingest`):
+
+```bash
+# one PDF, references only — should print N>0 parsed references
+PYTHONPATH=. python - <<'PY'
+from codex.parsing.grobid import extract_references
+refs = extract_references("/Users/tarikmoussa/Desktop/ConformalLabpp/papers/lutz-2024-thesis.pdf",
+                          grobid_url="http://jetson.local:8070")
+print(f"{len(refs)} references; first DOI/arXiv:",
+      next(((r['doi'], r['arxiv_id']) for r in refs if r['doi'] or r['arxiv_id']), None))
+PY
+```
+
+Then run the references-only backfill (DB tunnel up — see
+`docs/audit/DATA-QUALITY-2026-06-15.md`):
+
+```bash
+ssh -N -L 5433:localhost:5432 alfred@jetson.local &     # DB tunnel
+cp .env.jetson-ingest .env
+PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only   # dry-run first
+PYTHONPATH=. python scripts/ra_grobid_backfill.py --write            # apply
+```
+
+---
+
+## Troubleshooting
+
+- **Container killed mid-request / exit 137** — OOM. The CRF image needs ~3 GB for
+  references; raise the compose `memory` limit or check Postgres isn't starving the
+  box (`free -h`, `docker stats papers-grobid`).
+- **`isalive` true but `processReferences` 500s** — usually a `pdfalto` failure on a
+  malformed PDF; confirm with a clean PDF (`lutz-2024-thesis.pdf`) and check
+  `docker compose logs grobid`.
+- **`permission denied … docker.sock`** — the docker-group prerequisite above wasn't
+  applied, or the SSH session predates it (re-login).

docs/infra/jetson-ssh-stability.md

@@ -0,0 +1,66 @@
+# Infra TODO — stable long-running ops against the Jetson (tmux + autossh)
+
+**Status:** OPEN (infra). **Filed:** 2026-06-17. **Priority:** MED — it slows live ops
+but has a known workaround (short/idempotent ops).
+
+## Problem (observed)
+
+Live data ops run on the **Mac** and reach the Jetson Postgres through an SSH
+port-forward tunnel (`ssh -f -N -L 5433:localhost:5432 alfred@jetson.local`,
+`DATABASE_URL` → `localhost:5433`). During the R-A / R-C / R-F work this tunnel
+**dropped repeatedly mid-run**: any operation longer than a few minutes (corpus
+re-ingest, the bge-m3 embed loop, the GROBID sweep) had a high chance of failing
+partway when the tunnel died, leaving the work half-applied. At one point the Jetson
+went **fully offline** (ping 100 % loss, `ssh:22` timeout) mid-operation.
+
+`ingest_paper()` and the backfill scripts open one DB connection for the whole run, so
+a tunnel drop raises `OperationalError: connection refused` and aborts — partial state,
+manual retry. Adding `ServerAliveInterval` to the tunnel helped only marginally.
+
+## Recommended fix
+
+**1. Run long ops *on the Jetson* inside `tmux` (primary fix).**
+The Jetson is where Postgres (and GROBID) live, so running ingest there means the DB is
+`localhost` — **no tunnel at all** — and `tmux` keeps the process alive across SSH
+disconnects (detach / reattach to monitor). This removes the tunnel from the critical
+path for every long-running write.
+
+- [ ] Confirm the `codex` env is deployable on the Jetson (Python 3.12+, deps incl. the
+      bge-m3 embedder; the Orin Nano GPU should handle it — GROBID already runs there).
+      If not, stand up a venv / container for it.
+- [ ] Standard pattern: `ssh alfred@jetson`, then
+      `tmux new -s codex` → `set -a; source .env; set +a` →
+      `PYTHONPATH=. python scripts/<job>.py --write` → detach (`Ctrl-b d`).
+      Reattach later with `tmux attach -t codex`. The job survives the SSH drop.
+- [ ] Point the Jetson-side `DATABASE_URL` at `localhost:5432` directly (no `:5433`).
+- [ ] Document this as the default for any multi-minute write (re-ingest, sweeps,
+      embed loops) in `docs/audit/DATA-QUALITY-2026-06-15.md` "How to reach the data".
+
+**2. `autossh` for when a tunnel *is* needed (secondary).**
+For interactive/dev use from the Mac (psql, quick probes, read-only queries), replace
+the plain `ssh -f -N -L` with `autossh -M 0 -f -N -o ServerAliveInterval=15 -o
+ServerAliveCountMax=3 -L 5433:localhost:5432 alfred@jetson` so a dropped tunnel
+auto-reconnects. NB: autossh only restarts the *tunnel* — an in-flight transaction
+still fails, so this is for short/interactive use, not long writes (use tmux-on-Jetson
+for those).
+
+**3. (Optional, code) make backfill scripts resilient.**
+Lower-value once (1)/(2) are in place, but: open a fresh `get_conn()` per paper (not one
+for the whole run) and/or wrap the per-item write in a small retry, so a transient drop
+costs one item, not the whole batch. The R-A backfill already does per-paper connections;
+`rc_tex_reingest.py` opens a connection per paper too — but `ingest_paper` itself holds
+one connection for its whole body, which is the unit that fails.
+
+## Why this matters
+
+The chronic flakiness directly cost time this session and left one cosmetic task
+unfinished (the R-F accent re-clean — see DATA-QUALITY-2026-06-15.md R-F "Pending"). The
+**robust pattern that already works** is short, idempotent operations (per-paper, or a
+single quick `UPDATE`); tmux-on-Jetson makes even the long ops safe.
+
+## Acceptance
+
+- A documented, repeatable way to run a full corpus re-ingest that survives an SSH drop
+  (tmux-on-Jetson), verified by detaching/reattaching across a disconnect.
+- The DATA-QUALITY doc's "How to reach the data" section recommends tmux-on-Jetson for
+  long writes and `autossh` for interactive tunnels.

infra/docker-compose.yml

@@ -28,20 +28,34 @@ services:
     restart: unless-stopped
 
   grobid:
-    # CRF-only image is sufficient for reference extraction and is lighter than
+    # CRF-only image: sufficient for reference extraction (/api/processReferences)
-    # the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid
+    # and ~500MB vs ~10GB for the full deep-learning variant.
-    # for the latest 0.8.x tag before deploying.
+    #
-    image: grobid/grobid:0.8.2
+    # The `-crf` tag is also the ONLY GROBID variant published as a multi-arch
+    # manifest that includes linux/arm64 — the full image (grobid/grobid:0.9.0)
+    # and every 0.8.x tag are amd64-only. So this is what lets GROBID run
+    # *natively* on the aarch64 Jetson (no slow/RAM-heavy qemu emulation). arm64
+    # builds start at 0.9.0-crf; there is no arm64 0.8.x.
+    # Deploy + docker-permission fix: docs/infra/grobid-jetson.md
+    image: grobid/grobid:0.9.0-crf
     container_name: papers-grobid
     ports:
       - "8070:8070"
-    restart: unless-stopped
+    init: true            # reap zombie children (GROBID docs recommend --init)
-    # GROBID is RAM-hungry; uncomment to cap usage on constrained hardware
+    ulimits:
-    # (e.g. Nvidia Jetson):
+      core: 0             # disable core dumps (GROBID docs: --ulimit core=0)
-    # deploy:
+    # On-demand only: GROBID is needed solely during reference extraction
-    #   resources:
+    # (scripts/ra_grobid_backfill.py), not for normal corpus operations — start it
-    #     limits:
+    # for an R-A run, stop it after to reclaim ~3.5GB. "no" = no auto-start on boot.
-    #       memory: 4g
+    # See docs/infra/grobid-jetson.md ("Run on-demand").
+    restart: "no"
+    # GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB
+    # RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom.
+    # Honored by `docker compose up` under Compose v2 (no swarm needed).
+    deploy:
+      resources:
+        limits:
+          memory: 4g
 
 volumes:
   pgdata:

infra/reingest_canonical_ids.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+# One-time migration to canonical paper IDs (audit C-7).
+#
+# Before the C-7 fix, ingest stored OpenAlex's URL-form doi/id as papers.id
+# (e.g. "https://doi.org/10.48550/arxiv.math/0603097"). After the fix, ingest
+# keys papers.id on the bare caller id ("math/0603097"). The existing rows keep
+# their old URL-form primary keys, so a plain re-ingest would DUPLICATE every
+# paper (new bare-id row alongside the old URL-id row). This script wipes the
+# corpus and rebuilds it on canonical ids.
+#
+# DESTRUCTIVE: `TRUNCATE papers CASCADE` removes papers + chunks + citations +
+# formulas + figures + paper_identifiers (code_links.paper_id is set NULL). All
+# content is rebuilt by ingest_all.sh, which re-fetches OpenAlex and re-embeds
+# (minutes for ~36 papers).
+#
+# Prerequisites: SSH tunnel on :5433 (see ingest_all.sh) and .env.jetson-ingest.
+# Usage: bash infra/reingest_canonical_ids.sh
+set -euo pipefail
+
+CODEX_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+ENV_FILE="$CODEX_DIR/.env.jetson-ingest"
+PY="$CODEX_DIR/.venv/bin/python"
+TUNNEL_PORT=5433
+
+# ── 1. Preconditions ─────────────────────────────────────────────────────────
+if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then
+  echo "ERROR: SSH tunnel not active on :$TUNNEL_PORT"
+  echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@jetson.local"
+  exit 1
+fi
+[[ -f "$ENV_FILE" ]] || { echo "ERROR: $ENV_FILE missing"; exit 1; }
+[[ -x "$PY" ]] || { echo "ERROR: venv python not found at $PY (run uv sync)"; exit 1; }
+# Load DATABASE_URL into the environment (not echoed — keeps the password out of logs).
+set -a; source "$ENV_FILE"; set +a
+
+# run_sql <SQL>: execute one statement via psycopg, print any result rows.
+# Reads DATABASE_URL from the environment; suppresses the DSN on error.
+run_sql() {
+  "$PY" - "$1" <<'PYEOF'
+import os, sys, psycopg
+from psycopg.rows import dict_row
+try:
+    with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c:
+        cur = c.execute(sys.argv[1])
+        if cur.description:
+            for row in cur.fetchall():
+                print("   " + "  ".join(f"{k}={v!r}" for k, v in row.items()))
+        c.commit()
+except Exception as e:
+    print(f"   DB ERROR: {type(e).__name__} (details suppressed to avoid DSN leak)")
+    sys.exit(1)
+PYEOF
+}
+
+# ── 2. BEFORE snapshot ───────────────────────────────────────────────────────
+echo "── BEFORE migration ──────────────────────────────────────────"
+run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers"
+echo "   sample ids:"
+run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3"
+
+# ── 2b. Preflight: verify the schema is re-ingestable BEFORE wiping anything ──
+# ingest writes chunks.section (F-16). On a live DB that predates it the column
+# is missing, and the app user cannot ALTER tables owned by 'postgres' (audit
+# M-1). Check FIRST so we never TRUNCATE and then fail mid-re-ingest.
+echo "── Preflight: schema readiness ───────────────────────────────"
+SECTION_OK="$("$PY" - <<'PYEOF'
+import os, psycopg
+try:
+    with psycopg.connect(os.environ["DATABASE_URL"], connect_timeout=10) as c:
+        cols = [r[0] for r in c.execute(
+            "SELECT column_name FROM information_schema.columns WHERE table_name='chunks'").fetchall()]
+        print("yes" if "section" in cols else "no")
+except Exception:
+    print("error")
+PYEOF
+)"
+if [[ "$SECTION_OK" != "yes" ]]; then
+  echo "ABORT — schema not ready: chunks.section is missing (audit M-1)."
+  echo "The app user cannot add it (core tables are owned by 'postgres'). Sync the"
+  echo "schema with a privileged role, then re-run this script. Either:"
+  echo ""
+  echo "  (a) MIGRATION_DATABASE_URL=postgresql://postgres:PASS@HOST:PORT/papers \\"
+  echo "        \"$CODEX_DIR/.venv/bin/codex\" migrate     # applies schema.sql idempotently"
+  echo "  (b) sudo -u postgres psql -d papers \\"
+  echo "        -c \"ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;\""
+  echo ""
+  echo "Nothing was changed — no TRUNCATE was issued."
+  exit 1
+fi
+echo "   chunks.section present — schema OK."
+
+# ── 3. Confirmation gate (destructive) ───────────────────────────────────────
+echo ""
+echo "This TRUNCATEs papers CASCADE (papers/chunks/citations/formulas/figures/"
+echo "paper_identifiers) and re-ingests via ingest_all.sh. The DB content is"
+echo "rebuilt from scratch."
+read -r -p "Type 'MIGRATE' to proceed: " confirm
+[[ "$confirm" == "MIGRATE" ]] || { echo "Aborted — nothing changed."; exit 1; }
+
+# ── 4. Wipe ──────────────────────────────────────────────────────────────────
+echo "── TRUNCATE papers CASCADE ───────────────────────────────────"
+run_sql "TRUNCATE papers CASCADE"
+echo "   wiped."
+
+# ── 5. Re-ingest on canonical ids ────────────────────────────────────────────
+echo "── Re-ingest (ingest_all.sh) ─────────────────────────────────"
+bash "$CODEX_DIR/ingest_all.sh"
+
+# ── 6. AFTER snapshot + live verification ────────────────────────────────────
+echo "── AFTER migration ───────────────────────────────────────────"
+echo "   C-7 — url_form_ids should now be 0; sample ids should be bare:"
+run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers"
+run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3"
+
+# C-1 must be verified through the REAL resolver-based discovery_leads(), not a
+# raw cited_id check: cited_id/openalex_id stay in OpenAlex form, so the raw
+# "cited_id NOT IN papers.id" count is non-zero by design — the resolver is what
+# excludes ingested papers. Check that no ingested paper leaks into the leads.
+echo "   C-1 — real discovery_leads() must contain no already-ingested paper:"
+PYTHONPATH="$CODEX_DIR" "$PY" - <<'PYEOF'
+import os, psycopg
+from psycopg.rows import dict_row
+from codex.discover import discovery_leads
+try:
+    leads = discovery_leads(limit=100000)
+    with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c:
+        ingested = {r["id"] for r in c.execute("SELECT id FROM papers").fetchall()}
+        ingested |= {
+            r["openalex_id"]
+            for r in c.execute("SELECT openalex_id FROM papers WHERE openalex_id IS NOT NULL").fetchall()
+        }
+    leaked = sum(1 for lead in leads if lead["cited_id"] in ingested)
+    print(f"   leads={len(leads)}  ingested-papers-leaked-into-leads={leaked}")
+except Exception as e:
+    print(f"   DB ERROR: {type(e).__name__} (details suppressed)")
+PYEOF
+
+echo ""
+echo "✓ Migration complete."
+echo "  Expected: url_form_ids=0 (C-7 verified) and leaked=0 (C-1 verified) above."
+echo "  Next: re-run 'codex graph report' and refresh the ADR-F15 spike table (audit D-1)."

infra/schema.sql

@@ -13,7 +13,7 @@ CREATE EXTENSION IF NOT EXISTS vector;
 -- ---------------------------------------------------------------------
 -- Layer 1: Papers + full-text chunks
 -- ---------------------------------------------------------------------
-CREATE TABLE papers (
+CREATE TABLE IF NOT EXISTS papers (
     id            TEXT PRIMARY KEY,        -- canonical ID: arXiv ID or DOI
     openalex_id   TEXT UNIQUE,             -- e.g. W2741809807
     bibkey        TEXT UNIQUE,             -- BibTeX key -> .bib + Doxygen @cite
@@ -27,10 +27,10 @@ CREATE TABLE papers (
 );
 
 -- HNSW index for fast approximate nearest-neighbour search on abstracts.
-CREATE INDEX papers_abstract_emb_idx
+CREATE INDEX IF NOT EXISTS papers_abstract_emb_idx
     ON papers USING hnsw (abstract_emb vector_cosine_ops);
 
-CREATE TABLE chunks (
+CREATE TABLE IF NOT EXISTS chunks (
     id        BIGSERIAL PRIMARY KEY,
     paper_id  TEXT REFERENCES papers(id) ON DELETE CASCADE,
     ord       INT NOT NULL,               -- position within the paper
@@ -38,13 +38,13 @@ CREATE TABLE chunks (
     embedding vector(1024)
 );
 
-CREATE INDEX chunks_emb_idx
+CREATE INDEX IF NOT EXISTS chunks_emb_idx
     ON chunks USING hnsw (embedding vector_cosine_ops);
-CREATE INDEX chunks_paper_idx ON chunks (paper_id);
+CREATE INDEX IF NOT EXISTS chunks_paper_idx ON chunks (paper_id);
 
 -- Sparse / keyword hits for exact mathematical terminology (hybrid search).
 -- Dense (above) + full-text (below) combined = robust against math terms.
-CREATE INDEX chunks_fts_idx
+CREATE INDEX IF NOT EXISTS chunks_fts_idx
     ON chunks USING gin (to_tsvector('english', content));
 
 -- ---------------------------------------------------------------------
@@ -53,19 +53,19 @@ CREATE INDEX chunks_fts_idx
 --   edges to not-yet-ingested papers are intentionally preserved —
 --   those are your discovery leads.
 -- ---------------------------------------------------------------------
-CREATE TABLE citations (
+CREATE TABLE IF NOT EXISTS citations (
     citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
     cited_id  TEXT NOT NULL,               -- arXiv/DOI/OpenAlex ID of the target
     context   TEXT,                        -- optional: citation context (S2)
     PRIMARY KEY (citing_id, cited_id)
 );
 
-CREATE INDEX citations_cited_idx ON citations (cited_id);
+CREATE INDEX IF NOT EXISTS citations_cited_idx ON citations (cited_id);
 
 -- ---------------------------------------------------------------------
 -- Layer 3: Provenance  (the "cleanly couple" goal)
 -- ---------------------------------------------------------------------
-CREATE TABLE code_links (
+CREATE TABLE IF NOT EXISTS code_links (
     id        BIGSERIAL PRIMARY KEY,
     symbol    TEXT NOT NULL,               -- e.g. 'dec::hodge_star' or 'file.cpp:120'
     paper_id  TEXT REFERENCES papers(id) ON DELETE SET NULL,
@@ -74,8 +74,8 @@ CREATE TABLE code_links (
     added_at  TIMESTAMPTZ DEFAULT now()
 );
 
-CREATE INDEX code_links_symbol_idx ON code_links (symbol);
+CREATE INDEX IF NOT EXISTS code_links_symbol_idx ON code_links (symbol);
-CREATE INDEX code_links_paper_idx  ON code_links (paper_id);
+CREATE INDEX IF NOT EXISTS code_links_paper_idx  ON code_links (paper_id);
 
 -- ---------------------------------------------------------------------
 -- Example query: discovery leads
@@ -122,3 +122,24 @@ CREATE TABLE IF NOT EXISTS figures (
 CREATE INDEX IF NOT EXISTS figures_emb_idx
     ON figures USING hnsw (embedding vector_cosine_ops);
 CREATE INDEX IF NOT EXISTS figures_paper_idx ON figures (paper_id);
+
+-- F-16 Chunk Quality Gate: section classification
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;
+
+-- F-17 Paper identifier aliases: one paper → many OpenAlex IDs
+-- (arXiv preprint ID + journal published ID + any future version)
+-- Each OpenAlex ID maps to exactly one paper (UNIQUE on openalex_id).
+-- Populated automatically by ingest; add extra aliases manually.
+CREATE TABLE IF NOT EXISTS paper_identifiers (
+    paper_id    TEXT NOT NULL REFERENCES papers(id) ON DELETE CASCADE,
+    openalex_id TEXT NOT NULL,
+    PRIMARY KEY (paper_id, openalex_id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS paper_identifiers_oa_idx
+    ON paper_identifiers (openalex_id);
+
+-- Seed from existing papers.openalex_id
+INSERT INTO paper_identifiers (paper_id, openalex_id)
+SELECT id, openalex_id FROM papers WHERE openalex_id IS NOT NULL
+ON CONFLICT DO NOTHING;

ingest_all.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# Batch ingest — writes directly to Jetson PostgreSQL via SSH tunnel.
+#
+# Prerequisites:
+#   ssh -f -N -L 5433:localhost:5432 alfred@jetson.local
+#   uv sync  (done once)
+#
+# Usage: bash ingest_all.sh 2>&1 | tee ingest_all.log
+set -euo pipefail
+
+PAPERS_DIR="/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt"
+CODEX_DIR="$(cd "$(dirname "$0")" && pwd)"
+ENV_FILE="$CODEX_DIR/.env.jetson-ingest"
+TUNNEL_PORT=5433
+
+# ── 1. Verify SSH tunnel is active ───────────────────────────────────────────
+if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then
+  echo "ERROR: SSH tunnel not active on port $TUNNEL_PORT"
+  echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@jetson.local"
+  exit 1
+fi
+echo "✓ SSH tunnel active on port $TUNNEL_PORT"
+
+# ── 1b. Clear macOS UF_HIDDEN flag on editable-install .pth files ─────────────
+# Python 3.13 skips .pth files with this flag set (new in 3.13); uv sets it.
+# --no-sync below prevents uv from re-syncing (which would re-set the flag).
+SP="$CODEX_DIR/.venv/lib/python3.13/site-packages"
+for pth in "$SP"/_editable_impl_codex*.pth; do
+  [[ -f "$pth" ]] && chflags nohidden "$pth" 2>/dev/null || true
+done
+echo "✓ Cleared hidden flags on editable .pth files"
+
+# ── 2. Load env (DATABASE_URL → Jetson via tunnel) ───────────────────────────
+# pydantic-settings reads DATABASE_URL directly from the environment.
+# DOTENV_PATH is not supported; export each var explicitly.
+set -a; source "$ENV_FILE"; set +a
+export DATABASE_URL GROBID_URL OLLAMA_BASE_URL EMBEDDING_MODEL EMBEDDING_DIM OPENALEX_MAILTO
+echo "✓ DATABASE_URL: $DATABASE_URL"
+
+# ── 3. Paper ID → source file mapping ────────────────────────────────────────
+# Format: "PAPER_ID|filename.txt"
+# SKIP: farkas-kra-1992-riemann-surfaces.txt (textbook, no OpenAlex entry)
+declare -a PAPERS=(
+  # arXiv papers
+  "math/0603097|springborn-2008-weighted-delaunay-hyperideal.txt"
+  "1005.2698|bobenko-pinkall-springborn-2015-discrete-conformal-maps.txt"
+  "math/0306167|luo-2004-combinatorial-yamabe-flow.txt"
+  "math/0203250|bobenko-springborn-2004-circle-patterns.txt"
+  "math/0503219|bobenko-springborn-2007-discrete-laplace-beltrami.txt"
+  "2310.17529|bobenko-lutz-2025-non-euclidean-dce.txt"
+  "2305.10988|bobenko-lutz-2024-decorated-conformal-maps.txt"
+  "2206.13461|lutz-2023-canonical-tessellations.txt"
+  "2601.22903|bowers-bowers-lutz-2026-koebe-rigidity.txt"
+  "1911.00966|pinkall-springborn-2021-liouville.txt"
+  "1505.01341|born-bucking-springborn-2015-quasiconformal.txt"
+  "0906.1560|glickenstein-2011-discrete-conformal-variations.txt"
+  "math/0001176|rivin-schlenker-2000-schlafli-formula-arxiv-preprint.txt"
+  # DOI papers
+  "10.1007/s00454-019-00132-8|springborn-2020-ideal-hyperbolic-polyhedra.txt"
+  "10.1145/1964921.1964997|alexa-wardetzky-2011-discrete-laplacians-polygonal.txt"
+  "10.1111/cgf.13931|bunge-herholz-kazhdan-botsch-2020-polygon-laplacian.txt"
+  "10.1145/3450626.3459763|gillespie-springborn-crane-2021-discrete-conformal-equivalence.txt"
+  "10.1145/3306346.3323042|sharp-soliman-crane-2019-navigating-intrinsic-triangulations.txt"
+  "10.1145/3197517.3201367|soliman-slepcv-crane-2018-optimal-cone-singularities.txt"
+  "10.1145/3132705|sawhney-crane-2017-boundary-first-flattening.txt"
+  "10.1145/2767000|knoppel-crane-pinkall-schroder-2015-stripe-patterns.txt"
+  "10.5555/1070432.1070581|erickson-whittlesey-2005-greedy-homotopy-homology.txt"
+  "10.1145/1185657.1185665|desbrun-kanso-tong-2006-discrete-differential-forms.txt"
+  "10.1080/10586458.1993.10504266|pinkall-polthier-1993-computing-discrete-minimal-surfaces.txt"
+  "10.1007/s11040-021-09394-2|bobenko-bucking-2021-period-matrices.txt"
+  "10.14279/depositonce-20357|lutz-2024-thesis.txt"
+  "10.14279/depositonce-5415|sechelmann-2016-thesis.txt"
+  # Book chapters (txt contains full book; ID = cited chapter)
+  "10.1007/0-387-29555-0_13|precopa-molnar-eds-2006-non-euclidean-geometries-book.txt"
+  "10.1007/978-3-642-17413-1_7|bobenko-klein-eds-2011-computational-approach-riemann-surfaces-book.txt"
+)
+
+# ── 4. Ingest loop ────────────────────────────────────────────────────────────
+OK=0; FAIL=0; SKIP=0
+FAILED_IDS=()
+
+cd "$CODEX_DIR"
+
+for entry in "${PAPERS[@]}"; do
+  PAPER_ID="${entry%%|*}"
+  FILENAME="${entry##*|}"
+  SOURCE="$PAPERS_DIR/$FILENAME"
+
+  if [[ ! -f "$SOURCE" ]]; then
+    echo "SKIP (file missing): $FILENAME"
+    (( SKIP++ )) || true
+    continue
+  fi
+
+  echo ""
+  echo "── Ingesting: $PAPER_ID"
+  echo "   source: $FILENAME"
+
+  if PYTHONPATH="$CODEX_DIR" "$CODEX_DIR/.venv/bin/codex" ingest "$PAPER_ID" --source "$SOURCE"; then
+    (( OK++ )) || true
+  else
+    echo "FAILED: $PAPER_ID"
+    FAILED_IDS+=("$PAPER_ID")
+    (( FAIL++ )) || true
+  fi
+done
+
+# ── 5. Summary ────────────────────────────────────────────────────────────────
+echo ""
+echo "════════════════════════════════════════"
+echo "Ingest complete: $OK OK, $FAIL FAILED, $SKIP SKIPPED"
+if [[ ${#FAILED_IDS[@]} -gt 0 ]]; then
+  echo "Failed IDs:"
+  for id in "${FAILED_IDS[@]}"; do echo "  $id"; done
+fi
+echo "════════════════════════════════════════"

scripts/__init__.py

@@ -0,0 +1 @@
+"""Operational one-off scripts (importable for tests)."""

scripts/ra_grobid_backfill.py

@@ -0,0 +1,227 @@
+"""R-A — GROBID reference backfill (references-only, idempotent).
+
+Adds GROBID-parsed citations to the live corpus **without** re-chunking the body,
+so the DQ-3-verified-clean ``.txt`` chunks are preserved. Complements the DQ-1
+Semantic-Scholar supplement with references the APIs lack (notably the two
+TU-Berlin ``depositonce`` theses, which neither OpenAlex nor S2 indexes).
+
+Why references-only instead of ``ingest_paper(source_path=pdf)``
+----------------------------------------------------------------
+The PDF branch of :func:`codex.ingest.ingest_paper` runs Nougat OCR and then
+``DELETE FROM chunks`` + re-INSERT — it would replace the clean ``.txt`` chunks
+(DQ-3) with unmeasured PDF-OCR text and additionally requires the Nougat server.
+This backfill touches only the ``citations`` table (``ON CONFLICT DO NOTHING``),
+so it is safe, idempotent, and needs only a reachable GROBID server.
+
+Prerequisite
+------------
+A reachable GROBID server. The corpus' ``GROBID_URL`` points at the Jetson
+(``http://jetson.local:8070``). GROBID now runs natively on that aarch64 host
+via the multi-arch CRF-only image (``grobid/grobid:0.9.0-crf``) — see
+``docs/infra/grobid-jetson.md`` for the deploy + docker-permission fix and the
+``GET /api/isalive`` check. (The old ``grobid/grobid:0.8.2`` pin was amd64-only,
+which is why GROBID never came up; pass ``--grobid-url`` to override the default.)
+Use the SSH DB tunnel for ``DATABASE_URL`` as documented in
+``docs/audit/DATA-QUALITY-2026-06-15.md``.
+
+Usage
+-----
+    # dry run (no writes) over the two zero-edge theses
+    PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only
+
+    # write every paper's GROBID-parsed refs to the live DB
+    PYTHONPATH=. python scripts/ra_grobid_backfill.py --write
+
+Default is **dry-run**; pass ``--write`` to mutate the live DB.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+from pathlib import Path
+
+import psycopg
+import psycopg.rows
+
+from codex.config import get_settings
+from codex.db import get_conn
+from codex.models import Citation
+from codex.parsing.grobid import extract_references
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PDF_DIR = "/Users/tarikmoussa/Desktop/ConformalLabpp/papers"
+
+
+def _normalize_cited_id(doi: str = "", arxiv_id: str = "") -> str | None:
+    """Normalize a GROBID-extracted reference id to the corpus' canonical form.
+
+    DOIs → bare, lower-cased (strip ``https://doi.org/`` / ``http://`` / ``doi:``),
+    matching ``papers.id`` and ``codex.sources.openalex._normalize_doi`` (DQ-5).
+    arXiv ids → strip a leading ``arXiv:`` prefix, leaving the bare id
+    (``2305.10988``, ``math/0603097``). DOI takes precedence when both exist.
+    Kept self-contained rather than reusing ``ingest._norm_cited_id`` because it
+    additionally strips the ``arXiv:`` prefix (which GROBID can emit but that
+    helper leaves untouched), and to avoid importing a private cross-module name.
+    """
+    doi = (doi or "").strip()
+    if doi:
+        s = doi.lower()
+        for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
+            if s.startswith(prefix):
+                return s[len(prefix) :]
+        return s
+    arx = (arxiv_id or "").strip()
+    if arx:
+        return arx[len("arxiv:") :].strip() if arx.lower().startswith("arxiv:") else arx
+    return None
+
+
+def build_grobid_citations(
+    paper_id: str,
+    pdf_path: str | Path,
+    grobid_url: str | None = None,
+    timeout: float = 60.0,
+) -> list[Citation]:
+    """Extract a paper's references via GROBID and return normalized Citations.
+
+    ``citing_id`` is the canonical ``paper_id``; ``cited_id`` is normalized.
+    Self-citations and within-paper duplicates are dropped. Refs with neither a
+    DOI nor an arXiv id are skipped (they cannot join the graph).
+    """
+    refs = extract_references(str(pdf_path), grobid_url=grobid_url, timeout=timeout)
+    seen: set[str] = set()
+    citations: list[Citation] = []
+    for ref in refs:
+        cited = _normalize_cited_id(ref.get("doi", ""), ref.get("arxiv_id", ""))
+        if not cited or cited == paper_id or cited in seen:
+            continue
+        seen.add(cited)
+        citations.append(Citation(citing_id=paper_id, cited_id=cited))
+    return citations
+
+
+def _coverage(conn: psycopg.Connection[psycopg.rows.DictRow]) -> tuple[int, int, int]:
+    """Return (n_papers, n_papers_with_out_edges, n_citations) from the live DB."""
+    row = conn.execute(
+        """
+        SELECT (SELECT count(*) FROM papers)                    AS papers,
+               (SELECT count(DISTINCT citing_id) FROM citations) AS citing,
+               (SELECT count(*) FROM citations)                  AS edges
+        """
+    ).fetchone()
+    assert row is not None
+    return int(row["papers"]), int(row["citing"]), int(row["edges"])
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--pdf-dir", default=DEFAULT_PDF_DIR, help="directory of source PDFs")
+    parser.add_argument("--grobid-url", default=None, help="override Settings().grobid_url")
+    parser.add_argument("--only", default=None, help="restrict to a single paper id")
+    parser.add_argument(
+        "--zero-edge-only",
+        action="store_true",
+        help="only papers with 0 out-edges (the R-A primary targets)",
+    )
+    parser.add_argument("--limit", type=int, default=None, help="cap number of papers")
+    parser.add_argument(
+        "--timeout",
+        type=float,
+        default=300.0,
+        help="GROBID HTTP timeout (s); theses can need >60s, esp. emulated",
+    )
+    parser.add_argument(
+        "--write", action="store_true", help="WRITE to the live DB (default: dry-run)"
+    )
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    grobid_url = args.grobid_url or get_settings().grobid_url
+    pdf_dir = Path(args.pdf_dir)
+    pdfs = {p.stem: p for p in pdf_dir.glob("*.pdf")}
+
+    # Read the target list in a short-lived connection, then release it: the GROBID
+    # extraction below is a slow per-paper network loop and must NOT hold an
+    # idle-in-transaction connection open across it (a mid-loop SSH-tunnel drop would
+    # otherwise abort the whole batch — see docs/infra/jetson-ssh-stability.md).
+    with get_conn() as conn:
+        rows = conn.execute(
+            """
+            SELECT p.id, p.source_path,
+                   (SELECT count(*) FROM citations ci WHERE ci.citing_id = p.id) AS out_edges
+            FROM papers p
+            ORDER BY out_edges, p.id
+            """
+        ).fetchall()
+
+    targets: list[tuple[str, Path]] = []
+    for r in rows:
+        if args.only and r["id"] != args.only:
+            continue
+        if args.zero_edge_only and r["out_edges"] != 0:
+            continue
+        stem = Path(r["source_path"]).stem if r["source_path"] else None
+        pdf = pdfs.get(stem) if stem else None
+        if pdf is None:
+            logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem)
+            continue
+        targets.append((r["id"], pdf))
+    if args.limit is not None:
+        targets = targets[: args.limit]
+
+    logger.info(
+        "GROBID=%s | papers to process=%d | mode=%s",
+        grobid_url,
+        len(targets),
+        "WRITE" if args.write else "DRY-RUN",
+    )
+
+    # GROBID extraction — no DB connection held during this slow network loop.
+    all_citations: list[Citation] = []
+    for paper_id, pdf in targets:
+        try:
+            cits = build_grobid_citations(
+                paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout
+            )
+        except Exception as exc:
+            logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc)
+            continue
+        logger.info("%-38s %3d refs  (%s)", paper_id, len(cits), pdf.name)
+        all_citations.extend(cits)
+
+    logger.info("total candidate citations: %d", len(all_citations))
+
+    if not args.write:
+        logger.info("DRY-RUN — no rows written. Re-run with --write to apply.")
+        return 0
+
+    # Fresh connection for the write — the read connection was already released.
+    with get_conn() as conn:
+        before = _coverage(conn)
+        with conn.cursor() as cur:
+            cur.executemany(
+                """
+                INSERT INTO citations (citing_id, cited_id, context)
+                VALUES (%s, %s, %s)
+                ON CONFLICT (citing_id, cited_id) DO NOTHING
+                """,
+                [(c.citing_id, c.cited_id, c.context) for c in all_citations],
+            )
+        conn.commit()
+        after = _coverage(conn)
+    logger.info(
+        "coverage papers=%d  citing %d->%d  edges %d->%d (+%d)",
+        after[0],
+        before[1],
+        after[1],
+        before[2],
+        after[2],
+        after[2] - before[2],
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/rc_tex_reingest.py

@@ -0,0 +1,131 @@
+"""R-C — re-ingest arXiv papers from LaTeX source (section-aware, higher fidelity).
+
+For each arXiv paper, download its source via :func:`codex.sources.arxiv.fetch_source`
+(multi-file ``\\input``/``\\include`` flattened), write it to a temp ``.tex``, and
+re-ingest through :func:`codex.ingest.ingest_paper`. The ``.tex`` ingest path is
+section-aware (:func:`codex.parsing.tex.chunk_sections`), so the stored ``section``
+column is labelled from real ``\\section`` headings instead of mostly ``body`` —
+serving DQ-3 (fidelity vs the OCR'd ``.txt``) and audit R-12 (weak ``section`` column).
+
+Only arXiv papers have downloadable source; DOI/journal papers and books are skipped.
+Re-ingest REPLACES the paper's chunks (DELETE + re-INSERT) — that is the intended R-C
+improvement (flattened ``.txt`` → clean ``.tex`` chunks). It is idempotent and
+**dry-run by default**; pass ``--write`` to mutate the live corpus.
+
+Usage
+-----
+    # dry-run: list arXiv papers + source availability (no writes)
+    PYTHONPATH=. python scripts/rc_tex_reingest.py
+
+    # re-ingest every arXiv paper from .tex
+    PYTHONPATH=. python scripts/rc_tex_reingest.py --write
+
+    # one paper
+    PYTHONPATH=. python scripts/rc_tex_reingest.py --only 2305.10988 --write
+
+Needs ``DATABASE_URL`` (Jetson tunnel) as documented in
+``docs/audit/DATA-QUALITY-2026-06-15.md``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import tempfile
+
+import psycopg
+import psycopg.rows
+
+from codex.db import get_conn
+from codex.ingest import ingest_paper
+from codex.sources import arxiv
+
+logger = logging.getLogger(__name__)
+
+
+def is_arxiv_id(paper_id: str) -> bool:
+    """Return True for arXiv ids (the only papers with downloadable LaTeX source).
+
+    arXiv ids are modern (``2305.10988``) or legacy (``math/0603097``). DOIs start
+    with ``10.``; OpenAlex W-ids start with ``W`` or an ``https://openalex.org/`` URL.
+    """
+    p = (paper_id or "").strip()
+    # Exclude DOIs (incl. the arXiv DOI 10.48550/arXiv.*), OpenAlex W-ids, and any
+    # http(s) URL form; everything else is treated as a bare arXiv id.
+    return (
+        bool(p)
+        and not p.startswith(("10.", "W"))
+        and not p.lower().startswith(("http://", "https://"))
+    )
+
+
+def _section_dist(conn: psycopg.Connection[psycopg.rows.DictRow], paper_id: str) -> dict[str, int]:
+    """Return the section-label distribution for a paper's stored chunks."""
+    rows = conn.execute(
+        "SELECT coalesce(section, '(null)') AS s, count(*) AS n "
+        "FROM chunks WHERE paper_id = %s GROUP BY 1 ORDER BY 2 DESC",
+        (paper_id,),
+    ).fetchall()
+    return {r["s"]: int(r["n"]) for r in rows}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--only", default=None, help="restrict to a single arXiv id")
+    parser.add_argument("--limit", type=int, default=None, help="cap number of papers")
+    parser.add_argument(
+        "--write", action="store_true", help="re-ingest into the live DB (default: dry-run)"
+    )
+    args = parser.parse_args(argv)
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+
+    with get_conn() as conn:
+        rows = conn.execute("SELECT id FROM papers ORDER BY id").fetchall()
+    targets = [
+        r["id"]
+        for r in rows
+        if is_arxiv_id(r["id"]) and (args.only is None or r["id"] == args.only)
+    ]
+    if args.limit is not None:
+        targets = targets[: args.limit]
+
+    logger.info(
+        "arXiv papers to process=%d | mode=%s",
+        len(targets),
+        "WRITE" if args.write else "DRY-RUN",
+    )
+
+    for pid in targets:
+        try:
+            src = arxiv.fetch_source(pid)
+        except Exception as exc:
+            logger.warning("%s: source fetch failed: %s", pid, exc)
+            continue
+        if not src:
+            logger.warning("%s: no LaTeX source available — skipping", pid)
+            continue
+        if not args.write:
+            logger.info("%s: source OK (%d chars)", pid, len(src))
+            continue
+
+        with get_conn() as conn:
+            before = _section_dist(conn, pid)
+        with tempfile.NamedTemporaryFile("w", suffix=".tex", delete=True) as tf:
+            tf.write(src)
+            tf.flush()
+            res = ingest_paper(pid, source_path=tf.name)
+        with get_conn() as conn:
+            after = _section_dist(conn, pid)
+        logger.info(
+            "%s: chunks %d->%d | section before=%s after=%s",
+            pid,
+            sum(before.values()),
+            res.chunks_upserted,
+            before,
+            after,
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

spike/F13_synthesis_spike.py

@@ -3,7 +3,7 @@
 NOT a unit test. Not maintained. Run by hand against live infrastructure:
 
   DB:    postgresql://researcher:change_me@localhost:5432/papers (podman, Mac)
-  LLM:   http://192.168.178.103:11434 (Jetson, qwen2.5:7b)
+  LLM:   http://jetson.local:11434 (Jetson, qwen2.5:7b)
 
 Goal: emit 3-5 grounded leads (connection + gap) and a couple of
 conjectures, then eyeball:
@@ -25,7 +25,7 @@ from pathlib import Path
 
 # Wire up the live infra without polluting ~/.env
 os.environ.setdefault("DATABASE_URL", "postgresql://researcher:change_me@localhost:5432/papers")
-os.environ.setdefault("OLLAMA_BASE_URL", "http://192.168.178.103:11434")
+os.environ.setdefault("OLLAMA_BASE_URL", "http://jetson.local:11434")
 os.environ.setdefault("SYNTHESIS_LLM_MODEL", "qwen2.5:7b")
 os.environ.setdefault("SYNTHESIS_TOP_K", "12")
 

tests/cli/test_cli.py

@@ -106,6 +106,13 @@ class TestSearch:
             assert "Test Paper" in result.stdout
             assert "0.123" in result.stdout
 
+            # Regression guard (DQ-4): the pgvector `<->` operator requires the
+            # query embedding cast to ::vector. Without it, real Postgres raises
+            # "operator does not exist: vector <-> double precision[]". Mocked DBs
+            # don't catch the type error, so assert the cast is in the SQL.
+            executed_sql = mock_conn.execute.call_args_list[0][0][0]
+            assert "<->" in executed_sql and "::vector" in executed_sql
+
     def test_search_formula_command(self) -> None:
         """Test `search formula` command with mocked DB."""
         with patch("codex.db.get_conn") as mock_get_conn:

tests/graph/test_cli.py

@@ -0,0 +1,256 @@
+"""CLI tests for `codex graph` sub-commands (F-15)."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from unittest.mock import MagicMock, patch
+
+import networkx as nx
+import numpy as np
+from typer.testing import CliRunner
+
+from codex.cli import app
+
+runner = CliRunner()
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_conn_cm(conn: MagicMock):
+    @contextmanager
+    def _cm():
+        yield conn
+
+    return _cm
+
+
+def _make_conn_with_paper_ids(*paper_ids: str) -> MagicMock:
+    """Mock conn that returns paper rows for SELECT id, title FROM papers."""
+    conn = MagicMock()
+    conn.execute.return_value.fetchall.return_value = [
+        {"id": pid, "title": f"Title of {pid}"} for pid in paper_ids
+    ]
+    return conn
+
+
+def _sample_graph() -> nx.DiGraph:
+    """A→B, A→C, B→C, B→D, C→D — 5 nodes, D is dangling (not in papers)."""
+    g = nx.DiGraph()
+    g.add_edges_from([("A", "B"), ("A", "C"), ("B", "C"), ("B", "D"), ("C", "D")])
+    return g
+
+
+# ---------------------------------------------------------------------------
+# codex graph report
+# ---------------------------------------------------------------------------
+
+
+class TestGraphReport:
+    def _run(self, graph=None, known_ids=("A", "B", "C"), extra_args=()):
+        if graph is None:
+            graph = _sample_graph()
+        conn = _make_conn_with_paper_ids(*known_ids)
+
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.graph.build_citation_graph", return_value=graph),
+        ):
+            return runner.invoke(app, ["graph", "report", *extra_args])
+
+    def test_exit_zero(self):
+        assert self._run().exit_code == 0
+
+    def test_shows_hub_section(self):
+        assert "HUB" in self._run().output.upper()
+
+    def test_shows_dangling_section(self):
+        assert "DANGLING" in self._run().output.upper()
+
+    def test_dangling_node_in_output(self):
+        # D is cited but not in known_ids
+        assert "D" in self._run().output
+
+    def test_json_output_valid(self):
+        import json
+
+        graph = _sample_graph()
+        conn = _make_conn_with_paper_ids("A", "B", "C")
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.graph.build_citation_graph", return_value=graph),
+        ):
+            result = runner.invoke(app, ["graph", "report", "--json"])
+        assert result.exit_code == 0
+        data = json.loads(result.output)
+        assert "hubs" in data
+        assert "dangling" in data
+        assert "nodes" in data
+        assert data["dangling_count"] == 1
+
+    def test_empty_graph_message(self):
+        conn = _make_conn_with_paper_ids()
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.graph.build_citation_graph", return_value=nx.DiGraph()),
+        ):
+            result = runner.invoke(app, ["graph", "report"])
+        assert result.exit_code == 0
+        assert "empty" in result.output.lower()
+
+    def test_top_n_respected(self):
+        # Build graph with 8 nodes, ask for top 3
+        g = nx.DiGraph()
+        for i in range(8):
+            g.add_edge(f"P{i}", "HUB")
+        conn = _make_conn_with_paper_ids(*[f"P{i}" for i in range(8)], "HUB")
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.graph.build_citation_graph", return_value=g),
+        ):
+            result = runner.invoke(app, ["graph", "report", "--top-n", "3"])
+        assert result.exit_code == 0
+
+    def test_small_corpus_warning_shown(self):
+        # known_ids has only 3 papers (< graph_min_corpus_size default 15)
+        result = self._run(known_ids=("A", "B", "C"))
+        assert result.exit_code == 0
+        assert "Warning" in result.output or "warning" in result.output
+
+    def test_shows_citing_coverage_line(self):
+        # _sample_graph: A, B, C all cite → 3/3 = 100%, so the line shows but no warning.
+        out = self._run(known_ids=("A", "B", "C")).output
+        assert "Citing coverage" in out
+        assert "weakens" not in out  # 100% coverage → no R-D warning
+
+    def test_low_citing_coverage_warning(self):
+        # Only A has an out-edge; B and C are ingested but cite nothing → 1/3 = 33%.
+        g = nx.DiGraph()
+        g.add_edge("A", "X")
+        g.add_nodes_from(["B", "C"])
+        result = self._run(graph=g, known_ids=("A", "B", "C"))
+        assert result.exit_code == 0
+        assert "weakens PageRank" in result.output  # the R-D coverage warning
+
+    def test_json_includes_citing_coverage(self):
+        import json
+
+        g = nx.DiGraph()
+        g.add_edge("A", "X")
+        g.add_nodes_from(["B", "C"])
+        conn = _make_conn_with_paper_ids("A", "B", "C")
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.graph.build_citation_graph", return_value=g),
+        ):
+            result = runner.invoke(app, ["graph", "report", "--json"])
+        data = json.loads(result.output)
+        assert data["citing_coverage"] == {"citing": 1, "papers": 3, "fraction": 0.3333}
+
+
+# ---------------------------------------------------------------------------
+# codex graph related
+# ---------------------------------------------------------------------------
+
+
+class TestGraphRelated:
+    def _run(self, paper_id: str, graph=None, extra_args=()):
+        if graph is None:
+            graph = _sample_graph()
+        conn = MagicMock()
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.graph.build_citation_graph", return_value=graph),
+        ):
+            return runner.invoke(app, ["graph", "related", paper_id, *extra_args])
+
+    def test_exit_zero_known_paper(self):
+        assert self._run("A").exit_code == 0
+
+    def test_unknown_paper_graceful(self):
+        result = self._run("UNKNOWN_ID")
+        assert result.exit_code == 0
+        assert "No papers" in result.output
+
+    def test_no_results_message_when_min_shared_high(self):
+        result = self._run("A", _sample_graph(), extra_args=("--min-shared", "99"))
+        assert "No papers" in result.output
+
+    def test_related_papers_shown(self):
+        # A cites {B,C}; B also cites C → shared=1 with A
+        result = self._run("A", _sample_graph(), extra_args=("--min-shared", "1"))
+        assert result.exit_code == 0
+        # B shares C with A; should appear
+        assert "B" in result.output
+
+
+# ---------------------------------------------------------------------------
+# codex search paper --cite-boost
+# ---------------------------------------------------------------------------
+
+
+class TestSearchCiteBoost:
+    def _make_two_paper_conn(self, pr_winner: str) -> MagicMock:
+        """Two papers A and B at equal distance; pr_winner has a high PageRank score."""
+        conn = MagicMock()
+        conn.execute.return_value.fetchall.return_value = [
+            {"id": "A", "title": "Paper A", "year": 2020, "distance": 0.5},
+            {"id": "B", "title": "Paper B", "year": 2021, "distance": 0.5},
+        ]
+        return conn
+
+    def test_cite_boost_exits_zero(self):
+        conn = MagicMock()
+        conn.execute.return_value.fetchall.return_value = [
+            {"id": "A", "title": "Paper A", "year": 2020, "distance": 0.2},
+        ]
+        graph = _sample_graph()
+
+        def _fake_encode(texts):
+            return np.zeros((len(texts), 1024), dtype="float32")
+
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.embed.get_embedder") as mock_emb,
+            patch("codex.graph.build_citation_graph", return_value=graph),
+        ):
+            mock_emb.return_value.encode_dense.side_effect = _fake_encode
+            result = runner.invoke(app, ["search", "paper", "hodge star", "--cite-boost"])
+        assert result.exit_code == 0
+
+    def test_cite_boost_promotes_high_pr_paper(self):
+        """High-PR paper should appear first (lower boosted distance) via divide formula."""
+        import networkx as nx
+
+        # Build a graph where A has high PageRank (many papers cite A)
+        g = nx.DiGraph()
+        for i in range(10):
+            g.add_edge(f"P{i}", "A")  # A is heavily cited → high PR
+        g.add_edge("P0", "B")  # B is barely cited → low PR
+
+        conn = MagicMock()
+        conn.execute.return_value.fetchall.return_value = [
+            {"id": "A", "title": "Hub", "year": 2020, "distance": 0.5},
+            {"id": "B", "title": "Niche", "year": 2021, "distance": 0.5},
+        ]
+
+        def _fake_encode(texts):
+            return np.zeros((len(texts), 1024), dtype="float32")
+
+        with (
+            patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
+            patch("codex.embed.get_embedder") as mock_emb,
+            patch("codex.graph.build_citation_graph", return_value=g),
+        ):
+            mock_emb.return_value.encode_dense.side_effect = _fake_encode
+            result = runner.invoke(app, ["search", "paper", "query", "--cite-boost"])
+
+        assert result.exit_code == 0
+        lines = [ln for ln in result.output.strip().splitlines() if "Hub" in ln or "Niche" in ln]
+        assert len(lines) == 2
+        # A (Hub) should appear before B (Niche) — lower boosted distance
+        hub_line = next(line for line in lines if "Hub" in line)
+        niche_line = next(line for line in lines if "Niche" in line)
+        assert lines.index(hub_line) < lines.index(niche_line)

tests/graph/test_graph.py

@@ -0,0 +1,311 @@
+"""Tests for codex.graph — citation graph analytics (F-15)."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import networkx as nx
+import pytest
+
+from codex.graph import (
+    build_citation_graph,
+    citation_pagerank,
+    citing_coverage,
+    dangling_citations,
+    find_co_cited,
+    find_related,
+)
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def _make_conn(rows: list[dict]) -> MagicMock:
+    conn = MagicMock()
+    conn.execute.return_value.fetchall.return_value = rows
+    return conn
+
+
+def _citation_rows(*pairs: tuple[str, str]) -> list[dict]:
+    return [{"citing_id": a, "cited_id": b} for a, b in pairs]
+
+
+def _small_graph() -> nx.DiGraph:
+    """
+    A→C, A→D, B→C, B→D, B→E, C→D
+    Nodes: A B C D E  (5 nodes ingested)
+    """
+    rows = _citation_rows(
+        ("A", "C"),
+        ("A", "D"),
+        ("B", "C"),
+        ("B", "D"),
+        ("B", "E"),
+        ("C", "D"),
+    )
+    return build_citation_graph(_make_conn(rows))
+
+
+# ---------------------------------------------------------------------------
+# build_citation_graph
+# ---------------------------------------------------------------------------
+
+
+class TestBuildCitationGraph:
+    def test_empty_db_returns_empty_graph(self):
+        g = build_citation_graph(_make_conn([]))
+        assert isinstance(g, nx.DiGraph)
+        assert g.number_of_nodes() == 0
+        assert g.number_of_edges() == 0
+
+    def test_nodes_and_edges_populated(self):
+        rows = _citation_rows(("P1", "P2"), ("P1", "P3"), ("P2", "P3"))
+        g = build_citation_graph(_make_conn(rows))
+        assert set(g.nodes()) == {"P1", "P2", "P3"}
+        assert g.number_of_edges() == 3
+
+    def test_edge_direction(self):
+        rows = _citation_rows(("citing", "cited"))
+        g = build_citation_graph(_make_conn(rows))
+        assert g.has_edge("citing", "cited")
+        assert not g.has_edge("cited", "citing")
+
+    def test_duplicate_edges_collapsed(self):
+        rows = _citation_rows(("A", "B"), ("A", "B"))
+        g = build_citation_graph(_make_conn(rows))
+        assert g.number_of_edges() == 1
+
+    def test_dangling_nodes_included(self):
+        rows = _citation_rows(("P1", "not_in_papers"))
+        g = build_citation_graph(_make_conn(rows))
+        assert "not_in_papers" in g.nodes()
+
+    def test_cross_citation_uses_doi_when_in_kb(self):
+        # Simulate DB COALESCE: cited_id already resolved to papers.id (DOI)
+        # for in-KB papers; dangling references keep their OpenAlex ID.
+        rows = [
+            {
+                "citing_id": "https://doi.org/10.1234/paper-a",
+                "cited_id": "https://doi.org/10.5678/paper-b",
+            },
+            {
+                "citing_id": "https://doi.org/10.5678/paper-b",
+                "cited_id": "https://openalex.org/W999",
+            },
+        ]
+        g = build_citation_graph(_make_conn(rows))
+        assert g.has_edge("https://doi.org/10.1234/paper-a", "https://doi.org/10.5678/paper-b")
+        assert g.has_edge("https://doi.org/10.5678/paper-b", "https://openalex.org/W999")
+        assert g.number_of_nodes() == 3
+
+
+# ---------------------------------------------------------------------------
+# citation_pagerank
+# ---------------------------------------------------------------------------
+
+
+class TestCitationPagerank:
+    def test_empty_graph_returns_empty_dict(self):
+        g = nx.DiGraph()
+        assert citation_pagerank(g) == {}
+
+    def test_scores_sum_to_one(self):
+        g = _small_graph()
+        pr = citation_pagerank(g)
+        assert abs(sum(pr.values()) - 1.0) < 1e-6
+
+    def test_hub_node_has_highest_score(self):
+        # D is pointed to by A, B, C → should rank highest
+        g = _small_graph()
+        pr = citation_pagerank(g)
+        assert pr["D"] == max(pr.values())
+
+    def test_all_nodes_present(self):
+        g = _small_graph()
+        pr = citation_pagerank(g)
+        assert set(pr.keys()) == set(g.nodes())
+
+    def test_small_graph_returns_uniform_scores(self):
+        # 3 nodes < _MIN_PAGERANK_NODES (5) → uniform
+        rows = _citation_rows(("A", "B"), ("B", "C"))
+        g = build_citation_graph(_make_conn(rows))
+        pr = citation_pagerank(g)
+        assert len(pr) == 3
+        scores = list(pr.values())
+        assert all(abs(s - scores[0]) < 1e-9 for s in scores)
+
+    def test_small_graph_logs_warning(self, caplog: pytest.LogCaptureFixture):
+        import logging
+
+        rows = _citation_rows(("A", "B"), ("B", "C"))
+        g = build_citation_graph(_make_conn(rows))
+        with caplog.at_level(logging.WARNING, logger="codex.graph"):
+            citation_pagerank(g)
+        assert any("corpus too small" in rec.message for rec in caplog.records)
+
+    def test_custom_damping(self):
+        g = _small_graph()
+        pr_85 = citation_pagerank(g, damping=0.85)
+        pr_50 = citation_pagerank(g, damping=0.50)
+        # Different damping → different scores (not all equal)
+        assert pr_85 != pr_50
+
+
+# ---------------------------------------------------------------------------
+# find_related  (bibliographic coupling)
+# ---------------------------------------------------------------------------
+
+
+class TestFindRelated:
+    def _coupling_graph(self) -> nx.DiGraph:
+        """
+        A cites: X Y Z
+        B cites: X Y
+        C cites: X
+        D cites: W   (no overlap with A)
+        """
+        rows = _citation_rows(
+            ("A", "X"),
+            ("A", "Y"),
+            ("A", "Z"),
+            ("B", "X"),
+            ("B", "Y"),
+            ("C", "X"),
+            ("D", "W"),
+        )
+        return build_citation_graph(_make_conn(rows))
+
+    def test_unknown_paper_returns_empty(self):
+        g = self._coupling_graph()
+        assert find_related("UNKNOWN", g) == []
+
+    def test_paper_with_no_refs_returns_empty(self):
+        # X has no outgoing edges → no references to couple on
+        g = self._coupling_graph()
+        assert find_related("X", g) == []
+
+    def test_related_by_two_shared_refs(self):
+        g = self._coupling_graph()
+        related = find_related("A", g, min_shared=2)
+        assert "B" in related
+
+    def test_not_related_below_threshold(self):
+        g = self._coupling_graph()
+        related = find_related("A", g, min_shared=2)
+        assert "C" not in related  # only 1 shared ref
+
+    def test_ordered_by_shared_count_desc(self):
+        g = self._coupling_graph()
+        related = find_related("A", g, min_shared=1)
+        # B shares 2 refs, C shares 1 → B before C
+        assert related.index("B") < related.index("C")
+
+    def test_self_not_in_results(self):
+        g = self._coupling_graph()
+        related = find_related("A", g, min_shared=1)
+        assert "A" not in related
+
+
+# ---------------------------------------------------------------------------
+# find_co_cited
+# ---------------------------------------------------------------------------
+
+
+class TestFindCoCited:
+    def _co_citation_graph(self) -> nx.DiGraph:
+        """
+        P1 cites: A, B, C
+        P2 cites: A, B
+        P3 cites: A
+        """
+        rows = _citation_rows(
+            ("P1", "A"),
+            ("P1", "B"),
+            ("P1", "C"),
+            ("P2", "A"),
+            ("P2", "B"),
+            ("P3", "A"),
+        )
+        return build_citation_graph(_make_conn(rows))
+
+    def test_unknown_paper_returns_empty(self):
+        g = self._co_citation_graph()
+        assert find_co_cited("UNKNOWN", g) == []
+
+    def test_paper_with_no_citers_returns_empty(self):
+        # C has citers but let's test a node that has no predecessors
+        rows = _citation_rows(("P1", "X"))
+        g = build_citation_graph(_make_conn(rows))
+        # "X" is cited once by P1, but P1 has no other citations
+        # X is co-cited with nothing
+        result = find_co_cited("X", g)
+        assert result == []
+
+    def test_most_co_cited_first(self):
+        g = self._co_citation_graph()
+        result = find_co_cited("A", g)
+        # B is co-cited twice (by P1 and P2); C once (by P1 only)
+        ids = [r[0] for r in result]
+        assert ids[0] == "B"
+
+    def test_self_not_included(self):
+        g = self._co_citation_graph()
+        result = find_co_cited("A", g)
+        assert all(pid != "A" for pid, _ in result)
+
+    def test_counts_correct(self):
+        g = self._co_citation_graph()
+        result = dict(find_co_cited("A", g))
+        assert result["B"] == 2  # P1 and P2 both cite A and B
+        assert result["C"] == 1  # only P1 cites A and C
+
+
+# ---------------------------------------------------------------------------
+# dangling_citations
+# ---------------------------------------------------------------------------
+
+
+class TestDanglingCitations:
+    def test_empty_graph(self):
+        assert dangling_citations(nx.DiGraph(), set()) == []
+
+    def test_all_known_returns_empty(self):
+        rows = _citation_rows(("A", "B"), ("B", "C"))
+        g = build_citation_graph(_make_conn(rows))
+        assert dangling_citations(g, {"A", "B", "C"}) == []
+
+    def test_identifies_dangling(self):
+        rows = _citation_rows(("A", "B"), ("A", "external_doi"))
+        g = build_citation_graph(_make_conn(rows))
+        dangling = dangling_citations(g, {"A", "B"})
+        assert "external_doi" in dangling
+        assert "A" not in dangling
+        assert "B" not in dangling
+
+    def test_empty_known_set(self):
+        rows = _citation_rows(("A", "B"))
+        g = build_citation_graph(_make_conn(rows))
+        dangling = dangling_citations(g, set())
+        assert set(dangling) == {"A", "B"}
+
+
+# ---------------------------------------------------------------------------
+# citing_coverage (R-D)
+# ---------------------------------------------------------------------------
+
+
+class TestCitingCoverage:
+    def test_counts_papers_with_out_edges(self):
+        # _small_graph: A→.., B→.., C→D have out-edges; D, E have none.
+        g = _small_graph()
+        n_citing, n_papers = citing_coverage(g, {"A", "B", "C", "D", "E"})
+        assert (n_citing, n_papers) == (3, 5)
+
+    def test_paper_absent_from_graph_counts_as_no_out_edges(self):
+        # A cites; Z was ingested but has no chunks/edges, so it isn't a graph node.
+        g = _small_graph()
+        assert citing_coverage(g, {"A", "Z"}) == (1, 2)
+
+    def test_empty(self):
+        assert citing_coverage(nx.DiGraph(), set()) == (0, 0)

tests/ingest/test_ingest.py

@@ -10,7 +10,9 @@ from contextlib import contextmanager
 from typing import Any
 from unittest.mock import MagicMock, patch
 
+import httpx
 import numpy as np
+import psycopg
 import pytest
 
 from codex.ingest import IngestResult, _make_bibkey, ingest_paper
@@ -106,13 +108,99 @@ def test_ingest_paper_basic() -> None:
     )
 
 
+def test_ingest_citation_fallback_to_crossref() -> None:
+    """R-B: OpenAlex + S2 both empty → Crossref references supply the edges.
+
+    Crossref is the third leg of the fallback chain. Its cited DOIs are
+    case-normalised and citing_id is rewritten to the canonical paper.id.
+    """
+    paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    # Upper-case suffix proves the cited DOI is normalised to bare lower-case.
+    crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")]
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),  # OpenAlex empty
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),  # S2 empty
+        patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr,
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper(paper.id)
+
+    # Crossref consulted with the bare DOI; one edge upserted.
+    mock_cr.assert_called_once_with(paper.id)
+    assert result.citations_upserted == 1
+    mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
+    cit_rows = mock_cursor.executemany.call_args_list[0][0][1]
+    assert cit_rows[0][0] == paper.id
+    assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7"
+
+
+def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
+    """R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback)."""
+    paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
+    mock_conn = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch(
+            "codex.ingest.openalex.fetch_citations",
+            return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")],
+        ),
+        patch("codex.ingest.crossref.fetch_references") as mock_cr,
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    mock_cr.assert_not_called()
+
+
+def test_ingest_canonicalises_paper_id_to_caller_arg() -> None:
+    """papers.id is the caller-supplied id, not OpenAlex's URL-form id (audit C-7).
+
+    Real OpenAlex returns the doi/id as a full URL; ingest must key the paper on
+    the bare id the caller passed (matching ingest scripts and CLI lookups), and
+    keep the OpenAlex work id only as openalex_id.
+    """
+    oa_paper = Paper(
+        id="https://doi.org/10.48550/arxiv.math/0603097",  # URL form, as OpenAlex emits
+        title="A Test Paper",
+        openalex_id="https://openalex.org/W4301005794",
+        authors=["Alice", "Bob"],
+        year=2008,
+        abstract="Test abstract.",
+    )
+    mock_conn = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=oa_paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper("math/0603097")
+
+    papers_params = mock_conn.execute.call_args_list[0][0][1]
+    assert papers_params["id"] == "math/0603097", "papers.id must be the caller's bare id"
+    assert papers_params["openalex_id"] == "https://openalex.org/W4301005794"
+    assert result.paper_id == "math/0603097"
+
+
 # ---------------------------------------------------------------------------
 # 2. test_ingest_paper_source_tex
 # ---------------------------------------------------------------------------
 
 
 def test_ingest_paper_source_tex(tmp_path: Any) -> None:
-    """``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB."""
+    """``.tex`` source → section-aware chunking; the stored ``section`` column is
+    labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12)."""
     tex_file = tmp_path / "paper.tex"
     tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.")
 
@@ -122,15 +210,20 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
     mock_conn.executemany = MagicMock()
     mock_conn.commit = MagicMock()
 
-    fake_chunks = ["chunk one text here.", "chunk two text here."]
+    # (section title, chunk content) pairs from the section-aware chunker.
+    section_pairs = [
+        ("Introduction", "Body of the introduction goes here."),
+        ("Proof of the Main Theorem", "Body of the proof goes here."),
+        ("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."),
+    ]
 
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=paper),
         patch("codex.ingest.openalex.fetch_citations", return_value=[]),
         patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
-        patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."),
+        patch("codex.parsing.tex.chunk_sections", return_value=section_pairs),
-        patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
+        patch("codex.ingest.is_quality_chunk", return_value=True),
     ):
         result = ingest_paper(paper.id, source_path=str(tex_file))
 
@@ -145,7 +238,16 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
     chunk_sql: str = chunk_insert_call[0][0]
     assert "INSERT INTO chunks" in chunk_sql
 
-    assert result.chunks_upserted == len(fake_chunks)
+    # R-F: a bare canonical heading → bucket ("Introduction"→intro); any other
+    # heading is stored as its cleaned real title ("proof of the main theorem",
+    # "discrete conformal maps") instead of being collapsed/mislabelled.
+    chunk_rows = chunk_insert_call[0][1]
+    assert [row[4] for row in chunk_rows] == [
+        "intro",
+        "proof of the main theorem",
+        "discrete conformal maps",
+    ]
+    assert result.chunks_upserted == len(section_pairs)
 
 
 # ---------------------------------------------------------------------------
@@ -173,11 +275,15 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=paper),
         patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1);
+        # mock it empty so only the GROBID refs contribute to the count.
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
         patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
         patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
         patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
         patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
+        patch("codex.ingest.is_quality_chunk", return_value=True),
     ):
         result = ingest_paper(paper.id, source_path=str(pdf_file))
 
@@ -193,6 +299,73 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
     assert mock_cursor.executemany.call_count >= 2
 
 
+def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
+    """R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so
+    it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each
+    must be normalized to the bare, lower-cased canonical form before the Citation
+    is built — otherwise the same reference splits into case-/URL-variant graph
+    nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no
+    DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1)."""
+    pdf_file = tmp_path / "paper.pdf"
+    pdf_file.write_bytes(b"%PDF-1.4 fake content")
+
+    paper = _make_paper()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
+        # Faithful to GROBID's output shape: all five keys always present.
+        return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id}
+
+    grobid_refs = [
+        _ref(doi="10.1007/S00454-019-00132-8"),  # mixed-case bare DOI
+        _ref(doi="https://doi.org/10.1145/AbC.123"),  # URL-form DOI
+        _ref(doi="doi:10.1/MixedCase"),  # doi:-prefixed DOI
+        _ref(arxiv_id="2101.00001"),  # arXiv-only ref → untouched
+    ]
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID
+        # refs contribute to the citation rows under inspection.
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
+        patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
+        patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
+        patch("codex.ingest.is_quality_chunk", return_value=True),
+    ):
+        result = ingest_paper(paper.id, source_path=str(pdf_file))
+
+    assert result.citations_upserted == 4
+
+    # Locate the citations INSERT among the cursor's executemany calls (chunks
+    # are also inserted via executemany, so filter by SQL rather than by index).
+    mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
+    citation_calls = [
+        c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0]
+    ]
+    assert len(citation_calls) == 1
+    inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1]
+
+    # Every edge hangs off the canonical paper.id (citing side).
+    assert all(row[0] == paper.id for row in inserted_rows)
+
+    cited_ids = {row[1] for row in inserted_rows}
+    assert cited_ids == {
+        "10.1007/s00454-019-00132-8",  # mixed-case → lower-cased
+        "10.1145/abc.123",  # URL prefix stripped + lower-cased
+        "10.1/mixedcase",  # doi: prefix stripped + lower-cased
+        "2101.00001",  # arXiv id untouched
+    }
+    # No raw URL-form / prefixed DOI leaked into the citation graph.
+    assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids)
+
+
 # ---------------------------------------------------------------------------
 # 4. test_ingest_paper_not_found
 # ---------------------------------------------------------------------------
@@ -245,7 +418,7 @@ def test_ingest_paper_idempotent() -> None:
 
 
 def test_ingest_paper_arxiv_s2_fallback() -> None:
-    """If OpenAlex returns None for an arXiv ID, S2 fallback produces a stub Paper."""
+    """OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted."""
     paper_id = "2301.07041"
     mock_conn = MagicMock()
     mock_conn.execute = MagicMock()
@@ -254,17 +427,121 @@ def test_ingest_paper_arxiv_s2_fallback() -> None:
 
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=None),
+        # arXiv + abstract sources also empty → genuine stub fallback path.
+        patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv,
+        patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
+        patch("codex.ingest.crossref.fetch_abstract", return_value=None),
         patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2,
         patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
     ):
         result = ingest_paper(paper_id)
 
-    # S2 was consulted as fallback
+    # arXiv metadata recovery attempted; S2 references consulted as fallback.
+    mock_arxiv.assert_called_once_with(paper_id)
     mock_s2.assert_called()
     assert result.paper_id == paper_id
 
 
+def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None:
+    """DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API,
+    bibkey auto-generated, abstract embedded (no zero vector)."""
+    recovered = Paper(
+        id="1911.00966",
+        title="A discrete version of Liouville's theorem on conformal maps",
+        authors=["Ulrich Pinkall", "Boris Springborn"],
+        year=2019,
+        abstract="Liouville's theorem says that in dimension greater than two...",
+    )
+    fake_emb = _fake_embedder()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=None),
+        patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv,
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=fake_emb),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper("1911.00966")
+
+    mock_arxiv.assert_called_once_with("1911.00966")
+    assert result.paper_id == "1911.00966"
+    params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert params["title"].startswith("A discrete version")
+    assert params["bibkey"] == "PinkallSpringborn2019"  # generated from recovered authors+year
+    # abstract present → real embedding computed, not a zero vector
+    fake_emb.encode_dense.assert_called_once()
+
+
+def test_ingest_paper_empty_abstract_recovered_from_s2() -> None:
+    """DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding."""
+    paper = _make_paper(abstract=None)  # has openalex_id, no abstract
+    fake_emb = _fake_embedder()
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch(
+            "codex.ingest.semanticscholar.fetch_abstract",
+            return_value="Recovered abstract text.",
+        ) as mock_abs,
+        patch("codex.ingest.get_embedder", return_value=fake_emb),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    mock_abs.assert_called_once()
+    params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert params["abstract"] == "Recovered abstract text."
+    fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."])
+
+
+def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
+    """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
+    rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
+    paper = _make_paper()  # openalex_id="W123", id="2301.07041"
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    # S2 keys citing_id by the id form passed in; ingest must canonicalise it.
+    s2_refs = [
+        Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"),
+        Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"),
+    ]
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2,
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper(paper.id)
+
+    # S2 consulted with a namespaced id (a bare arXiv id 404s on S2).
+    mock_s2.assert_called_once_with("arXiv:2301.07041")
+    assert result.citations_upserted == 2
+
+    mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
+    inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1]
+    # citing_id rewritten to paper.id, not the S2 id form.
+    assert all(row[0] == paper.id for row in inserted_rows)
+    cited = {row[1] for row in inserted_rows}
+    assert "10.1/upper" in cited  # DOI lowercased
+    assert "2202.00002" in cited  # arXiv id untouched
+
+
 # ---------------------------------------------------------------------------
 # 7. test_ingest_paper_no_abstract_uses_zero_vector
 # ---------------------------------------------------------------------------
@@ -283,6 +560,9 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=paper),
         patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # No abstract recoverable from any source (DQ-2) → genuine zero-vector case.
+        patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
+        patch("codex.ingest.crossref.fetch_abstract", return_value=None),
         patch("codex.ingest.get_embedder", return_value=fake_emb),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
     ):
@@ -318,9 +598,11 @@ def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> No
     mock_conn.executemany = MagicMock()
     mock_conn.commit = MagicMock()
 
-    fake_formula = FormulaChunk(paper_id=paper.id, page=1, raw_latex=r"\alpha", context="test")
+    # Extractors derive paper_id from the PDF filename stem ("paper"), which differs
+    # from papers.id — ingest must insert papers.id, not this stem (audit C-10).
+    fake_formula = FormulaChunk(paper_id="paper", page=1, raw_latex=r"\alpha", context="test")
     fake_figure = FigureChunk(
-        paper_id=paper.id, page=1, image_path="/tmp/fig.png", caption="Figure 1."
+        paper_id="paper", page=1, image_path="/tmp/fig.png", caption="Figure 1."
     )
 
     mock_settings = MagicMock()
@@ -347,6 +629,16 @@ def test_ingest_paper_rich_calls_formula_and_figure_parsers(tmp_path: Any) -> No
     assert result.formulas_upserted == 1
     assert result.figures_upserted == 1
 
+    # C-10: formula/figure rows must be keyed on papers.id, not the extractor's
+    # filename-stem paper_id, or the FK to papers(id) is violated.
+    cursor = mock_conn.cursor.return_value.__enter__.return_value
+    inserted_paper_ids = {
+        row[0] for call in cursor.executemany.call_args_list for row in call[0][1]
+    }
+    assert inserted_paper_ids == {paper.id}, (
+        f"formula/figure inserts must use papers.id, got {inserted_paper_ids}"
+    )
+
 
 def test_ingest_paper_no_rich_skips_parsers(tmp_path: Any) -> None:
     """When rich=False (default), formula and figure parsers are NOT called."""
@@ -480,6 +772,156 @@ def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None:
     assert upsert_params["bibkey"] == "AliceBob2023"
 
 
+def test_ingest_doi_paper_upserts_bare_canonical_id() -> None:
+    """DQ-5 regression: re-ingesting an existing DOI paper must be idempotent.
+
+    OpenAlex returns the ``doi`` as a full URL (and possibly mixed-case), but the
+    stored canonical ``papers.id`` is the BARE, lower-cased DOI (M-1 migration).
+    The papers upsert must therefore carry the BARE id into ``%(id)s`` so
+    ``ON CONFLICT (id)`` matches the existing row and UPDATEs it — rather than
+    attempting a fresh INSERT that trips the separate ``papers_openalex_id_key``
+    unique constraint (the UniqueViolation crash).
+
+    The real ``_map_paper``/``fetch_paper`` run here (only the HTTP ``_get``, the
+    embedder and the DB are mocked), so this exercises the actual normalization
+    through the ingest path. The DB is a MagicMock, so it cannot raise the real
+    constraint error; the live re-ingest of 10.1007/s00454-019-00132-8 is the
+    end-to-end UniqueViolation proof (see DATA-QUALITY-2026-06-15.md, DQ-5).
+    """
+    bare = "10.1007/s00454-019-00132-8"
+    work = {
+        "id": "https://openalex.org/W2899262408",
+        # URL form + an upper-case suffix char — the worst case for ON CONFLICT.
+        "doi": "https://doi.org/10.1007/S00454-019-00132-8",
+        "title": "A Test Paper",
+        "publication_year": 2019,
+        "authorships": [{"author": {"display_name": "Alice Smith"}}],
+        "abstract_inverted_index": {"Hello": [0], "world": [1]},
+        # Non-empty so fetch_citations yields edges and the S2 fallback is skipped.
+        "referenced_works": ["https://openalex.org/W1"],
+    }
+
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch(
+            "codex.ingest.openalex._get",
+            return_value=httpx.Response(200, json=work),
+        ),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper(bare)
+
+    # The IngestResult and the upsert must use the bare canonical id, not the URL.
+    assert result.paper_id == bare
+
+    upsert_sql: str = mock_conn.execute.call_args_list[0][0][0]
+    upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert "INSERT INTO papers" in upsert_sql
+    assert "ON CONFLICT (id)" in upsert_sql
+    assert upsert_params["id"] == bare, (
+        f"upsert id must be the bare canonical DOI '{bare}', "
+        f"not the OpenAlex URL form '{upsert_params['id']}'"
+    )
+
+
+def test_ingest_normalizes_non_bare_caller_id() -> None:
+    """Review #1: a non-bare CALLER id (URL-form / mixed-case DOI) is normalized to the
+    bare canonical form before being pinned to papers.id (the C-7 pin runs it through
+    _norm_cited_id), so a sloppy caller cannot store a URL-form papers.id that would
+    defeat ON CONFLICT (id) idempotency or the ``paper.id.startswith('10.')`` gates."""
+    paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W1")
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
+        patch("codex.ingest.crossref.fetch_references", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        # Caller passes the URL form with an upper-case suffix char (worst case).
+        result = ingest_paper("https://doi.org/10.1007/S00454-019-00132-8")
+
+    bare = "10.1007/s00454-019-00132-8"
+    assert result.paper_id == bare
+    upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
+    assert upsert_params["id"] == bare, (
+        f"non-bare caller must be normalized to '{bare}', got '{upsert_params['id']}'"
+    )
+
+
+def test_ingest_disambiguates_bibkey_on_unique_violation() -> None:
+    """A papers.bibkey UNIQUE collision is retried with an a/b/c suffix (audit C-15)."""
+    paper = _make_paper()  # bibkey auto-generates to "AliceBob2023"
+    mock_conn = MagicMock()
+    state = {"first": True}
+
+    def execute_side_effect(sql: str, params: Any = None) -> MagicMock:
+        # The first papers INSERT collides on the bibkey; the retry must succeed.
+        if "INSERT INTO papers" in sql and state["first"]:
+            state["first"] = False
+            raise psycopg.errors.UniqueViolation(
+                'duplicate key value violates unique constraint "papers_bibkey_key"'
+            )
+        return MagicMock()
+
+    mock_conn.execute.side_effect = execute_side_effect
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        ingest_paper(paper.id)
+
+    papers_inserts = [
+        c for c in mock_conn.execute.call_args_list if "INSERT INTO papers" in c[0][0]
+    ]
+    assert len(papers_inserts) == 2, "expected one retry after the bibkey collision"
+    assert papers_inserts[0][0][1]["bibkey"] == "AliceBob2023"
+    assert papers_inserts[1][0][1]["bibkey"] == "AliceBob2023a"
+    mock_conn.rollback.assert_called_once()
+
+
+def test_ingest_openalex_collision_raises_clear_error() -> None:
+    """An openalex_id UNIQUE collision raises a clear, actionable error (audit C-15).
+
+    Unlike a bibkey clash (auto-suffixed), a shared openalex_id means two papers
+    claim the same OpenAlex work — a real data conflict that must surface, not the
+    raw psycopg error.
+    """
+    paper = _make_paper()
+    mock_conn = MagicMock()
+
+    def execute_side_effect(sql: str, params: Any = None) -> MagicMock:
+        if "INSERT INTO papers" in sql:
+            raise psycopg.errors.UniqueViolation(
+                'duplicate key value violates unique constraint "papers_openalex_id_key"'
+            )
+        return MagicMock()
+
+    mock_conn.execute.side_effect = execute_side_effect
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+        pytest.raises(ValueError, match="openalex_id"),
+    ):
+        ingest_paper(paper.id)
+
+
 # ---------------------------------------------------------------------------
 
 

tests/mcp/test_http_auth.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 from unittest.mock import MagicMock, patch
 
 import pytest
+from pydantic import SecretStr
 
 # ---------------------------------------------------------------------------
 # _require_http_token
@@ -37,7 +38,7 @@ def test_require_http_token_returns_token_when_set() -> None:
     from codex.mcp_server import _require_http_token
 
     mock_settings = MagicMock()
-    mock_settings.mcp_auth_token = "super-secret-token"
+    mock_settings.mcp_auth_token = SecretStr("super-secret-token")
 
     with patch("codex.config.get_settings", return_value=mock_settings):
         result = _require_http_token()
@@ -97,7 +98,7 @@ def test_main_http_with_token_calls_uvicorn() -> None:
 
     mock_settings = MagicMock()
     mock_settings.mcp_transport = "http"
-    mock_settings.mcp_auth_token = "test-token"
+    mock_settings.mcp_auth_token = SecretStr("test-token")
     mock_settings.mcp_host = "127.0.0.1"
     mock_settings.mcp_port = 8765
 

tests/parsing/test_mathpix.py

@@ -10,6 +10,7 @@ from typing import Any
 from unittest.mock import MagicMock, patch
 
 import pytest
+from pydantic import SecretStr
 
 from codex.models import FormulaChunk
 
@@ -295,8 +296,8 @@ class TestRouteSelection:
     def test_mathpix_selected_when_creds_present(self) -> None:
         """extract_formulas calls MathPix backend when both creds are set."""
         mock_settings = MagicMock()
-        mock_settings.mathpix_app_id = "my_id"
+        mock_settings.mathpix_app_id = SecretStr("my_id")
-        mock_settings.mathpix_app_key = "my_key"
+        mock_settings.mathpix_app_key = SecretStr("my_key")
         mock_settings.pix2tex_fallback = True
 
         with (

tests/parsing/test_tex.py

@@ -4,7 +4,13 @@ from __future__ import annotations
 
 import pytest
 
-from codex.parsing.tex import chunk_text, extract_sections, latex_to_text
+from codex.parsing.tex import (
+    chunk_sections,
+    chunk_text,
+    extract_sections,
+    flatten_inputs,
+    latex_to_text,
+)
 
 
 class TestExtractSections:
@@ -110,3 +116,62 @@ class TestLatexToText:
         result = latex_to_text(latex)
         assert r"\cite" not in result
         assert "%" not in result
+
+
+class TestFlattenInputs:
+    def test_inlines_input_and_include(self) -> None:
+        files = {
+            "main.tex": r"Start \input{sec/intro} mid \include{proof} end",
+            "sec/intro.tex": "INTRO BODY",
+            "proof.tex": "PROOF BODY",
+        }
+        out = flatten_inputs(files["main.tex"], files)
+        assert "INTRO BODY" in out
+        assert "PROOF BODY" in out
+        assert r"\input" not in out and r"\include" not in out
+
+    def test_resolves_without_tex_suffix(self) -> None:
+        # \input{intro} must resolve the archive member "intro.tex".
+        files = {"main.tex": r"\input{intro}", "intro.tex": "BODY"}
+        assert "BODY" in flatten_inputs(files["main.tex"], files)
+
+    def test_recursive_includes(self) -> None:
+        files = {"a.tex": r"X \input{b}", "b.tex": r"Y \input{c}", "c.tex": "Z"}
+        out = flatten_inputs(files["a.tex"], files)
+        assert "X" in out and "Y" in out and "Z" in out
+
+    def test_unresolved_input_dropped(self) -> None:
+        # A reference with no matching file is removed, not left as a directive.
+        out = flatten_inputs(r"A \input{missing} B", {})
+        assert r"\input" not in out
+        assert "A" in out and "B" in out
+
+    def test_input_cycle_does_not_leak_directive(self) -> None:
+        # a -> b -> a: the depth cap must strip the unresolved directive, not leak it.
+        files = {"a.tex": r"AA \input{b}", "b.tex": r"BB \input{a}"}
+        out = flatten_inputs(files["a.tex"], files)
+        assert r"\input" not in out
+        assert "AA" in out and "BB" in out
+
+    def test_dot_slash_prefix_resolves_without_overstrip(self) -> None:
+        # "./intro" resolves intro.tex; lstrip must not eat a real leading dot/run.
+        files = {"main.tex": r"\input{./intro}", "intro.tex": "BODY"}
+        assert "BODY" in flatten_inputs(files["main.tex"], files)
+
+
+class TestChunkSections:
+    def test_pairs_carry_section_titles(self) -> None:
+        latex = (
+            r"\section{Introduction}" + "\nIntro body text here.\n"
+            r"\section{Proof}" + "\nProof body text here.\n"
+        )
+        pairs = chunk_sections(latex)
+        titles = [t for t, _ in pairs]
+        assert "Introduction" in titles
+        assert "Proof" in titles
+        # No chunk spans a section boundary: each chunk belongs to one title.
+        assert all(isinstance(c, str) and c for _, c in pairs)
+
+    def test_fallback_titleless_when_no_sections(self) -> None:
+        pairs = chunk_sections("Plain text with no section commands at all here.")
+        assert pairs and all(title is None for title, _ in pairs)

tests/quality/test_quality.py

@@ -0,0 +1,302 @@
+"""Tests for codex.quality — chunk quality gate and section classification (F-16)."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from codex.quality import (
+    _bib_score,
+    _clean_title,
+    classify_section,
+    filter_chunks,
+    is_quality_chunk,
+    run_quality_pass,
+    section_label,
+)
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _settings(min_chars: int = 60, min_alpha: float = 0.40, max_bib: float = 0.70):
+    s = MagicMock()
+    s.chunk_min_chars = min_chars
+    s.chunk_min_alpha_ratio = min_alpha
+    s.chunk_max_bib_score = max_bib
+    return s
+
+
+GOOD_TEXT = (
+    "In differential geometry, the discrete Laplace–Beltrami operator "
+    "generalises the smooth operator to polyhedral surfaces by using cotangent "
+    "weights. This discretisation preserves key spectral properties."
+)
+
+BIB_TEXT = (
+    "Bobenko, A. and Pinkall, U. (2015) Discrete differential geometry. "
+    "doi:10.1145/2010324.1964978 Smith et al. (2018) Surface remeshing. "
+    "doi:10.1007/s00211-019-01082-w Jones et al. (2020) doi:10.1145/1234567.890123"
+)
+
+
+# ---------------------------------------------------------------------------
+# _bib_score
+# ---------------------------------------------------------------------------
+
+
+class TestBibScore:
+    def test_empty_string(self):
+        assert _bib_score("") == 0.0
+
+    def test_clean_prose_is_low(self):
+        assert _bib_score(GOOD_TEXT) < 0.10
+
+    def test_bib_like_text_is_high(self):
+        assert _bib_score(BIB_TEXT) > 0.70
+
+    def test_single_doi_short_text_spikes(self):
+        text = "See doi:10.1145/2010324.1964978 for details."
+        # 8 words, 1 DOI (weight 3) → raw = 3 / max(0.8, 1) = 3.0 → capped 1.0
+        assert _bib_score(text) == 1.0
+
+    def test_score_capped_at_one(self):
+        # _DOI_RE requires \d{4,} — use a valid registrant length
+        heavy = "10.1145/abcdef " * 20
+        assert _bib_score(heavy) == 1.0
+
+
+# ---------------------------------------------------------------------------
+# is_quality_chunk
+# ---------------------------------------------------------------------------
+
+
+class TestIsQualityChunk:
+    def test_passes_good_text(self):
+        assert is_quality_chunk(GOOD_TEXT, settings=_settings()) is True
+
+    def test_fails_too_short(self):
+        assert is_quality_chunk("Short.", settings=_settings(min_chars=60)) is False
+
+    def test_fails_low_alpha_ratio(self):
+        # Mostly digits and symbols
+        junk = "1234 5678 9012 3456 7890 |||| ==== ---- ____ .... ;;;; 0000 9999 8888 7777"
+        assert is_quality_chunk(junk, settings=_settings(min_alpha=0.40)) is False
+
+    def test_fails_high_bib_score(self):
+        assert is_quality_chunk(BIB_TEXT, settings=_settings(max_bib=0.70)) is False
+
+    def test_bib_threshold_is_strict_gt(self):
+        # A text that scores exactly at the threshold should still pass (> not >=)
+        s = _settings(max_bib=1.0)
+        assert is_quality_chunk(GOOD_TEXT, settings=s) is True
+
+    def test_exactly_min_chars_passes(self):
+        # check is `len < min_chars`, so exactly at the threshold passes
+        text = "a" * 60
+        assert is_quality_chunk(text, settings=_settings(min_chars=60)) is True
+
+    def test_one_below_min_chars_fails(self):
+        text = "a" * 59
+        assert is_quality_chunk(text, settings=_settings(min_chars=60)) is False
+
+
+# ---------------------------------------------------------------------------
+# classify_section
+# ---------------------------------------------------------------------------
+
+
+class TestClassifySection:
+    def test_abstract_header(self):
+        assert classify_section("Abstract\n\nThis paper presents...") == "abstract"
+
+    def test_abstract_german(self):
+        assert classify_section("Zusammenfassung\nDiese Arbeit...") == "abstract"
+
+    def test_intro_header(self):
+        assert classify_section("Introduction\nOver the past decade...") == "intro"
+
+    def test_intro_numbered(self):
+        assert classify_section("1. Introduction\nWe study...") == "intro"
+
+    def test_theorem_header(self):
+        assert classify_section("Theorem 3.1 (Existence). Let M be...") == "theorem"
+
+    def test_lemma_header(self):
+        assert classify_section("Lemma 2. If f is smooth then...") == "theorem"
+
+    def test_proof_header(self):
+        assert classify_section("Proof. We proceed by induction...") == "proof"
+
+    def test_bibliography_header(self):
+        assert classify_section("References\n[1] Bobenko, A....") == "bibliography"
+
+    def test_bibliography_german(self):
+        assert classify_section("Literatur\n[1] Müller, H....") == "bibliography"
+
+    def test_body_fallback(self):
+        assert classify_section("The cotangent weight for edge (i,j) is...") == "body"
+
+    def test_doi_density_fallback_to_bibliography(self):
+        # No "References" header but heavy DOI density → bibliography
+        dense = BIB_TEXT
+        assert classify_section(dense) == "bibliography"
+
+    def test_only_first_200_chars_checked(self):
+        # Pattern in position >200 should NOT be matched as section header
+        prefix = "x" * 201
+        text = prefix + "\nAbstract"
+        assert classify_section(text) == "body"
+
+
+# ---------------------------------------------------------------------------
+# filter_chunks
+# ---------------------------------------------------------------------------
+
+
+class TestFilterChunks:
+    def test_empty_list(self):
+        assert filter_chunks([], settings=_settings()) == []
+
+    def test_all_good(self):
+        chunks = [GOOD_TEXT, GOOD_TEXT + " More content here."]
+        assert filter_chunks(chunks, settings=_settings()) == chunks
+
+    def test_all_bad(self):
+        chunks = ["x", "y"]
+        assert filter_chunks(chunks, settings=_settings()) == []
+
+    def test_mixed(self):
+        chunks = [GOOD_TEXT, "Short.", GOOD_TEXT]
+        result = filter_chunks(chunks, settings=_settings())
+        assert len(result) == 2
+        assert all(c == GOOD_TEXT for c in result)
+
+
+# ---------------------------------------------------------------------------
+# run_quality_pass
+# ---------------------------------------------------------------------------
+
+
+def _make_mock_conn(rows: list[dict]) -> MagicMock:
+    """Return a mock psycopg connection whose execute().fetchall() returns rows."""
+    conn = MagicMock()
+    conn.execute.return_value.fetchall.return_value = rows
+    return conn
+
+
+class TestRunQualityPass:
+    def test_removes_bad_chunk(self):
+        conn = _make_mock_conn([{"id": 1, "content": "x"}])
+        stats = run_quality_pass(conn=conn, settings=_settings())
+        assert stats["removed"] == 1
+        assert stats["kept"] == 0
+        # The DELETE must have been called with chunk id=1
+        delete_calls = [c for c in conn.execute.call_args_list if "DELETE" in str(c)]
+        assert len(delete_calls) == 1
+
+    def test_tags_good_chunk(self):
+        conn = _make_mock_conn([{"id": 2, "content": GOOD_TEXT}])
+        stats = run_quality_pass(conn=conn, settings=_settings())
+        assert stats["kept"] == 1
+        assert stats["tagged"] == 1
+        assert stats["removed"] == 0
+        # UPDATE must have been called
+        update_calls = [c for c in conn.execute.call_args_list if "UPDATE" in str(c)]
+        assert len(update_calls) == 1
+
+    def test_mixed_batch(self):
+        rows = [
+            {"id": 1, "content": "x"},
+            {"id": 2, "content": GOOD_TEXT},
+            {"id": 3, "content": "y"},
+            {"id": 4, "content": GOOD_TEXT + " extra words here."},
+        ]
+        conn = _make_mock_conn(rows)
+        stats = run_quality_pass(conn=conn, settings=_settings())
+        assert stats["removed"] == 2
+        assert stats["kept"] == 2
+        assert stats["tagged"] == 2
+
+    def test_paper_id_scopes_query(self):
+        conn = _make_mock_conn([])
+        run_quality_pass(paper_id="2301.07041", conn=conn, settings=_settings())
+        first_call = conn.execute.call_args_list[0]
+        sql = first_call[0][0]
+        assert "paper_id" in sql
+
+    def test_all_papers_query_has_no_filter(self):
+        conn = _make_mock_conn([])
+        run_quality_pass(paper_id=None, conn=conn, settings=_settings())
+        first_call = conn.execute.call_args_list[0]
+        sql = first_call[0][0]
+        assert "paper_id" not in sql
+
+    def test_commit_is_called(self):
+        conn = _make_mock_conn([])
+        run_quality_pass(conn=conn, settings=_settings())
+        conn.commit.assert_called_once()
+
+    def test_section_label_stored_correctly(self):
+        conn = _make_mock_conn([{"id": 5, "content": "Abstract\n\n" + GOOD_TEXT}])
+        run_quality_pass(conn=conn, settings=_settings())
+        update_calls = [c for c in conn.execute.call_args_list if "UPDATE" in str(c)]
+        assert len(update_calls) == 1
+        kwargs = update_calls[0][0][1]
+        assert kwargs["section"] == "abstract"
+        assert kwargs["id"] == 5
+
+
+class TestCleanTitle:
+    def test_strips_leading_numbering(self):
+        assert _clean_title("3.2 Variational Principles") == "variational principles"
+
+    def test_strips_latex_and_braces(self):
+        assert _clean_title(r"The \emph{Discrete} Laplacian") == "the discrete laplacian"
+
+    def test_collapses_and_lowercases(self):
+        assert _clean_title("  Rigidity   Results  ") == "rigidity results"
+
+    def test_strips_accent_macros(self):
+        assert _clean_title(r"M\"obius Invariance") == "mobius invariance"
+
+    def test_strips_ties_and_accents(self):
+        assert _clean_title(r"Colin~de~Verdi\`ere") == "colin de verdiere"
+
+    def test_truncates_at_word_boundary(self):
+        long = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu omega"
+        out = _clean_title(long)
+        assert len(out) <= 60
+        assert not out.endswith(" ")
+        assert long.lower().startswith(out)  # whole words from the start, no partial tail
+
+
+class TestSectionLabel:
+    def test_bare_canonical_heading_maps_to_bucket(self):
+        # Only an EXACT canonical heading collapses to its controlled bucket.
+        assert section_label("Introduction", "body text") == "intro"
+        assert section_label("Proof", "body text") == "proof"
+        assert section_label("References", "body text") == "bibliography"
+        assert section_label("Abstract", "body text") == "abstract"
+
+    def test_descriptive_title_stored_verbatim(self):
+        # R-F: a heading that merely STARTS with a keyword keeps its real title,
+        # not the bucket — fixes classify_section prefix-match over-bucketing.
+        assert section_label("Preliminaries", "body text") == "preliminaries"
+        assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps"
+        assert section_label("Proof of Theorem 3", "x") == "proof of theorem 3"
+        assert section_label("Abstract Nonsense and Categories", "x") == (
+            "abstract nonsense and categories"
+        )
+        assert section_label("Introduction to Operator Algebras", "x") == (
+            "introduction to operator algebras"
+        )
+
+    def test_none_or_blank_title_falls_back_to_content(self):
+        # .pdf/.txt path: unchanged content-based classification.
+        assert section_label(None, "Theorem 1. Let X be ...") == "theorem"
+        assert section_label("", "ordinary body prose here") == "body"
+
+    def test_title_cleaning_to_empty_falls_back_to_content(self):
+        # A pure-numbering heading cleans to "" → classify by content instead.
+        assert section_label("3.2", "Lemma 2. Suppose ...") == "theorem"

tests/scaffold/test_config.py

@@ -8,7 +8,10 @@ from codex.config import Settings, get_settings
 def test_settings_defaults() -> None:
     """Settings() is constructable without env vars; expected defaults are set."""
     s = Settings()
-    assert s.database_url == "postgresql://researcher:change_me@localhost:5432/papers"
+    assert (
+        s.database_url.get_secret_value()
+        == "postgresql://researcher:change_me@localhost:5432/papers"
+    )
     assert s.embedding_model == "BAAI/bge-m3"
     assert s.embedding_dim == 1024
 
@@ -20,7 +23,7 @@ def test_settings_env_override(monkeypatch: object) -> None:
     mp: pytest.MonkeyPatch = monkeypatch  # type: ignore[assignment]
     mp.setenv("DATABASE_URL", "postgresql://x:y@h:5432/db")
     s = Settings()
-    assert s.database_url == "postgresql://x:y@h:5432/db"
+    assert s.database_url.get_secret_value() == "postgresql://x:y@h:5432/db"
 
 
 def test_get_settings_is_singleton() -> None:

tests/scaffold/test_db.py

@@ -33,3 +33,40 @@ def test_apply_schema_executes_sql() -> None:
 
     mock_conn.execute.assert_called_once()
     mock_conn.commit.assert_called_once()
+
+
+def test_schema_sql_is_idempotent() -> None:
+    """Every CREATE TABLE/INDEX in infra/schema.sql uses IF NOT EXISTS (audit M-1, T-2).
+
+    Re-applying schema.sql must be a safe no-op so the migration can run
+    repeatedly against an existing database.
+    """
+    import re
+    from pathlib import Path
+
+    schema = (Path(__file__).resolve().parents[2] / "infra" / "schema.sql").read_text()
+    # Drop line comments so commented-out example queries don't count.
+    code = "\n".join(line.split("--", 1)[0] for line in schema.splitlines())
+    non_idempotent = re.findall(
+        r"CREATE\s+(?:UNIQUE\s+)?(?:TABLE|INDEX)\s+(?!IF\s+NOT\s+EXISTS)(\w+)",
+        code,
+        re.IGNORECASE,
+    )
+    assert not non_idempotent, f"schema.sql has non-idempotent CREATE statements: {non_idempotent}"
+
+
+def test_migrate_command_reports_privilege_error() -> None:
+    """`codex migrate` surfaces InsufficientPrivilege with guidance and exits 1 (M-1)."""
+    import psycopg
+    from typer.testing import CliRunner
+
+    from codex.cli import app
+
+    def boom(*args: object, **kwargs: object) -> object:
+        raise psycopg.errors.InsufficientPrivilege("must be owner of table chunks")
+
+    with patch("psycopg.connect", boom):
+        result = CliRunner().invoke(app, ["migrate"])
+
+    assert result.exit_code == 1
+    assert "MIGRATION_DATABASE_URL" in result.output

tests/scripts/test_ra_grobid_backfill.py

@@ -0,0 +1,112 @@
+"""Tests for scripts.ra_grobid_backfill — the R-A GROBID references backfill.
+
+Only the pure, offline logic is covered (id normalization + Citation assembly);
+``extract_references`` is mocked so no GROBID server or DB is required.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+from codex.models import Citation
+from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations, main
+
+# ---------------------------------------------------------------------------
+# _normalize_cited_id
+# ---------------------------------------------------------------------------
+
+
+def test_normalize_doi_url_to_bare_lowercase() -> None:
+    assert _normalize_cited_id(doi="https://doi.org/10.1007/S00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_normalize_doi_prefixes() -> None:
+    assert _normalize_cited_id(doi="http://doi.org/10.1/X") == "10.1/x"
+    assert _normalize_cited_id(doi="doi:10.1/X") == "10.1/x"
+    assert _normalize_cited_id(doi="10.1/x") == "10.1/x"
+
+
+def test_normalize_arxiv_prefix_stripped() -> None:
+    assert _normalize_cited_id(arxiv_id="arXiv:2305.10988") == "2305.10988"
+    assert _normalize_cited_id(arxiv_id="2305.10988") == "2305.10988"
+    # Legacy ids keep their slash.
+    assert _normalize_cited_id(arxiv_id="math/0603097") == "math/0603097"
+
+
+def test_normalize_doi_takes_precedence_over_arxiv() -> None:
+    assert _normalize_cited_id(doi="10.5/y", arxiv_id="2305.10988") == "10.5/y"
+
+
+def test_normalize_empty_returns_none() -> None:
+    assert _normalize_cited_id() is None
+    assert _normalize_cited_id(doi="   ", arxiv_id="") is None
+
+
+# ---------------------------------------------------------------------------
+# build_grobid_citations
+# ---------------------------------------------------------------------------
+
+
+def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
+    return {"title": "t", "authors": "a", "year": "2020", "doi": doi, "arxiv_id": arxiv_id}
+
+
+def test_build_citations_normalizes_dedups_and_drops_self() -> None:
+    paper_id = "10.1007/s00454-019-00132-8"
+    refs = [
+        _ref(doi="https://doi.org/10.1/A"),  # → 10.1/a
+        _ref(doi="DOI:10.1/a"),  # duplicate of the above after normalization
+        _ref(arxiv_id="arXiv:2305.10988"),  # → 2305.10988
+        _ref(doi="", arxiv_id=""),  # no id → skipped
+        _ref(doi="https://doi.org/10.1007/S00454-019-00132-8"),  # self-cite → dropped
+    ]
+    with patch("scripts.ra_grobid_backfill.extract_references", return_value=refs) as mock_ex:
+        cits = build_grobid_citations(paper_id, "some.pdf", grobid_url="http://grobid")
+
+    mock_ex.assert_called_once_with("some.pdf", grobid_url="http://grobid", timeout=60.0)
+    assert all(isinstance(c, Citation) for c in cits)
+    assert all(c.citing_id == paper_id for c in cits)
+    assert [c.cited_id for c in cits] == ["10.1/a", "2305.10988"]
+
+
+def test_build_citations_empty_when_no_refs() -> None:
+    with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]):
+        assert build_grobid_citations("10.1/x", "x.pdf") == []
+
+
+# ---------------------------------------------------------------------------
+# main — connection scope (review finding #4)
+# ---------------------------------------------------------------------------
+
+
+def test_main_uses_separate_connections_for_read_and_write(tmp_path: Any) -> None:
+    """Review #4: the read connection is released BEFORE the slow GROBID loop, and a
+    fresh connection is opened for the write — so no idle-in-transaction connection
+    spans the per-paper network loop (which would abort on an SSH-tunnel drop)."""
+    (tmp_path / "paper.pdf").write_bytes(b"%PDF-1.4")
+    conns: list[MagicMock] = []
+
+    @contextmanager
+    def fake_get_conn() -> Any:
+        c = MagicMock()
+        c.execute.return_value.fetchall.return_value = [
+            {"id": "10.1/x", "source_path": "/d/paper.txt", "out_edges": 0}
+        ]
+        c.execute.return_value.fetchone.return_value = {"papers": 1, "citing": 1, "edges": 1}
+        conns.append(c)
+        yield c
+
+    with (
+        patch("scripts.ra_grobid_backfill.get_conn", side_effect=fake_get_conn),
+        patch("scripts.ra_grobid_backfill.build_grobid_citations", return_value=[]),
+    ):
+        rc = main(["--pdf-dir", str(tmp_path), "--write"])
+
+    assert rc == 0
+    # Two DISTINCT get_conn() context managers (read, then write) — not one held
+    # open across the GROBID loop.
+    assert len(conns) == 2

tests/scripts/test_rc_tex_reingest.py

@@ -0,0 +1,21 @@
+"""Tests for scripts.rc_tex_reingest."""
+
+from __future__ import annotations
+
+from scripts.rc_tex_reingest import is_arxiv_id
+
+
+def test_is_arxiv_id_accepts_modern_and_legacy() -> None:
+    assert is_arxiv_id("2305.10988")
+    assert is_arxiv_id("math/0603097")
+    assert is_arxiv_id("1005.2698")
+
+
+def test_is_arxiv_id_rejects_dois_and_wids() -> None:
+    assert not is_arxiv_id("10.1007/s00454-019-00132-8")
+    assert not is_arxiv_id("10.14279/depositonce-20357")
+    assert not is_arxiv_id("W2971636899")
+    assert not is_arxiv_id("https://openalex.org/W2971636899")
+    assert not is_arxiv_id("http://openalex.org/W2971636899")
+    assert not is_arxiv_id("10.48550/arXiv.2305.10988")  # arXiv DOI form, not bare id
+    assert not is_arxiv_id("")

tests/sources/test_arxiv.py

@@ -53,6 +53,94 @@ def test_fetch_source_returns_latex(monkeypatch: pytest.MonkeyPatch) -> None:
     assert "Hello world" in result
 
 
+def test_fetch_source_flattens_multifile(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Multi-file projects: \\input/\\include directives are inlined so the body
+    from the included files is returned, not just the primary skeleton (R-C)."""
+    main = (
+        b"\\documentclass{article}\\begin{document}"
+        b"\\input{sections/intro}\\include{proof}\\end{document}"
+    )
+    tar_bytes = _make_tar_gz(
+        {
+            "main.tex": main,
+            "sections/intro.tex": b"INTRODUCTION BODY TEXT",
+            "proof.tex": b"PROOF BODY TEXT",
+        }
+    )
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("2301.07041")
+
+    assert result is not None
+    assert "INTRODUCTION BODY TEXT" in result  # \input was inlined
+    assert "PROOF BODY TEXT" in result  # \include was inlined
+    assert "\\input" not in result and "\\include" not in result
+
+
+def test_fetch_source_bare_gzip_single_file(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Legacy arXiv submissions are served as a single bare gzipped .tex (no tar
+    wrapper); fetch_source must gunzip the body directly rather than fail."""
+    import gzip
+
+    latex = b"\\documentstyle[a4]{article}\n\\section{Intro}\nOld single-file paper body."
+    gz_bytes = gzip.compress(latex)
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(200, content=gz_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("math/0001176")
+
+    assert result is not None
+    assert "Old single-file paper body." in result
+
+
+def test_fetch_source_resolves_non_tex_include(monkeypatch: pytest.MonkeyPatch) -> None:
+    """\\input of a non-.tex body member (e.g. a .def file) is inlined, not dropped (R-C)."""
+    main = b"\\documentclass{article}\\begin{document}\\input{body.def}\\end{document}"
+    tar_bytes = _make_tar_gz({"main.tex": main, "body.def": b"NON TEX BODY TEXT"})
+
+    def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    result = arxiv.fetch_source("2301.07041")
+    assert result is not None
+    assert "NON TEX BODY TEXT" in result
+
+
+def test_fetch_source_ignores_commented_documentclass(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A commented %\\documentclass must not select a file as the primary document."""
+    decoy = b"% \\documentclass{article}\n\\section{Stub}\nDECOY ONLY"
+    real = b"\\documentclass{book}\\begin{document}REAL BODY HERE\\end{document}"
+    # decoy sorts/inserts first; without the fix it would win on the substring match.
+    tar_bytes = _make_tar_gz({"aaa_decoy.tex": decoy, "real.tex": real})
+
+    def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    result = arxiv.fetch_source("2301.07041")
+    assert result is not None
+    assert "REAL BODY HERE" in result
+    assert "DECOY ONLY" not in result
+
+
 # ---------------------------------------------------------------------------
 # fetch_source — 404 → None
 # ---------------------------------------------------------------------------
@@ -138,3 +226,63 @@ def test_fetch_pdf_url_pure_computation() -> None:
 
     url2 = arxiv.fetch_pdf_url("1234.56789")
     assert url2 == "https://arxiv.org/pdf/1234.56789.pdf"
+
+
+# ---------------------------------------------------------------------------
+# fetch_metadata — parse the Atom feed into a Paper (DQ-2)
+# ---------------------------------------------------------------------------
+
+_ATOM_FEED = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <entry>
+    <title>A discrete version of Liouville's theorem on conformal maps</title>
+    <summary>Liouville's theorem says that in dimension greater than two,
+    all conformal maps are Moebius transformations.</summary>
+    <published>2019-11-03T00:00:00Z</published>
+    <author><name>Ulrich Pinkall</name></author>
+    <author><name>Boris Springborn</name></author>
+  </entry>
+</feed>"""
+
+_ATOM_EMPTY = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"><title>ArXiv Query</title></feed>"""
+
+
+def test_fetch_metadata_parses_entry(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A valid Atom feed yields a populated Paper."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    paper = arxiv.fetch_metadata("1911.00966")
+    assert paper is not None
+    assert paper.id == "1911.00966"
+    assert paper.title.startswith("A discrete version of Liouville")
+    assert paper.authors == ["Ulrich Pinkall", "Boris Springborn"]
+    assert paper.year == 2019
+    assert paper.abstract and "conformal maps" in paper.abstract
+    assert paper.openalex_id is None
+
+
+def test_fetch_metadata_no_entry_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A feed with no <entry> (id not found) returns None."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_EMPTY, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    assert arxiv.fetch_metadata("0000.00000") is None
+
+
+def test_fetch_metadata_strips_arxiv_prefix(monkeypatch: pytest.MonkeyPatch) -> None:
+    """An ``arXiv:`` prefix is stripped so Paper.id is the bare id."""
+
+    def mock_get(url: str, **kwargs: object) -> httpx.Response:
+        return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+    paper = arxiv.fetch_metadata("arXiv:1911.00966")
+    assert paper is not None
+    assert paper.id == "1911.00966"

tests/sources/test_crossref.py

@@ -0,0 +1,104 @@
+"""Tests for codex.sources.crossref."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from codex.models import Citation
+from codex.sources import crossref
+
+
+def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A JATS-tagged abstract is returned as plain text with tags removed."""
+    body = {
+        "message": {
+            "abstract": "<jats:p>We prove a <jats:italic>theorem</jats:italic> about "
+            "polyhedra.</jats:p>"
+        }
+    }
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=body)
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+
+    result = crossref.fetch_abstract("10.1007/s00454-019-00132-8")
+    assert result == "We prove a theorem about polyhedra."
+
+
+def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A work with no abstract field (e.g. a book chapter) returns None."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}})
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None
+
+
+def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 returns None rather than raising."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "404",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_abstract("10.0000/nonexistent") is None
+
+
+# ---------------------------------------------------------------------------
+# fetch_references (roadmap R-B)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_references_returns_doi_edges(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Only references carrying a DOI become Citations; citing_id is the queried DOI."""
+    body = {
+        "message": {
+            "reference": [
+                {"key": "ref1", "DOI": "10.1090/s0002-9947-04-03545-7"},
+                {"key": "ref2", "unstructured": "A book with no DOI, 1992."},  # skipped
+                {"key": "ref3", "DOI": "10.1007/bf02392449", "article-title": "X"},
+            ]
+        }
+    }
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=body)
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    refs = crossref.fetch_references("10.1007/s00454-019-00132-8")
+
+    assert refs == [
+        Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1090/s0002-9947-04-03545-7"),
+        Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1007/bf02392449"),
+    ]
+
+
+def test_fetch_references_no_reference_key_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A work with no deposited reference list returns []."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"message": {"title": ["No refs deposited"]}})
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_references("10.1007/978-3-642-17413-1_7") == []
+
+
+def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A 404 returns [] rather than raising."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        raise httpx.HTTPStatusError(
+            "404",
+            request=httpx.Request("GET", url),
+            response=httpx.Response(404, request=httpx.Request("GET", url)),
+        )
+
+    monkeypatch.setattr(crossref, "_get", mock_get)
+    assert crossref.fetch_references("10.0000/nonexistent") == []

tests/sources/test_openalex.py

@@ -7,7 +7,7 @@ import pytest
 
 from codex.models import Citation, Paper
 from codex.sources import openalex
-from codex.sources.openalex import _resolve_id
+from codex.sources.openalex import _canonical_id, _normalize_doi, _resolve_id
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -15,7 +15,9 @@ from codex.sources.openalex import _resolve_id
 
 _SAMPLE_WORK = {
     "id": "https://openalex.org/W2741809807",
-    "doi": "10.1145/3592430",
+    # Real OpenAlex returns the DOI as a full URL, not a bare DOI (audit T-3 / DQ-5
+    # — the old bare fixture hid the URL-form behaviour both fixes address).
+    "doi": "https://doi.org/10.1145/3592430",
     "title": "Conformal Prediction: A Review",
     "publication_year": 2023,
     "authorships": [
@@ -67,6 +69,87 @@ def test_resolve_prefixed_unchanged() -> None:
     assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x"
 
 
+# ---------------------------------------------------------------------------
+# _normalize_doi — canonical bare, lower-cased form (DQ-5)
+# ---------------------------------------------------------------------------
+
+
+def test_normalize_doi_strips_https_url() -> None:
+    # The exact form OpenAlex returns in the `doi` field.
+    assert _normalize_doi("https://doi.org/10.1007/s00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_normalize_doi_strips_http_and_doi_prefix() -> None:
+    assert _normalize_doi("http://doi.org/10.1145/3592430") == "10.1145/3592430"
+    assert _normalize_doi("doi:10.1145/3592430") == "10.1145/3592430"
+
+
+def test_normalize_doi_lowercases() -> None:
+    # DOIs are case-insensitive; the canonical id (and _norm_cited_id) is lower-case.
+    assert _normalize_doi("https://doi.org/10.1007/S00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_normalize_doi_already_bare_is_idempotent() -> None:
+    assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8"
+
+
+# ---------------------------------------------------------------------------
+# _canonical_id — arXiv DOI → bare arXiv id (arXiv half of DQ-5)
+# ---------------------------------------------------------------------------
+
+
+def test_canonical_id_strips_arxiv_doi_modern() -> None:
+    # arXiv works carry the arXiv DOI; the canonical id is the bare arXiv id.
+    assert _canonical_id("https://doi.org/10.48550/arXiv.2305.10988") == "2305.10988"
+
+
+def test_canonical_id_strips_arxiv_doi_legacy() -> None:
+    assert _canonical_id("https://doi.org/10.48550/arXiv.math/0603097") == "math/0603097"
+
+
+def test_canonical_id_leaves_regular_doi_bare() -> None:
+    # A normal journal DOI is only normalized (bare + lower-case), not stripped.
+    assert _canonical_id("https://doi.org/10.1007/S00454-019-00132-8") == (
+        "10.1007/s00454-019-00132-8"
+    )
+
+
+def test_fetch_paper_arxiv_doi_yields_bare_id(monkeypatch: pytest.MonkeyPatch) -> None:
+    """An arXiv work maps to the BARE arXiv id (not the arXiv DOI), so re-ingesting
+    by arXiv id hits ON CONFLICT (id) instead of tripping papers_openalex_id_key."""
+    work = {**_SAMPLE_WORK, "doi": "https://doi.org/10.48550/arXiv.2305.10988"}
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=work)
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+    paper = openalex.fetch_paper("2305.10988")
+
+    assert paper is not None
+    assert paper.id == "2305.10988"
+
+
+def test_map_paper_falls_back_to_openalex_id_without_doi(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A work with no DOI keeps the OpenAlex W-id (URL form) as id, unchanged —
+    normalization only rewrites the DOI branch."""
+    work = {k: v for k, v in _SAMPLE_WORK.items() if k != "doi"}
+
+    def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
+        return httpx.Response(200, json=work)
+
+    monkeypatch.setattr(openalex, "_get", mock_get)
+    paper = openalex.fetch_paper("W2741809807")
+
+    assert paper is not None
+    assert paper.id == "https://openalex.org/W2741809807"
+
+
 # ---------------------------------------------------------------------------
 # fetch_paper — success
 # ---------------------------------------------------------------------------
@@ -85,6 +168,12 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
 
     assert paper is not None
     assert isinstance(paper, Paper)
+    # DQ-5: openalex._canonical_id normalizes OpenAlex's URL-form doi to the BARE
+    # lower-cased DOI at the source layer; ingest additionally pins papers.id to the
+    # caller's id (audit C-7). Both converge on the bare canonical id — which is what
+    # makes the ingest ON CONFLICT (id) upsert idempotent. (T-3: the fixture is
+    # URL-form so this mapping can't silently regress.)
+    assert paper.id == "10.1145/3592430"
     assert paper.title == "Conformal Prediction: A Review"
     assert paper.year == 2023
     assert paper.authors == ["Alice Smith", "Bob Jones"]

tests/sources/test_semanticscholar.py

@@ -124,3 +124,46 @@ def test_fetch_recommendations_404_returns_empty(
 
     result = semanticscholar.fetch_recommendations("nonexistent")
     assert result == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_references — {"data": null} guard (DQ-1 regression)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_references_null_data_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
+    parsed references; this must yield [] rather than raising TypeError."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"data": None, "citingPaperInfo": {}})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_references("DOI:10.14279/depositonce-5415") == []
+
+
+# ---------------------------------------------------------------------------
+# fetch_abstract (DQ-2)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_abstract_returns_text(monkeypatch: pytest.MonkeyPatch) -> None:
+    """fetch_abstract returns the abstract string when present."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"abstract": "We study ideal hyperbolic polyhedra."})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_abstract("DOI:10.1007/s00454-019-00132-8") == (
+        "We study ideal hyperbolic polyhedra."
+    )
+
+
+def test_fetch_abstract_null_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A null/absent abstract yields None."""
+
+    def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
+        return httpx.Response(200, json={"abstract": None})
+
+    monkeypatch.setattr(semanticscholar, "_get", mock_get)
+    assert semanticscholar.fetch_abstract("arXiv:1234.5678") is None

tests/synthesis/test_cli.py

@@ -55,9 +55,7 @@ def isolated_leads_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
 
     monkeypatch.setattr("codex.cli.get_settings", lambda: settings, raising=False)
     monkeypatch.setattr("codex.synthesis.get_settings", lambda: settings)
-    monkeypatch.setattr(
+    monkeypatch.setattr("codex.config.get_settings", lambda: settings, raising=False)
-        "codex.config.get_settings", lambda: settings, raising=False
-    )
     return leads_path
 
 

tests/synthesis/test_gaps.py

@@ -112,10 +112,35 @@ def test_find_gaps_coverage_map_low_coverage_triggers_llm(
         "For an ideal tetrahedron with dihedral angles the hyperbolic volume is "
         "V = L(gamma1) + L(gamma2) + L(gamma3). [Springborn2008 #chunk 16]\n"
     )
-    leads = find_gaps(llm=StubLLM(grounded_body))
+    # Coverage gaps are opt-in (audit R-8): pass the topic explicitly.
+    leads = find_gaps(llm=StubLLM(grounded_body), topics=["hyperbolic volume formula"])
     assert any(lead.kind == "gap" for lead in leads)
 
 
+def test_find_gaps_no_coverage_gaps_without_explicit_topics(
+    monkeypatch: pytest.MonkeyPatch,
+    mock_paper_titles: None,
+    mock_settings: Path,
+) -> None:
+    """Without explicit topics, paper titles do NOT auto-generate coverage gaps (R-8)."""
+    sparse_chunks: list[dict[str, Any]] = [
+        {
+            "id": 10,
+            "paper_id": "springborn-2008",
+            "ord": 16,
+            "content": "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)",
+            "bibkey": "Springborn2008",
+        },
+    ]
+    monkeypatch.setattr(
+        "codex.synthesis._retrieve_chunks",
+        lambda queries, top_k: [dict(c) for c in sparse_chunks],
+    )
+    # No lib_path, no topics → the coverage map must not run despite sparse coverage.
+    leads = find_gaps(llm=StubLLM("a gap. [Springborn2008 #chunk 16]"))
+    assert leads == []
+
+
 def test_find_gaps_explicit_topics_override_default(
     monkeypatch: pytest.MonkeyPatch,
     mock_paper_titles: None,

tests/synthesis/test_grounded.py

@@ -76,7 +76,7 @@ def test_finalise_grounded_lead_succeeds_when_grounded(
     )
     assert lead is not None
     assert lead.kind == "connection"
-    assert lead.id == "L-0001"
+    assert lead.id == "L-C-0001"  # kind-prefixed id (audit C-11)
     assert lead.confidence >= 0.5
     assert lead.status == "unverified"
     assert lead.suggested_validation == ""
@@ -148,7 +148,8 @@ def test_find_gaps_topic_with_no_chunks(
 ) -> None:
     """When retrieval returns no chunks, a gap lead is emitted directly."""
     monkeypatch.setattr("codex.synthesis._retrieve_chunks", lambda queries, top_k: [])
-    leads = find_gaps(llm=StubLLM(""))
+    # Coverage gaps are opt-in now (audit R-8): pass an explicit topic.
+    leads = find_gaps(llm=StubLLM(""), topics=["nonexistent topic"])
     assert len(leads) >= 1
     assert all(lead.kind == "gap" for lead in leads)
     # Direct gap leads carry a validation hint (re-ingest)

tests/synthesis/test_model.py

@@ -92,9 +92,21 @@ def test_lead_empty_provenance_raises() -> None:
 
 
 def test_lead_id_format_helper() -> None:
-    """The internal _make_lead_id helper produces zero-padded L-XXXX strings."""
+    """_make_lead_id produces a kind-prefixed, zero-padded L-<K>-XXXX string."""
     from codex.synthesis import _make_lead_id
 
-    assert _make_lead_id(1) == "L-0001"
+    assert _make_lead_id(1, "connection") == "L-C-0001"
-    assert _make_lead_id(42) == "L-0042"
+    assert _make_lead_id(42, "gap") == "L-G-0042"
-    assert _make_lead_id(9999) == "L-9999"
+    assert _make_lead_id(9999, "improvement") == "L-I-9999"
+    assert _make_lead_id(1, "conjecture") == "L-X-0001"
+
+
+def test_lead_id_namespaced_per_kind_no_collision() -> None:
+    """Regression for audit C-11: each stage numbers from 1, so without a kind
+    prefix connection/gap/improvement all collide on L-0001 and overwrite one
+    another in grounded/. The prefix must keep same-seq ids distinct.
+    """
+    from codex.synthesis import _make_lead_id
+
+    ids = {_make_lead_id(1, k) for k in ("connection", "gap", "improvement", "conjecture")}
+    assert len(ids) == 4, f"same-seq ids must stay distinct across kinds, got {ids}"

tests/wiki/test_compile.py

@@ -201,9 +201,7 @@ def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored()
     This test verifies that a claim whose text = the last sentence grounds correctly.
     """
     # The last sentence of the claim text is directly grounded in the fixture chunk.
-    grounded_sentence = (
+    grounded_sentence = "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)"
-        "the hyperbolic volume is V = L(gamma1) + L(gamma2) + L(gamma3)"
-    )
     claim = Claim(
         text=grounded_sentence,
         bibkey="Springborn2008",
@@ -215,6 +213,47 @@ def test_grounding_guard_last_sentence_grounded_earlier_hallucination_ignored()
     )
 
 
+def test_grounding_guard_robust_to_trailing_comma() -> None:
+    """A faithful 5-word claim grounds despite a trailing comma (audit R-1).
+
+    The chunk says "... the volume function V0 is strictly concave on the angle
+    domain." Before edge-punctuation stripping, the claim token "concave," != the
+    chunk token "concave", so the only content-5-gram failed to match and the
+    claim was wrongly marked ungrounded.
+    """
+    claim = Claim(
+        text="the volume function V0 is strictly concave,",
+        bibkey="Springborn2008",
+        locator="chunk 16",
+    )
+    result = _run_grounding_guard([claim], FIXTURE_CHUNKS)
+    assert result[0].grounded is True
+
+
+def test_grounding_guard_no_substring_bleed_across_tokens() -> None:
+    """A 5-gram must match whole tokens, not bleed into a longer chunk word (audit C-6).
+
+    The chunk contains 'subset'; a claim whose first token is 'set' must NOT ground
+    by substring-matching the suffix of 'subset'.
+    """
+    chunks = [
+        {
+            "id": 99,
+            "paper_id": "p",
+            "ord": 0,
+            "bibkey": "TestBib",
+            "content": "the subset conformal map energy functional is minimized here",
+        }
+    ]
+    claim = Claim(
+        text="the set conformal map energy functional",
+        bibkey="TestBib",
+        locator="chunk 0",
+    )
+    result = _run_grounding_guard([claim], chunks)
+    assert result[0].grounded is False
+
+
 # ---------------------------------------------------------------------------
 # _inject_cross_refs
 # ---------------------------------------------------------------------------
@@ -276,6 +315,19 @@ def test_inject_cross_refs_skips_inline_code() -> None:
     assert "[[circle-packing]]" in result  # bare occurrence replaced
 
 
+def test_inject_cross_refs_skips_math_spans() -> None:
+    """Concept names inside LaTeX math are NOT rewritten to [[slug]] links (audit R-9)."""
+    concepts = [
+        Concept(slug="energy", title="Energy", aliases=[]),
+        FIXTURE_CONCEPT,
+    ]
+    text = "Prose Energy here. Math $x = Energy^2$ and inline \\(Energy\\) stay literal."
+    result = _inject_cross_refs(text, concepts, current_slug="lobachevsky-function")
+    assert "$x = Energy^2$" in result  # display/inline $...$ math untouched
+    assert "\\(Energy\\)" in result  # \(...\) math untouched
+    assert "Prose [[energy]] here." in result  # prose occurrence still linked
+
+
 def test_inject_cross_refs_full_list_even_when_single_concept() -> None:
     """Cross-ref injection uses the full concept list, not just the compiled concept."""
     concepts = [
@@ -506,6 +558,47 @@ def test_log_append_records_skipped(
     assert "circle-packing" in content
 
 
+# ---------------------------------------------------------------------------
+# Audit C-4: zero-citation pages must NOT bypass the quarantine
+# ---------------------------------------------------------------------------
+
+
+def test_zero_citation_page_is_quarantined_not_published(
+    mock_retrieve: None,
+    mock_formulas: None,
+    mock_settings: Path,
+) -> None:
+    """A confident-but-uncited LLM response yields zero parseable claims
+    (grounding_rate 0.0). It must be quarantined — not written to wiki/ and
+    marked compiled. Regression for audit C-4: the old ``total_claims > 0``
+    guard let zero-claim (and LLM-outage empty) pages fall through to wiki/.
+    """
+    from codex.wiki import compile_all
+
+    (mock_settings / "concepts.yaml").write_text(
+        textwrap.dedent(
+            """\
+            concepts:
+              - slug: lobachevsky-function
+                title: Lobachevsky Function
+                aliases: [Milnor Lobachevsky]
+            """
+        ),
+        encoding="utf-8",
+    )
+    # No [BibKey] citations anywhere → zero claims → grounding_rate 0.0.
+    uncited = MockLLM(
+        "The Lobachevsky function is obviously central to all of geometry. "
+        "Every result follows from it. This paragraph cites nothing at all."
+    )
+
+    report = compile_all(changed_only=False, llm=uncited, output_dir=str(mock_settings))
+
+    assert "lobachevsky-function" in report.quarantined
+    assert "lobachevsky-function" not in report.compiled
+    assert not (mock_settings / "lobachevsky-function.md").exists()
+
+
 # ---------------------------------------------------------------------------
 # Idempotency: second compile without chunk change → no rewrite
 # ---------------------------------------------------------------------------
@@ -523,9 +616,7 @@ def test_idempotent_no_rewrite_when_unchanged(
     call_count = 0
     original_compile = compile_concept
 
-    def counting_compile(
+    def counting_compile(concept: Concept, chunks: list[Any], **kwargs: Any) -> ConceptPage:
-        concept: Concept, chunks: list[Any], **kwargs: Any
-    ) -> ConceptPage:
         nonlocal call_count
         call_count += 1
         return original_compile(concept, chunks, **kwargs)
@@ -569,9 +660,7 @@ def test_force_all_recompiles_even_when_unchanged(
     call_count = 0
     original_compile = compile_concept
 
-    def counting_compile(
+    def counting_compile(concept: Concept, chunks: list[Any], **kwargs: Any) -> ConceptPage:
-        concept: Concept, chunks: list[Any], **kwargs: Any
-    ) -> ConceptPage:
         nonlocal call_count
         call_count += 1
         return original_compile(concept, chunks, **kwargs)