merge: reconcile data-quality roadmap with audit-remediation main (#12-15)

Resolved 4 conflicts: cli.py (R-D citing-coverage warning + main's small-corpus JSON/hub-title changes both kept); ingest.py (C-7 paper.id=caller-id pin first, then DQ-2 abstract recovery); test_ingest.py + test_openalex.py (kept both branches' tests).

C-7/DQ-5 overlap (both canonicalise papers.id): kept both layers — openalex._canonical_id normalizes fetch_paper's id (DQ-5); ingest pins papers.id to the caller's bare id (C-7); they converge on the bare canonical id. Fixed the auto-merged test_fetch_paper_success that had contradictory bare/URL assertions. 402 tests pass; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codex/cli.py

@@ -29,6 +29,37 @@ app.add_typer(graph_app, name="graph")
 app.add_typer(search_app, name="search")
 
 
+@app.command()
+def migrate() -> None:
+    """Apply infra/schema.sql to bring the database schema up to date (idempotent).
+
+    Must connect as a role that can run DDL (owns / can CREATE + ALTER the tables).
+    The application's DATABASE_URL is usually a least-privilege DML role and will
+    fail with InsufficientPrivilege; set MIGRATION_DATABASE_URL to a privileged
+    (owner/superuser) connection (audit M-1).
+    """
+    import psycopg
+    import psycopg.rows
+
+    from codex.config import get_settings
+    from codex.db import apply_schema
+
+    settings = get_settings()
+    url = (settings.migration_database_url or settings.database_url).get_secret_value()
+    try:
+        with psycopg.connect(url, row_factory=psycopg.rows.dict_row) as conn:
+            apply_schema(conn)
+    except psycopg.errors.InsufficientPrivilege as exc:
+        typer.echo(
+            "ERROR: schema migration needs a role that owns the tables "
+            "(CREATE/ALTER). Set MIGRATION_DATABASE_URL to a privileged connection "
+            f"and retry. ({exc})",
+            err=True,
+        )
+        raise typer.Exit(1) from exc
+    typer.echo("Schema applied — database is up to date.")
+
+
 @app.command()
 def ingest(
     paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
@@ -71,7 +102,8 @@ def search_paper(
     cite_boost: bool = typer.Option(
         False,
         "--cite-boost",
-        help="Weight results by citation PageRank (F-15). Graceful when corpus < 5 papers.",
+        help="Re-rank the top results by citation PageRank — a within-page "
+        "tie-breaker, not a hard re-ranking (F-15). Graceful when corpus < 5 papers.",
     ),
 ) -> None:
     """Semantic similarity search over paper abstracts."""
@@ -157,6 +189,24 @@ def discover_leads(
         typer.echo(f"{item['pull']:>4}×  {item['cited_id']}")
 
 
+@discover_app.command("recommend")
+def discover_recommend(
+    paper_id: str = typer.Argument(
+        ..., help="Semantic Scholar paper ID (or arXiv:/DOI: prefixed)."
+    ),
+    limit: int = typer.Option(20, "--limit", "-n", help="Number of recommendations."),
+) -> None:
+    """Recommended papers for PAPER_ID via Semantic Scholar."""
+    from codex.sources.semanticscholar import fetch_recommendations
+
+    rec = fetch_recommendations(paper_id, limit=limit)
+    if not rec:
+        typer.echo(f"No recommendations found for {paper_id}.")
+        return
+    for pid in rec:
+        typer.echo(pid)
+
+
 @discover_app.command("citing")
 def discover_citing(paper_id: str = typer.Argument(...)) -> None:
     """List papers that cite PAPER_ID."""
@@ -391,7 +441,15 @@ def synthesis_leads(
         "--lib-path",
         help=(
             "Path to a C++ source tree. Enables improvement leads (@cite-aware) and "
-            "code-vs-corpus gap detection. Without it only connection + topic gaps run."
+            "code-vs-corpus gap detection."
+        ),
+    ),
+    topic: list[str] | None = typer.Option(  # noqa: B008
+        None,
+        "--topic",
+        help=(
+            "Topic to check for coverage gaps (repeatable). Opt-in: without --topic "
+            "no coverage-map gaps run (avoids per-paper false positives, audit R-8)."
         ),
     ),
     output_dir: Optional[str] = typer.Option(  # noqa: UP045
@@ -413,7 +471,7 @@ def synthesis_leads(
     llm = default_llm_client()
 
     connections = find_connections(top_k=settings.synthesis_top_k, llm=llm)
-    gaps = find_gaps(lib_path=lib_path, llm=llm)
+    gaps = find_gaps(lib_path=lib_path, llm=llm, topics=topic or None)
     improvements = (
         find_improvements(lib_path, top_k=settings.synthesis_top_k, llm=llm) if lib_path else []
     )
@@ -548,7 +606,9 @@ def graph_report(
     settings = get_settings()
     with get_conn() as conn:
         graph = build_citation_graph(conn)
-        known_ids = {row["id"] for row in conn.execute("SELECT id FROM papers").fetchall()}
+        paper_rows = conn.execute("SELECT id, title FROM papers").fetchall()
+    known_ids = {row["id"] for row in paper_rows}
+    titles = {row["id"]: row["title"] for row in paper_rows}
 
     if graph.number_of_nodes() == 0:
         typer.echo("Citation graph is empty — ingest some papers first.")
@@ -560,6 +620,13 @@ def graph_report(
     dangling = sorted(dangling_citations(graph, known_ids))
     n_citing, _ = citing_coverage(graph, known_ids)
     coverage = n_citing / n_papers if n_papers else 0.0
+    small_corpus = n_papers < settings.graph_min_corpus_size
+    warning = (
+        f"only {n_papers} ingested papers; recommended >= {settings.graph_min_corpus_size} "
+        "for meaningful ranking"
+        if small_corpus
+        else None
+    )
 
     if output_json:
         typer.echo(
@@ -572,7 +639,11 @@ def graph_report(
                         "papers": n_papers,
                         "fraction": round(coverage, 4),
                     },
-                    "hubs": [{"paper_id": pid, "pagerank": score} for pid, score in hubs],
+                    "small_corpus_warning": warning,  # null unless below threshold (audit U-3)
+                    "hubs": [
+                        {"paper_id": pid, "title": titles.get(pid), "pagerank": score}
+                        for pid, score in hubs
+                    ],
                     "dangling_count": len(dangling),
                     "dangling": dangling,
                 },
@@ -581,12 +652,8 @@ def graph_report(
         )
         return
 
-    if n_papers < settings.graph_min_corpus_size:
-        typer.echo(
-            f"Warning: only {n_papers} ingested papers "
-            f"(recommended ≥ {settings.graph_min_corpus_size} for meaningful ranking).",
-            err=True,
-        )
+    if warning:
+        typer.echo(f"Warning: {warning}.", err=True)
     # R-D: low citing-paper coverage starves PageRank/coupling even at a healthy
     # paper count — warn on the share of papers that actually have out-edges.
     if coverage < settings.graph_min_citing_coverage:
@@ -602,7 +669,9 @@ def graph_report(
     typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)")
     typer.echo("-" * 48)
     for pid, score in hubs:
-        typer.echo(f"  {score:.4f}  {pid}")
+        # in-KB hubs show their title; dangling (OpenAlex-id) hubs are flagged (U-1)
+        label = titles.get(pid) or "(dangling — not ingested)"
+        typer.echo(f"  {score:.4f}  {pid}  {label}")
     typer.echo("")
     typer.echo(f"DANGLING CITATIONS ({len(dangling)} cited but not ingested)")
     typer.echo("-" * 48)
@@ -635,3 +704,23 @@ def graph_related(
     )
     for pid in related:
         typer.echo(f"  {pid}")
+
+
+@graph_app.command("cocited")
+def graph_cocited(
+    paper_id: str = typer.Argument(..., help="Paper ID to find co-cited papers for."),
+) -> None:
+    """List papers frequently co-cited with PAPER_ID (graph co-citation)."""
+    from codex.db import get_conn
+    from codex.graph import build_citation_graph, find_co_cited
+
+    with get_conn() as conn:
+        graph = build_citation_graph(conn)
+
+    results = find_co_cited(paper_id, graph)
+    if not results:
+        typer.echo(f"No papers co-cited with {paper_id}.")
+        return
+    typer.echo(f"Papers co-cited with {paper_id} (count desc):")
+    for pid, count in results:
+        typer.echo(f"  {count:>3}×  {pid}")