45c3b16dd7
- C-14: embed.py header now states that only the dense path is wired (search 'hybrid' is dense + Postgres FTS); encode_sparse/encode are reserved for a future sparse-retrieval layer, not yet consumed. - U-2: --cite-boost help clarifies it re-ranks within the top results (a tie-breaker), not a hard re-ranking — matching the small alpha effect. Accepted (documented heuristics, no change): R-3 conflict detection is a keyword-only signal labelled as such; R-12 classify_section is mostly 'body' because chunks are word-windows that rarely start at a section header. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
705 lines
24 KiB
Python
705 lines
24 KiB
Python
"""codex CLI — commands wired to domain modules."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import UTC, datetime
|
||
from typing import Optional
|
||
|
||
import typer
|
||
|
||
app = typer.Typer(help="codex — personal knowledge base for scientific papers.")
|
||
discover_app = typer.Typer(help="Discovery queries over the citation graph.")
|
||
prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||
synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
|
||
quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
|
||
graph_app = typer.Typer(help="Citation graph analytics: PageRank, coupling, dangling leads (F-15).")
|
||
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
|
||
search_app = typer.Typer(
|
||
help="Search: semantic paper search and formula full-text search.",
|
||
invoke_without_command=True,
|
||
)
|
||
app.add_typer(discover_app, name="discover")
|
||
app.add_typer(prov_app, name="provenance")
|
||
app.add_typer(wiki_app, name="wiki")
|
||
app.add_typer(synth_app, name="synthesis")
|
||
app.add_typer(quality_app, name="quality")
|
||
app.add_typer(graph_app, name="graph")
|
||
app.add_typer(search_app, name="search")
|
||
|
||
|
||
@app.command()
|
||
def migrate() -> None:
|
||
"""Apply infra/schema.sql to bring the database schema up to date (idempotent).
|
||
|
||
Must connect as a role that can run DDL (owns / can CREATE + ALTER the tables).
|
||
The application's DATABASE_URL is usually a least-privilege DML role and will
|
||
fail with InsufficientPrivilege; set MIGRATION_DATABASE_URL to a privileged
|
||
(owner/superuser) connection (audit M-1).
|
||
"""
|
||
import psycopg
|
||
import psycopg.rows
|
||
|
||
from codex.config import get_settings
|
||
from codex.db import apply_schema
|
||
|
||
settings = get_settings()
|
||
url = (settings.migration_database_url or settings.database_url).get_secret_value()
|
||
try:
|
||
with psycopg.connect(url, row_factory=psycopg.rows.dict_row) as conn:
|
||
apply_schema(conn)
|
||
except psycopg.errors.InsufficientPrivilege as exc:
|
||
typer.echo(
|
||
"ERROR: schema migration needs a role that owns the tables "
|
||
"(CREATE/ALTER). Set MIGRATION_DATABASE_URL to a privileged connection "
|
||
f"and retry. ({exc})",
|
||
err=True,
|
||
)
|
||
raise typer.Exit(1) from exc
|
||
typer.echo("Schema applied — database is up to date.")
|
||
|
||
|
||
@app.command()
|
||
def ingest(
|
||
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
|
||
source: Optional[str] = typer.Option(None, "--source", "-s", help="Path to .tex or .pdf"), # noqa: UP045
|
||
rich: bool = typer.Option( # noqa: A002
|
||
False,
|
||
"--rich",
|
||
help="Extract formulas and figures (requires PDF source, F-09).",
|
||
),
|
||
) -> None:
|
||
"""Ingest a paper into the knowledge base."""
|
||
from codex.ingest import ingest_paper
|
||
|
||
result = ingest_paper(paper_id, source_path=source, rich=rich)
|
||
typer.echo(
|
||
f"Ingested {result.paper_id}: {result.chunks_upserted} chunks, "
|
||
f"{result.citations_upserted} citations, "
|
||
f"{result.formulas_upserted} formulas, "
|
||
f"{result.figures_upserted} figures"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F-09: Search command group
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@search_app.callback(invoke_without_command=True)
|
||
def search_callback(ctx: typer.Context) -> None:
|
||
"""Search subcommands: ``paper`` for semantic search, ``formula`` for LaTeX FTS."""
|
||
if ctx.invoked_subcommand is None:
|
||
typer.echo(ctx.get_help())
|
||
raise typer.Exit(0)
|
||
|
||
|
||
@search_app.command("paper")
|
||
def search_paper(
|
||
query: str = typer.Argument(..., help="Natural-language search query"),
|
||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||
cite_boost: bool = typer.Option(
|
||
False,
|
||
"--cite-boost",
|
||
help="Re-rank the top results by citation PageRank — a within-page "
|
||
"tie-breaker, not a hard re-ranking (F-15). Graceful when corpus < 5 papers.",
|
||
),
|
||
) -> None:
|
||
"""Semantic similarity search over paper abstracts."""
|
||
from codex.config import get_settings
|
||
from codex.db import get_conn
|
||
from codex.embed import get_embedder
|
||
|
||
emb = get_embedder().encode_dense([query])[0].tolist()
|
||
with get_conn() as conn:
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT id, title, year,
|
||
abstract_emb <-> %(emb)s AS distance
|
||
FROM papers
|
||
WHERE abstract_emb IS NOT NULL
|
||
ORDER BY abstract_emb <-> %(emb)s
|
||
LIMIT %(limit)s
|
||
""",
|
||
{"emb": emb, "limit": limit},
|
||
).fetchall()
|
||
|
||
if cite_boost and rows:
|
||
from codex.graph import build_citation_graph, citation_pagerank
|
||
|
||
settings = get_settings()
|
||
graph = build_citation_graph(conn)
|
||
pr = citation_pagerank(graph)
|
||
alpha = settings.graph_cite_boost_alpha
|
||
results = []
|
||
for row in rows:
|
||
pr_score = pr.get(row["id"], 0.0)
|
||
# distance is lower=better; divide to reduce distance for high-PR papers
|
||
boosted = row["distance"] / (1.0 + alpha * pr_score)
|
||
results.append((boosted, row))
|
||
results.sort(key=lambda x: x[0])
|
||
for boosted_dist, row in results:
|
||
typer.echo(f"[{boosted_dist:.3f}] {row['id']} ({row['year']}) — {row['title']}")
|
||
return
|
||
|
||
for row in rows:
|
||
typer.echo(f"[{row['distance']:.3f}] {row['id']} ({row['year']}) — {row['title']}")
|
||
|
||
|
||
@search_app.command("formula")
|
||
def search_formula(
|
||
query: str = typer.Argument(..., help="LaTeX snippet or keyword to search in formulas"),
|
||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||
) -> None:
|
||
"""Full-text search over extracted LaTeX formulas (F-09)."""
|
||
from codex.db import get_conn
|
||
|
||
with get_conn() as conn:
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT paper_id, page, raw_latex, context,
|
||
ts_rank(to_tsvector('english', raw_latex || ' ' || coalesce(context, '')),
|
||
plainto_tsquery('english', %(query)s)) AS rank
|
||
FROM formulas
|
||
WHERE to_tsvector('english', raw_latex || ' ' || coalesce(context, ''))
|
||
@@ plainto_tsquery('english', %(query)s)
|
||
ORDER BY rank DESC
|
||
LIMIT %(limit)s
|
||
""",
|
||
{"query": query, "limit": limit},
|
||
).fetchall()
|
||
if not rows:
|
||
typer.echo("No matching formulas found.")
|
||
return
|
||
for row in rows:
|
||
typer.echo(
|
||
f"[{row['rank']:.3f}] {row['paper_id']} p.{row['page']} {row['raw_latex'][:80]}"
|
||
)
|
||
|
||
|
||
@discover_app.command("leads")
|
||
def discover_leads(
|
||
limit: int = typer.Option(20, "--limit", "-n"),
|
||
) -> None:
|
||
"""Show discovery leads: cited papers not yet ingested, ranked by pull."""
|
||
from codex.discover import discovery_leads
|
||
|
||
for item in discovery_leads(limit=limit):
|
||
typer.echo(f"{item['pull']:>4}× {item['cited_id']}")
|
||
|
||
|
||
@discover_app.command("recommend")
|
||
def discover_recommend(
|
||
paper_id: str = typer.Argument(
|
||
..., help="Semantic Scholar paper ID (or arXiv:/DOI: prefixed)."
|
||
),
|
||
limit: int = typer.Option(20, "--limit", "-n", help="Number of recommendations."),
|
||
) -> None:
|
||
"""Recommended papers for PAPER_ID via Semantic Scholar."""
|
||
from codex.sources.semanticscholar import fetch_recommendations
|
||
|
||
rec = fetch_recommendations(paper_id, limit=limit)
|
||
if not rec:
|
||
typer.echo(f"No recommendations found for {paper_id}.")
|
||
return
|
||
for pid in rec:
|
||
typer.echo(pid)
|
||
|
||
|
||
@discover_app.command("citing")
|
||
def discover_citing(paper_id: str = typer.Argument(...)) -> None:
|
||
"""List papers that cite PAPER_ID."""
|
||
from codex.discover import citing_papers
|
||
|
||
for pid in citing_papers(paper_id):
|
||
typer.echo(pid)
|
||
|
||
|
||
@discover_app.command("cited-by")
|
||
def discover_cited_by(paper_id: str = typer.Argument(...)) -> None:
|
||
"""List papers that PAPER_ID cites."""
|
||
from codex.discover import cited_by
|
||
|
||
for pid in cited_by(paper_id):
|
||
typer.echo(pid)
|
||
|
||
|
||
@discover_app.command("cocited")
|
||
def discover_cocited(
|
||
paper_id: str = typer.Argument(...),
|
||
limit: int = typer.Option(10, "--limit", "-n"),
|
||
) -> None:
|
||
"""List papers frequently co-cited with PAPER_ID."""
|
||
from codex.discover import cocited_papers
|
||
|
||
for item in cocited_papers(paper_id, limit=limit):
|
||
typer.echo(f"{item['co_citations']:>4}× {item['paper_id']}")
|
||
|
||
|
||
@prov_app.command("scan")
|
||
def prov_scan(root_dir: str = typer.Argument(..., help="Root directory to scan")) -> None:
|
||
"""Scan C++ sources for @cite tags."""
|
||
from codex.provenance import scan_cite_tags
|
||
|
||
hits = scan_cite_tags(root_dir)
|
||
typer.echo(json.dumps(hits, indent=2))
|
||
|
||
|
||
@prov_app.command("add-link")
|
||
def prov_add_link(
|
||
symbol: str = typer.Argument(...),
|
||
paper_id: Optional[str] = typer.Option(None, "--paper-id"), # noqa: UP045
|
||
role: Optional[str] = typer.Option(None, "--role"), # noqa: UP045
|
||
note: Optional[str] = typer.Option(None, "--note"), # noqa: UP045
|
||
) -> None:
|
||
"""Add a code → paper link."""
|
||
from codex.provenance import add_code_link
|
||
|
||
link = add_code_link(symbol, paper_id=paper_id, role=role, note=note)
|
||
typer.echo(f"Created code_link id={link.id}: {symbol} → {paper_id}")
|
||
|
||
|
||
@prov_app.command("links")
|
||
def prov_links(
|
||
paper_id: Optional[str] = typer.Option(None, "--paper-id"), # noqa: UP045
|
||
) -> None:
|
||
"""List code → paper links."""
|
||
from codex.provenance import list_code_links
|
||
|
||
for link in list_code_links(paper_id=paper_id):
|
||
typer.echo(f"[{link.id}] {link.symbol} → {link.paper_id} ({link.role})")
|
||
|
||
|
||
@prov_app.command("bib")
|
||
def prov_bib(
|
||
paper_ids: Optional[list[str]] = typer.Argument(None), # noqa: UP045, B008
|
||
output: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output", "-o", help="Write to file instead of stdout"
|
||
),
|
||
) -> None:
|
||
"""Export papers as BibTeX."""
|
||
from codex.provenance import export_bib
|
||
|
||
bib = export_bib(paper_ids if paper_ids else None)
|
||
if output:
|
||
from pathlib import Path
|
||
|
||
Path(output).write_text(bib, encoding="utf-8")
|
||
typer.echo(f"Wrote {output}")
|
||
else:
|
||
typer.echo(bib)
|
||
|
||
|
||
@app.command()
|
||
def ask(question: str = typer.Argument(..., help="Question to answer")) -> None:
|
||
"""Ask a question (RAG — not yet implemented)."""
|
||
typer.echo("ask: not yet implemented", err=True)
|
||
raise typer.Exit(1)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F-12: Wiki command group
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@wiki_app.command("compile")
|
||
def wiki_compile(
|
||
concept: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--concept", help="Compile only this concept slug."
|
||
),
|
||
all_concepts: bool = typer.Option(
|
||
False, "--all", help="Force full recompile (ignore change detection)."
|
||
),
|
||
top_k: int = typer.Option(0, "--top-k", help="Override config.wiki_top_k (0 = use config)."),
|
||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output-dir", help="Override config.wiki_dir."
|
||
),
|
||
) -> None:
|
||
"""Compile concept pages from retrieved chunks via local LLM.
|
||
|
||
By default only recompiles concepts whose source chunks have changed
|
||
(hash-based incremental). Use --all to force full recompile.
|
||
"""
|
||
from codex.wiki import compile_all
|
||
|
||
report = compile_all(
|
||
changed_only=not all_concepts,
|
||
top_k=top_k if top_k > 0 else None,
|
||
concept_filter=concept,
|
||
output_dir=output_dir,
|
||
)
|
||
|
||
if report.compiled:
|
||
typer.echo(f"Compiled: {', '.join(report.compiled)}")
|
||
if report.skipped:
|
||
typer.echo(f"Skipped (unchanged): {', '.join(report.skipped)}")
|
||
if report.ungrounded:
|
||
typer.echo(f"⚠ Ungrounded claims: {len(report.ungrounded)}", err=True)
|
||
for slug, text in report.ungrounded:
|
||
typer.echo(f" [{slug}] {text[:100]}", err=True)
|
||
|
||
|
||
@wiki_app.command("list")
|
||
def wiki_list(
|
||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output-dir", help="Override config.wiki_dir."
|
||
),
|
||
) -> None:
|
||
"""List compiled concept pages with freshness information."""
|
||
import json as _json
|
||
from pathlib import Path
|
||
|
||
from codex.config import get_settings
|
||
|
||
settings = get_settings()
|
||
wiki_dir = Path(output_dir or settings.wiki_dir)
|
||
state_path = wiki_dir / ".compile-state.json"
|
||
|
||
state: dict[str, str] = {}
|
||
if state_path.exists():
|
||
try:
|
||
state = _json.loads(state_path.read_text(encoding="utf-8"))
|
||
except (_json.JSONDecodeError, OSError):
|
||
state = {}
|
||
|
||
pages = sorted(wiki_dir.glob("*.md"))
|
||
if not pages:
|
||
typer.echo("No compiled pages found.")
|
||
return
|
||
|
||
for page in pages:
|
||
if page.name in ("index.md", "log.md"):
|
||
continue
|
||
slug = page.stem
|
||
hash_val = state.get(slug, "—")[:8]
|
||
content = page.read_text(encoding="utf-8")
|
||
n_claims = content.count("[")
|
||
n_ungrounded = content.count("⚠")
|
||
mtime = datetime.fromtimestamp(page.stat().st_mtime, tz=UTC).strftime("%Y-%m-%d %H:%M")
|
||
typer.echo(
|
||
f"{slug:40s} last={mtime} hash={hash_val} claims≈{n_claims} ⚠={n_ungrounded}"
|
||
)
|
||
|
||
|
||
@wiki_app.command("check")
|
||
def wiki_check(
|
||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output-dir", help="Override config.wiki_dir."
|
||
),
|
||
) -> None:
|
||
"""Check all compiled pages for ungrounded claims.
|
||
|
||
Exits with code 0 if all claims are grounded, 1 if any are ungrounded.
|
||
Suitable for CI pipelines.
|
||
"""
|
||
from pathlib import Path
|
||
|
||
from codex.config import get_settings
|
||
|
||
settings = get_settings()
|
||
wiki_dir = Path(output_dir or settings.wiki_dir)
|
||
|
||
pages = sorted(wiki_dir.glob("*.md"))
|
||
found_ungrounded = False
|
||
for page in pages:
|
||
if page.name in ("index.md", "log.md"):
|
||
continue
|
||
content = page.read_text(encoding="utf-8")
|
||
if "⚠" in content:
|
||
typer.echo(f"⚠ Ungrounded claim(s) in: {page.name}", err=True)
|
||
found_ungrounded = True
|
||
|
||
# Check for quarantined pages in wiki/draft/
|
||
draft_dir = wiki_dir / "draft"
|
||
draft_pages = sorted(draft_dir.glob("*.md")) if draft_dir.exists() else []
|
||
if draft_pages:
|
||
n = len(draft_pages)
|
||
typer.echo(
|
||
f"⚠ {n} page(s) quarantined in wiki/draft/ (grounding rate below threshold)",
|
||
err=True,
|
||
)
|
||
for dp in draft_pages:
|
||
typer.echo(f" - {dp.name}", err=True)
|
||
raise typer.Exit(2)
|
||
|
||
if found_ungrounded:
|
||
raise typer.Exit(1)
|
||
else:
|
||
typer.echo("All claims grounded.")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F-13: Synthesis command group
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@synth_app.command("leads")
|
||
def synthesis_leads(
|
||
lib_path: Optional[str] = typer.Option( # noqa: UP045
|
||
None,
|
||
"--lib-path",
|
||
help=(
|
||
"Path to a C++ source tree. Enables improvement leads (@cite-aware) and "
|
||
"code-vs-corpus gap detection."
|
||
),
|
||
),
|
||
topic: list[str] | None = typer.Option( # noqa: B008
|
||
None,
|
||
"--topic",
|
||
help=(
|
||
"Topic to check for coverage gaps (repeatable). Opt-in: without --topic "
|
||
"no coverage-map gaps run (avoids per-paper false positives, audit R-8)."
|
||
),
|
||
),
|
||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output-dir", help="Override config.leads_dir."
|
||
),
|
||
) -> None:
|
||
"""Generate grounded leads (stages 2-4) and write them to leads/grounded/."""
|
||
from codex.config import get_settings
|
||
from codex.synthesis import (
|
||
default_llm_client,
|
||
find_connections,
|
||
find_gaps,
|
||
find_improvements,
|
||
write_leads,
|
||
)
|
||
|
||
settings = get_settings()
|
||
leads_dir = output_dir or settings.leads_dir
|
||
llm = default_llm_client()
|
||
|
||
connections = find_connections(top_k=settings.synthesis_top_k, llm=llm)
|
||
gaps = find_gaps(lib_path=lib_path, llm=llm, topics=topic or None)
|
||
improvements = (
|
||
find_improvements(lib_path, top_k=settings.synthesis_top_k, llm=llm) if lib_path else []
|
||
)
|
||
|
||
all_leads = connections + gaps + improvements
|
||
write_leads(all_leads, leads_dir)
|
||
|
||
typer.echo(
|
||
f"Grounded leads written to {leads_dir}/grounded/: "
|
||
f"{len(connections)} connection · {len(gaps)} gap · {len(improvements)} improvement"
|
||
)
|
||
|
||
|
||
@synth_app.command("conjectures")
|
||
def synthesis_conjectures(
|
||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output-dir", help="Override config.leads_dir."
|
||
),
|
||
) -> None:
|
||
"""Generate stage-5 conjecture leads — quarantined to leads/conjectures/.
|
||
|
||
INVARIANT (F-13 HARD): conjectures are NEVER written to wiki/. They
|
||
require a ``suggested_validation`` spike before they may be promoted.
|
||
"""
|
||
from codex.config import get_settings
|
||
from codex.synthesis import default_llm_client, propose_conjectures, write_leads
|
||
|
||
settings = get_settings()
|
||
leads_dir = output_dir or settings.leads_dir
|
||
llm = default_llm_client()
|
||
|
||
conjectures = propose_conjectures(top_k=settings.synthesis_top_k, llm=llm)
|
||
write_leads(conjectures, leads_dir)
|
||
|
||
typer.echo(f"Conjectures (UNVERIFIED) written to {leads_dir}/conjectures/: {len(conjectures)}")
|
||
|
||
|
||
@synth_app.command("report")
|
||
def synthesis_report(
|
||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||
None, "--output-dir", help="Override config.leads_dir."
|
||
),
|
||
) -> None:
|
||
"""Show grounded leads and conjectures — visibly separated."""
|
||
from pathlib import Path
|
||
|
||
from codex.config import get_settings
|
||
|
||
settings = get_settings()
|
||
leads_dir = Path(output_dir or settings.leads_dir)
|
||
grounded_dir = leads_dir / "grounded"
|
||
conj_dir = leads_dir / "conjectures"
|
||
|
||
grounded_files = sorted(grounded_dir.glob("*.md")) if grounded_dir.exists() else []
|
||
conj_files = sorted(conj_dir.glob("*.md")) if conj_dir.exists() else []
|
||
|
||
typer.echo("=" * 72)
|
||
typer.echo(f"GROUNDED LEADS ({len(grounded_files)}) → {grounded_dir}")
|
||
typer.echo("=" * 72)
|
||
if not grounded_files:
|
||
typer.echo("(none)")
|
||
for f in grounded_files:
|
||
first = f.read_text(encoding="utf-8").splitlines()[0]
|
||
typer.echo(f" {first}")
|
||
|
||
typer.echo("")
|
||
typer.echo("=" * 72)
|
||
typer.echo(f"CONJECTURES ({len(conj_files)}) → {conj_dir} [UNVERIFIED — never in wiki/]")
|
||
typer.echo("=" * 72)
|
||
if not conj_files:
|
||
typer.echo("(none)")
|
||
for f in conj_files:
|
||
first = f.read_text(encoding="utf-8").splitlines()[0]
|
||
typer.echo(f" {first}")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F-16 — quality sub-commands
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@quality_app.command("run")
|
||
def quality_run(
|
||
paper_id: Optional[str] = typer.Option( # noqa: UP045
|
||
None,
|
||
"--paper-id",
|
||
help="Restrict pass to one paper (omit to process all papers).",
|
||
),
|
||
) -> None:
|
||
"""Apply quality gate + section classification to existing chunks.
|
||
|
||
Iterates over chunks in the database, removes those that fail the
|
||
quality thresholds, and back-fills the ``section`` column for the rest.
|
||
"""
|
||
from codex.config import get_settings
|
||
from codex.db import get_conn
|
||
from codex.quality import run_quality_pass
|
||
|
||
settings = get_settings()
|
||
with get_conn() as conn:
|
||
stats = run_quality_pass(paper_id=paper_id, conn=conn, settings=settings)
|
||
|
||
scope = f"paper {paper_id}" if paper_id else "all papers"
|
||
typer.echo(
|
||
f"Quality pass ({scope}): "
|
||
f"{stats['kept']} kept, "
|
||
f"{stats['removed']} removed, "
|
||
f"{stats['tagged']} section-tagged"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# F-15 — graph sub-commands
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@graph_app.command("report")
|
||
def graph_report(
|
||
top_n: int = typer.Option(10, "--top-n", "-n", help="Number of hub papers to show."),
|
||
output_json: bool = typer.Option(False, "--json", help="Output as JSON."),
|
||
) -> None:
|
||
"""Show citation graph report: top hub papers, dangling citations, cluster summary."""
|
||
from codex.config import get_settings
|
||
from codex.db import get_conn
|
||
from codex.graph import build_citation_graph, citation_pagerank, dangling_citations
|
||
|
||
settings = get_settings()
|
||
with get_conn() as conn:
|
||
graph = build_citation_graph(conn)
|
||
paper_rows = conn.execute("SELECT id, title FROM papers").fetchall()
|
||
known_ids = {row["id"] for row in paper_rows}
|
||
titles = {row["id"]: row["title"] for row in paper_rows}
|
||
|
||
if graph.number_of_nodes() == 0:
|
||
typer.echo("Citation graph is empty — ingest some papers first.")
|
||
return
|
||
|
||
n_papers = len(known_ids)
|
||
pr = citation_pagerank(graph)
|
||
hubs = sorted(pr.items(), key=lambda x: -x[1])[:top_n]
|
||
dangling = sorted(dangling_citations(graph, known_ids))
|
||
small_corpus = n_papers < settings.graph_min_corpus_size
|
||
warning = (
|
||
f"only {n_papers} ingested papers; recommended >= {settings.graph_min_corpus_size} "
|
||
"for meaningful ranking"
|
||
if small_corpus
|
||
else None
|
||
)
|
||
|
||
if output_json:
|
||
typer.echo(
|
||
json.dumps(
|
||
{
|
||
"nodes": graph.number_of_nodes(),
|
||
"edges": graph.number_of_edges(),
|
||
"small_corpus_warning": warning, # null unless below threshold (audit U-3)
|
||
"hubs": [
|
||
{"paper_id": pid, "title": titles.get(pid), "pagerank": score}
|
||
for pid, score in hubs
|
||
],
|
||
"dangling_count": len(dangling),
|
||
"dangling": dangling,
|
||
},
|
||
indent=2,
|
||
)
|
||
)
|
||
return
|
||
|
||
if warning:
|
||
typer.echo(f"Warning: {warning}.", err=True)
|
||
typer.echo(f"Graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges")
|
||
typer.echo("")
|
||
typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)")
|
||
typer.echo("-" * 48)
|
||
for pid, score in hubs:
|
||
# in-KB hubs show their title; dangling (OpenAlex-id) hubs are flagged (U-1)
|
||
label = titles.get(pid) or "(dangling — not ingested)"
|
||
typer.echo(f" {score:.4f} {pid} {label}")
|
||
typer.echo("")
|
||
typer.echo(f"DANGLING CITATIONS ({len(dangling)} cited but not ingested)")
|
||
typer.echo("-" * 48)
|
||
for pid in dangling[:top_n]:
|
||
typer.echo(f" {pid}")
|
||
if len(dangling) > top_n:
|
||
typer.echo(f" … and {len(dangling) - top_n} more")
|
||
|
||
|
||
@graph_app.command("related")
|
||
def graph_related(
|
||
paper_id: str = typer.Argument(..., help="Paper ID to find related papers for."),
|
||
min_shared: int = typer.Option(
|
||
2, "--min-shared", help="Minimum shared references for bibliographic coupling."
|
||
),
|
||
) -> None:
|
||
"""List bibliographically related papers (shared references ≥ min-shared)."""
|
||
from codex.db import get_conn
|
||
from codex.graph import build_citation_graph, find_related
|
||
|
||
with get_conn() as conn:
|
||
graph = build_citation_graph(conn)
|
||
|
||
related = find_related(paper_id, graph, min_shared=min_shared)
|
||
if not related:
|
||
typer.echo(f"No papers with ≥ {min_shared} shared references found for {paper_id}.")
|
||
return
|
||
typer.echo(
|
||
f"Papers related to {paper_id} (bibliographic coupling, ≥ {min_shared} shared refs):"
|
||
)
|
||
for pid in related:
|
||
typer.echo(f" {pid}")
|
||
|
||
|
||
@graph_app.command("cocited")
|
||
def graph_cocited(
|
||
paper_id: str = typer.Argument(..., help="Paper ID to find co-cited papers for."),
|
||
) -> None:
|
||
"""List papers frequently co-cited with PAPER_ID (graph co-citation)."""
|
||
from codex.db import get_conn
|
||
from codex.graph import build_citation_graph, find_co_cited
|
||
|
||
with get_conn() as conn:
|
||
graph = build_citation_graph(conn)
|
||
|
||
results = find_co_cited(paper_id, graph)
|
||
if not results:
|
||
typer.echo(f"No papers co-cited with {paper_id}.")
|
||
return
|
||
typer.echo(f"Papers co-cited with {paper_id} (count desc):")
|
||
for pid, count in results:
|
||
typer.echo(f" {count:>3}× {pid}")
|