feat(F-15): citation graph — PageRank, coupling, co-citation, CLI
- codex/graph.py: build_citation_graph (DiGraph from DB), citation_pagerank (graceful uniform fallback < 5 nodes), find_related (bibliographic coupling), find_co_cited, dangling_citations - codex/config.py: graph_cite_boost_alpha=0.3, graph_min_corpus_size=15 - codex/cli.py: graph report [--top-n] [--json], graph related <paper-id> [--min-shared]; search paper --cite-boost (PageRank score weighting) - tests/graph/: 39 tests (graph functions + CLI) — 326 total green Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
107
codex/cli.py
107
codex/cli.py
@@ -14,6 +14,7 @@ prov_app = typer.Typer(help="Provenance: @cite scan, code_links, bib export.")
|
||||
wiki_app = typer.Typer(help="Wiki-compile: grounded concept pages over the RAG substrate.")
|
||||
synth_app = typer.Typer(help="Synthesis: grounded leads and quarantined conjectures (F-13).")
|
||||
quality_app = typer.Typer(help="Chunk quality gate and section classification (F-16).")
|
||||
graph_app = typer.Typer(help="Citation graph analytics: PageRank, coupling, dangling leads (F-15).")
|
||||
# F-09: search sub-app with paper (semantic) and formula (FTS) subcommands
|
||||
search_app = typer.Typer(
|
||||
help="Search: semantic paper search and formula full-text search.",
|
||||
@@ -24,6 +25,7 @@ app.add_typer(prov_app, name="provenance")
|
||||
app.add_typer(wiki_app, name="wiki")
|
||||
app.add_typer(synth_app, name="synthesis")
|
||||
app.add_typer(quality_app, name="quality")
|
||||
app.add_typer(graph_app, name="graph")
|
||||
app.add_typer(search_app, name="search")
|
||||
|
||||
|
||||
@@ -66,8 +68,14 @@ def search_callback(ctx: typer.Context) -> None:
|
||||
def search_paper(
|
||||
query: str = typer.Argument(..., help="Natural-language search query"),
|
||||
limit: int = typer.Option(10, "--limit", "-n", help="Number of results"),
|
||||
cite_boost: bool = typer.Option(
|
||||
False,
|
||||
"--cite-boost",
|
||||
help="Weight results by citation PageRank (F-15). Graceful when corpus < 5 papers.",
|
||||
),
|
||||
) -> None:
|
||||
"""Semantic similarity search over paper abstracts."""
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.embed import get_embedder
|
||||
|
||||
@@ -84,6 +92,24 @@ def search_paper(
|
||||
""",
|
||||
{"emb": emb, "limit": limit},
|
||||
).fetchall()
|
||||
|
||||
if cite_boost and rows:
|
||||
from codex.graph import build_citation_graph, citation_pagerank
|
||||
|
||||
settings = get_settings()
|
||||
graph = build_citation_graph(conn)
|
||||
pr = citation_pagerank(graph)
|
||||
alpha = settings.graph_cite_boost_alpha
|
||||
results = []
|
||||
for row in rows:
|
||||
pr_score = pr.get(row["id"], 0.0)
|
||||
boosted = row["distance"] * (1.0 + alpha * pr_score)
|
||||
results.append((boosted, row))
|
||||
results.sort(key=lambda x: x[0])
|
||||
for boosted_dist, row in results:
|
||||
typer.echo(f"[{boosted_dist:.3f}] {row['id']} ({row['year']}) — {row['title']}")
|
||||
return
|
||||
|
||||
for row in rows:
|
||||
typer.echo(f"[{row['distance']:.3f}] {row['id']} ({row['year']}) — {row['title']}")
|
||||
|
||||
@@ -496,3 +522,84 @@ def quality_run(
|
||||
f"{stats['removed']} removed, "
|
||||
f"{stats['tagged']} section-tagged"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F-15 — graph sub-commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@graph_app.command("report")
|
||||
def graph_report(
|
||||
top_n: int = typer.Option(10, "--top-n", "-n", help="Number of hub papers to show."),
|
||||
output_json: bool = typer.Option(False, "--json", help="Output as JSON."),
|
||||
) -> None:
|
||||
"""Show citation graph report: top hub papers, dangling citations, cluster summary."""
|
||||
from codex.db import get_conn
|
||||
from codex.graph import build_citation_graph, citation_pagerank, dangling_citations
|
||||
|
||||
with get_conn() as conn:
|
||||
graph = build_citation_graph(conn)
|
||||
known_ids = {row["id"] for row in conn.execute("SELECT id FROM papers").fetchall()}
|
||||
|
||||
if graph.number_of_nodes() == 0:
|
||||
typer.echo("Citation graph is empty — ingest some papers first.")
|
||||
raise typer.Exit(0)
|
||||
|
||||
pr = citation_pagerank(graph)
|
||||
hubs = sorted(pr.items(), key=lambda x: -x[1])[:top_n]
|
||||
dangling = dangling_citations(graph, known_ids)
|
||||
|
||||
if output_json:
|
||||
typer.echo(
|
||||
json.dumps(
|
||||
{
|
||||
"nodes": graph.number_of_nodes(),
|
||||
"edges": graph.number_of_edges(),
|
||||
"hubs": [{"paper_id": pid, "pagerank": score} for pid, score in hubs],
|
||||
"dangling_count": len(dangling),
|
||||
"dangling": dangling,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
typer.echo(f"Graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges")
|
||||
typer.echo("")
|
||||
typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)")
|
||||
typer.echo("-" * 48)
|
||||
for pid, score in hubs:
|
||||
typer.echo(f" {score:.4f} {pid}")
|
||||
typer.echo("")
|
||||
typer.echo(f"DANGLING CITATIONS ({len(dangling)} cited but not ingested)")
|
||||
typer.echo("-" * 48)
|
||||
for pid in dangling[:top_n]:
|
||||
typer.echo(f" {pid}")
|
||||
if len(dangling) > top_n:
|
||||
typer.echo(f" … and {len(dangling) - top_n} more")
|
||||
|
||||
|
||||
@graph_app.command("related")
|
||||
def graph_related(
|
||||
paper_id: str = typer.Argument(..., help="Paper ID to find related papers for."),
|
||||
min_shared: int = typer.Option(
|
||||
2, "--min-shared", help="Minimum shared references for bibliographic coupling."
|
||||
),
|
||||
) -> None:
|
||||
"""List bibliographically related papers (shared references ≥ min-shared)."""
|
||||
from codex.db import get_conn
|
||||
from codex.graph import build_citation_graph, find_related
|
||||
|
||||
with get_conn() as conn:
|
||||
graph = build_citation_graph(conn)
|
||||
|
||||
related = find_related(paper_id, graph, min_shared=min_shared)
|
||||
if not related:
|
||||
typer.echo(f"No papers with ≥ {min_shared} shared references found for {paper_id}.")
|
||||
return
|
||||
typer.echo(
|
||||
f"Papers related to {paper_id} (bibliographic coupling, ≥ {min_shared} shared refs):"
|
||||
)
|
||||
for pid in related:
|
||||
typer.echo(f" {pid}")
|
||||
|
||||
@@ -286,6 +286,30 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# F-15 Literature Graph
|
||||
# ------------------------------------------------------------------
|
||||
graph_cite_boost_alpha: float = Field(
|
||||
default=0.3,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description=(
|
||||
"Weight of the PageRank citation-boost in hybrid search. "
|
||||
"Final score = dense_score * (1 + alpha * pagerank_score). "
|
||||
"Set to 0.0 to disable the boost without removing the flag."
|
||||
),
|
||||
)
|
||||
|
||||
graph_min_corpus_size: int = Field(
|
||||
default=15,
|
||||
gt=0,
|
||||
description=(
|
||||
"Minimum number of ingested papers for citation_pagerank to yield "
|
||||
"meaningful rankings. Below this threshold a warning is logged and "
|
||||
"uniform scores are returned."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
|
||||
169
codex/graph.py
Normal file
169
codex/graph.py
Normal file
@@ -0,0 +1,169 @@
|
||||
"""Citation graph analytics (F-15).
|
||||
|
||||
Builds an in-memory NetworkX DiGraph from the ``citations`` table and
|
||||
provides PageRank, bibliographic coupling, co-citation, and dangling-
|
||||
citation queries. The graph is ephemeral — rebuilt per call, < 1 s for
|
||||
≤ 100 papers.
|
||||
|
||||
Graceful degradation
|
||||
--------------------
|
||||
* ``citation_pagerank`` returns uniform scores when the graph has fewer
|
||||
than 5 nodes, and logs a warning.
|
||||
* All functions accept an empty graph without raising.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, cast
|
||||
|
||||
import networkx as nx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MIN_PAGERANK_NODES = 5
|
||||
|
||||
|
||||
def build_citation_graph(conn: Any) -> nx.DiGraph:
|
||||
"""Load the ``citations`` table into a directed graph.
|
||||
|
||||
Nodes are paper IDs (strings). An edge ``citing → cited`` means
|
||||
the citing paper references the cited paper. Both ingested papers
|
||||
and dangling targets (cited but not yet ingested) appear as nodes.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
conn:
|
||||
Open psycopg connection (dict-row factory assumed).
|
||||
|
||||
Returns
|
||||
-------
|
||||
nx.DiGraph with every (citing_id, cited_id) pair as an edge.
|
||||
"""
|
||||
rows = conn.execute("SELECT citing_id, cited_id FROM citations").fetchall()
|
||||
g: nx.DiGraph = nx.DiGraph()
|
||||
for row in rows:
|
||||
g.add_edge(row["citing_id"], row["cited_id"])
|
||||
logger.debug(
|
||||
"Built citation graph: %d nodes, %d edges", g.number_of_nodes(), g.number_of_edges()
|
||||
)
|
||||
return g
|
||||
|
||||
|
||||
def citation_pagerank(graph: nx.DiGraph, *, damping: float = 0.85) -> dict[str, float]:
|
||||
"""Compute PageRank over the citation graph.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
DiGraph from :func:`build_citation_graph`.
|
||||
damping:
|
||||
PageRank damping factor (default 0.85).
|
||||
|
||||
Returns
|
||||
-------
|
||||
``{paper_id: score}`` dict. Scores sum to approximately 1.0.
|
||||
|
||||
When ``graph`` has fewer than :data:`_MIN_PAGERANK_NODES` nodes,
|
||||
returns a uniform distribution and logs a warning — the signal is
|
||||
too thin to be meaningful.
|
||||
"""
|
||||
n = graph.number_of_nodes()
|
||||
if n == 0:
|
||||
return {}
|
||||
if n < _MIN_PAGERANK_NODES:
|
||||
logger.warning(
|
||||
"citation_pagerank: only %d nodes — corpus too small for meaningful ranking "
|
||||
"(need ≥ %d). Returning uniform scores.",
|
||||
n,
|
||||
_MIN_PAGERANK_NODES,
|
||||
)
|
||||
uniform = 1.0 / n
|
||||
return {node: uniform for node in graph.nodes()}
|
||||
return cast(dict[str, float], nx.pagerank(graph, alpha=damping))
|
||||
|
||||
|
||||
def find_related(paper_id: str, graph: nx.DiGraph, *, min_shared: int = 2) -> list[str]:
|
||||
"""Bibliographic coupling: papers sharing ≥ ``min_shared`` references with ``paper_id``.
|
||||
|
||||
Two papers are bibliographically coupled when they cite the same sources.
|
||||
The more shared references, the more likely they treat related topics.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
paper_id:
|
||||
Source paper whose references form the seed set.
|
||||
graph:
|
||||
DiGraph from :func:`build_citation_graph`.
|
||||
min_shared:
|
||||
Minimum number of shared references to be included.
|
||||
|
||||
Returns
|
||||
-------
|
||||
List of paper IDs ordered by shared-reference count DESC.
|
||||
"""
|
||||
if paper_id not in graph:
|
||||
return []
|
||||
# papers cited by paper_id
|
||||
references: set[str] = set(graph.successors(paper_id))
|
||||
if not references:
|
||||
return []
|
||||
|
||||
shared: dict[str, int] = {}
|
||||
for ref in references:
|
||||
# other papers that also cite this reference
|
||||
for co_citer in graph.predecessors(ref):
|
||||
if co_citer == paper_id:
|
||||
continue
|
||||
shared[co_citer] = shared.get(co_citer, 0) + 1
|
||||
|
||||
return [
|
||||
pid for pid, count in sorted(shared.items(), key=lambda x: -x[1]) if count >= min_shared
|
||||
]
|
||||
|
||||
|
||||
def find_co_cited(paper_id: str, graph: nx.DiGraph) -> list[tuple[str, int]]:
|
||||
"""Co-citation: papers frequently cited alongside ``paper_id``.
|
||||
|
||||
A paper X is co-cited with ``paper_id`` when some third paper cites both.
|
||||
|
||||
Returns
|
||||
-------
|
||||
``[(paper_id, count)]`` ordered by count DESC.
|
||||
"""
|
||||
if paper_id not in graph:
|
||||
return []
|
||||
# papers that cite paper_id
|
||||
citers: set[str] = set(graph.predecessors(paper_id))
|
||||
if not citers:
|
||||
return []
|
||||
|
||||
co_cited: dict[str, int] = {}
|
||||
for citer in citers:
|
||||
for other in graph.successors(citer):
|
||||
if other == paper_id:
|
||||
continue
|
||||
co_cited[other] = co_cited.get(other, 0) + 1
|
||||
|
||||
return sorted(co_cited.items(), key=lambda x: -x[1])
|
||||
|
||||
|
||||
def dangling_citations(graph: nx.DiGraph, known_ids: set[str]) -> list[str]:
|
||||
"""Return graph nodes not present in ``known_ids`` (cited but not yet ingested).
|
||||
|
||||
Complements :func:`codex.discover.discovery_leads` with graph context:
|
||||
the returned IDs are reachable in the citation graph but have no paper
|
||||
record in the DB.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
DiGraph from :func:`build_citation_graph`.
|
||||
known_ids:
|
||||
Set of paper IDs that exist in the ``papers`` table.
|
||||
|
||||
Returns
|
||||
-------
|
||||
List of dangling paper IDs (unordered).
|
||||
"""
|
||||
return [node for node in graph.nodes() if node not in known_ids]
|
||||
Reference in New Issue
Block a user