merge: reconcile data-quality roadmap with audit-remediation main (#12-15)
Resolved 4 conflicts: cli.py (R-D citing-coverage warning + main's small-corpus JSON/hub-title changes both kept); ingest.py (C-7 paper.id=caller-id pin first, then DQ-2 abstract recovery); test_ingest.py + test_openalex.py (kept both branches' tests). C-7/DQ-5 overlap (both canonicalise papers.id): kept both layers — openalex._canonical_id normalizes fetch_paper's id (DQ-5); ingest pins papers.id to the caller's bare id (C-7); they converge on the bare canonical id. Fixed the auto-merged test_fetch_paper_success that had contradictory bare/URL assertions. 402 tests pass; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
113
codex/cli.py
113
codex/cli.py
@@ -29,6 +29,37 @@ app.add_typer(graph_app, name="graph")
|
||||
app.add_typer(search_app, name="search")
|
||||
|
||||
|
||||
@app.command()
|
||||
def migrate() -> None:
|
||||
"""Apply infra/schema.sql to bring the database schema up to date (idempotent).
|
||||
|
||||
Must connect as a role that can run DDL (owns / can CREATE + ALTER the tables).
|
||||
The application's DATABASE_URL is usually a least-privilege DML role and will
|
||||
fail with InsufficientPrivilege; set MIGRATION_DATABASE_URL to a privileged
|
||||
(owner/superuser) connection (audit M-1).
|
||||
"""
|
||||
import psycopg
|
||||
import psycopg.rows
|
||||
|
||||
from codex.config import get_settings
|
||||
from codex.db import apply_schema
|
||||
|
||||
settings = get_settings()
|
||||
url = (settings.migration_database_url or settings.database_url).get_secret_value()
|
||||
try:
|
||||
with psycopg.connect(url, row_factory=psycopg.rows.dict_row) as conn:
|
||||
apply_schema(conn)
|
||||
except psycopg.errors.InsufficientPrivilege as exc:
|
||||
typer.echo(
|
||||
"ERROR: schema migration needs a role that owns the tables "
|
||||
"(CREATE/ALTER). Set MIGRATION_DATABASE_URL to a privileged connection "
|
||||
f"and retry. ({exc})",
|
||||
err=True,
|
||||
)
|
||||
raise typer.Exit(1) from exc
|
||||
typer.echo("Schema applied — database is up to date.")
|
||||
|
||||
|
||||
@app.command()
|
||||
def ingest(
|
||||
paper_id: str = typer.Argument(..., help="arXiv ID, DOI, or OpenAlex W-ID"),
|
||||
@@ -71,7 +102,8 @@ def search_paper(
|
||||
cite_boost: bool = typer.Option(
|
||||
False,
|
||||
"--cite-boost",
|
||||
help="Weight results by citation PageRank (F-15). Graceful when corpus < 5 papers.",
|
||||
help="Re-rank the top results by citation PageRank — a within-page "
|
||||
"tie-breaker, not a hard re-ranking (F-15). Graceful when corpus < 5 papers.",
|
||||
),
|
||||
) -> None:
|
||||
"""Semantic similarity search over paper abstracts."""
|
||||
@@ -157,6 +189,24 @@ def discover_leads(
|
||||
typer.echo(f"{item['pull']:>4}× {item['cited_id']}")
|
||||
|
||||
|
||||
@discover_app.command("recommend")
|
||||
def discover_recommend(
|
||||
paper_id: str = typer.Argument(
|
||||
..., help="Semantic Scholar paper ID (or arXiv:/DOI: prefixed)."
|
||||
),
|
||||
limit: int = typer.Option(20, "--limit", "-n", help="Number of recommendations."),
|
||||
) -> None:
|
||||
"""Recommended papers for PAPER_ID via Semantic Scholar."""
|
||||
from codex.sources.semanticscholar import fetch_recommendations
|
||||
|
||||
rec = fetch_recommendations(paper_id, limit=limit)
|
||||
if not rec:
|
||||
typer.echo(f"No recommendations found for {paper_id}.")
|
||||
return
|
||||
for pid in rec:
|
||||
typer.echo(pid)
|
||||
|
||||
|
||||
@discover_app.command("citing")
|
||||
def discover_citing(paper_id: str = typer.Argument(...)) -> None:
|
||||
"""List papers that cite PAPER_ID."""
|
||||
@@ -391,7 +441,15 @@ def synthesis_leads(
|
||||
"--lib-path",
|
||||
help=(
|
||||
"Path to a C++ source tree. Enables improvement leads (@cite-aware) and "
|
||||
"code-vs-corpus gap detection. Without it only connection + topic gaps run."
|
||||
"code-vs-corpus gap detection."
|
||||
),
|
||||
),
|
||||
topic: list[str] | None = typer.Option( # noqa: B008
|
||||
None,
|
||||
"--topic",
|
||||
help=(
|
||||
"Topic to check for coverage gaps (repeatable). Opt-in: without --topic "
|
||||
"no coverage-map gaps run (avoids per-paper false positives, audit R-8)."
|
||||
),
|
||||
),
|
||||
output_dir: Optional[str] = typer.Option( # noqa: UP045
|
||||
@@ -413,7 +471,7 @@ def synthesis_leads(
|
||||
llm = default_llm_client()
|
||||
|
||||
connections = find_connections(top_k=settings.synthesis_top_k, llm=llm)
|
||||
gaps = find_gaps(lib_path=lib_path, llm=llm)
|
||||
gaps = find_gaps(lib_path=lib_path, llm=llm, topics=topic or None)
|
||||
improvements = (
|
||||
find_improvements(lib_path, top_k=settings.synthesis_top_k, llm=llm) if lib_path else []
|
||||
)
|
||||
@@ -548,7 +606,9 @@ def graph_report(
|
||||
settings = get_settings()
|
||||
with get_conn() as conn:
|
||||
graph = build_citation_graph(conn)
|
||||
known_ids = {row["id"] for row in conn.execute("SELECT id FROM papers").fetchall()}
|
||||
paper_rows = conn.execute("SELECT id, title FROM papers").fetchall()
|
||||
known_ids = {row["id"] for row in paper_rows}
|
||||
titles = {row["id"]: row["title"] for row in paper_rows}
|
||||
|
||||
if graph.number_of_nodes() == 0:
|
||||
typer.echo("Citation graph is empty — ingest some papers first.")
|
||||
@@ -560,6 +620,13 @@ def graph_report(
|
||||
dangling = sorted(dangling_citations(graph, known_ids))
|
||||
n_citing, _ = citing_coverage(graph, known_ids)
|
||||
coverage = n_citing / n_papers if n_papers else 0.0
|
||||
small_corpus = n_papers < settings.graph_min_corpus_size
|
||||
warning = (
|
||||
f"only {n_papers} ingested papers; recommended >= {settings.graph_min_corpus_size} "
|
||||
"for meaningful ranking"
|
||||
if small_corpus
|
||||
else None
|
||||
)
|
||||
|
||||
if output_json:
|
||||
typer.echo(
|
||||
@@ -572,7 +639,11 @@ def graph_report(
|
||||
"papers": n_papers,
|
||||
"fraction": round(coverage, 4),
|
||||
},
|
||||
"hubs": [{"paper_id": pid, "pagerank": score} for pid, score in hubs],
|
||||
"small_corpus_warning": warning, # null unless below threshold (audit U-3)
|
||||
"hubs": [
|
||||
{"paper_id": pid, "title": titles.get(pid), "pagerank": score}
|
||||
for pid, score in hubs
|
||||
],
|
||||
"dangling_count": len(dangling),
|
||||
"dangling": dangling,
|
||||
},
|
||||
@@ -581,12 +652,8 @@ def graph_report(
|
||||
)
|
||||
return
|
||||
|
||||
if n_papers < settings.graph_min_corpus_size:
|
||||
typer.echo(
|
||||
f"Warning: only {n_papers} ingested papers "
|
||||
f"(recommended ≥ {settings.graph_min_corpus_size} for meaningful ranking).",
|
||||
err=True,
|
||||
)
|
||||
if warning:
|
||||
typer.echo(f"Warning: {warning}.", err=True)
|
||||
# R-D: low citing-paper coverage starves PageRank/coupling even at a healthy
|
||||
# paper count — warn on the share of papers that actually have out-edges.
|
||||
if coverage < settings.graph_min_citing_coverage:
|
||||
@@ -602,7 +669,9 @@ def graph_report(
|
||||
typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)")
|
||||
typer.echo("-" * 48)
|
||||
for pid, score in hubs:
|
||||
typer.echo(f" {score:.4f} {pid}")
|
||||
# in-KB hubs show their title; dangling (OpenAlex-id) hubs are flagged (U-1)
|
||||
label = titles.get(pid) or "(dangling — not ingested)"
|
||||
typer.echo(f" {score:.4f} {pid} {label}")
|
||||
typer.echo("")
|
||||
typer.echo(f"DANGLING CITATIONS ({len(dangling)} cited but not ingested)")
|
||||
typer.echo("-" * 48)
|
||||
@@ -635,3 +704,23 @@ def graph_related(
|
||||
)
|
||||
for pid in related:
|
||||
typer.echo(f" {pid}")
|
||||
|
||||
|
||||
@graph_app.command("cocited")
|
||||
def graph_cocited(
|
||||
paper_id: str = typer.Argument(..., help="Paper ID to find co-cited papers for."),
|
||||
) -> None:
|
||||
"""List papers frequently co-cited with PAPER_ID (graph co-citation)."""
|
||||
from codex.db import get_conn
|
||||
from codex.graph import build_citation_graph, find_co_cited
|
||||
|
||||
with get_conn() as conn:
|
||||
graph = build_citation_graph(conn)
|
||||
|
||||
results = find_co_cited(paper_id, graph)
|
||||
if not results:
|
||||
typer.echo(f"No papers co-cited with {paper_id}.")
|
||||
return
|
||||
typer.echo(f"Papers co-cited with {paper_id} (count desc):")
|
||||
for pid, count in results:
|
||||
typer.echo(f" {count:>3}× {pid}")
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic import AliasChoices, Field
|
||||
from pydantic import AliasChoices, Field, SecretStr
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
@@ -26,14 +26,25 @@ class Settings(BaseSettings):
|
||||
# ------------------------------------------------------------------
|
||||
# Database
|
||||
# ------------------------------------------------------------------
|
||||
database_url: str = Field(
|
||||
default="postgresql://researcher:change_me@localhost:5432/papers",
|
||||
database_url: SecretStr = Field(
|
||||
default=SecretStr("postgresql://researcher:change_me@localhost:5432/papers"),
|
||||
description=(
|
||||
"libpq-compatible connection string consumed by psycopg. "
|
||||
"Example: postgresql://user:pass@host:5432/dbname"
|
||||
),
|
||||
)
|
||||
|
||||
migration_database_url: SecretStr | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Optional privileged connection string used by `codex migrate` to apply "
|
||||
"schema DDL. The app's database_url is typically a least-privilege DML "
|
||||
"role that cannot CREATE/ALTER owner-held tables (raises "
|
||||
"InsufficientPrivilege); point this at an owner/superuser connection. "
|
||||
"Falls back to database_url when unset."
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# External services
|
||||
# ------------------------------------------------------------------
|
||||
@@ -137,7 +148,7 @@ class Settings(BaseSettings):
|
||||
# ------------------------------------------------------------------
|
||||
# F-09 Rich Parsing
|
||||
# ------------------------------------------------------------------
|
||||
mathpix_app_id: str | None = Field(
|
||||
mathpix_app_id: SecretStr | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"MathPix App ID for cloud formula extraction. "
|
||||
@@ -145,7 +156,7 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
mathpix_app_key: str | None = Field(
|
||||
mathpix_app_key: SecretStr | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"MathPix App Key for cloud formula extraction. "
|
||||
@@ -246,7 +257,7 @@ class Settings(BaseSettings):
|
||||
description="Bind port for HTTP transport (ignored for stdio).",
|
||||
)
|
||||
|
||||
mcp_auth_token: str | None = Field(
|
||||
mcp_auth_token: SecretStr | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Bearer token required when mcp_transport='http'. "
|
||||
@@ -295,7 +306,8 @@ class Settings(BaseSettings):
|
||||
le=1.0,
|
||||
description=(
|
||||
"Weight of the PageRank citation-boost in hybrid search. "
|
||||
"Final score = dense_score * (1 + alpha * pagerank_score). "
|
||||
"boosted_distance = distance / (1 + alpha * pagerank_score) — dividing "
|
||||
"lowers the cosine distance of high-PageRank papers (lower = closer). "
|
||||
"Set to 0.0 to disable the boost without removing the flag."
|
||||
),
|
||||
)
|
||||
|
||||
13
codex/db.py
13
codex/db.py
@@ -36,7 +36,7 @@ def get_conn() -> Generator[psycopg.Connection[psycopg.rows.DictRow], None, None
|
||||
"""
|
||||
settings = get_settings()
|
||||
with psycopg.connect(
|
||||
settings.database_url,
|
||||
settings.database_url.get_secret_value(),
|
||||
row_factory=psycopg.rows.dict_row,
|
||||
) as conn:
|
||||
yield conn
|
||||
@@ -45,11 +45,16 @@ def get_conn() -> Generator[psycopg.Connection[psycopg.rows.DictRow], None, None
|
||||
def apply_schema(conn: psycopg.Connection[psycopg.rows.DictRow]) -> None:
|
||||
"""Idempotently apply ``infra/schema.sql`` to the connected database.
|
||||
|
||||
Safe to call on every startup — all DDL statements use
|
||||
``CREATE … IF NOT EXISTS``.
|
||||
Safe to call repeatedly — every statement is ``CREATE … IF NOT EXISTS`` or
|
||||
``ADD COLUMN IF NOT EXISTS``, so re-application is a no-op.
|
||||
|
||||
Requires a connection whose role can run DDL (owns / can CREATE + ALTER the
|
||||
tables). The least-privilege application role is typically DML-only and will
|
||||
raise ``InsufficientPrivilege``; use a privileged connection — see
|
||||
``codex migrate`` and ``MIGRATION_DATABASE_URL`` (audit M-1).
|
||||
|
||||
Args:
|
||||
conn: An open psycopg connection (obtained via :func:`get_conn`).
|
||||
conn: An open psycopg connection with DDL privileges.
|
||||
"""
|
||||
schema_path = Path(__file__).parent.parent / "infra" / "schema.sql"
|
||||
sql = schema_path.read_text(encoding="utf-8")
|
||||
|
||||
@@ -1,22 +1,32 @@
|
||||
"""Discovery queries over the citation graph stored in PostgreSQL."""
|
||||
"""Discovery queries over the citation graph stored in PostgreSQL.
|
||||
|
||||
Every query resolves ``citations.cited_id`` to a canonical ``papers.id`` through
|
||||
the shared :data:`codex.graph.RESOLVED_CITATIONS_SQL`, so this module and the
|
||||
citation-graph layer agree on what counts as "ingested". Previously this module
|
||||
compared the raw OpenAlex ``cited_id`` against ``papers.id`` (a DOI/arXiv id),
|
||||
which mis-reported already-ingested papers as discovery leads (audit C-1).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from codex.db import get_conn
|
||||
from codex.graph import RESOLVED_CITATIONS_SQL
|
||||
|
||||
|
||||
def discovery_leads(limit: int = 20) -> list[dict[str, int | str]]:
|
||||
"""Return papers referenced by already-ingested papers but not yet collected.
|
||||
|
||||
Returns list of dicts with keys:
|
||||
cited_id (str) — arXiv ID, DOI, or OpenAlex W-ID of the target
|
||||
cited_id (str) — canonical id of the referenced (not-yet-ingested) work
|
||||
pull (int) — how many already-ingested papers cite this target
|
||||
|
||||
Ordered by pull DESC, limited to `limit` rows.
|
||||
Ordered by pull DESC, limited to `limit` rows. References that resolve to a
|
||||
paper already in the corpus are excluded — they are not leads.
|
||||
"""
|
||||
sql = """
|
||||
sql = f"""
|
||||
WITH resolved AS ({RESOLVED_CITATIONS_SQL})
|
||||
SELECT cited_id, count(*) AS pull
|
||||
FROM citations
|
||||
FROM resolved
|
||||
WHERE cited_id NOT IN (SELECT id FROM papers)
|
||||
GROUP BY cited_id
|
||||
ORDER BY pull DESC
|
||||
@@ -29,7 +39,10 @@ def discovery_leads(limit: int = 20) -> list[dict[str, int | str]]:
|
||||
|
||||
def citing_papers(paper_id: str) -> list[str]:
|
||||
"""Return IDs of all papers that cite `paper_id` (in-graph reverse citations)."""
|
||||
sql = "SELECT citing_id FROM citations WHERE cited_id = %(paper_id)s"
|
||||
sql = f"""
|
||||
WITH resolved AS ({RESOLVED_CITATIONS_SQL})
|
||||
SELECT citing_id FROM resolved WHERE cited_id = %(paper_id)s
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(sql, {"paper_id": paper_id}).fetchall()
|
||||
return [row["citing_id"] for row in rows]
|
||||
@@ -37,7 +50,10 @@ def citing_papers(paper_id: str) -> list[str]:
|
||||
|
||||
def cited_by(paper_id: str) -> list[str]:
|
||||
"""Return IDs of all papers that `paper_id` cites (forward citations)."""
|
||||
sql = "SELECT cited_id FROM citations WHERE citing_id = %(paper_id)s"
|
||||
sql = f"""
|
||||
WITH resolved AS ({RESOLVED_CITATIONS_SQL})
|
||||
SELECT cited_id FROM resolved WHERE citing_id = %(paper_id)s
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(sql, {"paper_id": paper_id}).fetchall()
|
||||
return [row["cited_id"] for row in rows]
|
||||
@@ -49,10 +65,11 @@ def cocited_papers(paper_id: str, limit: int = 10) -> list[dict[str, int | str]]
|
||||
A paper X is co-cited with `paper_id` if some paper cites both.
|
||||
Returns list of dicts: {paper_id: str, co_citations: int}, ordered DESC.
|
||||
"""
|
||||
sql = """
|
||||
sql = f"""
|
||||
WITH resolved AS ({RESOLVED_CITATIONS_SQL})
|
||||
SELECT c2.cited_id AS paper_id, count(*) AS co_citations
|
||||
FROM citations c1
|
||||
JOIN citations c2 ON c1.citing_id = c2.citing_id
|
||||
FROM resolved c1
|
||||
JOIN resolved c2 ON c1.citing_id = c2.citing_id
|
||||
WHERE c1.cited_id = %(paper_id)s
|
||||
AND c2.cited_id != %(paper_id)s
|
||||
GROUP BY c2.cited_id
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Hybrid dense + sparse embeddings via BGE-M3 (ADR-0002).
|
||||
"""Dense (+ optional sparse) embeddings via BGE-M3 (ADR-0002).
|
||||
|
||||
Uses :class:`FlagEmbedding.BGEM3FlagModel` for encoding because the
|
||||
``return_dense`` / ``return_sparse`` kwargs are part of the FlagEmbedding
|
||||
@@ -7,6 +7,11 @@ vanilla ``SentenceTransformer`` load of ``BAAI/bge-m3`` (same weights,
|
||||
same model); sparse output is a list of ``{token_id: weight}`` dicts per
|
||||
text.
|
||||
|
||||
Status: only the **dense** path is wired into ingest/search today; the search
|
||||
"hybrid" is dense + Postgres FTS. :meth:`Embedder.encode_sparse` / :meth:`encode`
|
||||
are provided for a future sparse-retrieval layer but are not yet consumed by the
|
||||
pipeline (audit C-14).
|
||||
|
||||
Notes for callers
|
||||
-----------------
|
||||
* Empty input is handled explicitly — no model call is issued.
|
||||
|
||||
@@ -23,6 +23,23 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_MIN_PAGERANK_NODES = 5
|
||||
|
||||
# Single source of truth for resolving ``citations.cited_id`` to a canonical
|
||||
# paper id. ``cited_id`` stores OpenAlex IDs while ``papers.id`` stores DOIs /
|
||||
# arXiv ids, so an in-KB reference cited by its OpenAlex id does not match
|
||||
# ``papers.id`` directly. This resolves it to the canonical ``papers.id`` (via
|
||||
# ``papers.openalex_id``, then the ``paper_identifiers`` alias table); dangling
|
||||
# references keep their raw id. Both :func:`build_citation_graph` and
|
||||
# :mod:`codex.discover` query through this so the "what counts as ingested" rule
|
||||
# cannot drift between them again (audit C-1). Trusted constant — no user input.
|
||||
RESOLVED_CITATIONS_SQL = """
|
||||
SELECT c.citing_id,
|
||||
COALESCE(p.id, pi.paper_id, c.cited_id) AS cited_id
|
||||
FROM citations c
|
||||
LEFT JOIN papers p ON p.openalex_id = c.cited_id
|
||||
LEFT JOIN paper_identifiers pi
|
||||
ON pi.openalex_id = c.cited_id AND p.id IS NULL
|
||||
"""
|
||||
|
||||
|
||||
def build_citation_graph(conn: Any) -> nx.DiGraph:
|
||||
"""Load the ``citations`` table into a directed graph.
|
||||
@@ -45,15 +62,7 @@ def build_citation_graph(conn: Any) -> nx.DiGraph:
|
||||
-------
|
||||
nx.DiGraph with every (citing_id, cited_id) pair as an edge.
|
||||
"""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.citing_id, COALESCE(p.id, pi.paper_id, c.cited_id) AS cited_id
|
||||
FROM citations c
|
||||
LEFT JOIN papers p ON p.openalex_id = c.cited_id
|
||||
LEFT JOIN paper_identifiers pi
|
||||
ON pi.openalex_id = c.cited_id AND p.id IS NULL
|
||||
"""
|
||||
).fetchall()
|
||||
rows = conn.execute(RESOLVED_CITATIONS_SQL).fetchall()
|
||||
g: nx.DiGraph = nx.DiGraph()
|
||||
for row in rows:
|
||||
g.add_edge(row["citing_id"], row["cited_id"])
|
||||
|
||||
107
codex/ingest.py
107
codex/ingest.py
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import psycopg
|
||||
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
@@ -205,9 +206,18 @@ def ingest_paper(
|
||||
if paper is None:
|
||||
raise ValueError(f"Paper not found: {paper_id}")
|
||||
|
||||
# Canonical identity is the caller-supplied id — what the CLI / ingest scripts
|
||||
# use and what papers.id is contracted to hold (arXiv id or DOI). OpenAlex's
|
||||
# fetch_paper fills paper.id with the URL-form doi/id (audit C-7); since DQ-5
|
||||
# `openalex._canonical_id` already normalizes that to the bare form, this pin
|
||||
# is belt-and-suspenders and keeps re-ingest idempotent regardless. The
|
||||
# OpenAlex work id is retained as openalex_id and in paper_identifiers.
|
||||
paper.id = paper_id.strip()
|
||||
|
||||
# DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute
|
||||
# some publishers' abstracts). Supplement from S2, then Crossref, so the
|
||||
# paper gets a real (non-zero) abstract embedding instead of a zero vector.
|
||||
# some publishers' abstracts). Supplement from S2, then Crossref (uses the
|
||||
# canonical paper.id set above), so the paper gets a real (non-zero) abstract
|
||||
# embedding instead of a zero vector.
|
||||
if not (paper.abstract or "").strip():
|
||||
recovered = _recover_abstract(paper)
|
||||
if recovered:
|
||||
@@ -232,35 +242,64 @@ def ingest_paper(
|
||||
# ---------------------------------------------------------------
|
||||
# 3. Upsert paper (INSERT … ON CONFLICT (id) DO UPDATE SET …)
|
||||
# ---------------------------------------------------------------
|
||||
upsert_sql = """
|
||||
INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
|
||||
source_path, abstract_emb)
|
||||
VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
|
||||
%(abstract)s, %(source_path)s, %(abstract_emb)s)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
openalex_id = EXCLUDED.openalex_id,
|
||||
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
|
||||
title = EXCLUDED.title,
|
||||
authors = EXCLUDED.authors,
|
||||
year = EXCLUDED.year,
|
||||
abstract = EXCLUDED.abstract,
|
||||
source_path = COALESCE(EXCLUDED.source_path, papers.source_path),
|
||||
abstract_emb = EXCLUDED.abstract_emb
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
|
||||
source_path, abstract_emb)
|
||||
VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
|
||||
%(abstract)s, %(source_path)s, %(abstract_emb)s)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
openalex_id = EXCLUDED.openalex_id,
|
||||
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
|
||||
title = EXCLUDED.title,
|
||||
authors = EXCLUDED.authors,
|
||||
year = EXCLUDED.year,
|
||||
abstract = EXCLUDED.abstract,
|
||||
source_path = COALESCE(EXCLUDED.source_path, papers.source_path),
|
||||
abstract_emb = EXCLUDED.abstract_emb
|
||||
""",
|
||||
{
|
||||
"id": paper.id,
|
||||
"openalex_id": paper.openalex_id,
|
||||
"bibkey": paper.bibkey,
|
||||
"title": paper.title,
|
||||
"authors": paper.authors,
|
||||
"year": paper.year,
|
||||
"abstract": paper.abstract,
|
||||
"source_path": source_path,
|
||||
"abstract_emb": abstract_emb,
|
||||
},
|
||||
)
|
||||
# The id-keyed upsert does not catch a papers.bibkey UNIQUE collision (two
|
||||
# same-author/year papers generate the same key). On such a collision retry
|
||||
# with an a/b/c… suffix instead of failing the whole ingest (audit C-15).
|
||||
base_bibkey = paper.bibkey
|
||||
for attempt in range(27):
|
||||
try:
|
||||
conn.execute(
|
||||
upsert_sql,
|
||||
{
|
||||
"id": paper.id,
|
||||
"openalex_id": paper.openalex_id,
|
||||
"bibkey": paper.bibkey,
|
||||
"title": paper.title,
|
||||
"authors": paper.authors,
|
||||
"year": paper.year,
|
||||
"abstract": paper.abstract,
|
||||
"source_path": source_path,
|
||||
"abstract_emb": abstract_emb,
|
||||
},
|
||||
)
|
||||
break
|
||||
except psycopg.errors.UniqueViolation as exc:
|
||||
msg = str(exc).lower()
|
||||
conn.rollback()
|
||||
if "openalex" in msg:
|
||||
# Two different papers.id claim the same OpenAlex work — a real data
|
||||
# conflict, not something to auto-rename. Surface it clearly instead
|
||||
# of the raw psycopg error (companion gap to C-15's bibkey case).
|
||||
raise ValueError(
|
||||
f"openalex_id {paper.openalex_id!r} is already attached to a "
|
||||
f"different paper; ingesting '{paper.id}' would duplicate it — "
|
||||
"resolve the conflicting paper first."
|
||||
) from exc
|
||||
if base_bibkey is None or "bibkey" not in msg or attempt == 26:
|
||||
raise
|
||||
paper.bibkey = f"{base_bibkey}{chr(ord('a') + attempt)}"
|
||||
logger.warning(
|
||||
"bibkey %r already exists; ingesting '%s' as %r instead",
|
||||
base_bibkey,
|
||||
paper.id,
|
||||
paper.bibkey,
|
||||
)
|
||||
|
||||
# Register openalex_id in paper_identifiers (alias table).
|
||||
# Every ingest records the primary openalex_id so the graph JOIN
|
||||
@@ -429,7 +468,10 @@ def ingest_paper(
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
""",
|
||||
[
|
||||
(f.paper_id, f.page, f.raw_latex, f.context, f.eq_label)
|
||||
# paper.id, not f.paper_id: the extractor derives its
|
||||
# paper_id from the PDF filename stem, which need not match
|
||||
# papers.id and would violate the FK (audit C-10).
|
||||
(paper.id, f.page, f.raw_latex, f.context, f.eq_label)
|
||||
for f in formulas
|
||||
],
|
||||
)
|
||||
@@ -443,7 +485,8 @@ def ingest_paper(
|
||||
VALUES (%s, %s, %s, %s)
|
||||
""",
|
||||
[
|
||||
(fig.paper_id, fig.page, fig.caption, fig.image_path)
|
||||
# paper.id, not fig.paper_id (filename stem) — see C-10 above.
|
||||
(paper.id, fig.page, fig.caption, fig.image_path)
|
||||
for fig in figures
|
||||
],
|
||||
)
|
||||
|
||||
@@ -24,6 +24,7 @@ Transport
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
@@ -121,7 +122,6 @@ def wiki_read(concept_slug: str) -> dict[str, object]:
|
||||
directory or the requested page does not exist.
|
||||
"""
|
||||
try:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from codex.config import get_settings
|
||||
@@ -135,16 +135,12 @@ def wiki_read(concept_slug: str) -> dict[str, object]:
|
||||
|
||||
markdown = page_path.read_text(encoding="utf-8")
|
||||
|
||||
# Collect cited bibkeys from the compile state
|
||||
state_path = wiki_dir / ".compile-state.json"
|
||||
sources: list[str] = []
|
||||
if state_path.exists():
|
||||
try:
|
||||
state: dict[str, str] = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
if concept_slug in state:
|
||||
sources = [concept_slug]
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
# Collect the actual cited bibkeys from the page's inline [BibKey #loc]
|
||||
# citations (audit C-3: this previously returned the concept slug itself,
|
||||
# not the cited sources).
|
||||
from codex.wiki import _parse_claims
|
||||
|
||||
sources = sorted({claim.bibkey for claim in _parse_claims(markdown)})
|
||||
|
||||
return {"markdown": markdown, "sources": sources}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
@@ -298,12 +294,13 @@ def _require_http_token() -> str:
|
||||
from codex.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
if not settings.mcp_auth_token:
|
||||
token = settings.mcp_auth_token.get_secret_value() if settings.mcp_auth_token else ""
|
||||
if not token:
|
||||
raise RuntimeError(
|
||||
"HTTP transport requires MCP_AUTH_TOKEN to be set. "
|
||||
"Set the environment variable or add it to .env."
|
||||
)
|
||||
return settings.mcp_auth_token
|
||||
return token
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -337,7 +334,9 @@ def main() -> None:
|
||||
self, request: Request, call_next: RequestResponseEndpoint
|
||||
) -> Response:
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if auth != f"Bearer {token}":
|
||||
# Constant-time comparison to avoid a token-length/equality timing
|
||||
# side-channel (audit S-2).
|
||||
if not secrets.compare_digest(auth, f"Bearer {token}"):
|
||||
return Response("Unauthorized", status_code=401)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
@@ -302,8 +302,10 @@ def extract_formulas(
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
||||
app_id = mathpix_app_id or settings.mathpix_app_id or ""
|
||||
app_key = mathpix_app_key or settings.mathpix_app_key or ""
|
||||
settings_id = settings.mathpix_app_id.get_secret_value() if settings.mathpix_app_id else ""
|
||||
settings_key = settings.mathpix_app_key.get_secret_value() if settings.mathpix_app_key else ""
|
||||
app_id = mathpix_app_id or settings_id
|
||||
app_key = mathpix_app_key or settings_key
|
||||
|
||||
if app_id and app_key:
|
||||
logger.info("Using MathPix backend for %s", pdf_path)
|
||||
|
||||
@@ -118,10 +118,7 @@ def fetch_source(arxiv_id: str) -> str | None:
|
||||
file is present (signals Nougat fallback).
|
||||
"""
|
||||
url = f"{_BASE}/src/{arxiv_id}"
|
||||
try:
|
||||
response = httpx.get(url, timeout=60, follow_redirects=True)
|
||||
except httpx.RequestError:
|
||||
raise
|
||||
response = httpx.get(url, timeout=60, follow_redirects=True)
|
||||
if response.status_code == 404:
|
||||
logger.debug("arXiv 404 for source id=%s", arxiv_id)
|
||||
return None
|
||||
|
||||
@@ -14,6 +14,7 @@ import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
||||
@@ -77,7 +78,9 @@ def fetch_references(paper_id: str) -> list[Citation]:
|
||||
list[Citation]
|
||||
One Citation per reference, with optional context snippet.
|
||||
"""
|
||||
url = f"{_BASE_GRAPH}/paper/{paper_id}/references"
|
||||
# quote the id so a legacy-arXiv "/" (e.g. arXiv:math/0603097) doesn't break
|
||||
# the URL path; keep ":" for the arXiv:/DOI: scheme prefix (audit R-5).
|
||||
url = f"{_BASE_GRAPH}/paper/{quote(paper_id, safe=':')}/references"
|
||||
params: dict[str, Any] = {"fields": "externalIds,contexts"}
|
||||
try:
|
||||
response = _get(url, params=params)
|
||||
@@ -140,7 +143,7 @@ def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]:
|
||||
list[str]
|
||||
List of recommended paper IDs.
|
||||
"""
|
||||
url = f"{_BASE_RECS}/papers/forpaper/{paper_id}"
|
||||
url = f"{_BASE_RECS}/papers/forpaper/{quote(paper_id, safe=':')}"
|
||||
params: dict[str, Any] = {"limit": limit}
|
||||
try:
|
||||
response = _get(url, params=params)
|
||||
|
||||
@@ -248,9 +248,23 @@ def _grounded_ratio(claims: list[Claim]) -> float:
|
||||
return n_grounded / len(claims)
|
||||
|
||||
|
||||
def _make_lead_id(seq: int) -> str:
|
||||
"""Format a sequential lead id like ``L-0001``."""
|
||||
return f"L-{seq:04d}"
|
||||
_KIND_PREFIX: dict[LeadKind, str] = {
|
||||
"connection": "C",
|
||||
"gap": "G",
|
||||
"improvement": "I",
|
||||
"conjecture": "X",
|
||||
}
|
||||
|
||||
|
||||
def _make_lead_id(seq: int, kind: LeadKind) -> str:
|
||||
"""Format a per-kind sequential lead id like ``L-C-0001``.
|
||||
|
||||
Each synthesis stage numbers its leads from 1, so without a kind prefix a
|
||||
connection, a gap, and an improvement all produce ``L-0001`` and clobber
|
||||
one another in the shared ``grounded/`` directory (audit C-11). The kind
|
||||
prefix namespaces the ids so they stay distinct.
|
||||
"""
|
||||
return f"L-{_KIND_PREFIX[kind]}-{seq:04d}"
|
||||
|
||||
|
||||
def _safe_llm_generate(llm: LLMClient, prompt: str, model: str) -> str:
|
||||
@@ -296,7 +310,7 @@ def _finalise_grounded_lead(
|
||||
return None
|
||||
marked = _mark_ungrounded(body, claims)
|
||||
return Lead(
|
||||
id=_make_lead_id(seq),
|
||||
id=_make_lead_id(seq, kind),
|
||||
kind=kind,
|
||||
title=title,
|
||||
body=marked,
|
||||
@@ -402,11 +416,12 @@ def find_gaps(
|
||||
|
||||
Two complementary signals are used:
|
||||
|
||||
* **Topic coverage map** — for each topic in ``topics`` (or, if
|
||||
``None``, the union of paper titles), retrieve chunks and check
|
||||
bibkey coverage. A topic covered by < ``synthesis_gap_min_coverage``
|
||||
bibkeys is reported as a gap candidate and the LLM is asked to
|
||||
articulate it.
|
||||
* **Topic coverage map** — for each **explicitly-requested** topic in
|
||||
``topics``, retrieve chunks and check bibkey coverage. A topic covered by
|
||||
< ``synthesis_gap_min_coverage`` bibkeys is a gap candidate and the LLM is
|
||||
asked to articulate it. Defaulting to paper titles flagged almost every
|
||||
paper as a gap (a title retrieves mostly its own single bibkey), so the
|
||||
coverage map is opt-in now; pass ``topics`` / CLI ``--topic`` (audit R-8).
|
||||
* **Code-vs-corpus gap** — when ``lib_path`` is given, any @cite
|
||||
bibkey scanned from code that is NOT present in the corpus is
|
||||
flagged. This catches the "cited in code, never ingested" case.
|
||||
@@ -415,9 +430,9 @@ def find_gaps(
|
||||
seq = 1
|
||||
leads: list[Lead] = []
|
||||
|
||||
# Coverage-map probes are the explicit topics only — no default-to-titles,
|
||||
# which turned every paper into a spurious gap (audit R-8).
|
||||
probe_topics: list[str] = list(topics or [])
|
||||
if not probe_topics:
|
||||
probe_topics = [title for _, title in _list_paper_titles(db_conn)]
|
||||
|
||||
known_bibkeys = {bk for bk, _ in _list_paper_titles(db_conn)}
|
||||
|
||||
@@ -428,7 +443,7 @@ def find_gaps(
|
||||
# No coverage at all → record the gap directly (LLM not strictly needed).
|
||||
leads.append(
|
||||
Lead(
|
||||
id=_make_lead_id(seq),
|
||||
id=_make_lead_id(seq, "gap"),
|
||||
kind="gap",
|
||||
title=f"Gap: corpus has no material on '{topic}'",
|
||||
body=(
|
||||
@@ -481,7 +496,7 @@ def find_gaps(
|
||||
for bibkey in sorted(missing):
|
||||
leads.append(
|
||||
Lead(
|
||||
id=_make_lead_id(seq),
|
||||
id=_make_lead_id(seq, "gap"),
|
||||
kind="gap",
|
||||
title=f"Gap: code cites '{bibkey}' but corpus does not contain it",
|
||||
body=(
|
||||
@@ -658,7 +673,7 @@ def propose_conjectures(
|
||||
continue
|
||||
leads.append(
|
||||
Lead(
|
||||
id=_make_lead_id(seq),
|
||||
id=_make_lead_id(seq, "conjecture"),
|
||||
kind="conjecture",
|
||||
title=title,
|
||||
body=body,
|
||||
@@ -759,10 +774,11 @@ def write_leads(leads: list[Lead], leads_dir: str) -> None:
|
||||
* Grounded leads → ``leads_dir/grounded/<id>.md``
|
||||
* Conjectures → ``leads_dir/conjectures/<id>.md`` (HARD invariant)
|
||||
|
||||
The function ALSO maintains an append-only ``leads_dir/INDEX.md``:
|
||||
a previously-recorded id is kept on subsequent runs even if it is
|
||||
overwritten with a newer body. The two sections (grounded /
|
||||
conjectures) are visibly separated.
|
||||
The function ALSO rebuilds ``leads_dir/INDEX.md`` from the on-disk files on
|
||||
every run (see :func:`_update_index`): it reflects whatever lead files
|
||||
currently exist under ``grounded/`` and ``conjectures/``, listed as two
|
||||
visibly separated sections. It is not append-only — a deleted lead file
|
||||
drops out of the index on the next run.
|
||||
|
||||
Notes
|
||||
-----
|
||||
|
||||
102
codex/wiki.py
102
codex/wiki.py
@@ -186,8 +186,15 @@ def _retrieve_chunks(
|
||||
).fetchall()
|
||||
|
||||
# Filter reference-list chunks: skip chunks whose content looks like a bibliography
|
||||
# (heuristic: > 60 % of lines match "^\[\d+\]" or "^[A-Z][a-z]+,?\s+[A-Z]\.").
|
||||
ref_pattern = re.compile(r"^\s*(\[\d+\]|[A-Z][a-z]+,?\s+[A-Z]\.)", re.MULTILINE)
|
||||
# (heuristic: > 60 % of lines look like "[12]" refs or "Surname, I." authors).
|
||||
# The author alternative excludes common math-structure words ("Theorem A.",
|
||||
# "Lemma B.") so theorem-dense chunks aren't mistaken for a reference list (R-2).
|
||||
ref_pattern = re.compile(
|
||||
r"^\s*(\[\d+\]|(?!(?:Theorem|Lemma|Proposition|Corollary|Definition|Proof|"
|
||||
r"Section|Figure|Fig|Table|Equation|Eq|Remark|Example|Chapter)\b)"
|
||||
r"[A-Z][a-z]+,?\s+[A-Z]\.)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
filtered: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
content: str = row["content"]
|
||||
@@ -273,7 +280,13 @@ _STOPWORDS: frozenset[str] = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
_SENTENCE_SPLIT_RE = re.compile(r"[.!?]")
|
||||
# Split on sentence-ending punctuation only when followed by whitespace/end, so a
|
||||
# decimal like "1.5" does not create a spurious sentence boundary (audit R-1).
|
||||
_SENTENCE_SPLIT_RE = re.compile(r"[.!?]+(?=\s|$)")
|
||||
|
||||
# Punctuation stripped from token edges before grounding comparison — sentence/
|
||||
# clause marks and quotes, NOT math delimiters like ()=+ (audit R-1).
|
||||
_EDGE_PUNCT = ".,;:!?\"'"
|
||||
|
||||
|
||||
def _last_sentence(text: str) -> str:
|
||||
@@ -292,8 +305,16 @@ def _last_sentence(text: str) -> str:
|
||||
|
||||
|
||||
def _content_words(text: str) -> list[str]:
|
||||
"""Return lowercased tokens with stopwords removed."""
|
||||
return [w for w in text.lower().split() if w not in _STOPWORDS]
|
||||
"""Return lowercased content tokens, edge-punctuation-stripped, stopwords removed.
|
||||
|
||||
Stripping leading/trailing sentence punctuation makes grounding robust to
|
||||
surface variation ("volume," vs "volume") so a faithful paraphrase is not
|
||||
marked ungrounded over a comma (audit R-1). Math delimiters (``()=+``) are
|
||||
preserved. Claim and chunk tokens are normalised identically, so matching
|
||||
stays consistent.
|
||||
"""
|
||||
tokens = (w.strip(_EDGE_PUNCT) for w in text.lower().split())
|
||||
return [w for w in tokens if w and w not in _STOPWORDS]
|
||||
|
||||
|
||||
def _run_grounding_guard(
|
||||
@@ -322,8 +343,9 @@ def _run_grounding_guard(
|
||||
for chunk in chunks:
|
||||
bk = str(chunk.get("bibkey") or "")
|
||||
if bk:
|
||||
# Content words of the chunk (stopwords removed, lowercased)
|
||||
cw = " ".join(_content_words(chunk["content"]))
|
||||
# Content words of the chunk, space-padded so a 5-gram match is bounded
|
||||
# by whole tokens, not a substring bleeding across words (audit C-6).
|
||||
cw = " " + " ".join(_content_words(chunk["content"])) + " "
|
||||
bib_index.setdefault(bk, []).append(cw)
|
||||
|
||||
for claim in claims:
|
||||
@@ -345,7 +367,7 @@ def _run_grounding_guard(
|
||||
|
||||
found = False
|
||||
for i in range(len(content) - 4): # content-5-grams
|
||||
gram = " ".join(content[i : i + 5])
|
||||
gram = " " + " ".join(content[i : i + 5]) + " " # padded: token-boundary match
|
||||
if any(gram in src for src in sources):
|
||||
found = True
|
||||
break
|
||||
@@ -402,6 +424,13 @@ def _detect_conflicts(
|
||||
|
||||
_INLINE_CODE_RE = re.compile(r"`[^`]+`")
|
||||
|
||||
# LaTeX math spans — protected from cross-ref injection so a concept name inside
|
||||
# a formula is not rewritten to a [[slug]] link, corrupting the LaTeX (audit R-9).
|
||||
_MATH_SPAN_RE = re.compile(
|
||||
r"\$\$.*?\$\$|\$[^$\n]*?\$|\\\(.*?\\\)|\\\[.*?\\\]",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _inject_cross_refs(
|
||||
markdown: str,
|
||||
@@ -411,11 +440,12 @@ def _inject_cross_refs(
|
||||
"""Replace occurrences of other concept titles/aliases with ``[[slug]]`` links.
|
||||
|
||||
Only exact case-insensitive whole-word matches outside of existing
|
||||
``[[…]]`` blocks or inline code spans are replaced.
|
||||
Inline-code spans (`` `…` ``) are temporarily protected by null-byte
|
||||
placeholders and restored after injection.
|
||||
``[[…]]`` blocks, inline code spans, or LaTeX math spans are replaced.
|
||||
Inline-code (`` `…` ``) and math (``$…$``, ``$$…$$``, ``\\(…\\)``,
|
||||
``\\[…\\]``) spans are temporarily protected by null-byte placeholders and
|
||||
restored after injection.
|
||||
"""
|
||||
# Step 1: protect inline-code spans from replacement
|
||||
# Step 1: protect inline-code and math spans from replacement
|
||||
placeholders: dict[str, str] = {}
|
||||
|
||||
def _protect(m: re.Match[str]) -> str:
|
||||
@@ -424,6 +454,7 @@ def _inject_cross_refs(
|
||||
return key
|
||||
|
||||
markdown = _INLINE_CODE_RE.sub(_protect, markdown)
|
||||
markdown = _MATH_SPAN_RE.sub(_protect, markdown) # don't rewrite names inside LaTeX (R-9)
|
||||
|
||||
# Step 2: inject cross-refs on unprotected text
|
||||
for concept in all_concepts:
|
||||
@@ -602,14 +633,14 @@ def compile_concept(
|
||||
claims = _parse_claims(raw_output)
|
||||
claims = _run_grounding_guard(claims, chunks)
|
||||
|
||||
_all_concepts = all_concepts or []
|
||||
raw_output = _inject_cross_refs(raw_output, _all_concepts, concept.slug)
|
||||
|
||||
# Graceful: try to embed formula chunks (F-09) — skip if table missing
|
||||
# Graceful: formula-embedding hook (F-09) — currently a no-op (audit C-12).
|
||||
_try_embed_formulas(concept, chunks)
|
||||
|
||||
compiled_at = datetime.now(UTC)
|
||||
# Mark ungrounded claims on the raw body FIRST, then inject cross-refs: a
|
||||
# concept name inside a claim must not stop the ⚠ regex from matching (C-13).
|
||||
markdown = _render_page_markdown(concept, raw_output, claims, compiled_at)
|
||||
markdown = _inject_cross_refs(markdown, all_concepts or [], concept.slug)
|
||||
|
||||
return ConceptPage(
|
||||
concept=concept,
|
||||
@@ -621,20 +652,13 @@ def compile_concept(
|
||||
|
||||
|
||||
def _try_embed_formulas(concept: Concept, chunks: list[dict[str, Any]]) -> None:
|
||||
"""Attempt to look up formula chunks for the concept — graceful no-op if F-09 absent."""
|
||||
try:
|
||||
from codex.db import get_conn
|
||||
"""Reserved hook for embedding F-09 formula chunks into a page.
|
||||
|
||||
with get_conn() as conn:
|
||||
# Check if formulas table exists
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_name = 'formulas'"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return # F-09 not present
|
||||
# (Future: embed relevant raw_latex into the page)
|
||||
except Exception: # noqa: BLE001
|
||||
return # DB not reachable or other error — degrade gracefully
|
||||
Currently a no-op — formula embedding is not implemented yet. Kept as a stable
|
||||
seam for compile_concept's call site. The previous body issued a per-concept
|
||||
DB round-trip (information_schema lookup) that did nothing (audit C-12).
|
||||
"""
|
||||
return
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -722,17 +746,23 @@ def compile_all(
|
||||
# Detect conflicts between chunks from different bibkeys
|
||||
report.conflicts.extend(_detect_conflicts(concept.slug, chunks))
|
||||
|
||||
# Quarantine check: compute grounding rate
|
||||
# Quarantine check: compute grounding rate.
|
||||
# A page with NO citations at all (total_claims == 0) carries zero grounding
|
||||
# evidence — this also covers the LLM-outage case where compile_concept
|
||||
# returns an empty page — and must NOT be published (audit C-4). Previously
|
||||
# the `total_claims > 0` guard let such pages fall through to wiki/.
|
||||
total_claims = len(page.claims)
|
||||
grounded_claims = sum(1 for c in page.claims if c.grounded)
|
||||
grounding_rate = grounded_claims / total_claims if total_claims > 0 else 0.0
|
||||
|
||||
if total_claims > 0 and grounding_rate < settings.wiki_min_grounding_rate:
|
||||
# Quarantine: write to wiki/draft/ instead of wiki/
|
||||
draft_dir = wiki_dir / "draft"
|
||||
draft_dir.mkdir(parents=True, exist_ok=True)
|
||||
page_path = draft_dir / f"{concept.slug}.md"
|
||||
page_path.write_text(page.markdown, encoding="utf-8")
|
||||
if total_claims == 0 or grounding_rate < settings.wiki_min_grounding_rate:
|
||||
# Quarantine and do NOT update the compile-state, so a transient failure
|
||||
# (e.g. LLM outage) is retried on the next run instead of being frozen as
|
||||
# "compiled". An empty body is not worth a draft file — just record it.
|
||||
if page.markdown.strip():
|
||||
draft_dir = wiki_dir / "draft"
|
||||
draft_dir.mkdir(parents=True, exist_ok=True)
|
||||
(draft_dir / f"{concept.slug}.md").write_text(page.markdown, encoding="utf-8")
|
||||
report.quarantined.append(concept.slug)
|
||||
else:
|
||||
# Write page to wiki/
|
||||
|
||||
Reference in New Issue
Block a user