9e5184385a
Resolved 4 conflicts: cli.py (R-D citing-coverage warning + main's small-corpus JSON/hub-title changes both kept); ingest.py (C-7 paper.id=caller-id pin first, then DQ-2 abstract recovery); test_ingest.py + test_openalex.py (kept both branches' tests). C-7/DQ-5 overlap (both canonicalise papers.id): kept both layers — openalex._canonical_id normalizes fetch_paper's id (DQ-5); ingest pins papers.id to the caller's bare id (C-7); they converge on the bare canonical id. Fixed the auto-merged test_fetch_paper_success that had contradictory bare/URL assertions. 402 tests pass; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
342 lines
11 KiB
Python
342 lines
11 KiB
Python
"""Application configuration via environment variables / .env file.
|
|
|
|
All settings are read from the environment (or a .env file in the project
|
|
root). Import :func:`get_settings` wherever you need configuration; the
|
|
returned object is cached after the first call.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
|
|
from pydantic import AliasChoices, Field, SecretStr
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
"""Centralised, env-driven configuration for the codex application."""
|
|
|
|
model_config = SettingsConfigDict(
|
|
env_file=".env",
|
|
env_file_encoding="utf-8",
|
|
case_sensitive=False,
|
|
extra="ignore",
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Database
|
|
# ------------------------------------------------------------------
|
|
database_url: SecretStr = Field(
|
|
default=SecretStr("postgresql://researcher:change_me@localhost:5432/papers"),
|
|
description=(
|
|
"libpq-compatible connection string consumed by psycopg. "
|
|
"Example: postgresql://user:pass@host:5432/dbname"
|
|
),
|
|
)
|
|
|
|
migration_database_url: SecretStr | None = Field(
|
|
default=None,
|
|
description=(
|
|
"Optional privileged connection string used by `codex migrate` to apply "
|
|
"schema DDL. The app's database_url is typically a least-privilege DML "
|
|
"role that cannot CREATE/ALTER owner-held tables (raises "
|
|
"InsufficientPrivilege); point this at an owner/superuser connection. "
|
|
"Falls back to database_url when unset."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# External services
|
|
# ------------------------------------------------------------------
|
|
grobid_url: str = Field(
|
|
default="http://localhost:8070",
|
|
description="Base URL of the GROBID HTTP API (containerised).",
|
|
)
|
|
|
|
nougat_url: str = Field(
|
|
default="http://localhost:8080",
|
|
validation_alias=AliasChoices("NOUGAT_URL", "nougat_url"),
|
|
description="Base URL of the Nougat OCR HTTP API (containerised).",
|
|
)
|
|
|
|
ollama_base_url: str = Field(
|
|
default="http://localhost:11434",
|
|
description="Base URL of the local Ollama endpoint (optional Q&A layer).",
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Embeddings
|
|
# ------------------------------------------------------------------
|
|
embedding_model: str = Field(
|
|
default="BAAI/bge-m3",
|
|
description=(
|
|
"sentence-transformers model identifier. "
|
|
"Must match EMBEDDING_DIM. Default: BAAI/bge-m3 (1024 dims)."
|
|
),
|
|
)
|
|
|
|
embedding_dim: int = Field(
|
|
default=1024,
|
|
gt=0,
|
|
description=(
|
|
"Dimension of the dense embedding vectors. "
|
|
"Must match the output dimension of EMBEDDING_MODEL."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# API etiquette
|
|
# ------------------------------------------------------------------
|
|
openalex_mailto: str = Field(
|
|
default="",
|
|
description=(
|
|
"E-mail address for the OpenAlex Polite Pool (faster rate limits). "
|
|
"Required by OpenAlex ToS for automated access."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# F-12 Wiki-Compile
|
|
# ------------------------------------------------------------------
|
|
wiki_dir: str = Field(
|
|
default="wiki/",
|
|
description=(
|
|
"Directory where compiled wiki pages are written. "
|
|
"Relative paths are resolved from the current working directory."
|
|
),
|
|
)
|
|
|
|
wiki_llm_model: str = Field(
|
|
default="qwen2.5:7b",
|
|
description=(
|
|
"Ollama model name used for wiki synthesis. "
|
|
"Must be available at the configured Ollama endpoint (WIKI_LLM_URL or "
|
|
"OLLAMA_BASE_URL). Default: qwen2.5:7b (qwen-light profile on Jetson)."
|
|
),
|
|
)
|
|
|
|
wiki_llm_url: str | None = Field(
|
|
default=None,
|
|
description=(
|
|
"Ollama base URL for wiki synthesis. "
|
|
"When None, falls back to OLLAMA_BASE_URL "
|
|
"(http://192.168.178.103:11434 for the Jetson). "
|
|
"Set WIKI_LLM_URL to override."
|
|
),
|
|
)
|
|
|
|
wiki_top_k: int = Field(
|
|
default=12,
|
|
gt=0,
|
|
description=(
|
|
"Number of top chunks retrieved per concept for wiki synthesis. "
|
|
"Higher values improve recall at the cost of a larger LLM prompt."
|
|
),
|
|
)
|
|
|
|
wiki_min_grounding_rate: float = Field(
|
|
default=0.5,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description=(
|
|
"Minimum fraction of claims that must be grounded for a page to be written "
|
|
"to wiki/<slug>.md. Pages below this threshold are quarantined to "
|
|
"wiki/draft/<slug>.md instead."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# F-09 Rich Parsing
|
|
# ------------------------------------------------------------------
|
|
mathpix_app_id: SecretStr | None = Field(
|
|
default=None,
|
|
description=(
|
|
"MathPix App ID for cloud formula extraction. "
|
|
"When None, pix2tex local OCR is used as fallback."
|
|
),
|
|
)
|
|
|
|
mathpix_app_key: SecretStr | None = Field(
|
|
default=None,
|
|
description=(
|
|
"MathPix App Key for cloud formula extraction. "
|
|
"Must be set together with MATHPIX_APP_ID."
|
|
),
|
|
)
|
|
|
|
pix2tex_fallback: bool = Field(
|
|
default=True,
|
|
description=(
|
|
"Enable pix2tex (local LaTeX OCR) as fallback when MathPix credentials "
|
|
"are absent. Set to False to disable formula extraction entirely without creds."
|
|
),
|
|
)
|
|
|
|
figures_dir: str = Field(
|
|
default="figures/",
|
|
description=(
|
|
"Directory where extracted figure images are written. "
|
|
"Relative paths are resolved from the current working directory."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# F-13 Synthesis / Lead-Engine
|
|
# ------------------------------------------------------------------
|
|
leads_dir: str = Field(
|
|
default="leads/",
|
|
description=(
|
|
"Directory where synthesis leads are written. "
|
|
"Grounded leads land in <leads_dir>/grounded/, conjectures in "
|
|
"<leads_dir>/conjectures/. INVARIANT: conjectures NEVER touch wiki/."
|
|
),
|
|
)
|
|
|
|
synthesis_llm_model: str = Field(
|
|
default="qwen2.5:7b",
|
|
description=(
|
|
"Ollama model name used for synthesis (connection / gap / improvement / "
|
|
"conjecture). Defaults to the qwen-light profile on the Jetson."
|
|
),
|
|
)
|
|
|
|
synthesis_llm_url: str | None = Field(
|
|
default=None,
|
|
description=("Ollama base URL for synthesis. When None, falls back to OLLAMA_BASE_URL."),
|
|
)
|
|
|
|
synthesis_top_k: int = Field(
|
|
default=20,
|
|
gt=0,
|
|
description=(
|
|
"Number of top chunks retrieved per synthesis probe. Higher values "
|
|
"improve recall at the cost of a larger LLM prompt."
|
|
),
|
|
)
|
|
|
|
synthesis_min_grounded_ratio: float = Field(
|
|
default=0.5,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description=(
|
|
"Minimum fraction of claims that must be grounded for a grounded lead "
|
|
"(connection / gap / improvement) to be emitted. Leads below this "
|
|
"threshold are dropped rather than written to leads/grounded/."
|
|
),
|
|
)
|
|
|
|
synthesis_gap_min_coverage: int = Field(
|
|
default=2,
|
|
gt=0,
|
|
description=(
|
|
"A topic is considered a 'gap' candidate when it is covered by fewer "
|
|
"than this many distinct bibkeys in the retrieved top-K."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# F-14 MCP Server
|
|
# ------------------------------------------------------------------
|
|
mcp_transport: str = Field(
|
|
default="stdio",
|
|
description=(
|
|
"Transport protocol for the MCP server. "
|
|
"Allowed values: 'stdio' (default) or 'http'. "
|
|
"HTTP transport requires MCP_AUTH_TOKEN to be set."
|
|
),
|
|
)
|
|
|
|
mcp_host: str = Field(
|
|
default="127.0.0.1",
|
|
description="Bind host for HTTP transport (ignored for stdio).",
|
|
)
|
|
|
|
mcp_port: int = Field(
|
|
default=8765,
|
|
gt=0,
|
|
description="Bind port for HTTP transport (ignored for stdio).",
|
|
)
|
|
|
|
mcp_auth_token: SecretStr | None = Field(
|
|
default=None,
|
|
description=(
|
|
"Bearer token required when mcp_transport='http'. "
|
|
"If HTTP transport is requested but this is None, the server refuses to start."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# F-16 Chunk Quality Gate
|
|
# ------------------------------------------------------------------
|
|
chunk_min_chars: int = Field(
|
|
default=60,
|
|
gt=0,
|
|
description=(
|
|
"Minimum character count for a chunk to pass the quality gate. "
|
|
"Chunks shorter than this value are discarded before embedding."
|
|
),
|
|
)
|
|
|
|
chunk_min_alpha_ratio: float = Field(
|
|
default=0.40,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description=(
|
|
"Minimum fraction of alphabetic characters in a chunk. "
|
|
"Chunks with a lower ratio are treated as OCR artefacts and discarded."
|
|
),
|
|
)
|
|
|
|
chunk_max_bib_score: float = Field(
|
|
default=0.70,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description=(
|
|
"Maximum bibliography heuristic score before a chunk is rejected. "
|
|
"Score is based on DOI density, 'et al.' and year-in-brackets patterns."
|
|
),
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# F-15 Literature Graph
|
|
# ------------------------------------------------------------------
|
|
graph_cite_boost_alpha: float = Field(
|
|
default=0.3,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description=(
|
|
"Weight of the PageRank citation-boost in hybrid search. "
|
|
"boosted_distance = distance / (1 + alpha * pagerank_score) — dividing "
|
|
"lowers the cosine distance of high-PageRank papers (lower = closer). "
|
|
"Set to 0.0 to disable the boost without removing the flag."
|
|
),
|
|
)
|
|
|
|
graph_min_corpus_size: int = Field(
|
|
default=15,
|
|
gt=0,
|
|
description=(
|
|
"Minimum number of ingested papers for citation_pagerank to yield "
|
|
"meaningful rankings. Below this threshold a warning is logged and "
|
|
"uniform scores are returned."
|
|
),
|
|
)
|
|
|
|
graph_min_citing_coverage: float = Field(
|
|
default=0.8,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description=(
|
|
"Minimum share of ingested papers that must have ≥1 out-edge (i.e. "
|
|
"cite something) before `codex graph report` warns. Low citing "
|
|
"coverage starves PageRank/coupling even when the paper count looks "
|
|
"healthy — the share of *citing* papers is what matters (R-D)."
|
|
),
|
|
)
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_settings() -> Settings:
|
|
"""Return the cached application settings singleton."""
|
|
return Settings()
|