feat: initial project scaffold

pyproject.toml (Python 3.12, uv), codex/ package (config, db, models),
infra/ (docker-compose + schema), .env.example, .gitignore, README.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.example

@@ -0,0 +1,25 @@
+# Copy this file to .env and fill in your values.
+# Never commit .env to version control.
+
+# PostgreSQL connection string (psycopg / asyncpg format)
+# Example: postgresql://researcher:change_me@localhost:5432/papers
+DATABASE_URL=postgresql://researcher:change_me@localhost:5432/papers
+
+# GROBID service base URL (containerised — see infra/docker-compose.yml)
+GROBID_URL=http://localhost:8070
+
+# Ollama base URL for optional local LLM Q&A layer
+OLLAMA_BASE_URL=http://localhost:11434
+
+# Sentence-transformers model for dense embeddings.
+# BGE-M3 (BAAI/bge-m3) produces 1024-dimensional vectors and supports
+# dense + sparse (hybrid) retrieval. Change together with EMBEDDING_DIM.
+EMBEDDING_MODEL=BAAI/bge-m3
+
+# Dimension of the embedding vectors. Must match EMBEDDING_MODEL output.
+# BGE-M3 = 1024 | Jina v4 = 2048 | Qwen3-Embedding-0.6B = 1024
+EMBEDDING_DIM=1024
+
+# E-mail address for the OpenAlex Polite Pool (faster rate limits).
+# Required by OpenAlex ToS when making automated requests.
+OPENALEX_MAILTO=you@example.com

.gitignore

@@ -0,0 +1,35 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+*.egg
+
+# Environment secrets — NEVER commit
+.env
+
+# Type-checker and linter caches
+.mypy_cache/
+.ruff_cache/
+
+# Test caches
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Editor / OS artefacts
+.DS_Store
+.idea/
+.vscode/
+*.swp

README.md

@@ -0,0 +1,113 @@
+# codex — Personal Paper Knowledge Base
+
+A self-hostable tool for managing scientific papers: ingest PDFs and arXiv
+sources, build a citation graph, and link C++ implementations back to the
+papers that describe them.
+
+---
+
+## Quick start
+
+```bash
+# 1. Start Postgres (pgvector) + GROBID
+docker compose -f infra/docker-compose.yml up -d
+
+# 2. Copy and edit environment variables
+cp .env.example .env
+$EDITOR .env
+
+# 3. Install Python dependencies (requires uv)
+uv sync
+
+# 4. Apply the database schema (first run only)
+uv run python -c "
+from codex.db import get_conn, apply_schema
+with get_conn() as conn:
+    apply_schema(conn)
+print('Schema applied.')
+"
+```
+
+---
+
+## Environment variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `DATABASE_URL` | `postgresql://researcher:change_me@localhost:5432/papers` | libpq connection string |
+| `GROBID_URL` | `http://localhost:8070` | GROBID HTTP API base URL |
+| `OLLAMA_BASE_URL` | `http://localhost:11434` | Local Ollama endpoint (optional) |
+| `EMBEDDING_MODEL` | `BAAI/bge-m3` | sentence-transformers model name |
+| `EMBEDDING_DIM` | `1024` | Embedding vector dimension (must match model) |
+| `OPENALEX_MAILTO` | *(empty)* | E-mail for OpenAlex Polite Pool (required for automated use) |
+
+See `.env.example` for a documented template.
+
+---
+
+## Three-layer data model
+
+All data lives in a single Postgres instance with the pgvector extension.
+
+### Layer 1 — Semantics (`papers`, `chunks`)
+
+Papers are stored with metadata and an abstract-level dense embedding
+(BGE-M3, 1024 dimensions). Full-text is split into `chunks`, each with its
+own dense embedding and a Postgres full-text (GIN) index.  
+Hybrid search combines nearest-neighbour vector search with keyword (FTS)
+retrieval for robust handling of exact mathematical terminology.
+
+### Layer 2 — Citations (`citations`)
+
+Directed edges in the citation graph.  The `cited_id` column has **no
+foreign-key constraint** on purpose: edges pointing to papers that have not
+yet been ingested are kept as-is.  Those "dangling" targets are your
+**discovery leads** — papers frequently cited by your collection that you
+have not yet read.
+
+```sql
+-- Top discovery leads
+SELECT cited_id, count(*) AS pull
+FROM   citations
+WHERE  cited_id NOT IN (SELECT id FROM papers)
+GROUP  BY cited_id
+ORDER  BY pull DESC
+LIMIT  20;
+```
+
+### Layer 3 — Provenance (`code_links`)
+
+Maps C++ symbols (qualified names or `file.cpp:line` references) to the
+papers they implement.  The workflow:
+
+1. Tag C++ source with `@cite <bibkey>` in Doxygen comments.
+2. Run `codex provenance sync --lib-path <path>` to scan and resolve tags.
+3. Run `codex provenance export-bib <out.bib>` to generate a `.bib` file
+   containing **only the cited subset** of your collection.
+
+The exported `.bib` is a derived view of the master catalogue — regenerate
+it at any time; it is not the source of truth.
+
+---
+
+## CLI reference (coming in F-07)
+
+```
+codex ingest <id>               # ingest one paper by arXiv ID or DOI
+codex ingest-file <ids.txt>     # bulk ingest from a file of IDs
+codex search "<query>" [--hybrid]
+codex discover leads
+codex provenance sync --lib-path <path>
+codex provenance export-bib <out.bib>
+codex ask "<question>"          # optional LLM Q&A via Ollama
+```
+
+---
+
+## Development
+
+```bash
+uv run ruff check . && uv run ruff format --check .   # lint
+uv run mypy codex/                                     # type-check
+uv run pytest                                          # tests
+```

codex/config.py

@@ -0,0 +1,85 @@
+"""Application configuration via environment variables / .env file.
+
+All settings are read from the environment (or a .env file in the project
+root).  Import :func:`get_settings` wherever you need configuration; the
+returned object is cached after the first call.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Centralised, env-driven configuration for the codex application."""
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    # ------------------------------------------------------------------
+    # Database
+    # ------------------------------------------------------------------
+    database_url: str = Field(
+        default="postgresql://researcher:change_me@localhost:5432/papers",
+        description=(
+            "libpq-compatible connection string consumed by psycopg. "
+            "Example: postgresql://user:pass@host:5432/dbname"
+        ),
+    )
+
+    # ------------------------------------------------------------------
+    # External services
+    # ------------------------------------------------------------------
+    grobid_url: str = Field(
+        default="http://localhost:8070",
+        description="Base URL of the GROBID HTTP API (containerised).",
+    )
+
+    ollama_base_url: str = Field(
+        default="http://localhost:11434",
+        description="Base URL of the local Ollama endpoint (optional Q&A layer).",
+    )
+
+    # ------------------------------------------------------------------
+    # Embeddings
+    # ------------------------------------------------------------------
+    embedding_model: str = Field(
+        default="BAAI/bge-m3",
+        description=(
+            "sentence-transformers model identifier. "
+            "Must match EMBEDDING_DIM. Default: BAAI/bge-m3 (1024 dims)."
+        ),
+    )
+
+    embedding_dim: int = Field(
+        default=1024,
+        gt=0,
+        description=(
+            "Dimension of the dense embedding vectors. "
+            "Must match the output dimension of EMBEDDING_MODEL."
+        ),
+    )
+
+    # ------------------------------------------------------------------
+    # API etiquette
+    # ------------------------------------------------------------------
+    openalex_mailto: str = Field(
+        default="",
+        description=(
+            "E-mail address for the OpenAlex Polite Pool (faster rate limits). "
+            "Required by OpenAlex ToS for automated access."
+        ),
+    )
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    """Return the cached application settings singleton."""
+    return Settings()

codex/db.py

@@ -0,0 +1,57 @@
+"""Database connection helpers.
+
+Provides:
+- :func:`get_conn` — a context manager that yields a psycopg connection.
+- :func:`apply_schema` — idempotently applies ``infra/schema.sql``.
+
+The connection pool (psycopg_pool) is intentionally deferred to F-05
+(ingest); here we expose a simple per-call ``psycopg.connect`` wrapper
+that is sufficient for schema migration and unit-testing without requiring
+an additional ``psycopg[pool]`` dependency at scaffold stage.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+
+import psycopg
+import psycopg.rows
+
+from codex.config import get_settings
+
+
+@contextmanager
+def get_conn() -> Generator[psycopg.Connection[psycopg.rows.DictRow], None, None]:
+    """Yield a psycopg connection with dict-row factory.
+
+    The connection is opened from :func:`codex.config.get_settings`
+    ``database_url`` and closed automatically when the context exits.
+
+    Usage::
+
+        with get_conn() as conn:
+            conn.execute("SELECT 1")
+    """
+    settings = get_settings()
+    with psycopg.connect(
+        settings.database_url,
+        row_factory=psycopg.rows.dict_row,
+    ) as conn:
+        yield conn
+
+
+def apply_schema(conn: psycopg.Connection[psycopg.rows.DictRow]) -> None:
+    """Idempotently apply ``infra/schema.sql`` to the connected database.
+
+    Safe to call on every startup — all DDL statements use
+    ``CREATE … IF NOT EXISTS``.
+
+    Args:
+        conn: An open psycopg connection (obtained via :func:`get_conn`).
+    """
+    schema_path = Path(__file__).parent.parent / "infra" / "schema.sql"
+    sql = schema_path.read_text(encoding="utf-8")
+    conn.execute(sql)
+    conn.commit()

codex/models.py

@@ -0,0 +1,80 @@
+"""Domain dataclasses.
+
+Each class maps directly to a table in ``infra/schema.sql``.
+Field names and types are kept in sync with the DDL.
+
+All fields that are nullable in the schema are typed as ``… | None``
+with a default of ``None`` so instances can be constructed incrementally.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass
+class Paper:
+    """Maps to the ``papers`` table (Layer 1 — semantic search).
+
+    ``id`` is the canonical identifier: an arXiv ID (e.g. ``2301.00001``)
+    or a DOI (e.g. ``10.1145/3592430``).
+    """
+
+    id: str
+    title: str
+    openalex_id: str | None = None
+    bibkey: str | None = None
+    authors: list[str] = field(default_factory=list)
+    year: int | None = None
+    abstract: str | None = None
+    source_path: str | None = None
+    # abstract_emb is a list of floats representing the pgvector column;
+    # pgvector's Python driver accepts list[float] for vector columns.
+    abstract_emb: list[float] | None = None
+    added_at: datetime | None = None
+
+
+@dataclass
+class Chunk:
+    """Maps to the ``chunks`` table (Layer 1 — full-text chunks).
+
+    ``id`` is set by the database (BIGSERIAL); leave it as ``None``
+    when constructing a new chunk before insertion.
+    """
+
+    paper_id: str
+    ord: int
+    content: str
+    id: int | None = None
+    embedding: list[float] | None = None
+
+
+@dataclass
+class Citation:
+    """Maps to the ``citations`` table (Layer 2 — citation graph).
+
+    ``cited_id`` intentionally has no foreign-key constraint in the schema;
+    citations to not-yet-ingested papers are preserved as discovery leads.
+    """
+
+    citing_id: str
+    cited_id: str
+    context: str | None = None
+
+
+@dataclass
+class CodeLink:
+    """Maps to the ``code_links`` table (Layer 3 — provenance).
+
+    ``symbol`` is a C++ qualified name or ``file.cpp:line`` reference.
+    ``role`` is a free-text description such as 'implements Thm 3.2'.
+    ``id`` is set by the database (BIGSERIAL).
+    """
+
+    symbol: str
+    paper_id: str | None = None
+    role: str | None = None
+    note: str | None = None
+    id: int | None = None
+    added_at: datetime | None = None

infra/docker-compose.yml

@@ -0,0 +1,47 @@
+# Paper knowledge base: Provenance + Citation graph + Semantic search
+# A single Postgres instance (pgvector) covers all three layers.
+# GROBID parses references / structure from PDFs without an arXiv source.
+#
+# Quick start:
+#   docker compose up -d
+#   psql "${DATABASE_URL}"          # see .env.example for default value
+
+services:
+  db:
+    image: pgvector/pgvector:pg17       # official pgvector image
+    container_name: papers-db
+    environment:
+      POSTGRES_DB: papers
+      POSTGRES_USER: researcher
+      POSTGRES_PASSWORD: change_me      # change before production use
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+      # schema.sql is executed automatically on first start:
+      - ./schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U researcher -d papers"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  grobid:
+    # CRF-only image is sufficient for reference extraction and is lighter than
+    # the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid
+    # for the latest 0.8.x tag before deploying.
+    image: grobid/grobid:0.8.2
+    container_name: papers-grobid
+    ports:
+      - "8070:8070"
+    restart: unless-stopped
+    # GROBID is RAM-hungry; uncomment to cap usage on constrained hardware
+    # (e.g. Nvidia Jetson):
+    # deploy:
+    #   resources:
+    #     limits:
+    #       memory: 4g
+
+volumes:
+  pgdata:

infra/schema.sql

@@ -0,0 +1,90 @@
+-- =====================================================================
+-- Schema: Paper knowledge base
+--   Layer 1  papers, chunks   -> semantic search (pgvector)
+--   Layer 2  citations        -> citation graph / discovery
+--   Layer 3  code_links       -> provenance (C++ symbol <-> paper)
+--
+-- Adjust EMBEDDING_DIM to match your model (see .env.example):
+--   BGE-M3 = 1024 | Qwen3-Embedding-0.6B = 1024 | Jina v4 = 2048
+-- =====================================================================
+
+CREATE EXTENSION IF NOT EXISTS vector;
+
+-- ---------------------------------------------------------------------
+-- Layer 1: Papers + full-text chunks
+-- ---------------------------------------------------------------------
+CREATE TABLE papers (
+    id            TEXT PRIMARY KEY,        -- canonical ID: arXiv ID or DOI
+    openalex_id   TEXT UNIQUE,             -- e.g. W2741809807
+    bibkey        TEXT UNIQUE,             -- BibTeX key -> .bib + Doxygen @cite
+    title         TEXT NOT NULL,
+    authors       TEXT[],
+    year          INT,
+    abstract      TEXT,
+    source_path   TEXT,                    -- path to parsed .tex / .mmd file
+    abstract_emb  vector(1024),            -- paper-level embedding (similarity)
+    added_at      TIMESTAMPTZ DEFAULT now()
+);
+
+-- HNSW index for fast approximate nearest-neighbour search on abstracts.
+CREATE INDEX papers_abstract_emb_idx
+    ON papers USING hnsw (abstract_emb vector_cosine_ops);
+
+CREATE TABLE chunks (
+    id        BIGSERIAL PRIMARY KEY,
+    paper_id  TEXT REFERENCES papers(id) ON DELETE CASCADE,
+    ord       INT NOT NULL,               -- position within the paper
+    content   TEXT NOT NULL,
+    embedding vector(1024)
+);
+
+CREATE INDEX chunks_emb_idx
+    ON chunks USING hnsw (embedding vector_cosine_ops);
+CREATE INDEX chunks_paper_idx ON chunks (paper_id);
+
+-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
+-- Dense (above) + full-text (below) combined = robust against math terms.
+CREATE INDEX chunks_fts_idx
+    ON chunks USING gin (to_tsvector('english', content));
+
+-- ---------------------------------------------------------------------
+-- Layer 2: Citation graph
+--   cited_id has NO foreign key ON PURPOSE:
+--   edges to not-yet-ingested papers are intentionally preserved —
+--   those are your discovery leads.
+-- ---------------------------------------------------------------------
+CREATE TABLE citations (
+    citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
+    cited_id  TEXT NOT NULL,               -- arXiv/DOI/OpenAlex ID of the target
+    context   TEXT,                        -- optional: citation context (S2)
+    PRIMARY KEY (citing_id, cited_id)
+);
+
+CREATE INDEX citations_cited_idx ON citations (cited_id);
+
+-- ---------------------------------------------------------------------
+-- Layer 3: Provenance  (the "cleanly couple" goal)
+-- ---------------------------------------------------------------------
+CREATE TABLE code_links (
+    id        BIGSERIAL PRIMARY KEY,
+    symbol    TEXT NOT NULL,               -- e.g. 'dec::hodge_star' or 'file.cpp:120'
+    paper_id  TEXT REFERENCES papers(id) ON DELETE SET NULL,
+    role      TEXT,                        -- e.g. 'implements Thm 3.2', 'uses Eq 5'
+    note      TEXT,
+    added_at  TIMESTAMPTZ DEFAULT now()
+);
+
+CREATE INDEX code_links_symbol_idx ON code_links (symbol);
+CREATE INDEX code_links_paper_idx  ON code_links (paper_id);
+
+-- ---------------------------------------------------------------------
+-- Example query: discovery leads
+-- Papers referenced by multiple already-ingested papers but not yet
+-- collected themselves — ranked by how often they are cited locally.
+-- ---------------------------------------------------------------------
+-- SELECT cited_id, count(*) AS pull
+-- FROM   citations
+-- WHERE  cited_id NOT IN (SELECT id FROM papers)
+-- GROUP  BY cited_id
+-- ORDER  BY pull DESC
+-- LIMIT  20;

pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "codex"
+version = "0.1.0"
+description = "Personal knowledge base for scientific papers — provenance, citation graph, semantic search."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "psycopg[binary]>=3.1",
+    "pgvector>=0.3",
+    "pydantic-settings>=2",
+    "sentence-transformers>=3",
+    "typer>=0.12",
+    "httpx>=0.27",
+    "tenacity>=8",
+]
+
+[project.scripts]
+codex = "codex.cli:app"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.4",
+    "mypy>=1.10",
+    "pytest>=8",
+    "pytest-mock>=3",
+]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+ignore = []
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]