feat: initial project scaffold
pyproject.toml (Python 3.12, uv), codex/ package (config, db, models), infra/ (docker-compose + schema), .env.example, .gitignore, README. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
0
codex/__init__.py
Normal file
0
codex/__init__.py
Normal file
85
codex/config.py
Normal file
85
codex/config.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""Application configuration via environment variables / .env file.
|
||||
|
||||
All settings are read from the environment (or a .env file in the project
|
||||
root). Import :func:`get_settings` wherever you need configuration; the
|
||||
returned object is cached after the first call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Centralised, env-driven configuration for the codex application."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Database
|
||||
# ------------------------------------------------------------------
|
||||
database_url: str = Field(
|
||||
default="postgresql://researcher:change_me@localhost:5432/papers",
|
||||
description=(
|
||||
"libpq-compatible connection string consumed by psycopg. "
|
||||
"Example: postgresql://user:pass@host:5432/dbname"
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# External services
|
||||
# ------------------------------------------------------------------
|
||||
grobid_url: str = Field(
|
||||
default="http://localhost:8070",
|
||||
description="Base URL of the GROBID HTTP API (containerised).",
|
||||
)
|
||||
|
||||
ollama_base_url: str = Field(
|
||||
default="http://localhost:11434",
|
||||
description="Base URL of the local Ollama endpoint (optional Q&A layer).",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Embeddings
|
||||
# ------------------------------------------------------------------
|
||||
embedding_model: str = Field(
|
||||
default="BAAI/bge-m3",
|
||||
description=(
|
||||
"sentence-transformers model identifier. "
|
||||
"Must match EMBEDDING_DIM. Default: BAAI/bge-m3 (1024 dims)."
|
||||
),
|
||||
)
|
||||
|
||||
embedding_dim: int = Field(
|
||||
default=1024,
|
||||
gt=0,
|
||||
description=(
|
||||
"Dimension of the dense embedding vectors. "
|
||||
"Must match the output dimension of EMBEDDING_MODEL."
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# API etiquette
|
||||
# ------------------------------------------------------------------
|
||||
openalex_mailto: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"E-mail address for the OpenAlex Polite Pool (faster rate limits). "
|
||||
"Required by OpenAlex ToS for automated access."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
"""Return the cached application settings singleton."""
|
||||
return Settings()
|
||||
57
codex/db.py
Normal file
57
codex/db.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""Database connection helpers.
|
||||
|
||||
Provides:
|
||||
- :func:`get_conn` — a context manager that yields a psycopg connection.
|
||||
- :func:`apply_schema` — idempotently applies ``infra/schema.sql``.
|
||||
|
||||
The connection pool (psycopg_pool) is intentionally deferred to F-05
|
||||
(ingest); here we expose a simple per-call ``psycopg.connect`` wrapper
|
||||
that is sufficient for schema migration and unit-testing without requiring
|
||||
an additional ``psycopg[pool]`` dependency at scaffold stage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
import psycopg.rows
|
||||
|
||||
from codex.config import get_settings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_conn() -> Generator[psycopg.Connection[psycopg.rows.DictRow], None, None]:
|
||||
"""Yield a psycopg connection with dict-row factory.
|
||||
|
||||
The connection is opened from :func:`codex.config.get_settings`
|
||||
``database_url`` and closed automatically when the context exits.
|
||||
|
||||
Usage::
|
||||
|
||||
with get_conn() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
"""
|
||||
settings = get_settings()
|
||||
with psycopg.connect(
|
||||
settings.database_url,
|
||||
row_factory=psycopg.rows.dict_row,
|
||||
) as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
def apply_schema(conn: psycopg.Connection[psycopg.rows.DictRow]) -> None:
|
||||
"""Idempotently apply ``infra/schema.sql`` to the connected database.
|
||||
|
||||
Safe to call on every startup — all DDL statements use
|
||||
``CREATE … IF NOT EXISTS``.
|
||||
|
||||
Args:
|
||||
conn: An open psycopg connection (obtained via :func:`get_conn`).
|
||||
"""
|
||||
schema_path = Path(__file__).parent.parent / "infra" / "schema.sql"
|
||||
sql = schema_path.read_text(encoding="utf-8")
|
||||
conn.execute(sql)
|
||||
conn.commit()
|
||||
80
codex/models.py
Normal file
80
codex/models.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Domain dataclasses.
|
||||
|
||||
Each class maps directly to a table in ``infra/schema.sql``.
|
||||
Field names and types are kept in sync with the DDL.
|
||||
|
||||
All fields that are nullable in the schema are typed as ``… | None``
|
||||
with a default of ``None`` so instances can be constructed incrementally.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class Paper:
|
||||
"""Maps to the ``papers`` table (Layer 1 — semantic search).
|
||||
|
||||
``id`` is the canonical identifier: an arXiv ID (e.g. ``2301.00001``)
|
||||
or a DOI (e.g. ``10.1145/3592430``).
|
||||
"""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
openalex_id: str | None = None
|
||||
bibkey: str | None = None
|
||||
authors: list[str] = field(default_factory=list)
|
||||
year: int | None = None
|
||||
abstract: str | None = None
|
||||
source_path: str | None = None
|
||||
# abstract_emb is a list of floats representing the pgvector column;
|
||||
# pgvector's Python driver accepts list[float] for vector columns.
|
||||
abstract_emb: list[float] | None = None
|
||||
added_at: datetime | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
"""Maps to the ``chunks`` table (Layer 1 — full-text chunks).
|
||||
|
||||
``id`` is set by the database (BIGSERIAL); leave it as ``None``
|
||||
when constructing a new chunk before insertion.
|
||||
"""
|
||||
|
||||
paper_id: str
|
||||
ord: int
|
||||
content: str
|
||||
id: int | None = None
|
||||
embedding: list[float] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Citation:
|
||||
"""Maps to the ``citations`` table (Layer 2 — citation graph).
|
||||
|
||||
``cited_id`` intentionally has no foreign-key constraint in the schema;
|
||||
citations to not-yet-ingested papers are preserved as discovery leads.
|
||||
"""
|
||||
|
||||
citing_id: str
|
||||
cited_id: str
|
||||
context: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodeLink:
|
||||
"""Maps to the ``code_links`` table (Layer 3 — provenance).
|
||||
|
||||
``symbol`` is a C++ qualified name or ``file.cpp:line`` reference.
|
||||
``role`` is a free-text description such as 'implements Thm 3.2'.
|
||||
``id`` is set by the database (BIGSERIAL).
|
||||
"""
|
||||
|
||||
symbol: str
|
||||
paper_id: str | None = None
|
||||
role: str | None = None
|
||||
note: str | None = None
|
||||
id: int | None = None
|
||||
added_at: datetime | None = None
|
||||
0
codex/parsing/__init__.py
Normal file
0
codex/parsing/__init__.py
Normal file
0
codex/sources/__init__.py
Normal file
0
codex/sources/__init__.py
Normal file
Reference in New Issue
Block a user