feat: initial project scaffold

pyproject.toml (Python 3.12, uv), codex/ package (config, db, models),
infra/ (docker-compose + schema), .env.example, .gitignore, README.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-04 11:19:10 +02:00
commit e7ac7766a5
13 changed files with 2136 additions and 0 deletions

47
infra/docker-compose.yml Normal file
View File

@@ -0,0 +1,47 @@
# Paper knowledge base: Provenance + Citation graph + Semantic search
# A single Postgres instance (pgvector) covers all three layers.
# GROBID parses references / structure from PDFs without an arXiv source.
#
# Quick start:
# docker compose up -d
# psql "${DATABASE_URL}" # see .env.example for default value
services:
db:
image: pgvector/pgvector:pg17 # official pgvector image
container_name: papers-db
environment:
POSTGRES_DB: papers
POSTGRES_USER: researcher
POSTGRES_PASSWORD: change_me # change before production use
ports:
- "5432:5432"
volumes:
- pgdata:/var/lib/postgresql/data
# schema.sql is executed automatically on first start:
- ./schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U researcher -d papers"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped
grobid:
# CRF-only image is sufficient for reference extraction and is lighter than
# the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid
# for the latest 0.8.x tag before deploying.
image: grobid/grobid:0.8.2
container_name: papers-grobid
ports:
- "8070:8070"
restart: unless-stopped
# GROBID is RAM-hungry; uncomment to cap usage on constrained hardware
# (e.g. Nvidia Jetson):
# deploy:
# resources:
# limits:
# memory: 4g
volumes:
pgdata:

90
infra/schema.sql Normal file
View File

@@ -0,0 +1,90 @@
-- =====================================================================
-- Schema: Paper knowledge base
-- Layer 1 papers, chunks -> semantic search (pgvector)
-- Layer 2 citations -> citation graph / discovery
-- Layer 3 code_links -> provenance (C++ symbol <-> paper)
--
-- Adjust EMBEDDING_DIM to match your model (see .env.example):
-- BGE-M3 = 1024 | Qwen3-Embedding-0.6B = 1024 | Jina v4 = 2048
-- =====================================================================
CREATE EXTENSION IF NOT EXISTS vector;
-- ---------------------------------------------------------------------
-- Layer 1: Papers + full-text chunks
-- ---------------------------------------------------------------------
CREATE TABLE papers (
id TEXT PRIMARY KEY, -- canonical ID: arXiv ID or DOI
openalex_id TEXT UNIQUE, -- e.g. W2741809807
bibkey TEXT UNIQUE, -- BibTeX key -> .bib + Doxygen @cite
title TEXT NOT NULL,
authors TEXT[],
year INT,
abstract TEXT,
source_path TEXT, -- path to parsed .tex / .mmd file
abstract_emb vector(1024), -- paper-level embedding (similarity)
added_at TIMESTAMPTZ DEFAULT now()
);
-- HNSW index for fast approximate nearest-neighbour search on abstracts.
CREATE INDEX papers_abstract_emb_idx
ON papers USING hnsw (abstract_emb vector_cosine_ops);
CREATE TABLE chunks (
id BIGSERIAL PRIMARY KEY,
paper_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
ord INT NOT NULL, -- position within the paper
content TEXT NOT NULL,
embedding vector(1024)
);
CREATE INDEX chunks_emb_idx
ON chunks USING hnsw (embedding vector_cosine_ops);
CREATE INDEX chunks_paper_idx ON chunks (paper_id);
-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
-- Dense (above) + full-text (below) combined = robust against math terms.
CREATE INDEX chunks_fts_idx
ON chunks USING gin (to_tsvector('english', content));
-- ---------------------------------------------------------------------
-- Layer 2: Citation graph
-- cited_id has NO foreign key ON PURPOSE:
-- edges to not-yet-ingested papers are intentionally preserved —
-- those are your discovery leads.
-- ---------------------------------------------------------------------
CREATE TABLE citations (
citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
cited_id TEXT NOT NULL, -- arXiv/DOI/OpenAlex ID of the target
context TEXT, -- optional: citation context (S2)
PRIMARY KEY (citing_id, cited_id)
);
CREATE INDEX citations_cited_idx ON citations (cited_id);
-- ---------------------------------------------------------------------
-- Layer 3: Provenance (the "cleanly couple" goal)
-- ---------------------------------------------------------------------
CREATE TABLE code_links (
id BIGSERIAL PRIMARY KEY,
symbol TEXT NOT NULL, -- e.g. 'dec::hodge_star' or 'file.cpp:120'
paper_id TEXT REFERENCES papers(id) ON DELETE SET NULL,
role TEXT, -- e.g. 'implements Thm 3.2', 'uses Eq 5'
note TEXT,
added_at TIMESTAMPTZ DEFAULT now()
);
CREATE INDEX code_links_symbol_idx ON code_links (symbol);
CREATE INDEX code_links_paper_idx ON code_links (paper_id);
-- ---------------------------------------------------------------------
-- Example query: discovery leads
-- Papers referenced by multiple already-ingested papers but not yet
-- collected themselves — ranked by how often they are cited locally.
-- ---------------------------------------------------------------------
-- SELECT cited_id, count(*) AS pull
-- FROM citations
-- WHERE cited_id NOT IN (SELECT id FROM papers)
-- GROUP BY cited_id
-- ORDER BY pull DESC
-- LIMIT 20;