feat: initial project scaffold
pyproject.toml (Python 3.12, uv), codex/ package (config, db, models), infra/ (docker-compose + schema), .env.example, .gitignore, README. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
47
infra/docker-compose.yml
Normal file
47
infra/docker-compose.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
# Paper knowledge base: Provenance + Citation graph + Semantic search
|
||||
# A single Postgres instance (pgvector) covers all three layers.
|
||||
# GROBID parses references / structure from PDFs without an arXiv source.
|
||||
#
|
||||
# Quick start:
|
||||
# docker compose up -d
|
||||
# psql "${DATABASE_URL}" # see .env.example for default value
|
||||
|
||||
services:
|
||||
db:
|
||||
image: pgvector/pgvector:pg17 # official pgvector image
|
||||
container_name: papers-db
|
||||
environment:
|
||||
POSTGRES_DB: papers
|
||||
POSTGRES_USER: researcher
|
||||
POSTGRES_PASSWORD: change_me # change before production use
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
# schema.sql is executed automatically on first start:
|
||||
- ./schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U researcher -d papers"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
grobid:
|
||||
# CRF-only image is sufficient for reference extraction and is lighter than
|
||||
# the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid
|
||||
# for the latest 0.8.x tag before deploying.
|
||||
image: grobid/grobid:0.8.2
|
||||
container_name: papers-grobid
|
||||
ports:
|
||||
- "8070:8070"
|
||||
restart: unless-stopped
|
||||
# GROBID is RAM-hungry; uncomment to cap usage on constrained hardware
|
||||
# (e.g. Nvidia Jetson):
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# memory: 4g
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
90
infra/schema.sql
Normal file
90
infra/schema.sql
Normal file
@@ -0,0 +1,90 @@
|
||||
-- =====================================================================
|
||||
-- Schema: Paper knowledge base
|
||||
-- Layer 1 papers, chunks -> semantic search (pgvector)
|
||||
-- Layer 2 citations -> citation graph / discovery
|
||||
-- Layer 3 code_links -> provenance (C++ symbol <-> paper)
|
||||
--
|
||||
-- Adjust EMBEDDING_DIM to match your model (see .env.example):
|
||||
-- BGE-M3 = 1024 | Qwen3-Embedding-0.6B = 1024 | Jina v4 = 2048
|
||||
-- =====================================================================
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 1: Papers + full-text chunks
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE papers (
|
||||
id TEXT PRIMARY KEY, -- canonical ID: arXiv ID or DOI
|
||||
openalex_id TEXT UNIQUE, -- e.g. W2741809807
|
||||
bibkey TEXT UNIQUE, -- BibTeX key -> .bib + Doxygen @cite
|
||||
title TEXT NOT NULL,
|
||||
authors TEXT[],
|
||||
year INT,
|
||||
abstract TEXT,
|
||||
source_path TEXT, -- path to parsed .tex / .mmd file
|
||||
abstract_emb vector(1024), -- paper-level embedding (similarity)
|
||||
added_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
-- HNSW index for fast approximate nearest-neighbour search on abstracts.
|
||||
CREATE INDEX papers_abstract_emb_idx
|
||||
ON papers USING hnsw (abstract_emb vector_cosine_ops);
|
||||
|
||||
CREATE TABLE chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
paper_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
||||
ord INT NOT NULL, -- position within the paper
|
||||
content TEXT NOT NULL,
|
||||
embedding vector(1024)
|
||||
);
|
||||
|
||||
CREATE INDEX chunks_emb_idx
|
||||
ON chunks USING hnsw (embedding vector_cosine_ops);
|
||||
CREATE INDEX chunks_paper_idx ON chunks (paper_id);
|
||||
|
||||
-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
|
||||
-- Dense (above) + full-text (below) combined = robust against math terms.
|
||||
CREATE INDEX chunks_fts_idx
|
||||
ON chunks USING gin (to_tsvector('english', content));
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 2: Citation graph
|
||||
-- cited_id has NO foreign key ON PURPOSE:
|
||||
-- edges to not-yet-ingested papers are intentionally preserved —
|
||||
-- those are your discovery leads.
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE citations (
|
||||
citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
||||
cited_id TEXT NOT NULL, -- arXiv/DOI/OpenAlex ID of the target
|
||||
context TEXT, -- optional: citation context (S2)
|
||||
PRIMARY KEY (citing_id, cited_id)
|
||||
);
|
||||
|
||||
CREATE INDEX citations_cited_idx ON citations (cited_id);
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 3: Provenance (the "cleanly couple" goal)
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE code_links (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
symbol TEXT NOT NULL, -- e.g. 'dec::hodge_star' or 'file.cpp:120'
|
||||
paper_id TEXT REFERENCES papers(id) ON DELETE SET NULL,
|
||||
role TEXT, -- e.g. 'implements Thm 3.2', 'uses Eq 5'
|
||||
note TEXT,
|
||||
added_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX code_links_symbol_idx ON code_links (symbol);
|
||||
CREATE INDEX code_links_paper_idx ON code_links (paper_id);
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Example query: discovery leads
|
||||
-- Papers referenced by multiple already-ingested papers but not yet
|
||||
-- collected themselves — ranked by how often they are cited locally.
|
||||
-- ---------------------------------------------------------------------
|
||||
-- SELECT cited_id, count(*) AS pull
|
||||
-- FROM citations
|
||||
-- WHERE cited_id NOT IN (SELECT id FROM papers)
|
||||
-- GROUP BY cited_id
|
||||
-- ORDER BY pull DESC
|
||||
-- LIMIT 20;
|
||||
Reference in New Issue
Block a user