e7ac7766a5
pyproject.toml (Python 3.12, uv), codex/ package (config, db, models), infra/ (docker-compose + schema), .env.example, .gitignore, README. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
91 lines
3.8 KiB
SQL
91 lines
3.8 KiB
SQL
-- =====================================================================
|
|
-- Schema: Paper knowledge base
|
|
-- Layer 1 papers, chunks -> semantic search (pgvector)
|
|
-- Layer 2 citations -> citation graph / discovery
|
|
-- Layer 3 code_links -> provenance (C++ symbol <-> paper)
|
|
--
|
|
-- Adjust EMBEDDING_DIM to match your model (see .env.example):
|
|
-- BGE-M3 = 1024 | Qwen3-Embedding-0.6B = 1024 | Jina v4 = 2048
|
|
-- =====================================================================
|
|
|
|
CREATE EXTENSION IF NOT EXISTS vector;
|
|
|
|
-- ---------------------------------------------------------------------
|
|
-- Layer 1: Papers + full-text chunks
|
|
-- ---------------------------------------------------------------------
|
|
CREATE TABLE papers (
|
|
id TEXT PRIMARY KEY, -- canonical ID: arXiv ID or DOI
|
|
openalex_id TEXT UNIQUE, -- e.g. W2741809807
|
|
bibkey TEXT UNIQUE, -- BibTeX key -> .bib + Doxygen @cite
|
|
title TEXT NOT NULL,
|
|
authors TEXT[],
|
|
year INT,
|
|
abstract TEXT,
|
|
source_path TEXT, -- path to parsed .tex / .mmd file
|
|
abstract_emb vector(1024), -- paper-level embedding (similarity)
|
|
added_at TIMESTAMPTZ DEFAULT now()
|
|
);
|
|
|
|
-- HNSW index for fast approximate nearest-neighbour search on abstracts.
|
|
CREATE INDEX papers_abstract_emb_idx
|
|
ON papers USING hnsw (abstract_emb vector_cosine_ops);
|
|
|
|
CREATE TABLE chunks (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
paper_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
|
ord INT NOT NULL, -- position within the paper
|
|
content TEXT NOT NULL,
|
|
embedding vector(1024)
|
|
);
|
|
|
|
CREATE INDEX chunks_emb_idx
|
|
ON chunks USING hnsw (embedding vector_cosine_ops);
|
|
CREATE INDEX chunks_paper_idx ON chunks (paper_id);
|
|
|
|
-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
|
|
-- Dense (above) + full-text (below) combined = robust against math terms.
|
|
CREATE INDEX chunks_fts_idx
|
|
ON chunks USING gin (to_tsvector('english', content));
|
|
|
|
-- ---------------------------------------------------------------------
|
|
-- Layer 2: Citation graph
|
|
-- cited_id has NO foreign key ON PURPOSE:
|
|
-- edges to not-yet-ingested papers are intentionally preserved —
|
|
-- those are your discovery leads.
|
|
-- ---------------------------------------------------------------------
|
|
CREATE TABLE citations (
|
|
citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
|
cited_id TEXT NOT NULL, -- arXiv/DOI/OpenAlex ID of the target
|
|
context TEXT, -- optional: citation context (S2)
|
|
PRIMARY KEY (citing_id, cited_id)
|
|
);
|
|
|
|
CREATE INDEX citations_cited_idx ON citations (cited_id);
|
|
|
|
-- ---------------------------------------------------------------------
|
|
-- Layer 3: Provenance (the "cleanly couple" goal)
|
|
-- ---------------------------------------------------------------------
|
|
CREATE TABLE code_links (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
symbol TEXT NOT NULL, -- e.g. 'dec::hodge_star' or 'file.cpp:120'
|
|
paper_id TEXT REFERENCES papers(id) ON DELETE SET NULL,
|
|
role TEXT, -- e.g. 'implements Thm 3.2', 'uses Eq 5'
|
|
note TEXT,
|
|
added_at TIMESTAMPTZ DEFAULT now()
|
|
);
|
|
|
|
CREATE INDEX code_links_symbol_idx ON code_links (symbol);
|
|
CREATE INDEX code_links_paper_idx ON code_links (paper_id);
|
|
|
|
-- ---------------------------------------------------------------------
|
|
-- Example query: discovery leads
|
|
-- Papers referenced by multiple already-ingested papers but not yet
|
|
-- collected themselves — ranked by how often they are cited locally.
|
|
-- ---------------------------------------------------------------------
|
|
-- SELECT cited_id, count(*) AS pull
|
|
-- FROM citations
|
|
-- WHERE cited_id NOT IN (SELECT id FROM papers)
|
|
-- GROUP BY cited_id
|
|
-- ORDER BY pull DESC
|
|
-- LIMIT 20;
|