feat: initial project scaffold
pyproject.toml (Python 3.12, uv), codex/ package (config, db, models), infra/ (docker-compose + schema), .env.example, .gitignore, README. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
90
infra/schema.sql
Normal file
90
infra/schema.sql
Normal file
@@ -0,0 +1,90 @@
|
||||
-- =====================================================================
|
||||
-- Schema: Paper knowledge base
|
||||
-- Layer 1 papers, chunks -> semantic search (pgvector)
|
||||
-- Layer 2 citations -> citation graph / discovery
|
||||
-- Layer 3 code_links -> provenance (C++ symbol <-> paper)
|
||||
--
|
||||
-- Adjust EMBEDDING_DIM to match your model (see .env.example):
|
||||
-- BGE-M3 = 1024 | Qwen3-Embedding-0.6B = 1024 | Jina v4 = 2048
|
||||
-- =====================================================================
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 1: Papers + full-text chunks
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE papers (
|
||||
id TEXT PRIMARY KEY, -- canonical ID: arXiv ID or DOI
|
||||
openalex_id TEXT UNIQUE, -- e.g. W2741809807
|
||||
bibkey TEXT UNIQUE, -- BibTeX key -> .bib + Doxygen @cite
|
||||
title TEXT NOT NULL,
|
||||
authors TEXT[],
|
||||
year INT,
|
||||
abstract TEXT,
|
||||
source_path TEXT, -- path to parsed .tex / .mmd file
|
||||
abstract_emb vector(1024), -- paper-level embedding (similarity)
|
||||
added_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
-- HNSW index for fast approximate nearest-neighbour search on abstracts.
|
||||
CREATE INDEX papers_abstract_emb_idx
|
||||
ON papers USING hnsw (abstract_emb vector_cosine_ops);
|
||||
|
||||
CREATE TABLE chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
paper_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
||||
ord INT NOT NULL, -- position within the paper
|
||||
content TEXT NOT NULL,
|
||||
embedding vector(1024)
|
||||
);
|
||||
|
||||
CREATE INDEX chunks_emb_idx
|
||||
ON chunks USING hnsw (embedding vector_cosine_ops);
|
||||
CREATE INDEX chunks_paper_idx ON chunks (paper_id);
|
||||
|
||||
-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
|
||||
-- Dense (above) + full-text (below) combined = robust against math terms.
|
||||
CREATE INDEX chunks_fts_idx
|
||||
ON chunks USING gin (to_tsvector('english', content));
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 2: Citation graph
|
||||
-- cited_id has NO foreign key ON PURPOSE:
|
||||
-- edges to not-yet-ingested papers are intentionally preserved —
|
||||
-- those are your discovery leads.
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE citations (
|
||||
citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
||||
cited_id TEXT NOT NULL, -- arXiv/DOI/OpenAlex ID of the target
|
||||
context TEXT, -- optional: citation context (S2)
|
||||
PRIMARY KEY (citing_id, cited_id)
|
||||
);
|
||||
|
||||
CREATE INDEX citations_cited_idx ON citations (cited_id);
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 3: Provenance (the "cleanly couple" goal)
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE code_links (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
symbol TEXT NOT NULL, -- e.g. 'dec::hodge_star' or 'file.cpp:120'
|
||||
paper_id TEXT REFERENCES papers(id) ON DELETE SET NULL,
|
||||
role TEXT, -- e.g. 'implements Thm 3.2', 'uses Eq 5'
|
||||
note TEXT,
|
||||
added_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX code_links_symbol_idx ON code_links (symbol);
|
||||
CREATE INDEX code_links_paper_idx ON code_links (paper_id);
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Example query: discovery leads
|
||||
-- Papers referenced by multiple already-ingested papers but not yet
|
||||
-- collected themselves — ranked by how often they are cited locally.
|
||||
-- ---------------------------------------------------------------------
|
||||
-- SELECT cited_id, count(*) AS pull
|
||||
-- FROM citations
|
||||
-- WHERE cited_id NOT IN (SELECT id FROM papers)
|
||||
-- GROUP BY cited_id
|
||||
-- ORDER BY pull DESC
|
||||
-- LIMIT 20;
|
||||
Reference in New Issue
Block a user