codex-py/infra/schema.sql

-- =====================================================================
-- Schema: Paper knowledge base
--   Layer 1  papers, chunks   -> semantic search (pgvector)
--   Layer 2  citations        -> citation graph / discovery
--   Layer 3  code_links       -> provenance (C++ symbol <-> paper)
--
-- Adjust EMBEDDING_DIM to match your model (see .env.example):
--   BGE-M3 = 1024 | Qwen3-Embedding-0.6B = 1024 | Jina v4 = 2048
-- =====================================================================

CREATE EXTENSION IF NOT EXISTS vector;

-- ---------------------------------------------------------------------
-- Layer 1: Papers + full-text chunks
-- ---------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS papers (
    id            TEXT PRIMARY KEY,        -- canonical ID: arXiv ID or DOI
    openalex_id   TEXT UNIQUE,             -- e.g. W2741809807
    bibkey        TEXT UNIQUE,             -- BibTeX key -> .bib + Doxygen @cite
    title         TEXT NOT NULL,
    authors       TEXT[],
    year          INT,
    abstract      TEXT,
    source_path   TEXT,                    -- path to parsed .tex / .mmd file
    abstract_emb  vector(1024),            -- paper-level embedding (similarity)
    added_at      TIMESTAMPTZ DEFAULT now()
);

-- HNSW index for fast approximate nearest-neighbour search on abstracts.
CREATE INDEX IF NOT EXISTS papers_abstract_emb_idx
    ON papers USING hnsw (abstract_emb vector_cosine_ops);

CREATE TABLE IF NOT EXISTS chunks (
    id        BIGSERIAL PRIMARY KEY,
    paper_id  TEXT REFERENCES papers(id) ON DELETE CASCADE,
    ord       INT NOT NULL,               -- position within the paper
    content   TEXT NOT NULL,
    embedding vector(1024)
);

CREATE INDEX IF NOT EXISTS chunks_emb_idx
    ON chunks USING hnsw (embedding vector_cosine_ops);
CREATE INDEX IF NOT EXISTS chunks_paper_idx ON chunks (paper_id);

-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
-- Dense (above) + full-text (below) combined = robust against math terms.
CREATE INDEX IF NOT EXISTS chunks_fts_idx
    ON chunks USING gin (to_tsvector('english', content));

-- ---------------------------------------------------------------------
-- Layer 2: Citation graph
--   cited_id has NO foreign key ON PURPOSE:
--   edges to not-yet-ingested papers are intentionally preserved —
--   those are your discovery leads.
-- ---------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS citations (
    citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
    cited_id  TEXT NOT NULL,               -- arXiv/DOI/OpenAlex ID of the target
    context   TEXT,                        -- optional: citation context (S2)
    PRIMARY KEY (citing_id, cited_id)
);

CREATE INDEX IF NOT EXISTS citations_cited_idx ON citations (cited_id);

-- ---------------------------------------------------------------------
-- Layer 3: Provenance  (the "cleanly couple" goal)
-- ---------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS code_links (
    id        BIGSERIAL PRIMARY KEY,
    symbol    TEXT NOT NULL,               -- e.g. 'dec::hodge_star' or 'file.cpp:120'
    paper_id  TEXT REFERENCES papers(id) ON DELETE SET NULL,
    role      TEXT,                        -- e.g. 'implements Thm 3.2', 'uses Eq 5'
    note      TEXT,
    added_at  TIMESTAMPTZ DEFAULT now()
);

CREATE INDEX IF NOT EXISTS code_links_symbol_idx ON code_links (symbol);
CREATE INDEX IF NOT EXISTS code_links_paper_idx  ON code_links (paper_id);

-- ---------------------------------------------------------------------
-- Example query: discovery leads
-- Papers referenced by multiple already-ingested papers but not yet
-- collected themselves — ranked by how often they are cited locally.
-- ---------------------------------------------------------------------
-- SELECT cited_id, count(*) AS pull
-- FROM   citations
-- WHERE  cited_id NOT IN (SELECT id FROM papers)
-- GROUP  BY cited_id
-- ORDER  BY pull DESC
-- LIMIT  20;

-- ---------------------------------------------------------------------
-- F-09 Rich Parsing: formulas + figures
-- ---------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS formulas (
    id          BIGSERIAL PRIMARY KEY,
    paper_id    TEXT REFERENCES papers(id) ON DELETE CASCADE,
    page        INT,
    raw_latex   TEXT NOT NULL,
    context     TEXT,
    eq_label    TEXT,
    embedding   vector(1024)
);

CREATE INDEX IF NOT EXISTS formulas_emb_idx
    ON formulas USING hnsw (embedding vector_cosine_ops);
CREATE INDEX IF NOT EXISTS formulas_paper_idx ON formulas (paper_id);
CREATE INDEX IF NOT EXISTS formulas_fts_idx
    ON formulas USING gin (
        to_tsvector('english', raw_latex || ' ' || coalesce(context, ''))
    );

CREATE TABLE IF NOT EXISTS figures (
    id          BIGSERIAL PRIMARY KEY,
    paper_id    TEXT REFERENCES papers(id) ON DELETE CASCADE,
    page        INT,
    caption     TEXT,
    image_path  TEXT,
    embedding   vector(1024)
);

CREATE INDEX IF NOT EXISTS figures_emb_idx
    ON figures USING hnsw (embedding vector_cosine_ops);
CREATE INDEX IF NOT EXISTS figures_paper_idx ON figures (paper_id);

-- F-16 Chunk Quality Gate: section classification
ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;

-- F-17 Paper identifier aliases: one paper → many OpenAlex IDs
-- (arXiv preprint ID + journal published ID + any future version)
-- Each OpenAlex ID maps to exactly one paper (UNIQUE on openalex_id).
-- Populated automatically by ingest; add extra aliases manually.
CREATE TABLE IF NOT EXISTS paper_identifiers (
    paper_id    TEXT NOT NULL REFERENCES papers(id) ON DELETE CASCADE,
    openalex_id TEXT NOT NULL,
    PRIMARY KEY (paper_id, openalex_id)
);

CREATE UNIQUE INDEX IF NOT EXISTS paper_identifiers_oa_idx
    ON paper_identifiers (openalex_id);

-- Seed from existing papers.openalex_id
INSERT INTO paper_identifiers (paper_id, openalex_id)
SELECT id, openalex_id FROM papers WHERE openalex_id IS NOT NULL
ON CONFLICT DO NOTHING;