feat(graph): paper_identifiers table — multi-ID alias support
Fixes preprint/journal duplicate problem: arXiv papers were ingested under their preprint OpenAlex ID, but corpus citations reference the published-version OpenAlex ID → those citations were invisible in the graph (appeared as dangling). Changes: - infra/schema.sql: CREATE TABLE paper_identifiers (paper_id, openalex_id) with UNIQUE index; seeded from papers.openalex_id on migration - codex/graph.py: build_citation_graph LEFT JOINs paper_identifiers as a second resolution path: COALESCE(p.id, pi.paper_id, c.cited_id) - codex/ingest.py: every ingest inserts openalex_id into paper_identifiers (ON CONFLICT DO NOTHING) — aliases can be added manually alongside Live DB: 35 rows seeded + 4 published-version aliases added: math/0503219 → W2163787581 (DCG 2007) math/0306167 → W2136126748 (Commun Analysis Geom 2004) math/0203250 → W1567166970 (Trans AMS 2003) 1005.2698 → W1511400044 (Geom & Topol 2015) Effect: dangling 574→570; Bobenko+Springborn 2007 now IN-KB rank 9. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -47,9 +47,11 @@ def build_citation_graph(conn: Any) -> nx.DiGraph:
|
||||
"""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.citing_id, COALESCE(p.id, c.cited_id) AS cited_id
|
||||
SELECT c.citing_id, COALESCE(p.id, pi.paper_id, c.cited_id) AS cited_id
|
||||
FROM citations c
|
||||
LEFT JOIN papers p ON p.openalex_id = c.cited_id
|
||||
LEFT JOIN paper_identifiers pi
|
||||
ON pi.openalex_id = c.cited_id AND p.id IS NULL
|
||||
"""
|
||||
).fetchall()
|
||||
g: nx.DiGraph = nx.DiGraph()
|
||||
|
||||
@@ -147,6 +147,19 @@ def ingest_paper(
|
||||
},
|
||||
)
|
||||
|
||||
# Register openalex_id in paper_identifiers (alias table).
|
||||
# Every ingest records the primary openalex_id so the graph JOIN
|
||||
# resolves it even if papers.openalex_id is later updated.
|
||||
if paper.openalex_id:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO paper_identifiers (paper_id, openalex_id)
|
||||
VALUES (%s, %s)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
(paper.id, paper.openalex_id),
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# 4. Parse + chunk + embed source file (if provided)
|
||||
# DELETE existing chunks for this paper first, then bulk INSERT
|
||||
|
||||
@@ -125,3 +125,21 @@ CREATE INDEX IF NOT EXISTS figures_paper_idx ON figures (paper_id);
|
||||
|
||||
-- F-16 Chunk Quality Gate: section classification
|
||||
ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;
|
||||
|
||||
-- F-17 Paper identifier aliases: one paper → many OpenAlex IDs
|
||||
-- (arXiv preprint ID + journal published ID + any future version)
|
||||
-- Each OpenAlex ID maps to exactly one paper (UNIQUE on openalex_id).
|
||||
-- Populated automatically by ingest; add extra aliases manually.
|
||||
CREATE TABLE IF NOT EXISTS paper_identifiers (
|
||||
paper_id TEXT NOT NULL REFERENCES papers(id) ON DELETE CASCADE,
|
||||
openalex_id TEXT NOT NULL,
|
||||
PRIMARY KEY (paper_id, openalex_id)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS paper_identifiers_oa_idx
|
||||
ON paper_identifiers (openalex_id);
|
||||
|
||||
-- Seed from existing papers.openalex_id
|
||||
INSERT INTO paper_identifiers (paper_id, openalex_id)
|
||||
SELECT id, openalex_id FROM papers WHERE openalex_id IS NOT NULL
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
Reference in New Issue
Block a user