feat(graph): paper_identifiers table — multi-ID alias support
Fixes preprint/journal duplicate problem: arXiv papers were ingested under their preprint OpenAlex ID, but corpus citations reference the published-version OpenAlex ID → those citations were invisible in the graph (appeared as dangling). Changes: - infra/schema.sql: CREATE TABLE paper_identifiers (paper_id, openalex_id) with UNIQUE index; seeded from papers.openalex_id on migration - codex/graph.py: build_citation_graph LEFT JOINs paper_identifiers as a second resolution path: COALESCE(p.id, pi.paper_id, c.cited_id) - codex/ingest.py: every ingest inserts openalex_id into paper_identifiers (ON CONFLICT DO NOTHING) — aliases can be added manually alongside Live DB: 35 rows seeded + 4 published-version aliases added: math/0503219 → W2163787581 (DCG 2007) math/0306167 → W2136126748 (Commun Analysis Geom 2004) math/0203250 → W1567166970 (Trans AMS 2003) 1005.2698 → W1511400044 (Geom & Topol 2015) Effect: dangling 574→570; Bobenko+Springborn 2007 now IN-KB rank 9. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -47,9 +47,11 @@ def build_citation_graph(conn: Any) -> nx.DiGraph:
|
|||||||
"""
|
"""
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"""
|
"""
|
||||||
SELECT c.citing_id, COALESCE(p.id, c.cited_id) AS cited_id
|
SELECT c.citing_id, COALESCE(p.id, pi.paper_id, c.cited_id) AS cited_id
|
||||||
FROM citations c
|
FROM citations c
|
||||||
LEFT JOIN papers p ON p.openalex_id = c.cited_id
|
LEFT JOIN papers p ON p.openalex_id = c.cited_id
|
||||||
|
LEFT JOIN paper_identifiers pi
|
||||||
|
ON pi.openalex_id = c.cited_id AND p.id IS NULL
|
||||||
"""
|
"""
|
||||||
).fetchall()
|
).fetchall()
|
||||||
g: nx.DiGraph = nx.DiGraph()
|
g: nx.DiGraph = nx.DiGraph()
|
||||||
|
|||||||
@@ -147,6 +147,19 @@ def ingest_paper(
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Register openalex_id in paper_identifiers (alias table).
|
||||||
|
# Every ingest records the primary openalex_id so the graph JOIN
|
||||||
|
# resolves it even if papers.openalex_id is later updated.
|
||||||
|
if paper.openalex_id:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO paper_identifiers (paper_id, openalex_id)
|
||||||
|
VALUES (%s, %s)
|
||||||
|
ON CONFLICT DO NOTHING
|
||||||
|
""",
|
||||||
|
(paper.id, paper.openalex_id),
|
||||||
|
)
|
||||||
|
|
||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
# 4. Parse + chunk + embed source file (if provided)
|
# 4. Parse + chunk + embed source file (if provided)
|
||||||
# DELETE existing chunks for this paper first, then bulk INSERT
|
# DELETE existing chunks for this paper first, then bulk INSERT
|
||||||
|
|||||||
@@ -125,3 +125,21 @@ CREATE INDEX IF NOT EXISTS figures_paper_idx ON figures (paper_id);
|
|||||||
|
|
||||||
-- F-16 Chunk Quality Gate: section classification
|
-- F-16 Chunk Quality Gate: section classification
|
||||||
ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;
|
ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;
|
||||||
|
|
||||||
|
-- F-17 Paper identifier aliases: one paper → many OpenAlex IDs
|
||||||
|
-- (arXiv preprint ID + journal published ID + any future version)
|
||||||
|
-- Each OpenAlex ID maps to exactly one paper (UNIQUE on openalex_id).
|
||||||
|
-- Populated automatically by ingest; add extra aliases manually.
|
||||||
|
CREATE TABLE IF NOT EXISTS paper_identifiers (
|
||||||
|
paper_id TEXT NOT NULL REFERENCES papers(id) ON DELETE CASCADE,
|
||||||
|
openalex_id TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (paper_id, openalex_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS paper_identifiers_oa_idx
|
||||||
|
ON paper_identifiers (openalex_id);
|
||||||
|
|
||||||
|
-- Seed from existing papers.openalex_id
|
||||||
|
INSERT INTO paper_identifiers (paper_id, openalex_id)
|
||||||
|
SELECT id, openalex_id FROM papers WHERE openalex_id IS NOT NULL
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|||||||
Reference in New Issue
Block a user