user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ingest): retry on bibkey UNIQUE collision (audit C-15)

Browse Source 
The id-keyed upsert (ON CONFLICT (id)) does not catch the papers.bibkey UNIQUE
constraint, so a second paper whose auto-generated key matches an existing one
(two same-author/year works → e.g. 'BobenkoLutz2023') failed the whole ingest
with a UniqueViolation. Surfaced by the canonical-id re-ingest: 2305.10988
collided with 2310.17529.

ingest now catches a bibkey UniqueViolation, rolls back, and retries the upsert
with an a/b/c… suffix. Non-bibkey violations re-raise unchanged. Mocks don't
raise, so the existing happy-path tests are untouched; a new test drives the
collision-and-retry path.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-15 22:11:14 +02:00
parent 45c3b16dd7
commit b52303d22b
2 changed files with 77 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
codex/ingest.py
View File

@@ -8,6 +8,7 @@ from dataclasses import dataclass
from pathlib import Path
import numpy as np
import psycopg
from codex.config import get_settings
from codex.db import get_conn
@@ -125,35 +126,48 @@ def ingest_paper(
# ---------------------------------------------------------------
# 3. Upsert paper (INSERT … ON CONFLICT (id) DO UPDATE SET …)
# ---------------------------------------------------------------
upsert_sql = """
INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
source_path, abstract_emb)
VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
%(abstract)s, %(source_path)s, %(abstract_emb)s)
ON CONFLICT (id) DO UPDATE SET
openalex_id = EXCLUDED.openalex_id,
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
title = EXCLUDED.title,
authors = EXCLUDED.authors,
year = EXCLUDED.year,
abstract = EXCLUDED.abstract,
source_path = COALESCE(EXCLUDED.source_path, papers.source_path),
abstract_emb = EXCLUDED.abstract_emb
"""
with get_conn() as conn:
conn.execute(
"""
INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
source_path, abstract_emb)
VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
%(abstract)s, %(source_path)s, %(abstract_emb)s)
ON CONFLICT (id) DO UPDATE SET
openalex_id = EXCLUDED.openalex_id,
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
title = EXCLUDED.title,
authors = EXCLUDED.authors,
year = EXCLUDED.year,
abstract = EXCLUDED.abstract,
source_path = COALESCE(EXCLUDED.source_path, papers.source_path),
abstract_emb = EXCLUDED.abstract_emb
""",
{
"id": paper.id,
"openalex_id": paper.openalex_id,
"bibkey": paper.bibkey,
"title": paper.title,
"authors": paper.authors,
"year": paper.year,
"abstract": paper.abstract,
"source_path": source_path,
"abstract_emb": abstract_emb,
},
)
# The id-keyed upsert does not catch a papers.bibkey UNIQUE collision (two
# same-author/year papers generate the same key). On such a collision retry
# with an a/b/c… suffix instead of failing the whole ingest (audit C-15).
base_bibkey = paper.bibkey
for attempt in range(27):
try:
conn.execute(
upsert_sql,
{
"id": paper.id,
"openalex_id": paper.openalex_id,
"bibkey": paper.bibkey,
"title": paper.title,
"authors": paper.authors,
"year": paper.year,
"abstract": paper.abstract,
"source_path": source_path,
"abstract_emb": abstract_emb,
},
)
break
except psycopg.errors.UniqueViolation as exc:
if base_bibkey is None or "bibkey" not in str(exc).lower() or attempt == 26:
raise
conn.rollback()
paper.bibkey = f"{base_bibkey}{chr(ord('a') + attempt)}"
# Register openalex_id in paper_identifiers (alias table).
# Every ingest records the primary openalex_id so the graph JOIN