user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: reconcile data-quality roadmap with audit-remediation main (#12-15)

Browse Source 
Resolved 4 conflicts: cli.py (R-D citing-coverage warning + main's small-corpus JSON/hub-title changes both kept); ingest.py (C-7 paper.id=caller-id pin first, then DQ-2 abstract recovery); test_ingest.py + test_openalex.py (kept both branches' tests).

C-7/DQ-5 overlap (both canonicalise papers.id): kept both layers — openalex._canonical_id normalizes fetch_paper's id (DQ-5); ingest pins papers.id to the caller's bare id (C-7); they converge on the bare canonical id. Fixed the auto-merged test_fetch_paper_success that had contradictory bare/URL assertions. 402 tests pass; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-22 15:11:26 +02:00
parent e817022097 d64edfc324
commit 9e5184385a
29 changed files with 900 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
codex/ingest.py
View File

@@ -7,6 +7,7 @@ from dataclasses import dataclass
from pathlib import Path
import numpy as np
import psycopg
from codex.config import get_settings
from codex.db import get_conn
@@ -205,9 +206,18 @@ def ingest_paper(
if paper is None:
raise ValueError(f"Paper not found: {paper_id}")
# Canonical identity is the caller-supplied id — what the CLI / ingest scripts
# use and what papers.id is contracted to hold (arXiv id or DOI). OpenAlex's
# fetch_paper fills paper.id with the URL-form doi/id (audit C-7); since DQ-5
# `openalex._canonical_id` already normalizes that to the bare form, this pin
# is belt-and-suspenders and keeps re-ingest idempotent regardless. The
# OpenAlex work id is retained as openalex_id and in paper_identifiers.
paper.id = paper_id.strip()
# DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute
# some publishers' abstracts). Supplement from S2, then Crossref, so the
# paper gets a real (non-zero) abstract embedding instead of a zero vector.
# some publishers' abstracts). Supplement from S2, then Crossref (uses the
# canonical paper.id set above), so the paper gets a real (non-zero) abstract
# embedding instead of a zero vector.
if not (paper.abstract or "").strip():
recovered = _recover_abstract(paper)
if recovered:
@@ -232,35 +242,64 @@ def ingest_paper(
# ---------------------------------------------------------------
# 3. Upsert paper (INSERT … ON CONFLICT (id) DO UPDATE SET …)
# ---------------------------------------------------------------
upsert_sql = """
INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
source_path, abstract_emb)
VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
%(abstract)s, %(source_path)s, %(abstract_emb)s)
ON CONFLICT (id) DO UPDATE SET
openalex_id = EXCLUDED.openalex_id,
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
title = EXCLUDED.title,
authors = EXCLUDED.authors,
year = EXCLUDED.year,
abstract = EXCLUDED.abstract,
source_path = COALESCE(EXCLUDED.source_path, papers.source_path),
abstract_emb = EXCLUDED.abstract_emb
"""
with get_conn() as conn:
conn.execute(
"""
INSERT INTO papers (id, openalex_id, bibkey, title, authors, year, abstract,
source_path, abstract_emb)
VALUES (%(id)s, %(openalex_id)s, %(bibkey)s, %(title)s, %(authors)s, %(year)s,
%(abstract)s, %(source_path)s, %(abstract_emb)s)
ON CONFLICT (id) DO UPDATE SET
openalex_id = EXCLUDED.openalex_id,
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
title = EXCLUDED.title,
authors = EXCLUDED.authors,
year = EXCLUDED.year,
abstract = EXCLUDED.abstract,
source_path = COALESCE(EXCLUDED.source_path, papers.source_path),
abstract_emb = EXCLUDED.abstract_emb
""",
{
"id": paper.id,
"openalex_id": paper.openalex_id,
"bibkey": paper.bibkey,
"title": paper.title,
"authors": paper.authors,
"year": paper.year,
"abstract": paper.abstract,
"source_path": source_path,
"abstract_emb": abstract_emb,
},
)
# The id-keyed upsert does not catch a papers.bibkey UNIQUE collision (two
# same-author/year papers generate the same key). On such a collision retry
# with an a/b/c… suffix instead of failing the whole ingest (audit C-15).
base_bibkey = paper.bibkey
for attempt in range(27):
try:
conn.execute(
upsert_sql,
{
"id": paper.id,
"openalex_id": paper.openalex_id,
"bibkey": paper.bibkey,
"title": paper.title,
"authors": paper.authors,
"year": paper.year,
"abstract": paper.abstract,
"source_path": source_path,
"abstract_emb": abstract_emb,
},
)
break
except psycopg.errors.UniqueViolation as exc:
msg = str(exc).lower()
conn.rollback()
if "openalex" in msg:
# Two different papers.id claim the same OpenAlex work — a real data
# conflict, not something to auto-rename. Surface it clearly instead
# of the raw psycopg error (companion gap to C-15's bibkey case).
raise ValueError(
f"openalex_id {paper.openalex_id!r} is already attached to a "
f"different paper; ingesting '{paper.id}' would duplicate it — "
"resolve the conflicting paper first."
) from exc
if base_bibkey is None or "bibkey" not in msg or attempt == 26:
raise
paper.bibkey = f"{base_bibkey}{chr(ord('a') + attempt)}"
logger.warning(
"bibkey %r already exists; ingesting '%s' as %r instead",
base_bibkey,
paper.id,
paper.bibkey,
)
# Register openalex_id in paper_identifiers (alias table).
# Every ingest records the primary openalex_id so the graph JOIN
@@ -429,7 +468,10 @@ def ingest_paper(
VALUES (%s, %s, %s, %s, %s)
""",
[
(f.paper_id, f.page, f.raw_latex, f.context, f.eq_label)
# paper.id, not f.paper_id: the extractor derives its
# paper_id from the PDF filename stem, which need not match
# papers.id and would violate the FK (audit C-10).
(paper.id, f.page, f.raw_latex, f.context, f.eq_label)
for f in formulas
],
)
@@ -443,7 +485,8 @@ def ingest_paper(
VALUES (%s, %s, %s, %s)
""",
[
(fig.paper_id, fig.page, fig.caption, fig.image_path)
# paper.id, not fig.paper_id (filename stem) — see C-10 above.
(paper.id, fig.page, fig.caption, fig.image_path)
for fig in figures
],
)