fix(ingest): D-04 bibkey heuristic + D-05a/b/c regression tests + __main__ #9

Merged
user2595 merged 1 commits from fix/d04-d05-ingest-fixes into main 2026-06-15 01:07:36 +00:00
4 changed files with 139 additions and 5 deletions

3
codex/__main__.py Normal file
View File

@@ -0,0 +1,3 @@
from codex.cli import app
app()

View File

@@ -27,6 +27,25 @@ class IngestResult:
figures_upserted: int = 0 figures_upserted: int = 0
def _make_bibkey(paper: Paper) -> str | None:
"""Generate a BibTeX-style key from paper metadata.
Format: ConcatSurnames + Year (e.g. "BobenkoPinkallSpringborn2015").
Surnames are the last whitespace-separated token of each author name.
>3 authors: FirstSurname + "EtAl" + Year.
Returns None when authors or year are missing.
"""
if not paper.authors or not paper.year:
return None
surnames = [name.split()[-1] for name in paper.authors if name.strip()]
if not surnames:
return None
year = str(paper.year)
if len(surnames) <= 3:
return "".join(surnames) + year
return surnames[0] + "EtAl" + year
def ingest_paper( def ingest_paper(
paper_id: str, paper_id: str,
source_path: str | None = None, source_path: str | None = None,
@@ -61,6 +80,7 @@ def ingest_paper(
# --------------------------------------------------------------- # ---------------------------------------------------------------
paper: Paper | None = openalex.fetch_paper(paper_id) paper: Paper | None = openalex.fetch_paper(paper_id)
if paper is None: if paper is None:
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix # Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
pid_lower = paper_id.lower() pid_lower = paper_id.lower()
@@ -78,6 +98,10 @@ def ingest_paper(
if paper is None: if paper is None:
raise ValueError(f"Paper not found: {paper_id}") raise ValueError(f"Paper not found: {paper_id}")
# Auto-generate bibkey when source has none (D-04).
if paper.bibkey is None:
paper.bibkey = _make_bibkey(paper)
# --------------------------------------------------------------- # ---------------------------------------------------------------
# 2. Embed abstract (dense only — schema has one vector column) # 2. Embed abstract (dense only — schema has one vector column)
# --------------------------------------------------------------- # ---------------------------------------------------------------
@@ -102,6 +126,7 @@ def ingest_paper(
%(abstract)s, %(source_path)s, %(abstract_emb)s) %(abstract)s, %(source_path)s, %(abstract_emb)s)
ON CONFLICT (id) DO UPDATE SET ON CONFLICT (id) DO UPDATE SET
openalex_id = EXCLUDED.openalex_id, openalex_id = EXCLUDED.openalex_id,
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
title = EXCLUDED.title, title = EXCLUDED.title,
authors = EXCLUDED.authors, authors = EXCLUDED.authors,
year = EXCLUDED.year, year = EXCLUDED.year,
@@ -181,7 +206,10 @@ def ingest_paper(
raw = openalex.fetch_citations(paper.openalex_id) raw = openalex.fetch_citations(paper.openalex_id)
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI. # OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
# Rewrite citing_id to paper.id so the FK constraint is satisfied. # Rewrite citing_id to paper.id so the FK constraint is satisfied.
api_citations = [Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw] api_citations = [
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
for c in raw
]
else: else:
api_citations = semanticscholar.fetch_references(paper_id) api_citations = semanticscholar.fetch_references(paper_id)

View File

@@ -351,3 +351,27 @@ class TestAsk:
assert result.exit_code == 1 assert result.exit_code == 1
assert "not yet implemented" in result.stdout or "not yet implemented" in result.stderr assert "not yet implemented" in result.stdout or "not yet implemented" in result.stderr
# ---------------------------------------------------------------------------
# D-05a/b — Entry-point smoke tests (python -m codex, no uv run)
# ---------------------------------------------------------------------------
class TestEntryPoint:
"""Verify that `python -m codex` works without `uv run` (D-05a/b)."""
def test_python_m_codex_help(self) -> None:
"""`python -m codex --help` exits 0 and prints usage."""
import subprocess
import sys
proc = subprocess.run(
[sys.executable, "-m", "codex", "--help"],
capture_output=True,
text=True,
)
assert proc.returncode == 0, f"Non-zero exit: {proc.stderr}"
assert "Usage" in proc.stdout or "usage" in proc.stdout.lower(), (
f"No usage in output: {proc.stdout[:200]}"
)

View File

@@ -13,7 +13,7 @@ from unittest.mock import MagicMock, patch
import numpy as np import numpy as np
import pytest import pytest
from codex.ingest import IngestResult, ingest_paper from codex.ingest import IngestResult, _make_bibkey, ingest_paper
from codex.models import Citation, FigureChunk, FormulaChunk, Paper from codex.models import Citation, FigureChunk, FormulaChunk, Paper
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -71,13 +71,14 @@ def test_ingest_paper_basic() -> None:
mock_conn.executemany = MagicMock() mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock() mock_conn.commit = MagicMock()
api_citations = [ # OpenAlex returns citing_id=openalex_id; ingest.py must rewrite to paper.id (D-05c).
raw_api_citations = [
Citation(citing_id=paper.openalex_id or "", cited_id="W999"), Citation(citing_id=paper.openalex_id or "", cited_id="W999"),
] ]
with ( with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper) as mock_fetch, patch("codex.ingest.openalex.fetch_paper", return_value=paper) as mock_fetch,
patch("codex.ingest.openalex.fetch_citations", return_value=api_citations), patch("codex.ingest.openalex.fetch_citations", return_value=raw_api_citations),
patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
): ):
@@ -94,7 +95,15 @@ def test_ingest_paper_basic() -> None:
assert isinstance(result, IngestResult) assert isinstance(result, IngestResult)
assert result.paper_id == paper.id assert result.paper_id == paper.id
assert result.chunks_upserted == 0 assert result.chunks_upserted == 0
assert result.citations_upserted == len(api_citations) assert result.citations_upserted == len(raw_api_citations)
# D-05c regression: citing_id must be rewritten to paper.id (DOI), not openalex_id.
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
cit_call = mock_cursor.executemany.call_args_list[0]
inserted_rows: list[tuple[str, str, Any]] = cit_call[0][1]
assert inserted_rows[0][0] == paper.id, (
f"citing_id must be paper.id='{paper.id}', not openalex_id='{paper.openalex_id}'"
)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -404,6 +413,76 @@ def test_ingest_paper_rich_requires_pdf_source(tmp_path: Any) -> None:
assert result.figures_upserted == 0 assert result.figures_upserted == 0
# ---------------------------------------------------------------------------
# D-04 — bibkey auto-generation heuristic
# ---------------------------------------------------------------------------
def test_make_bibkey_single_author() -> None:
paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=2023)
assert _make_bibkey(paper) == "Smith2023"
def test_make_bibkey_two_authors() -> None:
paper = Paper(id="doi:10.1/x", title="T", authors=["Alice Foo", "Bob Bar"], year=2015)
assert _make_bibkey(paper) == "FooBar2015"
def test_make_bibkey_three_authors() -> None:
paper = Paper(
id="doi:10.1/x",
title="T",
authors=["Alexander Bobenko", "Ulrich Pinkall", "Boris Springborn"],
year=2015,
)
assert _make_bibkey(paper) == "BobenkoPinkallSpringborn2015"
def test_make_bibkey_many_authors_uses_etal() -> None:
paper = Paper(
id="doi:10.1/x",
title="T",
authors=["A One", "B Two", "C Three", "D Four"],
year=2020,
)
assert _make_bibkey(paper) == "OneEtAl2020"
def test_make_bibkey_no_authors_returns_none() -> None:
paper = Paper(id="doi:10.1/x", title="T", authors=[], year=2023)
assert _make_bibkey(paper) is None
def test_make_bibkey_no_year_returns_none() -> None:
paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=None)
assert _make_bibkey(paper) is None
def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None:
"""ingest_paper auto-generates bibkey when paper has none (D-04)."""
paper = _make_paper() # bibkey=None (default from _make_paper)
mock_conn = MagicMock()
mock_conn.execute = MagicMock()
mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock()
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
):
ingest_paper(paper.id)
upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
assert upsert_params["bibkey"] is not None, "bibkey must be auto-generated when paper has none"
# authors=["Alice", "Bob"], year=2023 → 2 authors ≤ 3 → concat last tokens
assert upsert_params["bibkey"] == "AliceBob2023"
# ---------------------------------------------------------------------------
def test_ingest_result_has_formula_figure_counts() -> None: def test_ingest_result_has_formula_figure_counts() -> None:
"""IngestResult has formulas_upserted and figures_upserted fields with default 0.""" """IngestResult has formulas_upserted and figures_upserted fields with default 0."""
result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0) result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0)