fix(ingest): D-04 bibkey heuristic + D-05a/b/c regression tests + __main__
D-04: add _make_bibkey(paper) — ConcatSurnames+Year, ≤3 authors full list,
>3 authors FirstEtAlYear. Applied in ingest_paper when paper.bibkey is
None. ON CONFLICT now fills NULL bibkeys via COALESCE(papers.bibkey,
EXCLUDED.bibkey) while preserving manually-set values.
D-05b: add codex/__main__.py so `python -m codex` dispatches to the CLI.
D-05c: fix test_ingest.py:75 — rename var to raw_api_citations (the raw
OpenAlex payload with citing_id=openalex_id) and add regression
assertion: citing_id written to DB must equal paper.id (DOI), not
paper.openalex_id.
D-05a (proxy): add TestEntryPoint.test_python_m_codex_help — runs
`sys.executable -m codex --help` via subprocess, covering the
non-uv-run entry path. 253 tests green, ruff+mypy clean.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
3
codex/__main__.py
Normal file
3
codex/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
from codex.cli import app
|
||||||
|
|
||||||
|
app()
|
||||||
@@ -27,6 +27,25 @@ class IngestResult:
|
|||||||
figures_upserted: int = 0
|
figures_upserted: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _make_bibkey(paper: Paper) -> str | None:
|
||||||
|
"""Generate a BibTeX-style key from paper metadata.
|
||||||
|
|
||||||
|
Format: ConcatSurnames + Year (e.g. "BobenkoPinkallSpringborn2015").
|
||||||
|
Surnames are the last whitespace-separated token of each author name.
|
||||||
|
>3 authors: FirstSurname + "EtAl" + Year.
|
||||||
|
Returns None when authors or year are missing.
|
||||||
|
"""
|
||||||
|
if not paper.authors or not paper.year:
|
||||||
|
return None
|
||||||
|
surnames = [name.split()[-1] for name in paper.authors if name.strip()]
|
||||||
|
if not surnames:
|
||||||
|
return None
|
||||||
|
year = str(paper.year)
|
||||||
|
if len(surnames) <= 3:
|
||||||
|
return "".join(surnames) + year
|
||||||
|
return surnames[0] + "EtAl" + year
|
||||||
|
|
||||||
|
|
||||||
def ingest_paper(
|
def ingest_paper(
|
||||||
paper_id: str,
|
paper_id: str,
|
||||||
source_path: str | None = None,
|
source_path: str | None = None,
|
||||||
@@ -61,6 +80,7 @@ def ingest_paper(
|
|||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
paper: Paper | None = openalex.fetch_paper(paper_id)
|
paper: Paper | None = openalex.fetch_paper(paper_id)
|
||||||
|
|
||||||
|
|
||||||
if paper is None:
|
if paper is None:
|
||||||
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
|
# Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix
|
||||||
pid_lower = paper_id.lower()
|
pid_lower = paper_id.lower()
|
||||||
@@ -78,6 +98,10 @@ def ingest_paper(
|
|||||||
if paper is None:
|
if paper is None:
|
||||||
raise ValueError(f"Paper not found: {paper_id}")
|
raise ValueError(f"Paper not found: {paper_id}")
|
||||||
|
|
||||||
|
# Auto-generate bibkey when source has none (D-04).
|
||||||
|
if paper.bibkey is None:
|
||||||
|
paper.bibkey = _make_bibkey(paper)
|
||||||
|
|
||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
# 2. Embed abstract (dense only — schema has one vector column)
|
# 2. Embed abstract (dense only — schema has one vector column)
|
||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
@@ -102,6 +126,7 @@ def ingest_paper(
|
|||||||
%(abstract)s, %(source_path)s, %(abstract_emb)s)
|
%(abstract)s, %(source_path)s, %(abstract_emb)s)
|
||||||
ON CONFLICT (id) DO UPDATE SET
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
openalex_id = EXCLUDED.openalex_id,
|
openalex_id = EXCLUDED.openalex_id,
|
||||||
|
bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey),
|
||||||
title = EXCLUDED.title,
|
title = EXCLUDED.title,
|
||||||
authors = EXCLUDED.authors,
|
authors = EXCLUDED.authors,
|
||||||
year = EXCLUDED.year,
|
year = EXCLUDED.year,
|
||||||
@@ -181,7 +206,10 @@ def ingest_paper(
|
|||||||
raw = openalex.fetch_citations(paper.openalex_id)
|
raw = openalex.fetch_citations(paper.openalex_id)
|
||||||
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
|
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
|
||||||
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
|
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
|
||||||
api_citations = [Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw]
|
api_citations = [
|
||||||
|
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context)
|
||||||
|
for c in raw
|
||||||
|
]
|
||||||
else:
|
else:
|
||||||
api_citations = semanticscholar.fetch_references(paper_id)
|
api_citations = semanticscholar.fetch_references(paper_id)
|
||||||
|
|
||||||
|
|||||||
@@ -351,3 +351,27 @@ class TestAsk:
|
|||||||
|
|
||||||
assert result.exit_code == 1
|
assert result.exit_code == 1
|
||||||
assert "not yet implemented" in result.stdout or "not yet implemented" in result.stderr
|
assert "not yet implemented" in result.stdout or "not yet implemented" in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# D-05a/b — Entry-point smoke tests (python -m codex, no uv run)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestEntryPoint:
|
||||||
|
"""Verify that `python -m codex` works without `uv run` (D-05a/b)."""
|
||||||
|
|
||||||
|
def test_python_m_codex_help(self) -> None:
|
||||||
|
"""`python -m codex --help` exits 0 and prints usage."""
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
proc = subprocess.run(
|
||||||
|
[sys.executable, "-m", "codex", "--help"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert proc.returncode == 0, f"Non-zero exit: {proc.stderr}"
|
||||||
|
assert "Usage" in proc.stdout or "usage" in proc.stdout.lower(), (
|
||||||
|
f"No usage in output: {proc.stdout[:200]}"
|
||||||
|
)
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from codex.ingest import IngestResult, ingest_paper
|
from codex.ingest import IngestResult, _make_bibkey, ingest_paper
|
||||||
from codex.models import Citation, FigureChunk, FormulaChunk, Paper
|
from codex.models import Citation, FigureChunk, FormulaChunk, Paper
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -71,13 +71,14 @@ def test_ingest_paper_basic() -> None:
|
|||||||
mock_conn.executemany = MagicMock()
|
mock_conn.executemany = MagicMock()
|
||||||
mock_conn.commit = MagicMock()
|
mock_conn.commit = MagicMock()
|
||||||
|
|
||||||
api_citations = [
|
# OpenAlex returns citing_id=openalex_id; ingest.py must rewrite to paper.id (D-05c).
|
||||||
|
raw_api_citations = [
|
||||||
Citation(citing_id=paper.openalex_id or "", cited_id="W999"),
|
Citation(citing_id=paper.openalex_id or "", cited_id="W999"),
|
||||||
]
|
]
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper) as mock_fetch,
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper) as mock_fetch,
|
||||||
patch("codex.ingest.openalex.fetch_citations", return_value=api_citations),
|
patch("codex.ingest.openalex.fetch_citations", return_value=raw_api_citations),
|
||||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||||
):
|
):
|
||||||
@@ -94,7 +95,15 @@ def test_ingest_paper_basic() -> None:
|
|||||||
assert isinstance(result, IngestResult)
|
assert isinstance(result, IngestResult)
|
||||||
assert result.paper_id == paper.id
|
assert result.paper_id == paper.id
|
||||||
assert result.chunks_upserted == 0
|
assert result.chunks_upserted == 0
|
||||||
assert result.citations_upserted == len(api_citations)
|
assert result.citations_upserted == len(raw_api_citations)
|
||||||
|
|
||||||
|
# D-05c regression: citing_id must be rewritten to paper.id (DOI), not openalex_id.
|
||||||
|
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
||||||
|
cit_call = mock_cursor.executemany.call_args_list[0]
|
||||||
|
inserted_rows: list[tuple[str, str, Any]] = cit_call[0][1]
|
||||||
|
assert inserted_rows[0][0] == paper.id, (
|
||||||
|
f"citing_id must be paper.id='{paper.id}', not openalex_id='{paper.openalex_id}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -404,6 +413,76 @@ def test_ingest_paper_rich_requires_pdf_source(tmp_path: Any) -> None:
|
|||||||
assert result.figures_upserted == 0
|
assert result.figures_upserted == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# D-04 — bibkey auto-generation heuristic
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_bibkey_single_author() -> None:
|
||||||
|
paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=2023)
|
||||||
|
assert _make_bibkey(paper) == "Smith2023"
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_bibkey_two_authors() -> None:
|
||||||
|
paper = Paper(id="doi:10.1/x", title="T", authors=["Alice Foo", "Bob Bar"], year=2015)
|
||||||
|
assert _make_bibkey(paper) == "FooBar2015"
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_bibkey_three_authors() -> None:
|
||||||
|
paper = Paper(
|
||||||
|
id="doi:10.1/x",
|
||||||
|
title="T",
|
||||||
|
authors=["Alexander Bobenko", "Ulrich Pinkall", "Boris Springborn"],
|
||||||
|
year=2015,
|
||||||
|
)
|
||||||
|
assert _make_bibkey(paper) == "BobenkoPinkallSpringborn2015"
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_bibkey_many_authors_uses_etal() -> None:
|
||||||
|
paper = Paper(
|
||||||
|
id="doi:10.1/x",
|
||||||
|
title="T",
|
||||||
|
authors=["A One", "B Two", "C Three", "D Four"],
|
||||||
|
year=2020,
|
||||||
|
)
|
||||||
|
assert _make_bibkey(paper) == "OneEtAl2020"
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_bibkey_no_authors_returns_none() -> None:
|
||||||
|
paper = Paper(id="doi:10.1/x", title="T", authors=[], year=2023)
|
||||||
|
assert _make_bibkey(paper) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_bibkey_no_year_returns_none() -> None:
|
||||||
|
paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=None)
|
||||||
|
assert _make_bibkey(paper) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None:
|
||||||
|
"""ingest_paper auto-generates bibkey when paper has none (D-04)."""
|
||||||
|
paper = _make_paper() # bibkey=None (default from _make_paper)
|
||||||
|
mock_conn = MagicMock()
|
||||||
|
mock_conn.execute = MagicMock()
|
||||||
|
mock_conn.executemany = MagicMock()
|
||||||
|
mock_conn.commit = MagicMock()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||||
|
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||||
|
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||||
|
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||||
|
):
|
||||||
|
ingest_paper(paper.id)
|
||||||
|
|
||||||
|
upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
||||||
|
assert upsert_params["bibkey"] is not None, "bibkey must be auto-generated when paper has none"
|
||||||
|
# authors=["Alice", "Bob"], year=2023 → 2 authors ≤ 3 → concat last tokens
|
||||||
|
assert upsert_params["bibkey"] == "AliceBob2023"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def test_ingest_result_has_formula_figure_counts() -> None:
|
def test_ingest_result_has_formula_figure_counts() -> None:
|
||||||
"""IngestResult has formulas_upserted and figures_upserted fields with default 0."""
|
"""IngestResult has formulas_upserted and figures_upserted fields with default 0."""
|
||||||
result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0)
|
result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0)
|
||||||
|
|||||||
Reference in New Issue
Block a user