From b87087d3c62a758e68afe740950ac2fb76dec78d Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 15 Jun 2026 03:07:06 +0200 Subject: [PATCH] fix(ingest): D-04 bibkey heuristic + D-05a/b/c regression tests + __main__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D-04: add _make_bibkey(paper) — ConcatSurnames+Year, ≤3 authors full list, >3 authors FirstEtAlYear. Applied in ingest_paper when paper.bibkey is None. ON CONFLICT now fills NULL bibkeys via COALESCE(papers.bibkey, EXCLUDED.bibkey) while preserving manually-set values. D-05b: add codex/__main__.py so `python -m codex` dispatches to the CLI. D-05c: fix test_ingest.py:75 — rename var to raw_api_citations (the raw OpenAlex payload with citing_id=openalex_id) and add regression assertion: citing_id written to DB must equal paper.id (DOI), not paper.openalex_id. D-05a (proxy): add TestEntryPoint.test_python_m_codex_help — runs `sys.executable -m codex --help` via subprocess, covering the non-uv-run entry path. 253 tests green, ruff+mypy clean. Co-Authored-By: Claude Sonnet 4.6 --- codex/__main__.py | 3 ++ codex/ingest.py | 30 ++++++++++++- tests/cli/test_cli.py | 24 ++++++++++ tests/ingest/test_ingest.py | 87 +++++++++++++++++++++++++++++++++++-- 4 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 codex/__main__.py diff --git a/codex/__main__.py b/codex/__main__.py new file mode 100644 index 0000000..aaec1a6 --- /dev/null +++ b/codex/__main__.py @@ -0,0 +1,3 @@ +from codex.cli import app + +app() diff --git a/codex/ingest.py b/codex/ingest.py index 72d2deb..2c2e912 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -27,6 +27,25 @@ class IngestResult: figures_upserted: int = 0 +def _make_bibkey(paper: Paper) -> str | None: + """Generate a BibTeX-style key from paper metadata. + + Format: ConcatSurnames + Year (e.g. "BobenkoPinkallSpringborn2015"). + Surnames are the last whitespace-separated token of each author name. + >3 authors: FirstSurname + "EtAl" + Year. + Returns None when authors or year are missing. + """ + if not paper.authors or not paper.year: + return None + surnames = [name.split()[-1] for name in paper.authors if name.strip()] + if not surnames: + return None + year = str(paper.year) + if len(surnames) <= 3: + return "".join(surnames) + year + return surnames[0] + "EtAl" + year + + def ingest_paper( paper_id: str, source_path: str | None = None, @@ -61,6 +80,7 @@ def ingest_paper( # --------------------------------------------------------------- paper: Paper | None = openalex.fetch_paper(paper_id) + if paper is None: # Detect arXiv IDs: numeric pattern like "2301.07041" or "arxiv:..." prefix pid_lower = paper_id.lower() @@ -78,6 +98,10 @@ def ingest_paper( if paper is None: raise ValueError(f"Paper not found: {paper_id}") + # Auto-generate bibkey when source has none (D-04). + if paper.bibkey is None: + paper.bibkey = _make_bibkey(paper) + # --------------------------------------------------------------- # 2. Embed abstract (dense only — schema has one vector column) # --------------------------------------------------------------- @@ -102,6 +126,7 @@ def ingest_paper( %(abstract)s, %(source_path)s, %(abstract_emb)s) ON CONFLICT (id) DO UPDATE SET openalex_id = EXCLUDED.openalex_id, + bibkey = COALESCE(papers.bibkey, EXCLUDED.bibkey), title = EXCLUDED.title, authors = EXCLUDED.authors, year = EXCLUDED.year, @@ -181,7 +206,10 @@ def ingest_paper( raw = openalex.fetch_citations(paper.openalex_id) # OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI. # Rewrite citing_id to paper.id so the FK constraint is satisfied. - api_citations = [Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw] + api_citations = [ + Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) + for c in raw + ] else: api_citations = semanticscholar.fetch_references(paper_id) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index d4ada3a..4d15713 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -351,3 +351,27 @@ class TestAsk: assert result.exit_code == 1 assert "not yet implemented" in result.stdout or "not yet implemented" in result.stderr + + +# --------------------------------------------------------------------------- +# D-05a/b — Entry-point smoke tests (python -m codex, no uv run) +# --------------------------------------------------------------------------- + + +class TestEntryPoint: + """Verify that `python -m codex` works without `uv run` (D-05a/b).""" + + def test_python_m_codex_help(self) -> None: + """`python -m codex --help` exits 0 and prints usage.""" + import subprocess + import sys + + proc = subprocess.run( + [sys.executable, "-m", "codex", "--help"], + capture_output=True, + text=True, + ) + assert proc.returncode == 0, f"Non-zero exit: {proc.stderr}" + assert "Usage" in proc.stdout or "usage" in proc.stdout.lower(), ( + f"No usage in output: {proc.stdout[:200]}" + ) diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index e34d227..d5b012c 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -13,7 +13,7 @@ from unittest.mock import MagicMock, patch import numpy as np import pytest -from codex.ingest import IngestResult, ingest_paper +from codex.ingest import IngestResult, _make_bibkey, ingest_paper from codex.models import Citation, FigureChunk, FormulaChunk, Paper # --------------------------------------------------------------------------- @@ -71,13 +71,14 @@ def test_ingest_paper_basic() -> None: mock_conn.executemany = MagicMock() mock_conn.commit = MagicMock() - api_citations = [ + # OpenAlex returns citing_id=openalex_id; ingest.py must rewrite to paper.id (D-05c). + raw_api_citations = [ Citation(citing_id=paper.openalex_id or "", cited_id="W999"), ] with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper) as mock_fetch, - patch("codex.ingest.openalex.fetch_citations", return_value=api_citations), + patch("codex.ingest.openalex.fetch_citations", return_value=raw_api_citations), patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), ): @@ -94,7 +95,15 @@ def test_ingest_paper_basic() -> None: assert isinstance(result, IngestResult) assert result.paper_id == paper.id assert result.chunks_upserted == 0 - assert result.citations_upserted == len(api_citations) + assert result.citations_upserted == len(raw_api_citations) + + # D-05c regression: citing_id must be rewritten to paper.id (DOI), not openalex_id. + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + cit_call = mock_cursor.executemany.call_args_list[0] + inserted_rows: list[tuple[str, str, Any]] = cit_call[0][1] + assert inserted_rows[0][0] == paper.id, ( + f"citing_id must be paper.id='{paper.id}', not openalex_id='{paper.openalex_id}'" + ) # --------------------------------------------------------------------------- @@ -404,6 +413,76 @@ def test_ingest_paper_rich_requires_pdf_source(tmp_path: Any) -> None: assert result.figures_upserted == 0 +# --------------------------------------------------------------------------- +# D-04 — bibkey auto-generation heuristic +# --------------------------------------------------------------------------- + + +def test_make_bibkey_single_author() -> None: + paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=2023) + assert _make_bibkey(paper) == "Smith2023" + + +def test_make_bibkey_two_authors() -> None: + paper = Paper(id="doi:10.1/x", title="T", authors=["Alice Foo", "Bob Bar"], year=2015) + assert _make_bibkey(paper) == "FooBar2015" + + +def test_make_bibkey_three_authors() -> None: + paper = Paper( + id="doi:10.1/x", + title="T", + authors=["Alexander Bobenko", "Ulrich Pinkall", "Boris Springborn"], + year=2015, + ) + assert _make_bibkey(paper) == "BobenkoPinkallSpringborn2015" + + +def test_make_bibkey_many_authors_uses_etal() -> None: + paper = Paper( + id="doi:10.1/x", + title="T", + authors=["A One", "B Two", "C Three", "D Four"], + year=2020, + ) + assert _make_bibkey(paper) == "OneEtAl2020" + + +def test_make_bibkey_no_authors_returns_none() -> None: + paper = Paper(id="doi:10.1/x", title="T", authors=[], year=2023) + assert _make_bibkey(paper) is None + + +def test_make_bibkey_no_year_returns_none() -> None: + paper = Paper(id="doi:10.1/x", title="T", authors=["John Smith"], year=None) + assert _make_bibkey(paper) is None + + +def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None: + """ingest_paper auto-generates bibkey when paper has none (D-04).""" + paper = _make_paper() # bibkey=None (default from _make_paper) + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + ingest_paper(paper.id) + + upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert upsert_params["bibkey"] is not None, "bibkey must be auto-generated when paper has none" + # authors=["Alice", "Bob"], year=2023 → 2 authors ≤ 3 → concat last tokens + assert upsert_params["bibkey"] == "AliceBob2023" + + +# --------------------------------------------------------------------------- + + def test_ingest_result_has_formula_figure_counts() -> None: """IngestResult has formulas_upserted and figures_upserted fields with default 0.""" result = IngestResult(paper_id="test", chunks_upserted=0, citations_upserted=0) -- 2.49.1