ff03443af4
Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404 -> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in: - arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API (authoritative for preprints). Used as an ingest fallback on OpenAlex 404, replacing the empty stub. - crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool. - semanticscholar.fetch_abstract: fetch just the abstract field. - ingest.py: _recover_abstract() supplements an empty abstract from S2 then Crossref *before* embedding, so the paper gets a real (non-zero) vector. Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter 10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented limit. Corpus now has 1 paper without an abstract. Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers (openalex returns full-URL id vs the bare canonical id) -> re-ingest trips papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix. Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
51 lines
1.7 KiB
Python
51 lines
1.7 KiB
Python
"""Tests for codex.sources.crossref."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from codex.sources import crossref
|
|
|
|
|
|
def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A JATS-tagged abstract is returned as plain text with tags removed."""
|
|
body = {
|
|
"message": {
|
|
"abstract": "<jats:p>We prove a <jats:italic>theorem</jats:italic> about "
|
|
"polyhedra.</jats:p>"
|
|
}
|
|
}
|
|
|
|
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
|
return httpx.Response(200, json=body)
|
|
|
|
monkeypatch.setattr(crossref, "_get", mock_get)
|
|
|
|
result = crossref.fetch_abstract("10.1007/s00454-019-00132-8")
|
|
assert result == "We prove a theorem about polyhedra."
|
|
|
|
|
|
def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A work with no abstract field (e.g. a book chapter) returns None."""
|
|
|
|
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
|
return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}})
|
|
|
|
monkeypatch.setattr(crossref, "_get", mock_get)
|
|
assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None
|
|
|
|
|
|
def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A 404 returns None rather than raising."""
|
|
|
|
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
|
raise httpx.HTTPStatusError(
|
|
"404",
|
|
request=httpx.Request("GET", url),
|
|
response=httpx.Response(404, request=httpx.Request("GET", url)),
|
|
)
|
|
|
|
monkeypatch.setattr(crossref, "_get", mock_get)
|
|
assert crossref.fetch_abstract("10.0000/nonexistent") is None
|