fix(parsing): case-insensitive arXiv idno match; fix tmp_path types; add extract_structure test
Review-Gate finding: GROBID emits type="arXiv" (camel-case), not "arxiv". XPath literal match silently returned empty strings for all real responses. Fix: iterate idno elements and compare .lower() == "arxiv". Test fixture updated to reflect real GROBID output (type="arXiv"). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2,13 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from codex.parsing.grobid import extract_references, extract_structure
|
||||
|
||||
from codex.parsing.grobid import extract_references
|
||||
|
||||
# Minimal valid TEI XML with 2 biblStruct elements
|
||||
# Real GROBID TEI output uses type="arXiv" (camel-case), not "arxiv".
|
||||
_TEI_XML = """\
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<TEI xmlns="http://www.tei-c.org/ns/1.0">
|
||||
@@ -32,7 +31,7 @@ _TEI_XML = """\
|
||||
</imprint>
|
||||
</monogr>
|
||||
<idno type="DOI">10.1234/nlp.2021</idno>
|
||||
<idno type="arxiv">2101.00001</idno>
|
||||
<idno type="arXiv">2101.00001</idno>
|
||||
</biblStruct>
|
||||
<biblStruct>
|
||||
<analytic>
|
||||
@@ -73,28 +72,36 @@ class _FakeResponse:
|
||||
pass
|
||||
|
||||
|
||||
def _mock_post(xml: str = _TEI_XML) -> MagicMock:
|
||||
fake = MagicMock()
|
||||
fake.text = xml
|
||||
fake.status_code = 200
|
||||
fake.raise_for_status = MagicMock()
|
||||
return fake
|
||||
|
||||
|
||||
class TestExtractReferences:
|
||||
def test_returns_two_dicts(self, tmp_path: pytest.TempdirFactory) -> None:
|
||||
pdf = tmp_path / "paper.pdf" # type: ignore[operator]
|
||||
def test_returns_two_dicts(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _FakeResponse()
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
assert len(refs) == 2
|
||||
|
||||
def test_all_keys_present(self, tmp_path: pytest.TempdirFactory) -> None:
|
||||
pdf = tmp_path / "paper.pdf" # type: ignore[operator]
|
||||
def test_all_keys_present(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _FakeResponse()
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
@@ -102,14 +109,14 @@ class TestExtractReferences:
|
||||
for ref in refs:
|
||||
assert required_keys == set(ref.keys()), f"Missing keys in {ref}"
|
||||
|
||||
def test_first_ref_values(self, tmp_path: pytest.TempdirFactory) -> None:
|
||||
pdf = tmp_path / "paper.pdf" # type: ignore[operator]
|
||||
def test_first_ref_values(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _FakeResponse()
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
@@ -120,16 +127,14 @@ class TestExtractReferences:
|
||||
assert first["doi"] == "10.1234/nlp.2021"
|
||||
assert first["arxiv_id"] == "2101.00001"
|
||||
|
||||
def test_second_ref_missing_arxiv_is_empty_string(
|
||||
self, tmp_path: pytest.TempdirFactory
|
||||
) -> None:
|
||||
pdf = tmp_path / "paper.pdf" # type: ignore[operator]
|
||||
def test_second_ref_missing_arxiv_is_empty_string(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _FakeResponse()
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
@@ -137,3 +142,20 @@ class TestExtractReferences:
|
||||
assert second["arxiv_id"] == ""
|
||||
assert second["title"] == "Attention Is All You Need"
|
||||
assert second["year"] == "2017"
|
||||
|
||||
|
||||
class TestExtractStructure:
|
||||
def test_returns_xml_string(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _mock_post("<TEI>raw xml</TEI>")
|
||||
|
||||
result = extract_structure(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
assert result == "<TEI>raw xml</TEI>"
|
||||
mock_client.post.assert_called_once()
|
||||
assert "processFulltextDocument" in mock_client.post.call_args[0][0]
|
||||
|
||||
Reference in New Issue
Block a user