feat(parsing): LaTeX, Nougat, GROBID parsers with chunking

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-04 23:30:58 +02:00
parent b52a6a7412
commit c6a428d335
8 changed files with 672 additions and 1 deletions
--- a/tests/parsing/test_grobid.py
+++ b/tests/parsing/test_grobid.py
@@ -0,0 +1,139 @@
+"""Tests for codex.parsing.grobid."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from codex.parsing.grobid import extract_references
+
+# Minimal valid TEI XML with 2 biblStruct elements
+_TEI_XML = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+  <text>
+    <back>
+      <div type="references">
+        <listBibl>
+          <biblStruct>
+            <analytic>
+              <title level="a">Deep Learning for NLP</title>
+              <author>
+                <persName>
+                  <forename>Jane</forename>
+                  <surname>Doe</surname>
+                </persName>
+              </author>
+            </analytic>
+            <monogr>
+              <imprint>
+                <date type="published" when="2021-06"/>
+              </imprint>
+            </monogr>
+            <idno type="DOI">10.1234/nlp.2021</idno>
+            <idno type="arxiv">2101.00001</idno>
+          </biblStruct>
+          <biblStruct>
+            <analytic>
+              <title level="a">Attention Is All You Need</title>
+              <author>
+                <persName>
+                  <forename>Ashish</forename>
+                  <surname>Vaswani</surname>
+                </persName>
+              </author>
+              <author>
+                <persName>
+                  <forename>Noam</forename>
+                  <surname>Shazeer</surname>
+                </persName>
+              </author>
+            </analytic>
+            <monogr>
+              <imprint>
+                <date type="published" when="2017"/>
+              </imprint>
+            </monogr>
+            <idno type="DOI">10.5678/attention</idno>
+          </biblStruct>
+        </listBibl>
+      </div>
+    </back>
+  </text>
+</TEI>
+"""
+
+
+class _FakeResponse:
+    text = _TEI_XML
+    status_code = 200
+
+    def raise_for_status(self) -> None:
+        pass
+
+
+class TestExtractReferences:
+    def test_returns_two_dicts(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        assert len(refs) == 2
+
+    def test_all_keys_present(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        required_keys = {"title", "authors", "year", "doi", "arxiv_id"}
+        for ref in refs:
+            assert required_keys == set(ref.keys()), f"Missing keys in {ref}"
+
+    def test_first_ref_values(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        first = refs[0]
+        assert first["title"] == "Deep Learning for NLP"
+        assert "Jane Doe" in first["authors"]
+        assert first["year"] == "2021"
+        assert first["doi"] == "10.1234/nlp.2021"
+        assert first["arxiv_id"] == "2101.00001"
+
+    def test_second_ref_missing_arxiv_is_empty_string(
+        self, tmp_path: pytest.TempdirFactory
+    ) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        second = refs[1]
+        assert second["arxiv_id"] == ""
+        assert second["title"] == "Attention Is All You Need"
+        assert second["year"] == "2017"