feat(parsing): LaTeX, Nougat, GROBID parsers with chunking

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tests/parsing/test_grobid.py

@@ -0,0 +1,139 @@
+"""Tests for codex.parsing.grobid."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from codex.parsing.grobid import extract_references
+
+# Minimal valid TEI XML with 2 biblStruct elements
+_TEI_XML = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+  <text>
+    <back>
+      <div type="references">
+        <listBibl>
+          <biblStruct>
+            <analytic>
+              <title level="a">Deep Learning for NLP</title>
+              <author>
+                <persName>
+                  <forename>Jane</forename>
+                  <surname>Doe</surname>
+                </persName>
+              </author>
+            </analytic>
+            <monogr>
+              <imprint>
+                <date type="published" when="2021-06"/>
+              </imprint>
+            </monogr>
+            <idno type="DOI">10.1234/nlp.2021</idno>
+            <idno type="arxiv">2101.00001</idno>
+          </biblStruct>
+          <biblStruct>
+            <analytic>
+              <title level="a">Attention Is All You Need</title>
+              <author>
+                <persName>
+                  <forename>Ashish</forename>
+                  <surname>Vaswani</surname>
+                </persName>
+              </author>
+              <author>
+                <persName>
+                  <forename>Noam</forename>
+                  <surname>Shazeer</surname>
+                </persName>
+              </author>
+            </analytic>
+            <monogr>
+              <imprint>
+                <date type="published" when="2017"/>
+              </imprint>
+            </monogr>
+            <idno type="DOI">10.5678/attention</idno>
+          </biblStruct>
+        </listBibl>
+      </div>
+    </back>
+  </text>
+</TEI>
+"""
+
+
+class _FakeResponse:
+    text = _TEI_XML
+    status_code = 200
+
+    def raise_for_status(self) -> None:
+        pass
+
+
+class TestExtractReferences:
+    def test_returns_two_dicts(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        assert len(refs) == 2
+
+    def test_all_keys_present(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        required_keys = {"title", "authors", "year", "doi", "arxiv_id"}
+        for ref in refs:
+            assert required_keys == set(ref.keys()), f"Missing keys in {ref}"
+
+    def test_first_ref_values(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        first = refs[0]
+        assert first["title"] == "Deep Learning for NLP"
+        assert "Jane Doe" in first["authors"]
+        assert first["year"] == "2021"
+        assert first["doi"] == "10.1234/nlp.2021"
+        assert first["arxiv_id"] == "2101.00001"
+
+    def test_second_ref_missing_arxiv_is_empty_string(
+        self, tmp_path: pytest.TempdirFactory
+    ) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = _FakeResponse()
+
+            refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
+
+        second = refs[1]
+        assert second["arxiv_id"] == ""
+        assert second["title"] == "Attention Is All You Need"
+        assert second["year"] == "2017"

tests/parsing/test_nougat.py

@@ -0,0 +1,70 @@
+"""Tests for codex.parsing.nougat."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import httpx
+import pytest
+
+from codex.parsing.nougat import pdf_to_markdown
+
+
+class _FakeResponse:
+    """Minimal stand-in for httpx.Response."""
+
+    def __init__(self, text: str, status_code: int = 200) -> None:
+        self.text = text
+        self.status_code = status_code
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise httpx.HTTPStatusError(
+                "error",
+                request=MagicMock(),
+                response=MagicMock(status_code=self.status_code),
+            )
+
+
+class TestPdfToMarkdown:
+    def test_success_returns_response_text(self, tmp_path: pytest.TempdirFactory) -> None:
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        expected = "# Title\nContent"
+        fake_response = _FakeResponse(expected)
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.return_value = fake_response
+
+            result = pdf_to_markdown(str(pdf), nougat_url="http://localhost:8080")
+
+        assert result == expected
+        mock_client.post.assert_called_once()
+        call_kwargs = mock_client.post.call_args
+        assert "/predict" in call_kwargs[0][0]
+
+    def test_connect_error_retried_exactly_twice(self, tmp_path: pytest.TempdirFactory) -> None:
+        """ConnectError must trigger 2 retries (3 total attempts)."""
+        pdf = tmp_path / "paper.pdf"  # type: ignore[operator]
+        pdf.write_bytes(b"%PDF fake")
+
+        call_count = 0
+
+        def _raising_post(*args: object, **kwargs: object) -> _FakeResponse:
+            nonlocal call_count
+            call_count += 1
+            raise httpx.ConnectError("refused")
+
+        with patch("httpx.Client") as mock_client_cls:
+            mock_client = MagicMock()
+            mock_client_cls.return_value.__enter__.return_value = mock_client
+            mock_client.post.side_effect = _raising_post
+
+            with pytest.raises(httpx.ConnectError):
+                pdf_to_markdown(str(pdf), nougat_url="http://localhost:8080")
+
+        # tenacity: stop_after_attempt(3) → 1 original + 2 retries = 3 calls
+        assert call_count == 3

tests/parsing/test_tex.py

@@ -0,0 +1,112 @@
+"""Tests for codex.parsing.tex."""
+
+from __future__ import annotations
+
+import pytest
+
+from codex.parsing.tex import chunk_text, extract_sections, latex_to_text
+
+
+class TestExtractSections:
+    def test_two_sections_returned(self) -> None:
+        latex = r"\section{Intro}" + "\nHello \\cite{foo} world\n"
+        latex += r"\section{Method}" + "\nResult"
+        sections = extract_sections(latex)
+        assert len(sections) == 2
+
+    def test_section_titles(self) -> None:
+        latex = r"\section{Intro}" + "\nHello\n" + r"\section{Method}" + "\nResult"
+        sections = extract_sections(latex)
+        assert sections[0][0] == "Intro"
+        assert sections[1][0] == "Method"
+
+    def test_cite_removed_from_text(self) -> None:
+        latex = r"\section{Intro}" + "\nHello \\cite{foo} world\n"
+        latex += r"\section{Method}" + "\nResult"
+        sections = extract_sections(latex)
+        body = sections[0][1]
+        assert r"\cite" not in body
+        assert "foo" not in body
+
+    def test_no_sections_returns_empty(self) -> None:
+        assert extract_sections("No sections here at all.") == []
+
+    def test_subsection_recognised(self) -> None:
+        latex = r"\subsection{Background}" + "\nSome text."
+        sections = extract_sections(latex)
+        assert len(sections) == 1
+        assert sections[0][0] == "Background"
+
+    def test_comments_stripped(self) -> None:
+        latex = r"\section{A}" + "\nHello % this is a comment\nWorld"
+        sections = extract_sections(latex)
+        assert "%" not in sections[0][1]
+        assert "comment" not in sections[0][1]
+
+    def test_math_stripped(self) -> None:
+        latex = r"\section{A}" + "\nLet $x = 1$ and $$y = 2$$ be numbers."
+        sections = extract_sections(latex)
+        body = sections[0][1]
+        assert "$" not in body
+
+
+class TestChunkText:
+    @pytest.fixture()
+    def long_text(self) -> str:
+        # Build a 600-word string with real sentences
+        sentence = "The quick brown fox jumps over the lazy dog near the river. "
+        words_per_sentence = len(sentence.split())
+        repeats = (600 // words_per_sentence) + 1
+        return (sentence * repeats).strip()
+
+    def test_at_least_two_chunks(self, long_text: str) -> None:
+        chunks = chunk_text(long_text, size=512, overlap=64)
+        assert len(chunks) >= 2
+
+    def test_each_chunk_within_size_bound(self, long_text: str) -> None:
+        size, overlap = 512, 64
+        chunks = chunk_text(long_text, size=size, overlap=overlap)
+        for chunk in chunks:
+            word_count = len(chunk.split())
+            assert word_count <= size + overlap, (
+                f"Chunk has {word_count} words, expected <= {size + overlap}"
+            )
+
+    def test_empty_text_returns_empty_list(self) -> None:
+        assert chunk_text("") == []
+
+    def test_short_text_returns_single_chunk(self) -> None:
+        chunks = chunk_text("Hello world.", size=512, overlap=64)
+        assert len(chunks) == 1
+
+    def test_chunks_cover_all_content(self) -> None:
+        # First chunk must start with first word, last chunk must end with last word
+        text = " ".join(f"word{i}" for i in range(600))
+        chunks = chunk_text(text, size=200, overlap=50)
+        assert chunks[0].startswith("word0")
+        assert chunks[-1].endswith("word599")
+
+
+class TestLatexToText:
+    def test_no_percent_in_output(self) -> None:
+        latex = r"\section{A}" + "\nSome % comment\ntext."
+        result = latex_to_text(latex)
+        assert "%" not in result
+
+    def test_no_cite_in_output(self) -> None:
+        latex = r"\section{A}" + "\nHello \\cite{bar} world."
+        result = latex_to_text(latex)
+        assert r"\cite" not in result
+        assert "bar" not in result
+
+    def test_returns_string(self) -> None:
+        latex = r"\section{A}" + "\nSimple text."
+        result = latex_to_text(latex)
+        assert isinstance(result, str)
+        assert len(result) > 0
+
+    def test_no_section_fallback(self) -> None:
+        latex = r"Some \cite{foo} text with % comment"
+        result = latex_to_text(latex)
+        assert r"\cite" not in result
+        assert "%" not in result