feat: merge F-03 parsing layer (LaTeX, Nougat, GROBID)
Review-Gate: APPROVE (Opus, cold, 2 passes) 23 tests, ruff+mypy clean. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
0
tests/parsing/__init__.py
Normal file
0
tests/parsing/__init__.py
Normal file
161
tests/parsing/test_grobid.py
Normal file
161
tests/parsing/test_grobid.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""Tests for codex.parsing.grobid."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from codex.parsing.grobid import extract_references, extract_structure
|
||||
|
||||
# Real GROBID TEI output uses type="arXiv" (camel-case), not "arxiv".
|
||||
_TEI_XML = """\
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<TEI xmlns="http://www.tei-c.org/ns/1.0">
|
||||
<text>
|
||||
<back>
|
||||
<div type="references">
|
||||
<listBibl>
|
||||
<biblStruct>
|
||||
<analytic>
|
||||
<title level="a">Deep Learning for NLP</title>
|
||||
<author>
|
||||
<persName>
|
||||
<forename>Jane</forename>
|
||||
<surname>Doe</surname>
|
||||
</persName>
|
||||
</author>
|
||||
</analytic>
|
||||
<monogr>
|
||||
<imprint>
|
||||
<date type="published" when="2021-06"/>
|
||||
</imprint>
|
||||
</monogr>
|
||||
<idno type="DOI">10.1234/nlp.2021</idno>
|
||||
<idno type="arXiv">2101.00001</idno>
|
||||
</biblStruct>
|
||||
<biblStruct>
|
||||
<analytic>
|
||||
<title level="a">Attention Is All You Need</title>
|
||||
<author>
|
||||
<persName>
|
||||
<forename>Ashish</forename>
|
||||
<surname>Vaswani</surname>
|
||||
</persName>
|
||||
</author>
|
||||
<author>
|
||||
<persName>
|
||||
<forename>Noam</forename>
|
||||
<surname>Shazeer</surname>
|
||||
</persName>
|
||||
</author>
|
||||
</analytic>
|
||||
<monogr>
|
||||
<imprint>
|
||||
<date type="published" when="2017"/>
|
||||
</imprint>
|
||||
</monogr>
|
||||
<idno type="DOI">10.5678/attention</idno>
|
||||
</biblStruct>
|
||||
</listBibl>
|
||||
</div>
|
||||
</back>
|
||||
</text>
|
||||
</TEI>
|
||||
"""
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
text = _TEI_XML
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _mock_post(xml: str = _TEI_XML) -> MagicMock:
|
||||
fake = MagicMock()
|
||||
fake.text = xml
|
||||
fake.status_code = 200
|
||||
fake.raise_for_status = MagicMock()
|
||||
return fake
|
||||
|
||||
|
||||
class TestExtractReferences:
|
||||
def test_returns_two_dicts(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
assert len(refs) == 2
|
||||
|
||||
def test_all_keys_present(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
required_keys = {"title", "authors", "year", "doi", "arxiv_id"}
|
||||
for ref in refs:
|
||||
assert required_keys == set(ref.keys()), f"Missing keys in {ref}"
|
||||
|
||||
def test_first_ref_values(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
first = refs[0]
|
||||
assert first["title"] == "Deep Learning for NLP"
|
||||
assert "Jane Doe" in first["authors"]
|
||||
assert first["year"] == "2021"
|
||||
assert first["doi"] == "10.1234/nlp.2021"
|
||||
assert first["arxiv_id"] == "2101.00001"
|
||||
|
||||
def test_second_ref_missing_arxiv_is_empty_string(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _mock_post()
|
||||
|
||||
refs = extract_references(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
second = refs[1]
|
||||
assert second["arxiv_id"] == ""
|
||||
assert second["title"] == "Attention Is All You Need"
|
||||
assert second["year"] == "2017"
|
||||
|
||||
|
||||
class TestExtractStructure:
|
||||
def test_returns_xml_string(self, tmp_path: Path) -> None:
|
||||
pdf = tmp_path / "paper.pdf"
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = _mock_post("<TEI>raw xml</TEI>")
|
||||
|
||||
result = extract_structure(str(pdf), grobid_url="http://localhost:8070")
|
||||
|
||||
assert result == "<TEI>raw xml</TEI>"
|
||||
mock_client.post.assert_called_once()
|
||||
assert "processFulltextDocument" in mock_client.post.call_args[0][0]
|
||||
70
tests/parsing/test_nougat.py
Normal file
70
tests/parsing/test_nougat.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""Tests for codex.parsing.nougat."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from codex.parsing.nougat import pdf_to_markdown
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
"""Minimal stand-in for httpx.Response."""
|
||||
|
||||
def __init__(self, text: str, status_code: int = 200) -> None:
|
||||
self.text = text
|
||||
self.status_code = status_code
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_code >= 400:
|
||||
raise httpx.HTTPStatusError(
|
||||
"error",
|
||||
request=MagicMock(),
|
||||
response=MagicMock(status_code=self.status_code),
|
||||
)
|
||||
|
||||
|
||||
class TestPdfToMarkdown:
|
||||
def test_success_returns_response_text(self, tmp_path: pytest.TempdirFactory) -> None:
|
||||
pdf = tmp_path / "paper.pdf" # type: ignore[operator]
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
expected = "# Title\nContent"
|
||||
fake_response = _FakeResponse(expected)
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.return_value = fake_response
|
||||
|
||||
result = pdf_to_markdown(str(pdf), nougat_url="http://localhost:8080")
|
||||
|
||||
assert result == expected
|
||||
mock_client.post.assert_called_once()
|
||||
call_kwargs = mock_client.post.call_args
|
||||
assert "/predict" in call_kwargs[0][0]
|
||||
|
||||
def test_connect_error_retried_exactly_twice(self, tmp_path: pytest.TempdirFactory) -> None:
|
||||
"""ConnectError must trigger 2 retries (3 total attempts)."""
|
||||
pdf = tmp_path / "paper.pdf" # type: ignore[operator]
|
||||
pdf.write_bytes(b"%PDF fake")
|
||||
|
||||
call_count = 0
|
||||
|
||||
def _raising_post(*args: object, **kwargs: object) -> _FakeResponse:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
raise httpx.ConnectError("refused")
|
||||
|
||||
with patch("httpx.Client") as mock_client_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client_cls.return_value.__enter__.return_value = mock_client
|
||||
mock_client.post.side_effect = _raising_post
|
||||
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
pdf_to_markdown(str(pdf), nougat_url="http://localhost:8080")
|
||||
|
||||
# tenacity: stop_after_attempt(3) → 1 original + 2 retries = 3 calls
|
||||
assert call_count == 3
|
||||
112
tests/parsing/test_tex.py
Normal file
112
tests/parsing/test_tex.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Tests for codex.parsing.tex."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from codex.parsing.tex import chunk_text, extract_sections, latex_to_text
|
||||
|
||||
|
||||
class TestExtractSections:
|
||||
def test_two_sections_returned(self) -> None:
|
||||
latex = r"\section{Intro}" + "\nHello \\cite{foo} world\n"
|
||||
latex += r"\section{Method}" + "\nResult"
|
||||
sections = extract_sections(latex)
|
||||
assert len(sections) == 2
|
||||
|
||||
def test_section_titles(self) -> None:
|
||||
latex = r"\section{Intro}" + "\nHello\n" + r"\section{Method}" + "\nResult"
|
||||
sections = extract_sections(latex)
|
||||
assert sections[0][0] == "Intro"
|
||||
assert sections[1][0] == "Method"
|
||||
|
||||
def test_cite_removed_from_text(self) -> None:
|
||||
latex = r"\section{Intro}" + "\nHello \\cite{foo} world\n"
|
||||
latex += r"\section{Method}" + "\nResult"
|
||||
sections = extract_sections(latex)
|
||||
body = sections[0][1]
|
||||
assert r"\cite" not in body
|
||||
assert "foo" not in body
|
||||
|
||||
def test_no_sections_returns_empty(self) -> None:
|
||||
assert extract_sections("No sections here at all.") == []
|
||||
|
||||
def test_subsection_recognised(self) -> None:
|
||||
latex = r"\subsection{Background}" + "\nSome text."
|
||||
sections = extract_sections(latex)
|
||||
assert len(sections) == 1
|
||||
assert sections[0][0] == "Background"
|
||||
|
||||
def test_comments_stripped(self) -> None:
|
||||
latex = r"\section{A}" + "\nHello % this is a comment\nWorld"
|
||||
sections = extract_sections(latex)
|
||||
assert "%" not in sections[0][1]
|
||||
assert "comment" not in sections[0][1]
|
||||
|
||||
def test_math_stripped(self) -> None:
|
||||
latex = r"\section{A}" + "\nLet $x = 1$ and $$y = 2$$ be numbers."
|
||||
sections = extract_sections(latex)
|
||||
body = sections[0][1]
|
||||
assert "$" not in body
|
||||
|
||||
|
||||
class TestChunkText:
|
||||
@pytest.fixture()
|
||||
def long_text(self) -> str:
|
||||
# Build a 600-word string with real sentences
|
||||
sentence = "The quick brown fox jumps over the lazy dog near the river. "
|
||||
words_per_sentence = len(sentence.split())
|
||||
repeats = (600 // words_per_sentence) + 1
|
||||
return (sentence * repeats).strip()
|
||||
|
||||
def test_at_least_two_chunks(self, long_text: str) -> None:
|
||||
chunks = chunk_text(long_text, size=512, overlap=64)
|
||||
assert len(chunks) >= 2
|
||||
|
||||
def test_each_chunk_within_size_bound(self, long_text: str) -> None:
|
||||
size, overlap = 512, 64
|
||||
chunks = chunk_text(long_text, size=size, overlap=overlap)
|
||||
for chunk in chunks:
|
||||
word_count = len(chunk.split())
|
||||
assert word_count <= size + overlap, (
|
||||
f"Chunk has {word_count} words, expected <= {size + overlap}"
|
||||
)
|
||||
|
||||
def test_empty_text_returns_empty_list(self) -> None:
|
||||
assert chunk_text("") == []
|
||||
|
||||
def test_short_text_returns_single_chunk(self) -> None:
|
||||
chunks = chunk_text("Hello world.", size=512, overlap=64)
|
||||
assert len(chunks) == 1
|
||||
|
||||
def test_chunks_cover_all_content(self) -> None:
|
||||
# First chunk must start with first word, last chunk must end with last word
|
||||
text = " ".join(f"word{i}" for i in range(600))
|
||||
chunks = chunk_text(text, size=200, overlap=50)
|
||||
assert chunks[0].startswith("word0")
|
||||
assert chunks[-1].endswith("word599")
|
||||
|
||||
|
||||
class TestLatexToText:
|
||||
def test_no_percent_in_output(self) -> None:
|
||||
latex = r"\section{A}" + "\nSome % comment\ntext."
|
||||
result = latex_to_text(latex)
|
||||
assert "%" not in result
|
||||
|
||||
def test_no_cite_in_output(self) -> None:
|
||||
latex = r"\section{A}" + "\nHello \\cite{bar} world."
|
||||
result = latex_to_text(latex)
|
||||
assert r"\cite" not in result
|
||||
assert "bar" not in result
|
||||
|
||||
def test_returns_string(self) -> None:
|
||||
latex = r"\section{A}" + "\nSimple text."
|
||||
result = latex_to_text(latex)
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
def test_no_section_fallback(self) -> None:
|
||||
latex = r"Some \cite{foo} text with % comment"
|
||||
result = latex_to_text(latex)
|
||||
assert r"\cite" not in result
|
||||
assert "%" not in result
|
||||
Reference in New Issue
Block a user