"""Tests for codex.parsing.tex.""" from __future__ import annotations import pytest from codex.parsing.tex import chunk_text, extract_sections, latex_to_text class TestExtractSections: def test_two_sections_returned(self) -> None: latex = r"\section{Intro}" + "\nHello \\cite{foo} world\n" latex += r"\section{Method}" + "\nResult" sections = extract_sections(latex) assert len(sections) == 2 def test_section_titles(self) -> None: latex = r"\section{Intro}" + "\nHello\n" + r"\section{Method}" + "\nResult" sections = extract_sections(latex) assert sections[0][0] == "Intro" assert sections[1][0] == "Method" def test_cite_removed_from_text(self) -> None: latex = r"\section{Intro}" + "\nHello \\cite{foo} world\n" latex += r"\section{Method}" + "\nResult" sections = extract_sections(latex) body = sections[0][1] assert r"\cite" not in body assert "foo" not in body def test_no_sections_returns_empty(self) -> None: assert extract_sections("No sections here at all.") == [] def test_subsection_recognised(self) -> None: latex = r"\subsection{Background}" + "\nSome text." sections = extract_sections(latex) assert len(sections) == 1 assert sections[0][0] == "Background" def test_comments_stripped(self) -> None: latex = r"\section{A}" + "\nHello % this is a comment\nWorld" sections = extract_sections(latex) assert "%" not in sections[0][1] assert "comment" not in sections[0][1] def test_math_stripped(self) -> None: latex = r"\section{A}" + "\nLet $x = 1$ and $$y = 2$$ be numbers." sections = extract_sections(latex) body = sections[0][1] assert "$" not in body class TestChunkText: @pytest.fixture() def long_text(self) -> str: # Build a 600-word string with real sentences sentence = "The quick brown fox jumps over the lazy dog near the river. " words_per_sentence = len(sentence.split()) repeats = (600 // words_per_sentence) + 1 return (sentence * repeats).strip() def test_at_least_two_chunks(self, long_text: str) -> None: chunks = chunk_text(long_text, size=512, overlap=64) assert len(chunks) >= 2 def test_each_chunk_within_size_bound(self, long_text: str) -> None: size, overlap = 512, 64 chunks = chunk_text(long_text, size=size, overlap=overlap) for chunk in chunks: word_count = len(chunk.split()) assert word_count <= size + overlap, ( f"Chunk has {word_count} words, expected <= {size + overlap}" ) def test_empty_text_returns_empty_list(self) -> None: assert chunk_text("") == [] def test_short_text_returns_single_chunk(self) -> None: chunks = chunk_text("Hello world.", size=512, overlap=64) assert len(chunks) == 1 def test_chunks_cover_all_content(self) -> None: # First chunk must start with first word, last chunk must end with last word text = " ".join(f"word{i}" for i in range(600)) chunks = chunk_text(text, size=200, overlap=50) assert chunks[0].startswith("word0") assert chunks[-1].endswith("word599") class TestLatexToText: def test_no_percent_in_output(self) -> None: latex = r"\section{A}" + "\nSome % comment\ntext." result = latex_to_text(latex) assert "%" not in result def test_no_cite_in_output(self) -> None: latex = r"\section{A}" + "\nHello \\cite{bar} world." result = latex_to_text(latex) assert r"\cite" not in result assert "bar" not in result def test_returns_string(self) -> None: latex = r"\section{A}" + "\nSimple text." result = latex_to_text(latex) assert isinstance(result, str) assert len(result) > 0 def test_no_section_fallback(self) -> None: latex = r"Some \cite{foo} text with % comment" result = latex_to_text(latex) assert r"\cite" not in result assert "%" not in result