feat(parsing): LaTeX, Nougat, GROBID parsers with chunking

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

codex/parsing/tex.py

@@ -0,0 +1,159 @@
+"""LaTeX parsing utilities.
+
+Provides helpers to extract sections, chunk text, and convert LaTeX to plain
+readable prose by stripping markup that is not useful for NLP/embedding.
+"""
+
+from __future__ import annotations
+
+import re
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+_SECTION_RE = re.compile(
+    r"\\(?:sub)*section\*?\s*\{([^}]*)\}",
+    re.DOTALL,
+)
+
+# Patterns for LaTeX noise removal (applied in order).
+_COMMENT_RE = re.compile(r"%[^\n]*")
+_CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}")
+_LABEL_RE = re.compile(r"\\label\{[^}]*\}")
+_REF_RE = re.compile(r"\\ref\{[^}]*\}")
+# Display math: $$...$$  (before single $ to avoid greedy mismatch)
+_DISPLAY_DOLLAR_RE = re.compile(r"\$\$.*?\$\$", re.DOTALL)
+# Inline math: $...$
+_INLINE_MATH_RE = re.compile(r"\$[^$\n]*?\$", re.DOTALL)
+# \begin{equation}...\end{equation}
+_ENV_EQUATION_RE = re.compile(
+    r"\\begin\{equation\*?\}.*?\\end\{equation\*?\}",
+    re.DOTALL,
+)
+# \begin{align}...\end{align}  (covers align, align*, aligned, …)
+_ENV_ALIGN_RE = re.compile(
+    r"\\begin\{align[^}]*\}.*?\\end\{align[^}]*\}",
+    re.DOTALL,
+)
+# Collapse excess whitespace
+_WHITESPACE_RE = re.compile(r"[ \t]+")
+_BLANK_LINES_RE = re.compile(r"\n{3,}")
+
+
+def _clean_latex(text: str) -> str:
+    """Strip LaTeX markup and return readable prose."""
+    text = _COMMENT_RE.sub("", text)
+    text = _ENV_EQUATION_RE.sub("", text)
+    text = _ENV_ALIGN_RE.sub("", text)
+    text = _DISPLAY_DOLLAR_RE.sub("", text)
+    text = _INLINE_MATH_RE.sub("", text)
+    text = _CITE_RE.sub("", text)
+    text = _LABEL_RE.sub("", text)
+    text = _REF_RE.sub("", text)
+    text = _WHITESPACE_RE.sub(" ", text)
+    text = _BLANK_LINES_RE.sub("\n\n", text)
+    return text.strip()
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def extract_sections(latex: str) -> list[tuple[str, str]]:
+    """Split *latex* on \\section / \\subsection boundaries.
+
+    Returns a list of ``(title, cleaned_text)`` tuples, one per section.
+    Text between the document start and the first section command is
+    discarded (preamble / abstract handling is out of scope).
+    """
+    # Find all section positions
+    matches = list(_SECTION_RE.finditer(latex))
+    if not matches:
+        return []
+
+    sections: list[tuple[str, str]] = []
+    for idx, match in enumerate(matches):
+        title = match.group(1).strip()
+        body_start = match.end()
+        body_end = matches[idx + 1].start() if idx + 1 < len(matches) else len(latex)
+        body = latex[body_start:body_end]
+        cleaned = _clean_latex(body)
+        sections.append((title, cleaned))
+
+    return sections
+
+
+def chunk_text(text: str, size: int = 512, overlap: int = 64) -> list[str]:
+    """Split *text* into overlapping word-based chunks.
+
+    Parameters
+    ----------
+    text:
+        Plain text to chunk (not raw LaTeX).
+    size:
+        Target number of words per chunk.
+    overlap:
+        Number of words to carry over into the next chunk.
+
+    Each chunk is at most ``size + overlap`` words.  Boundaries are snapped
+    to the nearest ``". "`` (sentence end) within ±20 words of the nominal
+    boundary when possible.
+    """
+    words = text.split()
+    if not words:
+        return []
+
+    snap_window = 20
+    chunks: list[str] = []
+    start = 0
+
+    while start < len(words):
+        end = min(start + size, len(words))
+
+        # Try to snap *end* to a sentence boundary within ±snap_window words.
+        if end < len(words):
+            best = end
+            # Build a small search window
+            lo = max(start + 1, end - snap_window)
+            hi = min(len(words), end + snap_window + 1)
+            # Prefer the closest sentence-end ". " to *end*
+            for offset in range(0, snap_window + 1):
+                for candidate in (end - offset, end + offset):
+                    if (
+                        lo <= candidate < hi
+                        and candidate > start
+                        and words[candidate - 1].endswith(".")
+                    ):
+                        # Sentence boundary: word at (candidate-1) ends with ".".
+                        best = candidate
+                        break
+                if best != end:
+                    break
+            end = best
+
+        chunk_words = words[start:end]
+        chunks.append(" ".join(chunk_words))
+
+        if end >= len(words):
+            break
+
+        # Next chunk starts *overlap* words before *end*.
+        start = max(start + 1, end - overlap)
+
+    return chunks
+
+
+def latex_to_text(latex: str) -> str:
+    """Convert a LaTeX document to plain text.
+
+    Extracts all sections, cleans each one, and joins them with a blank line.
+    If no section commands are found, the whole document is cleaned and
+    returned as a single block.
+    """
+    sections = extract_sections(latex)
+    if sections:
+        return "\n\n".join(body for _, body in sections)
+    # Fallback: no section structure — clean the whole string.
+    return _clean_latex(latex)