feat(tex): section-aware multi-file .tex ingest (R-C)

Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12).

Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/ingest/test_ingest.py

@@ -113,7 +113,8 @@ def test_ingest_paper_basic() -> None:
 
 
 def test_ingest_paper_source_tex(tmp_path: Any) -> None:
-    """``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB."""
+    """``.tex`` source → section-aware chunking; the stored ``section`` column is
+    labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12)."""
     tex_file = tmp_path / "paper.tex"
     tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.")
 
@@ -123,16 +124,19 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
     mock_conn.executemany = MagicMock()
     mock_conn.commit = MagicMock()
 
-    fake_chunks = ["chunk one text here.", "chunk two text here."]
+    # (section title, chunk content) pairs from the section-aware chunker.
+    section_pairs = [
+        ("Introduction", "Body of the introduction goes here."),
+        ("Proof of the Main Theorem", "Body of the proof goes here."),
+    ]
 
     with (
         patch("codex.ingest.openalex.fetch_paper", return_value=paper),
         patch("codex.ingest.openalex.fetch_citations", return_value=[]),
         patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
         patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
-        patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."),
-        patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
-        patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
+        patch("codex.parsing.tex.chunk_sections", return_value=section_pairs),
+        patch("codex.ingest.is_quality_chunk", return_value=True),
     ):
         result = ingest_paper(paper.id, source_path=str(tex_file))
 
@@ -147,7 +151,10 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
     chunk_sql: str = chunk_insert_call[0][0]
     assert "INSERT INTO chunks" in chunk_sql
 
-    assert result.chunks_upserted == len(fake_chunks)
+    # Section label comes from the heading: "Introduction" → intro, "Proof of…" → proof.
+    chunk_rows = chunk_insert_call[0][1]
+    assert [row[4] for row in chunk_rows] == ["intro", "proof"]
+    assert result.chunks_upserted == len(section_pairs)
 
 
 # ---------------------------------------------------------------------------
@@ -183,7 +190,7 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
         patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
         patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
         patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
-        patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
+        patch("codex.ingest.is_quality_chunk", return_value=True),
     ):
         result = ingest_paper(paper.id, source_path=str(pdf_file))
 
@@ -237,7 +244,7 @@ def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
         patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
         patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
         patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
-        patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
+        patch("codex.ingest.is_quality_chunk", return_value=True),
     ):
         result = ingest_paper(paper.id, source_path=str(pdf_file))
 

tests/parsing/test_tex.py

@@ -4,7 +4,13 @@ from __future__ import annotations
 
 import pytest
 
-from codex.parsing.tex import chunk_text, extract_sections, latex_to_text
+from codex.parsing.tex import (
+    chunk_sections,
+    chunk_text,
+    extract_sections,
+    flatten_inputs,
+    latex_to_text,
+)
 
 
 class TestExtractSections:
@@ -110,3 +116,50 @@ class TestLatexToText:
         result = latex_to_text(latex)
         assert r"\cite" not in result
         assert "%" not in result
+
+
+class TestFlattenInputs:
+    def test_inlines_input_and_include(self) -> None:
+        files = {
+            "main.tex": r"Start \input{sec/intro} mid \include{proof} end",
+            "sec/intro.tex": "INTRO BODY",
+            "proof.tex": "PROOF BODY",
+        }
+        out = flatten_inputs(files["main.tex"], files)
+        assert "INTRO BODY" in out
+        assert "PROOF BODY" in out
+        assert r"\input" not in out and r"\include" not in out
+
+    def test_resolves_without_tex_suffix(self) -> None:
+        # \input{intro} must resolve the archive member "intro.tex".
+        files = {"main.tex": r"\input{intro}", "intro.tex": "BODY"}
+        assert "BODY" in flatten_inputs(files["main.tex"], files)
+
+    def test_recursive_includes(self) -> None:
+        files = {"a.tex": r"X \input{b}", "b.tex": r"Y \input{c}", "c.tex": "Z"}
+        out = flatten_inputs(files["a.tex"], files)
+        assert "X" in out and "Y" in out and "Z" in out
+
+    def test_unresolved_input_dropped(self) -> None:
+        # A reference with no matching file is removed, not left as a directive.
+        out = flatten_inputs(r"A \input{missing} B", {})
+        assert r"\input" not in out
+        assert "A" in out and "B" in out
+
+
+class TestChunkSections:
+    def test_pairs_carry_section_titles(self) -> None:
+        latex = (
+            r"\section{Introduction}" + "\nIntro body text here.\n"
+            r"\section{Proof}" + "\nProof body text here.\n"
+        )
+        pairs = chunk_sections(latex)
+        titles = [t for t, _ in pairs]
+        assert "Introduction" in titles
+        assert "Proof" in titles
+        # No chunk spans a section boundary: each chunk belongs to one title.
+        assert all(isinstance(c, str) and c for _, c in pairs)
+
+    def test_fallback_titleless_when_no_sections(self) -> None:
+        pairs = chunk_sections("Plain text with no section commands at all here.")
+        assert pairs and all(title is None for title, _ in pairs)

tests/scripts/test_rc_tex_reingest.py

@@ -0,0 +1,19 @@
+"""Tests for scripts.rc_tex_reingest."""
+
+from __future__ import annotations
+
+from scripts.rc_tex_reingest import is_arxiv_id
+
+
+def test_is_arxiv_id_accepts_modern_and_legacy() -> None:
+    assert is_arxiv_id("2305.10988")
+    assert is_arxiv_id("math/0603097")
+    assert is_arxiv_id("1005.2698")
+
+
+def test_is_arxiv_id_rejects_dois_and_wids() -> None:
+    assert not is_arxiv_id("10.1007/s00454-019-00132-8")
+    assert not is_arxiv_id("10.14279/depositonce-20357")
+    assert not is_arxiv_id("W2971636899")
+    assert not is_arxiv_id("https://openalex.org/W2971636899")
+    assert not is_arxiv_id("")

tests/sources/test_arxiv.py

@@ -53,6 +53,39 @@ def test_fetch_source_returns_latex(monkeypatch: pytest.MonkeyPatch) -> None:
     assert "Hello world" in result
 
 
+def test_fetch_source_flattens_multifile(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Multi-file projects: \\input/\\include directives are inlined so the body
+    from the included files is returned, not just the primary skeleton (R-C)."""
+    main = (
+        b"\\documentclass{article}\\begin{document}"
+        b"\\input{sections/intro}\\include{proof}\\end{document}"
+    )
+    tar_bytes = _make_tar_gz(
+        {
+            "main.tex": main,
+            "sections/intro.tex": b"INTRODUCTION BODY TEXT",
+            "proof.tex": b"PROOF BODY TEXT",
+        }
+    )
+
+    def mock_get(
+        url: str,
+        *,
+        timeout: int = 60,
+        follow_redirects: bool = True,
+    ) -> httpx.Response:
+        return httpx.Response(200, content=tar_bytes)
+
+    monkeypatch.setattr(httpx, "get", mock_get)
+
+    result = arxiv.fetch_source("2301.07041")
+
+    assert result is not None
+    assert "INTRODUCTION BODY TEXT" in result  # \input was inlined
+    assert "PROOF BODY TEXT" in result  # \include was inlined
+    assert "\\input" not in result and "\\include" not in result
+
+
 # ---------------------------------------------------------------------------
 # fetch_source — 404 → None
 # ---------------------------------------------------------------------------