feat(tex): section-aware multi-file .tex ingest (R-C)
Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12). Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,13 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from codex.parsing.tex import chunk_text, extract_sections, latex_to_text
|
||||
from codex.parsing.tex import (
|
||||
chunk_sections,
|
||||
chunk_text,
|
||||
extract_sections,
|
||||
flatten_inputs,
|
||||
latex_to_text,
|
||||
)
|
||||
|
||||
|
||||
class TestExtractSections:
|
||||
@@ -110,3 +116,50 @@ class TestLatexToText:
|
||||
result = latex_to_text(latex)
|
||||
assert r"\cite" not in result
|
||||
assert "%" not in result
|
||||
|
||||
|
||||
class TestFlattenInputs:
|
||||
def test_inlines_input_and_include(self) -> None:
|
||||
files = {
|
||||
"main.tex": r"Start \input{sec/intro} mid \include{proof} end",
|
||||
"sec/intro.tex": "INTRO BODY",
|
||||
"proof.tex": "PROOF BODY",
|
||||
}
|
||||
out = flatten_inputs(files["main.tex"], files)
|
||||
assert "INTRO BODY" in out
|
||||
assert "PROOF BODY" in out
|
||||
assert r"\input" not in out and r"\include" not in out
|
||||
|
||||
def test_resolves_without_tex_suffix(self) -> None:
|
||||
# \input{intro} must resolve the archive member "intro.tex".
|
||||
files = {"main.tex": r"\input{intro}", "intro.tex": "BODY"}
|
||||
assert "BODY" in flatten_inputs(files["main.tex"], files)
|
||||
|
||||
def test_recursive_includes(self) -> None:
|
||||
files = {"a.tex": r"X \input{b}", "b.tex": r"Y \input{c}", "c.tex": "Z"}
|
||||
out = flatten_inputs(files["a.tex"], files)
|
||||
assert "X" in out and "Y" in out and "Z" in out
|
||||
|
||||
def test_unresolved_input_dropped(self) -> None:
|
||||
# A reference with no matching file is removed, not left as a directive.
|
||||
out = flatten_inputs(r"A \input{missing} B", {})
|
||||
assert r"\input" not in out
|
||||
assert "A" in out and "B" in out
|
||||
|
||||
|
||||
class TestChunkSections:
|
||||
def test_pairs_carry_section_titles(self) -> None:
|
||||
latex = (
|
||||
r"\section{Introduction}" + "\nIntro body text here.\n"
|
||||
r"\section{Proof}" + "\nProof body text here.\n"
|
||||
)
|
||||
pairs = chunk_sections(latex)
|
||||
titles = [t for t, _ in pairs]
|
||||
assert "Introduction" in titles
|
||||
assert "Proof" in titles
|
||||
# No chunk spans a section boundary: each chunk belongs to one title.
|
||||
assert all(isinstance(c, str) and c for _, c in pairs)
|
||||
|
||||
def test_fallback_titleless_when_no_sections(self) -> None:
|
||||
pairs = chunk_sections("Plain text with no section commands at all here.")
|
||||
assert pairs and all(title is None for title, _ in pairs)
|
||||
|
||||
Reference in New Issue
Block a user