feat(tex): section-aware multi-file .tex ingest (R-C)

Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12).

Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-18 04:15:20 +02:00
parent 1516684bbb
commit 1da81c7b28
8 changed files with 356 additions and 42 deletions

View File

@@ -17,6 +17,10 @@ _SECTION_RE = re.compile(
re.DOTALL,
)
# \input{file} / \include{file} — arXiv papers routinely split the body across
# files, so these must be inlined before parsing (R-C multi-file flattening).
_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}")
# Patterns for LaTeX noise removal (applied in order).
_COMMENT_RE = re.compile(r"%[^\n]*")
_CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}")
@@ -157,3 +161,59 @@ def latex_to_text(latex: str) -> str:
return "\n\n".join(body for _, body in sections)
# Fallback: no section structure — clean the whole string.
return _clean_latex(latex)
def _norm_texkey(name: str) -> str:
"""Normalize a tex file name / ``\\input`` argument to a comparable key.
Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing
``.tex`` so ``\\input{sections/intro}`` matches the archive member
``sections/intro.tex``.
"""
n = name.strip().lstrip("./")
return n[:-4] if n.endswith(".tex") else n
def flatten_inputs(primary: str, files: dict[str, str]) -> str:
"""Recursively inline ``\\input``/``\\include`` directives (R-C).
*files* maps each source file's archive name to its contents (keys may carry
a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv
multi-file projects leave the ``\\documentclass`` file as little more than a
skeleton of ``\\input`` lines, so the body must be assembled before parsing.
Unresolved references are dropped; recursion is capped at depth 10 to guard
against include cycles.
"""
norm = {_norm_texkey(k): v for k, v in files.items()}
def _expand(text: str, depth: int) -> str:
if depth > 10:
return text
def _repl(match: re.Match[str]) -> str:
content = norm.get(_norm_texkey(match.group(1)))
return _expand(content, depth + 1) if content is not None else ""
return _INPUT_RE.sub(_repl, text)
return _expand(primary, 0)
def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]:
"""Chunk LaTeX *within* each section, pairing every chunk with its title.
Unlike :func:`latex_to_text` (which discards titles and flattens the whole
document into one block before word-window chunking), this keeps chunks from
spanning section boundaries and remembers which ``\\section`` each came from.
The caller can then label the stored ``section`` column from the real heading
instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less
chunks of the whole cleaned document when there are no section commands.
"""
sections = extract_sections(latex)
if not sections:
return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)]
pairs: list[tuple[str | None, str]] = []
for title, body in sections:
for chunk in chunk_text(body, size, overlap):
pairs.append((title or None, chunk))
return pairs