feat(tex): section-aware multi-file .tex ingest (R-C)
Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12). Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,10 @@ _SECTION_RE = re.compile(
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# \input{file} / \include{file} — arXiv papers routinely split the body across
|
||||
# files, so these must be inlined before parsing (R-C multi-file flattening).
|
||||
_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}")
|
||||
|
||||
# Patterns for LaTeX noise removal (applied in order).
|
||||
_COMMENT_RE = re.compile(r"%[^\n]*")
|
||||
_CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}")
|
||||
@@ -157,3 +161,59 @@ def latex_to_text(latex: str) -> str:
|
||||
return "\n\n".join(body for _, body in sections)
|
||||
# Fallback: no section structure — clean the whole string.
|
||||
return _clean_latex(latex)
|
||||
|
||||
|
||||
def _norm_texkey(name: str) -> str:
|
||||
"""Normalize a tex file name / ``\\input`` argument to a comparable key.
|
||||
|
||||
Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing
|
||||
``.tex`` so ``\\input{sections/intro}`` matches the archive member
|
||||
``sections/intro.tex``.
|
||||
"""
|
||||
n = name.strip().lstrip("./")
|
||||
return n[:-4] if n.endswith(".tex") else n
|
||||
|
||||
|
||||
def flatten_inputs(primary: str, files: dict[str, str]) -> str:
|
||||
"""Recursively inline ``\\input``/``\\include`` directives (R-C).
|
||||
|
||||
*files* maps each source file's archive name to its contents (keys may carry
|
||||
a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv
|
||||
multi-file projects leave the ``\\documentclass`` file as little more than a
|
||||
skeleton of ``\\input`` lines, so the body must be assembled before parsing.
|
||||
Unresolved references are dropped; recursion is capped at depth 10 to guard
|
||||
against include cycles.
|
||||
"""
|
||||
norm = {_norm_texkey(k): v for k, v in files.items()}
|
||||
|
||||
def _expand(text: str, depth: int) -> str:
|
||||
if depth > 10:
|
||||
return text
|
||||
|
||||
def _repl(match: re.Match[str]) -> str:
|
||||
content = norm.get(_norm_texkey(match.group(1)))
|
||||
return _expand(content, depth + 1) if content is not None else ""
|
||||
|
||||
return _INPUT_RE.sub(_repl, text)
|
||||
|
||||
return _expand(primary, 0)
|
||||
|
||||
|
||||
def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]:
|
||||
"""Chunk LaTeX *within* each section, pairing every chunk with its title.
|
||||
|
||||
Unlike :func:`latex_to_text` (which discards titles and flattens the whole
|
||||
document into one block before word-window chunking), this keeps chunks from
|
||||
spanning section boundaries and remembers which ``\\section`` each came from.
|
||||
The caller can then label the stored ``section`` column from the real heading
|
||||
instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less
|
||||
chunks of the whole cleaned document when there are no section commands.
|
||||
"""
|
||||
sections = extract_sections(latex)
|
||||
if not sections:
|
||||
return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)]
|
||||
pairs: list[tuple[str | None, str]] = []
|
||||
for title, body in sections:
|
||||
for chunk in chunk_text(body, size, overlap):
|
||||
pairs.append((title or None, chunk))
|
||||
return pairs
|
||||
|
||||
Reference in New Issue
Block a user