user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tex): section-aware multi-file .tex ingest (R-C)

Browse Source 
Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12).

Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-18 04:15:20 +02:00
parent 1516684bbb
commit 1da81c7b28
8 changed files with 356 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
codex/ingest.py
View File

@@ -12,7 +12,7 @@ from codex.config import get_settings
from codex.db import get_conn
from codex.embed import get_embedder
from codex.models import Citation, Paper
from codex.quality import classify_section, filter_chunks
from codex.quality import classify_section, is_quality_chunk
from codex.sources import arxiv, crossref, openalex, semanticscholar
logger = logging.getLogger(__name__)
@@ -262,17 +262,21 @@ def ingest_paper(
if source_path is not None:
from codex.parsing.grobid import extract_references
from codex.parsing.nougat import pdf_to_markdown
from codex.parsing.tex import chunk_text, latex_to_text
from codex.parsing.tex import chunk_sections, chunk_text
suffix = Path(source_path).suffix.lower()
text = ""
# (section_title | None, chunk_text) pairs. `.tex` is section-aware so
# the stored `section` column reflects the real heading (R-C / R-12);
# `.txt`/`.pdf` carry no section structure, so the title is None.
raw_pairs: list[tuple[str | None, str]] = []
if suffix == ".tex":
text = latex_to_text(Path(source_path).read_text())
raw_pairs = chunk_sections(Path(source_path).read_text())
elif suffix == ".txt":
text = Path(source_path).read_text(encoding="utf-8", errors="replace")
raw_pairs = [(None, c) for c in chunk_text(text)]
elif suffix == ".pdf":
text = pdf_to_markdown(source_path)
raw_pairs = [(None, c) for c in chunk_text(pdf_to_markdown(source_path))]
# Also extract GROBID refs
grobid_refs = extract_references(source_path)
for ref in grobid_refs:
@@ -287,24 +291,33 @@ def ingest_paper(
else:
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
raw_chunks = chunk_text(text) if text else []
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
# Quality-gate the pairs (keeps each chunk aligned with its section title).
settings = get_settings()
kept_pairs = [
(title, content)
for title, content in raw_pairs
if is_quality_chunk(content, settings=settings)
]
# Delete existing chunks for this paper
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
if chunks_text:
if kept_pairs:
# Embed all chunks in one batch
chunk_embeddings = embedder.encode_dense(chunks_text)
chunk_contents = [content for _, content in kept_pairs]
chunk_embeddings = embedder.encode_dense(chunk_contents)
chunk_rows = [
(
paper.id,
ord_idx,
content,
chunk_embeddings[ord_idx].tolist(),
classify_section(content),
# .tex: classify from the section heading (prepended to the
# snippet) so the label reflects the real section; otherwise
# fall back to content-only classification.
classify_section(f"{title}. {content}" if title else content),
)
for ord_idx, content in enumerate(chunks_text)
for ord_idx, (title, content) in enumerate(kept_pairs)
]
with conn.cursor() as cur:
cur.executemany(

60
codex/parsing/tex.py
View File

@@ -17,6 +17,10 @@ _SECTION_RE = re.compile(
re.DOTALL,
)
# \input{file} / \include{file} — arXiv papers routinely split the body across
# files, so these must be inlined before parsing (R-C multi-file flattening).
_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}")
# Patterns for LaTeX noise removal (applied in order).
_COMMENT_RE = re.compile(r"%[^\n]*")
_CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}")
@@ -157,3 +161,59 @@ def latex_to_text(latex: str) -> str:
return "\n\n".join(body for _, body in sections)
# Fallback: no section structure — clean the whole string.
return _clean_latex(latex)
def _norm_texkey(name: str) -> str:
"""Normalize a tex file name / ``\\input`` argument to a comparable key.
Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing
``.tex`` so ``\\input{sections/intro}`` matches the archive member
``sections/intro.tex``.
"""
n = name.strip().lstrip("./")
return n[:-4] if n.endswith(".tex") else n
def flatten_inputs(primary: str, files: dict[str, str]) -> str:
"""Recursively inline ``\\input``/``\\include`` directives (R-C).
*files* maps each source file's archive name to its contents (keys may carry
a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv
multi-file projects leave the ``\\documentclass`` file as little more than a
skeleton of ``\\input`` lines, so the body must be assembled before parsing.
Unresolved references are dropped; recursion is capped at depth 10 to guard
against include cycles.
"""
norm = {_norm_texkey(k): v for k, v in files.items()}
def _expand(text: str, depth: int) -> str:
if depth > 10:
return text
def _repl(match: re.Match[str]) -> str:
content = norm.get(_norm_texkey(match.group(1)))
return _expand(content, depth + 1) if content is not None else ""
return _INPUT_RE.sub(_repl, text)
return _expand(primary, 0)
def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]:
"""Chunk LaTeX *within* each section, pairing every chunk with its title.
Unlike :func:`latex_to_text` (which discards titles and flattens the whole
document into one block before word-window chunking), this keeps chunks from
spanning section boundaries and remembers which ``\\section`` each came from.
The caller can then label the stored ``section`` column from the real heading
instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less
chunks of the whole cleaned document when there are no section commands.
"""
sections = extract_sections(latex)
if not sections:
return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)]
pairs: list[tuple[str | None, str]] = []
for title, body in sections:
for chunk in chunk_text(body, size, overlap):
pairs.append((title or None, chunk))
return pairs

48
codex/sources/arxiv.py
View File

@@ -101,8 +101,9 @@ def fetch_source(arxiv_id: str) -> str | None:
Downloads the .tar.gz source bundle from ``https://arxiv.org/src/{arxiv_id}``,
locates the primary .tex file (preferring any file containing ``\\documentclass``,
falling back to the largest .tex by size), and returns its contents as a UTF-8
string.
falling back to the largest .tex by size), and inlines its ``\\input``/
``\\include`` directives from the other archive members so multi-file projects
return the full document body, not just the primary file's include skeleton.
Parameters
----------
@@ -126,33 +127,36 @@ def fetch_source(arxiv_id: str) -> str | None:
if response.status_code != 200:
response.raise_for_status()
from codex.parsing.tex import flatten_inputs
raw = response.content
try:
with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf:
tex_members = [m for m in tf.getmembers() if m.name.endswith(".tex")]
if not tex_members:
logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
return None
# Prefer the file containing \documentclass (primary document)
primary: tarfile.TarInfo | None = None
for member in tex_members:
files: dict[str, str] = {}
primary_name: str | None = None
for member in tf.getmembers():
if not member.name.endswith(".tex"):
continue
f = tf.extractfile(member)
if f is None:
continue
content_bytes = f.read()
if b"\\documentclass" in content_bytes:
primary = member
# Decode and return immediately — first match wins
return content_bytes.decode("utf-8", errors="replace")
content = f.read().decode("utf-8", errors="replace")
files[member.name] = content
# Prefer the \documentclass file as the primary document.
if primary_name is None and "\\documentclass" in content:
primary_name = member.name
if primary is None:
# Fallback: largest .tex by size
largest = max(tex_members, key=lambda m: m.size)
f = tf.extractfile(largest)
if f is None:
return None
return f.read().decode("utf-8", errors="replace")
if not files:
logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
return None
# No \documentclass anywhere → fall back to the largest .tex.
if primary_name is None:
primary_name = max(files, key=lambda k: len(files[k]))
# Inline \input/\include so multi-file projects yield the full body,
# not just the primary file's skeleton of include directives (R-C).
return flatten_inputs(files[primary_name], files)
except tarfile.TarError as exc:
logger.warning("Failed to open tar archive for %s: %s", arxiv_id, exc)
return None