user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tex): section-aware multi-file .tex ingest (R-C)

Browse Source 
Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12).

Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-18 04:15:20 +02:00
parent 1516684bbb
commit 1da81c7b28
8 changed files with 356 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
codex/ingest.py
View File

@@ -12,7 +12,7 @@ from codex.config import get_settings
from codex.db import get_conn
from codex.embed import get_embedder
from codex.models import Citation, Paper
from codex.quality import classify_section, filter_chunks
from codex.quality import classify_section, is_quality_chunk
from codex.sources import arxiv, crossref, openalex, semanticscholar
logger = logging.getLogger(__name__)
@@ -262,17 +262,21 @@ def ingest_paper(
if source_path is not None:
from codex.parsing.grobid import extract_references
from codex.parsing.nougat import pdf_to_markdown
from codex.parsing.tex import chunk_text, latex_to_text
from codex.parsing.tex import chunk_sections, chunk_text
suffix = Path(source_path).suffix.lower()
text = ""
# (section_title | None, chunk_text) pairs. `.tex` is section-aware so
# the stored `section` column reflects the real heading (R-C / R-12);
# `.txt`/`.pdf` carry no section structure, so the title is None.
raw_pairs: list[tuple[str | None, str]] = []
if suffix == ".tex":
text = latex_to_text(Path(source_path).read_text())
raw_pairs = chunk_sections(Path(source_path).read_text())
elif suffix == ".txt":
text = Path(source_path).read_text(encoding="utf-8", errors="replace")
raw_pairs = [(None, c) for c in chunk_text(text)]
elif suffix == ".pdf":
text = pdf_to_markdown(source_path)
raw_pairs = [(None, c) for c in chunk_text(pdf_to_markdown(source_path))]
# Also extract GROBID refs
grobid_refs = extract_references(source_path)
for ref in grobid_refs:
@@ -287,24 +291,33 @@ def ingest_paper(
else:
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
raw_chunks = chunk_text(text) if text else []
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
# Quality-gate the pairs (keeps each chunk aligned with its section title).
settings = get_settings()
kept_pairs = [
(title, content)
for title, content in raw_pairs
if is_quality_chunk(content, settings=settings)
]
# Delete existing chunks for this paper
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
if chunks_text:
if kept_pairs:
# Embed all chunks in one batch
chunk_embeddings = embedder.encode_dense(chunks_text)
chunk_contents = [content for _, content in kept_pairs]
chunk_embeddings = embedder.encode_dense(chunk_contents)
chunk_rows = [
(
paper.id,
ord_idx,
content,
chunk_embeddings[ord_idx].tolist(),
classify_section(content),
# .tex: classify from the section heading (prepended to the
# snippet) so the label reflects the real section; otherwise
# fall back to content-only classification.
classify_section(f"{title}. {content}" if title else content),
)
for ord_idx, content in enumerate(chunks_text)
for ord_idx, (title, content) in enumerate(kept_pairs)
]
with conn.cursor() as cur:
cur.executemany(