feat(tex): section-aware multi-file .tex ingest (R-C)

Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12).

Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-18 04:15:20 +02:00
parent 1516684bbb
commit 1da81c7b28
8 changed files with 356 additions and 42 deletions

View File

@@ -113,7 +113,8 @@ def test_ingest_paper_basic() -> None:
def test_ingest_paper_source_tex(tmp_path: Any) -> None:
"""``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB."""
"""``.tex`` source → section-aware chunking; the stored ``section`` column is
labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12)."""
tex_file = tmp_path / "paper.tex"
tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.")
@@ -123,16 +124,19 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
mock_conn.executemany = MagicMock()
mock_conn.commit = MagicMock()
fake_chunks = ["chunk one text here.", "chunk two text here."]
# (section title, chunk content) pairs from the section-aware chunker.
section_pairs = [
("Introduction", "Body of the introduction goes here."),
("Proof of the Main Theorem", "Body of the proof goes here."),
]
with (
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."),
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
patch("codex.parsing.tex.chunk_sections", return_value=section_pairs),
patch("codex.ingest.is_quality_chunk", return_value=True),
):
result = ingest_paper(paper.id, source_path=str(tex_file))
@@ -147,7 +151,10 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
chunk_sql: str = chunk_insert_call[0][0]
assert "INSERT INTO chunks" in chunk_sql
assert result.chunks_upserted == len(fake_chunks)
# Section label comes from the heading: "Introduction" → intro, "Proof of…" → proof.
chunk_rows = chunk_insert_call[0][1]
assert [row[4] for row in chunk_rows] == ["intro", "proof"]
assert result.chunks_upserted == len(section_pairs)
# ---------------------------------------------------------------------------
@@ -183,7 +190,7 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
patch("codex.ingest.is_quality_chunk", return_value=True),
):
result = ingest_paper(paper.id, source_path=str(pdf_file))
@@ -237,7 +244,7 @@ def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
patch("codex.ingest.is_quality_chunk", return_value=True),
):
result = ingest_paper(paper.id, source_path=str(pdf_file))