feat(tex): section-aware multi-file .tex ingest (R-C)
Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12). Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -113,7 +113,8 @@ def test_ingest_paper_basic() -> None:
|
||||
|
||||
|
||||
def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
"""``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB."""
|
||||
"""``.tex`` source → section-aware chunking; the stored ``section`` column is
|
||||
labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12)."""
|
||||
tex_file = tmp_path / "paper.tex"
|
||||
tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.")
|
||||
|
||||
@@ -123,16 +124,19 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
fake_chunks = ["chunk one text here.", "chunk two text here."]
|
||||
# (section title, chunk content) pairs from the section-aware chunker.
|
||||
section_pairs = [
|
||||
("Introduction", "Body of the introduction goes here."),
|
||||
("Proof of the Main Theorem", "Body of the proof goes here."),
|
||||
]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
patch("codex.parsing.tex.chunk_sections", return_value=section_pairs),
|
||||
patch("codex.ingest.is_quality_chunk", return_value=True),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(tex_file))
|
||||
|
||||
@@ -147,7 +151,10 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
chunk_sql: str = chunk_insert_call[0][0]
|
||||
assert "INSERT INTO chunks" in chunk_sql
|
||||
|
||||
assert result.chunks_upserted == len(fake_chunks)
|
||||
# Section label comes from the heading: "Introduction" → intro, "Proof of…" → proof.
|
||||
chunk_rows = chunk_insert_call[0][1]
|
||||
assert [row[4] for row in chunk_rows] == ["intro", "proof"]
|
||||
assert result.chunks_upserted == len(section_pairs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -183,7 +190,7 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
|
||||
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
||||
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
patch("codex.ingest.is_quality_chunk", return_value=True),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
||||
|
||||
@@ -237,7 +244,7 @@ def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
|
||||
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
|
||||
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
patch("codex.ingest.is_quality_chunk", return_value=True),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user