feat(parsing): LaTeX, Nougat, GROBID parsers with chunking

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-04 23:30:58 +02:00
parent b52a6a7412
commit c6a428d335
8 changed files with 672 additions and 1 deletions

54
codex/parsing/nougat.py Normal file
View File

@@ -0,0 +1,54 @@
"""Nougat OCR integration.
Converts PDF files to Mathpix Markdown (mmd) by calling a self-hosted
Nougat HTTP server. No network fetching of papers happens here — the
caller is expected to pass a local PDF path.
"""
from __future__ import annotations
import httpx
import tenacity
from codex.config import Settings
def pdf_to_markdown(pdf_path: str, nougat_url: str | None = None) -> str:
"""Convert a local PDF to Mathpix Markdown via the Nougat HTTP API.
Parameters
----------
pdf_path:
Absolute (or relative) path to the PDF file on disk.
nougat_url:
Base URL of the Nougat server. Defaults to ``Settings().nougat_url``.
Returns
-------
str
The raw ``.mmd`` text returned by the server.
Raises
------
httpx.HTTPStatusError
If the server returns a non-2xx status code.
"""
if nougat_url is None:
nougat_url = Settings().nougat_url
@tenacity.retry(
retry=tenacity.retry_if_exception_type(httpx.ConnectError),
stop=tenacity.stop_after_attempt(3), # 1 original + 2 retries
wait=tenacity.wait_fixed(0),
reraise=True,
)
def _post() -> str:
with open(pdf_path, "rb") as fh, httpx.Client(timeout=120.0) as client:
response = client.post(
f"{nougat_url}/predict",
files={"file": (pdf_path, fh, "application/pdf")},
)
response.raise_for_status()
return response.text
return _post()