feat(parsing): LaTeX, Nougat, GROBID parsers with chunking
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
54
codex/parsing/nougat.py
Normal file
54
codex/parsing/nougat.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Nougat OCR integration.
|
||||
|
||||
Converts PDF files to Mathpix Markdown (mmd) by calling a self-hosted
|
||||
Nougat HTTP server. No network fetching of papers happens here — the
|
||||
caller is expected to pass a local PDF path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import tenacity
|
||||
|
||||
from codex.config import Settings
|
||||
|
||||
|
||||
def pdf_to_markdown(pdf_path: str, nougat_url: str | None = None) -> str:
|
||||
"""Convert a local PDF to Mathpix Markdown via the Nougat HTTP API.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pdf_path:
|
||||
Absolute (or relative) path to the PDF file on disk.
|
||||
nougat_url:
|
||||
Base URL of the Nougat server. Defaults to ``Settings().nougat_url``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The raw ``.mmd`` text returned by the server.
|
||||
|
||||
Raises
|
||||
------
|
||||
httpx.HTTPStatusError
|
||||
If the server returns a non-2xx status code.
|
||||
"""
|
||||
if nougat_url is None:
|
||||
nougat_url = Settings().nougat_url
|
||||
|
||||
@tenacity.retry(
|
||||
retry=tenacity.retry_if_exception_type(httpx.ConnectError),
|
||||
stop=tenacity.stop_after_attempt(3), # 1 original + 2 retries
|
||||
wait=tenacity.wait_fixed(0),
|
||||
reraise=True,
|
||||
)
|
||||
def _post() -> str:
|
||||
with open(pdf_path, "rb") as fh, httpx.Client(timeout=120.0) as client:
|
||||
response = client.post(
|
||||
f"{nougat_url}/predict",
|
||||
files={"file": (pdf_path, fh, "application/pdf")},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
return _post()
|
||||
Reference in New Issue
Block a user