user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(F-09): address review-gate findings — resource leak + idempotency

Browse Source 
- mathpix.py + figures.py: wrap fitz page-loop in try/finally so
  doc.close() is guaranteed even on exception (no file-handle leak)
- ingest.py: DELETE FROM formulas/figures WHERE paper_id before re-insert
  so --rich re-ingest is idempotent (BIGSERIAL has no natural UNIQUE key;
  ON CONFLICT DO NOTHING was a no-op)

Gate: 158 passed, ruff clean, mypy clean

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-14 02:23:21 +02:00
parent 1df9be6563
commit cd42e9abc8
3 changed files with 88 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
codex/ingest.py
View File

@@ -220,13 +220,18 @@ def ingest_paper(
if formulas or figures: if formulas or figures:
with get_conn() as conn2: with get_conn() as conn2:
with conn2.cursor() as cur:
# Delete-before-insert keeps re-ingest idempotent (BIGSERIAL has no
# natural UNIQUE key, so ON CONFLICT DO NOTHING never fires).
cur.execute("DELETE FROM formulas WHERE paper_id = %s", (paper.id,))
cur.execute("DELETE FROM figures WHERE paper_id = %s", (paper.id,))
if formulas: if formulas:
with conn2.cursor() as cur: with conn2.cursor() as cur:
cur.executemany( cur.executemany(
""" """
INSERT INTO formulas (paper_id, page, raw_latex, context, eq_label) INSERT INTO formulas (paper_id, page, raw_latex, context, eq_label)
VALUES (%s, %s, %s, %s, %s) VALUES (%s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING
""", """,
[ [
(f.paper_id, f.page, f.raw_latex, f.context, f.eq_label) (f.paper_id, f.page, f.raw_latex, f.context, f.eq_label)
@@ -241,7 +246,6 @@ def ingest_paper(
""" """
INSERT INTO figures (paper_id, page, caption, image_path) INSERT INTO figures (paper_id, page, caption, image_path)
VALUES (%s, %s, %s, %s) VALUES (%s, %s, %s, %s)
ON CONFLICT DO NOTHING
""", """,
[ [
(fig.paper_id, fig.page, fig.caption, fig.image_path) (fig.paper_id, fig.page, fig.caption, fig.image_path)

3
codex/parsing/figures.py
View File

@@ -126,6 +126,7 @@ def extract_figures(
logger.exception("fitz.open failed for %s", pdf_path) logger.exception("fitz.open failed for %s", pdf_path)
return results return results
try:
for page_num in range(len(doc)): for page_num in range(len(doc)):
page = doc[page_num] page = doc[page_num]
images: list[Any] = page.get_images(full=True) images: list[Any] = page.get_images(full=True)
@@ -162,5 +163,7 @@ def extract_figures(
caption=caption, caption=caption,
) )
) )
finally:
doc.close()
return results return results

8
codex/parsing/mathpix.py
View File

@@ -112,6 +112,7 @@ def _extract_formulas_pix2tex(pdf_path: str) -> list[FormulaChunk]:
scale = fitz.Matrix(3, 3) scale = fitz.Matrix(3, 3)
try:
for page_num in range(len(doc)): for page_num in range(len(doc)):
page = doc[page_num] page = doc[page_num]
blocks: list[Any] = page.get_text("blocks") blocks: list[Any] = page.get_text("blocks")
@@ -130,7 +131,10 @@ def _extract_formulas_pix2tex(pdf_path: str) -> list[FormulaChunk]:
continue continue
# Math content filter # Math content filter
if _math_symbol_count(text) < _MIN_MATH_SYMBOLS or _math_ratio(text) < _MIN_MATH_RATIO: if (
_math_symbol_count(text) < _MIN_MATH_SYMBOLS
or _math_ratio(text) < _MIN_MATH_RATIO
):
continue continue
# Crop at 3× for OCR quality # Crop at 3× for OCR quality
@@ -160,6 +164,8 @@ def _extract_formulas_pix2tex(pdf_path: str) -> list[FormulaChunk]:
context=context, context=context,
) )
) )
finally:
doc.close()
return results return results