From 5025ff3a882923e48d5409e5b2deccd26ad43382 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Sun, 14 Jun 2026 15:54:05 +0200 Subject: [PATCH] fix(ingest): support .txt sources + fix citing_id FK violation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs exposed during first batch ingest run after D-03 fix: 1. `.txt` files were silently skipped ("Unbekannter Dateityp") — added plain-text read path alongside .tex and .pdf. 2. After D-03, OpenAlex resolves arXiv IDs → paper.id uses the DOI (https://doi.org/…) but fetch_citations emits citing_id=openalex_id (https://openalex.org/W…). FK constraint citations_citing_id_fkey → papers.id violated. Fix: rewrite citing_id to paper.id after fetching from OpenAlex. Co-Authored-By: Claude Sonnet 4.6 --- codex/ingest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codex/ingest.py b/codex/ingest.py index a6e7ff6..72d2deb 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -139,6 +139,8 @@ def ingest_paper( if suffix == ".tex": text = latex_to_text(Path(source_path).read_text()) + elif suffix == ".txt": + text = Path(source_path).read_text(encoding="utf-8", errors="replace") elif suffix == ".pdf": text = pdf_to_markdown(source_path) # Also extract GROBID refs @@ -176,7 +178,10 @@ def ingest_paper( # --------------------------------------------------------------- api_citations: list[Citation] if paper.openalex_id: - api_citations = openalex.fetch_citations(paper.openalex_id) + raw = openalex.fetch_citations(paper.openalex_id) + # OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI. + # Rewrite citing_id to paper.id so the FK constraint is satisfied. + api_citations = [Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw] else: api_citations = semanticscholar.fetch_references(paper_id) -- 2.49.1