user2595/codex-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'fix(ingest): support .txt sources + fix citing_id FK violation' (#7) from fix/ingest-txt-and-citing-id into main

Browse Source 
fix(ingest): support .txt sources + fix citing_id FK violation — Opus APPROVE
This commit is contained in:
user2595
2026-06-14 13:56:04 +00:00
parent 6c28446644 5025ff3a88
commit 1bec18ae82
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
codex/ingest.py
View File

@@ -139,6 +139,8 @@ def ingest_paper(
if suffix == ".tex": if suffix == ".tex":
text = latex_to_text(Path(source_path).read_text()) text = latex_to_text(Path(source_path).read_text())
elif suffix == ".txt":
text = Path(source_path).read_text(encoding="utf-8", errors="replace")
elif suffix == ".pdf": elif suffix == ".pdf":
text = pdf_to_markdown(source_path) text = pdf_to_markdown(source_path)
# Also extract GROBID refs # Also extract GROBID refs
@@ -176,7 +178,10 @@ def ingest_paper(
# --------------------------------------------------------------- # ---------------------------------------------------------------
api_citations: list[Citation] api_citations: list[Citation]
if paper.openalex_id: if paper.openalex_id:
api_citations = openalex.fetch_citations(paper.openalex_id) raw = openalex.fetch_citations(paper.openalex_id)
# OpenAlex returns citing_id=openalex_id, but papers.id uses the DOI.
# Rewrite citing_id to paper.id so the FK constraint is satisfied.
api_citations = [Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw]
else: else:
api_citations = semanticscholar.fetch_references(paper_id) api_citations = semanticscholar.fetch_references(paper_id)