diff --git a/codex/parsing/grobid.py b/codex/parsing/grobid.py index b2b5443..7644803 100644 --- a/codex/parsing/grobid.py +++ b/codex/parsing/grobid.py @@ -26,6 +26,7 @@ def _text(element: ET.Element | None) -> str: def extract_references( pdf_path: str, grobid_url: str | None = None, + timeout: float = 60.0, ) -> list[dict[str, str]]: """Extract a structured reference list from a PDF via GROBID. @@ -35,6 +36,9 @@ def extract_references( Path to the PDF file on disk. grobid_url: Base URL of the GROBID server. Defaults to ``Settings().grobid_url``. + timeout: + HTTP client timeout in seconds. Large PDFs (e.g. theses) can exceed the + 60s default, especially against a slower/emulated GROBID; raise it then. Returns ------- @@ -46,7 +50,7 @@ def extract_references( if grobid_url is None: grobid_url = Settings().grobid_url - with open(pdf_path, "rb") as fh, httpx.Client(timeout=60.0) as client: + with open(pdf_path, "rb") as fh, httpx.Client(timeout=timeout) as client: response = client.post( f"{grobid_url}/api/processReferences", files={"input": (pdf_path, fh, "application/pdf")}, diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index c942325..5719674 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -338,7 +338,33 @@ so the quality lever is *combining more free sources*, not buying paywall access Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item is self-contained so a cold session can pick it up. -### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort** +### R-A — Run GROBID reference extraction on PDFs · **CORE DONE 2026-06-17** (full sweep optional) + +**Resolution (what was done) — citing coverage 27/29 → 29/29 (100%); edges 920 → 1022:** +- **References-only backfill, not full re-ingest.** Added + [scripts/ra_grobid_backfill.py](../../scripts/ra_grobid_backfill.py): per paper, + run `grobid.extract_references(pdf)`, normalize cited-ids to the canonical bare + form (DOIs de-URL'd + lower-cased; `arXiv:` stripped), INSERT into `citations` + (`ON CONFLICT DO NOTHING`). Deliberately avoids `ingest_paper(source_path=pdf)`, + whose PDF branch re-runs Nougat + `DELETE FROM chunks` and would overwrite the + DQ-3-verified-clean `.txt` chunks (and needs Nougat). Idempotent; dry-run by + default; 7 tests; ruff/mypy clean. +- **GROBID provisioning:** the pinned `grobid/grobid:0.8.2` is amd64-only, but the + Jetson and the dev Mac are both arm64 (this is why GROBID never ran). Ran the + lighter CRF image (`grobid/grobid:0.8.2-crf`) on the Mac under Podman+Rosetta + (PDFs are local; citations written over the SSH DB tunnel). Made + `extract_references` timeout configurable (theses need >60s emulated). A separate + task is provisioning a native arm64 GROBID on the Jetson (`docs/infra/grobid-jetson.md`). +- **Both `depositonce` theses closed:** `…depositonce-20357` (lutz-2024) **+96** + edges (5 in-corpus); `…depositonce-5415` (sechelmann-2016) **+6** edges — its + 147 MB/173-page scan blew pdfalto's 120s limit, so the 9 bibliography pages were + extracted with PyMuPDF first (77 refs, only 6 with a DOI/arXiv). **No paper has + zero out-edges now.** +- **Optional remaining (secondary acceptance):** sweep the other 27 already-covered + papers for *extra* GROBID refs via `python scripts/ra_grobid_backfill.py --write` + (the 2 book PDFs need the same bib-page extraction as sechelmann). + +**Original plan (for context):** - **✓ Unblocked (DQ-5 RESOLVED 2026-06-17):** DOI re-ingest is now idempotent, so the GROBID re-ingest no longer aborts on `papers_openalex_id_key`. - **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), diff --git a/docs/infra/grobid-jetson.md b/docs/infra/grobid-jetson.md new file mode 100644 index 0000000..4e6a0c6 --- /dev/null +++ b/docs/infra/grobid-jetson.md @@ -0,0 +1,163 @@ +# GROBID on the Jetson (aarch64) — deploy runbook + +**Status:** ACTIVE +**Host:** `alfred@192.168.178.103` — NVIDIA Jetson Orin Nano (aarch64, 8 GB RAM, 8 GB swap, 6 cores) +**Service:** GROBID reference extraction for roadmap **R-A** (`scripts/ra_grobid_backfill.py`) +**Last updated:** 2026-06-17 +**Verified:** 2026-06-17 — `grobid/grobid:0.9.0-crf` pulled as native `arm64`, `/api/isalive`→`true`, +end-to-end `extract_references` on `lutz-2024-thesis.pdf` → 116 refs (101 with DOI/arXiv) in ~36s. + +--- + +## Why this image (`grobid/grobid:0.9.0-crf`) + +GROBID had never come up on the Jetson because `infra/docker-compose.yml` pinned +`grobid/grobid:0.8.2`, and **every `0.8.x` tag (and the full `0.9.0`) is published +amd64-only**. A Docker tag resolves to a multi-arch *manifest list*; on aarch64 +the `0.8.2` list has no `linux/arm64` entry, so there is simply nothing to pull or +run (hence `curl localhost:8070` → connection refused, nothing listening). + +The fix is to select a tag whose manifest list actually contains a `linux/arm64` +build: + +| Candidate | arm64? | Size | Notes | +|-------------------------------|:------:|--------:|----------------------------------------| +| `grobid/grobid:0.8.2` | ✗ | ~9.5 GB | full DL image, amd64-only (old pin) | +| `grobid/grobid:0.9.0` / `-full`| ✗ | ~10 GB | full DL image, amd64-only | +| **`grobid/grobid:0.9.0-crf`** | **✓** | ~500 MB | **CRF-only, official, multi-arch** ← | +| `lfoppiano/grobid:0.9.0-crf` | ✓ | ~500 MB | same build, community namespace (alt.) | + +We use **`grobid/grobid:0.9.0-crf`**: + +- **CRF-only** is sufficient for reference extraction (`/api/processReferences`) + and is ~500 MB vs ~10 GB for the deep-learning image. +- **Native arm64** — no qemu emulation (too slow / RAM-heavy on a Jetson). +- **Official namespace**, **pinned version** (not `latest-crf`, which is a moving + target). arm64 builds only exist from `0.9.0-crf` onward — there is no arm64 + `0.8.x`, so this is a forced (but compatible) bump from the old `0.8.2` pin. The + TEI that `codex/parsing/grobid.py` parses (`biblStruct`, `persName`, + `idno[@type='DOI']`, …) is unchanged across the bump. + +> Caveat from upstream: the arm64 image is documented as "tested only on macOS, +> not linux/arm64". The container is the *same* `linux/arm64` ELF either way, so +> bare-metal Jetson works; the end-to-end check below is what de-risks it. + +--- + +## One-time prerequisites + +### 1. Docker permission for `alfred` + +The SSH user `alfred` is in the `sudo` group but **not** the `docker` group, so +`docker …` fails with `permission denied … /var/run/docker.sock`. Grant access +once (requires the sudo password): + +```bash +ssh alfred@192.168.178.103 +sudo usermod -aG docker alfred # add to docker group (persistent) +``` + +Then **start a fresh login** so the new group takes effect (group membership is +only re-evaluated at login — an existing session/`ssh` won't see it; a *new* +`ssh` connection will): + +```bash +exit && ssh alfred@192.168.178.103 +docker ps # should now work without sudo +``` + +(Alternatively run all `docker` commands below under `sudo`, but group membership +is cleaner and lets `ra_grobid_backfill.py` / curl checks be driven over plain SSH.) + +### 2. Docker Compose v2 plugin (arm64) + +This host has Docker Engine but **not** the Compose v2 CLI plugin (on Linux they +are separate packages; only Docker Desktop bundles Compose). Without it +`docker compose …` fails with `docker: unknown command: docker compose`. Install +the arm64 plugin into the user-local plugin dir (no sudo needed): + +```bash +mkdir -p ~/.docker/cli-plugins +curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-aarch64 \ + -o ~/.docker/cli-plugins/docker-compose +chmod +x ~/.docker/cli-plugins/docker-compose +docker compose version # confirm it resolves +``` + +--- + +## Deploy + +GROBID is **service-scoped** on purpose: Postgres (`papers-db`) is already running +on this host, so we bring up only the `grobid` service and never `docker compose up` +the whole stack (that would try to start a second `papers-db`). + +```bash +# copy this repo's compose file to the Jetson (first time only) +scp infra/docker-compose.yml infra/schema.sql alfred@192.168.178.103:~/codex-infra/ + +ssh alfred@192.168.178.103 +cd ~/codex-infra +docker compose up -d grobid # pulls grobid/grobid:0.9.0-crf (arm64), starts papers-grobid +docker compose logs -f grobid # wait for "Started ... in N seconds" (model load ~20–40s) +``` + +The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by +Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves +headroom alongside Postgres. + +--- + +## Verify `GET /api/isalive` → `true` + +```bash +# on the Jetson +curl -s http://localhost:8070/api/isalive # => true + +# from the Mac (LAN) — same value the corpus' GROBID_URL points at +curl -s http://192.168.178.103:8070/api/isalive # => true +``` + +Anything other than `true` (timeout / connection refused / `false`) means the +container isn't up — check `docker compose logs grobid` and that nothing else +holds port 8070. + +--- + +## End-to-end reference-extraction check + running R-A + +With `GROBID_URL=http://192.168.178.103:8070` (already set in `.env.jetson-ingest`): + +```bash +# one PDF, references only — should print N>0 parsed references +PYTHONPATH=. python - <<'PY' +from codex.parsing.grobid import extract_references +refs = extract_references("/Users/tarikmoussa/Desktop/ConformalLabpp/papers/lutz-2024-thesis.pdf", + grobid_url="http://192.168.178.103:8070") +print(f"{len(refs)} references; first DOI/arXiv:", + next(((r['doi'], r['arxiv_id']) for r in refs if r['doi'] or r['arxiv_id']), None)) +PY +``` + +Then run the references-only backfill (DB tunnel up — see +`docs/audit/DATA-QUALITY-2026-06-15.md`): + +```bash +ssh -N -L 5433:localhost:5432 alfred@192.168.178.103 & # DB tunnel +cp .env.jetson-ingest .env +PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only # dry-run first +PYTHONPATH=. python scripts/ra_grobid_backfill.py --write # apply +``` + +--- + +## Troubleshooting + +- **Container killed mid-request / exit 137** — OOM. The CRF image needs ~3 GB for + references; raise the compose `memory` limit or check Postgres isn't starving the + box (`free -h`, `docker stats papers-grobid`). +- **`isalive` true but `processReferences` 500s** — usually a `pdfalto` failure on a + malformed PDF; confirm with a clean PDF (`lutz-2024-thesis.pdf`) and check + `docker compose logs grobid`. +- **`permission denied … docker.sock`** — the docker-group prerequisite above wasn't + applied, or the SSH session predates it (re-login). diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index f7c8912..a1e822b 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -28,20 +28,30 @@ services: restart: unless-stopped grobid: - # CRF-only image is sufficient for reference extraction and is lighter than - # the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid - # for the latest 0.8.x tag before deploying. - image: grobid/grobid:0.8.2 + # CRF-only image: sufficient for reference extraction (/api/processReferences) + # and ~500MB vs ~10GB for the full deep-learning variant. + # + # The `-crf` tag is also the ONLY GROBID variant published as a multi-arch + # manifest that includes linux/arm64 — the full image (grobid/grobid:0.9.0) + # and every 0.8.x tag are amd64-only. So this is what lets GROBID run + # *natively* on the aarch64 Jetson (no slow/RAM-heavy qemu emulation). arm64 + # builds start at 0.9.0-crf; there is no arm64 0.8.x. + # Deploy + docker-permission fix: docs/infra/grobid-jetson.md + image: grobid/grobid:0.9.0-crf container_name: papers-grobid ports: - "8070:8070" + init: true # reap zombie children (GROBID docs recommend --init) + ulimits: + core: 0 # disable core dumps (GROBID docs: --ulimit core=0) restart: unless-stopped - # GROBID is RAM-hungry; uncomment to cap usage on constrained hardware - # (e.g. Nvidia Jetson): - # deploy: - # resources: - # limits: - # memory: 4g + # GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB + # RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom. + # Honored by `docker compose up` under Compose v2 (no swarm needed). + deploy: + resources: + limits: + memory: 4g volumes: pgdata: diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..82057c3 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Operational one-off scripts (importable for tests).""" diff --git a/scripts/ra_grobid_backfill.py b/scripts/ra_grobid_backfill.py new file mode 100644 index 0000000..798230e --- /dev/null +++ b/scripts/ra_grobid_backfill.py @@ -0,0 +1,220 @@ +"""R-A — GROBID reference backfill (references-only, idempotent). + +Adds GROBID-parsed citations to the live corpus **without** re-chunking the body, +so the DQ-3-verified-clean ``.txt`` chunks are preserved. Complements the DQ-1 +Semantic-Scholar supplement with references the APIs lack (notably the two +TU-Berlin ``depositonce`` theses, which neither OpenAlex nor S2 indexes). + +Why references-only instead of ``ingest_paper(source_path=pdf)`` +---------------------------------------------------------------- +The PDF branch of :func:`codex.ingest.ingest_paper` runs Nougat OCR and then +``DELETE FROM chunks`` + re-INSERT — it would replace the clean ``.txt`` chunks +(DQ-3) with unmeasured PDF-OCR text and additionally requires the Nougat server. +This backfill touches only the ``citations`` table (``ON CONFLICT DO NOTHING``), +so it is safe, idempotent, and needs only a reachable GROBID server. + +Prerequisite +------------ +A reachable GROBID server. The corpus' ``GROBID_URL`` points at the Jetson +(``http://192.168.178.103:8070``). GROBID now runs natively on that aarch64 host +via the multi-arch CRF-only image (``grobid/grobid:0.9.0-crf``) — see +``docs/infra/grobid-jetson.md`` for the deploy + docker-permission fix and the +``GET /api/isalive`` check. (The old ``grobid/grobid:0.8.2`` pin was amd64-only, +which is why GROBID never came up; pass ``--grobid-url`` to override the default.) +Use the SSH DB tunnel for ``DATABASE_URL`` as documented in +``docs/audit/DATA-QUALITY-2026-06-15.md``. + +Usage +----- + # dry run (no writes) over the two zero-edge theses + PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only + + # write every paper's GROBID-parsed refs to the live DB + PYTHONPATH=. python scripts/ra_grobid_backfill.py --write + +Default is **dry-run**; pass ``--write`` to mutate the live DB. +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +import psycopg +import psycopg.rows + +from codex.config import get_settings +from codex.db import get_conn +from codex.models import Citation +from codex.parsing.grobid import extract_references + +logger = logging.getLogger(__name__) + +DEFAULT_PDF_DIR = "/Users/tarikmoussa/Desktop/ConformalLabpp/papers" + + +def _normalize_cited_id(doi: str = "", arxiv_id: str = "") -> str | None: + """Normalize a GROBID-extracted reference id to the corpus' canonical form. + + DOIs → bare, lower-cased (strip ``https://doi.org/`` / ``http://`` / ``doi:``), + matching ``papers.id`` and ``codex.sources.openalex._normalize_doi`` (DQ-5). + arXiv ids → strip a leading ``arXiv:`` prefix, leaving the bare id + (``2305.10988``, ``math/0603097``). DOI takes precedence when both exist. + Kept self-contained rather than reusing ``ingest._norm_cited_id`` because it + additionally strips the ``arXiv:`` prefix (which GROBID can emit but that + helper leaves untouched), and to avoid importing a private cross-module name. + """ + doi = (doi or "").strip() + if doi: + s = doi.lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if s.startswith(prefix): + return s[len(prefix) :] + return s + arx = (arxiv_id or "").strip() + if arx: + return arx[len("arxiv:") :].strip() if arx.lower().startswith("arxiv:") else arx + return None + + +def build_grobid_citations( + paper_id: str, + pdf_path: str | Path, + grobid_url: str | None = None, + timeout: float = 60.0, +) -> list[Citation]: + """Extract a paper's references via GROBID and return normalized Citations. + + ``citing_id`` is the canonical ``paper_id``; ``cited_id`` is normalized. + Self-citations and within-paper duplicates are dropped. Refs with neither a + DOI nor an arXiv id are skipped (they cannot join the graph). + """ + refs = extract_references(str(pdf_path), grobid_url=grobid_url, timeout=timeout) + seen: set[str] = set() + citations: list[Citation] = [] + for ref in refs: + cited = _normalize_cited_id(ref.get("doi", ""), ref.get("arxiv_id", "")) + if not cited or cited == paper_id or cited in seen: + continue + seen.add(cited) + citations.append(Citation(citing_id=paper_id, cited_id=cited)) + return citations + + +def _coverage(conn: psycopg.Connection[psycopg.rows.DictRow]) -> tuple[int, int, int]: + """Return (n_papers, n_papers_with_out_edges, n_citations) from the live DB.""" + row = conn.execute( + """ + SELECT (SELECT count(*) FROM papers) AS papers, + (SELECT count(DISTINCT citing_id) FROM citations) AS citing, + (SELECT count(*) FROM citations) AS edges + """ + ).fetchone() + assert row is not None + return int(row["papers"]), int(row["citing"]), int(row["edges"]) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pdf-dir", default=DEFAULT_PDF_DIR, help="directory of source PDFs") + parser.add_argument("--grobid-url", default=None, help="override Settings().grobid_url") + parser.add_argument("--only", default=None, help="restrict to a single paper id") + parser.add_argument( + "--zero-edge-only", + action="store_true", + help="only papers with 0 out-edges (the R-A primary targets)", + ) + parser.add_argument("--limit", type=int, default=None, help="cap number of papers") + parser.add_argument( + "--timeout", + type=float, + default=300.0, + help="GROBID HTTP timeout (s); theses can need >60s, esp. emulated", + ) + parser.add_argument( + "--write", action="store_true", help="WRITE to the live DB (default: dry-run)" + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + grobid_url = args.grobid_url or get_settings().grobid_url + pdf_dir = Path(args.pdf_dir) + pdfs = {p.stem: p for p in pdf_dir.glob("*.pdf")} + + with get_conn() as conn: + rows = conn.execute( + """ + SELECT p.id, p.source_path, + (SELECT count(*) FROM citations ci WHERE ci.citing_id = p.id) AS out_edges + FROM papers p + ORDER BY out_edges, p.id + """ + ).fetchall() + + targets: list[tuple[str, Path]] = [] + for r in rows: + if args.only and r["id"] != args.only: + continue + if args.zero_edge_only and r["out_edges"] != 0: + continue + stem = Path(r["source_path"]).stem if r["source_path"] else None + pdf = pdfs.get(stem) if stem else None + if pdf is None: + logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem) + continue + targets.append((r["id"], pdf)) + if args.limit is not None: + targets = targets[: args.limit] + + logger.info( + "GROBID=%s | papers to process=%d | mode=%s", + grobid_url, + len(targets), + "WRITE" if args.write else "DRY-RUN", + ) + + all_citations: list[Citation] = [] + for paper_id, pdf in targets: + try: + cits = build_grobid_citations( + paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout + ) + except Exception as exc: + logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc) + continue + logger.info("%-38s %3d refs (%s)", paper_id, len(cits), pdf.name) + all_citations.extend(cits) + + logger.info("total candidate citations: %d", len(all_citations)) + + if not args.write: + logger.info("DRY-RUN — no rows written. Re-run with --write to apply.") + return 0 + + before = _coverage(conn) + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO citations (citing_id, cited_id, context) + VALUES (%s, %s, %s) + ON CONFLICT (citing_id, cited_id) DO NOTHING + """, + [(c.citing_id, c.cited_id, c.context) for c in all_citations], + ) + conn.commit() + after = _coverage(conn) + logger.info( + "coverage papers=%d citing %d->%d edges %d->%d (+%d)", + after[0], + before[1], + after[1], + before[2], + after[2], + after[2] - before[2], + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scripts/test_ra_grobid_backfill.py b/tests/scripts/test_ra_grobid_backfill.py new file mode 100644 index 0000000..8e97b2f --- /dev/null +++ b/tests/scripts/test_ra_grobid_backfill.py @@ -0,0 +1,76 @@ +"""Tests for scripts.ra_grobid_backfill — the R-A GROBID references backfill. + +Only the pure, offline logic is covered (id normalization + Citation assembly); +``extract_references`` is mocked so no GROBID server or DB is required. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from codex.models import Citation +from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations + +# --------------------------------------------------------------------------- +# _normalize_cited_id +# --------------------------------------------------------------------------- + + +def test_normalize_doi_url_to_bare_lowercase() -> None: + assert _normalize_cited_id(doi="https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_prefixes() -> None: + assert _normalize_cited_id(doi="http://doi.org/10.1/X") == "10.1/x" + assert _normalize_cited_id(doi="doi:10.1/X") == "10.1/x" + assert _normalize_cited_id(doi="10.1/x") == "10.1/x" + + +def test_normalize_arxiv_prefix_stripped() -> None: + assert _normalize_cited_id(arxiv_id="arXiv:2305.10988") == "2305.10988" + assert _normalize_cited_id(arxiv_id="2305.10988") == "2305.10988" + # Legacy ids keep their slash. + assert _normalize_cited_id(arxiv_id="math/0603097") == "math/0603097" + + +def test_normalize_doi_takes_precedence_over_arxiv() -> None: + assert _normalize_cited_id(doi="10.5/y", arxiv_id="2305.10988") == "10.5/y" + + +def test_normalize_empty_returns_none() -> None: + assert _normalize_cited_id() is None + assert _normalize_cited_id(doi=" ", arxiv_id="") is None + + +# --------------------------------------------------------------------------- +# build_grobid_citations +# --------------------------------------------------------------------------- + + +def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]: + return {"title": "t", "authors": "a", "year": "2020", "doi": doi, "arxiv_id": arxiv_id} + + +def test_build_citations_normalizes_dedups_and_drops_self() -> None: + paper_id = "10.1007/s00454-019-00132-8" + refs = [ + _ref(doi="https://doi.org/10.1/A"), # → 10.1/a + _ref(doi="DOI:10.1/a"), # duplicate of the above after normalization + _ref(arxiv_id="arXiv:2305.10988"), # → 2305.10988 + _ref(doi="", arxiv_id=""), # no id → skipped + _ref(doi="https://doi.org/10.1007/S00454-019-00132-8"), # self-cite → dropped + ] + with patch("scripts.ra_grobid_backfill.extract_references", return_value=refs) as mock_ex: + cits = build_grobid_citations(paper_id, "some.pdf", grobid_url="http://grobid") + + mock_ex.assert_called_once_with("some.pdf", grobid_url="http://grobid", timeout=60.0) + assert all(isinstance(c, Citation) for c in cits) + assert all(c.citing_id == paper_id for c in cits) + assert [c.cited_id for c in cits] == ["10.1/a", "2305.10988"] + + +def test_build_citations_empty_when_no_refs() -> None: + with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]): + assert build_grobid_citations("10.1/x", "x.pdf") == []