1516684bbb
Enable roadmap R-A (GROBID reference extraction) end-to-end and run it against local Jetson infra instead of a Mac/Rosetta emulation. Backfill: - Add scripts/ra_grobid_backfill.py: references-only, idempotent citation backfill (dry-run default). Deliberately not ingest_paper(source_path=pdf), which re-OCRs and replaces the DQ-3-clean .txt chunks; this touches only the citations table (ON CONFLICT DO NOTHING). 7 tests. - Make extract_references timeout configurable (large theses exceed the 60s default, especially against a slower GROBID). Jetson infra: - Bump grobid/grobid 0.8.2 -> 0.9.0-crf. The -crf tag is the only GROBID variant published as an arm64 multi-arch manifest; every 0.8.x tag and the full deep-learning image are amd64-only, which is why GROBID never ran on the aarch64 Jetson. Enable the 4g memory cap + init/ulimits per GROBID docs. - Add docs/infra/grobid-jetson.md runbook: arm64 image rationale, the two host prerequisites (docker-group membership + the Compose v2 CLI plugin, both missing on the Jetson), service-scoped deploy, and the GET /api/isalive check. Verified live 2026-06-17: native arm64 pull, isalive=true, end-to-end extract_references on lutz-2024-thesis.pdf = 116 refs (101 with DOI/arXiv). docs(audit): mark R-A core done (citing coverage 27/29 -> 29/29; edges 920 -> 1022). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
77 lines
3.0 KiB
Python
77 lines
3.0 KiB
Python
"""Tests for scripts.ra_grobid_backfill — the R-A GROBID references backfill.
|
|
|
|
Only the pure, offline logic is covered (id normalization + Citation assembly);
|
|
``extract_references`` is mocked so no GROBID server or DB is required.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
from codex.models import Citation
|
|
from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _normalize_cited_id
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_normalize_doi_url_to_bare_lowercase() -> None:
|
|
assert _normalize_cited_id(doi="https://doi.org/10.1007/S00454-019-00132-8") == (
|
|
"10.1007/s00454-019-00132-8"
|
|
)
|
|
|
|
|
|
def test_normalize_doi_prefixes() -> None:
|
|
assert _normalize_cited_id(doi="http://doi.org/10.1/X") == "10.1/x"
|
|
assert _normalize_cited_id(doi="doi:10.1/X") == "10.1/x"
|
|
assert _normalize_cited_id(doi="10.1/x") == "10.1/x"
|
|
|
|
|
|
def test_normalize_arxiv_prefix_stripped() -> None:
|
|
assert _normalize_cited_id(arxiv_id="arXiv:2305.10988") == "2305.10988"
|
|
assert _normalize_cited_id(arxiv_id="2305.10988") == "2305.10988"
|
|
# Legacy ids keep their slash.
|
|
assert _normalize_cited_id(arxiv_id="math/0603097") == "math/0603097"
|
|
|
|
|
|
def test_normalize_doi_takes_precedence_over_arxiv() -> None:
|
|
assert _normalize_cited_id(doi="10.5/y", arxiv_id="2305.10988") == "10.5/y"
|
|
|
|
|
|
def test_normalize_empty_returns_none() -> None:
|
|
assert _normalize_cited_id() is None
|
|
assert _normalize_cited_id(doi=" ", arxiv_id="") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# build_grobid_citations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
|
|
return {"title": "t", "authors": "a", "year": "2020", "doi": doi, "arxiv_id": arxiv_id}
|
|
|
|
|
|
def test_build_citations_normalizes_dedups_and_drops_self() -> None:
|
|
paper_id = "10.1007/s00454-019-00132-8"
|
|
refs = [
|
|
_ref(doi="https://doi.org/10.1/A"), # → 10.1/a
|
|
_ref(doi="DOI:10.1/a"), # duplicate of the above after normalization
|
|
_ref(arxiv_id="arXiv:2305.10988"), # → 2305.10988
|
|
_ref(doi="", arxiv_id=""), # no id → skipped
|
|
_ref(doi="https://doi.org/10.1007/S00454-019-00132-8"), # self-cite → dropped
|
|
]
|
|
with patch("scripts.ra_grobid_backfill.extract_references", return_value=refs) as mock_ex:
|
|
cits = build_grobid_citations(paper_id, "some.pdf", grobid_url="http://grobid")
|
|
|
|
mock_ex.assert_called_once_with("some.pdf", grobid_url="http://grobid", timeout=60.0)
|
|
assert all(isinstance(c, Citation) for c in cits)
|
|
assert all(c.citing_id == paper_id for c in cits)
|
|
assert [c.cited_id for c in cits] == ["10.1/a", "2305.10988"]
|
|
|
|
|
|
def test_build_citations_empty_when_no_refs() -> None:
|
|
with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]):
|
|
assert build_grobid_citations("10.1/x", "x.pdf") == []
|