From 82cea2a33b7f55bc901cc1f807de042ca20d9ca3 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 21:32:46 +0200 Subject: [PATCH 1/2] feat(crossref): reference extraction as third citation fallback (R-B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add crossref.fetch_references(doi), which parses message.reference[].DOI into Citations (DOI-less book/unstructured refs are skipped — they cannot join the graph). Wire it as the third leg of the ingest citation fallback chain: OpenAlex-empty -> S2 -> Crossref, for DOI papers only (Crossref's works endpoint is DOI-keyed). Cited DOIs are normalized to the canonical bare lower-case form via _norm_cited_id. As a fallback it fires only when OpenAlex and S2 both return no references, so it adds a forward-looking third source without changing the already-covered corpus. Tests: fetch_references unit tests (DOI edges / unstructured skipped / no refs / 404) and ingest tests for both fallback directions. Full suite 361 passed; ruff + mypy clean. Live-validated: 33/40/24 refs for Springer/ACM/Wiley DOIs. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 29 +++++++++++++- codex/sources/crossref.py | 39 ++++++++++++++++++- docs/audit/DATA-QUALITY-2026-06-15.md | 30 ++++++++++++++- tests/ingest/test_ingest.py | 54 +++++++++++++++++++++++++++ tests/sources/test_crossref.py | 54 +++++++++++++++++++++++++++ 5 files changed, 202 insertions(+), 4 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index c0f768f..63d3401 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -130,6 +130,29 @@ def _s2_reference_supplement(paper: Paper) -> list[Citation]: ] +def _crossref_reference_supplement(paper: Paper) -> list[Citation]: + """Fetch a paper's references from Crossref (roadmap R-B supplement). + + Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref). + Crossref carries publisher-deposited reference lists keyed on the citing + work's DOI, so only DOI papers can be looked up (arXiv-only ids → []). + ``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are + case-normalised. Network/parse failures degrade to [] rather than aborting. + """ + if not paper.id.startswith("10."): + return [] + try: + raw = crossref.fetch_references(paper.id) + except Exception: + logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True) + return [] + return [ + Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context) + for c in raw + if c.cited_id + ] + + def ingest_paper( paper_id: str, source_path: str | None = None, @@ -315,7 +338,7 @@ def ingest_paper( chunks_upserted = len(chunk_rows) # --------------------------------------------------------------- - # 5. Fetch citations (OpenAlex if openalex_id, else S2) + # 5. Fetch citations (fallback chain: OpenAlex → S2 → Crossref) # Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING # --------------------------------------------------------------- api_citations: list[Citation] @@ -333,6 +356,10 @@ def ingest_paper( api_citations = _s2_reference_supplement(paper) else: api_citations = _s2_reference_supplement(paper) + # R-B: if OpenAlex and S2 both came back empty, try Crossref's + # publisher-deposited references (DOI papers only) as a third leg. + if not api_citations: + api_citations = _crossref_reference_supplement(paper) # Merge API citations and GROBID PDF citations; dedup via set all_citations_set: set[tuple[str, str]] = set() diff --git a/codex/sources/crossref.py b/codex/sources/crossref.py index 4727d85..f263872 100644 --- a/codex/sources/crossref.py +++ b/codex/sources/crossref.py @@ -2,10 +2,11 @@ Provides: - fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI. +- fetch_references: retrieve a work's reference list (DOI edges) by DOI. Crossref is a *third* metadata source after OpenAlex and Semantic Scholar -(DQ-2 / roadmap R-B). Requests use the Polite Pool (mailto query parameter) -and are retried on 429/5xx with exponential back-off via tenacity. +(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto +query parameter) and are retried on 429/5xx with exponential back-off via tenacity. """ from __future__ import annotations @@ -18,6 +19,7 @@ import httpx from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential from codex.config import get_settings +from codex.models import Citation logger = logging.getLogger(__name__) @@ -89,3 +91,36 @@ def fetch_abstract(doi: str) -> str | None: raise message: dict[str, Any] = response.json().get("message", {}) return _strip_jats(message.get("abstract")) + + +def fetch_references(doi: str) -> list[Citation]: + """Fetch a work's reference list from Crossref by DOI (roadmap R-B). + + Crossref carries publisher-deposited reference lists in ``message.reference``. + Each entry that cites a DOI-registered work exposes a bare ``DOI`` field; + only those become citation-graph edges (book / older references carry just + ``unstructured`` text with no resolvable id and are skipped). ``citing_id`` + is the queried *doi* — the caller rewrites it to the canonical ``papers.id`` + and normalises the cited DOIs (mirrors the Semantic Scholar reference path). + + Parameters + ---------- + doi: + A bare DOI (``"10.1007/s00454-019-00132-8"``). + + Returns + ------- + list[Citation] + One Citation per reference that carries a DOI. Empty on 404 or when no + references are deposited for the work. + """ + url = f"{_BASE}/works/{doi}" + try: + response = _get(url) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return [] + raise + message: dict[str, Any] = response.json().get("message", {}) + references: list[dict[str, Any]] = message.get("reference") or [] + return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")] diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 5719674..9608dd8 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -382,7 +382,35 @@ is self-contained so a cold session can pick it up. - **Acceptance:** the 2 theses gain references; published papers gain GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29. -### R-B — Add Crossref as a third citation + abstract source · **MED lever, MED effort** +### R-B — Add Crossref as a third citation + abstract source · **DONE 2026-06-17** + +**Resolution (what was done):** +- **`crossref.fetch_references(doi)`** added (mirrors `semanticscholar.fetch_references`): + GETs `/works/{doi}`, returns one `Citation` per `message.reference[]` entry that + carries a registered `DOI` (book/`unstructured`-only refs are skipped — they + can't join the graph). `citing_id` is the queried DOI; the ingest supplement + rewrites it to `paper.id`. +- **Wired as the third leg of the citation fallback chain** in `ingest.py`: + OpenAlex-empty → S2 → **Crossref** (`_crossref_reference_supplement`, DOI papers + only — Crossref's works endpoint is DOI-keyed). cited DOIs pass through + `_norm_cited_id` (bare, lower-cased). Network/parse failures degrade to []. +- **Tests:** `fetch_references` unit tests (DOI edges / `unstructured` skipped / no + `reference` key / 404) + ingest tests asserting the Crossref leg fires when + OpenAlex+S2 are empty (cited DOI normalised) **and** does *not* fire when OpenAlex + has citations. Suite **361 green**, ruff/mypy clean. The DQ-2 abstract half + (`fetch_abstract`) already shipped earlier. +- **Live-validated:** `fetch_references` against the real API returned 33 / 40 / 24 + DOI-bearing refs for `10.1007/s00454-019-00132-8` / `10.1145/3132705` / + `10.1111/cgf.13931` (Springer / ACM / Wiley). +- **No live corpus change (by design):** as a *fallback*, Crossref fires only when + OpenAlex **and** S2 are both empty. Post-DQ-1/R-A the only such papers were the 2 + `depositonce` theses, which Crossref doesn't index either — so R-B is a + forward-looking third leg for future DOI ingests, not a backfill. (Harvesting + Crossref's *unique* refs for already-covered papers would need an always-merged + supplement instead — a deliberate scope choice not taken, per the doc's "fallback + chain" spec.) **Files:** `codex/sources/crossref.py`, `codex/ingest.py`, + tests. + +**Original plan (for context):** - **What:** new `codex.sources.crossref` client hitting Crossref REST `/works/{doi}` for the `reference` array and `abstract`. - **Why:** Crossref carries **publisher-deposited references** for many DOIs (free, diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 936e6cb..82311f1 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -107,6 +107,60 @@ def test_ingest_paper_basic() -> None: ) +def test_ingest_citation_fallback_to_crossref() -> None: + """R-B: OpenAlex + S2 both empty → Crossref references supply the edges. + + Crossref is the third leg of the fallback chain. Its cited DOIs are + case-normalised and citing_id is rewritten to the canonical paper.id. + """ + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123") + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + # Upper-case suffix proves the cited DOI is normalised to bare lower-case. + crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty + patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(paper.id) + + # Crossref consulted with the bare DOI; one edge upserted. + mock_cr.assert_called_once_with(paper.id) + assert result.citations_upserted == 1 + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + cit_rows = mock_cursor.executemany.call_args_list[0][0][1] + assert cit_rows[0][0] == paper.id + assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7" + + +def test_ingest_crossref_not_called_when_openalex_has_citations() -> None: + """R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback).""" + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123") + mock_conn = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch( + "codex.ingest.openalex.fetch_citations", + return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")], + ), + patch("codex.ingest.crossref.fetch_references") as mock_cr, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + ingest_paper(paper.id) + + mock_cr.assert_not_called() + + # --------------------------------------------------------------------------- # 2. test_ingest_paper_source_tex # --------------------------------------------------------------------------- diff --git a/tests/sources/test_crossref.py b/tests/sources/test_crossref.py index f88d007..6e5c633 100644 --- a/tests/sources/test_crossref.py +++ b/tests/sources/test_crossref.py @@ -5,6 +5,7 @@ from __future__ import annotations import httpx import pytest +from codex.models import Citation from codex.sources import crossref @@ -48,3 +49,56 @@ def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> Non monkeypatch.setattr(crossref, "_get", mock_get) assert crossref.fetch_abstract("10.0000/nonexistent") is None + + +# --------------------------------------------------------------------------- +# fetch_references (roadmap R-B) +# --------------------------------------------------------------------------- + + +def test_fetch_references_returns_doi_edges(monkeypatch: pytest.MonkeyPatch) -> None: + """Only references carrying a DOI become Citations; citing_id is the queried DOI.""" + body = { + "message": { + "reference": [ + {"key": "ref1", "DOI": "10.1090/s0002-9947-04-03545-7"}, + {"key": "ref2", "unstructured": "A book with no DOI, 1992."}, # skipped + {"key": "ref3", "DOI": "10.1007/bf02392449", "article-title": "X"}, + ] + } + } + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json=body) + + monkeypatch.setattr(crossref, "_get", mock_get) + refs = crossref.fetch_references("10.1007/s00454-019-00132-8") + + assert refs == [ + Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1090/s0002-9947-04-03545-7"), + Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1007/bf02392449"), + ] + + +def test_fetch_references_no_reference_key_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """A work with no deposited reference list returns [].""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"message": {"title": ["No refs deposited"]}}) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_references("10.1007/978-3-642-17413-1_7") == [] + + +def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """A 404 returns [] rather than raising.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + raise httpx.HTTPStatusError( + "404", + request=httpx.Request("GET", url), + response=httpx.Response(404, request=httpx.Request("GET", url)), + ) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_references("10.0000/nonexistent") == [] From bf90c6cd57d2a91076dbeb8957506e691abf646e Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 21:36:04 +0200 Subject: [PATCH 2/2] feat(grobid): run GROBID on-demand (restart "no") to free ~3.5GB when idle GROBID is only needed during reference extraction (ra_grobid_backfill.py), not for normal corpus operations (search, embeddings, wiki). Idle it still holds ~3.5GB on the 8GB Jetson, so keep it stopped by default. - infra/docker-compose.yml: grobid restart unless-stopped -> "no" (no auto-start on boot; db keeps unless-stopped). Start/stop papers-grobid around an R-A run. - docs/infra/grobid-jetson.md: add "Run on-demand" section with the start/stop workflow, and record the refextract head-to-head evaluation on the Jetson that led to keeping GROBID: far worse DOI recall on this math corpus (lutz thesis 101 usable IDs vs refextract 2) and slower per PDF despite a smaller footprint. Co-Authored-By: Claude Opus 4.8 --- docs/infra/grobid-jetson.md | 31 ++++++++++++++++++++++++++++++- infra/docker-compose.yml | 6 +++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/infra/grobid-jetson.md b/docs/infra/grobid-jetson.md index 4e6a0c6..7a43e92 100644 --- a/docs/infra/grobid-jetson.md +++ b/docs/infra/grobid-jetson.md @@ -104,7 +104,36 @@ docker compose logs -f grobid # wait for "Started ... in N seconds" (mode The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves -headroom alongside Postgres. +headroom alongside Postgres. The `restart: "no"` policy means GROBID does **not** +auto-start on boot — see "Run on-demand" below. + +--- + +## Run on-demand (stop when idle) + +GROBID is only needed **during reference extraction** (`scripts/ra_grobid_backfill.py`); +nothing else in the stack uses it. Idle it still holds ~3.5 GB, so on the 8 GB Jetson +keep it stopped and start it only for an R-A run: + +```bash +ssh alfred@192.168.178.103 'docker start papers-grobid' # ~30s model load +curl -s --retry 30 --retry-delay 2 --retry-connrefused \ + http://192.168.178.103:8070/api/isalive # wait for: true + +# ... run R-A from the Mac ... +PYTHONPATH=. python scripts/ra_grobid_backfill.py --write + +ssh alfred@192.168.178.103 'docker stop papers-grobid' # reclaim ~3.5 GB +``` + +First-time deploy uses `docker compose up -d grobid` (above); afterwards the +container persists, so `docker start/stop papers-grobid` is all you need. + +> Why on-demand rather than a lighter tool: refextract was evaluated head-to-head +> on the Jetson and rejected — far worse DOI recall on this math corpus (lutz +> thesis: GROBID **101** usable IDs vs refextract **2**), and it was *slower* per +> PDF despite a smaller footprint. GROBID stays; running it on-demand removes its +> only real downside (idle RAM). --- diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index a1e822b..9304909 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -44,7 +44,11 @@ services: init: true # reap zombie children (GROBID docs recommend --init) ulimits: core: 0 # disable core dumps (GROBID docs: --ulimit core=0) - restart: unless-stopped + # On-demand only: GROBID is needed solely during reference extraction + # (scripts/ra_grobid_backfill.py), not for normal corpus operations — start it + # for an R-A run, stop it after to reclaim ~3.5GB. "no" = no auto-start on boot. + # See docs/infra/grobid-jetson.md ("Run on-demand"). + restart: "no" # GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB # RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom. # Honored by `docker compose up` under Compose v2 (no swarm needed).