From 51dc74470cf77516b69f60ac706db849d1f980bc Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Tue, 16 Jun 2026 23:51:48 +0200 Subject: [PATCH] feat(ingest): supplement citations from Semantic Scholar when OpenAlex is empty (DQ-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 12/29 papers had zero citations: OpenAlex indexes arXiv preprints and theses without a parsed reference list (verified referenced_works_count=0 server-side for all 11 with an openalex_id). Not an ingest bug — a source-coverage limit. - ingest.py: when OpenAlex returns no references, fall back to a Semantic Scholar reference supplement (_s2_reference_supplement); citing_id rewritten to canonical papers.id, DOI cited-ids lowercased. Removed a now-redundant discarded S2 probe in the OpenAlex-404 arXiv branch. - semanticscholar.py: fix TypeError on S2's HTTP-200 {"data": null} no-refs responses (.get("data", []) returns None when the key is present-but-null). - tests: regression test for the OpenAlex-empty -> S2 path. - Live DB backfilled idempotently: +330 citations (590->920), zero-out-edge papers 12->2, citing coverage 17->27/29. Only the 2 TU-Berlin depositonce theses remain (no references in any source). - docs: DATA-QUALITY-2026-06-15.md — DQ-1 resolution, DQ-4 result, and a free-source acquisition roadmap (R-A..R-E). Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 63 +++++++++- codex/sources/semanticscholar.py | 6 +- docs/audit/DATA-QUALITY-2026-06-15.md | 167 ++++++++++++++++++++++++-- tests/ingest/test_ingest.py | 40 ++++++ 4 files changed, 260 insertions(+), 16 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index b63799d..5dab3b8 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -2,7 +2,6 @@ from __future__ import annotations -import contextlib import logging from dataclasses import dataclass from pathlib import Path @@ -47,6 +46,55 @@ def _make_bibkey(paper: Paper) -> str | None: return surnames[0] + "EtAl" + year +def _s2_id_for(pid: str) -> str | None: + """Format a canonical paper id for the Semantic Scholar paper endpoint. + + S2 needs a namespaced id (``arXiv:``, ``DOI:``) or a bare 40-hex + S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by + the ``10.`` prefix; everything else is assumed to be an arXiv id (modern + ``2305.10988`` or legacy ``math/0603097``). + """ + p = (pid or "").strip() + if not p: + return None + if p.lower().startswith(("arxiv:", "doi:")): + return p + if p.startswith("10."): + return f"DOI:{p}" + return f"arXiv:{p}" + + +def _norm_cited_id(cited_id: str) -> str: + """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join + does not fragment the same reference into distinct case-variant nodes. + arXiv ids and S2 paperIds are left untouched.""" + return cited_id.lower() if cited_id.startswith("10.") else cited_id + + +def _s2_reference_supplement(paper: Paper) -> list[Citation]: + """Fetch a paper's references from Semantic Scholar (DQ-1 supplement). + + Used when OpenAlex returns an empty ``referenced_works`` list — common for + arXiv preprints and institutional theses that OpenAlex indexes without a + parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id`` + so the FK holds, and cited DOIs are case-normalised. Network/parse failures + degrade to an empty list rather than aborting the ingest. + """ + s2_id = _s2_id_for(paper.id) + if s2_id is None: + return [] + try: + raw = semanticscholar.fetch_references(s2_id) + except Exception: + logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True) + return [] + return [ + Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context) + for c in raw + if c.cited_id + ] + + def ingest_paper( paper_id: str, source_path: str | None = None, @@ -88,9 +136,9 @@ def ingest_paper( len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit() ) if looks_like_arxiv: - arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}" - with contextlib.suppress(Exception): - semanticscholar.fetch_references(arxiv_key) + # OpenAlex 404 on an arXiv id → keep a minimal stub. Citation + # recovery from S2 happens in the citation block below (DQ-1 + # supplement), so no probe fetch is needed here. paper = Paper(id=paper_id, title="") else: raise ValueError(f"Paper not found: {paper_id}") @@ -229,8 +277,13 @@ def ingest_paper( api_citations = [ Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw ] + # DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a + # parsed reference list (referenced_works == []). Fall back to the + # Semantic Scholar references, which often has them. + if not api_citations: + api_citations = _s2_reference_supplement(paper) else: - api_citations = semanticscholar.fetch_references(paper_id) + api_citations = _s2_reference_supplement(paper) # Merge API citations and GROBID PDF citations; dedup via set all_citations_set: set[tuple[str, str]] = set() diff --git a/codex/sources/semanticscholar.py b/codex/sources/semanticscholar.py index e82d75a..0e92268 100644 --- a/codex/sources/semanticscholar.py +++ b/codex/sources/semanticscholar.py @@ -87,7 +87,11 @@ def fetch_references(paper_id: str) -> list[Citation]: raise data = response.json() - raw_refs: list[dict[str, Any]] = data.get("data", []) + # S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no + # parsed reference list (e.g. some theses). ``.get("data", [])`` would yield + # the explicit ``None`` (the default only applies when the key is absent), + # so iterate over a guaranteed list instead. + raw_refs: list[dict[str, Any]] = data.get("data") or [] citations: list[Citation] = [] for entry in raw_refs: cited_paper: dict[str, Any] = entry.get("citedPaper", {}) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index f016a79..b3f87e8 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -1,6 +1,9 @@ # DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session) -**Status:** baseline scan done 2026-06-15; four investigation items open (DQ-1…DQ-4). +**Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement +applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is +strong after fixing a P0 `codex search paper` crash. **DQ-2/DQ-3 still open.** +**Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section). **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory of the audit conversation — everything needed to continue is in this file. @@ -50,7 +53,35 @@ incomplete, or unrepresentative data. This document is that second axis — a ## Findings (open investigation items) -### DQ-1 — Citation coverage: 12 / 29 papers (41 %) have ZERO citations · **HIGH** +### DQ-1 — Citation coverage: 12 / 29 papers (41 %) had ZERO citations · **RESOLVED 2026-06-16** + +**Resolution (what was done):** +- **Verified upstream, not a bug.** Queried the OpenAlex API directly for all 11 + zero-citation papers that have an `openalex_id`: every one returns + `referenced_works_count = 0` server-side, W-ids all match what we stored. They + are `type: preprint` (arXiv) / `article` / `dissertation` — works OpenAlex + indexes without a parsed bibliography. Confirmed source-coverage limit. +- **Semantic Scholar supplement applied to the live DB** (idempotent backfill, + `ON CONFLICT (citing_id,cited_id) DO NOTHING`): **+330 citations (590 → 920)**. + Zero-out-edge papers **12 → 2**; in-corpus citing coverage **17/29 → 27/29**; + graph 489→704 nodes, 590→920 edges; discovery-lead targets 472→677. 9 of the + new edges are in-corpus (e.g. `2305.10988 → 10.1007/s00454-019-00132-8`). +- **Wired into ingest** so future re-ingests self-heal: when OpenAlex returns an + empty `referenced_works`, `ingest.py` now falls back to + `_s2_reference_supplement()` (citing_id rewritten to canonical `papers.id`, + DOI cited-ids lowercased). Same helper drove the one-off backfill. +- **Bug fixed (would have crashed the batch):** `semanticscholar.fetch_references` + used `data.get("data", [])`, but S2 returns HTTP 200 `{"data": null}` for a + known paper with no parsed refs (e.g. `depositonce-5415`) → `TypeError`. + Changed to `data.get("data") or []`. Removed a now-redundant discarded S2 probe + in the OpenAlex-404 arXiv branch. New regression test added. +- **Irreducible remainder (documented limit):** the **2 `depositonce` theses** + (`10.14279/depositonce-20357`, `-5415`) stay at zero — neither OpenAlex nor S2 + indexes references for TU-Berlin institutional DOIs. Accept as inherent. +- **Files touched (uncommitted in working tree):** `codex/sources/semanticscholar.py`, + `codex/ingest.py`, `tests/ingest/test_ingest.py`. Full suite: 331 passed. + +**Original finding (for context):** - **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph comes from only ~17 papers. The whole F-15 layer (PageRank / coupling / co-citation / discovery leads) therefore runs on ~59 % of the corpus. @@ -111,7 +142,40 @@ incomplete, or unrepresentative data. This document is that second axis — a - **Acceptance:** sampled papers retain ≳ the expected fraction of source text; no paper is silently gutted by the quality gate. -### DQ-4 — Retrieval quality: NOT YET MEASURED · **MED** +### DQ-4 — Retrieval quality · **MEASURED 2026-06-16 — good, after fixing a P0 crash** + +**P0 bug found + fixed: `codex search paper` was crash-broken on the live DB.** +The CLI passed the query embedding uncast, so pgvector raised +`operator does not exist: vector <-> double precision[]` on every paper search. +The working call sites (`mcp_server.py`, `wiki.py`) cast `%(emb)s::vector`; the +CLI did not. Fixed both `<->` occurrences in `codex/cli.py search_paper`; added a +regression guard in `tests/cli/test_cli.py` asserting the cast (mocked-DB unit +tests can't catch the type error — this is why it survived the code audit). + +**Relevance verdict (post-fix): strong.** Six domain queries, both surfaces +(paper-level abstract search + chunk-level). **rank-1 was the exactly-correct +paper in 6/6 queries** for chunk search and 5/6 for paper-level (Laplace–Beltrami +put the exact paper at rank-2 behind the closely-related "Polygon Laplacian Made +Simple" — acceptable). Examples: "combinatorial Yamabe flow" → `math/0306167` +(exact, all 5 top chunks that paper); "variational principle for Delaunay +triangulations" → `math/0603097` (exact); "circle packing rigidity" → +`2601.22903` (exact). Three of the DQ-1-rescued papers (`2305.10988`, +`1005.2698`, `depositonce-5415`) now surface as top hits. The KB is trustworthy +for lookups. + +**Secondary finding → reinforces DQ-2: zero-vector abstract pollution.** Papers +with no abstract get a zero `abstract_emb`, which sits at constant L2 distance +≈1.000 from every query and appears as filler in the paper-level tail whenever +< 5 strongly-relevant papers exist. `1911.00966` (fully degraded, DQ-2) showed up +at distance 1.000 in 4/6 queries. Harmless at rank>1 with a visible 1.000 score, +but noise — and a concrete reason to fix DQ-2. Chunk-level search is immune (it +filters `bibkey IS NOT NULL`, and 1911.00966 has no bibkey/chunks). + +**Note:** the MCP `search` docstring claims "hybrid dense + FTS" but the SQL is +dense-only; `score = 1.0 - dist` can go negative (bge-m3 L2 ranges 0–2). Ranking +is fine (monotonic); only the absolute score label is misleading. + +**Original open question (for context):** - **Open question:** end-to-end, do real queries return *relevant* results? Clean chunks + a working index don't guarantee useful retrieval. - **To investigate next:** run a handful of domain queries through the actual @@ -125,14 +189,97 @@ incomplete, or unrepresentative data. This document is that second axis — a --- -## Priority for the next session -1. **DQ-1** (biggest lever — 41 % of the corpus is invisible to the graph; the S2 - supplement is concrete and reuses existing code). -2. **DQ-4** (cheap, high-information — tells you if the KB is actually usable). -3. **DQ-2**, then **DQ-3**. +## Roadmap — data-acquisition levers (post-audit, all free sources) -All four are *data* work (queries + maybe a re-ingest/enrichment), not code-audit -work — the code is already remediated (see the AUDIT-* docs and PRs #12–#14). +These are *forward-looking acquisition improvements*, distinct from the DQ-1..DQ-4 +audit findings (which assessed existing data). They came out of a strategy +discussion on 2026-06-16: for this Math/CS corpus the valuable data (references, +abstracts, full text) is almost entirely **open** (arXiv, OpenAlex, Crossref, S2), +so the quality lever is *combining more free sources*, not buying paywall access. +Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item +is self-contained so a cold session can pick it up. + +### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort** +- **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), + so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper + so `codex.parsing.grobid.extract_references` parses each bibliography. +- **Why:** extracts references the APIs lack — a *complement* to DQ-1's S2 + supplement. Crucially, the **2 `depositonce` theses** that stayed at zero in + DQ-1 (no API references anywhere) almost certainly have a references section in + their open-access PDFs → GROBID could finally give them out-edges. Closes the + last 2/29. +- **How:** no new code — ingest already merges + dedups `api_citations + + pdf_citations` ([codex/ingest.py:288](../../codex/ingest.py)). GROBID server is + already configured (`GROBID_URL=http://192.168.178.103:8070` in + `.env.jetson-ingest`). Download free arXiv PDFs, re-ingest with `source_path`. +- **Acceptance:** the 2 theses gain references; published papers gain + GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29. + +### R-B — Add Crossref as a third citation + abstract source · **MED lever, MED effort** +- **What:** new `codex.sources.crossref` client hitting Crossref REST + `/works/{doi}` for the `reference` array and `abstract`. +- **Why:** Crossref carries **publisher-deposited references** for many DOIs (free, + polite pool via `mailto`) — a third leg after OpenAlex/S2. It also recovers + abstracts OpenAlex is contractually barred from redistributing (likely the 2 + Springer DOIs missing abstracts in DQ-2). cited_ids are DOIs → flow through the + existing graph resolver unchanged. +- **How:** mirror `codex/sources/openalex.py` (tenacity retry, polite pool). Extend + the ingest fallback chain to OpenAlex-empty → S2 → Crossref (same + `_s2_reference_supplement` shape). Add unit tests with mocked HTTP. +- **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where + OpenAlex+S2 are empty. + +### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **MED lever, MED-HIGH effort** +- **What:** re-ingest from arXiv LaTeX source through the existing + `codex.parsing.tex.latex_to_text` path instead of flattened `.txt`. +- **Why:** highest-fidelity input — preserves math, structure, and section + boundaries. Directly serves **DQ-3** (content fidelity) and audit **R-12** (the + `section` column is low-signal because word-window `.txt` chunks rarely start at + a real header; `.tex` headers fix this). +- **How:** download free arXiv source tarballs, feed `.tex` to ingest with + `source_path`. Handle multi-file projects (main `.tex` + includes). +- **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage + vs source improves; no regression in the F-16 quality gate. + +### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **SMALL** +- **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper + count. Add a warning when the share of papers with ≥1 out-edge is low (e.g. + < 80%), since that is what actually starves PageRank/coupling. +- **Why:** at 17/29 the graph silently ran on 59% of the corpus with no signal. + Post-DQ-1 it's 27/29 (93%) — but the guard should catch future regressions. +- **Acceptance:** `codex graph report` surfaces citing-coverage % and warns below + threshold. + +### R-E — Paywall / institutional access · **DEFERRED — only for a paywall-only expansion** +- **Decision (2026-06-16):** a uni login does **not** help the current corpus — + every DQ-1/DQ-2 gap is in openly-available works (arXiv preprints, OA theses), + and all metadata/citation sources are free APIs. Revisit *only* if the corpus + expands to papers that exist solely behind a paywall with no preprint; then + institutional access → full-text PDF → GROBID refs (R-A) + clean chunks (R-C). +- **ToS caveat:** manual download of individual papers you have legitimate access + to is fine; **automated bulk download through a uni proxy (EZproxy/Shibboleth) + violates most publisher terms and can get the whole institution's access + revoked.** Do not wire a publisher login into the ingest pipeline. + +--- + +## Priority for the next session +1. ~~**DQ-1**~~ — **DONE 2026-06-16** (live DB at 920 citations, 27/29 citing + coverage; remedy wired into ingest). +2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance + verified strong; surfaced the zero-vector pollution that motivates DQ-2). +3. **DQ-2** (NEXT — recover `1911.00966`; backfill 3 missing abstracts so they + stop polluting paper-level search with zero-vectors), then **DQ-3** (chunk-vs- + source `.txt` fidelity). +4. **Roadmap levers** (see section above), best sequenced after DQ-2/DQ-3: + **R-A** (GROBID on arXiv PDFs — closes the last 2 zero-citation theses, highest + ROI), then **R-B** (Crossref source — also recovers DQ-2 abstracts), **R-C** + (`.tex` ingest — serves DQ-3), **R-D** (F-15 coverage warning, quick). + **R-E** (paywall) is deferred. + +DQ-1..DQ-4 are *data*/assessment work; the roadmap items R-A..R-D add **new code** +(re-ingest scripts, a Crossref source module, a `.tex` path, a graph warning) on +top of the already-remediated pipeline (see the AUDIT-* docs and PRs #12–#14). --- diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 80e4d23..6aad09b 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -174,6 +174,9 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1); + # mock it empty so only the GROBID refs contribute to the count. + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), @@ -267,6 +270,43 @@ def test_ingest_paper_arxiv_s2_fallback() -> None: assert result.paper_id == paper_id +def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None: + """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2, + rewriting citing_id to paper.id and lowercasing DOI cited-ids.""" + paper = _make_paper() # openalex_id="W123", id="2301.07041" + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + # S2 keys citing_id by the id form passed in; ingest must canonicalise it. + s2_refs = [ + Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"), + Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"), + ] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(paper.id) + + # S2 consulted with a namespaced id (a bare arXiv id 404s on S2). + mock_s2.assert_called_once_with("arXiv:2301.07041") + assert result.citations_upserted == 2 + + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1] + # citing_id rewritten to paper.id, not the S2 id form. + assert all(row[0] == paper.id for row in inserted_rows) + cited = {row[1] for row in inserted_rows} + assert "10.1/upper" in cited # DOI lowercased + assert "2202.00002" in cited # arXiv id untouched + + # --------------------------------------------------------------------------- # 7. test_ingest_paper_no_abstract_uses_zero_vector # ---------------------------------------------------------------------------