feat(ingest): supplement citations from Semantic Scholar when OpenAlex is empty (DQ-1)

12/29 papers had zero citations: OpenAlex indexes arXiv preprints and theses without a parsed reference list (verified referenced_works_count=0 server-side for all 11 with an openalex_id). Not an ingest bug — a source-coverage limit. - ingest.py: when OpenAlex returns no references, fall back to a Semantic Scholar reference supplement (_s2_reference_supplement); citing_id rewritten to canonical papers.id, DOI cited-ids lowercased. Removed a now-redundant discarded S2 probe in the OpenAlex-404 arXiv branch. - semanticscholar.py: fix TypeError on S2's HTTP-200 {"data": null} no-refs responses (.get("data", []) returns None when the key is present-but-null). - tests: regression test for the OpenAlex-empty -> S2 path. - Live DB backfilled idempotently: +330 citations (590->920), zero-out-edge papers 12->2, citing coverage 17->27/29. Only the 2 TU-Berlin depositonce theses remain (no references in any source). - docs: DATA-QUALITY-2026-06-15.md — DQ-1 resolution, DQ-4 result, and a free-source acquisition roadmap (R-A..R-E). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-16 23:51:48 +02:00
parent 553ae3995d
commit 51dc74470c
4 changed files with 260 additions and 16 deletions
--- a/codex/ingest.py
+++ b/codex/ingest.py
@@ -2,7 +2,6 @@

 from __future__ import annotations

-import contextlib
 import logging
 from dataclasses import dataclass
 from pathlib import Path
@@ -47,6 +46,55 @@ def _make_bibkey(paper: Paper) -> str | None:
    return surnames[0] + "EtAl" + year


+def _s2_id_for(pid: str) -> str | None:
+    """Format a canonical paper id for the Semantic Scholar paper endpoint.
+
+    S2 needs a namespaced id (``arXiv:<id>``, ``DOI:<doi>``) or a bare 40-hex
+    S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by
+    the ``10.`` prefix; everything else is assumed to be an arXiv id (modern
+    ``2305.10988`` or legacy ``math/0603097``).
+    """
+    p = (pid or "").strip()
+    if not p:
+        return None
+    if p.lower().startswith(("arxiv:", "doi:")):
+        return p
+    if p.startswith("10."):
+        return f"DOI:{p}"
+    return f"arXiv:{p}"
+
+
+def _norm_cited_id(cited_id: str) -> str:
+    """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join
+    does not fragment the same reference into distinct case-variant nodes.
+    arXiv ids and S2 paperIds are left untouched."""
+    return cited_id.lower() if cited_id.startswith("10.") else cited_id
+
+
+def _s2_reference_supplement(paper: Paper) -> list[Citation]:
+    """Fetch a paper's references from Semantic Scholar (DQ-1 supplement).
+
+    Used when OpenAlex returns an empty ``referenced_works`` list — common for
+    arXiv preprints and institutional theses that OpenAlex indexes without a
+    parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id``
+    so the FK holds, and cited DOIs are case-normalised. Network/parse failures
+    degrade to an empty list rather than aborting the ingest.
+    """
+    s2_id = _s2_id_for(paper.id)
+    if s2_id is None:
+        return []
+    try:
+        raw = semanticscholar.fetch_references(s2_id)
+    except Exception:
+        logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True)
+        return []
+    return [
+        Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
+        for c in raw
+        if c.cited_id
+    ]
+
+
 def ingest_paper(
    paper_id: str,
    source_path: str | None = None,
@@ -88,9 +136,9 @@ def ingest_paper(
            len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit()
        )
        if looks_like_arxiv:
-            arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}"
-            with contextlib.suppress(Exception):
-                semanticscholar.fetch_references(arxiv_key)
+            # OpenAlex 404 on an arXiv id → keep a minimal stub. Citation
+            # recovery from S2 happens in the citation block below (DQ-1
+            # supplement), so no probe fetch is needed here.
            paper = Paper(id=paper_id, title="")
        else:
            raise ValueError(f"Paper not found: {paper_id}")
@@ -229,8 +277,13 @@ def ingest_paper(
            api_citations = [
                Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
            ]
+            # DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a
+            # parsed reference list (referenced_works == []). Fall back to the
+            # Semantic Scholar references, which often has them.
+            if not api_citations:
+                api_citations = _s2_reference_supplement(paper)
        else:
-            api_citations = semanticscholar.fetch_references(paper_id)
+            api_citations = _s2_reference_supplement(paper)

        # Merge API citations and GROBID PDF citations; dedup via set
        all_citations_set: set[tuple[str, str]] = set()
--- a/codex/sources/semanticscholar.py
+++ b/codex/sources/semanticscholar.py
@@ -87,7 +87,11 @@ def fetch_references(paper_id: str) -> list[Citation]:
        raise

    data = response.json()
-    raw_refs: list[dict[str, Any]] = data.get("data", [])
+    # S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
+    # parsed reference list (e.g. some theses). ``.get("data", [])`` would yield
+    # the explicit ``None`` (the default only applies when the key is absent),
+    # so iterate over a guaranteed list instead.
+    raw_refs: list[dict[str, Any]] = data.get("data") or []
    citations: list[Citation] = []
    for entry in raw_refs:
        cited_paper: dict[str, Any] = entry.get("citedPaper", {})
--- a/docs/audit/DATA-QUALITY-2026-06-15.md
+++ b/docs/audit/DATA-QUALITY-2026-06-15.md
@@ -1,6 +1,9 @@
 # DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session)

-**Status:** baseline scan done 2026-06-15; four investigation items open (DQ-1…DQ-4).
+**Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement
+applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is
+strong after fixing a P0 `codex search paper` crash. **DQ-2/DQ-3 still open.**
+**Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section).
 **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory
 of the audit conversation — everything needed to continue is in this file.

@@ -50,7 +53,35 @@ incomplete, or unrepresentative data. This document is that second axis — a

 ## Findings (open investigation items)

-### DQ-1 — Citation coverage: 12 / 29 papers (41 %) have ZERO citations · **HIGH**
+### DQ-1 — Citation coverage: 12 / 29 papers (41 %) had ZERO citations · **RESOLVED 2026-06-16**
+
+**Resolution (what was done):**
+- **Verified upstream, not a bug.** Queried the OpenAlex API directly for all 11
+  zero-citation papers that have an `openalex_id`: every one returns
+  `referenced_works_count = 0` server-side, W-ids all match what we stored. They
+  are `type: preprint` (arXiv) / `article` / `dissertation` — works OpenAlex
+  indexes without a parsed bibliography. Confirmed source-coverage limit.
+- **Semantic Scholar supplement applied to the live DB** (idempotent backfill,
+  `ON CONFLICT (citing_id,cited_id) DO NOTHING`): **+330 citations (590 → 920)**.
+  Zero-out-edge papers **12 → 2**; in-corpus citing coverage **17/29 → 27/29**;
+  graph 489→704 nodes, 590→920 edges; discovery-lead targets 472→677. 9 of the
+  new edges are in-corpus (e.g. `2305.10988 → 10.1007/s00454-019-00132-8`).
+- **Wired into ingest** so future re-ingests self-heal: when OpenAlex returns an
+  empty `referenced_works`, `ingest.py` now falls back to
+  `_s2_reference_supplement()` (citing_id rewritten to canonical `papers.id`,
+  DOI cited-ids lowercased). Same helper drove the one-off backfill.
+- **Bug fixed (would have crashed the batch):** `semanticscholar.fetch_references`
+  used `data.get("data", [])`, but S2 returns HTTP 200 `{"data": null}` for a
+  known paper with no parsed refs (e.g. `depositonce-5415`) → `TypeError`.
+  Changed to `data.get("data") or []`. Removed a now-redundant discarded S2 probe
+  in the OpenAlex-404 arXiv branch. New regression test added.
+- **Irreducible remainder (documented limit):** the **2 `depositonce` theses**
+  (`10.14279/depositonce-20357`, `-5415`) stay at zero — neither OpenAlex nor S2
+  indexes references for TU-Berlin institutional DOIs. Accept as inherent.
+- **Files touched (uncommitted in working tree):** `codex/sources/semanticscholar.py`,
+  `codex/ingest.py`, `tests/ingest/test_ingest.py`. Full suite: 331 passed.
+
+**Original finding (for context):**
 - **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph
  comes from only ~17 papers. The whole F-15 layer (PageRank / coupling /
  co-citation / discovery leads) therefore runs on ~59 % of the corpus.
@@ -111,7 +142,40 @@ incomplete, or unrepresentative data. This document is that second axis — a
 - **Acceptance:** sampled papers retain ≳ the expected fraction of source text;
  no paper is silently gutted by the quality gate.

-### DQ-4 — Retrieval quality: NOT YET MEASURED · **MED**
+### DQ-4 — Retrieval quality · **MEASURED 2026-06-16 — good, after fixing a P0 crash**
+
+**P0 bug found + fixed: `codex search paper` was crash-broken on the live DB.**
+The CLI passed the query embedding uncast, so pgvector raised
+`operator does not exist: vector <-> double precision[]` on every paper search.
+The working call sites (`mcp_server.py`, `wiki.py`) cast `%(emb)s::vector`; the
+CLI did not. Fixed both `<->` occurrences in `codex/cli.py search_paper`; added a
+regression guard in `tests/cli/test_cli.py` asserting the cast (mocked-DB unit
+tests can't catch the type error — this is why it survived the code audit).
+
+**Relevance verdict (post-fix): strong.** Six domain queries, both surfaces
+(paper-level abstract search + chunk-level). **rank-1 was the exactly-correct
+paper in 6/6 queries** for chunk search and 5/6 for paper-level (Laplace–Beltrami
+put the exact paper at rank-2 behind the closely-related "Polygon Laplacian Made
+Simple" — acceptable). Examples: "combinatorial Yamabe flow" → `math/0306167`
+(exact, all 5 top chunks that paper); "variational principle for Delaunay
+triangulations" → `math/0603097` (exact); "circle packing rigidity" →
+`2601.22903` (exact). Three of the DQ-1-rescued papers (`2305.10988`,
+`1005.2698`, `depositonce-5415`) now surface as top hits. The KB is trustworthy
+for lookups.
+
+**Secondary finding → reinforces DQ-2: zero-vector abstract pollution.** Papers
+with no abstract get a zero `abstract_emb`, which sits at constant L2 distance
+≈1.000 from every query and appears as filler in the paper-level tail whenever
+< 5 strongly-relevant papers exist. `1911.00966` (fully degraded, DQ-2) showed up
+at distance 1.000 in 4/6 queries. Harmless at rank>1 with a visible 1.000 score,
+but noise — and a concrete reason to fix DQ-2. Chunk-level search is immune (it
+filters `bibkey IS NOT NULL`, and 1911.00966 has no bibkey/chunks).
+
+**Note:** the MCP `search` docstring claims "hybrid dense + FTS" but the SQL is
+dense-only; `score = 1.0 - dist` can go negative (bge-m3 L2 ranges 0–2). Ranking
+is fine (monotonic); only the absolute score label is misleading.
+
+**Original open question (for context):**
 - **Open question:** end-to-end, do real queries return *relevant* results? Clean
  chunks + a working index don't guarantee useful retrieval.
 - **To investigate next:** run a handful of domain queries through the actual
@@ -125,14 +189,97 @@ incomplete, or unrepresentative data. This document is that second axis — a

 ---

-## Priority for the next session
-1. **DQ-1** (biggest lever — 41 % of the corpus is invisible to the graph; the S2
-   supplement is concrete and reuses existing code).
-2. **DQ-4** (cheap, high-information — tells you if the KB is actually usable).
-3. **DQ-2**, then **DQ-3**.
+## Roadmap — data-acquisition levers (post-audit, all free sources)

-All four are *data* work (queries + maybe a re-ingest/enrichment), not code-audit
-work — the code is already remediated (see the AUDIT-* docs and PRs #12–#14).
+These are *forward-looking acquisition improvements*, distinct from the DQ-1..DQ-4
+audit findings (which assessed existing data). They came out of a strategy
+discussion on 2026-06-16: for this Math/CS corpus the valuable data (references,
+abstracts, full text) is almost entirely **open** (arXiv, OpenAlex, Crossref, S2),
+so the quality lever is *combining more free sources*, not buying paywall access.
+Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item
+is self-contained so a cold session can pick it up.
+
+### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort**
+- **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`),
+  so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper
+  so `codex.parsing.grobid.extract_references` parses each bibliography.
+- **Why:** extracts references the APIs lack — a *complement* to DQ-1's S2
+  supplement. Crucially, the **2 `depositonce` theses** that stayed at zero in
+  DQ-1 (no API references anywhere) almost certainly have a references section in
+  their open-access PDFs → GROBID could finally give them out-edges. Closes the
+  last 2/29.
+- **How:** no new code — ingest already merges + dedups `api_citations +
+  pdf_citations` ([codex/ingest.py:288](../../codex/ingest.py)). GROBID server is
+  already configured (`GROBID_URL=http://192.168.178.103:8070` in
+  `.env.jetson-ingest`). Download free arXiv PDFs, re-ingest with `source_path`.
+- **Acceptance:** the 2 theses gain references; published papers gain
+  GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29.
+
+### R-B — Add Crossref as a third citation + abstract source · **MED lever, MED effort**
+- **What:** new `codex.sources.crossref` client hitting Crossref REST
+  `/works/{doi}` for the `reference` array and `abstract`.
+- **Why:** Crossref carries **publisher-deposited references** for many DOIs (free,
+  polite pool via `mailto`) — a third leg after OpenAlex/S2. It also recovers
+  abstracts OpenAlex is contractually barred from redistributing (likely the 2
+  Springer DOIs missing abstracts in DQ-2). cited_ids are DOIs → flow through the
+  existing graph resolver unchanged.
+- **How:** mirror `codex/sources/openalex.py` (tenacity retry, polite pool). Extend
+  the ingest fallback chain to OpenAlex-empty → S2 → Crossref (same
+  `_s2_reference_supplement` shape). Add unit tests with mocked HTTP.
+- **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where
+  OpenAlex+S2 are empty.
+
+### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **MED lever, MED-HIGH effort**
+- **What:** re-ingest from arXiv LaTeX source through the existing
+  `codex.parsing.tex.latex_to_text` path instead of flattened `.txt`.
+- **Why:** highest-fidelity input — preserves math, structure, and section
+  boundaries. Directly serves **DQ-3** (content fidelity) and audit **R-12** (the
+  `section` column is low-signal because word-window `.txt` chunks rarely start at
+  a real header; `.tex` headers fix this).
+- **How:** download free arXiv source tarballs, feed `.tex` to ingest with
+  `source_path`. Handle multi-file projects (main `.tex` + includes).
+- **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage
+  vs source improves; no regression in the F-16 quality gate.
+
+### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **SMALL**
+- **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper
+  count. Add a warning when the share of papers with ≥1 out-edge is low (e.g.
+  < 80%), since that is what actually starves PageRank/coupling.
+- **Why:** at 17/29 the graph silently ran on 59% of the corpus with no signal.
+  Post-DQ-1 it's 27/29 (93%) — but the guard should catch future regressions.
+- **Acceptance:** `codex graph report` surfaces citing-coverage % and warns below
+  threshold.
+
+### R-E — Paywall / institutional access · **DEFERRED — only for a paywall-only expansion**
+- **Decision (2026-06-16):** a uni login does **not** help the current corpus —
+  every DQ-1/DQ-2 gap is in openly-available works (arXiv preprints, OA theses),
+  and all metadata/citation sources are free APIs. Revisit *only* if the corpus
+  expands to papers that exist solely behind a paywall with no preprint; then
+  institutional access → full-text PDF → GROBID refs (R-A) + clean chunks (R-C).
+- **ToS caveat:** manual download of individual papers you have legitimate access
+  to is fine; **automated bulk download through a uni proxy (EZproxy/Shibboleth)
+  violates most publisher terms and can get the whole institution's access
+  revoked.** Do not wire a publisher login into the ingest pipeline.
+
+---
+
+## Priority for the next session
+1. ~~**DQ-1**~~ — **DONE 2026-06-16** (live DB at 920 citations, 27/29 citing
+   coverage; remedy wired into ingest).
+2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance
+   verified strong; surfaced the zero-vector pollution that motivates DQ-2).
+3. **DQ-2** (NEXT — recover `1911.00966`; backfill 3 missing abstracts so they
+   stop polluting paper-level search with zero-vectors), then **DQ-3** (chunk-vs-
+   source `.txt` fidelity).
+4. **Roadmap levers** (see section above), best sequenced after DQ-2/DQ-3:
+   **R-A** (GROBID on arXiv PDFs — closes the last 2 zero-citation theses, highest
+   ROI), then **R-B** (Crossref source — also recovers DQ-2 abstracts), **R-C**
+   (`.tex` ingest — serves DQ-3), **R-D** (F-15 coverage warning, quick).
+   **R-E** (paywall) is deferred.
+
+DQ-1..DQ-4 are *data*/assessment work; the roadmap items R-A..R-D add **new code**
+(re-ingest scripts, a Crossref source module, a `.tex` path, a graph warning) on
+top of the already-remediated pipeline (see the AUDIT-* docs and PRs #12–#14).

 ---

--- a/tests/ingest/test_ingest.py
+++ b/tests/ingest/test_ingest.py
@@ -174,6 +174,9 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
    with (
        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        # OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1);
+        # mock it empty so only the GROBID refs contribute to the count.
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
        patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
@@ -267,6 +270,43 @@ def test_ingest_paper_arxiv_s2_fallback() -> None:
    assert result.paper_id == paper_id


+def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
+    """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
+    rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
+    paper = _make_paper()  # openalex_id="W123", id="2301.07041"
+    mock_conn = MagicMock()
+    mock_conn.execute = MagicMock()
+    mock_conn.executemany = MagicMock()
+    mock_conn.commit = MagicMock()
+
+    # S2 keys citing_id by the id form passed in; ingest must canonicalise it.
+    s2_refs = [
+        Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"),
+        Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"),
+    ]
+
+    with (
+        patch("codex.ingest.openalex.fetch_paper", return_value=paper),
+        patch("codex.ingest.openalex.fetch_citations", return_value=[]),
+        patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2,
+        patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
+        patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
+    ):
+        result = ingest_paper(paper.id)
+
+    # S2 consulted with a namespaced id (a bare arXiv id 404s on S2).
+    mock_s2.assert_called_once_with("arXiv:2301.07041")
+    assert result.citations_upserted == 2
+
+    mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
+    inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1]
+    # citing_id rewritten to paper.id, not the S2 id form.
+    assert all(row[0] == paper.id for row in inserted_rows)
+    cited = {row[1] for row in inserted_rows}
+    assert "10.1/upper" in cited  # DOI lowercased
+    assert "2202.00002" in cited  # arXiv id untouched
+
+
 # ---------------------------------------------------------------------------
 # 7. test_ingest_paper_no_abstract_uses_zero_vector
 # ---------------------------------------------------------------------------