feat(ingest): supplement citations from Semantic Scholar when OpenAlex is empty (DQ-1)

12/29 papers had zero citations: OpenAlex indexes arXiv preprints and theses
without a parsed reference list (verified referenced_works_count=0 server-side
for all 11 with an openalex_id). Not an ingest bug — a source-coverage limit.

- ingest.py: when OpenAlex returns no references, fall back to a Semantic
  Scholar reference supplement (_s2_reference_supplement); citing_id rewritten
  to canonical papers.id, DOI cited-ids lowercased. Removed a now-redundant
  discarded S2 probe in the OpenAlex-404 arXiv branch.
- semanticscholar.py: fix TypeError on S2's HTTP-200 {"data": null} no-refs
  responses (.get("data", []) returns None when the key is present-but-null).
- tests: regression test for the OpenAlex-empty -> S2 path.
- Live DB backfilled idempotently: +330 citations (590->920), zero-out-edge
  papers 12->2, citing coverage 17->27/29. Only the 2 TU-Berlin depositonce
  theses remain (no references in any source).
- docs: DATA-QUALITY-2026-06-15.md — DQ-1 resolution, DQ-4 result, and a
  free-source acquisition roadmap (R-A..R-E).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

codex/ingest.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import contextlib
 import logging
 from dataclasses import dataclass
 from pathlib import Path
@@ -47,6 +46,55 @@ def _make_bibkey(paper: Paper) -> str | None:
     return surnames[0] + "EtAl" + year
 
 
+def _s2_id_for(pid: str) -> str | None:
+    """Format a canonical paper id for the Semantic Scholar paper endpoint.
+
+    S2 needs a namespaced id (``arXiv:<id>``, ``DOI:<doi>``) or a bare 40-hex
+    S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by
+    the ``10.`` prefix; everything else is assumed to be an arXiv id (modern
+    ``2305.10988`` or legacy ``math/0603097``).
+    """
+    p = (pid or "").strip()
+    if not p:
+        return None
+    if p.lower().startswith(("arxiv:", "doi:")):
+        return p
+    if p.startswith("10."):
+        return f"DOI:{p}"
+    return f"arXiv:{p}"
+
+
+def _norm_cited_id(cited_id: str) -> str:
+    """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join
+    does not fragment the same reference into distinct case-variant nodes.
+    arXiv ids and S2 paperIds are left untouched."""
+    return cited_id.lower() if cited_id.startswith("10.") else cited_id
+
+
+def _s2_reference_supplement(paper: Paper) -> list[Citation]:
+    """Fetch a paper's references from Semantic Scholar (DQ-1 supplement).
+
+    Used when OpenAlex returns an empty ``referenced_works`` list — common for
+    arXiv preprints and institutional theses that OpenAlex indexes without a
+    parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id``
+    so the FK holds, and cited DOIs are case-normalised. Network/parse failures
+    degrade to an empty list rather than aborting the ingest.
+    """
+    s2_id = _s2_id_for(paper.id)
+    if s2_id is None:
+        return []
+    try:
+        raw = semanticscholar.fetch_references(s2_id)
+    except Exception:
+        logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True)
+        return []
+    return [
+        Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
+        for c in raw
+        if c.cited_id
+    ]
+
+
 def ingest_paper(
     paper_id: str,
     source_path: str | None = None,
@@ -88,9 +136,9 @@ def ingest_paper(
             len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit()
         )
         if looks_like_arxiv:
-            arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}"
-            with contextlib.suppress(Exception):
-                semanticscholar.fetch_references(arxiv_key)
+            # OpenAlex 404 on an arXiv id → keep a minimal stub. Citation
+            # recovery from S2 happens in the citation block below (DQ-1
+            # supplement), so no probe fetch is needed here.
             paper = Paper(id=paper_id, title="")
         else:
             raise ValueError(f"Paper not found: {paper_id}")
@@ -229,8 +277,13 @@ def ingest_paper(
             api_citations = [
                 Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
             ]
+            # DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a
+            # parsed reference list (referenced_works == []). Fall back to the
+            # Semantic Scholar references, which often has them.
+            if not api_citations:
+                api_citations = _s2_reference_supplement(paper)
         else:
-            api_citations = semanticscholar.fetch_references(paper_id)
+            api_citations = _s2_reference_supplement(paper)
 
         # Merge API citations and GROBID PDF citations; dedup via set
         all_citations_set: set[tuple[str, str]] = set()

codex/sources/semanticscholar.py

@@ -87,7 +87,11 @@ def fetch_references(paper_id: str) -> list[Citation]:
         raise
 
     data = response.json()
-    raw_refs: list[dict[str, Any]] = data.get("data", [])
+    # S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
+    # parsed reference list (e.g. some theses). ``.get("data", [])`` would yield
+    # the explicit ``None`` (the default only applies when the key is absent),
+    # so iterate over a guaranteed list instead.
+    raw_refs: list[dict[str, Any]] = data.get("data") or []
     citations: list[Citation] = []
     for entry in raw_refs:
         cited_paper: dict[str, Any] = entry.get("citedPaper", {})