From 553ae3995de526587a4ed85d1897fba230a0810e Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Tue, 16 Jun 2026 23:51:38 +0200 Subject: [PATCH 01/26] fix(search): cast query embedding to ::vector in `search paper` (DQ-4) `codex search paper` crashed on the live DB with "operator does not exist: vector <-> double precision[]". The pgvector `<->` operator needs its RHS typed as vector; peer call sites (mcp_server.py, wiki.py) cast but the CLI did not. Mocked unit tests never exercise real pgvector, so it went unseen. Add the cast to both `<->` occurrences and a regression guard asserting the SQL contains `::vector`. Co-Authored-By: Claude Opus 4.8 --- codex/cli.py | 4 ++-- tests/cli/test_cli.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/codex/cli.py b/codex/cli.py index 33ab2c8..28130b7 100644 --- a/codex/cli.py +++ b/codex/cli.py @@ -84,10 +84,10 @@ def search_paper( rows = conn.execute( """ SELECT id, title, year, - abstract_emb <-> %(emb)s AS distance + abstract_emb <-> %(emb)s::vector AS distance FROM papers WHERE abstract_emb IS NOT NULL - ORDER BY abstract_emb <-> %(emb)s + ORDER BY abstract_emb <-> %(emb)s::vector LIMIT %(limit)s """, {"emb": emb, "limit": limit}, diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 4d15713..9561191 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -106,6 +106,13 @@ class TestSearch: assert "Test Paper" in result.stdout assert "0.123" in result.stdout + # Regression guard (DQ-4): the pgvector `<->` operator requires the + # query embedding cast to ::vector. Without it, real Postgres raises + # "operator does not exist: vector <-> double precision[]". Mocked DBs + # don't catch the type error, so assert the cast is in the SQL. + executed_sql = mock_conn.execute.call_args_list[0][0][0] + assert "<->" in executed_sql and "::vector" in executed_sql + def test_search_formula_command(self) -> None: """Test `search formula` command with mocked DB.""" with patch("codex.db.get_conn") as mock_get_conn: From 51dc74470cf77516b69f60ac706db849d1f980bc Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Tue, 16 Jun 2026 23:51:48 +0200 Subject: [PATCH 02/26] feat(ingest): supplement citations from Semantic Scholar when OpenAlex is empty (DQ-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 12/29 papers had zero citations: OpenAlex indexes arXiv preprints and theses without a parsed reference list (verified referenced_works_count=0 server-side for all 11 with an openalex_id). Not an ingest bug — a source-coverage limit. - ingest.py: when OpenAlex returns no references, fall back to a Semantic Scholar reference supplement (_s2_reference_supplement); citing_id rewritten to canonical papers.id, DOI cited-ids lowercased. Removed a now-redundant discarded S2 probe in the OpenAlex-404 arXiv branch. - semanticscholar.py: fix TypeError on S2's HTTP-200 {"data": null} no-refs responses (.get("data", []) returns None when the key is present-but-null). - tests: regression test for the OpenAlex-empty -> S2 path. - Live DB backfilled idempotently: +330 citations (590->920), zero-out-edge papers 12->2, citing coverage 17->27/29. Only the 2 TU-Berlin depositonce theses remain (no references in any source). - docs: DATA-QUALITY-2026-06-15.md — DQ-1 resolution, DQ-4 result, and a free-source acquisition roadmap (R-A..R-E). Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 63 +++++++++- codex/sources/semanticscholar.py | 6 +- docs/audit/DATA-QUALITY-2026-06-15.md | 167 ++++++++++++++++++++++++-- tests/ingest/test_ingest.py | 40 ++++++ 4 files changed, 260 insertions(+), 16 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index b63799d..5dab3b8 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -2,7 +2,6 @@ from __future__ import annotations -import contextlib import logging from dataclasses import dataclass from pathlib import Path @@ -47,6 +46,55 @@ def _make_bibkey(paper: Paper) -> str | None: return surnames[0] + "EtAl" + year +def _s2_id_for(pid: str) -> str | None: + """Format a canonical paper id for the Semantic Scholar paper endpoint. + + S2 needs a namespaced id (``arXiv:``, ``DOI:``) or a bare 40-hex + S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by + the ``10.`` prefix; everything else is assumed to be an arXiv id (modern + ``2305.10988`` or legacy ``math/0603097``). + """ + p = (pid or "").strip() + if not p: + return None + if p.lower().startswith(("arxiv:", "doi:")): + return p + if p.startswith("10."): + return f"DOI:{p}" + return f"arXiv:{p}" + + +def _norm_cited_id(cited_id: str) -> str: + """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join + does not fragment the same reference into distinct case-variant nodes. + arXiv ids and S2 paperIds are left untouched.""" + return cited_id.lower() if cited_id.startswith("10.") else cited_id + + +def _s2_reference_supplement(paper: Paper) -> list[Citation]: + """Fetch a paper's references from Semantic Scholar (DQ-1 supplement). + + Used when OpenAlex returns an empty ``referenced_works`` list — common for + arXiv preprints and institutional theses that OpenAlex indexes without a + parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id`` + so the FK holds, and cited DOIs are case-normalised. Network/parse failures + degrade to an empty list rather than aborting the ingest. + """ + s2_id = _s2_id_for(paper.id) + if s2_id is None: + return [] + try: + raw = semanticscholar.fetch_references(s2_id) + except Exception: + logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True) + return [] + return [ + Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context) + for c in raw + if c.cited_id + ] + + def ingest_paper( paper_id: str, source_path: str | None = None, @@ -88,9 +136,9 @@ def ingest_paper( len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit() ) if looks_like_arxiv: - arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}" - with contextlib.suppress(Exception): - semanticscholar.fetch_references(arxiv_key) + # OpenAlex 404 on an arXiv id → keep a minimal stub. Citation + # recovery from S2 happens in the citation block below (DQ-1 + # supplement), so no probe fetch is needed here. paper = Paper(id=paper_id, title="") else: raise ValueError(f"Paper not found: {paper_id}") @@ -229,8 +277,13 @@ def ingest_paper( api_citations = [ Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw ] + # DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a + # parsed reference list (referenced_works == []). Fall back to the + # Semantic Scholar references, which often has them. + if not api_citations: + api_citations = _s2_reference_supplement(paper) else: - api_citations = semanticscholar.fetch_references(paper_id) + api_citations = _s2_reference_supplement(paper) # Merge API citations and GROBID PDF citations; dedup via set all_citations_set: set[tuple[str, str]] = set() diff --git a/codex/sources/semanticscholar.py b/codex/sources/semanticscholar.py index e82d75a..0e92268 100644 --- a/codex/sources/semanticscholar.py +++ b/codex/sources/semanticscholar.py @@ -87,7 +87,11 @@ def fetch_references(paper_id: str) -> list[Citation]: raise data = response.json() - raw_refs: list[dict[str, Any]] = data.get("data", []) + # S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no + # parsed reference list (e.g. some theses). ``.get("data", [])`` would yield + # the explicit ``None`` (the default only applies when the key is absent), + # so iterate over a guaranteed list instead. + raw_refs: list[dict[str, Any]] = data.get("data") or [] citations: list[Citation] = [] for entry in raw_refs: cited_paper: dict[str, Any] = entry.get("citedPaper", {}) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index f016a79..b3f87e8 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -1,6 +1,9 @@ # DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session) -**Status:** baseline scan done 2026-06-15; four investigation items open (DQ-1…DQ-4). +**Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement +applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is +strong after fixing a P0 `codex search paper` crash. **DQ-2/DQ-3 still open.** +**Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section). **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory of the audit conversation — everything needed to continue is in this file. @@ -50,7 +53,35 @@ incomplete, or unrepresentative data. This document is that second axis — a ## Findings (open investigation items) -### DQ-1 — Citation coverage: 12 / 29 papers (41 %) have ZERO citations · **HIGH** +### DQ-1 — Citation coverage: 12 / 29 papers (41 %) had ZERO citations · **RESOLVED 2026-06-16** + +**Resolution (what was done):** +- **Verified upstream, not a bug.** Queried the OpenAlex API directly for all 11 + zero-citation papers that have an `openalex_id`: every one returns + `referenced_works_count = 0` server-side, W-ids all match what we stored. They + are `type: preprint` (arXiv) / `article` / `dissertation` — works OpenAlex + indexes without a parsed bibliography. Confirmed source-coverage limit. +- **Semantic Scholar supplement applied to the live DB** (idempotent backfill, + `ON CONFLICT (citing_id,cited_id) DO NOTHING`): **+330 citations (590 → 920)**. + Zero-out-edge papers **12 → 2**; in-corpus citing coverage **17/29 → 27/29**; + graph 489→704 nodes, 590→920 edges; discovery-lead targets 472→677. 9 of the + new edges are in-corpus (e.g. `2305.10988 → 10.1007/s00454-019-00132-8`). +- **Wired into ingest** so future re-ingests self-heal: when OpenAlex returns an + empty `referenced_works`, `ingest.py` now falls back to + `_s2_reference_supplement()` (citing_id rewritten to canonical `papers.id`, + DOI cited-ids lowercased). Same helper drove the one-off backfill. +- **Bug fixed (would have crashed the batch):** `semanticscholar.fetch_references` + used `data.get("data", [])`, but S2 returns HTTP 200 `{"data": null}` for a + known paper with no parsed refs (e.g. `depositonce-5415`) → `TypeError`. + Changed to `data.get("data") or []`. Removed a now-redundant discarded S2 probe + in the OpenAlex-404 arXiv branch. New regression test added. +- **Irreducible remainder (documented limit):** the **2 `depositonce` theses** + (`10.14279/depositonce-20357`, `-5415`) stay at zero — neither OpenAlex nor S2 + indexes references for TU-Berlin institutional DOIs. Accept as inherent. +- **Files touched (uncommitted in working tree):** `codex/sources/semanticscholar.py`, + `codex/ingest.py`, `tests/ingest/test_ingest.py`. Full suite: 331 passed. + +**Original finding (for context):** - **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph comes from only ~17 papers. The whole F-15 layer (PageRank / coupling / co-citation / discovery leads) therefore runs on ~59 % of the corpus. @@ -111,7 +142,40 @@ incomplete, or unrepresentative data. This document is that second axis — a - **Acceptance:** sampled papers retain ≳ the expected fraction of source text; no paper is silently gutted by the quality gate. -### DQ-4 — Retrieval quality: NOT YET MEASURED · **MED** +### DQ-4 — Retrieval quality · **MEASURED 2026-06-16 — good, after fixing a P0 crash** + +**P0 bug found + fixed: `codex search paper` was crash-broken on the live DB.** +The CLI passed the query embedding uncast, so pgvector raised +`operator does not exist: vector <-> double precision[]` on every paper search. +The working call sites (`mcp_server.py`, `wiki.py`) cast `%(emb)s::vector`; the +CLI did not. Fixed both `<->` occurrences in `codex/cli.py search_paper`; added a +regression guard in `tests/cli/test_cli.py` asserting the cast (mocked-DB unit +tests can't catch the type error — this is why it survived the code audit). + +**Relevance verdict (post-fix): strong.** Six domain queries, both surfaces +(paper-level abstract search + chunk-level). **rank-1 was the exactly-correct +paper in 6/6 queries** for chunk search and 5/6 for paper-level (Laplace–Beltrami +put the exact paper at rank-2 behind the closely-related "Polygon Laplacian Made +Simple" — acceptable). Examples: "combinatorial Yamabe flow" → `math/0306167` +(exact, all 5 top chunks that paper); "variational principle for Delaunay +triangulations" → `math/0603097` (exact); "circle packing rigidity" → +`2601.22903` (exact). Three of the DQ-1-rescued papers (`2305.10988`, +`1005.2698`, `depositonce-5415`) now surface as top hits. The KB is trustworthy +for lookups. + +**Secondary finding → reinforces DQ-2: zero-vector abstract pollution.** Papers +with no abstract get a zero `abstract_emb`, which sits at constant L2 distance +≈1.000 from every query and appears as filler in the paper-level tail whenever +< 5 strongly-relevant papers exist. `1911.00966` (fully degraded, DQ-2) showed up +at distance 1.000 in 4/6 queries. Harmless at rank>1 with a visible 1.000 score, +but noise — and a concrete reason to fix DQ-2. Chunk-level search is immune (it +filters `bibkey IS NOT NULL`, and 1911.00966 has no bibkey/chunks). + +**Note:** the MCP `search` docstring claims "hybrid dense + FTS" but the SQL is +dense-only; `score = 1.0 - dist` can go negative (bge-m3 L2 ranges 0–2). Ranking +is fine (monotonic); only the absolute score label is misleading. + +**Original open question (for context):** - **Open question:** end-to-end, do real queries return *relevant* results? Clean chunks + a working index don't guarantee useful retrieval. - **To investigate next:** run a handful of domain queries through the actual @@ -125,14 +189,97 @@ incomplete, or unrepresentative data. This document is that second axis — a --- -## Priority for the next session -1. **DQ-1** (biggest lever — 41 % of the corpus is invisible to the graph; the S2 - supplement is concrete and reuses existing code). -2. **DQ-4** (cheap, high-information — tells you if the KB is actually usable). -3. **DQ-2**, then **DQ-3**. +## Roadmap — data-acquisition levers (post-audit, all free sources) -All four are *data* work (queries + maybe a re-ingest/enrichment), not code-audit -work — the code is already remediated (see the AUDIT-* docs and PRs #12–#14). +These are *forward-looking acquisition improvements*, distinct from the DQ-1..DQ-4 +audit findings (which assessed existing data). They came out of a strategy +discussion on 2026-06-16: for this Math/CS corpus the valuable data (references, +abstracts, full text) is almost entirely **open** (arXiv, OpenAlex, Crossref, S2), +so the quality lever is *combining more free sources*, not buying paywall access. +Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item +is self-contained so a cold session can pick it up. + +### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort** +- **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), + so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper + so `codex.parsing.grobid.extract_references` parses each bibliography. +- **Why:** extracts references the APIs lack — a *complement* to DQ-1's S2 + supplement. Crucially, the **2 `depositonce` theses** that stayed at zero in + DQ-1 (no API references anywhere) almost certainly have a references section in + their open-access PDFs → GROBID could finally give them out-edges. Closes the + last 2/29. +- **How:** no new code — ingest already merges + dedups `api_citations + + pdf_citations` ([codex/ingest.py:288](../../codex/ingest.py)). GROBID server is + already configured (`GROBID_URL=http://192.168.178.103:8070` in + `.env.jetson-ingest`). Download free arXiv PDFs, re-ingest with `source_path`. +- **Acceptance:** the 2 theses gain references; published papers gain + GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29. + +### R-B — Add Crossref as a third citation + abstract source · **MED lever, MED effort** +- **What:** new `codex.sources.crossref` client hitting Crossref REST + `/works/{doi}` for the `reference` array and `abstract`. +- **Why:** Crossref carries **publisher-deposited references** for many DOIs (free, + polite pool via `mailto`) — a third leg after OpenAlex/S2. It also recovers + abstracts OpenAlex is contractually barred from redistributing (likely the 2 + Springer DOIs missing abstracts in DQ-2). cited_ids are DOIs → flow through the + existing graph resolver unchanged. +- **How:** mirror `codex/sources/openalex.py` (tenacity retry, polite pool). Extend + the ingest fallback chain to OpenAlex-empty → S2 → Crossref (same + `_s2_reference_supplement` shape). Add unit tests with mocked HTTP. +- **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where + OpenAlex+S2 are empty. + +### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **MED lever, MED-HIGH effort** +- **What:** re-ingest from arXiv LaTeX source through the existing + `codex.parsing.tex.latex_to_text` path instead of flattened `.txt`. +- **Why:** highest-fidelity input — preserves math, structure, and section + boundaries. Directly serves **DQ-3** (content fidelity) and audit **R-12** (the + `section` column is low-signal because word-window `.txt` chunks rarely start at + a real header; `.tex` headers fix this). +- **How:** download free arXiv source tarballs, feed `.tex` to ingest with + `source_path`. Handle multi-file projects (main `.tex` + includes). +- **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage + vs source improves; no regression in the F-16 quality gate. + +### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **SMALL** +- **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper + count. Add a warning when the share of papers with ≥1 out-edge is low (e.g. + < 80%), since that is what actually starves PageRank/coupling. +- **Why:** at 17/29 the graph silently ran on 59% of the corpus with no signal. + Post-DQ-1 it's 27/29 (93%) — but the guard should catch future regressions. +- **Acceptance:** `codex graph report` surfaces citing-coverage % and warns below + threshold. + +### R-E — Paywall / institutional access · **DEFERRED — only for a paywall-only expansion** +- **Decision (2026-06-16):** a uni login does **not** help the current corpus — + every DQ-1/DQ-2 gap is in openly-available works (arXiv preprints, OA theses), + and all metadata/citation sources are free APIs. Revisit *only* if the corpus + expands to papers that exist solely behind a paywall with no preprint; then + institutional access → full-text PDF → GROBID refs (R-A) + clean chunks (R-C). +- **ToS caveat:** manual download of individual papers you have legitimate access + to is fine; **automated bulk download through a uni proxy (EZproxy/Shibboleth) + violates most publisher terms and can get the whole institution's access + revoked.** Do not wire a publisher login into the ingest pipeline. + +--- + +## Priority for the next session +1. ~~**DQ-1**~~ — **DONE 2026-06-16** (live DB at 920 citations, 27/29 citing + coverage; remedy wired into ingest). +2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance + verified strong; surfaced the zero-vector pollution that motivates DQ-2). +3. **DQ-2** (NEXT — recover `1911.00966`; backfill 3 missing abstracts so they + stop polluting paper-level search with zero-vectors), then **DQ-3** (chunk-vs- + source `.txt` fidelity). +4. **Roadmap levers** (see section above), best sequenced after DQ-2/DQ-3: + **R-A** (GROBID on arXiv PDFs — closes the last 2 zero-citation theses, highest + ROI), then **R-B** (Crossref source — also recovers DQ-2 abstracts), **R-C** + (`.tex` ingest — serves DQ-3), **R-D** (F-15 coverage warning, quick). + **R-E** (paywall) is deferred. + +DQ-1..DQ-4 are *data*/assessment work; the roadmap items R-A..R-D add **new code** +(re-ingest scripts, a Crossref source module, a `.tex` path, a graph warning) on +top of the already-remediated pipeline (see the AUDIT-* docs and PRs #12–#14). --- diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 80e4d23..6aad09b 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -174,6 +174,9 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1); + # mock it empty so only the GROBID refs contribute to the count. + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), @@ -267,6 +270,43 @@ def test_ingest_paper_arxiv_s2_fallback() -> None: assert result.paper_id == paper_id +def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None: + """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2, + rewriting citing_id to paper.id and lowercasing DOI cited-ids.""" + paper = _make_paper() # openalex_id="W123", id="2301.07041" + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + # S2 keys citing_id by the id form passed in; ingest must canonicalise it. + s2_refs = [ + Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"), + Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"), + ] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(paper.id) + + # S2 consulted with a namespaced id (a bare arXiv id 404s on S2). + mock_s2.assert_called_once_with("arXiv:2301.07041") + assert result.citations_upserted == 2 + + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1] + # citing_id rewritten to paper.id, not the S2 id form. + assert all(row[0] == paper.id for row in inserted_rows) + cited = {row[1] for row in inserted_rows} + assert "10.1/upper" in cited # DOI lowercased + assert "2202.00002" in cited # arXiv id untouched + + # --------------------------------------------------------------------------- # 7. test_ingest_paper_no_abstract_uses_zero_vector # --------------------------------------------------------------------------- From ff03443af425460df7796e988ec16d4fa6ce7ba9 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:12:59 +0200 Subject: [PATCH 03/26] feat(ingest): recover missing metadata + abstracts from arXiv/S2/Crossref (DQ-2) Three papers lacked abstracts and 1911.00966 was fully degraded (OpenAlex 404 -> empty `Paper(id, title="")` stub). Recovery sources verified, then wired in: - arxiv.fetch_metadata: resolve an arXiv id to a Paper via the Atom export API (authoritative for preprints). Used as an ingest fallback on OpenAlex 404, replacing the empty stub. - crossref.py (new): fetch_abstract(doi), JATS-stripped, Crossref Polite Pool. - semanticscholar.fetch_abstract: fetch just the abstract field. - ingest.py: _recover_abstract() supplements an empty abstract from S2 then Crossref *before* embedding, so the paper gets a real (non-zero) vector. Live DB backfill: 1911.00966 fully recovered from arXiv (bibkey PinkallSpringborn2019, 384-char abstract, its 10 chunks now visible to chunk search); 10.1007/s00454-019-00132-8 abstract from S2 (1186 chars). Book chapter 10.1007/978-3-642-17413-1_7 has no abstract in OpenAlex/S2/Crossref -> documented limit. Corpus now has 1 paper without an abstract. Also documented DQ-5 (not fixed): ingest_paper is not idempotent for DOI papers (openalex returns full-URL id vs the bare canonical id) -> re-ingest trips papers_openalex_id_key. Blocks roadmap R-A/R-C; needs a careful _map_paper fix. Tests: arxiv/crossref/s2 fetchers + ingest recovery paths (342 passing). Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 44 +++++++++++-- codex/sources/arxiv.py | 78 +++++++++++++++++++++++ codex/sources/crossref.py | 91 +++++++++++++++++++++++++++ codex/sources/semanticscholar.py | 18 ++++++ docs/audit/DATA-QUALITY-2026-06-15.md | 83 ++++++++++++++++++++---- tests/ingest/test_ingest.py | 74 +++++++++++++++++++++- tests/sources/test_arxiv.py | 60 ++++++++++++++++++ tests/sources/test_crossref.py | 50 +++++++++++++++ tests/sources/test_semanticscholar.py | 43 +++++++++++++ 9 files changed, 521 insertions(+), 20 deletions(-) create mode 100644 codex/sources/crossref.py create mode 100644 tests/sources/test_crossref.py diff --git a/codex/ingest.py b/codex/ingest.py index 5dab3b8..cd950e7 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -13,7 +13,7 @@ from codex.db import get_conn from codex.embed import get_embedder from codex.models import Citation, Paper from codex.quality import classify_section, filter_chunks -from codex.sources import openalex, semanticscholar +from codex.sources import arxiv, crossref, openalex, semanticscholar logger = logging.getLogger(__name__) @@ -71,6 +71,31 @@ def _norm_cited_id(cited_id: str) -> str: return cited_id.lower() if cited_id.startswith("10.") else cited_id +def _recover_abstract(paper: Paper) -> str | None: + """Recover a missing abstract from S2, then Crossref (DQ-2 supplement). + + Tried in order of coverage for this corpus: Semantic Scholar (indexes most + arXiv + journal abstracts), then Crossref (publisher-deposited, DOI only). + Network/parse failures degrade to None rather than aborting the ingest. + """ + s2_id = _s2_id_for(paper.id) + if s2_id is not None: + try: + abstract = semanticscholar.fetch_abstract(s2_id) + if abstract and abstract.strip(): + return abstract.strip() + except Exception: + logger.warning("S2 abstract recovery failed for %s", paper.id, exc_info=True) + if paper.id.startswith("10."): + try: + abstract = crossref.fetch_abstract(paper.id) + if abstract and abstract.strip(): + return abstract.strip() + except Exception: + logger.warning("Crossref abstract recovery failed for %s", paper.id, exc_info=True) + return None + + def _s2_reference_supplement(paper: Paper) -> list[Citation]: """Fetch a paper's references from Semantic Scholar (DQ-1 supplement). @@ -136,16 +161,25 @@ def ingest_paper( len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit() ) if looks_like_arxiv: - # OpenAlex 404 on an arXiv id → keep a minimal stub. Citation - # recovery from S2 happens in the citation block below (DQ-1 - # supplement), so no probe fetch is needed here. - paper = Paper(id=paper_id, title="") + # OpenAlex 404 on an arXiv id → recover authoritative metadata from + # the arXiv API (DQ-2); fall back to a minimal stub if arXiv also + # has nothing. Citation recovery from S2 happens in the citation + # block below (DQ-1 supplement). + paper = arxiv.fetch_metadata(paper_id) or Paper(id=paper_id, title="") else: raise ValueError(f"Paper not found: {paper_id}") if paper is None: raise ValueError(f"Paper not found: {paper_id}") + # DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute + # some publishers' abstracts). Supplement from S2, then Crossref, so the + # paper gets a real (non-zero) abstract embedding instead of a zero vector. + if not (paper.abstract or "").strip(): + recovered = _recover_abstract(paper) + if recovered: + paper.abstract = recovered + # Auto-generate bibkey when source has none (D-04). if paper.bibkey is None: paper.bibkey = _make_bibkey(paper) diff --git a/codex/sources/arxiv.py b/codex/sources/arxiv.py index 1de3e38..9bb0303 100644 --- a/codex/sources/arxiv.py +++ b/codex/sources/arxiv.py @@ -1,6 +1,7 @@ """arXiv API client. Provides: +- fetch_metadata: resolve an arXiv ID to a Paper via the Atom export API. - fetch_source: download the .tar.gz source of a paper and extract the primary .tex file. - fetch_pdf_url: return the canonical PDF URL for a given arXiv ID. """ @@ -10,12 +11,89 @@ from __future__ import annotations import io import logging import tarfile +import xml.etree.ElementTree as ET import httpx +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential + +from codex.models import Paper logger = logging.getLogger(__name__) _BASE = "https://arxiv.org" +_EXPORT = "https://export.arxiv.org" +_ATOM = "{http://www.w3.org/2005/Atom}" + + +def _is_retryable(exc: BaseException) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code == 429 or exc.response.status_code >= 500 + return False + + +@retry( + retry=retry_if_exception(_is_retryable), + stop=stop_after_attempt(4), + wait=wait_exponential(min=1, max=20), + before_sleep=lambda rs: logger.warning("arXiv retry %d", rs.attempt_number), +) +def _query(arxiv_id: str) -> httpx.Response: + response = httpx.get( + f"{_EXPORT}/api/query", + params={"id_list": arxiv_id}, + timeout=30, + follow_redirects=True, + ) + response.raise_for_status() + return response + + +def fetch_metadata(arxiv_id: str) -> Paper | None: + """Resolve an arXiv ID to a Paper via the Atom export API. + + Authoritative metadata source for arXiv preprints — used as an ingest + fallback when OpenAlex 404s on an arXiv id (DQ-2). The arXiv id (bare, + e.g. ``"1911.00966"`` or legacy ``"math/0603097"``) becomes ``Paper.id``; + ``openalex_id`` and ``bibkey`` are left for the caller to populate. + + Returns + ------- + Paper | None + Populated Paper (title, authors, year, abstract), or None if arXiv + has no entry for the id. + """ + bare = arxiv_id[len("arxiv:") :] if arxiv_id.lower().startswith("arxiv:") else arxiv_id + try: + response = _query(bare) + except httpx.HTTPError: + logger.warning("arXiv metadata fetch failed for %s", bare, exc_info=True) + return None + try: + root = ET.fromstring(response.text) + except ET.ParseError: + return None + entry = root.find(f"{_ATOM}entry") + if entry is None: + return None + # An id-not-found query still returns a feed but with no . + title = (entry.findtext(f"{_ATOM}title") or "").strip() + if not title: + return None + summary = (entry.findtext(f"{_ATOM}summary") or "").strip() + published = entry.findtext(f"{_ATOM}published") or "" + year = int(published[:4]) if published[:4].isdigit() else None + authors = [ + name.strip() + for a in entry.findall(f"{_ATOM}author") + if (name := a.findtext(f"{_ATOM}name")) and name.strip() + ] + return Paper( + id=bare, + title=" ".join(title.split()), + authors=authors, + year=year, + abstract=" ".join(summary.split()) or None, + ) def fetch_source(arxiv_id: str) -> str | None: diff --git a/codex/sources/crossref.py b/codex/sources/crossref.py new file mode 100644 index 0000000..4727d85 --- /dev/null +++ b/codex/sources/crossref.py @@ -0,0 +1,91 @@ +"""Crossref API client. + +Provides: +- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI. + +Crossref is a *third* metadata source after OpenAlex and Semantic Scholar +(DQ-2 / roadmap R-B). Requests use the Polite Pool (mailto query parameter) +and are retried on 429/5xx with exponential back-off via tenacity. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +import httpx +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential + +from codex.config import get_settings + +logger = logging.getLogger(__name__) + +_BASE = "https://api.crossref.org" +_TAG_RE = re.compile(r"<[^>]+>") + + +def _is_retryable(exc: BaseException) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code == 429 or exc.response.status_code >= 500 + return False + + +def _polite_params() -> dict[str, str]: + # Reuse the OpenAlex mailto for Crossref's Polite Pool (same address). + mailto = get_settings().openalex_mailto + return {"mailto": mailto} if mailto else {} + + +@retry( + retry=retry_if_exception(_is_retryable), + stop=stop_after_attempt(5), + wait=wait_exponential(min=1, max=30), + before_sleep=lambda rs: logger.warning( + "Crossref retry %d after %s", + rs.attempt_number, + rs.outcome.exception(), # type: ignore[union-attr] + ), +) +def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response: + merged: dict[str, str] = {**_polite_params(), **(params or {})} + response = httpx.get(url, params=merged, timeout=30, follow_redirects=True) + response.raise_for_status() + return response + + +def _strip_jats(abstract: str | None) -> str | None: + """Strip JATS/XML tags from a Crossref abstract and collapse whitespace.""" + if not abstract: + return None + text = _TAG_RE.sub("", abstract) + text = " ".join(text.split()).strip() + return text or None + + +def fetch_abstract(doi: str) -> str | None: + """Fetch a work's abstract from Crossref by DOI. + + Crossref stores abstracts as JATS-tagged markup (``…``); + tags are stripped before returning. Many works (notably book chapters) + have no abstract deposited — returns None in that case and on 404. + + Parameters + ---------- + doi: + A bare DOI (``"10.1007/s00454-019-00132-8"``). + + Returns + ------- + str | None + The plain-text abstract, or None when absent. + """ + url = f"{_BASE}/works/{doi}" + try: + response = _get(url) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None + raise + message: dict[str, Any] = response.json().get("message", {}) + return _strip_jats(message.get("abstract")) diff --git a/codex/sources/semanticscholar.py b/codex/sources/semanticscholar.py index 0e92268..5862071 100644 --- a/codex/sources/semanticscholar.py +++ b/codex/sources/semanticscholar.py @@ -107,6 +107,24 @@ def fetch_references(paper_id: str) -> list[Citation]: return citations +def fetch_abstract(paper_id: str) -> str | None: + """Fetch just the abstract for a paper from Semantic Scholar. + + Used as an ingest abstract supplement (DQ-2) when OpenAlex has the paper + but no abstract. ``paper_id`` should be namespaced (``arXiv:…`` / ``DOI:…``). + Returns None on 404 or when S2 has no abstract. + """ + url = f"{_BASE_GRAPH}/paper/{paper_id}" + try: + response = _get(url, params={"fields": "abstract"}) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None + raise + abstract = response.json().get("abstract") + return abstract or None + + def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]: """Fetch recommended paper IDs from Semantic Scholar. diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index b3f87e8..81b8a4c 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -2,7 +2,10 @@ **Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is -strong after fixing a P0 `codex search paper` crash. **DQ-2/DQ-3 still open.** +strong after fixing a P0 `codex search paper` crash. **DQ-2 RESOLVED 2026-06-16** +(1911.00966 fully recovered from arXiv; s00454 abstract from S2; book chapter is a +documented limit; recovery wired into ingest). **DQ-3 still open. New: DQ-5** +(`ingest_paper` not idempotent for DOI papers — blocks R-A/R-C). **Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section). **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory of the audit conversation — everything needed to continue is in this file. @@ -110,7 +113,32 @@ incomplete, or unrepresentative data. This document is that second axis — a supplement, or (b) documented as an inherent OpenAlex-coverage limit with the graph caveated accordingly. -### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **MED** +### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **RESOLVED 2026-06-16** + +**Resolution (what was done):** +- **`1911.00966` fully recovered** from the arXiv Atom API (OpenAlex 404'd on it): + title "A discrete version of Liouville's theorem on conformal maps", + Pinkall & Springborn, 2019, 384-char abstract → bibkey `PinkallSpringborn2019` + auto-generated, abstract embedded (non-zero). It was the worst node in the + corpus — now `@cite`-able, in wiki grounding, and its **10 chunks are visible + to chunk search** (previously filtered out by `bibkey IS NOT NULL`). Citations + were already added in DQ-1 (27 S2 refs). +- **`10.1007/s00454-019-00132-8` abstract recovered from S2** (1186 chars) and + re-embedded (targeted `UPDATE`, not full re-ingest — see DQ-5). +- **`10.1007/978-3-642-17413-1_7` (book chapter): documented inherent limit.** + No abstract in OpenAlex, S2, **or Crossref** (verified). Left as the single + remaining zero-vector paper; user may add an abstract by hand. +- **Corpus now: 1 paper without abstract** (the book chapter), down from 3+1. + Zero-vector search pollution (DQ-4 secondary finding) reduced from 2 tail + fillers to 1. +- **Wired into ingest** so this self-heals: OpenAlex-404 on an arXiv id now + recovers metadata via `arxiv.fetch_metadata` (new); an empty abstract is + supplemented via `_recover_abstract` → S2 then Crossref (new + `codex/sources/crossref.py`). New `semanticscholar.fetch_abstract`. +- **Files touched:** `codex/sources/arxiv.py`, `codex/sources/crossref.py` (new), + `codex/sources/semanticscholar.py`, `codex/ingest.py`, + tests. Full suite: 342. + +**Original finding (for context):** - **Measured:** - **No abstract (3):** `10.1007/978-3-642-17413-1_7`, `10.1007/s00454-019-00132-8`, `1911.00966`. These get a **zero-vector** @@ -187,6 +215,30 @@ is fine (monotonic); only the absolute score label is misleading. - **Acceptance:** a short relevance table (query → top-5 → on/off-topic) good enough to trust the KB for lookups, or a list of failure modes to fix. +### DQ-5 — `ingest_paper` is NOT idempotent for DOI papers · **HIGH (blocks R-A/R-C)** · found 2026-06-16 +- **Symptom:** re-ingesting an existing DOI paper crashes with + `UniqueViolation: papers_openalex_id_key`. Surfaced trying to re-ingest + `10.1007/s00454-019-00132-8` during the DQ-2 backfill. +- **Root cause:** `openalex._map_paper` sets `Paper.id = data["doi"]`, which is the + **full URL** `https://doi.org/10.1007/…`. The stored canonical id is the **bare** + `10.1007/…` (post the M-1 canonical-id migration). So `INSERT … ON CONFLICT (id)` + finds no id match, attempts a fresh INSERT, and trips the *separate* unique + constraint on `openalex_id` (already held by the canonical-id row). +- **Confirmed:** `openalex.fetch_paper("10.1007/s00454-019-00132-8").id == + "https://doi.org/10.1007/s00454-019-00132-8"` (≠ stored bare id). +- **Impact:** any re-ingest of a DOI paper aborts. This **blocks roadmap R-A + (GROBID on PDFs) and R-C (.tex ingest)** — both re-ingest existing papers. Also + implies *new* DOI ingests store full-URL ids, inconsistent with the bare-id + corpus (latent split-identity risk). +- **Proposed fix (handle with care — M-1 incident territory):** normalize the DOI + in `_map_paper` to the canonical bare, lower-cased form (strip + `https://doi.org/`), matching what the M-1 migration produced. Add a re-ingest + idempotency test over a DOI paper. Did **not** fix inline this session — the + canonical-id derivation is exactly where the M-1 migration incident occurred, so + it needs its own focused change + a live re-ingest check. +- **Workaround used for DQ-2:** targeted `UPDATE … SET abstract, abstract_emb` + instead of full re-ingest. + --- ## Roadmap — data-acquisition levers (post-audit, all free sources) @@ -200,6 +252,8 @@ Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each ite is self-contained so a cold session can pick it up. ### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort** +- **⚠ Blocked by DQ-5:** re-ingesting existing DOI papers currently crashes + (`papers_openalex_id_key`). Fix DQ-5 first, or the GROBID re-ingest aborts. - **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper so `codex.parsing.grobid.extract_references` parses each bibliography. @@ -268,18 +322,21 @@ is self-contained so a cold session can pick it up. coverage; remedy wired into ingest). 2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance verified strong; surfaced the zero-vector pollution that motivates DQ-2). -3. **DQ-2** (NEXT — recover `1911.00966`; backfill 3 missing abstracts so they - stop polluting paper-level search with zero-vectors), then **DQ-3** (chunk-vs- - source `.txt` fidelity). -4. **Roadmap levers** (see section above), best sequenced after DQ-2/DQ-3: - **R-A** (GROBID on arXiv PDFs — closes the last 2 zero-citation theses, highest - ROI), then **R-B** (Crossref source — also recovers DQ-2 abstracts), **R-C** - (`.tex` ingest — serves DQ-3), **R-D** (F-15 coverage warning, quick). - **R-E** (paywall) is deferred. +3. ~~**DQ-2**~~ — **DONE 2026-06-16** (1911.00966 fully recovered; s00454 abstract + from S2; book chapter documented; recovery wired into ingest). +4. **DQ-5** (NEXT — `ingest_paper` not idempotent for DOI papers; **blocks R-A/R-C**; + careful — M-1 canonical-id territory), then **DQ-3** (chunk-vs-source fidelity). +5. **Roadmap levers** (see section above): **R-A** (GROBID on arXiv PDFs — closes + the last 2 zero-citation theses; needs DQ-5 first), **R-B** (Crossref refs — + abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3; + needs DQ-5 first), **R-D** (F-15 coverage warning, quick). **R-E** (paywall) + deferred. -DQ-1..DQ-4 are *data*/assessment work; the roadmap items R-A..R-D add **new code** -(re-ingest scripts, a Crossref source module, a `.tex` path, a graph warning) on -top of the already-remediated pipeline (see the AUDIT-* docs and PRs #12–#14). +DQ-1..DQ-4 are *data*/assessment work; DQ-5 + roadmap R-A..R-D add **new code** +(canonical-id fix, re-ingest scripts, Crossref references, a `.tex` path, a graph +warning) on top of the already-remediated pipeline (see the AUDIT-* docs and PRs +#12–#14). The DQ-1/DQ-2 ingest wiring + Crossref/arXiv abstract recovery already +landed this session. --- diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 6aad09b..5729466 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -250,7 +250,7 @@ def test_ingest_paper_idempotent() -> None: def test_ingest_paper_arxiv_s2_fallback() -> None: - """If OpenAlex returns None for an arXiv ID, S2 fallback produces a stub Paper.""" + """OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted.""" paper_id = "2301.07041" mock_conn = MagicMock() mock_conn.execute = MagicMock() @@ -259,17 +259,84 @@ def test_ingest_paper_arxiv_s2_fallback() -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=None), + # arXiv + abstract sources also empty → genuine stub fallback path. + patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv, + patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None), + patch("codex.ingest.crossref.fetch_abstract", return_value=None), patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2, patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), ): result = ingest_paper(paper_id) - # S2 was consulted as fallback + # arXiv metadata recovery attempted; S2 references consulted as fallback. + mock_arxiv.assert_called_once_with(paper_id) mock_s2.assert_called() assert result.paper_id == paper_id +def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None: + """DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API, + bibkey auto-generated, abstract embedded (no zero vector).""" + recovered = Paper( + id="1911.00966", + title="A discrete version of Liouville's theorem on conformal maps", + authors=["Ulrich Pinkall", "Boris Springborn"], + year=2019, + abstract="Liouville's theorem says that in dimension greater than two...", + ) + fake_emb = _fake_embedder() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=None), + patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv, + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=fake_emb), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper("1911.00966") + + mock_arxiv.assert_called_once_with("1911.00966") + assert result.paper_id == "1911.00966" + params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert params["title"].startswith("A discrete version") + assert params["bibkey"] == "PinkallSpringborn2019" # generated from recovered authors+year + # abstract present → real embedding computed, not a zero vector + fake_emb.encode_dense.assert_called_once() + + +def test_ingest_paper_empty_abstract_recovered_from_s2() -> None: + """DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding.""" + paper = _make_paper(abstract=None) # has openalex_id, no abstract + fake_emb = _fake_embedder() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch( + "codex.ingest.semanticscholar.fetch_abstract", + return_value="Recovered abstract text.", + ) as mock_abs, + patch("codex.ingest.get_embedder", return_value=fake_emb), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + ingest_paper(paper.id) + + mock_abs.assert_called_once() + params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert params["abstract"] == "Recovered abstract text." + fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."]) + + def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None: """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2, rewriting citing_id to paper.id and lowercasing DOI cited-ids.""" @@ -325,6 +392,9 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # No abstract recoverable from any source (DQ-2) → genuine zero-vector case. + patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None), + patch("codex.ingest.crossref.fetch_abstract", return_value=None), patch("codex.ingest.get_embedder", return_value=fake_emb), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), ): diff --git a/tests/sources/test_arxiv.py b/tests/sources/test_arxiv.py index 1c08500..c4ceb60 100644 --- a/tests/sources/test_arxiv.py +++ b/tests/sources/test_arxiv.py @@ -138,3 +138,63 @@ def test_fetch_pdf_url_pure_computation() -> None: url2 = arxiv.fetch_pdf_url("1234.56789") assert url2 == "https://arxiv.org/pdf/1234.56789.pdf" + + +# --------------------------------------------------------------------------- +# fetch_metadata — parse the Atom feed into a Paper (DQ-2) +# --------------------------------------------------------------------------- + +_ATOM_FEED = """ + + + A discrete version of Liouville's theorem on conformal maps + Liouville's theorem says that in dimension greater than two, + all conformal maps are Moebius transformations. + 2019-11-03T00:00:00Z + Ulrich Pinkall + Boris Springborn + +""" + +_ATOM_EMPTY = """ +ArXiv Query""" + + +def test_fetch_metadata_parses_entry(monkeypatch: pytest.MonkeyPatch) -> None: + """A valid Atom feed yields a populated Paper.""" + + def mock_get(url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", mock_get) + + paper = arxiv.fetch_metadata("1911.00966") + assert paper is not None + assert paper.id == "1911.00966" + assert paper.title.startswith("A discrete version of Liouville") + assert paper.authors == ["Ulrich Pinkall", "Boris Springborn"] + assert paper.year == 2019 + assert paper.abstract and "conformal maps" in paper.abstract + assert paper.openalex_id is None + + +def test_fetch_metadata_no_entry_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A feed with no (id not found) returns None.""" + + def mock_get(url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(200, text=_ATOM_EMPTY, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", mock_get) + assert arxiv.fetch_metadata("0000.00000") is None + + +def test_fetch_metadata_strips_arxiv_prefix(monkeypatch: pytest.MonkeyPatch) -> None: + """An ``arXiv:`` prefix is stripped so Paper.id is the bare id.""" + + def mock_get(url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", mock_get) + paper = arxiv.fetch_metadata("arXiv:1911.00966") + assert paper is not None + assert paper.id == "1911.00966" diff --git a/tests/sources/test_crossref.py b/tests/sources/test_crossref.py new file mode 100644 index 0000000..f88d007 --- /dev/null +++ b/tests/sources/test_crossref.py @@ -0,0 +1,50 @@ +"""Tests for codex.sources.crossref.""" + +from __future__ import annotations + +import httpx +import pytest + +from codex.sources import crossref + + +def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None: + """A JATS-tagged abstract is returned as plain text with tags removed.""" + body = { + "message": { + "abstract": "We prove a theorem about " + "polyhedra." + } + } + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json=body) + + monkeypatch.setattr(crossref, "_get", mock_get) + + result = crossref.fetch_abstract("10.1007/s00454-019-00132-8") + assert result == "We prove a theorem about polyhedra." + + +def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A work with no abstract field (e.g. a book chapter) returns None.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}}) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None + + +def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A 404 returns None rather than raising.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + raise httpx.HTTPStatusError( + "404", + request=httpx.Request("GET", url), + response=httpx.Response(404, request=httpx.Request("GET", url)), + ) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_abstract("10.0000/nonexistent") is None diff --git a/tests/sources/test_semanticscholar.py b/tests/sources/test_semanticscholar.py index 42066d6..a1fb726 100644 --- a/tests/sources/test_semanticscholar.py +++ b/tests/sources/test_semanticscholar.py @@ -124,3 +124,46 @@ def test_fetch_recommendations_404_returns_empty( result = semanticscholar.fetch_recommendations("nonexistent") assert result == [] + + +# --------------------------------------------------------------------------- +# fetch_references — {"data": null} guard (DQ-1 regression) +# --------------------------------------------------------------------------- + + +def test_fetch_references_null_data_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no + parsed references; this must yield [] rather than raising TypeError.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"data": None, "citingPaperInfo": {}}) + + monkeypatch.setattr(semanticscholar, "_get", mock_get) + assert semanticscholar.fetch_references("DOI:10.14279/depositonce-5415") == [] + + +# --------------------------------------------------------------------------- +# fetch_abstract (DQ-2) +# --------------------------------------------------------------------------- + + +def test_fetch_abstract_returns_text(monkeypatch: pytest.MonkeyPatch) -> None: + """fetch_abstract returns the abstract string when present.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"abstract": "We study ideal hyperbolic polyhedra."}) + + monkeypatch.setattr(semanticscholar, "_get", mock_get) + assert semanticscholar.fetch_abstract("DOI:10.1007/s00454-019-00132-8") == ( + "We study ideal hyperbolic polyhedra." + ) + + +def test_fetch_abstract_null_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A null/absent abstract yields None.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"abstract": None}) + + monkeypatch.setattr(semanticscholar, "_get", mock_get) + assert semanticscholar.fetch_abstract("arXiv:1234.5678") is None From fa43f69dfdbe3c54eef8c6ab1b3e2fd569eaeaf4 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:23:16 +0200 Subject: [PATCH 04/26] =?UTF-8?q?docs(audit):=20DQ-3=20content=20fidelity?= =?UTF-8?q?=20verified=20clean=20=E2=80=94=20assessment=20axis=20complete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-ran chunk_text + is_quality_chunk on all 29 source .txt files vs stored chunks. Coverage 109-114% everywhere (the >100% is the 64/512-word overlap, i.e. full retention); stored==kept for every paper. Only 18 chunks dropped corpus-wide, all low_alpha = garbled OCR tables in 2 scanned-book sources (0-387-29555-0_13, depositonce-5415) — no real math content lost. Head/tail aligned (no truncation). Closes the data-quality assessment axis: DQ-1 (citations), DQ-2 (metadata), DQ-3 (fidelity), DQ-4 (retrieval) all resolved. DQ-5 (re-ingest idempotency) remains, tracked separately. Co-Authored-By: Claude Opus 4.8 --- docs/audit/DATA-QUALITY-2026-06-15.md | 38 ++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 81b8a4c..1d46ea4 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -4,8 +4,10 @@ applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is strong after fixing a P0 `codex search paper` crash. **DQ-2 RESOLVED 2026-06-16** (1911.00966 fully recovered from arXiv; s00454 abstract from S2; book chapter is a -documented limit; recovery wired into ingest). **DQ-3 still open. New: DQ-5** -(`ingest_paper` not idempotent for DOI papers — blocks R-A/R-C). +documented limit; recovery wired into ingest). **DQ-3 RESOLVED 2026-06-16** +(content fidelity clean — full coverage, only OCR junk dropped). **All four +assessment items (DQ-1..DQ-4) DONE. New: DQ-5** (`ingest_paper` not idempotent +for DOI papers — blocks R-A/R-C, tracked separately). **Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section). **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory of the audit conversation — everything needed to continue is in this file. @@ -157,7 +159,29 @@ incomplete, or unrepresentative data. This document is that second axis — a - **Acceptance:** every paper has at least a bibkey + non-zero abstract embedding, or the exceptions are documented with rationale. -### DQ-3 — Content fidelity (chunks vs source `.txt`): NOT YET VERIFIED · **MED** +### DQ-3 — Content fidelity (chunks vs source `.txt`): VERIFIED CLEAN · **RESOLVED 2026-06-16** + +**Resolution (read-only; re-ran `chunk_text` + `is_quality_chunk` on all 29 +source files and compared to stored chunks):** +- **No paper is silently gutted.** Coverage (stored chunk chars / source chars) + is **109–114% for every paper** — the >100% is exactly the 64/512-word overlap + inflation, i.e. full retention. `stored == kept` for all 29, so the live DB + matches what the current pipeline produces. +- **Only 18 chunks dropped corpus-wide, all `low_alpha`** (0 short, 0 bib), + concentrated in `10.1007/0-387-29555-0_13` (6) and `depositonce-5415` (10). + Inspected: they are **garbled OCR tables** from scanned-book sources (alpha + 0.30–0.36, e.g. `"(u: 1) 7 < u D:= ud3424 — else H®: out.v. Family 1…"`) — + exactly the artefacts the alpha-ratio gate targets. **No real math content lost** + (the handoff's feared failure mode did not occur; math prose stays > 0.40 alpha). +- **Head/tail alignment exact** on sampled papers: chunk 0 begins at the source + head (title/authors/abstract captured), last chunk ends at the source tail → no + front/back truncation. +- **Note:** 0 bibliography drops across the corpus → the upstream `.txt` extraction + had already stripped reference lists, which is why DQ-1's citation graph relies + on the API/GROBID sources rather than text-mined refs. +- **Acceptance met** for all 29 papers (not just a sample). No action needed. + +**Original open question (for context):** - **Open question:** does the stored chunk set faithfully reconstruct each source file, or did the chunker / `filter_chunks` silently drop material (e.g. an abstract, a section, math-heavy passages)? The whole corpus was ingested from @@ -324,9 +348,11 @@ is self-contained so a cold session can pick it up. verified strong; surfaced the zero-vector pollution that motivates DQ-2). 3. ~~**DQ-2**~~ — **DONE 2026-06-16** (1911.00966 fully recovered; s00454 abstract from S2; book chapter documented; recovery wired into ingest). -4. **DQ-5** (NEXT — `ingest_paper` not idempotent for DOI papers; **blocks R-A/R-C**; - careful — M-1 canonical-id territory), then **DQ-3** (chunk-vs-source fidelity). -5. **Roadmap levers** (see section above): **R-A** (GROBID on arXiv PDFs — closes +4. ~~**DQ-3**~~ — **DONE 2026-06-16** (content fidelity clean; full coverage, only + 18 OCR-junk chunks dropped, no math lost). **Assessment axis DQ-1..DQ-4 complete.** +5. **DQ-5** (NEXT — `ingest_paper` not idempotent for DOI papers; **blocks R-A/R-C**; + careful — M-1 canonical-id territory). +6. **Roadmap levers** (see section above): **R-A** (GROBID on arXiv PDFs — closes the last 2 zero-citation theses; needs DQ-5 first), **R-B** (Crossref refs — abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3; needs DQ-5 first), **R-D** (F-15 coverage warning, quick). **R-E** (paywall) From ea73b1475c202caa19b109aa835487b46d7774b2 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:31:01 +0200 Subject: [PATCH 05/26] fix(openalex): normalize DOI to bare canonical id so re-ingest is idempotent (DQ-5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openalex._map_paper set Paper.id from the raw `doi` field, which OpenAlex returns as a full URL (https://doi.org/10.x). The canonical papers.id (M-1 migration) is the bare, lower-cased DOI, so re-ingesting a DOI paper missed ON CONFLICT (id) and tripped the separate papers_openalex_id_key constraint (UniqueViolation). Fix: _normalize_doi() strips https://doi.org//http://doi.org//doi: and lower-cases; applied in the DOI branch of _map_paper only (W-id fallback untouched). Also un-breaks _s2_id_for/Crossref recovery, which the URL form silently defeated. Corrected the _SAMPLE_WORK fixture to the URL form (the bare fixture hid the bug) + added _normalize_doi and re-ingest idempotency regression tests. Live verification: re-ingest of 10.1007/s00454-019-00132-8 (previously crashed) now UPDATEs in place — single row, added_at unchanged, no stray row, citations 45->45. Unblocks roadmap R-A/R-C. Full suite 348 green. Co-Authored-By: Claude Opus 4.8 --- codex/sources/openalex.py | 23 +++++++++- docs/audit/DATA-QUALITY-2026-06-15.md | 61 +++++++++++++++++++++------ tests/ingest/test_ingest.py | 58 +++++++++++++++++++++++++ tests/sources/test_openalex.py | 54 +++++++++++++++++++++++- 4 files changed, 181 insertions(+), 15 deletions(-) diff --git a/codex/sources/openalex.py b/codex/sources/openalex.py index 80b5307..f9be139 100644 --- a/codex/sources/openalex.py +++ b/codex/sources/openalex.py @@ -85,13 +85,34 @@ def _reconstruct_abstract(inverted_index: dict[str, list[int]] | None) -> str | return " ".join(w for _, w in word_positions) +def _normalize_doi(doi: str) -> str: + """Normalize an OpenAlex DOI to the canonical bare, lower-cased form. + + OpenAlex returns the ``doi`` field as a full URL + (``https://doi.org/10.1007/s00454-019-00132-8``), but the canonical + ``papers.id`` produced by the M-1 migration is the BARE, lower-cased DOI + (``10.1007/s00454-019-00132-8``). Stripping the ``https://doi.org/`` / + ``doi:`` prefix and lower-casing (DOIs are case-insensitive by spec) makes a + re-ingested DOI paper match its stored row, so the ingest + ``INSERT … ON CONFLICT (id)`` updates in place instead of attempting a fresh + INSERT that trips the separate ``papers_openalex_id_key`` unique constraint + (DQ-5). Mirrors the cited-id convention in ``ingest._norm_cited_id``. + """ + s = doi.strip().lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if s.startswith(prefix): + return s[len(prefix) :] + return s + + def _map_paper(data: dict[str, Any]) -> Paper: authors: list[str] = [ a.get("author", {}).get("display_name") or "" for a in data.get("authorships", []) ] abstract = _reconstruct_abstract(data.get("abstract_inverted_index")) + doi = data.get("doi") return Paper( - id=data.get("doi") or data.get("id") or "", + id=_normalize_doi(doi) if doi else (data.get("id") or ""), title=data.get("title") or "", openalex_id=data.get("id"), authors=authors, diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 1d46ea4..88d62ee 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -6,8 +6,9 @@ strong after fixing a P0 `codex search paper` crash. **DQ-2 RESOLVED 2026-06-16* (1911.00966 fully recovered from arXiv; s00454 abstract from S2; book chapter is a documented limit; recovery wired into ingest). **DQ-3 RESOLVED 2026-06-16** (content fidelity clean — full coverage, only OCR junk dropped). **All four -assessment items (DQ-1..DQ-4) DONE. New: DQ-5** (`ingest_paper` not idempotent -for DOI papers — blocks R-A/R-C, tracked separately). +assessment items (DQ-1..DQ-4) DONE. DQ-5 RESOLVED 2026-06-17** (DOI id normalized +to bare canonical form in `openalex._map_paper`; live re-ingest of +`10.1007/s00454-019-00132-8` confirmed idempotent → **unblocks R-A/R-C**). **Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section). **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory of the audit conversation — everything needed to continue is in this file. @@ -239,7 +240,43 @@ is fine (monotonic); only the absolute score label is misleading. - **Acceptance:** a short relevance table (query → top-5 → on/off-topic) good enough to trust the KB for lookups, or a list of failure modes to fix. -### DQ-5 — `ingest_paper` is NOT idempotent for DOI papers · **HIGH (blocks R-A/R-C)** · found 2026-06-16 +### DQ-5 — `ingest_paper` is NOT idempotent for DOI papers · **RESOLVED 2026-06-17** (found 2026-06-16) + +**Resolution (what was done):** +- **Root-cause fix, minimal + contained:** added `openalex._normalize_doi()` and + applied it in `_map_paper` so the DOI branch yields the canonical **bare, + lower-cased** form (strips `https://doi.org/` / `http://doi.org/` / `doi:`; + lower-cases — DOIs are case-insensitive, matching `ingest._norm_cited_id`). The + no-DOI fallback (OpenAlex W-id URL) is left untouched. `_map_paper`→`fetch_paper` + →`ingest_paper` is the only consumer chain, so blast radius is one function. +- **Checked nothing depended on the URL form.** It was the opposite: the full-URL + id silently *defeated* `_s2_id_for(paper.id)` and `_recover_abstract`'s + `paper.id.startswith("10.")` (a `https://…` string is neither `10.`- nor + `doi:`-prefixed), so S2/Crossref recovery never fired for DOI papers. The fix + repairs those paths too. +- **Tests (offline, mocked DB):** closed audit **T-3** — the OpenAlex fixture now + emits the real URL-form DOI and asserts `paper.id` is the bare form; added + `_normalize_doi` unit tests (https/http/`doi:`/upper-case/already-bare) and a + W-id-fallback test; added ingest regression `test_ingest_doi_paper_upserts_bare + _canonical_id` (runs the real `_map_paper` through ingest, asserts the papers + upsert carries the **bare** id into `%(id)s` and uses `ON CONFLICT (id)`). + Verified it has teeth (fails on the pre-fix code). Full suite **348 passed**; + ruff + mypy --strict clean. +- **Live idempotency confirmed (tunnel up):** re-ingested + `10.1007/s00454-019-00132-8` against the Jetson DB → no `UniqueViolation`; + `IngestResult(paper_id='10.1007/s00454-019-00132-8', citations_upserted=45)`. + After-state: **UPDATE in place** — `id` and `openalex_id` unchanged, **`added_at` + unchanged** (proves no fresh INSERT), exactly **1** row for the DOI (no URL-form + duplicate), citations 45→45, total papers 29→29. +- **Unblocks R-A (GROBID re-ingest) and R-C (.tex re-ingest)** — both re-ingest + existing papers. **Files touched:** `codex/sources/openalex.py`, + `tests/sources/test_openalex.py`, `tests/ingest/test_ingest.py`. +- **Note (out of scope, pre-existing):** arXiv papers stored as bare ids + (`2305.10988`) are still not round-trip idempotent if OpenAlex returns an + arXiv-DOI/W-id instead of the bare arXiv id — a separate canonicalization + concern, not this UniqueViolation. The DOI path (this finding) is fixed. + +**Original finding (for context):** - **Symptom:** re-ingesting an existing DOI paper crashes with `UniqueViolation: papers_openalex_id_key`. Surfaced trying to re-ingest `10.1007/s00454-019-00132-8` during the DQ-2 backfill. @@ -276,8 +313,8 @@ Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each ite is self-contained so a cold session can pick it up. ### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort** -- **⚠ Blocked by DQ-5:** re-ingesting existing DOI papers currently crashes - (`papers_openalex_id_key`). Fix DQ-5 first, or the GROBID re-ingest aborts. +- **✓ Unblocked (DQ-5 RESOLVED 2026-06-17):** DOI re-ingest is now idempotent, so + the GROBID re-ingest no longer aborts on `papers_openalex_id_key`. - **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper so `codex.parsing.grobid.extract_references` parses each bibliography. @@ -350,13 +387,13 @@ is self-contained so a cold session can pick it up. from S2; book chapter documented; recovery wired into ingest). 4. ~~**DQ-3**~~ — **DONE 2026-06-16** (content fidelity clean; full coverage, only 18 OCR-junk chunks dropped, no math lost). **Assessment axis DQ-1..DQ-4 complete.** -5. **DQ-5** (NEXT — `ingest_paper` not idempotent for DOI papers; **blocks R-A/R-C**; - careful — M-1 canonical-id territory). -6. **Roadmap levers** (see section above): **R-A** (GROBID on arXiv PDFs — closes - the last 2 zero-citation theses; needs DQ-5 first), **R-B** (Crossref refs — - abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3; - needs DQ-5 first), **R-D** (F-15 coverage warning, quick). **R-E** (paywall) - deferred. +5. ~~**DQ-5**~~ — **DONE 2026-06-17** (DOI id normalized to bare form in + `openalex._map_paper`; live re-ingest of `10.1007/s00454-019-00132-8` idempotent + — UPDATE in place, `added_at` unchanged; suite 348 green). **Unblocks R-A/R-C.** +6. **Roadmap levers** (see section above — now all unblocked): **R-A** (GROBID on + arXiv PDFs — closes the last 2 zero-citation theses), **R-B** (Crossref refs — + abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3), + **R-D** (F-15 coverage warning, quick). **R-E** (paywall) deferred. DQ-1..DQ-4 are *data*/assessment work; DQ-5 + roadmap R-A..R-D add **new code** (canonical-id fix, re-ingest scripts, Crossref references, a `.tex` path, a graph diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 5729466..5011fbf 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -10,6 +10,7 @@ from contextlib import contextmanager from typing import Any from unittest.mock import MagicMock, patch +import httpx import numpy as np import pytest @@ -592,6 +593,63 @@ def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None: assert upsert_params["bibkey"] == "AliceBob2023" +def test_ingest_doi_paper_upserts_bare_canonical_id() -> None: + """DQ-5 regression: re-ingesting an existing DOI paper must be idempotent. + + OpenAlex returns the ``doi`` as a full URL (and possibly mixed-case), but the + stored canonical ``papers.id`` is the BARE, lower-cased DOI (M-1 migration). + The papers upsert must therefore carry the BARE id into ``%(id)s`` so + ``ON CONFLICT (id)`` matches the existing row and UPDATEs it — rather than + attempting a fresh INSERT that trips the separate ``papers_openalex_id_key`` + unique constraint (the UniqueViolation crash). + + The real ``_map_paper``/``fetch_paper`` run here (only the HTTP ``_get``, the + embedder and the DB are mocked), so this exercises the actual normalization + through the ingest path. The DB is a MagicMock, so it cannot raise the real + constraint error; the live re-ingest of 10.1007/s00454-019-00132-8 is the + end-to-end UniqueViolation proof (see DATA-QUALITY-2026-06-15.md, DQ-5). + """ + bare = "10.1007/s00454-019-00132-8" + work = { + "id": "https://openalex.org/W2899262408", + # URL form + an upper-case suffix char — the worst case for ON CONFLICT. + "doi": "https://doi.org/10.1007/S00454-019-00132-8", + "title": "A Test Paper", + "publication_year": 2019, + "authorships": [{"author": {"display_name": "Alice Smith"}}], + "abstract_inverted_index": {"Hello": [0], "world": [1]}, + # Non-empty so fetch_citations yields edges and the S2 fallback is skipped. + "referenced_works": ["https://openalex.org/W1"], + } + + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch( + "codex.ingest.openalex._get", + return_value=httpx.Response(200, json=work), + ), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(bare) + + # The IngestResult and the upsert must use the bare canonical id, not the URL. + assert result.paper_id == bare + + upsert_sql: str = mock_conn.execute.call_args_list[0][0][0] + upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert "INSERT INTO papers" in upsert_sql + assert "ON CONFLICT (id)" in upsert_sql + assert upsert_params["id"] == bare, ( + f"upsert id must be the bare canonical DOI '{bare}', " + f"not the OpenAlex URL form '{upsert_params['id']}'" + ) + + # --------------------------------------------------------------------------- diff --git a/tests/sources/test_openalex.py b/tests/sources/test_openalex.py index d481aa2..6b1e9a2 100644 --- a/tests/sources/test_openalex.py +++ b/tests/sources/test_openalex.py @@ -7,7 +7,7 @@ import pytest from codex.models import Citation, Paper from codex.sources import openalex -from codex.sources.openalex import _resolve_id +from codex.sources.openalex import _normalize_doi, _resolve_id # --------------------------------------------------------------------------- # Helpers @@ -15,7 +15,9 @@ from codex.sources.openalex import _resolve_id _SAMPLE_WORK = { "id": "https://openalex.org/W2741809807", - "doi": "10.1145/3592430", + # Real OpenAlex returns the DOI as a full URL (T-3: the old bare fixture hid + # the URL-form behaviour that DQ-5 is about). + "doi": "https://doi.org/10.1145/3592430", "title": "Conformal Prediction: A Review", "publication_year": 2023, "authorships": [ @@ -67,6 +69,51 @@ def test_resolve_prefixed_unchanged() -> None: assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x" +# --------------------------------------------------------------------------- +# _normalize_doi — canonical bare, lower-cased form (DQ-5) +# --------------------------------------------------------------------------- + + +def test_normalize_doi_strips_https_url() -> None: + # The exact form OpenAlex returns in the `doi` field. + assert _normalize_doi("https://doi.org/10.1007/s00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_strips_http_and_doi_prefix() -> None: + assert _normalize_doi("http://doi.org/10.1145/3592430") == "10.1145/3592430" + assert _normalize_doi("doi:10.1145/3592430") == "10.1145/3592430" + + +def test_normalize_doi_lowercases() -> None: + # DOIs are case-insensitive; the canonical id (and _norm_cited_id) is lower-case. + assert _normalize_doi("https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_already_bare_is_idempotent() -> None: + assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8" + + +def test_map_paper_falls_back_to_openalex_id_without_doi( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A work with no DOI keeps the OpenAlex W-id (URL form) as id, unchanged — + normalization only rewrites the DOI branch.""" + work = {k: v for k, v in _SAMPLE_WORK.items() if k != "doi"} + + def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response: + return httpx.Response(200, json=work) + + monkeypatch.setattr(openalex, "_get", mock_get) + paper = openalex.fetch_paper("W2741809807") + + assert paper is not None + assert paper.id == "https://openalex.org/W2741809807" + + # --------------------------------------------------------------------------- # fetch_paper — success # --------------------------------------------------------------------------- @@ -85,6 +132,9 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None: assert paper is not None assert isinstance(paper, Paper) + # DQ-5 / C-7: paper.id is the canonical BARE DOI, not the full URL OpenAlex + # returns. This is what makes the ingest ON CONFLICT (id) upsert idempotent. + assert paper.id == "10.1145/3592430" assert paper.title == "Conformal Prediction: A Review" assert paper.year == 2023 assert paper.authors == ["Alice Smith", "Bob Jones"] From b371119264093c4ab0a9e11794003e95ce016dcd Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:33:45 +0200 Subject: [PATCH 06/26] =?UTF-8?q?docs(audit):=20scope=20DQ-3=20to=20txt?= =?UTF-8?q?=E2=86=92chunks=20+=20open=20DQ-6=20(PDF=E2=86=92txt=20extracti?= =?UTF-8?q?on=20fidelity)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarify that DQ-3's 'VERIFIED CLEAN' covers the ingest pipeline (txt -> chunks) only — it treats the .txt as ground truth. The upstream PDF/source -> .txt extraction (done outside codex-py) is unmeasured and shown to be lossy (DQ-3's dropped chunks were garbled OCR tables already present in the .txt). Add DQ-6 to measure that layer; 36 source PDFs are available. Note R-A/R-C (re-ingest from PDF/.tex) would largely sidestep it. Co-Authored-By: Claude Opus 4.8 --- docs/audit/DATA-QUALITY-2026-06-15.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 88d62ee..c942325 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -181,6 +181,32 @@ source files and compared to stored chunks):** had already stripped reference lists, which is why DQ-1's citation graph relies on the API/GROBID sources rather than text-mined refs. - **Acceptance met** for all 29 papers (not just a sample). No action needed. +- **⚠ SCOPE LIMIT — DQ-3 covers `.txt → chunks` ONLY.** It treats each source + `.txt` as ground truth and verifies the *ingest pipeline* preserves it. It does + **not** measure the layer above — **PDF/source → `.txt`** — which was done by an + upstream tool in `ConformalLabpp` (outside codex-py) and is unexamined. There is + already evidence that layer is lossy: the 18 dropped chunks were garbled OCR + tables *already present in the `.txt`* (i.e. the PDF→txt step garbled the + table/formula regions), and the bibliographies were stripped upstream. So + "VERIFIED CLEAN" means **the loader faithfully preserves whatever the `.txt` + contains, not that no information was lost from the source paper.** See DQ-6. + +### DQ-6 — PDF/source → `.txt` extraction fidelity: NOT MEASURED · **MED, open** +- **Open question:** how much did the upstream PDF→`.txt` extraction (done in + `ConformalLabpp`, outside codex-py) drop or garble vs the original papers? + Math/equations, tables, multi-column layouts, and figure captions are the usual + casualties of PDF text extraction — and we have direct evidence of garble + (DQ-3's low_alpha drops were already-garbled tables in the `.txt`). +- **Checkable:** the 36 original PDFs are present in `…/ConformalLabpp/papers/` + (map each paper's `source_path` `.txt` to its sibling `.pdf` by filename stem). +- **To investigate:** for a sample (incl. the known-bad scanned book + `0-387-29555-0_13` and a born-digital arXiv PDF), extract PDF text independently + and compare length / page-coverage to the `.txt`; spot-check whether specific + theorem statements & equations survive. Flag papers where the `.txt` is missing + large spans. +- **Acceptance:** a per-sample source→txt coverage estimate, or a list of papers + whose `.txt` is materially degraded (candidates for re-extraction — note roadmap + **R-A/R-C** would re-ingest from PDF/`.tex` directly and largely sidestep this). **Original open question (for context):** - **Open question:** does the stored chunk set faithfully reconstruct each source From 0021859094b38ed04c666b696e37255c90d18261 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:39:04 +0200 Subject: [PATCH 07/26] fix(ingest): normalize GROBID-derived citation DOIs to bare lower-case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GROBID extracts a cited DOI verbatim from PDF reference text, so it can be mixed-case, a https://doi.org/ URL, or doi:-prefixed. The PDF citation branch inserted it unchanged, while every other path (S2 supplement, OpenAlex, papers.id itself) uses the bare lower-cased DOI. Once R-A runs GROBID reference extraction over arXiv PDFs, those un-normalized cited-ids would never equal a bare-lowercase papers.id/cited_id — dangling the cross-citations and splitting one reference into case-/URL-variant graph nodes. Extend _norm_cited_id to strip https://doi.org//http://doi.org//doi: prefixes and lower-case (mirroring openalex._normalize_doi), making it the single canonical cited-id normalizer, and route the GROBID branch through it. arXiv ids and S2 paperIds still pass through untouched. Add test_ingest_pdf_grobid_citations_normalize_doi covering mixed-case, URL-form, and doi:-prefixed DOIs plus an arXiv-only ref. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 25 +++++++++++--- tests/ingest/test_ingest.py | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index cd950e7..c0f768f 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -65,10 +65,20 @@ def _s2_id_for(pid: str) -> str | None: def _norm_cited_id(cited_id: str) -> str: - """Lowercase DOI cited-ids (DOIs are case-insensitive) so the graph join - does not fragment the same reference into distinct case-variant nodes. - arXiv ids and S2 paperIds are left untouched.""" - return cited_id.lower() if cited_id.startswith("10.") else cited_id + """Normalize a DOI cited-id to its canonical bare, lower-cased form. + + DOIs are case-insensitive and may arrive bare (``10.…``), as a + ``https://doi.org/…`` URL, or ``doi:…``-prefixed: GROBID emits whichever + form the PDF reference text used, while S2/OpenAlex hand back bare DOIs. + Stripping the prefix and lower-casing keeps the citation-graph join from + fragmenting the same reference into distinct case-/URL-variant nodes. arXiv + ids and S2 paperIds are left untouched. Mirrors ``openalex._normalize_doi``.""" + s = cited_id.strip() + lowered = s.lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if lowered.startswith(prefix): + return lowered[len(prefix) :] + return lowered if lowered.startswith("10.") else s def _recover_abstract(paper: Paper) -> str | None: @@ -268,7 +278,12 @@ def ingest_paper( for ref in grobid_refs: cited_id = ref.get("doi") or ref.get("arxiv_id") if cited_id: - pdf_citations.append(Citation(citing_id=paper.id, cited_id=cited_id)) + # Normalize the GROBID DOI (bare/URL/``doi:`` → bare lower-case) + # so PDF-derived edges join the same graph nodes as the + # S2/OpenAlex paths; arXiv ids pass through untouched. + pdf_citations.append( + Citation(citing_id=paper.id, cited_id=_norm_cited_id(cited_id)) + ) else: logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path) diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 5011fbf..936e6cb 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -199,6 +199,73 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: assert mock_cursor.executemany.call_count >= 2 +def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None: + """R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so + it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each + must be normalized to the bare, lower-cased canonical form before the Citation + is built — otherwise the same reference splits into case-/URL-variant graph + nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no + DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1).""" + pdf_file = tmp_path / "paper.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake content") + + paper = _make_paper() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]: + # Faithful to GROBID's output shape: all five keys always present. + return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id} + + grobid_refs = [ + _ref(doi="10.1007/S00454-019-00132-8"), # mixed-case bare DOI + _ref(doi="https://doi.org/10.1145/AbC.123"), # URL-form DOI + _ref(doi="doi:10.1/MixedCase"), # doi:-prefixed DOI + _ref(arxiv_id="2101.00001"), # arXiv-only ref → untouched + ] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID + # refs contribute to the citation rows under inspection. + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), + patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]), + patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), + patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + ): + result = ingest_paper(paper.id, source_path=str(pdf_file)) + + assert result.citations_upserted == 4 + + # Locate the citations INSERT among the cursor's executemany calls (chunks + # are also inserted via executemany, so filter by SQL rather than by index). + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + citation_calls = [ + c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0] + ] + assert len(citation_calls) == 1 + inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1] + + # Every edge hangs off the canonical paper.id (citing side). + assert all(row[0] == paper.id for row in inserted_rows) + + cited_ids = {row[1] for row in inserted_rows} + assert cited_ids == { + "10.1007/s00454-019-00132-8", # mixed-case → lower-cased + "10.1145/abc.123", # URL prefix stripped + lower-cased + "10.1/mixedcase", # doi: prefix stripped + lower-cased + "2101.00001", # arXiv id untouched + } + # No raw URL-form / prefixed DOI leaked into the citation graph. + assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids) + + # --------------------------------------------------------------------------- # 4. test_ingest_paper_not_found # --------------------------------------------------------------------------- From b55aa5b4295abf1cb9ca04d67fd8bab3e486d749 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 00:42:37 +0200 Subject: [PATCH 08/26] chore: gitignore generated wiki/index.md wiki/index.md is regenerated by `codex wiki compile` from wiki/concepts.yaml (the tracked source), so the generated index should not be version-controlled. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 278f7af..01eb5e5 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ htmlcov/ # Claude Code — settings.json IS committed (team-wide); only personal overrides ignored .claude/settings.local.json + +# Generated wiki index (regenerated by `codex wiki compile`; concepts.yaml is the source) +wiki/index.md From 1516684bbbb4bab796fae68c92646c1dea17881d Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 08:28:34 +0200 Subject: [PATCH 09/26] feat(grobid): R-A reference backfill + native arm64 GROBID on Jetson Enable roadmap R-A (GROBID reference extraction) end-to-end and run it against local Jetson infra instead of a Mac/Rosetta emulation. Backfill: - Add scripts/ra_grobid_backfill.py: references-only, idempotent citation backfill (dry-run default). Deliberately not ingest_paper(source_path=pdf), which re-OCRs and replaces the DQ-3-clean .txt chunks; this touches only the citations table (ON CONFLICT DO NOTHING). 7 tests. - Make extract_references timeout configurable (large theses exceed the 60s default, especially against a slower GROBID). Jetson infra: - Bump grobid/grobid 0.8.2 -> 0.9.0-crf. The -crf tag is the only GROBID variant published as an arm64 multi-arch manifest; every 0.8.x tag and the full deep-learning image are amd64-only, which is why GROBID never ran on the aarch64 Jetson. Enable the 4g memory cap + init/ulimits per GROBID docs. - Add docs/infra/grobid-jetson.md runbook: arm64 image rationale, the two host prerequisites (docker-group membership + the Compose v2 CLI plugin, both missing on the Jetson), service-scoped deploy, and the GET /api/isalive check. Verified live 2026-06-17: native arm64 pull, isalive=true, end-to-end extract_references on lutz-2024-thesis.pdf = 116 refs (101 with DOI/arXiv). docs(audit): mark R-A core done (citing coverage 27/29 -> 29/29; edges 920 -> 1022). Co-Authored-By: Claude Opus 4.8 --- codex/parsing/grobid.py | 6 +- docs/audit/DATA-QUALITY-2026-06-15.md | 28 ++- docs/infra/grobid-jetson.md | 163 +++++++++++++++++ infra/docker-compose.yml | 30 ++-- scripts/__init__.py | 1 + scripts/ra_grobid_backfill.py | 220 +++++++++++++++++++++++ tests/scripts/__init__.py | 0 tests/scripts/test_ra_grobid_backfill.py | 76 ++++++++ 8 files changed, 512 insertions(+), 12 deletions(-) create mode 100644 docs/infra/grobid-jetson.md create mode 100644 scripts/__init__.py create mode 100644 scripts/ra_grobid_backfill.py create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/test_ra_grobid_backfill.py diff --git a/codex/parsing/grobid.py b/codex/parsing/grobid.py index b2b5443..7644803 100644 --- a/codex/parsing/grobid.py +++ b/codex/parsing/grobid.py @@ -26,6 +26,7 @@ def _text(element: ET.Element | None) -> str: def extract_references( pdf_path: str, grobid_url: str | None = None, + timeout: float = 60.0, ) -> list[dict[str, str]]: """Extract a structured reference list from a PDF via GROBID. @@ -35,6 +36,9 @@ def extract_references( Path to the PDF file on disk. grobid_url: Base URL of the GROBID server. Defaults to ``Settings().grobid_url``. + timeout: + HTTP client timeout in seconds. Large PDFs (e.g. theses) can exceed the + 60s default, especially against a slower/emulated GROBID; raise it then. Returns ------- @@ -46,7 +50,7 @@ def extract_references( if grobid_url is None: grobid_url = Settings().grobid_url - with open(pdf_path, "rb") as fh, httpx.Client(timeout=60.0) as client: + with open(pdf_path, "rb") as fh, httpx.Client(timeout=timeout) as client: response = client.post( f"{grobid_url}/api/processReferences", files={"input": (pdf_path, fh, "application/pdf")}, diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index c942325..5719674 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -338,7 +338,33 @@ so the quality lever is *combining more free sources*, not buying paywall access Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item is self-contained so a cold session can pick it up. -### R-A — Run GROBID reference extraction on arXiv PDFs · **HIGH lever, LOW-MED effort** +### R-A — Run GROBID reference extraction on PDFs · **CORE DONE 2026-06-17** (full sweep optional) + +**Resolution (what was done) — citing coverage 27/29 → 29/29 (100%); edges 920 → 1022:** +- **References-only backfill, not full re-ingest.** Added + [scripts/ra_grobid_backfill.py](../../scripts/ra_grobid_backfill.py): per paper, + run `grobid.extract_references(pdf)`, normalize cited-ids to the canonical bare + form (DOIs de-URL'd + lower-cased; `arXiv:` stripped), INSERT into `citations` + (`ON CONFLICT DO NOTHING`). Deliberately avoids `ingest_paper(source_path=pdf)`, + whose PDF branch re-runs Nougat + `DELETE FROM chunks` and would overwrite the + DQ-3-verified-clean `.txt` chunks (and needs Nougat). Idempotent; dry-run by + default; 7 tests; ruff/mypy clean. +- **GROBID provisioning:** the pinned `grobid/grobid:0.8.2` is amd64-only, but the + Jetson and the dev Mac are both arm64 (this is why GROBID never ran). Ran the + lighter CRF image (`grobid/grobid:0.8.2-crf`) on the Mac under Podman+Rosetta + (PDFs are local; citations written over the SSH DB tunnel). Made + `extract_references` timeout configurable (theses need >60s emulated). A separate + task is provisioning a native arm64 GROBID on the Jetson (`docs/infra/grobid-jetson.md`). +- **Both `depositonce` theses closed:** `…depositonce-20357` (lutz-2024) **+96** + edges (5 in-corpus); `…depositonce-5415` (sechelmann-2016) **+6** edges — its + 147 MB/173-page scan blew pdfalto's 120s limit, so the 9 bibliography pages were + extracted with PyMuPDF first (77 refs, only 6 with a DOI/arXiv). **No paper has + zero out-edges now.** +- **Optional remaining (secondary acceptance):** sweep the other 27 already-covered + papers for *extra* GROBID refs via `python scripts/ra_grobid_backfill.py --write` + (the 2 book PDFs need the same bib-page extraction as sechelmann). + +**Original plan (for context):** - **✓ Unblocked (DQ-5 RESOLVED 2026-06-17):** DOI re-ingest is now idempotent, so the GROBID re-ingest no longer aborts on `papers_openalex_id_key`. - **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), diff --git a/docs/infra/grobid-jetson.md b/docs/infra/grobid-jetson.md new file mode 100644 index 0000000..4e6a0c6 --- /dev/null +++ b/docs/infra/grobid-jetson.md @@ -0,0 +1,163 @@ +# GROBID on the Jetson (aarch64) — deploy runbook + +**Status:** ACTIVE +**Host:** `alfred@192.168.178.103` — NVIDIA Jetson Orin Nano (aarch64, 8 GB RAM, 8 GB swap, 6 cores) +**Service:** GROBID reference extraction for roadmap **R-A** (`scripts/ra_grobid_backfill.py`) +**Last updated:** 2026-06-17 +**Verified:** 2026-06-17 — `grobid/grobid:0.9.0-crf` pulled as native `arm64`, `/api/isalive`→`true`, +end-to-end `extract_references` on `lutz-2024-thesis.pdf` → 116 refs (101 with DOI/arXiv) in ~36s. + +--- + +## Why this image (`grobid/grobid:0.9.0-crf`) + +GROBID had never come up on the Jetson because `infra/docker-compose.yml` pinned +`grobid/grobid:0.8.2`, and **every `0.8.x` tag (and the full `0.9.0`) is published +amd64-only**. A Docker tag resolves to a multi-arch *manifest list*; on aarch64 +the `0.8.2` list has no `linux/arm64` entry, so there is simply nothing to pull or +run (hence `curl localhost:8070` → connection refused, nothing listening). + +The fix is to select a tag whose manifest list actually contains a `linux/arm64` +build: + +| Candidate | arm64? | Size | Notes | +|-------------------------------|:------:|--------:|----------------------------------------| +| `grobid/grobid:0.8.2` | ✗ | ~9.5 GB | full DL image, amd64-only (old pin) | +| `grobid/grobid:0.9.0` / `-full`| ✗ | ~10 GB | full DL image, amd64-only | +| **`grobid/grobid:0.9.0-crf`** | **✓** | ~500 MB | **CRF-only, official, multi-arch** ← | +| `lfoppiano/grobid:0.9.0-crf` | ✓ | ~500 MB | same build, community namespace (alt.) | + +We use **`grobid/grobid:0.9.0-crf`**: + +- **CRF-only** is sufficient for reference extraction (`/api/processReferences`) + and is ~500 MB vs ~10 GB for the deep-learning image. +- **Native arm64** — no qemu emulation (too slow / RAM-heavy on a Jetson). +- **Official namespace**, **pinned version** (not `latest-crf`, which is a moving + target). arm64 builds only exist from `0.9.0-crf` onward — there is no arm64 + `0.8.x`, so this is a forced (but compatible) bump from the old `0.8.2` pin. The + TEI that `codex/parsing/grobid.py` parses (`biblStruct`, `persName`, + `idno[@type='DOI']`, …) is unchanged across the bump. + +> Caveat from upstream: the arm64 image is documented as "tested only on macOS, +> not linux/arm64". The container is the *same* `linux/arm64` ELF either way, so +> bare-metal Jetson works; the end-to-end check below is what de-risks it. + +--- + +## One-time prerequisites + +### 1. Docker permission for `alfred` + +The SSH user `alfred` is in the `sudo` group but **not** the `docker` group, so +`docker …` fails with `permission denied … /var/run/docker.sock`. Grant access +once (requires the sudo password): + +```bash +ssh alfred@192.168.178.103 +sudo usermod -aG docker alfred # add to docker group (persistent) +``` + +Then **start a fresh login** so the new group takes effect (group membership is +only re-evaluated at login — an existing session/`ssh` won't see it; a *new* +`ssh` connection will): + +```bash +exit && ssh alfred@192.168.178.103 +docker ps # should now work without sudo +``` + +(Alternatively run all `docker` commands below under `sudo`, but group membership +is cleaner and lets `ra_grobid_backfill.py` / curl checks be driven over plain SSH.) + +### 2. Docker Compose v2 plugin (arm64) + +This host has Docker Engine but **not** the Compose v2 CLI plugin (on Linux they +are separate packages; only Docker Desktop bundles Compose). Without it +`docker compose …` fails with `docker: unknown command: docker compose`. Install +the arm64 plugin into the user-local plugin dir (no sudo needed): + +```bash +mkdir -p ~/.docker/cli-plugins +curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-aarch64 \ + -o ~/.docker/cli-plugins/docker-compose +chmod +x ~/.docker/cli-plugins/docker-compose +docker compose version # confirm it resolves +``` + +--- + +## Deploy + +GROBID is **service-scoped** on purpose: Postgres (`papers-db`) is already running +on this host, so we bring up only the `grobid` service and never `docker compose up` +the whole stack (that would try to start a second `papers-db`). + +```bash +# copy this repo's compose file to the Jetson (first time only) +scp infra/docker-compose.yml infra/schema.sql alfred@192.168.178.103:~/codex-infra/ + +ssh alfred@192.168.178.103 +cd ~/codex-infra +docker compose up -d grobid # pulls grobid/grobid:0.9.0-crf (arm64), starts papers-grobid +docker compose logs -f grobid # wait for "Started ... in N seconds" (model load ~20–40s) +``` + +The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by +Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves +headroom alongside Postgres. + +--- + +## Verify `GET /api/isalive` → `true` + +```bash +# on the Jetson +curl -s http://localhost:8070/api/isalive # => true + +# from the Mac (LAN) — same value the corpus' GROBID_URL points at +curl -s http://192.168.178.103:8070/api/isalive # => true +``` + +Anything other than `true` (timeout / connection refused / `false`) means the +container isn't up — check `docker compose logs grobid` and that nothing else +holds port 8070. + +--- + +## End-to-end reference-extraction check + running R-A + +With `GROBID_URL=http://192.168.178.103:8070` (already set in `.env.jetson-ingest`): + +```bash +# one PDF, references only — should print N>0 parsed references +PYTHONPATH=. python - <<'PY' +from codex.parsing.grobid import extract_references +refs = extract_references("/Users/tarikmoussa/Desktop/ConformalLabpp/papers/lutz-2024-thesis.pdf", + grobid_url="http://192.168.178.103:8070") +print(f"{len(refs)} references; first DOI/arXiv:", + next(((r['doi'], r['arxiv_id']) for r in refs if r['doi'] or r['arxiv_id']), None)) +PY +``` + +Then run the references-only backfill (DB tunnel up — see +`docs/audit/DATA-QUALITY-2026-06-15.md`): + +```bash +ssh -N -L 5433:localhost:5432 alfred@192.168.178.103 & # DB tunnel +cp .env.jetson-ingest .env +PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only # dry-run first +PYTHONPATH=. python scripts/ra_grobid_backfill.py --write # apply +``` + +--- + +## Troubleshooting + +- **Container killed mid-request / exit 137** — OOM. The CRF image needs ~3 GB for + references; raise the compose `memory` limit or check Postgres isn't starving the + box (`free -h`, `docker stats papers-grobid`). +- **`isalive` true but `processReferences` 500s** — usually a `pdfalto` failure on a + malformed PDF; confirm with a clean PDF (`lutz-2024-thesis.pdf`) and check + `docker compose logs grobid`. +- **`permission denied … docker.sock`** — the docker-group prerequisite above wasn't + applied, or the SSH session predates it (re-login). diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index f7c8912..a1e822b 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -28,20 +28,30 @@ services: restart: unless-stopped grobid: - # CRF-only image is sufficient for reference extraction and is lighter than - # the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid - # for the latest 0.8.x tag before deploying. - image: grobid/grobid:0.8.2 + # CRF-only image: sufficient for reference extraction (/api/processReferences) + # and ~500MB vs ~10GB for the full deep-learning variant. + # + # The `-crf` tag is also the ONLY GROBID variant published as a multi-arch + # manifest that includes linux/arm64 — the full image (grobid/grobid:0.9.0) + # and every 0.8.x tag are amd64-only. So this is what lets GROBID run + # *natively* on the aarch64 Jetson (no slow/RAM-heavy qemu emulation). arm64 + # builds start at 0.9.0-crf; there is no arm64 0.8.x. + # Deploy + docker-permission fix: docs/infra/grobid-jetson.md + image: grobid/grobid:0.9.0-crf container_name: papers-grobid ports: - "8070:8070" + init: true # reap zombie children (GROBID docs recommend --init) + ulimits: + core: 0 # disable core dumps (GROBID docs: --ulimit core=0) restart: unless-stopped - # GROBID is RAM-hungry; uncomment to cap usage on constrained hardware - # (e.g. Nvidia Jetson): - # deploy: - # resources: - # limits: - # memory: 4g + # GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB + # RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom. + # Honored by `docker compose up` under Compose v2 (no swarm needed). + deploy: + resources: + limits: + memory: 4g volumes: pgdata: diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..82057c3 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Operational one-off scripts (importable for tests).""" diff --git a/scripts/ra_grobid_backfill.py b/scripts/ra_grobid_backfill.py new file mode 100644 index 0000000..798230e --- /dev/null +++ b/scripts/ra_grobid_backfill.py @@ -0,0 +1,220 @@ +"""R-A — GROBID reference backfill (references-only, idempotent). + +Adds GROBID-parsed citations to the live corpus **without** re-chunking the body, +so the DQ-3-verified-clean ``.txt`` chunks are preserved. Complements the DQ-1 +Semantic-Scholar supplement with references the APIs lack (notably the two +TU-Berlin ``depositonce`` theses, which neither OpenAlex nor S2 indexes). + +Why references-only instead of ``ingest_paper(source_path=pdf)`` +---------------------------------------------------------------- +The PDF branch of :func:`codex.ingest.ingest_paper` runs Nougat OCR and then +``DELETE FROM chunks`` + re-INSERT — it would replace the clean ``.txt`` chunks +(DQ-3) with unmeasured PDF-OCR text and additionally requires the Nougat server. +This backfill touches only the ``citations`` table (``ON CONFLICT DO NOTHING``), +so it is safe, idempotent, and needs only a reachable GROBID server. + +Prerequisite +------------ +A reachable GROBID server. The corpus' ``GROBID_URL`` points at the Jetson +(``http://192.168.178.103:8070``). GROBID now runs natively on that aarch64 host +via the multi-arch CRF-only image (``grobid/grobid:0.9.0-crf``) — see +``docs/infra/grobid-jetson.md`` for the deploy + docker-permission fix and the +``GET /api/isalive`` check. (The old ``grobid/grobid:0.8.2`` pin was amd64-only, +which is why GROBID never came up; pass ``--grobid-url`` to override the default.) +Use the SSH DB tunnel for ``DATABASE_URL`` as documented in +``docs/audit/DATA-QUALITY-2026-06-15.md``. + +Usage +----- + # dry run (no writes) over the two zero-edge theses + PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only + + # write every paper's GROBID-parsed refs to the live DB + PYTHONPATH=. python scripts/ra_grobid_backfill.py --write + +Default is **dry-run**; pass ``--write`` to mutate the live DB. +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +import psycopg +import psycopg.rows + +from codex.config import get_settings +from codex.db import get_conn +from codex.models import Citation +from codex.parsing.grobid import extract_references + +logger = logging.getLogger(__name__) + +DEFAULT_PDF_DIR = "/Users/tarikmoussa/Desktop/ConformalLabpp/papers" + + +def _normalize_cited_id(doi: str = "", arxiv_id: str = "") -> str | None: + """Normalize a GROBID-extracted reference id to the corpus' canonical form. + + DOIs → bare, lower-cased (strip ``https://doi.org/`` / ``http://`` / ``doi:``), + matching ``papers.id`` and ``codex.sources.openalex._normalize_doi`` (DQ-5). + arXiv ids → strip a leading ``arXiv:`` prefix, leaving the bare id + (``2305.10988``, ``math/0603097``). DOI takes precedence when both exist. + Kept self-contained rather than reusing ``ingest._norm_cited_id`` because it + additionally strips the ``arXiv:`` prefix (which GROBID can emit but that + helper leaves untouched), and to avoid importing a private cross-module name. + """ + doi = (doi or "").strip() + if doi: + s = doi.lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if s.startswith(prefix): + return s[len(prefix) :] + return s + arx = (arxiv_id or "").strip() + if arx: + return arx[len("arxiv:") :].strip() if arx.lower().startswith("arxiv:") else arx + return None + + +def build_grobid_citations( + paper_id: str, + pdf_path: str | Path, + grobid_url: str | None = None, + timeout: float = 60.0, +) -> list[Citation]: + """Extract a paper's references via GROBID and return normalized Citations. + + ``citing_id`` is the canonical ``paper_id``; ``cited_id`` is normalized. + Self-citations and within-paper duplicates are dropped. Refs with neither a + DOI nor an arXiv id are skipped (they cannot join the graph). + """ + refs = extract_references(str(pdf_path), grobid_url=grobid_url, timeout=timeout) + seen: set[str] = set() + citations: list[Citation] = [] + for ref in refs: + cited = _normalize_cited_id(ref.get("doi", ""), ref.get("arxiv_id", "")) + if not cited or cited == paper_id or cited in seen: + continue + seen.add(cited) + citations.append(Citation(citing_id=paper_id, cited_id=cited)) + return citations + + +def _coverage(conn: psycopg.Connection[psycopg.rows.DictRow]) -> tuple[int, int, int]: + """Return (n_papers, n_papers_with_out_edges, n_citations) from the live DB.""" + row = conn.execute( + """ + SELECT (SELECT count(*) FROM papers) AS papers, + (SELECT count(DISTINCT citing_id) FROM citations) AS citing, + (SELECT count(*) FROM citations) AS edges + """ + ).fetchone() + assert row is not None + return int(row["papers"]), int(row["citing"]), int(row["edges"]) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pdf-dir", default=DEFAULT_PDF_DIR, help="directory of source PDFs") + parser.add_argument("--grobid-url", default=None, help="override Settings().grobid_url") + parser.add_argument("--only", default=None, help="restrict to a single paper id") + parser.add_argument( + "--zero-edge-only", + action="store_true", + help="only papers with 0 out-edges (the R-A primary targets)", + ) + parser.add_argument("--limit", type=int, default=None, help="cap number of papers") + parser.add_argument( + "--timeout", + type=float, + default=300.0, + help="GROBID HTTP timeout (s); theses can need >60s, esp. emulated", + ) + parser.add_argument( + "--write", action="store_true", help="WRITE to the live DB (default: dry-run)" + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + grobid_url = args.grobid_url or get_settings().grobid_url + pdf_dir = Path(args.pdf_dir) + pdfs = {p.stem: p for p in pdf_dir.glob("*.pdf")} + + with get_conn() as conn: + rows = conn.execute( + """ + SELECT p.id, p.source_path, + (SELECT count(*) FROM citations ci WHERE ci.citing_id = p.id) AS out_edges + FROM papers p + ORDER BY out_edges, p.id + """ + ).fetchall() + + targets: list[tuple[str, Path]] = [] + for r in rows: + if args.only and r["id"] != args.only: + continue + if args.zero_edge_only and r["out_edges"] != 0: + continue + stem = Path(r["source_path"]).stem if r["source_path"] else None + pdf = pdfs.get(stem) if stem else None + if pdf is None: + logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem) + continue + targets.append((r["id"], pdf)) + if args.limit is not None: + targets = targets[: args.limit] + + logger.info( + "GROBID=%s | papers to process=%d | mode=%s", + grobid_url, + len(targets), + "WRITE" if args.write else "DRY-RUN", + ) + + all_citations: list[Citation] = [] + for paper_id, pdf in targets: + try: + cits = build_grobid_citations( + paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout + ) + except Exception as exc: + logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc) + continue + logger.info("%-38s %3d refs (%s)", paper_id, len(cits), pdf.name) + all_citations.extend(cits) + + logger.info("total candidate citations: %d", len(all_citations)) + + if not args.write: + logger.info("DRY-RUN — no rows written. Re-run with --write to apply.") + return 0 + + before = _coverage(conn) + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO citations (citing_id, cited_id, context) + VALUES (%s, %s, %s) + ON CONFLICT (citing_id, cited_id) DO NOTHING + """, + [(c.citing_id, c.cited_id, c.context) for c in all_citations], + ) + conn.commit() + after = _coverage(conn) + logger.info( + "coverage papers=%d citing %d->%d edges %d->%d (+%d)", + after[0], + before[1], + after[1], + before[2], + after[2], + after[2] - before[2], + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scripts/test_ra_grobid_backfill.py b/tests/scripts/test_ra_grobid_backfill.py new file mode 100644 index 0000000..8e97b2f --- /dev/null +++ b/tests/scripts/test_ra_grobid_backfill.py @@ -0,0 +1,76 @@ +"""Tests for scripts.ra_grobid_backfill — the R-A GROBID references backfill. + +Only the pure, offline logic is covered (id normalization + Citation assembly); +``extract_references`` is mocked so no GROBID server or DB is required. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from codex.models import Citation +from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations + +# --------------------------------------------------------------------------- +# _normalize_cited_id +# --------------------------------------------------------------------------- + + +def test_normalize_doi_url_to_bare_lowercase() -> None: + assert _normalize_cited_id(doi="https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_prefixes() -> None: + assert _normalize_cited_id(doi="http://doi.org/10.1/X") == "10.1/x" + assert _normalize_cited_id(doi="doi:10.1/X") == "10.1/x" + assert _normalize_cited_id(doi="10.1/x") == "10.1/x" + + +def test_normalize_arxiv_prefix_stripped() -> None: + assert _normalize_cited_id(arxiv_id="arXiv:2305.10988") == "2305.10988" + assert _normalize_cited_id(arxiv_id="2305.10988") == "2305.10988" + # Legacy ids keep their slash. + assert _normalize_cited_id(arxiv_id="math/0603097") == "math/0603097" + + +def test_normalize_doi_takes_precedence_over_arxiv() -> None: + assert _normalize_cited_id(doi="10.5/y", arxiv_id="2305.10988") == "10.5/y" + + +def test_normalize_empty_returns_none() -> None: + assert _normalize_cited_id() is None + assert _normalize_cited_id(doi=" ", arxiv_id="") is None + + +# --------------------------------------------------------------------------- +# build_grobid_citations +# --------------------------------------------------------------------------- + + +def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]: + return {"title": "t", "authors": "a", "year": "2020", "doi": doi, "arxiv_id": arxiv_id} + + +def test_build_citations_normalizes_dedups_and_drops_self() -> None: + paper_id = "10.1007/s00454-019-00132-8" + refs = [ + _ref(doi="https://doi.org/10.1/A"), # → 10.1/a + _ref(doi="DOI:10.1/a"), # duplicate of the above after normalization + _ref(arxiv_id="arXiv:2305.10988"), # → 2305.10988 + _ref(doi="", arxiv_id=""), # no id → skipped + _ref(doi="https://doi.org/10.1007/S00454-019-00132-8"), # self-cite → dropped + ] + with patch("scripts.ra_grobid_backfill.extract_references", return_value=refs) as mock_ex: + cits = build_grobid_citations(paper_id, "some.pdf", grobid_url="http://grobid") + + mock_ex.assert_called_once_with("some.pdf", grobid_url="http://grobid", timeout=60.0) + assert all(isinstance(c, Citation) for c in cits) + assert all(c.citing_id == paper_id for c in cits) + assert [c.cited_id for c in cits] == ["10.1/a", "2305.10988"] + + +def test_build_citations_empty_when_no_refs() -> None: + with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]): + assert build_grobid_citations("10.1/x", "x.pdf") == [] From 82cea2a33b7f55bc901cc1f807de042ca20d9ca3 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 21:32:46 +0200 Subject: [PATCH 10/26] feat(crossref): reference extraction as third citation fallback (R-B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add crossref.fetch_references(doi), which parses message.reference[].DOI into Citations (DOI-less book/unstructured refs are skipped — they cannot join the graph). Wire it as the third leg of the ingest citation fallback chain: OpenAlex-empty -> S2 -> Crossref, for DOI papers only (Crossref's works endpoint is DOI-keyed). Cited DOIs are normalized to the canonical bare lower-case form via _norm_cited_id. As a fallback it fires only when OpenAlex and S2 both return no references, so it adds a forward-looking third source without changing the already-covered corpus. Tests: fetch_references unit tests (DOI edges / unstructured skipped / no refs / 404) and ingest tests for both fallback directions. Full suite 361 passed; ruff + mypy clean. Live-validated: 33/40/24 refs for Springer/ACM/Wiley DOIs. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 29 +++++++++++++- codex/sources/crossref.py | 39 ++++++++++++++++++- docs/audit/DATA-QUALITY-2026-06-15.md | 30 ++++++++++++++- tests/ingest/test_ingest.py | 54 +++++++++++++++++++++++++++ tests/sources/test_crossref.py | 54 +++++++++++++++++++++++++++ 5 files changed, 202 insertions(+), 4 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index c0f768f..63d3401 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -130,6 +130,29 @@ def _s2_reference_supplement(paper: Paper) -> list[Citation]: ] +def _crossref_reference_supplement(paper: Paper) -> list[Citation]: + """Fetch a paper's references from Crossref (roadmap R-B supplement). + + Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref). + Crossref carries publisher-deposited reference lists keyed on the citing + work's DOI, so only DOI papers can be looked up (arXiv-only ids → []). + ``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are + case-normalised. Network/parse failures degrade to [] rather than aborting. + """ + if not paper.id.startswith("10."): + return [] + try: + raw = crossref.fetch_references(paper.id) + except Exception: + logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True) + return [] + return [ + Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context) + for c in raw + if c.cited_id + ] + + def ingest_paper( paper_id: str, source_path: str | None = None, @@ -315,7 +338,7 @@ def ingest_paper( chunks_upserted = len(chunk_rows) # --------------------------------------------------------------- - # 5. Fetch citations (OpenAlex if openalex_id, else S2) + # 5. Fetch citations (fallback chain: OpenAlex → S2 → Crossref) # Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING # --------------------------------------------------------------- api_citations: list[Citation] @@ -333,6 +356,10 @@ def ingest_paper( api_citations = _s2_reference_supplement(paper) else: api_citations = _s2_reference_supplement(paper) + # R-B: if OpenAlex and S2 both came back empty, try Crossref's + # publisher-deposited references (DOI papers only) as a third leg. + if not api_citations: + api_citations = _crossref_reference_supplement(paper) # Merge API citations and GROBID PDF citations; dedup via set all_citations_set: set[tuple[str, str]] = set() diff --git a/codex/sources/crossref.py b/codex/sources/crossref.py index 4727d85..f263872 100644 --- a/codex/sources/crossref.py +++ b/codex/sources/crossref.py @@ -2,10 +2,11 @@ Provides: - fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI. +- fetch_references: retrieve a work's reference list (DOI edges) by DOI. Crossref is a *third* metadata source after OpenAlex and Semantic Scholar -(DQ-2 / roadmap R-B). Requests use the Polite Pool (mailto query parameter) -and are retried on 429/5xx with exponential back-off via tenacity. +(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto +query parameter) and are retried on 429/5xx with exponential back-off via tenacity. """ from __future__ import annotations @@ -18,6 +19,7 @@ import httpx from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential from codex.config import get_settings +from codex.models import Citation logger = logging.getLogger(__name__) @@ -89,3 +91,36 @@ def fetch_abstract(doi: str) -> str | None: raise message: dict[str, Any] = response.json().get("message", {}) return _strip_jats(message.get("abstract")) + + +def fetch_references(doi: str) -> list[Citation]: + """Fetch a work's reference list from Crossref by DOI (roadmap R-B). + + Crossref carries publisher-deposited reference lists in ``message.reference``. + Each entry that cites a DOI-registered work exposes a bare ``DOI`` field; + only those become citation-graph edges (book / older references carry just + ``unstructured`` text with no resolvable id and are skipped). ``citing_id`` + is the queried *doi* — the caller rewrites it to the canonical ``papers.id`` + and normalises the cited DOIs (mirrors the Semantic Scholar reference path). + + Parameters + ---------- + doi: + A bare DOI (``"10.1007/s00454-019-00132-8"``). + + Returns + ------- + list[Citation] + One Citation per reference that carries a DOI. Empty on 404 or when no + references are deposited for the work. + """ + url = f"{_BASE}/works/{doi}" + try: + response = _get(url) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return [] + raise + message: dict[str, Any] = response.json().get("message", {}) + references: list[dict[str, Any]] = message.get("reference") or [] + return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")] diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 5719674..9608dd8 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -382,7 +382,35 @@ is self-contained so a cold session can pick it up. - **Acceptance:** the 2 theses gain references; published papers gain GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29. -### R-B — Add Crossref as a third citation + abstract source · **MED lever, MED effort** +### R-B — Add Crossref as a third citation + abstract source · **DONE 2026-06-17** + +**Resolution (what was done):** +- **`crossref.fetch_references(doi)`** added (mirrors `semanticscholar.fetch_references`): + GETs `/works/{doi}`, returns one `Citation` per `message.reference[]` entry that + carries a registered `DOI` (book/`unstructured`-only refs are skipped — they + can't join the graph). `citing_id` is the queried DOI; the ingest supplement + rewrites it to `paper.id`. +- **Wired as the third leg of the citation fallback chain** in `ingest.py`: + OpenAlex-empty → S2 → **Crossref** (`_crossref_reference_supplement`, DOI papers + only — Crossref's works endpoint is DOI-keyed). cited DOIs pass through + `_norm_cited_id` (bare, lower-cased). Network/parse failures degrade to []. +- **Tests:** `fetch_references` unit tests (DOI edges / `unstructured` skipped / no + `reference` key / 404) + ingest tests asserting the Crossref leg fires when + OpenAlex+S2 are empty (cited DOI normalised) **and** does *not* fire when OpenAlex + has citations. Suite **361 green**, ruff/mypy clean. The DQ-2 abstract half + (`fetch_abstract`) already shipped earlier. +- **Live-validated:** `fetch_references` against the real API returned 33 / 40 / 24 + DOI-bearing refs for `10.1007/s00454-019-00132-8` / `10.1145/3132705` / + `10.1111/cgf.13931` (Springer / ACM / Wiley). +- **No live corpus change (by design):** as a *fallback*, Crossref fires only when + OpenAlex **and** S2 are both empty. Post-DQ-1/R-A the only such papers were the 2 + `depositonce` theses, which Crossref doesn't index either — so R-B is a + forward-looking third leg for future DOI ingests, not a backfill. (Harvesting + Crossref's *unique* refs for already-covered papers would need an always-merged + supplement instead — a deliberate scope choice not taken, per the doc's "fallback + chain" spec.) **Files:** `codex/sources/crossref.py`, `codex/ingest.py`, + tests. + +**Original plan (for context):** - **What:** new `codex.sources.crossref` client hitting Crossref REST `/works/{doi}` for the `reference` array and `abstract`. - **Why:** Crossref carries **publisher-deposited references** for many DOIs (free, diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 936e6cb..82311f1 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -107,6 +107,60 @@ def test_ingest_paper_basic() -> None: ) +def test_ingest_citation_fallback_to_crossref() -> None: + """R-B: OpenAlex + S2 both empty → Crossref references supply the edges. + + Crossref is the third leg of the fallback chain. Its cited DOIs are + case-normalised and citing_id is rewritten to the canonical paper.id. + """ + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123") + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + # Upper-case suffix proves the cited DOI is normalised to bare lower-case. + crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty + patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(paper.id) + + # Crossref consulted with the bare DOI; one edge upserted. + mock_cr.assert_called_once_with(paper.id) + assert result.citations_upserted == 1 + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + cit_rows = mock_cursor.executemany.call_args_list[0][0][1] + assert cit_rows[0][0] == paper.id + assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7" + + +def test_ingest_crossref_not_called_when_openalex_has_citations() -> None: + """R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback).""" + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123") + mock_conn = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch( + "codex.ingest.openalex.fetch_citations", + return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")], + ), + patch("codex.ingest.crossref.fetch_references") as mock_cr, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + ingest_paper(paper.id) + + mock_cr.assert_not_called() + + # --------------------------------------------------------------------------- # 2. test_ingest_paper_source_tex # --------------------------------------------------------------------------- diff --git a/tests/sources/test_crossref.py b/tests/sources/test_crossref.py index f88d007..6e5c633 100644 --- a/tests/sources/test_crossref.py +++ b/tests/sources/test_crossref.py @@ -5,6 +5,7 @@ from __future__ import annotations import httpx import pytest +from codex.models import Citation from codex.sources import crossref @@ -48,3 +49,56 @@ def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> Non monkeypatch.setattr(crossref, "_get", mock_get) assert crossref.fetch_abstract("10.0000/nonexistent") is None + + +# --------------------------------------------------------------------------- +# fetch_references (roadmap R-B) +# --------------------------------------------------------------------------- + + +def test_fetch_references_returns_doi_edges(monkeypatch: pytest.MonkeyPatch) -> None: + """Only references carrying a DOI become Citations; citing_id is the queried DOI.""" + body = { + "message": { + "reference": [ + {"key": "ref1", "DOI": "10.1090/s0002-9947-04-03545-7"}, + {"key": "ref2", "unstructured": "A book with no DOI, 1992."}, # skipped + {"key": "ref3", "DOI": "10.1007/bf02392449", "article-title": "X"}, + ] + } + } + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json=body) + + monkeypatch.setattr(crossref, "_get", mock_get) + refs = crossref.fetch_references("10.1007/s00454-019-00132-8") + + assert refs == [ + Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1090/s0002-9947-04-03545-7"), + Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1007/bf02392449"), + ] + + +def test_fetch_references_no_reference_key_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """A work with no deposited reference list returns [].""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"message": {"title": ["No refs deposited"]}}) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_references("10.1007/978-3-642-17413-1_7") == [] + + +def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """A 404 returns [] rather than raising.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + raise httpx.HTTPStatusError( + "404", + request=httpx.Request("GET", url), + response=httpx.Response(404, request=httpx.Request("GET", url)), + ) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_references("10.0000/nonexistent") == [] From bf90c6cd57d2a91076dbeb8957506e691abf646e Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Wed, 17 Jun 2026 21:36:04 +0200 Subject: [PATCH 11/26] feat(grobid): run GROBID on-demand (restart "no") to free ~3.5GB when idle GROBID is only needed during reference extraction (ra_grobid_backfill.py), not for normal corpus operations (search, embeddings, wiki). Idle it still holds ~3.5GB on the 8GB Jetson, so keep it stopped by default. - infra/docker-compose.yml: grobid restart unless-stopped -> "no" (no auto-start on boot; db keeps unless-stopped). Start/stop papers-grobid around an R-A run. - docs/infra/grobid-jetson.md: add "Run on-demand" section with the start/stop workflow, and record the refextract head-to-head evaluation on the Jetson that led to keeping GROBID: far worse DOI recall on this math corpus (lutz thesis 101 usable IDs vs refextract 2) and slower per PDF despite a smaller footprint. Co-Authored-By: Claude Opus 4.8 --- docs/infra/grobid-jetson.md | 31 ++++++++++++++++++++++++++++++- infra/docker-compose.yml | 6 +++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/infra/grobid-jetson.md b/docs/infra/grobid-jetson.md index 4e6a0c6..7a43e92 100644 --- a/docs/infra/grobid-jetson.md +++ b/docs/infra/grobid-jetson.md @@ -104,7 +104,36 @@ docker compose logs -f grobid # wait for "Started ... in N seconds" (mode The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves -headroom alongside Postgres. +headroom alongside Postgres. The `restart: "no"` policy means GROBID does **not** +auto-start on boot — see "Run on-demand" below. + +--- + +## Run on-demand (stop when idle) + +GROBID is only needed **during reference extraction** (`scripts/ra_grobid_backfill.py`); +nothing else in the stack uses it. Idle it still holds ~3.5 GB, so on the 8 GB Jetson +keep it stopped and start it only for an R-A run: + +```bash +ssh alfred@192.168.178.103 'docker start papers-grobid' # ~30s model load +curl -s --retry 30 --retry-delay 2 --retry-connrefused \ + http://192.168.178.103:8070/api/isalive # wait for: true + +# ... run R-A from the Mac ... +PYTHONPATH=. python scripts/ra_grobid_backfill.py --write + +ssh alfred@192.168.178.103 'docker stop papers-grobid' # reclaim ~3.5 GB +``` + +First-time deploy uses `docker compose up -d grobid` (above); afterwards the +container persists, so `docker start/stop papers-grobid` is all you need. + +> Why on-demand rather than a lighter tool: refextract was evaluated head-to-head +> on the Jetson and rejected — far worse DOI recall on this math corpus (lutz +> thesis: GROBID **101** usable IDs vs refextract **2**), and it was *slower* per +> PDF despite a smaller footprint. GROBID stays; running it on-demand removes its +> only real downside (idle RAM). --- diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index a1e822b..9304909 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -44,7 +44,11 @@ services: init: true # reap zombie children (GROBID docs recommend --init) ulimits: core: 0 # disable core dumps (GROBID docs: --ulimit core=0) - restart: unless-stopped + # On-demand only: GROBID is needed solely during reference extraction + # (scripts/ra_grobid_backfill.py), not for normal corpus operations — start it + # for an R-A run, stop it after to reclaim ~3.5GB. "no" = no auto-start on boot. + # See docs/infra/grobid-jetson.md ("Run on-demand"). + restart: "no" # GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB # RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom. # Honored by `docker compose up` under Compose v2 (no swarm needed). From 1da81c7b28edf9ea05be544735aa3b63ed364574 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 04:15:20 +0200 Subject: [PATCH 12/26] feat(tex): section-aware multi-file .tex ingest (R-C) Flatten multi-file arXiv LaTeX (\input/\include) in arxiv.fetch_source so the full body is assembled rather than just the primary file's include skeleton, and add section-aware chunking (tex.chunk_sections) so .tex chunks never span a \section boundary and are labelled by their real heading. The ingest .tex path now produces (section, chunk) pairs, quality-gated together so labels stay aligned; the stored 'section' column gains genuine signal instead of mostly 'body' (serves DQ-3 fidelity + audit R-12). Adds scripts/rc_tex_reingest.py to re-ingest arXiv papers from .tex (dry-run by default; replaces chunks). Tests: flatten_inputs, chunk_sections, multi-file fetch_source, section-label ingest, is_arxiv_id. Full suite 365 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 35 +++++--- codex/parsing/tex.py | 60 +++++++++++++ codex/sources/arxiv.py | 48 +++++----- scripts/rc_tex_reingest.py | 125 ++++++++++++++++++++++++++ tests/ingest/test_ingest.py | 23 +++-- tests/parsing/test_tex.py | 55 +++++++++++- tests/scripts/test_rc_tex_reingest.py | 19 ++++ tests/sources/test_arxiv.py | 33 +++++++ 8 files changed, 356 insertions(+), 42 deletions(-) create mode 100644 scripts/rc_tex_reingest.py create mode 100644 tests/scripts/test_rc_tex_reingest.py diff --git a/codex/ingest.py b/codex/ingest.py index c0f768f..ab79250 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -12,7 +12,7 @@ from codex.config import get_settings from codex.db import get_conn from codex.embed import get_embedder from codex.models import Citation, Paper -from codex.quality import classify_section, filter_chunks +from codex.quality import classify_section, is_quality_chunk from codex.sources import arxiv, crossref, openalex, semanticscholar logger = logging.getLogger(__name__) @@ -262,17 +262,21 @@ def ingest_paper( if source_path is not None: from codex.parsing.grobid import extract_references from codex.parsing.nougat import pdf_to_markdown - from codex.parsing.tex import chunk_text, latex_to_text + from codex.parsing.tex import chunk_sections, chunk_text suffix = Path(source_path).suffix.lower() - text = "" + # (section_title | None, chunk_text) pairs. `.tex` is section-aware so + # the stored `section` column reflects the real heading (R-C / R-12); + # `.txt`/`.pdf` carry no section structure, so the title is None. + raw_pairs: list[tuple[str | None, str]] = [] if suffix == ".tex": - text = latex_to_text(Path(source_path).read_text()) + raw_pairs = chunk_sections(Path(source_path).read_text()) elif suffix == ".txt": text = Path(source_path).read_text(encoding="utf-8", errors="replace") + raw_pairs = [(None, c) for c in chunk_text(text)] elif suffix == ".pdf": - text = pdf_to_markdown(source_path) + raw_pairs = [(None, c) for c in chunk_text(pdf_to_markdown(source_path))] # Also extract GROBID refs grobid_refs = extract_references(source_path) for ref in grobid_refs: @@ -287,24 +291,33 @@ def ingest_paper( else: logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path) - raw_chunks = chunk_text(text) if text else [] - chunks_text = filter_chunks(raw_chunks, settings=get_settings()) + # Quality-gate the pairs (keeps each chunk aligned with its section title). + settings = get_settings() + kept_pairs = [ + (title, content) + for title, content in raw_pairs + if is_quality_chunk(content, settings=settings) + ] # Delete existing chunks for this paper conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,)) - if chunks_text: + if kept_pairs: # Embed all chunks in one batch - chunk_embeddings = embedder.encode_dense(chunks_text) + chunk_contents = [content for _, content in kept_pairs] + chunk_embeddings = embedder.encode_dense(chunk_contents) chunk_rows = [ ( paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist(), - classify_section(content), + # .tex: classify from the section heading (prepended to the + # snippet) so the label reflects the real section; otherwise + # fall back to content-only classification. + classify_section(f"{title}. {content}" if title else content), ) - for ord_idx, content in enumerate(chunks_text) + for ord_idx, (title, content) in enumerate(kept_pairs) ] with conn.cursor() as cur: cur.executemany( diff --git a/codex/parsing/tex.py b/codex/parsing/tex.py index 43514d6..ed77ad7 100644 --- a/codex/parsing/tex.py +++ b/codex/parsing/tex.py @@ -17,6 +17,10 @@ _SECTION_RE = re.compile( re.DOTALL, ) +# \input{file} / \include{file} — arXiv papers routinely split the body across +# files, so these must be inlined before parsing (R-C multi-file flattening). +_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}") + # Patterns for LaTeX noise removal (applied in order). _COMMENT_RE = re.compile(r"%[^\n]*") _CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}") @@ -157,3 +161,59 @@ def latex_to_text(latex: str) -> str: return "\n\n".join(body for _, body in sections) # Fallback: no section structure — clean the whole string. return _clean_latex(latex) + + +def _norm_texkey(name: str) -> str: + """Normalize a tex file name / ``\\input`` argument to a comparable key. + + Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing + ``.tex`` so ``\\input{sections/intro}`` matches the archive member + ``sections/intro.tex``. + """ + n = name.strip().lstrip("./") + return n[:-4] if n.endswith(".tex") else n + + +def flatten_inputs(primary: str, files: dict[str, str]) -> str: + """Recursively inline ``\\input``/``\\include`` directives (R-C). + + *files* maps each source file's archive name to its contents (keys may carry + a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv + multi-file projects leave the ``\\documentclass`` file as little more than a + skeleton of ``\\input`` lines, so the body must be assembled before parsing. + Unresolved references are dropped; recursion is capped at depth 10 to guard + against include cycles. + """ + norm = {_norm_texkey(k): v for k, v in files.items()} + + def _expand(text: str, depth: int) -> str: + if depth > 10: + return text + + def _repl(match: re.Match[str]) -> str: + content = norm.get(_norm_texkey(match.group(1))) + return _expand(content, depth + 1) if content is not None else "" + + return _INPUT_RE.sub(_repl, text) + + return _expand(primary, 0) + + +def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]: + """Chunk LaTeX *within* each section, pairing every chunk with its title. + + Unlike :func:`latex_to_text` (which discards titles and flattens the whole + document into one block before word-window chunking), this keeps chunks from + spanning section boundaries and remembers which ``\\section`` each came from. + The caller can then label the stored ``section`` column from the real heading + instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less + chunks of the whole cleaned document when there are no section commands. + """ + sections = extract_sections(latex) + if not sections: + return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)] + pairs: list[tuple[str | None, str]] = [] + for title, body in sections: + for chunk in chunk_text(body, size, overlap): + pairs.append((title or None, chunk)) + return pairs diff --git a/codex/sources/arxiv.py b/codex/sources/arxiv.py index 9bb0303..dd61961 100644 --- a/codex/sources/arxiv.py +++ b/codex/sources/arxiv.py @@ -101,8 +101,9 @@ def fetch_source(arxiv_id: str) -> str | None: Downloads the .tar.gz source bundle from ``https://arxiv.org/src/{arxiv_id}``, locates the primary .tex file (preferring any file containing ``\\documentclass``, - falling back to the largest .tex by size), and returns its contents as a UTF-8 - string. + falling back to the largest .tex by size), and inlines its ``\\input``/ + ``\\include`` directives from the other archive members so multi-file projects + return the full document body, not just the primary file's include skeleton. Parameters ---------- @@ -126,33 +127,36 @@ def fetch_source(arxiv_id: str) -> str | None: if response.status_code != 200: response.raise_for_status() + from codex.parsing.tex import flatten_inputs + raw = response.content try: with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf: - tex_members = [m for m in tf.getmembers() if m.name.endswith(".tex")] - if not tex_members: - logger.debug("No .tex files found in arXiv source for %s", arxiv_id) - return None - - # Prefer the file containing \documentclass (primary document) - primary: tarfile.TarInfo | None = None - for member in tex_members: + files: dict[str, str] = {} + primary_name: str | None = None + for member in tf.getmembers(): + if not member.name.endswith(".tex"): + continue f = tf.extractfile(member) if f is None: continue - content_bytes = f.read() - if b"\\documentclass" in content_bytes: - primary = member - # Decode and return immediately — first match wins - return content_bytes.decode("utf-8", errors="replace") + content = f.read().decode("utf-8", errors="replace") + files[member.name] = content + # Prefer the \documentclass file as the primary document. + if primary_name is None and "\\documentclass" in content: + primary_name = member.name - if primary is None: - # Fallback: largest .tex by size - largest = max(tex_members, key=lambda m: m.size) - f = tf.extractfile(largest) - if f is None: - return None - return f.read().decode("utf-8", errors="replace") + if not files: + logger.debug("No .tex files found in arXiv source for %s", arxiv_id) + return None + + # No \documentclass anywhere → fall back to the largest .tex. + if primary_name is None: + primary_name = max(files, key=lambda k: len(files[k])) + + # Inline \input/\include so multi-file projects yield the full body, + # not just the primary file's skeleton of include directives (R-C). + return flatten_inputs(files[primary_name], files) except tarfile.TarError as exc: logger.warning("Failed to open tar archive for %s: %s", arxiv_id, exc) return None diff --git a/scripts/rc_tex_reingest.py b/scripts/rc_tex_reingest.py new file mode 100644 index 0000000..53839ac --- /dev/null +++ b/scripts/rc_tex_reingest.py @@ -0,0 +1,125 @@ +"""R-C — re-ingest arXiv papers from LaTeX source (section-aware, higher fidelity). + +For each arXiv paper, download its source via :func:`codex.sources.arxiv.fetch_source` +(multi-file ``\\input``/``\\include`` flattened), write it to a temp ``.tex``, and +re-ingest through :func:`codex.ingest.ingest_paper`. The ``.tex`` ingest path is +section-aware (:func:`codex.parsing.tex.chunk_sections`), so the stored ``section`` +column is labelled from real ``\\section`` headings instead of mostly ``body`` — +serving DQ-3 (fidelity vs the OCR'd ``.txt``) and audit R-12 (weak ``section`` column). + +Only arXiv papers have downloadable source; DOI/journal papers and books are skipped. +Re-ingest REPLACES the paper's chunks (DELETE + re-INSERT) — that is the intended R-C +improvement (flattened ``.txt`` → clean ``.tex`` chunks). It is idempotent and +**dry-run by default**; pass ``--write`` to mutate the live corpus. + +Usage +----- + # dry-run: list arXiv papers + source availability (no writes) + PYTHONPATH=. python scripts/rc_tex_reingest.py + + # re-ingest every arXiv paper from .tex + PYTHONPATH=. python scripts/rc_tex_reingest.py --write + + # one paper + PYTHONPATH=. python scripts/rc_tex_reingest.py --only 2305.10988 --write + +Needs ``DATABASE_URL`` (Jetson tunnel) as documented in +``docs/audit/DATA-QUALITY-2026-06-15.md``. +""" + +from __future__ import annotations + +import argparse +import logging +import tempfile + +import psycopg +import psycopg.rows + +from codex.db import get_conn +from codex.ingest import ingest_paper +from codex.sources import arxiv + +logger = logging.getLogger(__name__) + + +def is_arxiv_id(paper_id: str) -> bool: + """Return True for arXiv ids (the only papers with downloadable LaTeX source). + + arXiv ids are modern (``2305.10988``) or legacy (``math/0603097``). DOIs start + with ``10.``; OpenAlex W-ids start with ``W`` or an ``https://openalex.org/`` URL. + """ + p = (paper_id or "").strip() + return bool(p) and not p.startswith("10.") and not p.startswith(("W", "https://")) + + +def _section_dist(conn: psycopg.Connection[psycopg.rows.DictRow], paper_id: str) -> dict[str, int]: + """Return the section-label distribution for a paper's stored chunks.""" + rows = conn.execute( + "SELECT coalesce(section, '(null)') AS s, count(*) AS n " + "FROM chunks WHERE paper_id = %s GROUP BY 1 ORDER BY 2 DESC", + (paper_id,), + ).fetchall() + return {r["s"]: int(r["n"]) for r in rows} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--only", default=None, help="restrict to a single arXiv id") + parser.add_argument("--limit", type=int, default=None, help="cap number of papers") + parser.add_argument( + "--write", action="store_true", help="re-ingest into the live DB (default: dry-run)" + ) + args = parser.parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + + with get_conn() as conn: + rows = conn.execute("SELECT id FROM papers ORDER BY id").fetchall() + targets = [ + r["id"] + for r in rows + if is_arxiv_id(r["id"]) and (args.only is None or r["id"] == args.only) + ] + if args.limit is not None: + targets = targets[: args.limit] + + logger.info( + "arXiv papers to process=%d | mode=%s", + len(targets), + "WRITE" if args.write else "DRY-RUN", + ) + + for pid in targets: + try: + src = arxiv.fetch_source(pid) + except Exception as exc: + logger.warning("%s: source fetch failed: %s", pid, exc) + continue + if not src: + logger.warning("%s: no LaTeX source available — skipping", pid) + continue + if not args.write: + logger.info("%s: source OK (%d chars)", pid, len(src)) + continue + + with get_conn() as conn: + before = _section_dist(conn, pid) + with tempfile.NamedTemporaryFile("w", suffix=".tex", delete=True) as tf: + tf.write(src) + tf.flush() + res = ingest_paper(pid, source_path=tf.name) + with get_conn() as conn: + after = _section_dist(conn, pid) + logger.info( + "%s: chunks %d->%d | section before=%s after=%s", + pid, + sum(before.values()), + res.chunks_upserted, + before, + after, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 936e6cb..8828419 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -113,7 +113,8 @@ def test_ingest_paper_basic() -> None: def test_ingest_paper_source_tex(tmp_path: Any) -> None: - """``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB.""" + """``.tex`` source → section-aware chunking; the stored ``section`` column is + labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12).""" tex_file = tmp_path / "paper.tex" tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.") @@ -123,16 +124,19 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: mock_conn.executemany = MagicMock() mock_conn.commit = MagicMock() - fake_chunks = ["chunk one text here.", "chunk two text here."] + # (section title, chunk content) pairs from the section-aware chunker. + section_pairs = [ + ("Introduction", "Body of the introduction goes here."), + ("Proof of the Main Theorem", "Body of the proof goes here."), + ] with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), - patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."), - patch("codex.parsing.tex.chunk_text", return_value=fake_chunks), - patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + patch("codex.parsing.tex.chunk_sections", return_value=section_pairs), + patch("codex.ingest.is_quality_chunk", return_value=True), ): result = ingest_paper(paper.id, source_path=str(tex_file)) @@ -147,7 +151,10 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: chunk_sql: str = chunk_insert_call[0][0] assert "INSERT INTO chunks" in chunk_sql - assert result.chunks_upserted == len(fake_chunks) + # Section label comes from the heading: "Introduction" → intro, "Proof of…" → proof. + chunk_rows = chunk_insert_call[0][1] + assert [row[4] for row in chunk_rows] == ["intro", "proof"] + assert result.chunks_upserted == len(section_pairs) # --------------------------------------------------------------------------- @@ -183,7 +190,7 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), patch("codex.parsing.tex.chunk_text", return_value=fake_chunks), patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), - patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + patch("codex.ingest.is_quality_chunk", return_value=True), ): result = ingest_paper(paper.id, source_path=str(pdf_file)) @@ -237,7 +244,7 @@ def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None: patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]), patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), - patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + patch("codex.ingest.is_quality_chunk", return_value=True), ): result = ingest_paper(paper.id, source_path=str(pdf_file)) diff --git a/tests/parsing/test_tex.py b/tests/parsing/test_tex.py index 002cc23..13a8044 100644 --- a/tests/parsing/test_tex.py +++ b/tests/parsing/test_tex.py @@ -4,7 +4,13 @@ from __future__ import annotations import pytest -from codex.parsing.tex import chunk_text, extract_sections, latex_to_text +from codex.parsing.tex import ( + chunk_sections, + chunk_text, + extract_sections, + flatten_inputs, + latex_to_text, +) class TestExtractSections: @@ -110,3 +116,50 @@ class TestLatexToText: result = latex_to_text(latex) assert r"\cite" not in result assert "%" not in result + + +class TestFlattenInputs: + def test_inlines_input_and_include(self) -> None: + files = { + "main.tex": r"Start \input{sec/intro} mid \include{proof} end", + "sec/intro.tex": "INTRO BODY", + "proof.tex": "PROOF BODY", + } + out = flatten_inputs(files["main.tex"], files) + assert "INTRO BODY" in out + assert "PROOF BODY" in out + assert r"\input" not in out and r"\include" not in out + + def test_resolves_without_tex_suffix(self) -> None: + # \input{intro} must resolve the archive member "intro.tex". + files = {"main.tex": r"\input{intro}", "intro.tex": "BODY"} + assert "BODY" in flatten_inputs(files["main.tex"], files) + + def test_recursive_includes(self) -> None: + files = {"a.tex": r"X \input{b}", "b.tex": r"Y \input{c}", "c.tex": "Z"} + out = flatten_inputs(files["a.tex"], files) + assert "X" in out and "Y" in out and "Z" in out + + def test_unresolved_input_dropped(self) -> None: + # A reference with no matching file is removed, not left as a directive. + out = flatten_inputs(r"A \input{missing} B", {}) + assert r"\input" not in out + assert "A" in out and "B" in out + + +class TestChunkSections: + def test_pairs_carry_section_titles(self) -> None: + latex = ( + r"\section{Introduction}" + "\nIntro body text here.\n" + r"\section{Proof}" + "\nProof body text here.\n" + ) + pairs = chunk_sections(latex) + titles = [t for t, _ in pairs] + assert "Introduction" in titles + assert "Proof" in titles + # No chunk spans a section boundary: each chunk belongs to one title. + assert all(isinstance(c, str) and c for _, c in pairs) + + def test_fallback_titleless_when_no_sections(self) -> None: + pairs = chunk_sections("Plain text with no section commands at all here.") + assert pairs and all(title is None for title, _ in pairs) diff --git a/tests/scripts/test_rc_tex_reingest.py b/tests/scripts/test_rc_tex_reingest.py new file mode 100644 index 0000000..17a8770 --- /dev/null +++ b/tests/scripts/test_rc_tex_reingest.py @@ -0,0 +1,19 @@ +"""Tests for scripts.rc_tex_reingest.""" + +from __future__ import annotations + +from scripts.rc_tex_reingest import is_arxiv_id + + +def test_is_arxiv_id_accepts_modern_and_legacy() -> None: + assert is_arxiv_id("2305.10988") + assert is_arxiv_id("math/0603097") + assert is_arxiv_id("1005.2698") + + +def test_is_arxiv_id_rejects_dois_and_wids() -> None: + assert not is_arxiv_id("10.1007/s00454-019-00132-8") + assert not is_arxiv_id("10.14279/depositonce-20357") + assert not is_arxiv_id("W2971636899") + assert not is_arxiv_id("https://openalex.org/W2971636899") + assert not is_arxiv_id("") diff --git a/tests/sources/test_arxiv.py b/tests/sources/test_arxiv.py index c4ceb60..f62ff82 100644 --- a/tests/sources/test_arxiv.py +++ b/tests/sources/test_arxiv.py @@ -53,6 +53,39 @@ def test_fetch_source_returns_latex(monkeypatch: pytest.MonkeyPatch) -> None: assert "Hello world" in result +def test_fetch_source_flattens_multifile(monkeypatch: pytest.MonkeyPatch) -> None: + """Multi-file projects: \\input/\\include directives are inlined so the body + from the included files is returned, not just the primary skeleton (R-C).""" + main = ( + b"\\documentclass{article}\\begin{document}" + b"\\input{sections/intro}\\include{proof}\\end{document}" + ) + tar_bytes = _make_tar_gz( + { + "main.tex": main, + "sections/intro.tex": b"INTRODUCTION BODY TEXT", + "proof.tex": b"PROOF BODY TEXT", + } + ) + + def mock_get( + url: str, + *, + timeout: int = 60, + follow_redirects: bool = True, + ) -> httpx.Response: + return httpx.Response(200, content=tar_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + + result = arxiv.fetch_source("2301.07041") + + assert result is not None + assert "INTRODUCTION BODY TEXT" in result # \input was inlined + assert "PROOF BODY TEXT" in result # \include was inlined + assert "\\input" not in result and "\\include" not in result + + # --------------------------------------------------------------------------- # fetch_source — 404 → None # --------------------------------------------------------------------------- From 121b582f46be3b3079587b0be07811710a6c82df Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 04:18:18 +0200 Subject: [PATCH 13/26] fix(arxiv): handle legacy single-file gzipped source (no tar wrapper) Old arXiv submissions (e.g. legacy math/* papers) are served as a bare gzipped .tex rather than a .tar.gz, which tarfile.open rejected with 'invalid header'. Fall back to gunzipping the body directly when the tar parse fails, so R-C covers those papers too (recovers math/0001176 + math/0306167). Adds a regression test. Co-Authored-By: Claude Opus 4.8 --- codex/sources/arxiv.py | 16 ++++++++++++---- tests/sources/test_arxiv.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/codex/sources/arxiv.py b/codex/sources/arxiv.py index dd61961..7c1a803 100644 --- a/codex/sources/arxiv.py +++ b/codex/sources/arxiv.py @@ -8,6 +8,7 @@ Provides: from __future__ import annotations +import gzip import io import logging import tarfile @@ -157,12 +158,19 @@ def fetch_source(arxiv_id: str) -> str | None: # Inline \input/\include so multi-file projects yield the full body, # not just the primary file's skeleton of include directives (R-C). return flatten_inputs(files[primary_name], files) - except tarfile.TarError as exc: - logger.warning("Failed to open tar archive for %s: %s", arxiv_id, exc) + except tarfile.TarError: + # Old arXiv submissions are served as a single bare gzipped .tex with no + # tar wrapper (e.g. legacy math/* papers) — gunzip the body directly. + try: + single = gzip.decompress(raw).decode("utf-8", errors="replace") + except (OSError, EOFError) as exc: + logger.warning("Failed to read arXiv source for %s: %s", arxiv_id, exc) + return None + if "\\" in single[:5000]: # looks like LaTeX (has backslash commands) + return single + logger.debug("arXiv source for %s is not LaTeX", arxiv_id) return None - return None # unreachable but satisfies type checker - def fetch_pdf_url(arxiv_id: str) -> str: """Return the canonical PDF URL for an arXiv paper. diff --git a/tests/sources/test_arxiv.py b/tests/sources/test_arxiv.py index f62ff82..db376c1 100644 --- a/tests/sources/test_arxiv.py +++ b/tests/sources/test_arxiv.py @@ -86,6 +86,30 @@ def test_fetch_source_flattens_multifile(monkeypatch: pytest.MonkeyPatch) -> Non assert "\\input" not in result and "\\include" not in result +def test_fetch_source_bare_gzip_single_file(monkeypatch: pytest.MonkeyPatch) -> None: + """Legacy arXiv submissions are served as a single bare gzipped .tex (no tar + wrapper); fetch_source must gunzip the body directly rather than fail.""" + import gzip + + latex = b"\\documentstyle[a4]{article}\n\\section{Intro}\nOld single-file paper body." + gz_bytes = gzip.compress(latex) + + def mock_get( + url: str, + *, + timeout: int = 60, + follow_redirects: bool = True, + ) -> httpx.Response: + return httpx.Response(200, content=gz_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + + result = arxiv.fetch_source("math/0001176") + + assert result is not None + assert "Old single-file paper body." in result + + # --------------------------------------------------------------------------- # fetch_source — 404 → None # --------------------------------------------------------------------------- From f504ca62dba916fb0f1a93cbce488ef82d56cf82 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 04:25:22 +0200 Subject: [PATCH 14/26] fix(openalex): canonicalize arXiv DOI to bare arXiv id (DQ-5 arXiv half) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAlex returns arXiv works under the arXiv DOI (10.48550/arXiv.); _normalize_doi (DQ-5) left it as 10.48550/arxiv., but the corpus keys arXiv papers on the BARE id (2305.10988 / math/0603097). So re-ingesting an arXiv paper by its bare id missed INSERT ... ON CONFLICT (id) and tripped papers_openalex_id_key — the arXiv half of DQ-5, explicitly deferred there and surfaced again driving the R-C .tex re-ingest. Add _canonical_id() (normalize DOI, then strip the 10.48550/arxiv. prefix) and use it in _map_paper. Regression tests (modern + legacy arXiv DOI, regular DOI unchanged, fetch_paper bare id). Confirmed live: re-ingesting 2305.10988 now UPDATEs in place. Full suite 370 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 --- codex/sources/openalex.py | 23 +++++++++++++++++++- tests/sources/test_openalex.py | 38 +++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/codex/sources/openalex.py b/codex/sources/openalex.py index f9be139..2ad1073 100644 --- a/codex/sources/openalex.py +++ b/codex/sources/openalex.py @@ -105,6 +105,27 @@ def _normalize_doi(doi: str) -> str: return s +# arXiv works are registered under a DOI of the form 10.48550/arXiv.. +_ARXIV_DOI_PREFIX = "10.48550/arxiv." + + +def _canonical_id(doi: str) -> str: + """Return the canonical ``papers.id`` for an OpenAlex ``doi`` value. + + Builds on :func:`_normalize_doi` (bare, lower-cased DOI), then strips the + arXiv DOI prefix ``10.48550/arxiv.`` so an arXiv work's id is the BARE arXiv + id (``2305.10988`` / ``math/0603097``) — the form the corpus is keyed on — + not the arXiv DOI. Without this, re-ingesting an arXiv paper by its bare id + misses ``INSERT … ON CONFLICT (id)`` and trips ``papers_openalex_id_key``. + This is the arXiv half of DQ-5 (the DOI half was fixed there; this was + explicitly deferred and surfaced again driving the R-C .tex re-ingest). + """ + normalized = _normalize_doi(doi) + if normalized.startswith(_ARXIV_DOI_PREFIX): + return normalized[len(_ARXIV_DOI_PREFIX) :] + return normalized + + def _map_paper(data: dict[str, Any]) -> Paper: authors: list[str] = [ a.get("author", {}).get("display_name") or "" for a in data.get("authorships", []) @@ -112,7 +133,7 @@ def _map_paper(data: dict[str, Any]) -> Paper: abstract = _reconstruct_abstract(data.get("abstract_inverted_index")) doi = data.get("doi") return Paper( - id=_normalize_doi(doi) if doi else (data.get("id") or ""), + id=_canonical_id(doi) if doi else (data.get("id") or ""), title=data.get("title") or "", openalex_id=data.get("id"), authors=authors, diff --git a/tests/sources/test_openalex.py b/tests/sources/test_openalex.py index 6b1e9a2..82c329b 100644 --- a/tests/sources/test_openalex.py +++ b/tests/sources/test_openalex.py @@ -7,7 +7,7 @@ import pytest from codex.models import Citation, Paper from codex.sources import openalex -from codex.sources.openalex import _normalize_doi, _resolve_id +from codex.sources.openalex import _canonical_id, _normalize_doi, _resolve_id # --------------------------------------------------------------------------- # Helpers @@ -97,6 +97,42 @@ def test_normalize_doi_already_bare_is_idempotent() -> None: assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8" +# --------------------------------------------------------------------------- +# _canonical_id — arXiv DOI → bare arXiv id (arXiv half of DQ-5) +# --------------------------------------------------------------------------- + + +def test_canonical_id_strips_arxiv_doi_modern() -> None: + # arXiv works carry the arXiv DOI; the canonical id is the bare arXiv id. + assert _canonical_id("https://doi.org/10.48550/arXiv.2305.10988") == "2305.10988" + + +def test_canonical_id_strips_arxiv_doi_legacy() -> None: + assert _canonical_id("https://doi.org/10.48550/arXiv.math/0603097") == "math/0603097" + + +def test_canonical_id_leaves_regular_doi_bare() -> None: + # A normal journal DOI is only normalized (bare + lower-case), not stripped. + assert _canonical_id("https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_fetch_paper_arxiv_doi_yields_bare_id(monkeypatch: pytest.MonkeyPatch) -> None: + """An arXiv work maps to the BARE arXiv id (not the arXiv DOI), so re-ingesting + by arXiv id hits ON CONFLICT (id) instead of tripping papers_openalex_id_key.""" + work = {**_SAMPLE_WORK, "doi": "https://doi.org/10.48550/arXiv.2305.10988"} + + def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response: + return httpx.Response(200, json=work) + + monkeypatch.setattr(openalex, "_get", mock_get) + paper = openalex.fetch_paper("2305.10988") + + assert paper is not None + assert paper.id == "2305.10988" + + def test_map_paper_falls_back_to_openalex_id_without_doi( monkeypatch: pytest.MonkeyPatch, ) -> None: From c22c0db3d0c5199e829527876cbc64ae1c821bfa Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 07:47:43 +0200 Subject: [PATCH 15/26] docs(audit): mark R-C done (section-aware .tex ingest + DQ-5 arXiv half) Co-Authored-By: Claude Opus 4.8 --- docs/audit/DATA-QUALITY-2026-06-15.md | 28 ++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 5719674..e15ec46 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -396,7 +396,33 @@ is self-contained so a cold session can pick it up. - **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where OpenAlex+S2 are empty. -### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **MED lever, MED-HIGH effort** +### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **DONE 2026-06-17** + +**Resolution (what was done) — all 13 arXiv papers re-ingested from `.tex`:** +- **Multi-file flatten:** `arxiv.fetch_source` now inlines `\input`/`\include` + (`tex.flatten_inputs`) so multi-file projects yield the full body, not the + primary file's include skeleton (e.g. math/0603097 = 15 files / 12 includes; + 2305.10988 = 6 / 5). Also handles legacy single-file **bare-gzipped** `.tex` + (no tar wrapper) for old math/* papers (commit 121b582). +- **Section-aware chunking:** new `tex.chunk_sections` returns `(title, chunk)` + pairs (chunks never span a `\section`); the ingest `.tex` path labels the stored + `section` from the real heading instead of running `classify_section` on a + header-less word-window. Quality-gated as pairs so labels stay aligned. +- **Unblocked by the arXiv half of DQ-5** (commit f504ca6): `_canonical_id` + strips the `10.48550/arxiv.` DOI prefix so an arXiv paper's id is the bare + arXiv id — without it, re-ingest tripped `papers_openalex_id_key` (confirmed + live, then fixed; idempotent re-ingest verified). +- **Outcome:** `section` column gained signal — ~34/329 arXiv chunks now + intro/theorem/proof (was ~all `body`); math/0603097 `{body:31}`→`{body:29, + proof:9}`, 2310.17529 gained proof+intro, etc. Math papers with descriptively + titled sections stay `body` (classify_section's fixed vocab) — modest but real + (R-12). No paper gutted (chunk counts stable/▲); F-16 gate unchanged. +- **Files:** `codex/parsing/tex.py`, `codex/sources/arxiv.py`, `codex/ingest.py`, + `codex/sources/openalex.py`, `scripts/rc_tex_reingest.py` (driver, dry-run + default), + tests. Full suite **370 passed**; ruff/mypy clean. Branch + `feat/tex-ingest` (commits 1da81c7, 121b582, f504ca6). + +**Original plan (for context):** - **What:** re-ingest from arXiv LaTeX source through the existing `codex.parsing.tex.latex_to_text` path instead of flattened `.txt`. - **Why:** highest-fidelity input — preserves math, structure, and section From 5672358bbe0b9570cae19bd5e80bed6dde7cb4f1 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 07:59:25 +0200 Subject: [PATCH 16/26] feat(graph): warn on low citing-paper coverage in graph report (R-D) Add graph.citing_coverage() (papers with >=1 out-edge / total) and a graph_min_citing_coverage setting (default 0.8). codex graph report now surfaces 'Citing coverage: N/M papers with out-edges (P%)' in text and JSON, and warns when the share is below threshold -- low citing coverage starves PageRank/coupling even when the paper count looks healthy (the DQ-1 sub-item). Separate from the existing total-count graph_min_corpus_size warning. Tests: citing_coverage unit tests + CLI tests (line shown, warning fires/suppressed, JSON field). Full suite green; ruff + mypy clean. Live: graph report shows 29/29 (100%), no warning. Co-Authored-By: Claude Opus 4.8 --- codex/cli.py | 24 ++++++++++++++++++++- codex/config.py | 12 +++++++++++ codex/graph.py | 14 +++++++++++++ docs/audit/DATA-QUALITY-2026-06-15.md | 18 +++++++++++++++- tests/graph/test_cli.py | 30 +++++++++++++++++++++++++++ tests/graph/test_graph.py | 22 ++++++++++++++++++++ 6 files changed, 118 insertions(+), 2 deletions(-) diff --git a/codex/cli.py b/codex/cli.py index 28130b7..9128de1 100644 --- a/codex/cli.py +++ b/codex/cli.py @@ -538,7 +538,12 @@ def graph_report( """Show citation graph report: top hub papers, dangling citations, cluster summary.""" from codex.config import get_settings from codex.db import get_conn - from codex.graph import build_citation_graph, citation_pagerank, dangling_citations + from codex.graph import ( + build_citation_graph, + citation_pagerank, + citing_coverage, + dangling_citations, + ) settings = get_settings() with get_conn() as conn: @@ -553,6 +558,8 @@ def graph_report( pr = citation_pagerank(graph) hubs = sorted(pr.items(), key=lambda x: -x[1])[:top_n] dangling = sorted(dangling_citations(graph, known_ids)) + n_citing, _ = citing_coverage(graph, known_ids) + coverage = n_citing / n_papers if n_papers else 0.0 if output_json: typer.echo( @@ -560,6 +567,11 @@ def graph_report( { "nodes": graph.number_of_nodes(), "edges": graph.number_of_edges(), + "citing_coverage": { + "citing": n_citing, + "papers": n_papers, + "fraction": round(coverage, 4), + }, "hubs": [{"paper_id": pid, "pagerank": score} for pid, score in hubs], "dangling_count": len(dangling), "dangling": dangling, @@ -575,7 +587,17 @@ def graph_report( f"(recommended ≥ {settings.graph_min_corpus_size} for meaningful ranking).", err=True, ) + # R-D: low citing-paper coverage starves PageRank/coupling even at a healthy + # paper count — warn on the share of papers that actually have out-edges. + if coverage < settings.graph_min_citing_coverage: + typer.echo( + f"Warning: only {n_citing}/{n_papers} papers ({coverage:.0%}) have out-edges " + f"(recommended ≥ {settings.graph_min_citing_coverage:.0%}); " + f"low citing coverage weakens PageRank/coupling.", + err=True, + ) typer.echo(f"Graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges") + typer.echo(f"Citing coverage: {n_citing}/{n_papers} papers with out-edges ({coverage:.0%})") typer.echo("") typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)") typer.echo("-" * 48) diff --git a/codex/config.py b/codex/config.py index dfe27f2..fbe59e4 100644 --- a/codex/config.py +++ b/codex/config.py @@ -310,6 +310,18 @@ class Settings(BaseSettings): ), ) + graph_min_citing_coverage: float = Field( + default=0.8, + ge=0.0, + le=1.0, + description=( + "Minimum share of ingested papers that must have ≥1 out-edge (i.e. " + "cite something) before `codex graph report` warns. Low citing " + "coverage starves PageRank/coupling even when the paper count looks " + "healthy — the share of *citing* papers is what matters (R-D)." + ), + ) + @lru_cache(maxsize=1) def get_settings() -> Settings: diff --git a/codex/graph.py b/codex/graph.py index 981662d..5f4a5a3 100644 --- a/codex/graph.py +++ b/codex/graph.py @@ -100,6 +100,20 @@ def citation_pagerank(graph: nx.DiGraph, *, damping: float = 0.85) -> dict[str, return cast(dict[str, float], nx.pagerank(graph, alpha=damping)) +def citing_coverage(graph: nx.DiGraph, known_ids: set[str]) -> tuple[int, int]: + """Return ``(papers_with_out_edges, total_papers)`` — the citing-paper coverage. + + A paper "covers" the citation graph when it has ≥1 out-edge (it cites at least + one work). Low citing coverage starves the F-15 layer (PageRank / coupling / + co-citation) even when the total paper count looks healthy, because those + metrics are driven by out-edges, not by node count. Surfaced by + ``codex graph report`` (R-D); the warning threshold is + :attr:`codex.config.Settings.graph_min_citing_coverage`. + """ + n_citing = sum(1 for pid in known_ids if pid in graph and graph.out_degree(pid) > 0) + return n_citing, len(known_ids) + + def find_related(paper_id: str, graph: nx.DiGraph, *, min_shared: int = 2) -> list[str]: """Bibliographic coupling: papers sharing ≥ ``min_shared`` references with ``paper_id``. diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 5719674..70ef798 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -408,7 +408,23 @@ is self-contained so a cold session can pick it up. - **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage vs source improves; no regression in the F-16 quality gate. -### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **SMALL** +### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **DONE 2026-06-17** + +**Resolution (what was done):** +- Added `graph.citing_coverage(graph, known_ids)` → `(papers_with_out_edges, + total)`, a `graph_min_citing_coverage` setting (default 0.8), and wired both into + `codex graph report`: it now prints `Citing coverage: N/M papers with out-edges + (P%)` (and in `--json`), and emits a `Warning` when the share is below the + threshold ("low citing coverage weakens PageRank/coupling"). Separate from the + existing `graph_min_corpus_size` (total-count) warning. +- Tests: `citing_coverage` unit tests + CLI tests (line shown; warning fires at + low coverage; suppressed at 100%; JSON field). Full suite green; ruff/mypy clean. +- **Live:** `codex graph report` on the corpus prints `Citing coverage: 29/29 + papers with out-edges (100%)` with no warning (post-R-A). Branch + `feat/graph-coverage-warning`. Files: `codex/config.py`, `codex/graph.py`, + `codex/cli.py`, + tests. + +**Original plan (for context):** - **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper count. Add a warning when the share of papers with ≥1 out-edge is low (e.g. < 80%), since that is what actually starves PageRank/coupling. diff --git a/tests/graph/test_cli.py b/tests/graph/test_cli.py index 32ee60c..2dfb449 100644 --- a/tests/graph/test_cli.py +++ b/tests/graph/test_cli.py @@ -117,6 +117,36 @@ class TestGraphReport: assert result.exit_code == 0 assert "Warning" in result.output or "warning" in result.output + def test_shows_citing_coverage_line(self): + # _sample_graph: A, B, C all cite → 3/3 = 100%, so the line shows but no warning. + out = self._run(known_ids=("A", "B", "C")).output + assert "Citing coverage" in out + assert "weakens" not in out # 100% coverage → no R-D warning + + def test_low_citing_coverage_warning(self): + # Only A has an out-edge; B and C are ingested but cite nothing → 1/3 = 33%. + g = nx.DiGraph() + g.add_edge("A", "X") + g.add_nodes_from(["B", "C"]) + result = self._run(graph=g, known_ids=("A", "B", "C")) + assert result.exit_code == 0 + assert "weakens PageRank" in result.output # the R-D coverage warning + + def test_json_includes_citing_coverage(self): + import json + + g = nx.DiGraph() + g.add_edge("A", "X") + g.add_nodes_from(["B", "C"]) + conn = _make_conn_with_paper_ids("A", "B", "C") + with ( + patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)), + patch("codex.graph.build_citation_graph", return_value=g), + ): + result = runner.invoke(app, ["graph", "report", "--json"]) + data = json.loads(result.output) + assert data["citing_coverage"] == {"citing": 1, "papers": 3, "fraction": 0.3333} + # --------------------------------------------------------------------------- # codex graph related diff --git a/tests/graph/test_graph.py b/tests/graph/test_graph.py index c68bdfa..49c9abe 100644 --- a/tests/graph/test_graph.py +++ b/tests/graph/test_graph.py @@ -10,6 +10,7 @@ import pytest from codex.graph import ( build_citation_graph, citation_pagerank, + citing_coverage, dangling_citations, find_co_cited, find_related, @@ -287,3 +288,24 @@ class TestDanglingCitations: g = build_citation_graph(_make_conn(rows)) dangling = dangling_citations(g, set()) assert set(dangling) == {"A", "B"} + + +# --------------------------------------------------------------------------- +# citing_coverage (R-D) +# --------------------------------------------------------------------------- + + +class TestCitingCoverage: + def test_counts_papers_with_out_edges(self): + # _small_graph: A→.., B→.., C→D have out-edges; D, E have none. + g = _small_graph() + n_citing, n_papers = citing_coverage(g, {"A", "B", "C", "D", "E"}) + assert (n_citing, n_papers) == (3, 5) + + def test_paper_absent_from_graph_counts_as_no_out_edges(self): + # A cites; Z was ingested but has no chunks/edges, so it isn't a graph node. + g = _small_graph() + assert citing_coverage(g, {"A", "Z"}) == (1, 2) + + def test_empty(self): + assert citing_coverage(nx.DiGraph(), set()) == (0, 0) From c70ff9aa1e5b27854082ec0bd140cce130fda6d1 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 09:03:46 +0200 Subject: [PATCH 17/26] feat(quality): store real section titles for .tex chunks (R-F) Add quality.section_label(title, content): for section-aware .tex ingest, keep the controlled bucket (intro/theorem/proof/abstract/bibliography) when the real \section heading maps to one, else store the cleaned title verbatim (e.g. 'preliminaries', 'rigidity') instead of collapsing to 'body'. Math papers title most sections descriptively, so R-C's vocab-only mapping left ~90% as 'body'; this lifts the section signal toward near-full. .pdf/.txt/run_quality_pass keep content-based classify_section unchanged. Safe because the section column is write-only (no consumer filters on the vocab). New _clean_title (strip LaTeX/numbering, lower-case, truncate). Tests: section_label bucket/verbatim/fallback paths + _clean_title; .tex ingest stores a descriptive heading as its title. Full suite 377 passed; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 11 ++++++----- codex/quality.py | 36 +++++++++++++++++++++++++++++++++++ tests/ingest/test_ingest.py | 7 +++++-- tests/quality/test_quality.py | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index ab79250..43db683 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -12,7 +12,7 @@ from codex.config import get_settings from codex.db import get_conn from codex.embed import get_embedder from codex.models import Citation, Paper -from codex.quality import classify_section, is_quality_chunk +from codex.quality import is_quality_chunk, section_label from codex.sources import arxiv, crossref, openalex, semanticscholar logger = logging.getLogger(__name__) @@ -312,10 +312,11 @@ def ingest_paper( ord_idx, content, chunk_embeddings[ord_idx].tolist(), - # .tex: classify from the section heading (prepended to the - # snippet) so the label reflects the real section; otherwise - # fall back to content-only classification. - classify_section(f"{title}. {content}" if title else content), + # .tex: label from the real \section heading — keeps the + # controlled bucket when it matches, else stores the cleaned + # title (R-F); .txt/.pdf (title=None) classify by content. + # NB: `section` semantics now differ by source (write-only col). + section_label(title, content), ) for ord_idx, (title, content) in enumerate(kept_pairs) ] diff --git a/codex/quality.py b/codex/quality.py index ef6ee72..c2f1b94 100644 --- a/codex/quality.py +++ b/codex/quality.py @@ -110,6 +110,42 @@ def classify_section(text: str) -> str: return "body" +def _clean_title(title: str) -> str: + """Normalize a LaTeX ``\\section`` title for use as a section label. + + Strips LaTeX commands (``\\emph`` …) and ``{}$``, drops leading section + numbering (``3.2 ``), collapses whitespace, lower-cases, and truncates to + 60 chars so the stored ``section`` value is a clean, comparable string. + """ + t = re.sub(r"\\[a-zA-Z]+\*?", " ", title) # LaTeX commands + t = re.sub(r"[{}$]", "", t) # braces / math delimiters + t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 " + return " ".join(t.split()).strip().lower()[:60] + + +def section_label(title: str | None, content: str) -> str: + """Label a chunk's section, preferring the real ``\\section`` heading (R-F). + + For section-aware ``.tex`` ingest *title* is the real heading: if it maps to a + controlled bucket via :func:`classify_section` (intro / theorem / proof / + abstract / bibliography) that bucket is kept (cross-source consistency); + otherwise the cleaned title itself is stored (e.g. ``"preliminaries"``, + ``"rigidity"``) — far more signal than collapsing everything to ``body``. + + When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass`` + paths) this falls back to content-based :func:`classify_section` — unchanged + behaviour. Safe to store free-text titles because the ``section`` column is + write-only (no consumer filters on the controlled vocabulary). + """ + if not title or not title.strip(): + return classify_section(content) + cleaned = _clean_title(title) + if not cleaned: + return classify_section(content) + bucket = classify_section(cleaned) + return bucket if bucket != "body" else cleaned + + # --------------------------------------------------------------------------- # Batch filter # --------------------------------------------------------------------------- diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 8828419..0207b49 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -128,6 +128,7 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: section_pairs = [ ("Introduction", "Body of the introduction goes here."), ("Proof of the Main Theorem", "Body of the proof goes here."), + ("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."), ] with ( @@ -151,9 +152,11 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: chunk_sql: str = chunk_insert_call[0][0] assert "INSERT INTO chunks" in chunk_sql - # Section label comes from the heading: "Introduction" → intro, "Proof of…" → proof. + # R-F: heading → controlled bucket where it maps ("Introduction"→intro, + # "Proof of…"→proof), else the cleaned real title ("discrete conformal maps") + # instead of collapsing to "body". chunk_rows = chunk_insert_call[0][1] - assert [row[4] for row in chunk_rows] == ["intro", "proof"] + assert [row[4] for row in chunk_rows] == ["intro", "proof", "discrete conformal maps"] assert result.chunks_upserted == len(section_pairs) diff --git a/tests/quality/test_quality.py b/tests/quality/test_quality.py index b131d2f..2cfa631 100644 --- a/tests/quality/test_quality.py +++ b/tests/quality/test_quality.py @@ -6,10 +6,12 @@ from unittest.mock import MagicMock from codex.quality import ( _bib_score, + _clean_title, classify_section, filter_chunks, is_quality_chunk, run_quality_pass, + section_label, ) # --------------------------------------------------------------------------- @@ -243,3 +245,36 @@ class TestRunQualityPass: kwargs = update_calls[0][0][1] assert kwargs["section"] == "abstract" assert kwargs["id"] == 5 + + +class TestCleanTitle: + def test_strips_leading_numbering(self): + assert _clean_title("3.2 Variational Principles") == "variational principles" + + def test_strips_latex_and_braces(self): + assert _clean_title(r"The \emph{Discrete} Laplacian") == "the discrete laplacian" + + def test_collapses_and_lowercases(self): + assert _clean_title(" Rigidity Results ") == "rigidity results" + + +class TestSectionLabel: + def test_title_maps_to_controlled_bucket(self): + # Headings matching the controlled vocab keep the canonical bucket. + assert section_label("Introduction", "body text") == "intro" + assert section_label("Proof of Theorem 3", "body text") == "proof" + assert section_label("References", "body text") == "bibliography" + + def test_descriptive_title_stored_verbatim(self): + # R-F win: a non-vocab heading is stored as its cleaned title, not "body". + assert section_label("Preliminaries", "body text") == "preliminaries" + assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps" + + def test_none_or_blank_title_falls_back_to_content(self): + # .pdf/.txt path: unchanged content-based classification. + assert section_label(None, "Theorem 1. Let X be ...") == "theorem" + assert section_label("", "ordinary body prose here") == "body" + + def test_title_cleaning_to_empty_falls_back_to_content(self): + # A pure-numbering heading cleans to "" → classify by content instead. + assert section_label("3.2", "Lemma 2. Suppose ...") == "theorem" From 1a879d48277f41aa7de2488b04f57b2e518ad95b Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 09:17:43 +0200 Subject: [PATCH 18/26] feat(quality): clean LaTeX accents/ties + word-boundary truncation in section titles Polish _clean_title for R-F: strip accent/escape macros (backslash-quote o to o, backslash-amp), convert tilde ties to spaces, and truncate at a word boundary so stored section titles read cleanly (mobius invariance, colin de verdiere). Tests added; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 --- codex/quality.py | 14 +++++++++----- tests/quality/test_quality.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/codex/quality.py b/codex/quality.py index c2f1b94..7dc21f8 100644 --- a/codex/quality.py +++ b/codex/quality.py @@ -113,14 +113,18 @@ def classify_section(text: str) -> str: def _clean_title(title: str) -> str: """Normalize a LaTeX ``\\section`` title for use as a section label. - Strips LaTeX commands (``\\emph`` …) and ``{}$``, drops leading section - numbering (``3.2 ``), collapses whitespace, lower-cases, and truncates to - 60 chars so the stored ``section`` value is a clean, comparable string. + Strips LaTeX word-commands (``\\emph`` …), accent/escape macros (``\\"o`` → + ``o``, ``\\&``), ``~`` ties and ``{}$``, drops leading section numbering + (``3.2 ``), collapses whitespace, lower-cases, and truncates to 60 chars at a + word boundary so the stored ``section`` value is a clean, comparable string. """ - t = re.sub(r"\\[a-zA-Z]+\*?", " ", title) # LaTeX commands + t = title.replace("~", " ") # LaTeX non-breaking tie → space + t = re.sub(r"\\[a-zA-Z]+\*?", " ", t) # word-commands: \emph, \mathcal … + t = re.sub(r"\\[^a-zA-Z]", "", t) # accent/escape macros: \"o → o, \', \`, \& t = re.sub(r"[{}$]", "", t) # braces / math delimiters t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 " - return " ".join(t.split()).strip().lower()[:60] + t = " ".join(t.split()).strip().lower() + return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t def section_label(title: str | None, content: str) -> str: diff --git a/tests/quality/test_quality.py b/tests/quality/test_quality.py index 2cfa631..4895c30 100644 --- a/tests/quality/test_quality.py +++ b/tests/quality/test_quality.py @@ -257,6 +257,19 @@ class TestCleanTitle: def test_collapses_and_lowercases(self): assert _clean_title(" Rigidity Results ") == "rigidity results" + def test_strips_accent_macros(self): + assert _clean_title(r"M\"obius Invariance") == "mobius invariance" + + def test_strips_ties_and_accents(self): + assert _clean_title(r"Colin~de~Verdi\`ere") == "colin de verdiere" + + def test_truncates_at_word_boundary(self): + long = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu omega" + out = _clean_title(long) + assert len(out) <= 60 + assert not out.endswith(" ") + assert long.lower().startswith(out) # whole words from the start, no partial tail + class TestSectionLabel: def test_title_maps_to_controlled_bucket(self): From 5c60a7eda4ad3e5fdb87c45b5a6a83099d679e33 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 09:25:03 +0200 Subject: [PATCH 19/26] =?UTF-8?q?docs(audit):=20add=20R-F=20(richer=20sect?= =?UTF-8?q?ion=20labels)=20resolution=20=E2=80=94=2010%=20to=2095%=20non-b?= =?UTF-8?q?ody?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- docs/audit/DATA-QUALITY-2026-06-15.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index e15ec46..8058482 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -454,6 +454,31 @@ is self-contained so a cold session can pick it up. violates most publisher terms and can get the whole institution's access revoked.** Do not wire a publisher login into the ingest pipeline. +### R-F — Richer section labels: store real `\section` titles · **DONE 2026-06-17** + +**Resolution (what was done):** +- **Why:** R-C labelled `.tex` chunks by mapping the real `\section` heading through + `classify_section`'s fixed vocab, so descriptively-titled Math sections collapsed + to `body` (only **34/329 = 10 %** non-`body`). A research pass confirmed the + `section` column is **write-only** (nothing filters on the vocab — `mcp_server.py` + and `wiki.py` don't even SELECT it), so storing free-text titles is safe. +- **Change:** new `quality.section_label(title, content)` (+ `_clean_title`) — keep the + controlled bucket (intro/theorem/proof/abstract/bibliography) when the heading maps + to one, else store the cleaned real title ("the flip algorithm", "main results", …). + The `.tex` ingest path uses it; `.pdf`/`.txt`/`run_quality_pass` keep + `classify_section` unchanged. `_clean_title` strips LaTeX commands/accents/ties + + leading numbering, lower-cases, truncates at a word boundary. ~1 helper + 1 ingest + line; no schema change (column already free `TEXT`). +- **Result (live):** re-applied to the 13 arXiv papers → non-`body` **34/329 (10 %) → + 314/329 (95 %)**. Only `math/0306167` (old AMS-TeX, no `\section{}`) stays `body`. +- **Tests:** `section_label` / `_clean_title` units + `.tex` ingest stores a + descriptive heading verbatim. Full suite 377 passed; ruff/mypy clean. Branch + `feat/section-labels` (off `feat/tex-ingest`). +- **Pending (infra, not code):** an accent-cleanup polish (`m\"obius`→`mobius`) landed + in `_clean_title`, and a quick in-place `UPDATE` re-clean (no re-embed) was started, + but the Jetson went offline (ping 100 % loss) before it finished — a few live labels + still show LaTeX accents/ties. Re-run the in-place re-clean when the Jetson is back. + --- ## Priority for the next session From a050f6622a3ea7878bfdff0cce91f7e471f46514 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Thu, 18 Jun 2026 09:32:03 +0200 Subject: [PATCH 20/26] =?UTF-8?q?docs(infra):=20TODO=20=E2=80=94=20tmux-on?= =?UTF-8?q?-Jetson=20+=20autossh=20for=20stable=20long-running=20ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Mac->Jetson SSH tunnel dropped repeatedly during multi-minute live writes (re-ingest, embed loops, sweeps), aborting them partway. Document the fix: run long ops on the Jetson inside tmux (DB is localhost, no tunnel; survives SSH drops), use autossh for interactive tunnels, and optionally make backfill writes per-item resilient. Co-Authored-By: Claude Opus 4.8 --- docs/infra/jetson-ssh-stability.md | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/infra/jetson-ssh-stability.md diff --git a/docs/infra/jetson-ssh-stability.md b/docs/infra/jetson-ssh-stability.md new file mode 100644 index 0000000..2358f21 --- /dev/null +++ b/docs/infra/jetson-ssh-stability.md @@ -0,0 +1,66 @@ +# Infra TODO — stable long-running ops against the Jetson (tmux + autossh) + +**Status:** OPEN (infra). **Filed:** 2026-06-17. **Priority:** MED — it slows live ops +but has a known workaround (short/idempotent ops). + +## Problem (observed) + +Live data ops run on the **Mac** and reach the Jetson Postgres through an SSH +port-forward tunnel (`ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103`, +`DATABASE_URL` → `localhost:5433`). During the R-A / R-C / R-F work this tunnel +**dropped repeatedly mid-run**: any operation longer than a few minutes (corpus +re-ingest, the bge-m3 embed loop, the GROBID sweep) had a high chance of failing +partway when the tunnel died, leaving the work half-applied. At one point the Jetson +went **fully offline** (ping 100 % loss, `ssh:22` timeout) mid-operation. + +`ingest_paper()` and the backfill scripts open one DB connection for the whole run, so +a tunnel drop raises `OperationalError: connection refused` and aborts — partial state, +manual retry. Adding `ServerAliveInterval` to the tunnel helped only marginally. + +## Recommended fix + +**1. Run long ops *on the Jetson* inside `tmux` (primary fix).** +The Jetson is where Postgres (and GROBID) live, so running ingest there means the DB is +`localhost` — **no tunnel at all** — and `tmux` keeps the process alive across SSH +disconnects (detach / reattach to monitor). This removes the tunnel from the critical +path for every long-running write. + +- [ ] Confirm the `codex` env is deployable on the Jetson (Python 3.12+, deps incl. the + bge-m3 embedder; the Orin Nano GPU should handle it — GROBID already runs there). + If not, stand up a venv / container for it. +- [ ] Standard pattern: `ssh alfred@jetson`, then + `tmux new -s codex` → `set -a; source .env; set +a` → + `PYTHONPATH=. python scripts/.py --write` → detach (`Ctrl-b d`). + Reattach later with `tmux attach -t codex`. The job survives the SSH drop. +- [ ] Point the Jetson-side `DATABASE_URL` at `localhost:5432` directly (no `:5433`). +- [ ] Document this as the default for any multi-minute write (re-ingest, sweeps, + embed loops) in `docs/audit/DATA-QUALITY-2026-06-15.md` "How to reach the data". + +**2. `autossh` for when a tunnel *is* needed (secondary).** +For interactive/dev use from the Mac (psql, quick probes, read-only queries), replace +the plain `ssh -f -N -L` with `autossh -M 0 -f -N -o ServerAliveInterval=15 -o +ServerAliveCountMax=3 -L 5433:localhost:5432 alfred@jetson` so a dropped tunnel +auto-reconnects. NB: autossh only restarts the *tunnel* — an in-flight transaction +still fails, so this is for short/interactive use, not long writes (use tmux-on-Jetson +for those). + +**3. (Optional, code) make backfill scripts resilient.** +Lower-value once (1)/(2) are in place, but: open a fresh `get_conn()` per paper (not one +for the whole run) and/or wrap the per-item write in a small retry, so a transient drop +costs one item, not the whole batch. The R-A backfill already does per-paper connections; +`rc_tex_reingest.py` opens a connection per paper too — but `ingest_paper` itself holds +one connection for its whole body, which is the unit that fails. + +## Why this matters + +The chronic flakiness directly cost time this session and left one cosmetic task +unfinished (the R-F accent re-clean — see DATA-QUALITY-2026-06-15.md R-F "Pending"). The +**robust pattern that already works** is short, idempotent operations (per-paper, or a +single quick `UPDATE`); tmux-on-Jetson makes even the long ops safe. + +## Acceptance + +- A documented, repeatable way to run a full corpus re-ingest that survives an SSH drop + (tmux-on-Jetson), verified by detaching/reattaching across a disconnect. +- The DATA-QUALITY doc's "How to reach the data" section recommends tmux-on-Jetson for + long writes and `autossh` for interactive tunnels. From 200b7f8188af6f432315f553cf527ba980760da1 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 22 Jun 2026 01:48:47 +0200 Subject: [PATCH 21/26] docs: session 2026-06-17 open-items handoff notice Consolidates the session's loose ends: Jetson offline (blocker), R-F accent re-clean pending, 5 feature branches unmerged to main (+ merge order), tmux-on-Jetson infra TODO, and documented known limits. Co-Authored-By: Claude Opus 4.8 --- docs/SESSION-2026-06-17-open-items.md | 52 +++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/SESSION-2026-06-17-open-items.md diff --git a/docs/SESSION-2026-06-17-open-items.md b/docs/SESSION-2026-06-17-open-items.md new file mode 100644 index 0000000..4133877 --- /dev/null +++ b/docs/SESSION-2026-06-17-open-items.md @@ -0,0 +1,52 @@ +# Open items — session 2026-06-17 (handoff notice) + +Everything still open after this session. Done items are in +`docs/audit/DATA-QUALITY-2026-06-15.md` (DQ-1..DQ-5 + roadmap R-A,R-B,R-C,R-D,R-F all +DONE; R-E deferred). This file is only the **loose ends**. + +## 1. BLOCKER — Jetson is offline +- The Jetson (`192.168.178.103`) went **unreachable** mid-session: `ping` 100 % loss, + `ssh:22` timeout. It hosts Postgres (the live corpus) **and** GROBID. +- **Blocks every live-DB action below.** First step next session: confirm it's powered + + on the network, bring the SSH tunnel back up + (`ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103`). + +## 2. PENDING — R-F accent re-clean (cosmetic, code already merged into the branch) +- R-F is functionally DONE (section signal 34/329 → **314/329 = 95 %**), but a few live + `chunks.section` labels still carry LaTeX accents/ties (e.g. `m\"obius`, `colin~de~…`). +- The `_clean_title` polish that fixes these is committed on `feat/section-labels`; the + **in-place re-clean of existing rows didn't finish** (Jetson went offline, item #1). +- **Action when Jetson is back (short UPDATE, no re-ingest):** apply + `codex.quality._clean_title` to every `chunks.section` containing `\` or `~` and write + it back. Quick + idempotent — exactly the kind of short op that survives a flaky link. + +## 3. Git — feature branches not merged to `main` +This session's work is on 5 branches off `audit/loop-1` (itself **18 commits ahead of +`main`**). No PRs opened. Merge decision + order pending: + +| Branch | Contents | Commits ahead of main | +|---|---|---| +| `audit/loop-1` | base: DQ-1..DQ-5, **R-A** (GROBID backfill + native arm64 GROBID) | 18 | +| `feat/tex-ingest` | **R-C** (.tex section-aware ingest) + DQ-5 arXiv-id fix + gzip fallback | 22 | +| `feat/section-labels` | **R-F** (richer section labels) — **branched off `feat/tex-ingest`** | 25 | +| `feat/crossref-references` | **R-B** (Crossref refs) + a concurrent GROBID-on-demand tweak | 20 | +| `feat/graph-coverage-warning` | **R-D** (citing-coverage warning) | 19 | +| `chore/infra-ssh-stability` | infra TODO doc (#4) | 19 | + +- **Order matters:** `feat/section-labels` contains `feat/tex-ingest`, so merge + `feat/tex-ingest` first (or merge section-labels alone, which brings both). +- All branches share the `audit/loop-1` base, so merging any one brings that whole line + to `main` — decide whether to land `audit/loop-1` → `main` as the trunk first. + +## 4. Infra TODO — tmux-on-Jetson for stable long ops (documented, not implemented) +- `docs/infra/jetson-ssh-stability.md` (branch `chore/infra-ssh-stability`). The + Mac↔Jetson tunnel dropped repeatedly on multi-minute writes all session. Fix: run long + ops **on the Jetson in `tmux`** (DB is localhost there → no tunnel; survives SSH drops); + `autossh` for interactive tunnels. Open. + +## 5. Known limitations (documented, NOT action items) +- `math/0306167` (old AMS-TeX, no `\section{}`) stays `section=body` — inherent. +- The 2 edited-volume books yielded 0 GROBID-parsed DOI/arXiv refs in the R-A sweep — + inherent (per-chapter refs, no clean end-bibliography). +- **DQ-6** (upstream PDF→`.txt` extraction fidelity, done outside codex-py) — flagged + earlier, largely **mooted** by R-A/R-C ingesting from PDF/`.tex` directly. From f761ee63f82199fe02401c8d446cc5c52af40c58 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 22 Jun 2026 01:53:36 +0200 Subject: [PATCH 22/26] =?UTF-8?q?docs(audit):=20R-F=20accent=20re-clean=20?= =?UTF-8?q?done=20=E2=80=94=208=20labels=20fixed=20in=20place,=200=20artif?= =?UTF-8?q?acts=20remain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- docs/audit/DATA-QUALITY-2026-06-15.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index 8058482..1d2fe19 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -474,10 +474,11 @@ is self-contained so a cold session can pick it up. - **Tests:** `section_label` / `_clean_title` units + `.tex` ingest stores a descriptive heading verbatim. Full suite 377 passed; ruff/mypy clean. Branch `feat/section-labels` (off `feat/tex-ingest`). -- **Pending (infra, not code):** an accent-cleanup polish (`m\"obius`→`mobius`) landed - in `_clean_title`, and a quick in-place `UPDATE` re-clean (no re-embed) was started, - but the Jetson went offline (ping 100 % loss) before it finished — a few live labels - still show LaTeX accents/ties. Re-run the in-place re-clean when the Jetson is back. +- **Accent re-clean — DONE 2026-06-17.** The `_clean_title` polish (`m\"obius`→`mobius`, + `~` ties, word-boundary truncation) landed in code, and the existing live labels were + re-cleaned in place (apply `_clean_title` to any `chunks.section` containing `\` or + `~`; **8 labels** fixed, no re-embed). Verified: **0** labels with LaTeX artifacts + remain; non-`body` holds at **314/329 (95 %)**. R-F fully complete. --- From 49c251eafa21c2be49b7084b159c4dd4f639c6f6 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 22 Jun 2026 01:54:32 +0200 Subject: [PATCH 23/26] =?UTF-8?q?docs:=20open-items=20=E2=80=94=20Jetson?= =?UTF-8?q?=20back=20(tunnel=20had=20dropped),=20R-F=20accent=20re-clean?= =?UTF-8?q?=20done?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- docs/SESSION-2026-06-17-open-items.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/docs/SESSION-2026-06-17-open-items.md b/docs/SESSION-2026-06-17-open-items.md index 4133877..7e223d3 100644 --- a/docs/SESSION-2026-06-17-open-items.md +++ b/docs/SESSION-2026-06-17-open-items.md @@ -4,21 +4,18 @@ Everything still open after this session. Done items are in `docs/audit/DATA-QUALITY-2026-06-15.md` (DQ-1..DQ-5 + roadmap R-A,R-B,R-C,R-D,R-F all DONE; R-E deferred). This file is only the **loose ends**. -## 1. BLOCKER — Jetson is offline -- The Jetson (`192.168.178.103`) went **unreachable** mid-session: `ping` 100 % loss, - `ssh:22` timeout. It hosts Postgres (the live corpus) **and** GROBID. -- **Blocks every live-DB action below.** First step next session: confirm it's powered - + on the network, bring the SSH tunnel back up - (`ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103`). +## 1. ✅ RESOLVED — Jetson is online (the tunnel had dropped, not the host) +- What looked like the Jetson going offline was the **SSH tunnel** breaking, not the + host. The Jetson (`192.168.178.103`) is online. Re-open the tunnel with keepalive: + `ssh -o ServerAliveInterval=15 -f -N -L 5433:localhost:5432 alfred@192.168.178.103`. + Better still: run long ops in tmux-on-Jetson (item #4) to avoid the tunnel entirely. -## 2. PENDING — R-F accent re-clean (cosmetic, code already merged into the branch) -- R-F is functionally DONE (section signal 34/329 → **314/329 = 95 %**), but a few live - `chunks.section` labels still carry LaTeX accents/ties (e.g. `m\"obius`, `colin~de~…`). -- The `_clean_title` polish that fixes these is committed on `feat/section-labels`; the - **in-place re-clean of existing rows didn't finish** (Jetson went offline, item #1). -- **Action when Jetson is back (short UPDATE, no re-ingest):** apply - `codex.quality._clean_title` to every `chunks.section` containing `\` or `~` and write - it back. Quick + idempotent — exactly the kind of short op that survives a flaky link. +## 2. ✅ DONE — R-F accent re-clean +- R-F is functionally DONE (section signal 34/329 → **314/329 = 95 %**). The remaining + cosmetic accent/tie artifacts in live `chunks.section` labels were re-cleaned in place + once the tunnel was back: applied `codex.quality._clean_title` to every label + containing `\` or `~` — **8 labels fixed**, no re-embed. Verified **0 artifacts** + remain. R-F fully complete. ## 3. Git — feature branches not merged to `main` This session's work is on 5 branches off `audit/loop-1` (itself **18 commits ahead of From e817022097266792174654e038a83675a9e13584 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 22 Jun 2026 02:01:38 +0200 Subject: [PATCH 24/26] docs: mark integration/roadmap merge-ready (391 green) + cleanup notes Co-Authored-By: Claude Opus 4.8 --- docs/SESSION-2026-06-17-open-items.md | 40 +++++++++++++++++---------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/SESSION-2026-06-17-open-items.md b/docs/SESSION-2026-06-17-open-items.md index 7e223d3..1d0a8d7 100644 --- a/docs/SESSION-2026-06-17-open-items.md +++ b/docs/SESSION-2026-06-17-open-items.md @@ -17,23 +17,33 @@ DONE; R-E deferred). This file is only the **loose ends**. containing `\` or `~` — **8 labels fixed**, no re-embed. Verified **0 artifacts** remain. R-F fully complete. -## 3. Git — feature branches not merged to `main` -This session's work is on 5 branches off `audit/loop-1` (itself **18 commits ahead of -`main`**). No PRs opened. Merge decision + order pending: +## 3. ✅ READY TO MERGE — all branches integrated + validated +All five feature branches are merged (**zero conflicts**) into **`integration/roadmap`** +(off `audit/loop-1`) and validated green: **391 tests pass, ruff + ruff-format + mypy +clean, tree clean**. This is the single merge candidate for `main`. -| Branch | Contents | Commits ahead of main | -|---|---|---| -| `audit/loop-1` | base: DQ-1..DQ-5, **R-A** (GROBID backfill + native arm64 GROBID) | 18 | -| `feat/tex-ingest` | **R-C** (.tex section-aware ingest) + DQ-5 arXiv-id fix + gzip fallback | 22 | -| `feat/section-labels` | **R-F** (richer section labels) — **branched off `feat/tex-ingest`** | 25 | -| `feat/crossref-references` | **R-B** (Crossref refs) + a concurrent GROBID-on-demand tweak | 20 | -| `feat/graph-coverage-warning` | **R-D** (citing-coverage warning) | 19 | -| `chore/infra-ssh-stability` | infra TODO doc (#4) | 19 | +Contents (DQ-1..DQ-5 + R-A..R-F + infra docs): +| Source branch | Contents | +|---|---| +| `audit/loop-1` (base) | DQ-1..DQ-5, **R-A** (GROBID backfill + native arm64 GROBID), this notice | +| `feat/section-labels` (incl. `feat/tex-ingest`) | **R-C** (.tex ingest) + DQ-5 arXiv-id fix + gzip fallback + **R-F** (section labels) | +| `feat/crossref-references` | **R-B** (Crossref refs) + GROBID-on-demand tweak | +| `feat/graph-coverage-warning` | **R-D** (citing-coverage warning) | +| `chore/infra-ssh-stability` | infra TODO (#4) | -- **Order matters:** `feat/section-labels` contains `feat/tex-ingest`, so merge - `feat/tex-ingest` first (or merge section-labels alone, which brings both). -- All branches share the `audit/loop-1` base, so merging any one brings that whole line - to `main` — decide whether to land `audit/loop-1` → `main` as the trunk first. +**To land on `main`** (user-triggered): +``` +git checkout main && git merge --no-ff integration/roadmap +``` +…or open a PR `integration/roadmap` → `main`. The individual feature branches are +preserved unchanged for per-feature review if preferred. + +**Cleanup — branches prunable AFTER the merge lands** (not deleted here; destructive): +- Already in `main`: `fix/cli-readme-drift`, `fix/d04-d05-ingest-fixes`. +- Folded into `integration/roadmap`: the 5 feature branches + `audit/loop-1`. +- Pre-session / stale (verify before pruning): `feat/F-15-literature-graph`, + `feat/F-16-chunk-quality`, `fix/audit-wave-1`/`-2`/`-3`, `scratch/spike-embed`. +- Working tree is clean — only gitignored `.venv`/caches/`.env`; no stray tracked files. ## 4. Infra TODO — tmux-on-Jetson for stable long ops (documented, not implemented) - `docs/infra/jetson-ssh-stability.md` (branch `chore/infra-ssh-stability`). The From b2f01ce0108a4cec5f8bddb17a369b62d2433611 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Fri, 26 Jun 2026 22:50:23 +0200 Subject: [PATCH 25/26] fix(review): address PR #16 code-review findings #1 ingest: normalize the pinned caller id (_norm_cited_id) so a non-bare DOI caller cannot store a URL-form papers.id that defeats DQ-5 idempotency / the startswith(10.) recovery gates. #2 quality.section_label: collapse to a controlled bucket only for an EXACT canonical heading; descriptive titles ('Abstract Nonsense...') keep their real title instead of being mislabelled. #4 ra_grobid_backfill: release the read connection before the slow GROBID network loop, fresh connection for the write (no idle-in-transaction across the loop over the flaky tunnel). #5/#10 tex: flatten_inputs strips unresolved input/include at the depth cap (no literal leak on cycles); _norm_texkey strips only a single leading ./ . #6/#7 arxiv.fetch_source: keep non-.tex members resolvable for input; pick primary on an UN-commented documentclass line. #13 is_arxiv_id: also exclude http:// and arXiv-DOI forms. Tests added/updated for each. Left as deliberate decisions: #3 (pre-section text drop is pre-existing in extract_sections; abstract stored separately), #8 (script normalizer is intentionally self-contained, already documented), #9/#11/#12 (no current trigger / tightening the gzip heuristic would reject valid old LaTeX like documentstyle / title truncation is by design on a write-only column). ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 --- codex/ingest.py | 14 ++-- codex/parsing/tex.py | 8 ++- codex/quality.py | 33 ++++++++-- codex/sources/arxiv.py | 33 ++++++++-- scripts/ra_grobid_backfill.py | 95 ++++++++++++++------------- scripts/rc_tex_reingest.py | 8 ++- tests/ingest/test_ingest.py | 12 ++-- tests/parsing/test_tex.py | 12 ++++ tests/quality/test_quality.py | 17 +++-- tests/scripts/test_rc_tex_reingest.py | 2 + tests/sources/test_arxiv.py | 31 +++++++++ 11 files changed, 190 insertions(+), 75 deletions(-) diff --git a/codex/ingest.py b/codex/ingest.py index 6518d74..9381c69 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -207,12 +207,14 @@ def ingest_paper( raise ValueError(f"Paper not found: {paper_id}") # Canonical identity is the caller-supplied id — what the CLI / ingest scripts - # use and what papers.id is contracted to hold (arXiv id or DOI). OpenAlex's - # fetch_paper fills paper.id with the URL-form doi/id (audit C-7); since DQ-5 - # `openalex._canonical_id` already normalizes that to the bare form, this pin - # is belt-and-suspenders and keeps re-ingest idempotent regardless. The - # OpenAlex work id is retained as openalex_id and in paper_identifiers. - paper.id = paper_id.strip() + # use and what papers.id is contracted to hold (arXiv id or DOI). Pin it to the + # caller's id (audit C-7), but normalize it the same way cited-ids are (strip a + # https://doi.org/ or doi: prefix, lower-case DOIs) so a non-bare caller cannot + # store a URL-form papers.id that would defeat DQ-5's ON CONFLICT (id) idempotency + # or the `paper.id.startswith("10.")` recovery gates below. arXiv/W ids pass + # through unchanged. The OpenAlex work id is retained as openalex_id / in + # paper_identifiers. + paper.id = _norm_cited_id(paper_id.strip()) # DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute # some publishers' abstracts). Supplement from S2, then Crossref (uses the diff --git a/codex/parsing/tex.py b/codex/parsing/tex.py index ed77ad7..afc5b02 100644 --- a/codex/parsing/tex.py +++ b/codex/parsing/tex.py @@ -170,7 +170,9 @@ def _norm_texkey(name: str) -> str: ``.tex`` so ``\\input{sections/intro}`` matches the archive member ``sections/intro.tex``. """ - n = name.strip().lstrip("./") + n = name.strip() + if n.startswith("./"): # a single leading "./" only — don't eat "../" or ".hidden" + n = n[2:] return n[:-4] if n.endswith(".tex") else n @@ -188,7 +190,9 @@ def flatten_inputs(primary: str, files: dict[str, str]) -> str: def _expand(text: str, depth: int) -> str: if depth > 10: - return text + # Cap reached (likely an \input cycle): strip any still-unresolved + # directives rather than leak a literal "\input{…}" into the parsed text. + return _INPUT_RE.sub("", text) def _repl(match: re.Match[str]) -> str: content = norm.get(_norm_texkey(match.group(1))) diff --git a/codex/quality.py b/codex/quality.py index 7dc21f8..2bd23fc 100644 --- a/codex/quality.py +++ b/codex/quality.py @@ -127,14 +127,34 @@ def _clean_title(title: str) -> str: return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t +# A heading collapses to a controlled bucket only when it IS one of these canonical +# section words — NOT when it merely starts with one. Prefix-matching via +# classify_section mislabels real sections ("Abstract Nonsense and Categories" → +# abstract, "References Architecture" → bibliography); an exact map avoids that while +# keeping cross-source consistency for the bare headings. +_SECTION_TITLE_BUCKETS = { + "abstract": "abstract", + "zusammenfassung": "abstract", + "introduction": "intro", + "einleitung": "intro", + "proof": "proof", + "beweis": "proof", + "references": "bibliography", + "bibliography": "bibliography", + "bibliographie": "bibliography", + "literatur": "bibliography", +} + + def section_label(title: str | None, content: str) -> str: """Label a chunk's section, preferring the real ``\\section`` heading (R-F). - For section-aware ``.tex`` ingest *title* is the real heading: if it maps to a - controlled bucket via :func:`classify_section` (intro / theorem / proof / - abstract / bibliography) that bucket is kept (cross-source consistency); - otherwise the cleaned title itself is stored (e.g. ``"preliminaries"``, - ``"rigidity"``) — far more signal than collapsing everything to ``body``. + For section-aware ``.tex`` ingest *title* is the real heading: a bare canonical + heading ("Introduction", "References", "Proof") maps to its controlled bucket + (cross-source consistency); any other heading is stored as its cleaned real + title (e.g. ``"preliminaries"``, ``"introduction to operator algebras"``) — far + more signal than collapsing to ``body`` AND without the prefix-match mislabels + that ``classify_section`` would produce on descriptive titles. When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass`` paths) this falls back to content-based :func:`classify_section` — unchanged @@ -146,8 +166,7 @@ def section_label(title: str | None, content: str) -> str: cleaned = _clean_title(title) if not cleaned: return classify_section(content) - bucket = classify_section(cleaned) - return bucket if bucket != "body" else cleaned + return _SECTION_TITLE_BUCKETS.get(cleaned, cleaned) # --------------------------------------------------------------------------- diff --git a/codex/sources/arxiv.py b/codex/sources/arxiv.py index 80b6932..2452172 100644 --- a/codex/sources/arxiv.py +++ b/codex/sources/arxiv.py @@ -97,6 +97,16 @@ def fetch_metadata(arxiv_id: str) -> Paper | None: ) +def _has_documentclass(content: str) -> bool: + """True if *content* has an UN-commented ``\\documentclass`` line. + + A plain ``"\\documentclass" in content`` substring test also matches a + commented-out ``%\\documentclass`` (common in arXiv preambles); requiring the + command to start its line avoids selecting the wrong primary file. + """ + return any(line.lstrip().startswith("\\documentclass") for line in content.splitlines()) + + def fetch_source(arxiv_id: str) -> str | None: """Download and extract the primary LaTeX source for an arXiv paper. @@ -128,29 +138,38 @@ def fetch_source(arxiv_id: str) -> str | None: from codex.parsing.tex import flatten_inputs raw = response.content + # Image/font/binary members are never \input'd as text; skip them so the source + # map stays text-only (decode-with-replace would otherwise store garbled bytes). + skip_ext = (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".pdf", ".eps", ".ps", ".ttf", ".otf") try: with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf: files: dict[str, str] = {} primary_name: str | None = None for member in tf.getmembers(): - if not member.name.endswith(".tex"): + if not member.isfile() or member.name.lower().endswith(skip_ext): continue f = tf.extractfile(member) if f is None: continue - content = f.read().decode("utf-8", errors="replace") - files[member.name] = content - # Prefer the \documentclass file as the primary document. - if primary_name is None and "\\documentclass" in content: + # Keep ALL text members resolvable so \input of a non-.tex include + # (e.g. a .def or extension-less body file) is not silently dropped (R-C). + files[member.name] = f.read().decode("utf-8", errors="replace") + # Primary doc = a .tex with an UN-commented \documentclass. + if ( + primary_name is None + and member.name.endswith(".tex") + and _has_documentclass(files[member.name]) + ): primary_name = member.name - if not files: + tex_files = {k: v for k, v in files.items() if k.endswith(".tex")} + if not tex_files: logger.debug("No .tex files found in arXiv source for %s", arxiv_id) return None # No \documentclass anywhere → fall back to the largest .tex. if primary_name is None: - primary_name = max(files, key=lambda k: len(files[k])) + primary_name = max(tex_files, key=lambda k: len(tex_files[k])) # Inline \input/\include so multi-file projects yield the full body, # not just the primary file's skeleton of include directives (R-C). diff --git a/scripts/ra_grobid_backfill.py b/scripts/ra_grobid_backfill.py index 798230e..5f8ed04 100644 --- a/scripts/ra_grobid_backfill.py +++ b/scripts/ra_grobid_backfill.py @@ -142,6 +142,10 @@ def main(argv: list[str] | None = None) -> int: pdf_dir = Path(args.pdf_dir) pdfs = {p.stem: p for p in pdf_dir.glob("*.pdf")} + # Read the target list in a short-lived connection, then release it: the GROBID + # extraction below is a slow per-paper network loop and must NOT hold an + # idle-in-transaction connection open across it (a mid-loop SSH-tunnel drop would + # otherwise abort the whole batch — see docs/infra/jetson-ssh-stability.md). with get_conn() as conn: rows = conn.execute( """ @@ -152,46 +156,49 @@ def main(argv: list[str] | None = None) -> int: """ ).fetchall() - targets: list[tuple[str, Path]] = [] - for r in rows: - if args.only and r["id"] != args.only: - continue - if args.zero_edge_only and r["out_edges"] != 0: - continue - stem = Path(r["source_path"]).stem if r["source_path"] else None - pdf = pdfs.get(stem) if stem else None - if pdf is None: - logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem) - continue - targets.append((r["id"], pdf)) - if args.limit is not None: - targets = targets[: args.limit] + targets: list[tuple[str, Path]] = [] + for r in rows: + if args.only and r["id"] != args.only: + continue + if args.zero_edge_only and r["out_edges"] != 0: + continue + stem = Path(r["source_path"]).stem if r["source_path"] else None + pdf = pdfs.get(stem) if stem else None + if pdf is None: + logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem) + continue + targets.append((r["id"], pdf)) + if args.limit is not None: + targets = targets[: args.limit] - logger.info( - "GROBID=%s | papers to process=%d | mode=%s", - grobid_url, - len(targets), - "WRITE" if args.write else "DRY-RUN", - ) + logger.info( + "GROBID=%s | papers to process=%d | mode=%s", + grobid_url, + len(targets), + "WRITE" if args.write else "DRY-RUN", + ) - all_citations: list[Citation] = [] - for paper_id, pdf in targets: - try: - cits = build_grobid_citations( - paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout - ) - except Exception as exc: - logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc) - continue - logger.info("%-38s %3d refs (%s)", paper_id, len(cits), pdf.name) - all_citations.extend(cits) + # GROBID extraction — no DB connection held during this slow network loop. + all_citations: list[Citation] = [] + for paper_id, pdf in targets: + try: + cits = build_grobid_citations( + paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout + ) + except Exception as exc: + logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc) + continue + logger.info("%-38s %3d refs (%s)", paper_id, len(cits), pdf.name) + all_citations.extend(cits) - logger.info("total candidate citations: %d", len(all_citations)) + logger.info("total candidate citations: %d", len(all_citations)) - if not args.write: - logger.info("DRY-RUN — no rows written. Re-run with --write to apply.") - return 0 + if not args.write: + logger.info("DRY-RUN — no rows written. Re-run with --write to apply.") + return 0 + # Fresh connection for the write — the read connection was already released. + with get_conn() as conn: before = _coverage(conn) with conn.cursor() as cur: cur.executemany( @@ -204,15 +211,15 @@ def main(argv: list[str] | None = None) -> int: ) conn.commit() after = _coverage(conn) - logger.info( - "coverage papers=%d citing %d->%d edges %d->%d (+%d)", - after[0], - before[1], - after[1], - before[2], - after[2], - after[2] - before[2], - ) + logger.info( + "coverage papers=%d citing %d->%d edges %d->%d (+%d)", + after[0], + before[1], + after[1], + before[2], + after[2], + after[2] - before[2], + ) return 0 diff --git a/scripts/rc_tex_reingest.py b/scripts/rc_tex_reingest.py index 53839ac..7ccdf1e 100644 --- a/scripts/rc_tex_reingest.py +++ b/scripts/rc_tex_reingest.py @@ -50,7 +50,13 @@ def is_arxiv_id(paper_id: str) -> bool: with ``10.``; OpenAlex W-ids start with ``W`` or an ``https://openalex.org/`` URL. """ p = (paper_id or "").strip() - return bool(p) and not p.startswith("10.") and not p.startswith(("W", "https://")) + # Exclude DOIs (incl. the arXiv DOI 10.48550/arXiv.*), OpenAlex W-ids, and any + # http(s) URL form; everything else is treated as a bare arXiv id. + return ( + bool(p) + and not p.startswith(("10.", "W")) + and not p.lower().startswith(("http://", "https://")) + ) def _section_dist(conn: psycopg.Connection[psycopg.rows.DictRow], paper_id: str) -> dict[str, int]: diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 85b54de..06e4b36 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -238,11 +238,15 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: chunk_sql: str = chunk_insert_call[0][0] assert "INSERT INTO chunks" in chunk_sql - # R-F: heading → controlled bucket where it maps ("Introduction"→intro, - # "Proof of…"→proof), else the cleaned real title ("discrete conformal maps") - # instead of collapsing to "body". + # R-F: a bare canonical heading → bucket ("Introduction"→intro); any other + # heading is stored as its cleaned real title ("proof of the main theorem", + # "discrete conformal maps") instead of being collapsed/mislabelled. chunk_rows = chunk_insert_call[0][1] - assert [row[4] for row in chunk_rows] == ["intro", "proof", "discrete conformal maps"] + assert [row[4] for row in chunk_rows] == [ + "intro", + "proof of the main theorem", + "discrete conformal maps", + ] assert result.chunks_upserted == len(section_pairs) diff --git a/tests/parsing/test_tex.py b/tests/parsing/test_tex.py index 13a8044..7e97a5f 100644 --- a/tests/parsing/test_tex.py +++ b/tests/parsing/test_tex.py @@ -146,6 +146,18 @@ class TestFlattenInputs: assert r"\input" not in out assert "A" in out and "B" in out + def test_input_cycle_does_not_leak_directive(self) -> None: + # a -> b -> a: the depth cap must strip the unresolved directive, not leak it. + files = {"a.tex": r"AA \input{b}", "b.tex": r"BB \input{a}"} + out = flatten_inputs(files["a.tex"], files) + assert r"\input" not in out + assert "AA" in out and "BB" in out + + def test_dot_slash_prefix_resolves_without_overstrip(self) -> None: + # "./intro" resolves intro.tex; lstrip must not eat a real leading dot/run. + files = {"main.tex": r"\input{./intro}", "intro.tex": "BODY"} + assert "BODY" in flatten_inputs(files["main.tex"], files) + class TestChunkSections: def test_pairs_carry_section_titles(self) -> None: diff --git a/tests/quality/test_quality.py b/tests/quality/test_quality.py index 4895c30..240d802 100644 --- a/tests/quality/test_quality.py +++ b/tests/quality/test_quality.py @@ -272,16 +272,25 @@ class TestCleanTitle: class TestSectionLabel: - def test_title_maps_to_controlled_bucket(self): - # Headings matching the controlled vocab keep the canonical bucket. + def test_bare_canonical_heading_maps_to_bucket(self): + # Only an EXACT canonical heading collapses to its controlled bucket. assert section_label("Introduction", "body text") == "intro" - assert section_label("Proof of Theorem 3", "body text") == "proof" + assert section_label("Proof", "body text") == "proof" assert section_label("References", "body text") == "bibliography" + assert section_label("Abstract", "body text") == "abstract" def test_descriptive_title_stored_verbatim(self): - # R-F win: a non-vocab heading is stored as its cleaned title, not "body". + # R-F: a heading that merely STARTS with a keyword keeps its real title, + # not the bucket — fixes classify_section prefix-match over-bucketing. assert section_label("Preliminaries", "body text") == "preliminaries" assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps" + assert section_label("Proof of Theorem 3", "x") == "proof of theorem 3" + assert section_label("Abstract Nonsense and Categories", "x") == ( + "abstract nonsense and categories" + ) + assert section_label("Introduction to Operator Algebras", "x") == ( + "introduction to operator algebras" + ) def test_none_or_blank_title_falls_back_to_content(self): # .pdf/.txt path: unchanged content-based classification. diff --git a/tests/scripts/test_rc_tex_reingest.py b/tests/scripts/test_rc_tex_reingest.py index 17a8770..3f7239f 100644 --- a/tests/scripts/test_rc_tex_reingest.py +++ b/tests/scripts/test_rc_tex_reingest.py @@ -16,4 +16,6 @@ def test_is_arxiv_id_rejects_dois_and_wids() -> None: assert not is_arxiv_id("10.14279/depositonce-20357") assert not is_arxiv_id("W2971636899") assert not is_arxiv_id("https://openalex.org/W2971636899") + assert not is_arxiv_id("http://openalex.org/W2971636899") + assert not is_arxiv_id("10.48550/arXiv.2305.10988") # arXiv DOI form, not bare id assert not is_arxiv_id("") diff --git a/tests/sources/test_arxiv.py b/tests/sources/test_arxiv.py index db376c1..25e701b 100644 --- a/tests/sources/test_arxiv.py +++ b/tests/sources/test_arxiv.py @@ -110,6 +110,37 @@ def test_fetch_source_bare_gzip_single_file(monkeypatch: pytest.MonkeyPatch) -> assert "Old single-file paper body." in result +def test_fetch_source_resolves_non_tex_include(monkeypatch: pytest.MonkeyPatch) -> None: + """\\input of a non-.tex body member (e.g. a .def file) is inlined, not dropped (R-C).""" + main = b"\\documentclass{article}\\begin{document}\\input{body.def}\\end{document}" + tar_bytes = _make_tar_gz({"main.tex": main, "body.def": b"NON TEX BODY TEXT"}) + + def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response: + return httpx.Response(200, content=tar_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + result = arxiv.fetch_source("2301.07041") + assert result is not None + assert "NON TEX BODY TEXT" in result + + +def test_fetch_source_ignores_commented_documentclass(monkeypatch: pytest.MonkeyPatch) -> None: + """A commented %\\documentclass must not select a file as the primary document.""" + decoy = b"% \\documentclass{article}\n\\section{Stub}\nDECOY ONLY" + real = b"\\documentclass{book}\\begin{document}REAL BODY HERE\\end{document}" + # decoy sorts/inserts first; without the fix it would win on the substring match. + tar_bytes = _make_tar_gz({"aaa_decoy.tex": decoy, "real.tex": real}) + + def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response: + return httpx.Response(200, content=tar_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + result = arxiv.fetch_source("2301.07041") + assert result is not None + assert "REAL BODY HERE" in result + assert "DECOY ONLY" not in result + + # --------------------------------------------------------------------------- # fetch_source — 404 → None # --------------------------------------------------------------------------- From ab621fc5f566b27f995067d82dfc7949bb3a0d80 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Sat, 27 Jun 2026 09:09:39 +0200 Subject: [PATCH 26/26] test(review): cover findings #1 (non-bare caller id) and #4 (backfill connection scope) #1: assert a URL-form / mixed-case caller id is normalized to the bare canonical form in the papers upsert (the fix's whole point; existing DQ-5 test only used a bare caller). #4: assert ra_grobid_backfill.main() opens two separate connections (read released before the GROBID loop, fresh one for the write) rather than holding one across the loop. Both pass; ruff clean. Co-Authored-By: Claude Opus 4.8 --- tests/ingest/test_ingest.py | 30 ++++++++++++++++++ tests/scripts/test_ra_grobid_backfill.py | 40 ++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 06e4b36..4779f7e 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -829,6 +829,36 @@ def test_ingest_doi_paper_upserts_bare_canonical_id() -> None: ) +def test_ingest_normalizes_non_bare_caller_id() -> None: + """Review #1: a non-bare CALLER id (URL-form / mixed-case DOI) is normalized to the + bare canonical form before being pinned to papers.id (the C-7 pin runs it through + _norm_cited_id), so a sloppy caller cannot store a URL-form papers.id that would + defeat ON CONFLICT (id) idempotency or the ``paper.id.startswith('10.')`` gates.""" + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W1") + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.crossref.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + # Caller passes the URL form with an upper-case suffix char (worst case). + result = ingest_paper("https://doi.org/10.1007/S00454-019-00132-8") + + bare = "10.1007/s00454-019-00132-8" + assert result.paper_id == bare + upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert upsert_params["id"] == bare, ( + f"non-bare caller must be normalized to '{bare}', got '{upsert_params['id']}'" + ) + + def test_ingest_disambiguates_bibkey_on_unique_violation() -> None: """A papers.bibkey UNIQUE collision is retried with an a/b/c suffix (audit C-15).""" paper = _make_paper() # bibkey auto-generates to "AliceBob2023" diff --git a/tests/scripts/test_ra_grobid_backfill.py b/tests/scripts/test_ra_grobid_backfill.py index 8e97b2f..2ef8a6a 100644 --- a/tests/scripts/test_ra_grobid_backfill.py +++ b/tests/scripts/test_ra_grobid_backfill.py @@ -6,10 +6,12 @@ Only the pure, offline logic is covered (id normalization + Citation assembly); from __future__ import annotations -from unittest.mock import patch +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch from codex.models import Citation -from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations +from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations, main # --------------------------------------------------------------------------- # _normalize_cited_id @@ -74,3 +76,37 @@ def test_build_citations_normalizes_dedups_and_drops_self() -> None: def test_build_citations_empty_when_no_refs() -> None: with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]): assert build_grobid_citations("10.1/x", "x.pdf") == [] + + +# --------------------------------------------------------------------------- +# main — connection scope (review finding #4) +# --------------------------------------------------------------------------- + + +def test_main_uses_separate_connections_for_read_and_write(tmp_path: Any) -> None: + """Review #4: the read connection is released BEFORE the slow GROBID loop, and a + fresh connection is opened for the write — so no idle-in-transaction connection + spans the per-paper network loop (which would abort on an SSH-tunnel drop).""" + (tmp_path / "paper.pdf").write_bytes(b"%PDF-1.4") + conns: list[MagicMock] = [] + + @contextmanager + def fake_get_conn() -> Any: + c = MagicMock() + c.execute.return_value.fetchall.return_value = [ + {"id": "10.1/x", "source_path": "/d/paper.txt", "out_edges": 0} + ] + c.execute.return_value.fetchone.return_value = {"papers": 1, "citing": 1, "edges": 1} + conns.append(c) + yield c + + with ( + patch("scripts.ra_grobid_backfill.get_conn", side_effect=fake_get_conn), + patch("scripts.ra_grobid_backfill.build_grobid_citations", return_value=[]), + ): + rc = main(["--pdf-dir", str(tmp_path), "--write"]) + + assert rc == 0 + # Two DISTINCT get_conn() context managers (read, then write) — not one held + # open across the GROBID loop. + assert len(conns) == 2