diff --git a/.gitignore b/.gitignore index 278f7af..01eb5e5 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ htmlcov/ # Claude Code — settings.json IS committed (team-wide); only personal overrides ignored .claude/settings.local.json + +# Generated wiki index (regenerated by `codex wiki compile`; concepts.yaml is the source) +wiki/index.md diff --git a/codex/cli.py b/codex/cli.py index 4a186e0..44cff43 100644 --- a/codex/cli.py +++ b/codex/cli.py @@ -116,10 +116,10 @@ def search_paper( rows = conn.execute( """ SELECT id, title, year, - abstract_emb <-> %(emb)s AS distance + abstract_emb <-> %(emb)s::vector AS distance FROM papers WHERE abstract_emb IS NOT NULL - ORDER BY abstract_emb <-> %(emb)s + ORDER BY abstract_emb <-> %(emb)s::vector LIMIT %(limit)s """, {"emb": emb, "limit": limit}, @@ -596,7 +596,12 @@ def graph_report( """Show citation graph report: top hub papers, dangling citations, cluster summary.""" from codex.config import get_settings from codex.db import get_conn - from codex.graph import build_citation_graph, citation_pagerank, dangling_citations + from codex.graph import ( + build_citation_graph, + citation_pagerank, + citing_coverage, + dangling_citations, + ) settings = get_settings() with get_conn() as conn: @@ -613,6 +618,8 @@ def graph_report( pr = citation_pagerank(graph) hubs = sorted(pr.items(), key=lambda x: -x[1])[:top_n] dangling = sorted(dangling_citations(graph, known_ids)) + n_citing, _ = citing_coverage(graph, known_ids) + coverage = n_citing / n_papers if n_papers else 0.0 small_corpus = n_papers < settings.graph_min_corpus_size warning = ( f"only {n_papers} ingested papers; recommended >= {settings.graph_min_corpus_size} " @@ -627,6 +634,11 @@ def graph_report( { "nodes": graph.number_of_nodes(), "edges": graph.number_of_edges(), + "citing_coverage": { + "citing": n_citing, + "papers": n_papers, + "fraction": round(coverage, 4), + }, "small_corpus_warning": warning, # null unless below threshold (audit U-3) "hubs": [ {"paper_id": pid, "title": titles.get(pid), "pagerank": score} @@ -642,7 +654,17 @@ def graph_report( if warning: typer.echo(f"Warning: {warning}.", err=True) + # R-D: low citing-paper coverage starves PageRank/coupling even at a healthy + # paper count — warn on the share of papers that actually have out-edges. + if coverage < settings.graph_min_citing_coverage: + typer.echo( + f"Warning: only {n_citing}/{n_papers} papers ({coverage:.0%}) have out-edges " + f"(recommended ≥ {settings.graph_min_citing_coverage:.0%}); " + f"low citing coverage weakens PageRank/coupling.", + err=True, + ) typer.echo(f"Graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges") + typer.echo(f"Citing coverage: {n_citing}/{n_papers} papers with out-edges ({coverage:.0%})") typer.echo("") typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)") typer.echo("-" * 48) diff --git a/codex/config.py b/codex/config.py index 0adac76..fbc6eb6 100644 --- a/codex/config.py +++ b/codex/config.py @@ -322,6 +322,18 @@ class Settings(BaseSettings): ), ) + graph_min_citing_coverage: float = Field( + default=0.8, + ge=0.0, + le=1.0, + description=( + "Minimum share of ingested papers that must have ≥1 out-edge (i.e. " + "cite something) before `codex graph report` warns. Low citing " + "coverage starves PageRank/coupling even when the paper count looks " + "healthy — the share of *citing* papers is what matters (R-D)." + ), + ) + @lru_cache(maxsize=1) def get_settings() -> Settings: diff --git a/codex/graph.py b/codex/graph.py index 540139f..9b83b75 100644 --- a/codex/graph.py +++ b/codex/graph.py @@ -109,6 +109,20 @@ def citation_pagerank(graph: nx.DiGraph, *, damping: float = 0.85) -> dict[str, return cast(dict[str, float], nx.pagerank(graph, alpha=damping)) +def citing_coverage(graph: nx.DiGraph, known_ids: set[str]) -> tuple[int, int]: + """Return ``(papers_with_out_edges, total_papers)`` — the citing-paper coverage. + + A paper "covers" the citation graph when it has ≥1 out-edge (it cites at least + one work). Low citing coverage starves the F-15 layer (PageRank / coupling / + co-citation) even when the total paper count looks healthy, because those + metrics are driven by out-edges, not by node count. Surfaced by + ``codex graph report`` (R-D); the warning threshold is + :attr:`codex.config.Settings.graph_min_citing_coverage`. + """ + n_citing = sum(1 for pid in known_ids if pid in graph and graph.out_degree(pid) > 0) + return n_citing, len(known_ids) + + def find_related(paper_id: str, graph: nx.DiGraph, *, min_shared: int = 2) -> list[str]: """Bibliographic coupling: papers sharing ≥ ``min_shared`` references with ``paper_id``. diff --git a/codex/ingest.py b/codex/ingest.py index a6d1663..9381c69 100644 --- a/codex/ingest.py +++ b/codex/ingest.py @@ -2,7 +2,6 @@ from __future__ import annotations -import contextlib import logging from dataclasses import dataclass from pathlib import Path @@ -14,8 +13,8 @@ from codex.config import get_settings from codex.db import get_conn from codex.embed import get_embedder from codex.models import Citation, Paper -from codex.quality import classify_section, filter_chunks -from codex.sources import openalex, semanticscholar +from codex.quality import is_quality_chunk, section_label +from codex.sources import arxiv, crossref, openalex, semanticscholar logger = logging.getLogger(__name__) @@ -48,6 +47,113 @@ def _make_bibkey(paper: Paper) -> str | None: return surnames[0] + "EtAl" + year +def _s2_id_for(pid: str) -> str | None: + """Format a canonical paper id for the Semantic Scholar paper endpoint. + + S2 needs a namespaced id (``arXiv:``, ``DOI:``) or a bare 40-hex + S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by + the ``10.`` prefix; everything else is assumed to be an arXiv id (modern + ``2305.10988`` or legacy ``math/0603097``). + """ + p = (pid or "").strip() + if not p: + return None + if p.lower().startswith(("arxiv:", "doi:")): + return p + if p.startswith("10."): + return f"DOI:{p}" + return f"arXiv:{p}" + + +def _norm_cited_id(cited_id: str) -> str: + """Normalize a DOI cited-id to its canonical bare, lower-cased form. + + DOIs are case-insensitive and may arrive bare (``10.…``), as a + ``https://doi.org/…`` URL, or ``doi:…``-prefixed: GROBID emits whichever + form the PDF reference text used, while S2/OpenAlex hand back bare DOIs. + Stripping the prefix and lower-casing keeps the citation-graph join from + fragmenting the same reference into distinct case-/URL-variant nodes. arXiv + ids and S2 paperIds are left untouched. Mirrors ``openalex._normalize_doi``.""" + s = cited_id.strip() + lowered = s.lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if lowered.startswith(prefix): + return lowered[len(prefix) :] + return lowered if lowered.startswith("10.") else s + + +def _recover_abstract(paper: Paper) -> str | None: + """Recover a missing abstract from S2, then Crossref (DQ-2 supplement). + + Tried in order of coverage for this corpus: Semantic Scholar (indexes most + arXiv + journal abstracts), then Crossref (publisher-deposited, DOI only). + Network/parse failures degrade to None rather than aborting the ingest. + """ + s2_id = _s2_id_for(paper.id) + if s2_id is not None: + try: + abstract = semanticscholar.fetch_abstract(s2_id) + if abstract and abstract.strip(): + return abstract.strip() + except Exception: + logger.warning("S2 abstract recovery failed for %s", paper.id, exc_info=True) + if paper.id.startswith("10."): + try: + abstract = crossref.fetch_abstract(paper.id) + if abstract and abstract.strip(): + return abstract.strip() + except Exception: + logger.warning("Crossref abstract recovery failed for %s", paper.id, exc_info=True) + return None + + +def _s2_reference_supplement(paper: Paper) -> list[Citation]: + """Fetch a paper's references from Semantic Scholar (DQ-1 supplement). + + Used when OpenAlex returns an empty ``referenced_works`` list — common for + arXiv preprints and institutional theses that OpenAlex indexes without a + parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id`` + so the FK holds, and cited DOIs are case-normalised. Network/parse failures + degrade to an empty list rather than aborting the ingest. + """ + s2_id = _s2_id_for(paper.id) + if s2_id is None: + return [] + try: + raw = semanticscholar.fetch_references(s2_id) + except Exception: + logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True) + return [] + return [ + Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context) + for c in raw + if c.cited_id + ] + + +def _crossref_reference_supplement(paper: Paper) -> list[Citation]: + """Fetch a paper's references from Crossref (roadmap R-B supplement). + + Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref). + Crossref carries publisher-deposited reference lists keyed on the citing + work's DOI, so only DOI papers can be looked up (arXiv-only ids → []). + ``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are + case-normalised. Network/parse failures degrade to [] rather than aborting. + """ + if not paper.id.startswith("10."): + return [] + try: + raw = crossref.fetch_references(paper.id) + except Exception: + logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True) + return [] + return [ + Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context) + for c in raw + if c.cited_id + ] + + def ingest_paper( paper_id: str, source_path: str | None = None, @@ -89,10 +195,11 @@ def ingest_paper( len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit() ) if looks_like_arxiv: - arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}" - with contextlib.suppress(Exception): - semanticscholar.fetch_references(arxiv_key) - paper = Paper(id=paper_id, title="") + # OpenAlex 404 on an arXiv id → recover authoritative metadata from + # the arXiv API (DQ-2); fall back to a minimal stub if arXiv also + # has nothing. Citation recovery from S2 happens in the citation + # block below (DQ-1 supplement). + paper = arxiv.fetch_metadata(paper_id) or Paper(id=paper_id, title="") else: raise ValueError(f"Paper not found: {paper_id}") @@ -100,12 +207,23 @@ def ingest_paper( raise ValueError(f"Paper not found: {paper_id}") # Canonical identity is the caller-supplied id — what the CLI / ingest scripts - # use and what papers.id is contracted to hold (arXiv id or DOI). OpenAlex's - # fetch_paper fills paper.id with the URL-form doi/id (e.g. - # "https://doi.org/10.48550/arxiv.math/0603097"), which broke bare-id lookups - # and cross-citation matching (audit C-7). The OpenAlex work id is retained - # separately as openalex_id and in the paper_identifiers alias table. - paper.id = paper_id.strip() + # use and what papers.id is contracted to hold (arXiv id or DOI). Pin it to the + # caller's id (audit C-7), but normalize it the same way cited-ids are (strip a + # https://doi.org/ or doi: prefix, lower-case DOIs) so a non-bare caller cannot + # store a URL-form papers.id that would defeat DQ-5's ON CONFLICT (id) idempotency + # or the `paper.id.startswith("10.")` recovery gates below. arXiv/W ids pass + # through unchanged. The OpenAlex work id is retained as openalex_id / in + # paper_identifiers. + paper.id = _norm_cited_id(paper_id.strip()) + + # DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute + # some publishers' abstracts). Supplement from S2, then Crossref (uses the + # canonical paper.id set above), so the paper gets a real (non-zero) abstract + # embedding instead of a zero vector. + if not (paper.abstract or "").strip(): + recovered = _recover_abstract(paper) + if recovered: + paper.abstract = recovered # Auto-generate bibkey when source has none (D-04). if paper.bibkey is None: @@ -208,44 +326,63 @@ def ingest_paper( if source_path is not None: from codex.parsing.grobid import extract_references from codex.parsing.nougat import pdf_to_markdown - from codex.parsing.tex import chunk_text, latex_to_text + from codex.parsing.tex import chunk_sections, chunk_text suffix = Path(source_path).suffix.lower() - text = "" + # (section_title | None, chunk_text) pairs. `.tex` is section-aware so + # the stored `section` column reflects the real heading (R-C / R-12); + # `.txt`/`.pdf` carry no section structure, so the title is None. + raw_pairs: list[tuple[str | None, str]] = [] if suffix == ".tex": - text = latex_to_text(Path(source_path).read_text()) + raw_pairs = chunk_sections(Path(source_path).read_text()) elif suffix == ".txt": text = Path(source_path).read_text(encoding="utf-8", errors="replace") + raw_pairs = [(None, c) for c in chunk_text(text)] elif suffix == ".pdf": - text = pdf_to_markdown(source_path) + raw_pairs = [(None, c) for c in chunk_text(pdf_to_markdown(source_path))] # Also extract GROBID refs grobid_refs = extract_references(source_path) for ref in grobid_refs: cited_id = ref.get("doi") or ref.get("arxiv_id") if cited_id: - pdf_citations.append(Citation(citing_id=paper.id, cited_id=cited_id)) + # Normalize the GROBID DOI (bare/URL/``doi:`` → bare lower-case) + # so PDF-derived edges join the same graph nodes as the + # S2/OpenAlex paths; arXiv ids pass through untouched. + pdf_citations.append( + Citation(citing_id=paper.id, cited_id=_norm_cited_id(cited_id)) + ) else: logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path) - raw_chunks = chunk_text(text) if text else [] - chunks_text = filter_chunks(raw_chunks, settings=get_settings()) + # Quality-gate the pairs (keeps each chunk aligned with its section title). + settings = get_settings() + kept_pairs = [ + (title, content) + for title, content in raw_pairs + if is_quality_chunk(content, settings=settings) + ] # Delete existing chunks for this paper conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,)) - if chunks_text: + if kept_pairs: # Embed all chunks in one batch - chunk_embeddings = embedder.encode_dense(chunks_text) + chunk_contents = [content for _, content in kept_pairs] + chunk_embeddings = embedder.encode_dense(chunk_contents) chunk_rows = [ ( paper.id, ord_idx, content, chunk_embeddings[ord_idx].tolist(), - classify_section(content), + # .tex: label from the real \section heading — keeps the + # controlled bucket when it matches, else stores the cleaned + # title (R-F); .txt/.pdf (title=None) classify by content. + # NB: `section` semantics now differ by source (write-only col). + section_label(title, content), ) - for ord_idx, content in enumerate(chunks_text) + for ord_idx, (title, content) in enumerate(kept_pairs) ] with conn.cursor() as cur: cur.executemany( @@ -256,7 +393,7 @@ def ingest_paper( chunks_upserted = len(chunk_rows) # --------------------------------------------------------------- - # 5. Fetch citations (OpenAlex if openalex_id, else S2) + # 5. Fetch citations (fallback chain: OpenAlex → S2 → Crossref) # Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING # --------------------------------------------------------------- api_citations: list[Citation] @@ -267,8 +404,17 @@ def ingest_paper( api_citations = [ Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw ] + # DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a + # parsed reference list (referenced_works == []). Fall back to the + # Semantic Scholar references, which often has them. + if not api_citations: + api_citations = _s2_reference_supplement(paper) else: - api_citations = semanticscholar.fetch_references(paper_id) + api_citations = _s2_reference_supplement(paper) + # R-B: if OpenAlex and S2 both came back empty, try Crossref's + # publisher-deposited references (DOI papers only) as a third leg. + if not api_citations: + api_citations = _crossref_reference_supplement(paper) # Merge API citations and GROBID PDF citations; dedup via set all_citations_set: set[tuple[str, str]] = set() diff --git a/codex/parsing/grobid.py b/codex/parsing/grobid.py index b2b5443..7644803 100644 --- a/codex/parsing/grobid.py +++ b/codex/parsing/grobid.py @@ -26,6 +26,7 @@ def _text(element: ET.Element | None) -> str: def extract_references( pdf_path: str, grobid_url: str | None = None, + timeout: float = 60.0, ) -> list[dict[str, str]]: """Extract a structured reference list from a PDF via GROBID. @@ -35,6 +36,9 @@ def extract_references( Path to the PDF file on disk. grobid_url: Base URL of the GROBID server. Defaults to ``Settings().grobid_url``. + timeout: + HTTP client timeout in seconds. Large PDFs (e.g. theses) can exceed the + 60s default, especially against a slower/emulated GROBID; raise it then. Returns ------- @@ -46,7 +50,7 @@ def extract_references( if grobid_url is None: grobid_url = Settings().grobid_url - with open(pdf_path, "rb") as fh, httpx.Client(timeout=60.0) as client: + with open(pdf_path, "rb") as fh, httpx.Client(timeout=timeout) as client: response = client.post( f"{grobid_url}/api/processReferences", files={"input": (pdf_path, fh, "application/pdf")}, diff --git a/codex/parsing/tex.py b/codex/parsing/tex.py index 43514d6..afc5b02 100644 --- a/codex/parsing/tex.py +++ b/codex/parsing/tex.py @@ -17,6 +17,10 @@ _SECTION_RE = re.compile( re.DOTALL, ) +# \input{file} / \include{file} — arXiv papers routinely split the body across +# files, so these must be inlined before parsing (R-C multi-file flattening). +_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}") + # Patterns for LaTeX noise removal (applied in order). _COMMENT_RE = re.compile(r"%[^\n]*") _CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}") @@ -157,3 +161,63 @@ def latex_to_text(latex: str) -> str: return "\n\n".join(body for _, body in sections) # Fallback: no section structure — clean the whole string. return _clean_latex(latex) + + +def _norm_texkey(name: str) -> str: + """Normalize a tex file name / ``\\input`` argument to a comparable key. + + Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing + ``.tex`` so ``\\input{sections/intro}`` matches the archive member + ``sections/intro.tex``. + """ + n = name.strip() + if n.startswith("./"): # a single leading "./" only — don't eat "../" or ".hidden" + n = n[2:] + return n[:-4] if n.endswith(".tex") else n + + +def flatten_inputs(primary: str, files: dict[str, str]) -> str: + """Recursively inline ``\\input``/``\\include`` directives (R-C). + + *files* maps each source file's archive name to its contents (keys may carry + a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv + multi-file projects leave the ``\\documentclass`` file as little more than a + skeleton of ``\\input`` lines, so the body must be assembled before parsing. + Unresolved references are dropped; recursion is capped at depth 10 to guard + against include cycles. + """ + norm = {_norm_texkey(k): v for k, v in files.items()} + + def _expand(text: str, depth: int) -> str: + if depth > 10: + # Cap reached (likely an \input cycle): strip any still-unresolved + # directives rather than leak a literal "\input{…}" into the parsed text. + return _INPUT_RE.sub("", text) + + def _repl(match: re.Match[str]) -> str: + content = norm.get(_norm_texkey(match.group(1))) + return _expand(content, depth + 1) if content is not None else "" + + return _INPUT_RE.sub(_repl, text) + + return _expand(primary, 0) + + +def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]: + """Chunk LaTeX *within* each section, pairing every chunk with its title. + + Unlike :func:`latex_to_text` (which discards titles and flattens the whole + document into one block before word-window chunking), this keeps chunks from + spanning section boundaries and remembers which ``\\section`` each came from. + The caller can then label the stored ``section`` column from the real heading + instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less + chunks of the whole cleaned document when there are no section commands. + """ + sections = extract_sections(latex) + if not sections: + return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)] + pairs: list[tuple[str | None, str]] = [] + for title, body in sections: + for chunk in chunk_text(body, size, overlap): + pairs.append((title or None, chunk)) + return pairs diff --git a/codex/quality.py b/codex/quality.py index ef6ee72..2bd23fc 100644 --- a/codex/quality.py +++ b/codex/quality.py @@ -110,6 +110,65 @@ def classify_section(text: str) -> str: return "body" +def _clean_title(title: str) -> str: + """Normalize a LaTeX ``\\section`` title for use as a section label. + + Strips LaTeX word-commands (``\\emph`` …), accent/escape macros (``\\"o`` → + ``o``, ``\\&``), ``~`` ties and ``{}$``, drops leading section numbering + (``3.2 ``), collapses whitespace, lower-cases, and truncates to 60 chars at a + word boundary so the stored ``section`` value is a clean, comparable string. + """ + t = title.replace("~", " ") # LaTeX non-breaking tie → space + t = re.sub(r"\\[a-zA-Z]+\*?", " ", t) # word-commands: \emph, \mathcal … + t = re.sub(r"\\[^a-zA-Z]", "", t) # accent/escape macros: \"o → o, \', \`, \& + t = re.sub(r"[{}$]", "", t) # braces / math delimiters + t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 " + t = " ".join(t.split()).strip().lower() + return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t + + +# A heading collapses to a controlled bucket only when it IS one of these canonical +# section words — NOT when it merely starts with one. Prefix-matching via +# classify_section mislabels real sections ("Abstract Nonsense and Categories" → +# abstract, "References Architecture" → bibliography); an exact map avoids that while +# keeping cross-source consistency for the bare headings. +_SECTION_TITLE_BUCKETS = { + "abstract": "abstract", + "zusammenfassung": "abstract", + "introduction": "intro", + "einleitung": "intro", + "proof": "proof", + "beweis": "proof", + "references": "bibliography", + "bibliography": "bibliography", + "bibliographie": "bibliography", + "literatur": "bibliography", +} + + +def section_label(title: str | None, content: str) -> str: + """Label a chunk's section, preferring the real ``\\section`` heading (R-F). + + For section-aware ``.tex`` ingest *title* is the real heading: a bare canonical + heading ("Introduction", "References", "Proof") maps to its controlled bucket + (cross-source consistency); any other heading is stored as its cleaned real + title (e.g. ``"preliminaries"``, ``"introduction to operator algebras"``) — far + more signal than collapsing to ``body`` AND without the prefix-match mislabels + that ``classify_section`` would produce on descriptive titles. + + When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass`` + paths) this falls back to content-based :func:`classify_section` — unchanged + behaviour. Safe to store free-text titles because the ``section`` column is + write-only (no consumer filters on the controlled vocabulary). + """ + if not title or not title.strip(): + return classify_section(content) + cleaned = _clean_title(title) + if not cleaned: + return classify_section(content) + return _SECTION_TITLE_BUCKETS.get(cleaned, cleaned) + + # --------------------------------------------------------------------------- # Batch filter # --------------------------------------------------------------------------- diff --git a/codex/sources/arxiv.py b/codex/sources/arxiv.py index 5c81c13..2452172 100644 --- a/codex/sources/arxiv.py +++ b/codex/sources/arxiv.py @@ -1,21 +1,110 @@ """arXiv API client. Provides: +- fetch_metadata: resolve an arXiv ID to a Paper via the Atom export API. - fetch_source: download the .tar.gz source of a paper and extract the primary .tex file. - fetch_pdf_url: return the canonical PDF URL for a given arXiv ID. """ from __future__ import annotations +import gzip import io import logging import tarfile +import xml.etree.ElementTree as ET import httpx +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential + +from codex.models import Paper logger = logging.getLogger(__name__) _BASE = "https://arxiv.org" +_EXPORT = "https://export.arxiv.org" +_ATOM = "{http://www.w3.org/2005/Atom}" + + +def _is_retryable(exc: BaseException) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code == 429 or exc.response.status_code >= 500 + return False + + +@retry( + retry=retry_if_exception(_is_retryable), + stop=stop_after_attempt(4), + wait=wait_exponential(min=1, max=20), + before_sleep=lambda rs: logger.warning("arXiv retry %d", rs.attempt_number), +) +def _query(arxiv_id: str) -> httpx.Response: + response = httpx.get( + f"{_EXPORT}/api/query", + params={"id_list": arxiv_id}, + timeout=30, + follow_redirects=True, + ) + response.raise_for_status() + return response + + +def fetch_metadata(arxiv_id: str) -> Paper | None: + """Resolve an arXiv ID to a Paper via the Atom export API. + + Authoritative metadata source for arXiv preprints — used as an ingest + fallback when OpenAlex 404s on an arXiv id (DQ-2). The arXiv id (bare, + e.g. ``"1911.00966"`` or legacy ``"math/0603097"``) becomes ``Paper.id``; + ``openalex_id`` and ``bibkey`` are left for the caller to populate. + + Returns + ------- + Paper | None + Populated Paper (title, authors, year, abstract), or None if arXiv + has no entry for the id. + """ + bare = arxiv_id[len("arxiv:") :] if arxiv_id.lower().startswith("arxiv:") else arxiv_id + try: + response = _query(bare) + except httpx.HTTPError: + logger.warning("arXiv metadata fetch failed for %s", bare, exc_info=True) + return None + try: + root = ET.fromstring(response.text) + except ET.ParseError: + return None + entry = root.find(f"{_ATOM}entry") + if entry is None: + return None + # An id-not-found query still returns a feed but with no . + title = (entry.findtext(f"{_ATOM}title") or "").strip() + if not title: + return None + summary = (entry.findtext(f"{_ATOM}summary") or "").strip() + published = entry.findtext(f"{_ATOM}published") or "" + year = int(published[:4]) if published[:4].isdigit() else None + authors = [ + name.strip() + for a in entry.findall(f"{_ATOM}author") + if (name := a.findtext(f"{_ATOM}name")) and name.strip() + ] + return Paper( + id=bare, + title=" ".join(title.split()), + authors=authors, + year=year, + abstract=" ".join(summary.split()) or None, + ) + + +def _has_documentclass(content: str) -> bool: + """True if *content* has an UN-commented ``\\documentclass`` line. + + A plain ``"\\documentclass" in content`` substring test also matches a + commented-out ``%\\documentclass`` (common in arXiv preambles); requiring the + command to start its line avoids selecting the wrong primary file. + """ + return any(line.lstrip().startswith("\\documentclass") for line in content.splitlines()) def fetch_source(arxiv_id: str) -> str | None: @@ -23,8 +112,9 @@ def fetch_source(arxiv_id: str) -> str | None: Downloads the .tar.gz source bundle from ``https://arxiv.org/src/{arxiv_id}``, locates the primary .tex file (preferring any file containing ``\\documentclass``, - falling back to the largest .tex by size), and returns its contents as a UTF-8 - string. + falling back to the largest .tex by size), and inlines its ``\\input``/ + ``\\include`` directives from the other archive members so multi-file projects + return the full document body, not just the primary file's include skeleton. Parameters ---------- @@ -45,39 +135,58 @@ def fetch_source(arxiv_id: str) -> str | None: if response.status_code != 200: response.raise_for_status() + from codex.parsing.tex import flatten_inputs + raw = response.content + # Image/font/binary members are never \input'd as text; skip them so the source + # map stays text-only (decode-with-replace would otherwise store garbled bytes). + skip_ext = (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".pdf", ".eps", ".ps", ".ttf", ".otf") try: with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf: - tex_members = [m for m in tf.getmembers() if m.name.endswith(".tex")] - if not tex_members: - logger.debug("No .tex files found in arXiv source for %s", arxiv_id) - return None - - # Prefer the file containing \documentclass (primary document) - primary: tarfile.TarInfo | None = None - for member in tex_members: + files: dict[str, str] = {} + primary_name: str | None = None + for member in tf.getmembers(): + if not member.isfile() or member.name.lower().endswith(skip_ext): + continue f = tf.extractfile(member) if f is None: continue - content_bytes = f.read() - if b"\\documentclass" in content_bytes: - primary = member - # Decode and return immediately — first match wins - return content_bytes.decode("utf-8", errors="replace") + # Keep ALL text members resolvable so \input of a non-.tex include + # (e.g. a .def or extension-less body file) is not silently dropped (R-C). + files[member.name] = f.read().decode("utf-8", errors="replace") + # Primary doc = a .tex with an UN-commented \documentclass. + if ( + primary_name is None + and member.name.endswith(".tex") + and _has_documentclass(files[member.name]) + ): + primary_name = member.name - if primary is None: - # Fallback: largest .tex by size - largest = max(tex_members, key=lambda m: m.size) - f = tf.extractfile(largest) - if f is None: - return None - return f.read().decode("utf-8", errors="replace") - except tarfile.TarError as exc: - logger.warning("Failed to open tar archive for %s: %s", arxiv_id, exc) + tex_files = {k: v for k, v in files.items() if k.endswith(".tex")} + if not tex_files: + logger.debug("No .tex files found in arXiv source for %s", arxiv_id) + return None + + # No \documentclass anywhere → fall back to the largest .tex. + if primary_name is None: + primary_name = max(tex_files, key=lambda k: len(tex_files[k])) + + # Inline \input/\include so multi-file projects yield the full body, + # not just the primary file's skeleton of include directives (R-C). + return flatten_inputs(files[primary_name], files) + except tarfile.TarError: + # Old arXiv submissions are served as a single bare gzipped .tex with no + # tar wrapper (e.g. legacy math/* papers) — gunzip the body directly. + try: + single = gzip.decompress(raw).decode("utf-8", errors="replace") + except (OSError, EOFError) as exc: + logger.warning("Failed to read arXiv source for %s: %s", arxiv_id, exc) + return None + if "\\" in single[:5000]: # looks like LaTeX (has backslash commands) + return single + logger.debug("arXiv source for %s is not LaTeX", arxiv_id) return None - return None # unreachable but satisfies type checker - def fetch_pdf_url(arxiv_id: str) -> str: """Return the canonical PDF URL for an arXiv paper. diff --git a/codex/sources/crossref.py b/codex/sources/crossref.py new file mode 100644 index 0000000..f263872 --- /dev/null +++ b/codex/sources/crossref.py @@ -0,0 +1,126 @@ +"""Crossref API client. + +Provides: +- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI. +- fetch_references: retrieve a work's reference list (DOI edges) by DOI. + +Crossref is a *third* metadata source after OpenAlex and Semantic Scholar +(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto +query parameter) and are retried on 429/5xx with exponential back-off via tenacity. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +import httpx +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential + +from codex.config import get_settings +from codex.models import Citation + +logger = logging.getLogger(__name__) + +_BASE = "https://api.crossref.org" +_TAG_RE = re.compile(r"<[^>]+>") + + +def _is_retryable(exc: BaseException) -> bool: + if isinstance(exc, httpx.HTTPStatusError): + return exc.response.status_code == 429 or exc.response.status_code >= 500 + return False + + +def _polite_params() -> dict[str, str]: + # Reuse the OpenAlex mailto for Crossref's Polite Pool (same address). + mailto = get_settings().openalex_mailto + return {"mailto": mailto} if mailto else {} + + +@retry( + retry=retry_if_exception(_is_retryable), + stop=stop_after_attempt(5), + wait=wait_exponential(min=1, max=30), + before_sleep=lambda rs: logger.warning( + "Crossref retry %d after %s", + rs.attempt_number, + rs.outcome.exception(), # type: ignore[union-attr] + ), +) +def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response: + merged: dict[str, str] = {**_polite_params(), **(params or {})} + response = httpx.get(url, params=merged, timeout=30, follow_redirects=True) + response.raise_for_status() + return response + + +def _strip_jats(abstract: str | None) -> str | None: + """Strip JATS/XML tags from a Crossref abstract and collapse whitespace.""" + if not abstract: + return None + text = _TAG_RE.sub("", abstract) + text = " ".join(text.split()).strip() + return text or None + + +def fetch_abstract(doi: str) -> str | None: + """Fetch a work's abstract from Crossref by DOI. + + Crossref stores abstracts as JATS-tagged markup (``…``); + tags are stripped before returning. Many works (notably book chapters) + have no abstract deposited — returns None in that case and on 404. + + Parameters + ---------- + doi: + A bare DOI (``"10.1007/s00454-019-00132-8"``). + + Returns + ------- + str | None + The plain-text abstract, or None when absent. + """ + url = f"{_BASE}/works/{doi}" + try: + response = _get(url) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None + raise + message: dict[str, Any] = response.json().get("message", {}) + return _strip_jats(message.get("abstract")) + + +def fetch_references(doi: str) -> list[Citation]: + """Fetch a work's reference list from Crossref by DOI (roadmap R-B). + + Crossref carries publisher-deposited reference lists in ``message.reference``. + Each entry that cites a DOI-registered work exposes a bare ``DOI`` field; + only those become citation-graph edges (book / older references carry just + ``unstructured`` text with no resolvable id and are skipped). ``citing_id`` + is the queried *doi* — the caller rewrites it to the canonical ``papers.id`` + and normalises the cited DOIs (mirrors the Semantic Scholar reference path). + + Parameters + ---------- + doi: + A bare DOI (``"10.1007/s00454-019-00132-8"``). + + Returns + ------- + list[Citation] + One Citation per reference that carries a DOI. Empty on 404 or when no + references are deposited for the work. + """ + url = f"{_BASE}/works/{doi}" + try: + response = _get(url) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return [] + raise + message: dict[str, Any] = response.json().get("message", {}) + references: list[dict[str, Any]] = message.get("reference") or [] + return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")] diff --git a/codex/sources/openalex.py b/codex/sources/openalex.py index 80b5307..2ad1073 100644 --- a/codex/sources/openalex.py +++ b/codex/sources/openalex.py @@ -85,13 +85,55 @@ def _reconstruct_abstract(inverted_index: dict[str, list[int]] | None) -> str | return " ".join(w for _, w in word_positions) +def _normalize_doi(doi: str) -> str: + """Normalize an OpenAlex DOI to the canonical bare, lower-cased form. + + OpenAlex returns the ``doi`` field as a full URL + (``https://doi.org/10.1007/s00454-019-00132-8``), but the canonical + ``papers.id`` produced by the M-1 migration is the BARE, lower-cased DOI + (``10.1007/s00454-019-00132-8``). Stripping the ``https://doi.org/`` / + ``doi:`` prefix and lower-casing (DOIs are case-insensitive by spec) makes a + re-ingested DOI paper match its stored row, so the ingest + ``INSERT … ON CONFLICT (id)`` updates in place instead of attempting a fresh + INSERT that trips the separate ``papers_openalex_id_key`` unique constraint + (DQ-5). Mirrors the cited-id convention in ``ingest._norm_cited_id``. + """ + s = doi.strip().lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if s.startswith(prefix): + return s[len(prefix) :] + return s + + +# arXiv works are registered under a DOI of the form 10.48550/arXiv.. +_ARXIV_DOI_PREFIX = "10.48550/arxiv." + + +def _canonical_id(doi: str) -> str: + """Return the canonical ``papers.id`` for an OpenAlex ``doi`` value. + + Builds on :func:`_normalize_doi` (bare, lower-cased DOI), then strips the + arXiv DOI prefix ``10.48550/arxiv.`` so an arXiv work's id is the BARE arXiv + id (``2305.10988`` / ``math/0603097``) — the form the corpus is keyed on — + not the arXiv DOI. Without this, re-ingesting an arXiv paper by its bare id + misses ``INSERT … ON CONFLICT (id)`` and trips ``papers_openalex_id_key``. + This is the arXiv half of DQ-5 (the DOI half was fixed there; this was + explicitly deferred and surfaced again driving the R-C .tex re-ingest). + """ + normalized = _normalize_doi(doi) + if normalized.startswith(_ARXIV_DOI_PREFIX): + return normalized[len(_ARXIV_DOI_PREFIX) :] + return normalized + + def _map_paper(data: dict[str, Any]) -> Paper: authors: list[str] = [ a.get("author", {}).get("display_name") or "" for a in data.get("authorships", []) ] abstract = _reconstruct_abstract(data.get("abstract_inverted_index")) + doi = data.get("doi") return Paper( - id=data.get("doi") or data.get("id") or "", + id=_canonical_id(doi) if doi else (data.get("id") or ""), title=data.get("title") or "", openalex_id=data.get("id"), authors=authors, diff --git a/codex/sources/semanticscholar.py b/codex/sources/semanticscholar.py index 5c48c9a..591ca73 100644 --- a/codex/sources/semanticscholar.py +++ b/codex/sources/semanticscholar.py @@ -90,7 +90,11 @@ def fetch_references(paper_id: str) -> list[Citation]: raise data = response.json() - raw_refs: list[dict[str, Any]] = data.get("data", []) + # S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no + # parsed reference list (e.g. some theses). ``.get("data", [])`` would yield + # the explicit ``None`` (the default only applies when the key is absent), + # so iterate over a guaranteed list instead. + raw_refs: list[dict[str, Any]] = data.get("data") or [] citations: list[Citation] = [] for entry in raw_refs: cited_paper: dict[str, Any] = entry.get("citedPaper", {}) @@ -106,6 +110,24 @@ def fetch_references(paper_id: str) -> list[Citation]: return citations +def fetch_abstract(paper_id: str) -> str | None: + """Fetch just the abstract for a paper from Semantic Scholar. + + Used as an ingest abstract supplement (DQ-2) when OpenAlex has the paper + but no abstract. ``paper_id`` should be namespaced (``arXiv:…`` / ``DOI:…``). + Returns None on 404 or when S2 has no abstract. + """ + url = f"{_BASE_GRAPH}/paper/{paper_id}" + try: + response = _get(url, params={"fields": "abstract"}) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None + raise + abstract = response.json().get("abstract") + return abstract or None + + def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]: """Fetch recommended paper IDs from Semantic Scholar. diff --git a/docs/SESSION-2026-06-17-open-items.md b/docs/SESSION-2026-06-17-open-items.md new file mode 100644 index 0000000..1d0a8d7 --- /dev/null +++ b/docs/SESSION-2026-06-17-open-items.md @@ -0,0 +1,59 @@ +# Open items — session 2026-06-17 (handoff notice) + +Everything still open after this session. Done items are in +`docs/audit/DATA-QUALITY-2026-06-15.md` (DQ-1..DQ-5 + roadmap R-A,R-B,R-C,R-D,R-F all +DONE; R-E deferred). This file is only the **loose ends**. + +## 1. ✅ RESOLVED — Jetson is online (the tunnel had dropped, not the host) +- What looked like the Jetson going offline was the **SSH tunnel** breaking, not the + host. The Jetson (`192.168.178.103`) is online. Re-open the tunnel with keepalive: + `ssh -o ServerAliveInterval=15 -f -N -L 5433:localhost:5432 alfred@192.168.178.103`. + Better still: run long ops in tmux-on-Jetson (item #4) to avoid the tunnel entirely. + +## 2. ✅ DONE — R-F accent re-clean +- R-F is functionally DONE (section signal 34/329 → **314/329 = 95 %**). The remaining + cosmetic accent/tie artifacts in live `chunks.section` labels were re-cleaned in place + once the tunnel was back: applied `codex.quality._clean_title` to every label + containing `\` or `~` — **8 labels fixed**, no re-embed. Verified **0 artifacts** + remain. R-F fully complete. + +## 3. ✅ READY TO MERGE — all branches integrated + validated +All five feature branches are merged (**zero conflicts**) into **`integration/roadmap`** +(off `audit/loop-1`) and validated green: **391 tests pass, ruff + ruff-format + mypy +clean, tree clean**. This is the single merge candidate for `main`. + +Contents (DQ-1..DQ-5 + R-A..R-F + infra docs): +| Source branch | Contents | +|---|---| +| `audit/loop-1` (base) | DQ-1..DQ-5, **R-A** (GROBID backfill + native arm64 GROBID), this notice | +| `feat/section-labels` (incl. `feat/tex-ingest`) | **R-C** (.tex ingest) + DQ-5 arXiv-id fix + gzip fallback + **R-F** (section labels) | +| `feat/crossref-references` | **R-B** (Crossref refs) + GROBID-on-demand tweak | +| `feat/graph-coverage-warning` | **R-D** (citing-coverage warning) | +| `chore/infra-ssh-stability` | infra TODO (#4) | + +**To land on `main`** (user-triggered): +``` +git checkout main && git merge --no-ff integration/roadmap +``` +…or open a PR `integration/roadmap` → `main`. The individual feature branches are +preserved unchanged for per-feature review if preferred. + +**Cleanup — branches prunable AFTER the merge lands** (not deleted here; destructive): +- Already in `main`: `fix/cli-readme-drift`, `fix/d04-d05-ingest-fixes`. +- Folded into `integration/roadmap`: the 5 feature branches + `audit/loop-1`. +- Pre-session / stale (verify before pruning): `feat/F-15-literature-graph`, + `feat/F-16-chunk-quality`, `fix/audit-wave-1`/`-2`/`-3`, `scratch/spike-embed`. +- Working tree is clean — only gitignored `.venv`/caches/`.env`; no stray tracked files. + +## 4. Infra TODO — tmux-on-Jetson for stable long ops (documented, not implemented) +- `docs/infra/jetson-ssh-stability.md` (branch `chore/infra-ssh-stability`). The + Mac↔Jetson tunnel dropped repeatedly on multi-minute writes all session. Fix: run long + ops **on the Jetson in `tmux`** (DB is localhost there → no tunnel; survives SSH drops); + `autossh` for interactive tunnels. Open. + +## 5. Known limitations (documented, NOT action items) +- `math/0306167` (old AMS-TeX, no `\section{}`) stays `section=body` — inherent. +- The 2 edited-volume books yielded 0 GROBID-parsed DOI/arXiv refs in the R-A sweep — + inherent (per-chapter refs, no clean end-bibliography). +- **DQ-6** (upstream PDF→`.txt` extraction fidelity, done outside codex-py) — flagged + earlier, largely **mooted** by R-A/R-C ingesting from PDF/`.tex` directly. diff --git a/docs/audit/DATA-QUALITY-2026-06-15.md b/docs/audit/DATA-QUALITY-2026-06-15.md index f016a79..e5f3b5c 100644 --- a/docs/audit/DATA-QUALITY-2026-06-15.md +++ b/docs/audit/DATA-QUALITY-2026-06-15.md @@ -1,6 +1,15 @@ # DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session) -**Status:** baseline scan done 2026-06-15; four investigation items open (DQ-1…DQ-4). +**Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement +applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is +strong after fixing a P0 `codex search paper` crash. **DQ-2 RESOLVED 2026-06-16** +(1911.00966 fully recovered from arXiv; s00454 abstract from S2; book chapter is a +documented limit; recovery wired into ingest). **DQ-3 RESOLVED 2026-06-16** +(content fidelity clean — full coverage, only OCR junk dropped). **All four +assessment items (DQ-1..DQ-4) DONE. DQ-5 RESOLVED 2026-06-17** (DOI id normalized +to bare canonical form in `openalex._map_paper`; live re-ingest of +`10.1007/s00454-019-00132-8` confirmed idempotent → **unblocks R-A/R-C**). +**Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section). **Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory of the audit conversation — everything needed to continue is in this file. @@ -50,7 +59,35 @@ incomplete, or unrepresentative data. This document is that second axis — a ## Findings (open investigation items) -### DQ-1 — Citation coverage: 12 / 29 papers (41 %) have ZERO citations · **HIGH** +### DQ-1 — Citation coverage: 12 / 29 papers (41 %) had ZERO citations · **RESOLVED 2026-06-16** + +**Resolution (what was done):** +- **Verified upstream, not a bug.** Queried the OpenAlex API directly for all 11 + zero-citation papers that have an `openalex_id`: every one returns + `referenced_works_count = 0` server-side, W-ids all match what we stored. They + are `type: preprint` (arXiv) / `article` / `dissertation` — works OpenAlex + indexes without a parsed bibliography. Confirmed source-coverage limit. +- **Semantic Scholar supplement applied to the live DB** (idempotent backfill, + `ON CONFLICT (citing_id,cited_id) DO NOTHING`): **+330 citations (590 → 920)**. + Zero-out-edge papers **12 → 2**; in-corpus citing coverage **17/29 → 27/29**; + graph 489→704 nodes, 590→920 edges; discovery-lead targets 472→677. 9 of the + new edges are in-corpus (e.g. `2305.10988 → 10.1007/s00454-019-00132-8`). +- **Wired into ingest** so future re-ingests self-heal: when OpenAlex returns an + empty `referenced_works`, `ingest.py` now falls back to + `_s2_reference_supplement()` (citing_id rewritten to canonical `papers.id`, + DOI cited-ids lowercased). Same helper drove the one-off backfill. +- **Bug fixed (would have crashed the batch):** `semanticscholar.fetch_references` + used `data.get("data", [])`, but S2 returns HTTP 200 `{"data": null}` for a + known paper with no parsed refs (e.g. `depositonce-5415`) → `TypeError`. + Changed to `data.get("data") or []`. Removed a now-redundant discarded S2 probe + in the OpenAlex-404 arXiv branch. New regression test added. +- **Irreducible remainder (documented limit):** the **2 `depositonce` theses** + (`10.14279/depositonce-20357`, `-5415`) stay at zero — neither OpenAlex nor S2 + indexes references for TU-Berlin institutional DOIs. Accept as inherent. +- **Files touched (uncommitted in working tree):** `codex/sources/semanticscholar.py`, + `codex/ingest.py`, `tests/ingest/test_ingest.py`. Full suite: 331 passed. + +**Original finding (for context):** - **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph comes from only ~17 papers. The whole F-15 layer (PageRank / coupling / co-citation / discovery leads) therefore runs on ~59 % of the corpus. @@ -79,7 +116,32 @@ incomplete, or unrepresentative data. This document is that second axis — a supplement, or (b) documented as an inherent OpenAlex-coverage limit with the graph caveated accordingly. -### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **MED** +### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **RESOLVED 2026-06-16** + +**Resolution (what was done):** +- **`1911.00966` fully recovered** from the arXiv Atom API (OpenAlex 404'd on it): + title "A discrete version of Liouville's theorem on conformal maps", + Pinkall & Springborn, 2019, 384-char abstract → bibkey `PinkallSpringborn2019` + auto-generated, abstract embedded (non-zero). It was the worst node in the + corpus — now `@cite`-able, in wiki grounding, and its **10 chunks are visible + to chunk search** (previously filtered out by `bibkey IS NOT NULL`). Citations + were already added in DQ-1 (27 S2 refs). +- **`10.1007/s00454-019-00132-8` abstract recovered from S2** (1186 chars) and + re-embedded (targeted `UPDATE`, not full re-ingest — see DQ-5). +- **`10.1007/978-3-642-17413-1_7` (book chapter): documented inherent limit.** + No abstract in OpenAlex, S2, **or Crossref** (verified). Left as the single + remaining zero-vector paper; user may add an abstract by hand. +- **Corpus now: 1 paper without abstract** (the book chapter), down from 3+1. + Zero-vector search pollution (DQ-4 secondary finding) reduced from 2 tail + fillers to 1. +- **Wired into ingest** so this self-heals: OpenAlex-404 on an arXiv id now + recovers metadata via `arxiv.fetch_metadata` (new); an empty abstract is + supplemented via `_recover_abstract` → S2 then Crossref (new + `codex/sources/crossref.py`). New `semanticscholar.fetch_abstract`. +- **Files touched:** `codex/sources/arxiv.py`, `codex/sources/crossref.py` (new), + `codex/sources/semanticscholar.py`, `codex/ingest.py`, + tests. Full suite: 342. + +**Original finding (for context):** - **Measured:** - **No abstract (3):** `10.1007/978-3-642-17413-1_7`, `10.1007/s00454-019-00132-8`, `1911.00966`. These get a **zero-vector** @@ -98,7 +160,55 @@ incomplete, or unrepresentative data. This document is that second axis — a - **Acceptance:** every paper has at least a bibkey + non-zero abstract embedding, or the exceptions are documented with rationale. -### DQ-3 — Content fidelity (chunks vs source `.txt`): NOT YET VERIFIED · **MED** +### DQ-3 — Content fidelity (chunks vs source `.txt`): VERIFIED CLEAN · **RESOLVED 2026-06-16** + +**Resolution (read-only; re-ran `chunk_text` + `is_quality_chunk` on all 29 +source files and compared to stored chunks):** +- **No paper is silently gutted.** Coverage (stored chunk chars / source chars) + is **109–114% for every paper** — the >100% is exactly the 64/512-word overlap + inflation, i.e. full retention. `stored == kept` for all 29, so the live DB + matches what the current pipeline produces. +- **Only 18 chunks dropped corpus-wide, all `low_alpha`** (0 short, 0 bib), + concentrated in `10.1007/0-387-29555-0_13` (6) and `depositonce-5415` (10). + Inspected: they are **garbled OCR tables** from scanned-book sources (alpha + 0.30–0.36, e.g. `"(u: 1) 7 < u D:= ud3424 — else H®: out.v. Family 1…"`) — + exactly the artefacts the alpha-ratio gate targets. **No real math content lost** + (the handoff's feared failure mode did not occur; math prose stays > 0.40 alpha). +- **Head/tail alignment exact** on sampled papers: chunk 0 begins at the source + head (title/authors/abstract captured), last chunk ends at the source tail → no + front/back truncation. +- **Note:** 0 bibliography drops across the corpus → the upstream `.txt` extraction + had already stripped reference lists, which is why DQ-1's citation graph relies + on the API/GROBID sources rather than text-mined refs. +- **Acceptance met** for all 29 papers (not just a sample). No action needed. +- **⚠ SCOPE LIMIT — DQ-3 covers `.txt → chunks` ONLY.** It treats each source + `.txt` as ground truth and verifies the *ingest pipeline* preserves it. It does + **not** measure the layer above — **PDF/source → `.txt`** — which was done by an + upstream tool in `ConformalLabpp` (outside codex-py) and is unexamined. There is + already evidence that layer is lossy: the 18 dropped chunks were garbled OCR + tables *already present in the `.txt`* (i.e. the PDF→txt step garbled the + table/formula regions), and the bibliographies were stripped upstream. So + "VERIFIED CLEAN" means **the loader faithfully preserves whatever the `.txt` + contains, not that no information was lost from the source paper.** See DQ-6. + +### DQ-6 — PDF/source → `.txt` extraction fidelity: NOT MEASURED · **MED, open** +- **Open question:** how much did the upstream PDF→`.txt` extraction (done in + `ConformalLabpp`, outside codex-py) drop or garble vs the original papers? + Math/equations, tables, multi-column layouts, and figure captions are the usual + casualties of PDF text extraction — and we have direct evidence of garble + (DQ-3's low_alpha drops were already-garbled tables in the `.txt`). +- **Checkable:** the 36 original PDFs are present in `…/ConformalLabpp/papers/` + (map each paper's `source_path` `.txt` to its sibling `.pdf` by filename stem). +- **To investigate:** for a sample (incl. the known-bad scanned book + `0-387-29555-0_13` and a born-digital arXiv PDF), extract PDF text independently + and compare length / page-coverage to the `.txt`; spot-check whether specific + theorem statements & equations survive. Flag papers where the `.txt` is missing + large spans. +- **Acceptance:** a per-sample source→txt coverage estimate, or a list of papers + whose `.txt` is materially degraded (candidates for re-extraction — note roadmap + **R-A/R-C** would re-ingest from PDF/`.tex` directly and largely sidestep this). + +**Original open question (for context):** - **Open question:** does the stored chunk set faithfully reconstruct each source file, or did the chunker / `filter_chunks` silently drop material (e.g. an abstract, a section, math-heavy passages)? The whole corpus was ingested from @@ -111,7 +221,40 @@ incomplete, or unrepresentative data. This document is that second axis — a - **Acceptance:** sampled papers retain ≳ the expected fraction of source text; no paper is silently gutted by the quality gate. -### DQ-4 — Retrieval quality: NOT YET MEASURED · **MED** +### DQ-4 — Retrieval quality · **MEASURED 2026-06-16 — good, after fixing a P0 crash** + +**P0 bug found + fixed: `codex search paper` was crash-broken on the live DB.** +The CLI passed the query embedding uncast, so pgvector raised +`operator does not exist: vector <-> double precision[]` on every paper search. +The working call sites (`mcp_server.py`, `wiki.py`) cast `%(emb)s::vector`; the +CLI did not. Fixed both `<->` occurrences in `codex/cli.py search_paper`; added a +regression guard in `tests/cli/test_cli.py` asserting the cast (mocked-DB unit +tests can't catch the type error — this is why it survived the code audit). + +**Relevance verdict (post-fix): strong.** Six domain queries, both surfaces +(paper-level abstract search + chunk-level). **rank-1 was the exactly-correct +paper in 6/6 queries** for chunk search and 5/6 for paper-level (Laplace–Beltrami +put the exact paper at rank-2 behind the closely-related "Polygon Laplacian Made +Simple" — acceptable). Examples: "combinatorial Yamabe flow" → `math/0306167` +(exact, all 5 top chunks that paper); "variational principle for Delaunay +triangulations" → `math/0603097` (exact); "circle packing rigidity" → +`2601.22903` (exact). Three of the DQ-1-rescued papers (`2305.10988`, +`1005.2698`, `depositonce-5415`) now surface as top hits. The KB is trustworthy +for lookups. + +**Secondary finding → reinforces DQ-2: zero-vector abstract pollution.** Papers +with no abstract get a zero `abstract_emb`, which sits at constant L2 distance +≈1.000 from every query and appears as filler in the paper-level tail whenever +< 5 strongly-relevant papers exist. `1911.00966` (fully degraded, DQ-2) showed up +at distance 1.000 in 4/6 queries. Harmless at rank>1 with a visible 1.000 score, +but noise — and a concrete reason to fix DQ-2. Chunk-level search is immune (it +filters `bibkey IS NOT NULL`, and 1911.00966 has no bibkey/chunks). + +**Note:** the MCP `search` docstring claims "hybrid dense + FTS" but the SQL is +dense-only; `score = 1.0 - dist` can go negative (bge-m3 L2 ranges 0–2). Ranking +is fine (monotonic); only the absolute score label is misleading. + +**Original open question (for context):** - **Open question:** end-to-end, do real queries return *relevant* results? Clean chunks + a working index don't guarantee useful retrieval. - **To investigate next:** run a handful of domain queries through the actual @@ -123,16 +266,288 @@ incomplete, or unrepresentative data. This document is that second axis — a - **Acceptance:** a short relevance table (query → top-5 → on/off-topic) good enough to trust the KB for lookups, or a list of failure modes to fix. +### DQ-5 — `ingest_paper` is NOT idempotent for DOI papers · **RESOLVED 2026-06-17** (found 2026-06-16) + +**Resolution (what was done):** +- **Root-cause fix, minimal + contained:** added `openalex._normalize_doi()` and + applied it in `_map_paper` so the DOI branch yields the canonical **bare, + lower-cased** form (strips `https://doi.org/` / `http://doi.org/` / `doi:`; + lower-cases — DOIs are case-insensitive, matching `ingest._norm_cited_id`). The + no-DOI fallback (OpenAlex W-id URL) is left untouched. `_map_paper`→`fetch_paper` + →`ingest_paper` is the only consumer chain, so blast radius is one function. +- **Checked nothing depended on the URL form.** It was the opposite: the full-URL + id silently *defeated* `_s2_id_for(paper.id)` and `_recover_abstract`'s + `paper.id.startswith("10.")` (a `https://…` string is neither `10.`- nor + `doi:`-prefixed), so S2/Crossref recovery never fired for DOI papers. The fix + repairs those paths too. +- **Tests (offline, mocked DB):** closed audit **T-3** — the OpenAlex fixture now + emits the real URL-form DOI and asserts `paper.id` is the bare form; added + `_normalize_doi` unit tests (https/http/`doi:`/upper-case/already-bare) and a + W-id-fallback test; added ingest regression `test_ingest_doi_paper_upserts_bare + _canonical_id` (runs the real `_map_paper` through ingest, asserts the papers + upsert carries the **bare** id into `%(id)s` and uses `ON CONFLICT (id)`). + Verified it has teeth (fails on the pre-fix code). Full suite **348 passed**; + ruff + mypy --strict clean. +- **Live idempotency confirmed (tunnel up):** re-ingested + `10.1007/s00454-019-00132-8` against the Jetson DB → no `UniqueViolation`; + `IngestResult(paper_id='10.1007/s00454-019-00132-8', citations_upserted=45)`. + After-state: **UPDATE in place** — `id` and `openalex_id` unchanged, **`added_at` + unchanged** (proves no fresh INSERT), exactly **1** row for the DOI (no URL-form + duplicate), citations 45→45, total papers 29→29. +- **Unblocks R-A (GROBID re-ingest) and R-C (.tex re-ingest)** — both re-ingest + existing papers. **Files touched:** `codex/sources/openalex.py`, + `tests/sources/test_openalex.py`, `tests/ingest/test_ingest.py`. +- **Note (out of scope, pre-existing):** arXiv papers stored as bare ids + (`2305.10988`) are still not round-trip idempotent if OpenAlex returns an + arXiv-DOI/W-id instead of the bare arXiv id — a separate canonicalization + concern, not this UniqueViolation. The DOI path (this finding) is fixed. + +**Original finding (for context):** +- **Symptom:** re-ingesting an existing DOI paper crashes with + `UniqueViolation: papers_openalex_id_key`. Surfaced trying to re-ingest + `10.1007/s00454-019-00132-8` during the DQ-2 backfill. +- **Root cause:** `openalex._map_paper` sets `Paper.id = data["doi"]`, which is the + **full URL** `https://doi.org/10.1007/…`. The stored canonical id is the **bare** + `10.1007/…` (post the M-1 canonical-id migration). So `INSERT … ON CONFLICT (id)` + finds no id match, attempts a fresh INSERT, and trips the *separate* unique + constraint on `openalex_id` (already held by the canonical-id row). +- **Confirmed:** `openalex.fetch_paper("10.1007/s00454-019-00132-8").id == + "https://doi.org/10.1007/s00454-019-00132-8"` (≠ stored bare id). +- **Impact:** any re-ingest of a DOI paper aborts. This **blocks roadmap R-A + (GROBID on PDFs) and R-C (.tex ingest)** — both re-ingest existing papers. Also + implies *new* DOI ingests store full-URL ids, inconsistent with the bare-id + corpus (latent split-identity risk). +- **Proposed fix (handle with care — M-1 incident territory):** normalize the DOI + in `_map_paper` to the canonical bare, lower-cased form (strip + `https://doi.org/`), matching what the M-1 migration produced. Add a re-ingest + idempotency test over a DOI paper. Did **not** fix inline this session — the + canonical-id derivation is exactly where the M-1 migration incident occurred, so + it needs its own focused change + a live re-ingest check. +- **Workaround used for DQ-2:** targeted `UPDATE … SET abstract, abstract_emb` + instead of full re-ingest. + +--- + +## Roadmap — data-acquisition levers (post-audit, all free sources) + +These are *forward-looking acquisition improvements*, distinct from the DQ-1..DQ-4 +audit findings (which assessed existing data). They came out of a strategy +discussion on 2026-06-16: for this Math/CS corpus the valuable data (references, +abstracts, full text) is almost entirely **open** (arXiv, OpenAlex, Crossref, S2), +so the quality lever is *combining more free sources*, not buying paywall access. +Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item +is self-contained so a cold session can pick it up. + +### R-A — Run GROBID reference extraction on PDFs · **CORE DONE 2026-06-17** (full sweep optional) + +**Resolution (what was done) — citing coverage 27/29 → 29/29 (100%); edges 920 → 1022:** +- **References-only backfill, not full re-ingest.** Added + [scripts/ra_grobid_backfill.py](../../scripts/ra_grobid_backfill.py): per paper, + run `grobid.extract_references(pdf)`, normalize cited-ids to the canonical bare + form (DOIs de-URL'd + lower-cased; `arXiv:` stripped), INSERT into `citations` + (`ON CONFLICT DO NOTHING`). Deliberately avoids `ingest_paper(source_path=pdf)`, + whose PDF branch re-runs Nougat + `DELETE FROM chunks` and would overwrite the + DQ-3-verified-clean `.txt` chunks (and needs Nougat). Idempotent; dry-run by + default; 7 tests; ruff/mypy clean. +- **GROBID provisioning:** the pinned `grobid/grobid:0.8.2` is amd64-only, but the + Jetson and the dev Mac are both arm64 (this is why GROBID never ran). Ran the + lighter CRF image (`grobid/grobid:0.8.2-crf`) on the Mac under Podman+Rosetta + (PDFs are local; citations written over the SSH DB tunnel). Made + `extract_references` timeout configurable (theses need >60s emulated). A separate + task is provisioning a native arm64 GROBID on the Jetson (`docs/infra/grobid-jetson.md`). +- **Both `depositonce` theses closed:** `…depositonce-20357` (lutz-2024) **+96** + edges (5 in-corpus); `…depositonce-5415` (sechelmann-2016) **+6** edges — its + 147 MB/173-page scan blew pdfalto's 120s limit, so the 9 bibliography pages were + extracted with PyMuPDF first (77 refs, only 6 with a DOI/arXiv). **No paper has + zero out-edges now.** +- **Optional remaining (secondary acceptance):** sweep the other 27 already-covered + papers for *extra* GROBID refs via `python scripts/ra_grobid_backfill.py --write` + (the 2 book PDFs need the same bib-page extraction as sechelmann). + +**Original plan (for context):** +- **✓ Unblocked (DQ-5 RESOLVED 2026-06-17):** DOI re-ingest is now idempotent, so + the GROBID re-ingest no longer aborts on `papers_openalex_id_key`. +- **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`), + so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper + so `codex.parsing.grobid.extract_references` parses each bibliography. +- **Why:** extracts references the APIs lack — a *complement* to DQ-1's S2 + supplement. Crucially, the **2 `depositonce` theses** that stayed at zero in + DQ-1 (no API references anywhere) almost certainly have a references section in + their open-access PDFs → GROBID could finally give them out-edges. Closes the + last 2/29. +- **How:** no new code — ingest already merges + dedups `api_citations + + pdf_citations` ([codex/ingest.py:288](../../codex/ingest.py)). GROBID server is + already configured (`GROBID_URL=http://192.168.178.103:8070` in + `.env.jetson-ingest`). Download free arXiv PDFs, re-ingest with `source_path`. +- **Acceptance:** the 2 theses gain references; published papers gain + GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29. + +### R-B — Add Crossref as a third citation + abstract source · **DONE 2026-06-17** + +**Resolution (what was done):** +- **`crossref.fetch_references(doi)`** added (mirrors `semanticscholar.fetch_references`): + GETs `/works/{doi}`, returns one `Citation` per `message.reference[]` entry that + carries a registered `DOI` (book/`unstructured`-only refs are skipped — they + can't join the graph). `citing_id` is the queried DOI; the ingest supplement + rewrites it to `paper.id`. +- **Wired as the third leg of the citation fallback chain** in `ingest.py`: + OpenAlex-empty → S2 → **Crossref** (`_crossref_reference_supplement`, DOI papers + only — Crossref's works endpoint is DOI-keyed). cited DOIs pass through + `_norm_cited_id` (bare, lower-cased). Network/parse failures degrade to []. +- **Tests:** `fetch_references` unit tests (DOI edges / `unstructured` skipped / no + `reference` key / 404) + ingest tests asserting the Crossref leg fires when + OpenAlex+S2 are empty (cited DOI normalised) **and** does *not* fire when OpenAlex + has citations. Suite **361 green**, ruff/mypy clean. The DQ-2 abstract half + (`fetch_abstract`) already shipped earlier. +- **Live-validated:** `fetch_references` against the real API returned 33 / 40 / 24 + DOI-bearing refs for `10.1007/s00454-019-00132-8` / `10.1145/3132705` / + `10.1111/cgf.13931` (Springer / ACM / Wiley). +- **No live corpus change (by design):** as a *fallback*, Crossref fires only when + OpenAlex **and** S2 are both empty. Post-DQ-1/R-A the only such papers were the 2 + `depositonce` theses, which Crossref doesn't index either — so R-B is a + forward-looking third leg for future DOI ingests, not a backfill. (Harvesting + Crossref's *unique* refs for already-covered papers would need an always-merged + supplement instead — a deliberate scope choice not taken, per the doc's "fallback + chain" spec.) **Files:** `codex/sources/crossref.py`, `codex/ingest.py`, + tests. + +**Original plan (for context):** +- **What:** new `codex.sources.crossref` client hitting Crossref REST + `/works/{doi}` for the `reference` array and `abstract`. +- **Why:** Crossref carries **publisher-deposited references** for many DOIs (free, + polite pool via `mailto`) — a third leg after OpenAlex/S2. It also recovers + abstracts OpenAlex is contractually barred from redistributing (likely the 2 + Springer DOIs missing abstracts in DQ-2). cited_ids are DOIs → flow through the + existing graph resolver unchanged. +- **How:** mirror `codex/sources/openalex.py` (tenacity retry, polite pool). Extend + the ingest fallback chain to OpenAlex-empty → S2 → Crossref (same + `_s2_reference_supplement` shape). Add unit tests with mocked HTTP. +- **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where + OpenAlex+S2 are empty. + +### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **DONE 2026-06-17** + +**Resolution (what was done) — all 13 arXiv papers re-ingested from `.tex`:** +- **Multi-file flatten:** `arxiv.fetch_source` now inlines `\input`/`\include` + (`tex.flatten_inputs`) so multi-file projects yield the full body, not the + primary file's include skeleton (e.g. math/0603097 = 15 files / 12 includes; + 2305.10988 = 6 / 5). Also handles legacy single-file **bare-gzipped** `.tex` + (no tar wrapper) for old math/* papers (commit 121b582). +- **Section-aware chunking:** new `tex.chunk_sections` returns `(title, chunk)` + pairs (chunks never span a `\section`); the ingest `.tex` path labels the stored + `section` from the real heading instead of running `classify_section` on a + header-less word-window. Quality-gated as pairs so labels stay aligned. +- **Unblocked by the arXiv half of DQ-5** (commit f504ca6): `_canonical_id` + strips the `10.48550/arxiv.` DOI prefix so an arXiv paper's id is the bare + arXiv id — without it, re-ingest tripped `papers_openalex_id_key` (confirmed + live, then fixed; idempotent re-ingest verified). +- **Outcome:** `section` column gained signal — ~34/329 arXiv chunks now + intro/theorem/proof (was ~all `body`); math/0603097 `{body:31}`→`{body:29, + proof:9}`, 2310.17529 gained proof+intro, etc. Math papers with descriptively + titled sections stay `body` (classify_section's fixed vocab) — modest but real + (R-12). No paper gutted (chunk counts stable/▲); F-16 gate unchanged. +- **Files:** `codex/parsing/tex.py`, `codex/sources/arxiv.py`, `codex/ingest.py`, + `codex/sources/openalex.py`, `scripts/rc_tex_reingest.py` (driver, dry-run + default), + tests. Full suite **370 passed**; ruff/mypy clean. Branch + `feat/tex-ingest` (commits 1da81c7, 121b582, f504ca6). + +**Original plan (for context):** +- **What:** re-ingest from arXiv LaTeX source through the existing + `codex.parsing.tex.latex_to_text` path instead of flattened `.txt`. +- **Why:** highest-fidelity input — preserves math, structure, and section + boundaries. Directly serves **DQ-3** (content fidelity) and audit **R-12** (the + `section` column is low-signal because word-window `.txt` chunks rarely start at + a real header; `.tex` headers fix this). +- **How:** download free arXiv source tarballs, feed `.tex` to ingest with + `source_path`. Handle multi-file projects (main `.tex` + includes). +- **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage + vs source improves; no regression in the F-16 quality gate. + +### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **DONE 2026-06-17** + +**Resolution (what was done):** +- Added `graph.citing_coverage(graph, known_ids)` → `(papers_with_out_edges, + total)`, a `graph_min_citing_coverage` setting (default 0.8), and wired both into + `codex graph report`: it now prints `Citing coverage: N/M papers with out-edges + (P%)` (and in `--json`), and emits a `Warning` when the share is below the + threshold ("low citing coverage weakens PageRank/coupling"). Separate from the + existing `graph_min_corpus_size` (total-count) warning. +- Tests: `citing_coverage` unit tests + CLI tests (line shown; warning fires at + low coverage; suppressed at 100%; JSON field). Full suite green; ruff/mypy clean. +- **Live:** `codex graph report` on the corpus prints `Citing coverage: 29/29 + papers with out-edges (100%)` with no warning (post-R-A). Branch + `feat/graph-coverage-warning`. Files: `codex/config.py`, `codex/graph.py`, + `codex/cli.py`, + tests. + +**Original plan (for context):** +- **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper + count. Add a warning when the share of papers with ≥1 out-edge is low (e.g. + < 80%), since that is what actually starves PageRank/coupling. +- **Why:** at 17/29 the graph silently ran on 59% of the corpus with no signal. + Post-DQ-1 it's 27/29 (93%) — but the guard should catch future regressions. +- **Acceptance:** `codex graph report` surfaces citing-coverage % and warns below + threshold. + +### R-E — Paywall / institutional access · **DEFERRED — only for a paywall-only expansion** +- **Decision (2026-06-16):** a uni login does **not** help the current corpus — + every DQ-1/DQ-2 gap is in openly-available works (arXiv preprints, OA theses), + and all metadata/citation sources are free APIs. Revisit *only* if the corpus + expands to papers that exist solely behind a paywall with no preprint; then + institutional access → full-text PDF → GROBID refs (R-A) + clean chunks (R-C). +- **ToS caveat:** manual download of individual papers you have legitimate access + to is fine; **automated bulk download through a uni proxy (EZproxy/Shibboleth) + violates most publisher terms and can get the whole institution's access + revoked.** Do not wire a publisher login into the ingest pipeline. + +### R-F — Richer section labels: store real `\section` titles · **DONE 2026-06-17** + +**Resolution (what was done):** +- **Why:** R-C labelled `.tex` chunks by mapping the real `\section` heading through + `classify_section`'s fixed vocab, so descriptively-titled Math sections collapsed + to `body` (only **34/329 = 10 %** non-`body`). A research pass confirmed the + `section` column is **write-only** (nothing filters on the vocab — `mcp_server.py` + and `wiki.py` don't even SELECT it), so storing free-text titles is safe. +- **Change:** new `quality.section_label(title, content)` (+ `_clean_title`) — keep the + controlled bucket (intro/theorem/proof/abstract/bibliography) when the heading maps + to one, else store the cleaned real title ("the flip algorithm", "main results", …). + The `.tex` ingest path uses it; `.pdf`/`.txt`/`run_quality_pass` keep + `classify_section` unchanged. `_clean_title` strips LaTeX commands/accents/ties + + leading numbering, lower-cases, truncates at a word boundary. ~1 helper + 1 ingest + line; no schema change (column already free `TEXT`). +- **Result (live):** re-applied to the 13 arXiv papers → non-`body` **34/329 (10 %) → + 314/329 (95 %)**. Only `math/0306167` (old AMS-TeX, no `\section{}`) stays `body`. +- **Tests:** `section_label` / `_clean_title` units + `.tex` ingest stores a + descriptive heading verbatim. Full suite 377 passed; ruff/mypy clean. Branch + `feat/section-labels` (off `feat/tex-ingest`). +- **Accent re-clean — DONE 2026-06-17.** The `_clean_title` polish (`m\"obius`→`mobius`, + `~` ties, word-boundary truncation) landed in code, and the existing live labels were + re-cleaned in place (apply `_clean_title` to any `chunks.section` containing `\` or + `~`; **8 labels** fixed, no re-embed). Verified: **0** labels with LaTeX artifacts + remain; non-`body` holds at **314/329 (95 %)**. R-F fully complete. + --- ## Priority for the next session -1. **DQ-1** (biggest lever — 41 % of the corpus is invisible to the graph; the S2 - supplement is concrete and reuses existing code). -2. **DQ-4** (cheap, high-information — tells you if the KB is actually usable). -3. **DQ-2**, then **DQ-3**. +1. ~~**DQ-1**~~ — **DONE 2026-06-16** (live DB at 920 citations, 27/29 citing + coverage; remedy wired into ingest). +2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance + verified strong; surfaced the zero-vector pollution that motivates DQ-2). +3. ~~**DQ-2**~~ — **DONE 2026-06-16** (1911.00966 fully recovered; s00454 abstract + from S2; book chapter documented; recovery wired into ingest). +4. ~~**DQ-3**~~ — **DONE 2026-06-16** (content fidelity clean; full coverage, only + 18 OCR-junk chunks dropped, no math lost). **Assessment axis DQ-1..DQ-4 complete.** +5. ~~**DQ-5**~~ — **DONE 2026-06-17** (DOI id normalized to bare form in + `openalex._map_paper`; live re-ingest of `10.1007/s00454-019-00132-8` idempotent + — UPDATE in place, `added_at` unchanged; suite 348 green). **Unblocks R-A/R-C.** +6. **Roadmap levers** (see section above — now all unblocked): **R-A** (GROBID on + arXiv PDFs — closes the last 2 zero-citation theses), **R-B** (Crossref refs — + abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3), + **R-D** (F-15 coverage warning, quick). **R-E** (paywall) deferred. -All four are *data* work (queries + maybe a re-ingest/enrichment), not code-audit -work — the code is already remediated (see the AUDIT-* docs and PRs #12–#14). +DQ-1..DQ-4 are *data*/assessment work; DQ-5 + roadmap R-A..R-D add **new code** +(canonical-id fix, re-ingest scripts, Crossref references, a `.tex` path, a graph +warning) on top of the already-remediated pipeline (see the AUDIT-* docs and PRs +#12–#14). The DQ-1/DQ-2 ingest wiring + Crossref/arXiv abstract recovery already +landed this session. --- diff --git a/docs/infra/grobid-jetson.md b/docs/infra/grobid-jetson.md new file mode 100644 index 0000000..7a43e92 --- /dev/null +++ b/docs/infra/grobid-jetson.md @@ -0,0 +1,192 @@ +# GROBID on the Jetson (aarch64) — deploy runbook + +**Status:** ACTIVE +**Host:** `alfred@192.168.178.103` — NVIDIA Jetson Orin Nano (aarch64, 8 GB RAM, 8 GB swap, 6 cores) +**Service:** GROBID reference extraction for roadmap **R-A** (`scripts/ra_grobid_backfill.py`) +**Last updated:** 2026-06-17 +**Verified:** 2026-06-17 — `grobid/grobid:0.9.0-crf` pulled as native `arm64`, `/api/isalive`→`true`, +end-to-end `extract_references` on `lutz-2024-thesis.pdf` → 116 refs (101 with DOI/arXiv) in ~36s. + +--- + +## Why this image (`grobid/grobid:0.9.0-crf`) + +GROBID had never come up on the Jetson because `infra/docker-compose.yml` pinned +`grobid/grobid:0.8.2`, and **every `0.8.x` tag (and the full `0.9.0`) is published +amd64-only**. A Docker tag resolves to a multi-arch *manifest list*; on aarch64 +the `0.8.2` list has no `linux/arm64` entry, so there is simply nothing to pull or +run (hence `curl localhost:8070` → connection refused, nothing listening). + +The fix is to select a tag whose manifest list actually contains a `linux/arm64` +build: + +| Candidate | arm64? | Size | Notes | +|-------------------------------|:------:|--------:|----------------------------------------| +| `grobid/grobid:0.8.2` | ✗ | ~9.5 GB | full DL image, amd64-only (old pin) | +| `grobid/grobid:0.9.0` / `-full`| ✗ | ~10 GB | full DL image, amd64-only | +| **`grobid/grobid:0.9.0-crf`** | **✓** | ~500 MB | **CRF-only, official, multi-arch** ← | +| `lfoppiano/grobid:0.9.0-crf` | ✓ | ~500 MB | same build, community namespace (alt.) | + +We use **`grobid/grobid:0.9.0-crf`**: + +- **CRF-only** is sufficient for reference extraction (`/api/processReferences`) + and is ~500 MB vs ~10 GB for the deep-learning image. +- **Native arm64** — no qemu emulation (too slow / RAM-heavy on a Jetson). +- **Official namespace**, **pinned version** (not `latest-crf`, which is a moving + target). arm64 builds only exist from `0.9.0-crf` onward — there is no arm64 + `0.8.x`, so this is a forced (but compatible) bump from the old `0.8.2` pin. The + TEI that `codex/parsing/grobid.py` parses (`biblStruct`, `persName`, + `idno[@type='DOI']`, …) is unchanged across the bump. + +> Caveat from upstream: the arm64 image is documented as "tested only on macOS, +> not linux/arm64". The container is the *same* `linux/arm64` ELF either way, so +> bare-metal Jetson works; the end-to-end check below is what de-risks it. + +--- + +## One-time prerequisites + +### 1. Docker permission for `alfred` + +The SSH user `alfred` is in the `sudo` group but **not** the `docker` group, so +`docker …` fails with `permission denied … /var/run/docker.sock`. Grant access +once (requires the sudo password): + +```bash +ssh alfred@192.168.178.103 +sudo usermod -aG docker alfred # add to docker group (persistent) +``` + +Then **start a fresh login** so the new group takes effect (group membership is +only re-evaluated at login — an existing session/`ssh` won't see it; a *new* +`ssh` connection will): + +```bash +exit && ssh alfred@192.168.178.103 +docker ps # should now work without sudo +``` + +(Alternatively run all `docker` commands below under `sudo`, but group membership +is cleaner and lets `ra_grobid_backfill.py` / curl checks be driven over plain SSH.) + +### 2. Docker Compose v2 plugin (arm64) + +This host has Docker Engine but **not** the Compose v2 CLI plugin (on Linux they +are separate packages; only Docker Desktop bundles Compose). Without it +`docker compose …` fails with `docker: unknown command: docker compose`. Install +the arm64 plugin into the user-local plugin dir (no sudo needed): + +```bash +mkdir -p ~/.docker/cli-plugins +curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-aarch64 \ + -o ~/.docker/cli-plugins/docker-compose +chmod +x ~/.docker/cli-plugins/docker-compose +docker compose version # confirm it resolves +``` + +--- + +## Deploy + +GROBID is **service-scoped** on purpose: Postgres (`papers-db`) is already running +on this host, so we bring up only the `grobid` service and never `docker compose up` +the whole stack (that would try to start a second `papers-db`). + +```bash +# copy this repo's compose file to the Jetson (first time only) +scp infra/docker-compose.yml infra/schema.sql alfred@192.168.178.103:~/codex-infra/ + +ssh alfred@192.168.178.103 +cd ~/codex-infra +docker compose up -d grobid # pulls grobid/grobid:0.9.0-crf (arm64), starts papers-grobid +docker compose logs -f grobid # wait for "Started ... in N seconds" (model load ~20–40s) +``` + +The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by +Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves +headroom alongside Postgres. The `restart: "no"` policy means GROBID does **not** +auto-start on boot — see "Run on-demand" below. + +--- + +## Run on-demand (stop when idle) + +GROBID is only needed **during reference extraction** (`scripts/ra_grobid_backfill.py`); +nothing else in the stack uses it. Idle it still holds ~3.5 GB, so on the 8 GB Jetson +keep it stopped and start it only for an R-A run: + +```bash +ssh alfred@192.168.178.103 'docker start papers-grobid' # ~30s model load +curl -s --retry 30 --retry-delay 2 --retry-connrefused \ + http://192.168.178.103:8070/api/isalive # wait for: true + +# ... run R-A from the Mac ... +PYTHONPATH=. python scripts/ra_grobid_backfill.py --write + +ssh alfred@192.168.178.103 'docker stop papers-grobid' # reclaim ~3.5 GB +``` + +First-time deploy uses `docker compose up -d grobid` (above); afterwards the +container persists, so `docker start/stop papers-grobid` is all you need. + +> Why on-demand rather than a lighter tool: refextract was evaluated head-to-head +> on the Jetson and rejected — far worse DOI recall on this math corpus (lutz +> thesis: GROBID **101** usable IDs vs refextract **2**), and it was *slower* per +> PDF despite a smaller footprint. GROBID stays; running it on-demand removes its +> only real downside (idle RAM). + +--- + +## Verify `GET /api/isalive` → `true` + +```bash +# on the Jetson +curl -s http://localhost:8070/api/isalive # => true + +# from the Mac (LAN) — same value the corpus' GROBID_URL points at +curl -s http://192.168.178.103:8070/api/isalive # => true +``` + +Anything other than `true` (timeout / connection refused / `false`) means the +container isn't up — check `docker compose logs grobid` and that nothing else +holds port 8070. + +--- + +## End-to-end reference-extraction check + running R-A + +With `GROBID_URL=http://192.168.178.103:8070` (already set in `.env.jetson-ingest`): + +```bash +# one PDF, references only — should print N>0 parsed references +PYTHONPATH=. python - <<'PY' +from codex.parsing.grobid import extract_references +refs = extract_references("/Users/tarikmoussa/Desktop/ConformalLabpp/papers/lutz-2024-thesis.pdf", + grobid_url="http://192.168.178.103:8070") +print(f"{len(refs)} references; first DOI/arXiv:", + next(((r['doi'], r['arxiv_id']) for r in refs if r['doi'] or r['arxiv_id']), None)) +PY +``` + +Then run the references-only backfill (DB tunnel up — see +`docs/audit/DATA-QUALITY-2026-06-15.md`): + +```bash +ssh -N -L 5433:localhost:5432 alfred@192.168.178.103 & # DB tunnel +cp .env.jetson-ingest .env +PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only # dry-run first +PYTHONPATH=. python scripts/ra_grobid_backfill.py --write # apply +``` + +--- + +## Troubleshooting + +- **Container killed mid-request / exit 137** — OOM. The CRF image needs ~3 GB for + references; raise the compose `memory` limit or check Postgres isn't starving the + box (`free -h`, `docker stats papers-grobid`). +- **`isalive` true but `processReferences` 500s** — usually a `pdfalto` failure on a + malformed PDF; confirm with a clean PDF (`lutz-2024-thesis.pdf`) and check + `docker compose logs grobid`. +- **`permission denied … docker.sock`** — the docker-group prerequisite above wasn't + applied, or the SSH session predates it (re-login). diff --git a/docs/infra/jetson-ssh-stability.md b/docs/infra/jetson-ssh-stability.md new file mode 100644 index 0000000..2358f21 --- /dev/null +++ b/docs/infra/jetson-ssh-stability.md @@ -0,0 +1,66 @@ +# Infra TODO — stable long-running ops against the Jetson (tmux + autossh) + +**Status:** OPEN (infra). **Filed:** 2026-06-17. **Priority:** MED — it slows live ops +but has a known workaround (short/idempotent ops). + +## Problem (observed) + +Live data ops run on the **Mac** and reach the Jetson Postgres through an SSH +port-forward tunnel (`ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103`, +`DATABASE_URL` → `localhost:5433`). During the R-A / R-C / R-F work this tunnel +**dropped repeatedly mid-run**: any operation longer than a few minutes (corpus +re-ingest, the bge-m3 embed loop, the GROBID sweep) had a high chance of failing +partway when the tunnel died, leaving the work half-applied. At one point the Jetson +went **fully offline** (ping 100 % loss, `ssh:22` timeout) mid-operation. + +`ingest_paper()` and the backfill scripts open one DB connection for the whole run, so +a tunnel drop raises `OperationalError: connection refused` and aborts — partial state, +manual retry. Adding `ServerAliveInterval` to the tunnel helped only marginally. + +## Recommended fix + +**1. Run long ops *on the Jetson* inside `tmux` (primary fix).** +The Jetson is where Postgres (and GROBID) live, so running ingest there means the DB is +`localhost` — **no tunnel at all** — and `tmux` keeps the process alive across SSH +disconnects (detach / reattach to monitor). This removes the tunnel from the critical +path for every long-running write. + +- [ ] Confirm the `codex` env is deployable on the Jetson (Python 3.12+, deps incl. the + bge-m3 embedder; the Orin Nano GPU should handle it — GROBID already runs there). + If not, stand up a venv / container for it. +- [ ] Standard pattern: `ssh alfred@jetson`, then + `tmux new -s codex` → `set -a; source .env; set +a` → + `PYTHONPATH=. python scripts/.py --write` → detach (`Ctrl-b d`). + Reattach later with `tmux attach -t codex`. The job survives the SSH drop. +- [ ] Point the Jetson-side `DATABASE_URL` at `localhost:5432` directly (no `:5433`). +- [ ] Document this as the default for any multi-minute write (re-ingest, sweeps, + embed loops) in `docs/audit/DATA-QUALITY-2026-06-15.md` "How to reach the data". + +**2. `autossh` for when a tunnel *is* needed (secondary).** +For interactive/dev use from the Mac (psql, quick probes, read-only queries), replace +the plain `ssh -f -N -L` with `autossh -M 0 -f -N -o ServerAliveInterval=15 -o +ServerAliveCountMax=3 -L 5433:localhost:5432 alfred@jetson` so a dropped tunnel +auto-reconnects. NB: autossh only restarts the *tunnel* — an in-flight transaction +still fails, so this is for short/interactive use, not long writes (use tmux-on-Jetson +for those). + +**3. (Optional, code) make backfill scripts resilient.** +Lower-value once (1)/(2) are in place, but: open a fresh `get_conn()` per paper (not one +for the whole run) and/or wrap the per-item write in a small retry, so a transient drop +costs one item, not the whole batch. The R-A backfill already does per-paper connections; +`rc_tex_reingest.py` opens a connection per paper too — but `ingest_paper` itself holds +one connection for its whole body, which is the unit that fails. + +## Why this matters + +The chronic flakiness directly cost time this session and left one cosmetic task +unfinished (the R-F accent re-clean — see DATA-QUALITY-2026-06-15.md R-F "Pending"). The +**robust pattern that already works** is short, idempotent operations (per-paper, or a +single quick `UPDATE`); tmux-on-Jetson makes even the long ops safe. + +## Acceptance + +- A documented, repeatable way to run a full corpus re-ingest that survives an SSH drop + (tmux-on-Jetson), verified by detaching/reattaching across a disconnect. +- The DATA-QUALITY doc's "How to reach the data" section recommends tmux-on-Jetson for + long writes and `autossh` for interactive tunnels. diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index f7c8912..9304909 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -28,20 +28,34 @@ services: restart: unless-stopped grobid: - # CRF-only image is sufficient for reference extraction and is lighter than - # the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid - # for the latest 0.8.x tag before deploying. - image: grobid/grobid:0.8.2 + # CRF-only image: sufficient for reference extraction (/api/processReferences) + # and ~500MB vs ~10GB for the full deep-learning variant. + # + # The `-crf` tag is also the ONLY GROBID variant published as a multi-arch + # manifest that includes linux/arm64 — the full image (grobid/grobid:0.9.0) + # and every 0.8.x tag are amd64-only. So this is what lets GROBID run + # *natively* on the aarch64 Jetson (no slow/RAM-heavy qemu emulation). arm64 + # builds start at 0.9.0-crf; there is no arm64 0.8.x. + # Deploy + docker-permission fix: docs/infra/grobid-jetson.md + image: grobid/grobid:0.9.0-crf container_name: papers-grobid ports: - "8070:8070" - restart: unless-stopped - # GROBID is RAM-hungry; uncomment to cap usage on constrained hardware - # (e.g. Nvidia Jetson): - # deploy: - # resources: - # limits: - # memory: 4g + init: true # reap zombie children (GROBID docs recommend --init) + ulimits: + core: 0 # disable core dumps (GROBID docs: --ulimit core=0) + # On-demand only: GROBID is needed solely during reference extraction + # (scripts/ra_grobid_backfill.py), not for normal corpus operations — start it + # for an R-A run, stop it after to reclaim ~3.5GB. "no" = no auto-start on boot. + # See docs/infra/grobid-jetson.md ("Run on-demand"). + restart: "no" + # GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB + # RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom. + # Honored by `docker compose up` under Compose v2 (no swarm needed). + deploy: + resources: + limits: + memory: 4g volumes: pgdata: diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..82057c3 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Operational one-off scripts (importable for tests).""" diff --git a/scripts/ra_grobid_backfill.py b/scripts/ra_grobid_backfill.py new file mode 100644 index 0000000..5f8ed04 --- /dev/null +++ b/scripts/ra_grobid_backfill.py @@ -0,0 +1,227 @@ +"""R-A — GROBID reference backfill (references-only, idempotent). + +Adds GROBID-parsed citations to the live corpus **without** re-chunking the body, +so the DQ-3-verified-clean ``.txt`` chunks are preserved. Complements the DQ-1 +Semantic-Scholar supplement with references the APIs lack (notably the two +TU-Berlin ``depositonce`` theses, which neither OpenAlex nor S2 indexes). + +Why references-only instead of ``ingest_paper(source_path=pdf)`` +---------------------------------------------------------------- +The PDF branch of :func:`codex.ingest.ingest_paper` runs Nougat OCR and then +``DELETE FROM chunks`` + re-INSERT — it would replace the clean ``.txt`` chunks +(DQ-3) with unmeasured PDF-OCR text and additionally requires the Nougat server. +This backfill touches only the ``citations`` table (``ON CONFLICT DO NOTHING``), +so it is safe, idempotent, and needs only a reachable GROBID server. + +Prerequisite +------------ +A reachable GROBID server. The corpus' ``GROBID_URL`` points at the Jetson +(``http://192.168.178.103:8070``). GROBID now runs natively on that aarch64 host +via the multi-arch CRF-only image (``grobid/grobid:0.9.0-crf``) — see +``docs/infra/grobid-jetson.md`` for the deploy + docker-permission fix and the +``GET /api/isalive`` check. (The old ``grobid/grobid:0.8.2`` pin was amd64-only, +which is why GROBID never came up; pass ``--grobid-url`` to override the default.) +Use the SSH DB tunnel for ``DATABASE_URL`` as documented in +``docs/audit/DATA-QUALITY-2026-06-15.md``. + +Usage +----- + # dry run (no writes) over the two zero-edge theses + PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only + + # write every paper's GROBID-parsed refs to the live DB + PYTHONPATH=. python scripts/ra_grobid_backfill.py --write + +Default is **dry-run**; pass ``--write`` to mutate the live DB. +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +import psycopg +import psycopg.rows + +from codex.config import get_settings +from codex.db import get_conn +from codex.models import Citation +from codex.parsing.grobid import extract_references + +logger = logging.getLogger(__name__) + +DEFAULT_PDF_DIR = "/Users/tarikmoussa/Desktop/ConformalLabpp/papers" + + +def _normalize_cited_id(doi: str = "", arxiv_id: str = "") -> str | None: + """Normalize a GROBID-extracted reference id to the corpus' canonical form. + + DOIs → bare, lower-cased (strip ``https://doi.org/`` / ``http://`` / ``doi:``), + matching ``papers.id`` and ``codex.sources.openalex._normalize_doi`` (DQ-5). + arXiv ids → strip a leading ``arXiv:`` prefix, leaving the bare id + (``2305.10988``, ``math/0603097``). DOI takes precedence when both exist. + Kept self-contained rather than reusing ``ingest._norm_cited_id`` because it + additionally strips the ``arXiv:`` prefix (which GROBID can emit but that + helper leaves untouched), and to avoid importing a private cross-module name. + """ + doi = (doi or "").strip() + if doi: + s = doi.lower() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if s.startswith(prefix): + return s[len(prefix) :] + return s + arx = (arxiv_id or "").strip() + if arx: + return arx[len("arxiv:") :].strip() if arx.lower().startswith("arxiv:") else arx + return None + + +def build_grobid_citations( + paper_id: str, + pdf_path: str | Path, + grobid_url: str | None = None, + timeout: float = 60.0, +) -> list[Citation]: + """Extract a paper's references via GROBID and return normalized Citations. + + ``citing_id`` is the canonical ``paper_id``; ``cited_id`` is normalized. + Self-citations and within-paper duplicates are dropped. Refs with neither a + DOI nor an arXiv id are skipped (they cannot join the graph). + """ + refs = extract_references(str(pdf_path), grobid_url=grobid_url, timeout=timeout) + seen: set[str] = set() + citations: list[Citation] = [] + for ref in refs: + cited = _normalize_cited_id(ref.get("doi", ""), ref.get("arxiv_id", "")) + if not cited or cited == paper_id or cited in seen: + continue + seen.add(cited) + citations.append(Citation(citing_id=paper_id, cited_id=cited)) + return citations + + +def _coverage(conn: psycopg.Connection[psycopg.rows.DictRow]) -> tuple[int, int, int]: + """Return (n_papers, n_papers_with_out_edges, n_citations) from the live DB.""" + row = conn.execute( + """ + SELECT (SELECT count(*) FROM papers) AS papers, + (SELECT count(DISTINCT citing_id) FROM citations) AS citing, + (SELECT count(*) FROM citations) AS edges + """ + ).fetchone() + assert row is not None + return int(row["papers"]), int(row["citing"]), int(row["edges"]) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pdf-dir", default=DEFAULT_PDF_DIR, help="directory of source PDFs") + parser.add_argument("--grobid-url", default=None, help="override Settings().grobid_url") + parser.add_argument("--only", default=None, help="restrict to a single paper id") + parser.add_argument( + "--zero-edge-only", + action="store_true", + help="only papers with 0 out-edges (the R-A primary targets)", + ) + parser.add_argument("--limit", type=int, default=None, help="cap number of papers") + parser.add_argument( + "--timeout", + type=float, + default=300.0, + help="GROBID HTTP timeout (s); theses can need >60s, esp. emulated", + ) + parser.add_argument( + "--write", action="store_true", help="WRITE to the live DB (default: dry-run)" + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + grobid_url = args.grobid_url or get_settings().grobid_url + pdf_dir = Path(args.pdf_dir) + pdfs = {p.stem: p for p in pdf_dir.glob("*.pdf")} + + # Read the target list in a short-lived connection, then release it: the GROBID + # extraction below is a slow per-paper network loop and must NOT hold an + # idle-in-transaction connection open across it (a mid-loop SSH-tunnel drop would + # otherwise abort the whole batch — see docs/infra/jetson-ssh-stability.md). + with get_conn() as conn: + rows = conn.execute( + """ + SELECT p.id, p.source_path, + (SELECT count(*) FROM citations ci WHERE ci.citing_id = p.id) AS out_edges + FROM papers p + ORDER BY out_edges, p.id + """ + ).fetchall() + + targets: list[tuple[str, Path]] = [] + for r in rows: + if args.only and r["id"] != args.only: + continue + if args.zero_edge_only and r["out_edges"] != 0: + continue + stem = Path(r["source_path"]).stem if r["source_path"] else None + pdf = pdfs.get(stem) if stem else None + if pdf is None: + logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem) + continue + targets.append((r["id"], pdf)) + if args.limit is not None: + targets = targets[: args.limit] + + logger.info( + "GROBID=%s | papers to process=%d | mode=%s", + grobid_url, + len(targets), + "WRITE" if args.write else "DRY-RUN", + ) + + # GROBID extraction — no DB connection held during this slow network loop. + all_citations: list[Citation] = [] + for paper_id, pdf in targets: + try: + cits = build_grobid_citations( + paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout + ) + except Exception as exc: + logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc) + continue + logger.info("%-38s %3d refs (%s)", paper_id, len(cits), pdf.name) + all_citations.extend(cits) + + logger.info("total candidate citations: %d", len(all_citations)) + + if not args.write: + logger.info("DRY-RUN — no rows written. Re-run with --write to apply.") + return 0 + + # Fresh connection for the write — the read connection was already released. + with get_conn() as conn: + before = _coverage(conn) + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO citations (citing_id, cited_id, context) + VALUES (%s, %s, %s) + ON CONFLICT (citing_id, cited_id) DO NOTHING + """, + [(c.citing_id, c.cited_id, c.context) for c in all_citations], + ) + conn.commit() + after = _coverage(conn) + logger.info( + "coverage papers=%d citing %d->%d edges %d->%d (+%d)", + after[0], + before[1], + after[1], + before[2], + after[2], + after[2] - before[2], + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rc_tex_reingest.py b/scripts/rc_tex_reingest.py new file mode 100644 index 0000000..7ccdf1e --- /dev/null +++ b/scripts/rc_tex_reingest.py @@ -0,0 +1,131 @@ +"""R-C — re-ingest arXiv papers from LaTeX source (section-aware, higher fidelity). + +For each arXiv paper, download its source via :func:`codex.sources.arxiv.fetch_source` +(multi-file ``\\input``/``\\include`` flattened), write it to a temp ``.tex``, and +re-ingest through :func:`codex.ingest.ingest_paper`. The ``.tex`` ingest path is +section-aware (:func:`codex.parsing.tex.chunk_sections`), so the stored ``section`` +column is labelled from real ``\\section`` headings instead of mostly ``body`` — +serving DQ-3 (fidelity vs the OCR'd ``.txt``) and audit R-12 (weak ``section`` column). + +Only arXiv papers have downloadable source; DOI/journal papers and books are skipped. +Re-ingest REPLACES the paper's chunks (DELETE + re-INSERT) — that is the intended R-C +improvement (flattened ``.txt`` → clean ``.tex`` chunks). It is idempotent and +**dry-run by default**; pass ``--write`` to mutate the live corpus. + +Usage +----- + # dry-run: list arXiv papers + source availability (no writes) + PYTHONPATH=. python scripts/rc_tex_reingest.py + + # re-ingest every arXiv paper from .tex + PYTHONPATH=. python scripts/rc_tex_reingest.py --write + + # one paper + PYTHONPATH=. python scripts/rc_tex_reingest.py --only 2305.10988 --write + +Needs ``DATABASE_URL`` (Jetson tunnel) as documented in +``docs/audit/DATA-QUALITY-2026-06-15.md``. +""" + +from __future__ import annotations + +import argparse +import logging +import tempfile + +import psycopg +import psycopg.rows + +from codex.db import get_conn +from codex.ingest import ingest_paper +from codex.sources import arxiv + +logger = logging.getLogger(__name__) + + +def is_arxiv_id(paper_id: str) -> bool: + """Return True for arXiv ids (the only papers with downloadable LaTeX source). + + arXiv ids are modern (``2305.10988``) or legacy (``math/0603097``). DOIs start + with ``10.``; OpenAlex W-ids start with ``W`` or an ``https://openalex.org/`` URL. + """ + p = (paper_id or "").strip() + # Exclude DOIs (incl. the arXiv DOI 10.48550/arXiv.*), OpenAlex W-ids, and any + # http(s) URL form; everything else is treated as a bare arXiv id. + return ( + bool(p) + and not p.startswith(("10.", "W")) + and not p.lower().startswith(("http://", "https://")) + ) + + +def _section_dist(conn: psycopg.Connection[psycopg.rows.DictRow], paper_id: str) -> dict[str, int]: + """Return the section-label distribution for a paper's stored chunks.""" + rows = conn.execute( + "SELECT coalesce(section, '(null)') AS s, count(*) AS n " + "FROM chunks WHERE paper_id = %s GROUP BY 1 ORDER BY 2 DESC", + (paper_id,), + ).fetchall() + return {r["s"]: int(r["n"]) for r in rows} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--only", default=None, help="restrict to a single arXiv id") + parser.add_argument("--limit", type=int, default=None, help="cap number of papers") + parser.add_argument( + "--write", action="store_true", help="re-ingest into the live DB (default: dry-run)" + ) + args = parser.parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + + with get_conn() as conn: + rows = conn.execute("SELECT id FROM papers ORDER BY id").fetchall() + targets = [ + r["id"] + for r in rows + if is_arxiv_id(r["id"]) and (args.only is None or r["id"] == args.only) + ] + if args.limit is not None: + targets = targets[: args.limit] + + logger.info( + "arXiv papers to process=%d | mode=%s", + len(targets), + "WRITE" if args.write else "DRY-RUN", + ) + + for pid in targets: + try: + src = arxiv.fetch_source(pid) + except Exception as exc: + logger.warning("%s: source fetch failed: %s", pid, exc) + continue + if not src: + logger.warning("%s: no LaTeX source available — skipping", pid) + continue + if not args.write: + logger.info("%s: source OK (%d chars)", pid, len(src)) + continue + + with get_conn() as conn: + before = _section_dist(conn, pid) + with tempfile.NamedTemporaryFile("w", suffix=".tex", delete=True) as tf: + tf.write(src) + tf.flush() + res = ingest_paper(pid, source_path=tf.name) + with get_conn() as conn: + after = _section_dist(conn, pid) + logger.info( + "%s: chunks %d->%d | section before=%s after=%s", + pid, + sum(before.values()), + res.chunks_upserted, + before, + after, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 4d15713..9561191 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -106,6 +106,13 @@ class TestSearch: assert "Test Paper" in result.stdout assert "0.123" in result.stdout + # Regression guard (DQ-4): the pgvector `<->` operator requires the + # query embedding cast to ::vector. Without it, real Postgres raises + # "operator does not exist: vector <-> double precision[]". Mocked DBs + # don't catch the type error, so assert the cast is in the SQL. + executed_sql = mock_conn.execute.call_args_list[0][0][0] + assert "<->" in executed_sql and "::vector" in executed_sql + def test_search_formula_command(self) -> None: """Test `search formula` command with mocked DB.""" with patch("codex.db.get_conn") as mock_get_conn: diff --git a/tests/graph/test_cli.py b/tests/graph/test_cli.py index e99535b..dbf5c83 100644 --- a/tests/graph/test_cli.py +++ b/tests/graph/test_cli.py @@ -119,6 +119,36 @@ class TestGraphReport: assert result.exit_code == 0 assert "Warning" in result.output or "warning" in result.output + def test_shows_citing_coverage_line(self): + # _sample_graph: A, B, C all cite → 3/3 = 100%, so the line shows but no warning. + out = self._run(known_ids=("A", "B", "C")).output + assert "Citing coverage" in out + assert "weakens" not in out # 100% coverage → no R-D warning + + def test_low_citing_coverage_warning(self): + # Only A has an out-edge; B and C are ingested but cite nothing → 1/3 = 33%. + g = nx.DiGraph() + g.add_edge("A", "X") + g.add_nodes_from(["B", "C"]) + result = self._run(graph=g, known_ids=("A", "B", "C")) + assert result.exit_code == 0 + assert "weakens PageRank" in result.output # the R-D coverage warning + + def test_json_includes_citing_coverage(self): + import json + + g = nx.DiGraph() + g.add_edge("A", "X") + g.add_nodes_from(["B", "C"]) + conn = _make_conn_with_paper_ids("A", "B", "C") + with ( + patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)), + patch("codex.graph.build_citation_graph", return_value=g), + ): + result = runner.invoke(app, ["graph", "report", "--json"]) + data = json.loads(result.output) + assert data["citing_coverage"] == {"citing": 1, "papers": 3, "fraction": 0.3333} + # --------------------------------------------------------------------------- # codex graph related diff --git a/tests/graph/test_graph.py b/tests/graph/test_graph.py index c68bdfa..49c9abe 100644 --- a/tests/graph/test_graph.py +++ b/tests/graph/test_graph.py @@ -10,6 +10,7 @@ import pytest from codex.graph import ( build_citation_graph, citation_pagerank, + citing_coverage, dangling_citations, find_co_cited, find_related, @@ -287,3 +288,24 @@ class TestDanglingCitations: g = build_citation_graph(_make_conn(rows)) dangling = dangling_citations(g, set()) assert set(dangling) == {"A", "B"} + + +# --------------------------------------------------------------------------- +# citing_coverage (R-D) +# --------------------------------------------------------------------------- + + +class TestCitingCoverage: + def test_counts_papers_with_out_edges(self): + # _small_graph: A→.., B→.., C→D have out-edges; D, E have none. + g = _small_graph() + n_citing, n_papers = citing_coverage(g, {"A", "B", "C", "D", "E"}) + assert (n_citing, n_papers) == (3, 5) + + def test_paper_absent_from_graph_counts_as_no_out_edges(self): + # A cites; Z was ingested but has no chunks/edges, so it isn't a graph node. + g = _small_graph() + assert citing_coverage(g, {"A", "Z"}) == (1, 2) + + def test_empty(self): + assert citing_coverage(nx.DiGraph(), set()) == (0, 0) diff --git a/tests/ingest/test_ingest.py b/tests/ingest/test_ingest.py index 95f4423..4779f7e 100644 --- a/tests/ingest/test_ingest.py +++ b/tests/ingest/test_ingest.py @@ -10,6 +10,7 @@ from contextlib import contextmanager from typing import Any from unittest.mock import MagicMock, patch +import httpx import numpy as np import psycopg import pytest @@ -107,6 +108,60 @@ def test_ingest_paper_basic() -> None: ) +def test_ingest_citation_fallback_to_crossref() -> None: + """R-B: OpenAlex + S2 both empty → Crossref references supply the edges. + + Crossref is the third leg of the fallback chain. Its cited DOIs are + case-normalised and citing_id is rewritten to the canonical paper.id. + """ + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123") + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + # Upper-case suffix proves the cited DOI is normalised to bare lower-case. + crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty + patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(paper.id) + + # Crossref consulted with the bare DOI; one edge upserted. + mock_cr.assert_called_once_with(paper.id) + assert result.citations_upserted == 1 + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + cit_rows = mock_cursor.executemany.call_args_list[0][0][1] + assert cit_rows[0][0] == paper.id + assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7" + + +def test_ingest_crossref_not_called_when_openalex_has_citations() -> None: + """R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback).""" + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123") + mock_conn = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch( + "codex.ingest.openalex.fetch_citations", + return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")], + ), + patch("codex.ingest.crossref.fetch_references") as mock_cr, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + ingest_paper(paper.id) + + mock_cr.assert_not_called() + + def test_ingest_canonicalises_paper_id_to_caller_arg() -> None: """papers.id is the caller-supplied id, not OpenAlex's URL-form id (audit C-7). @@ -144,7 +199,8 @@ def test_ingest_canonicalises_paper_id_to_caller_arg() -> None: def test_ingest_paper_source_tex(tmp_path: Any) -> None: - """``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB.""" + """``.tex`` source → section-aware chunking; the stored ``section`` column is + labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12).""" tex_file = tmp_path / "paper.tex" tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.") @@ -154,16 +210,20 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: mock_conn.executemany = MagicMock() mock_conn.commit = MagicMock() - fake_chunks = ["chunk one text here.", "chunk two text here."] + # (section title, chunk content) pairs from the section-aware chunker. + section_pairs = [ + ("Introduction", "Body of the introduction goes here."), + ("Proof of the Main Theorem", "Body of the proof goes here."), + ("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."), + ] with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), - patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."), - patch("codex.parsing.tex.chunk_text", return_value=fake_chunks), - patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + patch("codex.parsing.tex.chunk_sections", return_value=section_pairs), + patch("codex.ingest.is_quality_chunk", return_value=True), ): result = ingest_paper(paper.id, source_path=str(tex_file)) @@ -178,7 +238,16 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None: chunk_sql: str = chunk_insert_call[0][0] assert "INSERT INTO chunks" in chunk_sql - assert result.chunks_upserted == len(fake_chunks) + # R-F: a bare canonical heading → bucket ("Introduction"→intro); any other + # heading is stored as its cleaned real title ("proof of the main theorem", + # "discrete conformal maps") instead of being collapsed/mislabelled. + chunk_rows = chunk_insert_call[0][1] + assert [row[4] for row in chunk_rows] == [ + "intro", + "proof of the main theorem", + "discrete conformal maps", + ] + assert result.chunks_upserted == len(section_pairs) # --------------------------------------------------------------------------- @@ -206,12 +275,15 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1); + # mock it empty so only the GROBID refs contribute to the count. + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), patch("codex.parsing.tex.chunk_text", return_value=fake_chunks), patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), - patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks), + patch("codex.ingest.is_quality_chunk", return_value=True), ): result = ingest_paper(paper.id, source_path=str(pdf_file)) @@ -227,6 +299,73 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None: assert mock_cursor.executemany.call_count >= 2 +def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None: + """R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so + it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each + must be normalized to the bare, lower-cased canonical form before the Citation + is built — otherwise the same reference splits into case-/URL-variant graph + nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no + DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1).""" + pdf_file = tmp_path / "paper.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake content") + + paper = _make_paper() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]: + # Faithful to GROBID's output shape: all five keys always present. + return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id} + + grobid_refs = [ + _ref(doi="10.1007/S00454-019-00132-8"), # mixed-case bare DOI + _ref(doi="https://doi.org/10.1145/AbC.123"), # URL-form DOI + _ref(doi="doi:10.1/MixedCase"), # doi:-prefixed DOI + _ref(arxiv_id="2101.00001"), # arXiv-only ref → untouched + ] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID + # refs contribute to the citation rows under inspection. + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."), + patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]), + patch("codex.parsing.grobid.extract_references", return_value=grobid_refs), + patch("codex.ingest.is_quality_chunk", return_value=True), + ): + result = ingest_paper(paper.id, source_path=str(pdf_file)) + + assert result.citations_upserted == 4 + + # Locate the citations INSERT among the cursor's executemany calls (chunks + # are also inserted via executemany, so filter by SQL rather than by index). + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + citation_calls = [ + c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0] + ] + assert len(citation_calls) == 1 + inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1] + + # Every edge hangs off the canonical paper.id (citing side). + assert all(row[0] == paper.id for row in inserted_rows) + + cited_ids = {row[1] for row in inserted_rows} + assert cited_ids == { + "10.1007/s00454-019-00132-8", # mixed-case → lower-cased + "10.1145/abc.123", # URL prefix stripped + lower-cased + "10.1/mixedcase", # doi: prefix stripped + lower-cased + "2101.00001", # arXiv id untouched + } + # No raw URL-form / prefixed DOI leaked into the citation graph. + assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids) + + # --------------------------------------------------------------------------- # 4. test_ingest_paper_not_found # --------------------------------------------------------------------------- @@ -279,7 +418,7 @@ def test_ingest_paper_idempotent() -> None: def test_ingest_paper_arxiv_s2_fallback() -> None: - """If OpenAlex returns None for an arXiv ID, S2 fallback produces a stub Paper.""" + """OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted.""" paper_id = "2301.07041" mock_conn = MagicMock() mock_conn.execute = MagicMock() @@ -288,17 +427,121 @@ def test_ingest_paper_arxiv_s2_fallback() -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=None), + # arXiv + abstract sources also empty → genuine stub fallback path. + patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv, + patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None), + patch("codex.ingest.crossref.fetch_abstract", return_value=None), patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2, patch("codex.ingest.get_embedder", return_value=_fake_embedder()), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), ): result = ingest_paper(paper_id) - # S2 was consulted as fallback + # arXiv metadata recovery attempted; S2 references consulted as fallback. + mock_arxiv.assert_called_once_with(paper_id) mock_s2.assert_called() assert result.paper_id == paper_id +def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None: + """DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API, + bibkey auto-generated, abstract embedded (no zero vector).""" + recovered = Paper( + id="1911.00966", + title="A discrete version of Liouville's theorem on conformal maps", + authors=["Ulrich Pinkall", "Boris Springborn"], + year=2019, + abstract="Liouville's theorem says that in dimension greater than two...", + ) + fake_emb = _fake_embedder() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=None), + patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv, + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=fake_emb), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper("1911.00966") + + mock_arxiv.assert_called_once_with("1911.00966") + assert result.paper_id == "1911.00966" + params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert params["title"].startswith("A discrete version") + assert params["bibkey"] == "PinkallSpringborn2019" # generated from recovered authors+year + # abstract present → real embedding computed, not a zero vector + fake_emb.encode_dense.assert_called_once() + + +def test_ingest_paper_empty_abstract_recovered_from_s2() -> None: + """DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding.""" + paper = _make_paper(abstract=None) # has openalex_id, no abstract + fake_emb = _fake_embedder() + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch( + "codex.ingest.semanticscholar.fetch_abstract", + return_value="Recovered abstract text.", + ) as mock_abs, + patch("codex.ingest.get_embedder", return_value=fake_emb), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + ingest_paper(paper.id) + + mock_abs.assert_called_once() + params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert params["abstract"] == "Recovered abstract text." + fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."]) + + +def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None: + """DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2, + rewriting citing_id to paper.id and lowercasing DOI cited-ids.""" + paper = _make_paper() # openalex_id="W123", id="2301.07041" + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + # S2 keys citing_id by the id form passed in; ingest must canonicalise it. + s2_refs = [ + Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"), + Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"), + ] + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2, + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(paper.id) + + # S2 consulted with a namespaced id (a bare arXiv id 404s on S2). + mock_s2.assert_called_once_with("arXiv:2301.07041") + assert result.citations_upserted == 2 + + mock_cursor = mock_conn.cursor.return_value.__enter__.return_value + inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1] + # citing_id rewritten to paper.id, not the S2 id form. + assert all(row[0] == paper.id for row in inserted_rows) + cited = {row[1] for row in inserted_rows} + assert "10.1/upper" in cited # DOI lowercased + assert "2202.00002" in cited # arXiv id untouched + + # --------------------------------------------------------------------------- # 7. test_ingest_paper_no_abstract_uses_zero_vector # --------------------------------------------------------------------------- @@ -317,6 +560,9 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None: with ( patch("codex.ingest.openalex.fetch_paper", return_value=paper), patch("codex.ingest.openalex.fetch_citations", return_value=[]), + # No abstract recoverable from any source (DQ-2) → genuine zero-vector case. + patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None), + patch("codex.ingest.crossref.fetch_abstract", return_value=None), patch("codex.ingest.get_embedder", return_value=fake_emb), patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), ): @@ -526,6 +772,93 @@ def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None: assert upsert_params["bibkey"] == "AliceBob2023" +def test_ingest_doi_paper_upserts_bare_canonical_id() -> None: + """DQ-5 regression: re-ingesting an existing DOI paper must be idempotent. + + OpenAlex returns the ``doi`` as a full URL (and possibly mixed-case), but the + stored canonical ``papers.id`` is the BARE, lower-cased DOI (M-1 migration). + The papers upsert must therefore carry the BARE id into ``%(id)s`` so + ``ON CONFLICT (id)`` matches the existing row and UPDATEs it — rather than + attempting a fresh INSERT that trips the separate ``papers_openalex_id_key`` + unique constraint (the UniqueViolation crash). + + The real ``_map_paper``/``fetch_paper`` run here (only the HTTP ``_get``, the + embedder and the DB are mocked), so this exercises the actual normalization + through the ingest path. The DB is a MagicMock, so it cannot raise the real + constraint error; the live re-ingest of 10.1007/s00454-019-00132-8 is the + end-to-end UniqueViolation proof (see DATA-QUALITY-2026-06-15.md, DQ-5). + """ + bare = "10.1007/s00454-019-00132-8" + work = { + "id": "https://openalex.org/W2899262408", + # URL form + an upper-case suffix char — the worst case for ON CONFLICT. + "doi": "https://doi.org/10.1007/S00454-019-00132-8", + "title": "A Test Paper", + "publication_year": 2019, + "authorships": [{"author": {"display_name": "Alice Smith"}}], + "abstract_inverted_index": {"Hello": [0], "world": [1]}, + # Non-empty so fetch_citations yields edges and the S2 fallback is skipped. + "referenced_works": ["https://openalex.org/W1"], + } + + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch( + "codex.ingest.openalex._get", + return_value=httpx.Response(200, json=work), + ), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + result = ingest_paper(bare) + + # The IngestResult and the upsert must use the bare canonical id, not the URL. + assert result.paper_id == bare + + upsert_sql: str = mock_conn.execute.call_args_list[0][0][0] + upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert "INSERT INTO papers" in upsert_sql + assert "ON CONFLICT (id)" in upsert_sql + assert upsert_params["id"] == bare, ( + f"upsert id must be the bare canonical DOI '{bare}', " + f"not the OpenAlex URL form '{upsert_params['id']}'" + ) + + +def test_ingest_normalizes_non_bare_caller_id() -> None: + """Review #1: a non-bare CALLER id (URL-form / mixed-case DOI) is normalized to the + bare canonical form before being pinned to papers.id (the C-7 pin runs it through + _norm_cited_id), so a sloppy caller cannot store a URL-form papers.id that would + defeat ON CONFLICT (id) idempotency or the ``paper.id.startswith('10.')`` gates.""" + paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W1") + mock_conn = MagicMock() + mock_conn.execute = MagicMock() + mock_conn.executemany = MagicMock() + mock_conn.commit = MagicMock() + + with ( + patch("codex.ingest.openalex.fetch_paper", return_value=paper), + patch("codex.ingest.openalex.fetch_citations", return_value=[]), + patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), + patch("codex.ingest.crossref.fetch_references", return_value=[]), + patch("codex.ingest.get_embedder", return_value=_fake_embedder()), + patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)), + ): + # Caller passes the URL form with an upper-case suffix char (worst case). + result = ingest_paper("https://doi.org/10.1007/S00454-019-00132-8") + + bare = "10.1007/s00454-019-00132-8" + assert result.paper_id == bare + upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1] + assert upsert_params["id"] == bare, ( + f"non-bare caller must be normalized to '{bare}', got '{upsert_params['id']}'" + ) + + def test_ingest_disambiguates_bibkey_on_unique_violation() -> None: """A papers.bibkey UNIQUE collision is retried with an a/b/c suffix (audit C-15).""" paper = _make_paper() # bibkey auto-generates to "AliceBob2023" diff --git a/tests/parsing/test_tex.py b/tests/parsing/test_tex.py index 002cc23..7e97a5f 100644 --- a/tests/parsing/test_tex.py +++ b/tests/parsing/test_tex.py @@ -4,7 +4,13 @@ from __future__ import annotations import pytest -from codex.parsing.tex import chunk_text, extract_sections, latex_to_text +from codex.parsing.tex import ( + chunk_sections, + chunk_text, + extract_sections, + flatten_inputs, + latex_to_text, +) class TestExtractSections: @@ -110,3 +116,62 @@ class TestLatexToText: result = latex_to_text(latex) assert r"\cite" not in result assert "%" not in result + + +class TestFlattenInputs: + def test_inlines_input_and_include(self) -> None: + files = { + "main.tex": r"Start \input{sec/intro} mid \include{proof} end", + "sec/intro.tex": "INTRO BODY", + "proof.tex": "PROOF BODY", + } + out = flatten_inputs(files["main.tex"], files) + assert "INTRO BODY" in out + assert "PROOF BODY" in out + assert r"\input" not in out and r"\include" not in out + + def test_resolves_without_tex_suffix(self) -> None: + # \input{intro} must resolve the archive member "intro.tex". + files = {"main.tex": r"\input{intro}", "intro.tex": "BODY"} + assert "BODY" in flatten_inputs(files["main.tex"], files) + + def test_recursive_includes(self) -> None: + files = {"a.tex": r"X \input{b}", "b.tex": r"Y \input{c}", "c.tex": "Z"} + out = flatten_inputs(files["a.tex"], files) + assert "X" in out and "Y" in out and "Z" in out + + def test_unresolved_input_dropped(self) -> None: + # A reference with no matching file is removed, not left as a directive. + out = flatten_inputs(r"A \input{missing} B", {}) + assert r"\input" not in out + assert "A" in out and "B" in out + + def test_input_cycle_does_not_leak_directive(self) -> None: + # a -> b -> a: the depth cap must strip the unresolved directive, not leak it. + files = {"a.tex": r"AA \input{b}", "b.tex": r"BB \input{a}"} + out = flatten_inputs(files["a.tex"], files) + assert r"\input" not in out + assert "AA" in out and "BB" in out + + def test_dot_slash_prefix_resolves_without_overstrip(self) -> None: + # "./intro" resolves intro.tex; lstrip must not eat a real leading dot/run. + files = {"main.tex": r"\input{./intro}", "intro.tex": "BODY"} + assert "BODY" in flatten_inputs(files["main.tex"], files) + + +class TestChunkSections: + def test_pairs_carry_section_titles(self) -> None: + latex = ( + r"\section{Introduction}" + "\nIntro body text here.\n" + r"\section{Proof}" + "\nProof body text here.\n" + ) + pairs = chunk_sections(latex) + titles = [t for t, _ in pairs] + assert "Introduction" in titles + assert "Proof" in titles + # No chunk spans a section boundary: each chunk belongs to one title. + assert all(isinstance(c, str) and c for _, c in pairs) + + def test_fallback_titleless_when_no_sections(self) -> None: + pairs = chunk_sections("Plain text with no section commands at all here.") + assert pairs and all(title is None for title, _ in pairs) diff --git a/tests/quality/test_quality.py b/tests/quality/test_quality.py index b131d2f..240d802 100644 --- a/tests/quality/test_quality.py +++ b/tests/quality/test_quality.py @@ -6,10 +6,12 @@ from unittest.mock import MagicMock from codex.quality import ( _bib_score, + _clean_title, classify_section, filter_chunks, is_quality_chunk, run_quality_pass, + section_label, ) # --------------------------------------------------------------------------- @@ -243,3 +245,58 @@ class TestRunQualityPass: kwargs = update_calls[0][0][1] assert kwargs["section"] == "abstract" assert kwargs["id"] == 5 + + +class TestCleanTitle: + def test_strips_leading_numbering(self): + assert _clean_title("3.2 Variational Principles") == "variational principles" + + def test_strips_latex_and_braces(self): + assert _clean_title(r"The \emph{Discrete} Laplacian") == "the discrete laplacian" + + def test_collapses_and_lowercases(self): + assert _clean_title(" Rigidity Results ") == "rigidity results" + + def test_strips_accent_macros(self): + assert _clean_title(r"M\"obius Invariance") == "mobius invariance" + + def test_strips_ties_and_accents(self): + assert _clean_title(r"Colin~de~Verdi\`ere") == "colin de verdiere" + + def test_truncates_at_word_boundary(self): + long = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu omega" + out = _clean_title(long) + assert len(out) <= 60 + assert not out.endswith(" ") + assert long.lower().startswith(out) # whole words from the start, no partial tail + + +class TestSectionLabel: + def test_bare_canonical_heading_maps_to_bucket(self): + # Only an EXACT canonical heading collapses to its controlled bucket. + assert section_label("Introduction", "body text") == "intro" + assert section_label("Proof", "body text") == "proof" + assert section_label("References", "body text") == "bibliography" + assert section_label("Abstract", "body text") == "abstract" + + def test_descriptive_title_stored_verbatim(self): + # R-F: a heading that merely STARTS with a keyword keeps its real title, + # not the bucket — fixes classify_section prefix-match over-bucketing. + assert section_label("Preliminaries", "body text") == "preliminaries" + assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps" + assert section_label("Proof of Theorem 3", "x") == "proof of theorem 3" + assert section_label("Abstract Nonsense and Categories", "x") == ( + "abstract nonsense and categories" + ) + assert section_label("Introduction to Operator Algebras", "x") == ( + "introduction to operator algebras" + ) + + def test_none_or_blank_title_falls_back_to_content(self): + # .pdf/.txt path: unchanged content-based classification. + assert section_label(None, "Theorem 1. Let X be ...") == "theorem" + assert section_label("", "ordinary body prose here") == "body" + + def test_title_cleaning_to_empty_falls_back_to_content(self): + # A pure-numbering heading cleans to "" → classify by content instead. + assert section_label("3.2", "Lemma 2. Suppose ...") == "theorem" diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scripts/test_ra_grobid_backfill.py b/tests/scripts/test_ra_grobid_backfill.py new file mode 100644 index 0000000..2ef8a6a --- /dev/null +++ b/tests/scripts/test_ra_grobid_backfill.py @@ -0,0 +1,112 @@ +"""Tests for scripts.ra_grobid_backfill — the R-A GROBID references backfill. + +Only the pure, offline logic is covered (id normalization + Citation assembly); +``extract_references`` is mocked so no GROBID server or DB is required. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +from codex.models import Citation +from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations, main + +# --------------------------------------------------------------------------- +# _normalize_cited_id +# --------------------------------------------------------------------------- + + +def test_normalize_doi_url_to_bare_lowercase() -> None: + assert _normalize_cited_id(doi="https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_prefixes() -> None: + assert _normalize_cited_id(doi="http://doi.org/10.1/X") == "10.1/x" + assert _normalize_cited_id(doi="doi:10.1/X") == "10.1/x" + assert _normalize_cited_id(doi="10.1/x") == "10.1/x" + + +def test_normalize_arxiv_prefix_stripped() -> None: + assert _normalize_cited_id(arxiv_id="arXiv:2305.10988") == "2305.10988" + assert _normalize_cited_id(arxiv_id="2305.10988") == "2305.10988" + # Legacy ids keep their slash. + assert _normalize_cited_id(arxiv_id="math/0603097") == "math/0603097" + + +def test_normalize_doi_takes_precedence_over_arxiv() -> None: + assert _normalize_cited_id(doi="10.5/y", arxiv_id="2305.10988") == "10.5/y" + + +def test_normalize_empty_returns_none() -> None: + assert _normalize_cited_id() is None + assert _normalize_cited_id(doi=" ", arxiv_id="") is None + + +# --------------------------------------------------------------------------- +# build_grobid_citations +# --------------------------------------------------------------------------- + + +def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]: + return {"title": "t", "authors": "a", "year": "2020", "doi": doi, "arxiv_id": arxiv_id} + + +def test_build_citations_normalizes_dedups_and_drops_self() -> None: + paper_id = "10.1007/s00454-019-00132-8" + refs = [ + _ref(doi="https://doi.org/10.1/A"), # → 10.1/a + _ref(doi="DOI:10.1/a"), # duplicate of the above after normalization + _ref(arxiv_id="arXiv:2305.10988"), # → 2305.10988 + _ref(doi="", arxiv_id=""), # no id → skipped + _ref(doi="https://doi.org/10.1007/S00454-019-00132-8"), # self-cite → dropped + ] + with patch("scripts.ra_grobid_backfill.extract_references", return_value=refs) as mock_ex: + cits = build_grobid_citations(paper_id, "some.pdf", grobid_url="http://grobid") + + mock_ex.assert_called_once_with("some.pdf", grobid_url="http://grobid", timeout=60.0) + assert all(isinstance(c, Citation) for c in cits) + assert all(c.citing_id == paper_id for c in cits) + assert [c.cited_id for c in cits] == ["10.1/a", "2305.10988"] + + +def test_build_citations_empty_when_no_refs() -> None: + with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]): + assert build_grobid_citations("10.1/x", "x.pdf") == [] + + +# --------------------------------------------------------------------------- +# main — connection scope (review finding #4) +# --------------------------------------------------------------------------- + + +def test_main_uses_separate_connections_for_read_and_write(tmp_path: Any) -> None: + """Review #4: the read connection is released BEFORE the slow GROBID loop, and a + fresh connection is opened for the write — so no idle-in-transaction connection + spans the per-paper network loop (which would abort on an SSH-tunnel drop).""" + (tmp_path / "paper.pdf").write_bytes(b"%PDF-1.4") + conns: list[MagicMock] = [] + + @contextmanager + def fake_get_conn() -> Any: + c = MagicMock() + c.execute.return_value.fetchall.return_value = [ + {"id": "10.1/x", "source_path": "/d/paper.txt", "out_edges": 0} + ] + c.execute.return_value.fetchone.return_value = {"papers": 1, "citing": 1, "edges": 1} + conns.append(c) + yield c + + with ( + patch("scripts.ra_grobid_backfill.get_conn", side_effect=fake_get_conn), + patch("scripts.ra_grobid_backfill.build_grobid_citations", return_value=[]), + ): + rc = main(["--pdf-dir", str(tmp_path), "--write"]) + + assert rc == 0 + # Two DISTINCT get_conn() context managers (read, then write) — not one held + # open across the GROBID loop. + assert len(conns) == 2 diff --git a/tests/scripts/test_rc_tex_reingest.py b/tests/scripts/test_rc_tex_reingest.py new file mode 100644 index 0000000..3f7239f --- /dev/null +++ b/tests/scripts/test_rc_tex_reingest.py @@ -0,0 +1,21 @@ +"""Tests for scripts.rc_tex_reingest.""" + +from __future__ import annotations + +from scripts.rc_tex_reingest import is_arxiv_id + + +def test_is_arxiv_id_accepts_modern_and_legacy() -> None: + assert is_arxiv_id("2305.10988") + assert is_arxiv_id("math/0603097") + assert is_arxiv_id("1005.2698") + + +def test_is_arxiv_id_rejects_dois_and_wids() -> None: + assert not is_arxiv_id("10.1007/s00454-019-00132-8") + assert not is_arxiv_id("10.14279/depositonce-20357") + assert not is_arxiv_id("W2971636899") + assert not is_arxiv_id("https://openalex.org/W2971636899") + assert not is_arxiv_id("http://openalex.org/W2971636899") + assert not is_arxiv_id("10.48550/arXiv.2305.10988") # arXiv DOI form, not bare id + assert not is_arxiv_id("") diff --git a/tests/sources/test_arxiv.py b/tests/sources/test_arxiv.py index 1c08500..25e701b 100644 --- a/tests/sources/test_arxiv.py +++ b/tests/sources/test_arxiv.py @@ -53,6 +53,94 @@ def test_fetch_source_returns_latex(monkeypatch: pytest.MonkeyPatch) -> None: assert "Hello world" in result +def test_fetch_source_flattens_multifile(monkeypatch: pytest.MonkeyPatch) -> None: + """Multi-file projects: \\input/\\include directives are inlined so the body + from the included files is returned, not just the primary skeleton (R-C).""" + main = ( + b"\\documentclass{article}\\begin{document}" + b"\\input{sections/intro}\\include{proof}\\end{document}" + ) + tar_bytes = _make_tar_gz( + { + "main.tex": main, + "sections/intro.tex": b"INTRODUCTION BODY TEXT", + "proof.tex": b"PROOF BODY TEXT", + } + ) + + def mock_get( + url: str, + *, + timeout: int = 60, + follow_redirects: bool = True, + ) -> httpx.Response: + return httpx.Response(200, content=tar_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + + result = arxiv.fetch_source("2301.07041") + + assert result is not None + assert "INTRODUCTION BODY TEXT" in result # \input was inlined + assert "PROOF BODY TEXT" in result # \include was inlined + assert "\\input" not in result and "\\include" not in result + + +def test_fetch_source_bare_gzip_single_file(monkeypatch: pytest.MonkeyPatch) -> None: + """Legacy arXiv submissions are served as a single bare gzipped .tex (no tar + wrapper); fetch_source must gunzip the body directly rather than fail.""" + import gzip + + latex = b"\\documentstyle[a4]{article}\n\\section{Intro}\nOld single-file paper body." + gz_bytes = gzip.compress(latex) + + def mock_get( + url: str, + *, + timeout: int = 60, + follow_redirects: bool = True, + ) -> httpx.Response: + return httpx.Response(200, content=gz_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + + result = arxiv.fetch_source("math/0001176") + + assert result is not None + assert "Old single-file paper body." in result + + +def test_fetch_source_resolves_non_tex_include(monkeypatch: pytest.MonkeyPatch) -> None: + """\\input of a non-.tex body member (e.g. a .def file) is inlined, not dropped (R-C).""" + main = b"\\documentclass{article}\\begin{document}\\input{body.def}\\end{document}" + tar_bytes = _make_tar_gz({"main.tex": main, "body.def": b"NON TEX BODY TEXT"}) + + def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response: + return httpx.Response(200, content=tar_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + result = arxiv.fetch_source("2301.07041") + assert result is not None + assert "NON TEX BODY TEXT" in result + + +def test_fetch_source_ignores_commented_documentclass(monkeypatch: pytest.MonkeyPatch) -> None: + """A commented %\\documentclass must not select a file as the primary document.""" + decoy = b"% \\documentclass{article}\n\\section{Stub}\nDECOY ONLY" + real = b"\\documentclass{book}\\begin{document}REAL BODY HERE\\end{document}" + # decoy sorts/inserts first; without the fix it would win on the substring match. + tar_bytes = _make_tar_gz({"aaa_decoy.tex": decoy, "real.tex": real}) + + def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response: + return httpx.Response(200, content=tar_bytes) + + monkeypatch.setattr(httpx, "get", mock_get) + result = arxiv.fetch_source("2301.07041") + assert result is not None + assert "REAL BODY HERE" in result + assert "DECOY ONLY" not in result + + # --------------------------------------------------------------------------- # fetch_source — 404 → None # --------------------------------------------------------------------------- @@ -138,3 +226,63 @@ def test_fetch_pdf_url_pure_computation() -> None: url2 = arxiv.fetch_pdf_url("1234.56789") assert url2 == "https://arxiv.org/pdf/1234.56789.pdf" + + +# --------------------------------------------------------------------------- +# fetch_metadata — parse the Atom feed into a Paper (DQ-2) +# --------------------------------------------------------------------------- + +_ATOM_FEED = """ + + + A discrete version of Liouville's theorem on conformal maps + Liouville's theorem says that in dimension greater than two, + all conformal maps are Moebius transformations. + 2019-11-03T00:00:00Z + Ulrich Pinkall + Boris Springborn + +""" + +_ATOM_EMPTY = """ +ArXiv Query""" + + +def test_fetch_metadata_parses_entry(monkeypatch: pytest.MonkeyPatch) -> None: + """A valid Atom feed yields a populated Paper.""" + + def mock_get(url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", mock_get) + + paper = arxiv.fetch_metadata("1911.00966") + assert paper is not None + assert paper.id == "1911.00966" + assert paper.title.startswith("A discrete version of Liouville") + assert paper.authors == ["Ulrich Pinkall", "Boris Springborn"] + assert paper.year == 2019 + assert paper.abstract and "conformal maps" in paper.abstract + assert paper.openalex_id is None + + +def test_fetch_metadata_no_entry_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A feed with no (id not found) returns None.""" + + def mock_get(url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(200, text=_ATOM_EMPTY, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", mock_get) + assert arxiv.fetch_metadata("0000.00000") is None + + +def test_fetch_metadata_strips_arxiv_prefix(monkeypatch: pytest.MonkeyPatch) -> None: + """An ``arXiv:`` prefix is stripped so Paper.id is the bare id.""" + + def mock_get(url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", mock_get) + paper = arxiv.fetch_metadata("arXiv:1911.00966") + assert paper is not None + assert paper.id == "1911.00966" diff --git a/tests/sources/test_crossref.py b/tests/sources/test_crossref.py new file mode 100644 index 0000000..6e5c633 --- /dev/null +++ b/tests/sources/test_crossref.py @@ -0,0 +1,104 @@ +"""Tests for codex.sources.crossref.""" + +from __future__ import annotations + +import httpx +import pytest + +from codex.models import Citation +from codex.sources import crossref + + +def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None: + """A JATS-tagged abstract is returned as plain text with tags removed.""" + body = { + "message": { + "abstract": "We prove a theorem about " + "polyhedra." + } + } + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json=body) + + monkeypatch.setattr(crossref, "_get", mock_get) + + result = crossref.fetch_abstract("10.1007/s00454-019-00132-8") + assert result == "We prove a theorem about polyhedra." + + +def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A work with no abstract field (e.g. a book chapter) returns None.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}}) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None + + +def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A 404 returns None rather than raising.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + raise httpx.HTTPStatusError( + "404", + request=httpx.Request("GET", url), + response=httpx.Response(404, request=httpx.Request("GET", url)), + ) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_abstract("10.0000/nonexistent") is None + + +# --------------------------------------------------------------------------- +# fetch_references (roadmap R-B) +# --------------------------------------------------------------------------- + + +def test_fetch_references_returns_doi_edges(monkeypatch: pytest.MonkeyPatch) -> None: + """Only references carrying a DOI become Citations; citing_id is the queried DOI.""" + body = { + "message": { + "reference": [ + {"key": "ref1", "DOI": "10.1090/s0002-9947-04-03545-7"}, + {"key": "ref2", "unstructured": "A book with no DOI, 1992."}, # skipped + {"key": "ref3", "DOI": "10.1007/bf02392449", "article-title": "X"}, + ] + } + } + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json=body) + + monkeypatch.setattr(crossref, "_get", mock_get) + refs = crossref.fetch_references("10.1007/s00454-019-00132-8") + + assert refs == [ + Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1090/s0002-9947-04-03545-7"), + Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1007/bf02392449"), + ] + + +def test_fetch_references_no_reference_key_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """A work with no deposited reference list returns [].""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"message": {"title": ["No refs deposited"]}}) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_references("10.1007/978-3-642-17413-1_7") == [] + + +def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """A 404 returns [] rather than raising.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + raise httpx.HTTPStatusError( + "404", + request=httpx.Request("GET", url), + response=httpx.Response(404, request=httpx.Request("GET", url)), + ) + + monkeypatch.setattr(crossref, "_get", mock_get) + assert crossref.fetch_references("10.0000/nonexistent") == [] diff --git a/tests/sources/test_openalex.py b/tests/sources/test_openalex.py index 5a78b99..de557ea 100644 --- a/tests/sources/test_openalex.py +++ b/tests/sources/test_openalex.py @@ -7,7 +7,7 @@ import pytest from codex.models import Citation, Paper from codex.sources import openalex -from codex.sources.openalex import _resolve_id +from codex.sources.openalex import _canonical_id, _normalize_doi, _resolve_id # --------------------------------------------------------------------------- # Helpers @@ -15,7 +15,8 @@ from codex.sources.openalex import _resolve_id _SAMPLE_WORK = { "id": "https://openalex.org/W2741809807", - # Real OpenAlex returns the DOI as a full URL, not a bare DOI (audit T-3). + # Real OpenAlex returns the DOI as a full URL, not a bare DOI (audit T-3 / DQ-5 + # — the old bare fixture hid the URL-form behaviour both fixes address). "doi": "https://doi.org/10.1145/3592430", "title": "Conformal Prediction: A Review", "publication_year": 2023, @@ -68,6 +69,87 @@ def test_resolve_prefixed_unchanged() -> None: assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x" +# --------------------------------------------------------------------------- +# _normalize_doi — canonical bare, lower-cased form (DQ-5) +# --------------------------------------------------------------------------- + + +def test_normalize_doi_strips_https_url() -> None: + # The exact form OpenAlex returns in the `doi` field. + assert _normalize_doi("https://doi.org/10.1007/s00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_strips_http_and_doi_prefix() -> None: + assert _normalize_doi("http://doi.org/10.1145/3592430") == "10.1145/3592430" + assert _normalize_doi("doi:10.1145/3592430") == "10.1145/3592430" + + +def test_normalize_doi_lowercases() -> None: + # DOIs are case-insensitive; the canonical id (and _norm_cited_id) is lower-case. + assert _normalize_doi("https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_normalize_doi_already_bare_is_idempotent() -> None: + assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8" + + +# --------------------------------------------------------------------------- +# _canonical_id — arXiv DOI → bare arXiv id (arXiv half of DQ-5) +# --------------------------------------------------------------------------- + + +def test_canonical_id_strips_arxiv_doi_modern() -> None: + # arXiv works carry the arXiv DOI; the canonical id is the bare arXiv id. + assert _canonical_id("https://doi.org/10.48550/arXiv.2305.10988") == "2305.10988" + + +def test_canonical_id_strips_arxiv_doi_legacy() -> None: + assert _canonical_id("https://doi.org/10.48550/arXiv.math/0603097") == "math/0603097" + + +def test_canonical_id_leaves_regular_doi_bare() -> None: + # A normal journal DOI is only normalized (bare + lower-case), not stripped. + assert _canonical_id("https://doi.org/10.1007/S00454-019-00132-8") == ( + "10.1007/s00454-019-00132-8" + ) + + +def test_fetch_paper_arxiv_doi_yields_bare_id(monkeypatch: pytest.MonkeyPatch) -> None: + """An arXiv work maps to the BARE arXiv id (not the arXiv DOI), so re-ingesting + by arXiv id hits ON CONFLICT (id) instead of tripping papers_openalex_id_key.""" + work = {**_SAMPLE_WORK, "doi": "https://doi.org/10.48550/arXiv.2305.10988"} + + def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response: + return httpx.Response(200, json=work) + + monkeypatch.setattr(openalex, "_get", mock_get) + paper = openalex.fetch_paper("2305.10988") + + assert paper is not None + assert paper.id == "2305.10988" + + +def test_map_paper_falls_back_to_openalex_id_without_doi( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A work with no DOI keeps the OpenAlex W-id (URL form) as id, unchanged — + normalization only rewrites the DOI branch.""" + work = {k: v for k, v in _SAMPLE_WORK.items() if k != "doi"} + + def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response: + return httpx.Response(200, json=work) + + monkeypatch.setattr(openalex, "_get", mock_get) + paper = openalex.fetch_paper("W2741809807") + + assert paper is not None + assert paper.id == "https://openalex.org/W2741809807" + + # --------------------------------------------------------------------------- # fetch_paper — success # --------------------------------------------------------------------------- @@ -86,15 +168,17 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None: assert paper is not None assert isinstance(paper, Paper) + # DQ-5: openalex._canonical_id normalizes OpenAlex's URL-form doi to the BARE + # lower-cased DOI at the source layer; ingest additionally pins papers.id to the + # caller's id (audit C-7). Both converge on the bare canonical id — which is what + # makes the ingest ON CONFLICT (id) upsert idempotent. (T-3: the fixture is + # URL-form so this mapping can't silently regress.) + assert paper.id == "10.1145/3592430" assert paper.title == "Conformal Prediction: A Review" assert paper.year == 2023 assert paper.authors == ["Alice Smith", "Bob Jones"] assert paper.abstract == "Conformal prediction is great" assert paper.openalex_id == "https://openalex.org/W2741809807" - # fetch_paper maps id from the (URL-form) doi; ingest later canonicalises - # papers.id to the caller's bare id (audit C-7). Pin the real mapping here so - # the fixture cannot silently misrepresent OpenAlex again (audit T-3). - assert paper.id == "https://doi.org/10.1145/3592430" def test_fetch_paper_arxiv_id_prefixed(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/sources/test_semanticscholar.py b/tests/sources/test_semanticscholar.py index 42066d6..a1fb726 100644 --- a/tests/sources/test_semanticscholar.py +++ b/tests/sources/test_semanticscholar.py @@ -124,3 +124,46 @@ def test_fetch_recommendations_404_returns_empty( result = semanticscholar.fetch_recommendations("nonexistent") assert result == [] + + +# --------------------------------------------------------------------------- +# fetch_references — {"data": null} guard (DQ-1 regression) +# --------------------------------------------------------------------------- + + +def test_fetch_references_null_data_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no + parsed references; this must yield [] rather than raising TypeError.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"data": None, "citingPaperInfo": {}}) + + monkeypatch.setattr(semanticscholar, "_get", mock_get) + assert semanticscholar.fetch_references("DOI:10.14279/depositonce-5415") == [] + + +# --------------------------------------------------------------------------- +# fetch_abstract (DQ-2) +# --------------------------------------------------------------------------- + + +def test_fetch_abstract_returns_text(monkeypatch: pytest.MonkeyPatch) -> None: + """fetch_abstract returns the abstract string when present.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"abstract": "We study ideal hyperbolic polyhedra."}) + + monkeypatch.setattr(semanticscholar, "_get", mock_get) + assert semanticscholar.fetch_abstract("DOI:10.1007/s00454-019-00132-8") == ( + "We study ideal hyperbolic polyhedra." + ) + + +def test_fetch_abstract_null_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """A null/absent abstract yields None.""" + + def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response: + return httpx.Response(200, json={"abstract": None}) + + monkeypatch.setattr(semanticscholar, "_get", mock_get) + assert semanticscholar.fetch_abstract("arXiv:1234.5678") is None