Merge PR #16: data-quality roadmap R-A–R-F + DQ-5 (reconciled, reviewed)
Data-quality roadmap (R-A GROBID backfill, R-B Crossref refs, R-C .tex ingest, R-D citing-coverage warning, R-F section labels) + DQ-5 canonical-id idempotency, reconciled with audit-remediation main (#12-15), code-reviewed with 8 findings fixed. 408 tests pass; ruff + mypy clean.
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -38,3 +38,6 @@ htmlcov/
|
||||
|
||||
# Claude Code — settings.json IS committed (team-wide); only personal overrides ignored
|
||||
.claude/settings.local.json
|
||||
|
||||
# Generated wiki index (regenerated by `codex wiki compile`; concepts.yaml is the source)
|
||||
wiki/index.md
|
||||
|
||||
28
codex/cli.py
28
codex/cli.py
@@ -116,10 +116,10 @@ def search_paper(
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, title, year,
|
||||
abstract_emb <-> %(emb)s AS distance
|
||||
abstract_emb <-> %(emb)s::vector AS distance
|
||||
FROM papers
|
||||
WHERE abstract_emb IS NOT NULL
|
||||
ORDER BY abstract_emb <-> %(emb)s
|
||||
ORDER BY abstract_emb <-> %(emb)s::vector
|
||||
LIMIT %(limit)s
|
||||
""",
|
||||
{"emb": emb, "limit": limit},
|
||||
@@ -596,7 +596,12 @@ def graph_report(
|
||||
"""Show citation graph report: top hub papers, dangling citations, cluster summary."""
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.graph import build_citation_graph, citation_pagerank, dangling_citations
|
||||
from codex.graph import (
|
||||
build_citation_graph,
|
||||
citation_pagerank,
|
||||
citing_coverage,
|
||||
dangling_citations,
|
||||
)
|
||||
|
||||
settings = get_settings()
|
||||
with get_conn() as conn:
|
||||
@@ -613,6 +618,8 @@ def graph_report(
|
||||
pr = citation_pagerank(graph)
|
||||
hubs = sorted(pr.items(), key=lambda x: -x[1])[:top_n]
|
||||
dangling = sorted(dangling_citations(graph, known_ids))
|
||||
n_citing, _ = citing_coverage(graph, known_ids)
|
||||
coverage = n_citing / n_papers if n_papers else 0.0
|
||||
small_corpus = n_papers < settings.graph_min_corpus_size
|
||||
warning = (
|
||||
f"only {n_papers} ingested papers; recommended >= {settings.graph_min_corpus_size} "
|
||||
@@ -627,6 +634,11 @@ def graph_report(
|
||||
{
|
||||
"nodes": graph.number_of_nodes(),
|
||||
"edges": graph.number_of_edges(),
|
||||
"citing_coverage": {
|
||||
"citing": n_citing,
|
||||
"papers": n_papers,
|
||||
"fraction": round(coverage, 4),
|
||||
},
|
||||
"small_corpus_warning": warning, # null unless below threshold (audit U-3)
|
||||
"hubs": [
|
||||
{"paper_id": pid, "title": titles.get(pid), "pagerank": score}
|
||||
@@ -642,7 +654,17 @@ def graph_report(
|
||||
|
||||
if warning:
|
||||
typer.echo(f"Warning: {warning}.", err=True)
|
||||
# R-D: low citing-paper coverage starves PageRank/coupling even at a healthy
|
||||
# paper count — warn on the share of papers that actually have out-edges.
|
||||
if coverage < settings.graph_min_citing_coverage:
|
||||
typer.echo(
|
||||
f"Warning: only {n_citing}/{n_papers} papers ({coverage:.0%}) have out-edges "
|
||||
f"(recommended ≥ {settings.graph_min_citing_coverage:.0%}); "
|
||||
f"low citing coverage weakens PageRank/coupling.",
|
||||
err=True,
|
||||
)
|
||||
typer.echo(f"Graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges")
|
||||
typer.echo(f"Citing coverage: {n_citing}/{n_papers} papers with out-edges ({coverage:.0%})")
|
||||
typer.echo("")
|
||||
typer.echo(f"TOP-{top_n} HUB PAPERS (PageRank)")
|
||||
typer.echo("-" * 48)
|
||||
|
||||
@@ -322,6 +322,18 @@ class Settings(BaseSettings):
|
||||
),
|
||||
)
|
||||
|
||||
graph_min_citing_coverage: float = Field(
|
||||
default=0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description=(
|
||||
"Minimum share of ingested papers that must have ≥1 out-edge (i.e. "
|
||||
"cite something) before `codex graph report` warns. Low citing "
|
||||
"coverage starves PageRank/coupling even when the paper count looks "
|
||||
"healthy — the share of *citing* papers is what matters (R-D)."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
|
||||
@@ -109,6 +109,20 @@ def citation_pagerank(graph: nx.DiGraph, *, damping: float = 0.85) -> dict[str,
|
||||
return cast(dict[str, float], nx.pagerank(graph, alpha=damping))
|
||||
|
||||
|
||||
def citing_coverage(graph: nx.DiGraph, known_ids: set[str]) -> tuple[int, int]:
|
||||
"""Return ``(papers_with_out_edges, total_papers)`` — the citing-paper coverage.
|
||||
|
||||
A paper "covers" the citation graph when it has ≥1 out-edge (it cites at least
|
||||
one work). Low citing coverage starves the F-15 layer (PageRank / coupling /
|
||||
co-citation) even when the total paper count looks healthy, because those
|
||||
metrics are driven by out-edges, not by node count. Surfaced by
|
||||
``codex graph report`` (R-D); the warning threshold is
|
||||
:attr:`codex.config.Settings.graph_min_citing_coverage`.
|
||||
"""
|
||||
n_citing = sum(1 for pid in known_ids if pid in graph and graph.out_degree(pid) > 0)
|
||||
return n_citing, len(known_ids)
|
||||
|
||||
|
||||
def find_related(paper_id: str, graph: nx.DiGraph, *, min_shared: int = 2) -> list[str]:
|
||||
"""Bibliographic coupling: papers sharing ≥ ``min_shared`` references with ``paper_id``.
|
||||
|
||||
|
||||
198
codex/ingest.py
198
codex/ingest.py
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -14,8 +13,8 @@ from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.embed import get_embedder
|
||||
from codex.models import Citation, Paper
|
||||
from codex.quality import classify_section, filter_chunks
|
||||
from codex.sources import openalex, semanticscholar
|
||||
from codex.quality import is_quality_chunk, section_label
|
||||
from codex.sources import arxiv, crossref, openalex, semanticscholar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -48,6 +47,113 @@ def _make_bibkey(paper: Paper) -> str | None:
|
||||
return surnames[0] + "EtAl" + year
|
||||
|
||||
|
||||
def _s2_id_for(pid: str) -> str | None:
|
||||
"""Format a canonical paper id for the Semantic Scholar paper endpoint.
|
||||
|
||||
S2 needs a namespaced id (``arXiv:<id>``, ``DOI:<doi>``) or a bare 40-hex
|
||||
S2 paperId — a bare arXiv id like ``2305.10988`` 404s. DOIs are detected by
|
||||
the ``10.`` prefix; everything else is assumed to be an arXiv id (modern
|
||||
``2305.10988`` or legacy ``math/0603097``).
|
||||
"""
|
||||
p = (pid or "").strip()
|
||||
if not p:
|
||||
return None
|
||||
if p.lower().startswith(("arxiv:", "doi:")):
|
||||
return p
|
||||
if p.startswith("10."):
|
||||
return f"DOI:{p}"
|
||||
return f"arXiv:{p}"
|
||||
|
||||
|
||||
def _norm_cited_id(cited_id: str) -> str:
|
||||
"""Normalize a DOI cited-id to its canonical bare, lower-cased form.
|
||||
|
||||
DOIs are case-insensitive and may arrive bare (``10.…``), as a
|
||||
``https://doi.org/…`` URL, or ``doi:…``-prefixed: GROBID emits whichever
|
||||
form the PDF reference text used, while S2/OpenAlex hand back bare DOIs.
|
||||
Stripping the prefix and lower-casing keeps the citation-graph join from
|
||||
fragmenting the same reference into distinct case-/URL-variant nodes. arXiv
|
||||
ids and S2 paperIds are left untouched. Mirrors ``openalex._normalize_doi``."""
|
||||
s = cited_id.strip()
|
||||
lowered = s.lower()
|
||||
for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
|
||||
if lowered.startswith(prefix):
|
||||
return lowered[len(prefix) :]
|
||||
return lowered if lowered.startswith("10.") else s
|
||||
|
||||
|
||||
def _recover_abstract(paper: Paper) -> str | None:
|
||||
"""Recover a missing abstract from S2, then Crossref (DQ-2 supplement).
|
||||
|
||||
Tried in order of coverage for this corpus: Semantic Scholar (indexes most
|
||||
arXiv + journal abstracts), then Crossref (publisher-deposited, DOI only).
|
||||
Network/parse failures degrade to None rather than aborting the ingest.
|
||||
"""
|
||||
s2_id = _s2_id_for(paper.id)
|
||||
if s2_id is not None:
|
||||
try:
|
||||
abstract = semanticscholar.fetch_abstract(s2_id)
|
||||
if abstract and abstract.strip():
|
||||
return abstract.strip()
|
||||
except Exception:
|
||||
logger.warning("S2 abstract recovery failed for %s", paper.id, exc_info=True)
|
||||
if paper.id.startswith("10."):
|
||||
try:
|
||||
abstract = crossref.fetch_abstract(paper.id)
|
||||
if abstract and abstract.strip():
|
||||
return abstract.strip()
|
||||
except Exception:
|
||||
logger.warning("Crossref abstract recovery failed for %s", paper.id, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _s2_reference_supplement(paper: Paper) -> list[Citation]:
|
||||
"""Fetch a paper's references from Semantic Scholar (DQ-1 supplement).
|
||||
|
||||
Used when OpenAlex returns an empty ``referenced_works`` list — common for
|
||||
arXiv preprints and institutional theses that OpenAlex indexes without a
|
||||
parsed bibliography. ``citing_id`` is rewritten to the canonical ``paper.id``
|
||||
so the FK holds, and cited DOIs are case-normalised. Network/parse failures
|
||||
degrade to an empty list rather than aborting the ingest.
|
||||
"""
|
||||
s2_id = _s2_id_for(paper.id)
|
||||
if s2_id is None:
|
||||
return []
|
||||
try:
|
||||
raw = semanticscholar.fetch_references(s2_id)
|
||||
except Exception:
|
||||
logger.warning("S2 reference supplement failed for %s", paper.id, exc_info=True)
|
||||
return []
|
||||
return [
|
||||
Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
|
||||
for c in raw
|
||||
if c.cited_id
|
||||
]
|
||||
|
||||
|
||||
def _crossref_reference_supplement(paper: Paper) -> list[Citation]:
|
||||
"""Fetch a paper's references from Crossref (roadmap R-B supplement).
|
||||
|
||||
Third leg of the citation fallback chain (OpenAlex-empty → S2 → Crossref).
|
||||
Crossref carries publisher-deposited reference lists keyed on the citing
|
||||
work's DOI, so only DOI papers can be looked up (arXiv-only ids → []).
|
||||
``citing_id`` is rewritten to the canonical ``paper.id`` and cited DOIs are
|
||||
case-normalised. Network/parse failures degrade to [] rather than aborting.
|
||||
"""
|
||||
if not paper.id.startswith("10."):
|
||||
return []
|
||||
try:
|
||||
raw = crossref.fetch_references(paper.id)
|
||||
except Exception:
|
||||
logger.warning("Crossref reference supplement failed for %s", paper.id, exc_info=True)
|
||||
return []
|
||||
return [
|
||||
Citation(citing_id=paper.id, cited_id=_norm_cited_id(c.cited_id), context=c.context)
|
||||
for c in raw
|
||||
if c.cited_id
|
||||
]
|
||||
|
||||
|
||||
def ingest_paper(
|
||||
paper_id: str,
|
||||
source_path: str | None = None,
|
||||
@@ -89,10 +195,11 @@ def ingest_paper(
|
||||
len(paper_id) > 4 and paper_id[4:5] == "." and paper_id[:4].isdigit()
|
||||
)
|
||||
if looks_like_arxiv:
|
||||
arxiv_key = paper_id if pid_lower.startswith("arxiv:") else f"arXiv:{paper_id}"
|
||||
with contextlib.suppress(Exception):
|
||||
semanticscholar.fetch_references(arxiv_key)
|
||||
paper = Paper(id=paper_id, title="")
|
||||
# OpenAlex 404 on an arXiv id → recover authoritative metadata from
|
||||
# the arXiv API (DQ-2); fall back to a minimal stub if arXiv also
|
||||
# has nothing. Citation recovery from S2 happens in the citation
|
||||
# block below (DQ-1 supplement).
|
||||
paper = arxiv.fetch_metadata(paper_id) or Paper(id=paper_id, title="")
|
||||
else:
|
||||
raise ValueError(f"Paper not found: {paper_id}")
|
||||
|
||||
@@ -100,12 +207,23 @@ def ingest_paper(
|
||||
raise ValueError(f"Paper not found: {paper_id}")
|
||||
|
||||
# Canonical identity is the caller-supplied id — what the CLI / ingest scripts
|
||||
# use and what papers.id is contracted to hold (arXiv id or DOI). OpenAlex's
|
||||
# fetch_paper fills paper.id with the URL-form doi/id (e.g.
|
||||
# "https://doi.org/10.48550/arxiv.math/0603097"), which broke bare-id lookups
|
||||
# and cross-citation matching (audit C-7). The OpenAlex work id is retained
|
||||
# separately as openalex_id and in the paper_identifiers alias table.
|
||||
paper.id = paper_id.strip()
|
||||
# use and what papers.id is contracted to hold (arXiv id or DOI). Pin it to the
|
||||
# caller's id (audit C-7), but normalize it the same way cited-ids are (strip a
|
||||
# https://doi.org/ or doi: prefix, lower-case DOIs) so a non-bare caller cannot
|
||||
# store a URL-form papers.id that would defeat DQ-5's ON CONFLICT (id) idempotency
|
||||
# or the `paper.id.startswith("10.")` recovery gates below. arXiv/W ids pass
|
||||
# through unchanged. The OpenAlex work id is retained as openalex_id / in
|
||||
# paper_identifiers.
|
||||
paper.id = _norm_cited_id(paper_id.strip())
|
||||
|
||||
# DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute
|
||||
# some publishers' abstracts). Supplement from S2, then Crossref (uses the
|
||||
# canonical paper.id set above), so the paper gets a real (non-zero) abstract
|
||||
# embedding instead of a zero vector.
|
||||
if not (paper.abstract or "").strip():
|
||||
recovered = _recover_abstract(paper)
|
||||
if recovered:
|
||||
paper.abstract = recovered
|
||||
|
||||
# Auto-generate bibkey when source has none (D-04).
|
||||
if paper.bibkey is None:
|
||||
@@ -208,44 +326,63 @@ def ingest_paper(
|
||||
if source_path is not None:
|
||||
from codex.parsing.grobid import extract_references
|
||||
from codex.parsing.nougat import pdf_to_markdown
|
||||
from codex.parsing.tex import chunk_text, latex_to_text
|
||||
from codex.parsing.tex import chunk_sections, chunk_text
|
||||
|
||||
suffix = Path(source_path).suffix.lower()
|
||||
text = ""
|
||||
# (section_title | None, chunk_text) pairs. `.tex` is section-aware so
|
||||
# the stored `section` column reflects the real heading (R-C / R-12);
|
||||
# `.txt`/`.pdf` carry no section structure, so the title is None.
|
||||
raw_pairs: list[tuple[str | None, str]] = []
|
||||
|
||||
if suffix == ".tex":
|
||||
text = latex_to_text(Path(source_path).read_text())
|
||||
raw_pairs = chunk_sections(Path(source_path).read_text())
|
||||
elif suffix == ".txt":
|
||||
text = Path(source_path).read_text(encoding="utf-8", errors="replace")
|
||||
raw_pairs = [(None, c) for c in chunk_text(text)]
|
||||
elif suffix == ".pdf":
|
||||
text = pdf_to_markdown(source_path)
|
||||
raw_pairs = [(None, c) for c in chunk_text(pdf_to_markdown(source_path))]
|
||||
# Also extract GROBID refs
|
||||
grobid_refs = extract_references(source_path)
|
||||
for ref in grobid_refs:
|
||||
cited_id = ref.get("doi") or ref.get("arxiv_id")
|
||||
if cited_id:
|
||||
pdf_citations.append(Citation(citing_id=paper.id, cited_id=cited_id))
|
||||
# Normalize the GROBID DOI (bare/URL/``doi:`` → bare lower-case)
|
||||
# so PDF-derived edges join the same graph nodes as the
|
||||
# S2/OpenAlex paths; arXiv ids pass through untouched.
|
||||
pdf_citations.append(
|
||||
Citation(citing_id=paper.id, cited_id=_norm_cited_id(cited_id))
|
||||
)
|
||||
else:
|
||||
logger.warning("Unbekannter Dateityp: %s — kein Text-Parsing", source_path)
|
||||
|
||||
raw_chunks = chunk_text(text) if text else []
|
||||
chunks_text = filter_chunks(raw_chunks, settings=get_settings())
|
||||
# Quality-gate the pairs (keeps each chunk aligned with its section title).
|
||||
settings = get_settings()
|
||||
kept_pairs = [
|
||||
(title, content)
|
||||
for title, content in raw_pairs
|
||||
if is_quality_chunk(content, settings=settings)
|
||||
]
|
||||
|
||||
# Delete existing chunks for this paper
|
||||
conn.execute("DELETE FROM chunks WHERE paper_id = %s", (paper.id,))
|
||||
|
||||
if chunks_text:
|
||||
if kept_pairs:
|
||||
# Embed all chunks in one batch
|
||||
chunk_embeddings = embedder.encode_dense(chunks_text)
|
||||
chunk_contents = [content for _, content in kept_pairs]
|
||||
chunk_embeddings = embedder.encode_dense(chunk_contents)
|
||||
chunk_rows = [
|
||||
(
|
||||
paper.id,
|
||||
ord_idx,
|
||||
content,
|
||||
chunk_embeddings[ord_idx].tolist(),
|
||||
classify_section(content),
|
||||
# .tex: label from the real \section heading — keeps the
|
||||
# controlled bucket when it matches, else stores the cleaned
|
||||
# title (R-F); .txt/.pdf (title=None) classify by content.
|
||||
# NB: `section` semantics now differ by source (write-only col).
|
||||
section_label(title, content),
|
||||
)
|
||||
for ord_idx, content in enumerate(chunks_text)
|
||||
for ord_idx, (title, content) in enumerate(kept_pairs)
|
||||
]
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
@@ -256,7 +393,7 @@ def ingest_paper(
|
||||
chunks_upserted = len(chunk_rows)
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# 5. Fetch citations (OpenAlex if openalex_id, else S2)
|
||||
# 5. Fetch citations (fallback chain: OpenAlex → S2 → Crossref)
|
||||
# Insert with ON CONFLICT (citing_id, cited_id) DO NOTHING
|
||||
# ---------------------------------------------------------------
|
||||
api_citations: list[Citation]
|
||||
@@ -267,8 +404,17 @@ def ingest_paper(
|
||||
api_citations = [
|
||||
Citation(citing_id=paper.id, cited_id=c.cited_id, context=c.context) for c in raw
|
||||
]
|
||||
# DQ-1: OpenAlex indexes many arXiv preprints / theses WITHOUT a
|
||||
# parsed reference list (referenced_works == []). Fall back to the
|
||||
# Semantic Scholar references, which often has them.
|
||||
if not api_citations:
|
||||
api_citations = _s2_reference_supplement(paper)
|
||||
else:
|
||||
api_citations = semanticscholar.fetch_references(paper_id)
|
||||
api_citations = _s2_reference_supplement(paper)
|
||||
# R-B: if OpenAlex and S2 both came back empty, try Crossref's
|
||||
# publisher-deposited references (DOI papers only) as a third leg.
|
||||
if not api_citations:
|
||||
api_citations = _crossref_reference_supplement(paper)
|
||||
|
||||
# Merge API citations and GROBID PDF citations; dedup via set
|
||||
all_citations_set: set[tuple[str, str]] = set()
|
||||
|
||||
@@ -26,6 +26,7 @@ def _text(element: ET.Element | None) -> str:
|
||||
def extract_references(
|
||||
pdf_path: str,
|
||||
grobid_url: str | None = None,
|
||||
timeout: float = 60.0,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Extract a structured reference list from a PDF via GROBID.
|
||||
|
||||
@@ -35,6 +36,9 @@ def extract_references(
|
||||
Path to the PDF file on disk.
|
||||
grobid_url:
|
||||
Base URL of the GROBID server. Defaults to ``Settings().grobid_url``.
|
||||
timeout:
|
||||
HTTP client timeout in seconds. Large PDFs (e.g. theses) can exceed the
|
||||
60s default, especially against a slower/emulated GROBID; raise it then.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -46,7 +50,7 @@ def extract_references(
|
||||
if grobid_url is None:
|
||||
grobid_url = Settings().grobid_url
|
||||
|
||||
with open(pdf_path, "rb") as fh, httpx.Client(timeout=60.0) as client:
|
||||
with open(pdf_path, "rb") as fh, httpx.Client(timeout=timeout) as client:
|
||||
response = client.post(
|
||||
f"{grobid_url}/api/processReferences",
|
||||
files={"input": (pdf_path, fh, "application/pdf")},
|
||||
|
||||
@@ -17,6 +17,10 @@ _SECTION_RE = re.compile(
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# \input{file} / \include{file} — arXiv papers routinely split the body across
|
||||
# files, so these must be inlined before parsing (R-C multi-file flattening).
|
||||
_INPUT_RE = re.compile(r"\\(?:input|include)\s*\{([^}]+)\}")
|
||||
|
||||
# Patterns for LaTeX noise removal (applied in order).
|
||||
_COMMENT_RE = re.compile(r"%[^\n]*")
|
||||
_CITE_RE = re.compile(r"\\cite\*?\{[^}]*\}")
|
||||
@@ -157,3 +161,63 @@ def latex_to_text(latex: str) -> str:
|
||||
return "\n\n".join(body for _, body in sections)
|
||||
# Fallback: no section structure — clean the whole string.
|
||||
return _clean_latex(latex)
|
||||
|
||||
|
||||
def _norm_texkey(name: str) -> str:
|
||||
"""Normalize a tex file name / ``\\input`` argument to a comparable key.
|
||||
|
||||
Strips a leading ``./``, any directory-irrelevant whitespace, and a trailing
|
||||
``.tex`` so ``\\input{sections/intro}`` matches the archive member
|
||||
``sections/intro.tex``.
|
||||
"""
|
||||
n = name.strip()
|
||||
if n.startswith("./"): # a single leading "./" only — don't eat "../" or ".hidden"
|
||||
n = n[2:]
|
||||
return n[:-4] if n.endswith(".tex") else n
|
||||
|
||||
|
||||
def flatten_inputs(primary: str, files: dict[str, str]) -> str:
|
||||
"""Recursively inline ``\\input``/``\\include`` directives (R-C).
|
||||
|
||||
*files* maps each source file's archive name to its contents (keys may carry
|
||||
a ``.tex`` suffix or a ``./`` prefix — they are normalized internally). arXiv
|
||||
multi-file projects leave the ``\\documentclass`` file as little more than a
|
||||
skeleton of ``\\input`` lines, so the body must be assembled before parsing.
|
||||
Unresolved references are dropped; recursion is capped at depth 10 to guard
|
||||
against include cycles.
|
||||
"""
|
||||
norm = {_norm_texkey(k): v for k, v in files.items()}
|
||||
|
||||
def _expand(text: str, depth: int) -> str:
|
||||
if depth > 10:
|
||||
# Cap reached (likely an \input cycle): strip any still-unresolved
|
||||
# directives rather than leak a literal "\input{…}" into the parsed text.
|
||||
return _INPUT_RE.sub("", text)
|
||||
|
||||
def _repl(match: re.Match[str]) -> str:
|
||||
content = norm.get(_norm_texkey(match.group(1)))
|
||||
return _expand(content, depth + 1) if content is not None else ""
|
||||
|
||||
return _INPUT_RE.sub(_repl, text)
|
||||
|
||||
return _expand(primary, 0)
|
||||
|
||||
|
||||
def chunk_sections(latex: str, size: int = 512, overlap: int = 64) -> list[tuple[str | None, str]]:
|
||||
"""Chunk LaTeX *within* each section, pairing every chunk with its title.
|
||||
|
||||
Unlike :func:`latex_to_text` (which discards titles and flattens the whole
|
||||
document into one block before word-window chunking), this keeps chunks from
|
||||
spanning section boundaries and remembers which ``\\section`` each came from.
|
||||
The caller can then label the stored ``section`` column from the real heading
|
||||
instead of mostly ``body`` (R-C / audit R-12). Falls back to title-less
|
||||
chunks of the whole cleaned document when there are no section commands.
|
||||
"""
|
||||
sections = extract_sections(latex)
|
||||
if not sections:
|
||||
return [(None, chunk) for chunk in chunk_text(_clean_latex(latex), size, overlap)]
|
||||
pairs: list[tuple[str | None, str]] = []
|
||||
for title, body in sections:
|
||||
for chunk in chunk_text(body, size, overlap):
|
||||
pairs.append((title or None, chunk))
|
||||
return pairs
|
||||
|
||||
@@ -110,6 +110,65 @@ def classify_section(text: str) -> str:
|
||||
return "body"
|
||||
|
||||
|
||||
def _clean_title(title: str) -> str:
|
||||
"""Normalize a LaTeX ``\\section`` title for use as a section label.
|
||||
|
||||
Strips LaTeX word-commands (``\\emph`` …), accent/escape macros (``\\"o`` →
|
||||
``o``, ``\\&``), ``~`` ties and ``{}$``, drops leading section numbering
|
||||
(``3.2 ``), collapses whitespace, lower-cases, and truncates to 60 chars at a
|
||||
word boundary so the stored ``section`` value is a clean, comparable string.
|
||||
"""
|
||||
t = title.replace("~", " ") # LaTeX non-breaking tie → space
|
||||
t = re.sub(r"\\[a-zA-Z]+\*?", " ", t) # word-commands: \emph, \mathcal …
|
||||
t = re.sub(r"\\[^a-zA-Z]", "", t) # accent/escape macros: \"o → o, \', \`, \&
|
||||
t = re.sub(r"[{}$]", "", t) # braces / math delimiters
|
||||
t = re.sub(r"^\s*\d+(?:\.\d+)*\.?\s*", "", t) # leading numbering "3.2 "
|
||||
t = " ".join(t.split()).strip().lower()
|
||||
return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t
|
||||
|
||||
|
||||
# A heading collapses to a controlled bucket only when it IS one of these canonical
|
||||
# section words — NOT when it merely starts with one. Prefix-matching via
|
||||
# classify_section mislabels real sections ("Abstract Nonsense and Categories" →
|
||||
# abstract, "References Architecture" → bibliography); an exact map avoids that while
|
||||
# keeping cross-source consistency for the bare headings.
|
||||
_SECTION_TITLE_BUCKETS = {
|
||||
"abstract": "abstract",
|
||||
"zusammenfassung": "abstract",
|
||||
"introduction": "intro",
|
||||
"einleitung": "intro",
|
||||
"proof": "proof",
|
||||
"beweis": "proof",
|
||||
"references": "bibliography",
|
||||
"bibliography": "bibliography",
|
||||
"bibliographie": "bibliography",
|
||||
"literatur": "bibliography",
|
||||
}
|
||||
|
||||
|
||||
def section_label(title: str | None, content: str) -> str:
|
||||
"""Label a chunk's section, preferring the real ``\\section`` heading (R-F).
|
||||
|
||||
For section-aware ``.tex`` ingest *title* is the real heading: a bare canonical
|
||||
heading ("Introduction", "References", "Proof") maps to its controlled bucket
|
||||
(cross-source consistency); any other heading is stored as its cleaned real
|
||||
title (e.g. ``"preliminaries"``, ``"introduction to operator algebras"``) — far
|
||||
more signal than collapsing to ``body`` AND without the prefix-match mislabels
|
||||
that ``classify_section`` would produce on descriptive titles.
|
||||
|
||||
When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass``
|
||||
paths) this falls back to content-based :func:`classify_section` — unchanged
|
||||
behaviour. Safe to store free-text titles because the ``section`` column is
|
||||
write-only (no consumer filters on the controlled vocabulary).
|
||||
"""
|
||||
if not title or not title.strip():
|
||||
return classify_section(content)
|
||||
cleaned = _clean_title(title)
|
||||
if not cleaned:
|
||||
return classify_section(content)
|
||||
return _SECTION_TITLE_BUCKETS.get(cleaned, cleaned)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Batch filter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,21 +1,110 @@
|
||||
"""arXiv API client.
|
||||
|
||||
Provides:
|
||||
- fetch_metadata: resolve an arXiv ID to a Paper via the Atom export API.
|
||||
- fetch_source: download the .tar.gz source of a paper and extract the primary .tex file.
|
||||
- fetch_pdf_url: return the canonical PDF URL for a given arXiv ID.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import tarfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
||||
|
||||
from codex.models import Paper
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://arxiv.org"
|
||||
_EXPORT = "https://export.arxiv.org"
|
||||
_ATOM = "{http://www.w3.org/2005/Atom}"
|
||||
|
||||
|
||||
def _is_retryable(exc: BaseException) -> bool:
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return exc.response.status_code == 429 or exc.response.status_code >= 500
|
||||
return False
|
||||
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception(_is_retryable),
|
||||
stop=stop_after_attempt(4),
|
||||
wait=wait_exponential(min=1, max=20),
|
||||
before_sleep=lambda rs: logger.warning("arXiv retry %d", rs.attempt_number),
|
||||
)
|
||||
def _query(arxiv_id: str) -> httpx.Response:
|
||||
response = httpx.get(
|
||||
f"{_EXPORT}/api/query",
|
||||
params={"id_list": arxiv_id},
|
||||
timeout=30,
|
||||
follow_redirects=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
|
||||
def fetch_metadata(arxiv_id: str) -> Paper | None:
|
||||
"""Resolve an arXiv ID to a Paper via the Atom export API.
|
||||
|
||||
Authoritative metadata source for arXiv preprints — used as an ingest
|
||||
fallback when OpenAlex 404s on an arXiv id (DQ-2). The arXiv id (bare,
|
||||
e.g. ``"1911.00966"`` or legacy ``"math/0603097"``) becomes ``Paper.id``;
|
||||
``openalex_id`` and ``bibkey`` are left for the caller to populate.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Paper | None
|
||||
Populated Paper (title, authors, year, abstract), or None if arXiv
|
||||
has no entry for the id.
|
||||
"""
|
||||
bare = arxiv_id[len("arxiv:") :] if arxiv_id.lower().startswith("arxiv:") else arxiv_id
|
||||
try:
|
||||
response = _query(bare)
|
||||
except httpx.HTTPError:
|
||||
logger.warning("arXiv metadata fetch failed for %s", bare, exc_info=True)
|
||||
return None
|
||||
try:
|
||||
root = ET.fromstring(response.text)
|
||||
except ET.ParseError:
|
||||
return None
|
||||
entry = root.find(f"{_ATOM}entry")
|
||||
if entry is None:
|
||||
return None
|
||||
# An id-not-found query still returns a feed but with no <entry>.
|
||||
title = (entry.findtext(f"{_ATOM}title") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
summary = (entry.findtext(f"{_ATOM}summary") or "").strip()
|
||||
published = entry.findtext(f"{_ATOM}published") or ""
|
||||
year = int(published[:4]) if published[:4].isdigit() else None
|
||||
authors = [
|
||||
name.strip()
|
||||
for a in entry.findall(f"{_ATOM}author")
|
||||
if (name := a.findtext(f"{_ATOM}name")) and name.strip()
|
||||
]
|
||||
return Paper(
|
||||
id=bare,
|
||||
title=" ".join(title.split()),
|
||||
authors=authors,
|
||||
year=year,
|
||||
abstract=" ".join(summary.split()) or None,
|
||||
)
|
||||
|
||||
|
||||
def _has_documentclass(content: str) -> bool:
|
||||
"""True if *content* has an UN-commented ``\\documentclass`` line.
|
||||
|
||||
A plain ``"\\documentclass" in content`` substring test also matches a
|
||||
commented-out ``%\\documentclass`` (common in arXiv preambles); requiring the
|
||||
command to start its line avoids selecting the wrong primary file.
|
||||
"""
|
||||
return any(line.lstrip().startswith("\\documentclass") for line in content.splitlines())
|
||||
|
||||
|
||||
def fetch_source(arxiv_id: str) -> str | None:
|
||||
@@ -23,8 +112,9 @@ def fetch_source(arxiv_id: str) -> str | None:
|
||||
|
||||
Downloads the .tar.gz source bundle from ``https://arxiv.org/src/{arxiv_id}``,
|
||||
locates the primary .tex file (preferring any file containing ``\\documentclass``,
|
||||
falling back to the largest .tex by size), and returns its contents as a UTF-8
|
||||
string.
|
||||
falling back to the largest .tex by size), and inlines its ``\\input``/
|
||||
``\\include`` directives from the other archive members so multi-file projects
|
||||
return the full document body, not just the primary file's include skeleton.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -45,38 +135,57 @@ def fetch_source(arxiv_id: str) -> str | None:
|
||||
if response.status_code != 200:
|
||||
response.raise_for_status()
|
||||
|
||||
from codex.parsing.tex import flatten_inputs
|
||||
|
||||
raw = response.content
|
||||
# Image/font/binary members are never \input'd as text; skip them so the source
|
||||
# map stays text-only (decode-with-replace would otherwise store garbled bytes).
|
||||
skip_ext = (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".pdf", ".eps", ".ps", ".ttf", ".otf")
|
||||
try:
|
||||
with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf:
|
||||
tex_members = [m for m in tf.getmembers() if m.name.endswith(".tex")]
|
||||
if not tex_members:
|
||||
logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
|
||||
return None
|
||||
|
||||
# Prefer the file containing \documentclass (primary document)
|
||||
primary: tarfile.TarInfo | None = None
|
||||
for member in tex_members:
|
||||
files: dict[str, str] = {}
|
||||
primary_name: str | None = None
|
||||
for member in tf.getmembers():
|
||||
if not member.isfile() or member.name.lower().endswith(skip_ext):
|
||||
continue
|
||||
f = tf.extractfile(member)
|
||||
if f is None:
|
||||
continue
|
||||
content_bytes = f.read()
|
||||
if b"\\documentclass" in content_bytes:
|
||||
primary = member
|
||||
# Decode and return immediately — first match wins
|
||||
return content_bytes.decode("utf-8", errors="replace")
|
||||
# Keep ALL text members resolvable so \input of a non-.tex include
|
||||
# (e.g. a .def or extension-less body file) is not silently dropped (R-C).
|
||||
files[member.name] = f.read().decode("utf-8", errors="replace")
|
||||
# Primary doc = a .tex with an UN-commented \documentclass.
|
||||
if (
|
||||
primary_name is None
|
||||
and member.name.endswith(".tex")
|
||||
and _has_documentclass(files[member.name])
|
||||
):
|
||||
primary_name = member.name
|
||||
|
||||
if primary is None:
|
||||
# Fallback: largest .tex by size
|
||||
largest = max(tex_members, key=lambda m: m.size)
|
||||
f = tf.extractfile(largest)
|
||||
if f is None:
|
||||
return None
|
||||
return f.read().decode("utf-8", errors="replace")
|
||||
except tarfile.TarError as exc:
|
||||
logger.warning("Failed to open tar archive for %s: %s", arxiv_id, exc)
|
||||
tex_files = {k: v for k, v in files.items() if k.endswith(".tex")}
|
||||
if not tex_files:
|
||||
logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
|
||||
return None
|
||||
|
||||
return None # unreachable but satisfies type checker
|
||||
# No \documentclass anywhere → fall back to the largest .tex.
|
||||
if primary_name is None:
|
||||
primary_name = max(tex_files, key=lambda k: len(tex_files[k]))
|
||||
|
||||
# Inline \input/\include so multi-file projects yield the full body,
|
||||
# not just the primary file's skeleton of include directives (R-C).
|
||||
return flatten_inputs(files[primary_name], files)
|
||||
except tarfile.TarError:
|
||||
# Old arXiv submissions are served as a single bare gzipped .tex with no
|
||||
# tar wrapper (e.g. legacy math/* papers) — gunzip the body directly.
|
||||
try:
|
||||
single = gzip.decompress(raw).decode("utf-8", errors="replace")
|
||||
except (OSError, EOFError) as exc:
|
||||
logger.warning("Failed to read arXiv source for %s: %s", arxiv_id, exc)
|
||||
return None
|
||||
if "\\" in single[:5000]: # looks like LaTeX (has backslash commands)
|
||||
return single
|
||||
logger.debug("arXiv source for %s is not LaTeX", arxiv_id)
|
||||
return None
|
||||
|
||||
|
||||
def fetch_pdf_url(arxiv_id: str) -> str:
|
||||
|
||||
126
codex/sources/crossref.py
Normal file
126
codex/sources/crossref.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Crossref API client.
|
||||
|
||||
Provides:
|
||||
- fetch_abstract: retrieve a work's abstract (JATS-stripped) by DOI.
|
||||
- fetch_references: retrieve a work's reference list (DOI edges) by DOI.
|
||||
|
||||
Crossref is a *third* metadata source after OpenAlex and Semantic Scholar
|
||||
(DQ-2 abstracts / roadmap R-B references). Requests use the Polite Pool (mailto
|
||||
query parameter) and are retried on 429/5xx with exponential back-off via tenacity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
||||
|
||||
from codex.config import get_settings
|
||||
from codex.models import Citation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://api.crossref.org"
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
|
||||
|
||||
def _is_retryable(exc: BaseException) -> bool:
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return exc.response.status_code == 429 or exc.response.status_code >= 500
|
||||
return False
|
||||
|
||||
|
||||
def _polite_params() -> dict[str, str]:
|
||||
# Reuse the OpenAlex mailto for Crossref's Polite Pool (same address).
|
||||
mailto = get_settings().openalex_mailto
|
||||
return {"mailto": mailto} if mailto else {}
|
||||
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception(_is_retryable),
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_exponential(min=1, max=30),
|
||||
before_sleep=lambda rs: logger.warning(
|
||||
"Crossref retry %d after %s",
|
||||
rs.attempt_number,
|
||||
rs.outcome.exception(), # type: ignore[union-attr]
|
||||
),
|
||||
)
|
||||
def _get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
merged: dict[str, str] = {**_polite_params(), **(params or {})}
|
||||
response = httpx.get(url, params=merged, timeout=30, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
|
||||
def _strip_jats(abstract: str | None) -> str | None:
|
||||
"""Strip JATS/XML tags from a Crossref abstract and collapse whitespace."""
|
||||
if not abstract:
|
||||
return None
|
||||
text = _TAG_RE.sub("", abstract)
|
||||
text = " ".join(text.split()).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def fetch_abstract(doi: str) -> str | None:
|
||||
"""Fetch a work's abstract from Crossref by DOI.
|
||||
|
||||
Crossref stores abstracts as JATS-tagged markup (``<jats:p>…</jats:p>``);
|
||||
tags are stripped before returning. Many works (notably book chapters)
|
||||
have no abstract deposited — returns None in that case and on 404.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
doi:
|
||||
A bare DOI (``"10.1007/s00454-019-00132-8"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
str | None
|
||||
The plain-text abstract, or None when absent.
|
||||
"""
|
||||
url = f"{_BASE}/works/{doi}"
|
||||
try:
|
||||
response = _get(url)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code == 404:
|
||||
return None
|
||||
raise
|
||||
message: dict[str, Any] = response.json().get("message", {})
|
||||
return _strip_jats(message.get("abstract"))
|
||||
|
||||
|
||||
def fetch_references(doi: str) -> list[Citation]:
|
||||
"""Fetch a work's reference list from Crossref by DOI (roadmap R-B).
|
||||
|
||||
Crossref carries publisher-deposited reference lists in ``message.reference``.
|
||||
Each entry that cites a DOI-registered work exposes a bare ``DOI`` field;
|
||||
only those become citation-graph edges (book / older references carry just
|
||||
``unstructured`` text with no resolvable id and are skipped). ``citing_id``
|
||||
is the queried *doi* — the caller rewrites it to the canonical ``papers.id``
|
||||
and normalises the cited DOIs (mirrors the Semantic Scholar reference path).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
doi:
|
||||
A bare DOI (``"10.1007/s00454-019-00132-8"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Citation]
|
||||
One Citation per reference that carries a DOI. Empty on 404 or when no
|
||||
references are deposited for the work.
|
||||
"""
|
||||
url = f"{_BASE}/works/{doi}"
|
||||
try:
|
||||
response = _get(url)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code == 404:
|
||||
return []
|
||||
raise
|
||||
message: dict[str, Any] = response.json().get("message", {})
|
||||
references: list[dict[str, Any]] = message.get("reference") or []
|
||||
return [Citation(citing_id=doi, cited_id=ref["DOI"]) for ref in references if ref.get("DOI")]
|
||||
@@ -85,13 +85,55 @@ def _reconstruct_abstract(inverted_index: dict[str, list[int]] | None) -> str |
|
||||
return " ".join(w for _, w in word_positions)
|
||||
|
||||
|
||||
def _normalize_doi(doi: str) -> str:
|
||||
"""Normalize an OpenAlex DOI to the canonical bare, lower-cased form.
|
||||
|
||||
OpenAlex returns the ``doi`` field as a full URL
|
||||
(``https://doi.org/10.1007/s00454-019-00132-8``), but the canonical
|
||||
``papers.id`` produced by the M-1 migration is the BARE, lower-cased DOI
|
||||
(``10.1007/s00454-019-00132-8``). Stripping the ``https://doi.org/`` /
|
||||
``doi:`` prefix and lower-casing (DOIs are case-insensitive by spec) makes a
|
||||
re-ingested DOI paper match its stored row, so the ingest
|
||||
``INSERT … ON CONFLICT (id)`` updates in place instead of attempting a fresh
|
||||
INSERT that trips the separate ``papers_openalex_id_key`` unique constraint
|
||||
(DQ-5). Mirrors the cited-id convention in ``ingest._norm_cited_id``.
|
||||
"""
|
||||
s = doi.strip().lower()
|
||||
for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
|
||||
if s.startswith(prefix):
|
||||
return s[len(prefix) :]
|
||||
return s
|
||||
|
||||
|
||||
# arXiv works are registered under a DOI of the form 10.48550/arXiv.<id>.
|
||||
_ARXIV_DOI_PREFIX = "10.48550/arxiv."
|
||||
|
||||
|
||||
def _canonical_id(doi: str) -> str:
|
||||
"""Return the canonical ``papers.id`` for an OpenAlex ``doi`` value.
|
||||
|
||||
Builds on :func:`_normalize_doi` (bare, lower-cased DOI), then strips the
|
||||
arXiv DOI prefix ``10.48550/arxiv.`` so an arXiv work's id is the BARE arXiv
|
||||
id (``2305.10988`` / ``math/0603097``) — the form the corpus is keyed on —
|
||||
not the arXiv DOI. Without this, re-ingesting an arXiv paper by its bare id
|
||||
misses ``INSERT … ON CONFLICT (id)`` and trips ``papers_openalex_id_key``.
|
||||
This is the arXiv half of DQ-5 (the DOI half was fixed there; this was
|
||||
explicitly deferred and surfaced again driving the R-C .tex re-ingest).
|
||||
"""
|
||||
normalized = _normalize_doi(doi)
|
||||
if normalized.startswith(_ARXIV_DOI_PREFIX):
|
||||
return normalized[len(_ARXIV_DOI_PREFIX) :]
|
||||
return normalized
|
||||
|
||||
|
||||
def _map_paper(data: dict[str, Any]) -> Paper:
|
||||
authors: list[str] = [
|
||||
a.get("author", {}).get("display_name") or "" for a in data.get("authorships", [])
|
||||
]
|
||||
abstract = _reconstruct_abstract(data.get("abstract_inverted_index"))
|
||||
doi = data.get("doi")
|
||||
return Paper(
|
||||
id=data.get("doi") or data.get("id") or "",
|
||||
id=_canonical_id(doi) if doi else (data.get("id") or ""),
|
||||
title=data.get("title") or "",
|
||||
openalex_id=data.get("id"),
|
||||
authors=authors,
|
||||
|
||||
@@ -90,7 +90,11 @@ def fetch_references(paper_id: str) -> list[Citation]:
|
||||
raise
|
||||
|
||||
data = response.json()
|
||||
raw_refs: list[dict[str, Any]] = data.get("data", [])
|
||||
# S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
|
||||
# parsed reference list (e.g. some theses). ``.get("data", [])`` would yield
|
||||
# the explicit ``None`` (the default only applies when the key is absent),
|
||||
# so iterate over a guaranteed list instead.
|
||||
raw_refs: list[dict[str, Any]] = data.get("data") or []
|
||||
citations: list[Citation] = []
|
||||
for entry in raw_refs:
|
||||
cited_paper: dict[str, Any] = entry.get("citedPaper", {})
|
||||
@@ -106,6 +110,24 @@ def fetch_references(paper_id: str) -> list[Citation]:
|
||||
return citations
|
||||
|
||||
|
||||
def fetch_abstract(paper_id: str) -> str | None:
|
||||
"""Fetch just the abstract for a paper from Semantic Scholar.
|
||||
|
||||
Used as an ingest abstract supplement (DQ-2) when OpenAlex has the paper
|
||||
but no abstract. ``paper_id`` should be namespaced (``arXiv:…`` / ``DOI:…``).
|
||||
Returns None on 404 or when S2 has no abstract.
|
||||
"""
|
||||
url = f"{_BASE_GRAPH}/paper/{paper_id}"
|
||||
try:
|
||||
response = _get(url, params={"fields": "abstract"})
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code == 404:
|
||||
return None
|
||||
raise
|
||||
abstract = response.json().get("abstract")
|
||||
return abstract or None
|
||||
|
||||
|
||||
def fetch_recommendations(paper_id: str, limit: int = 20) -> list[str]:
|
||||
"""Fetch recommended paper IDs from Semantic Scholar.
|
||||
|
||||
|
||||
59
docs/SESSION-2026-06-17-open-items.md
Normal file
59
docs/SESSION-2026-06-17-open-items.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# Open items — session 2026-06-17 (handoff notice)
|
||||
|
||||
Everything still open after this session. Done items are in
|
||||
`docs/audit/DATA-QUALITY-2026-06-15.md` (DQ-1..DQ-5 + roadmap R-A,R-B,R-C,R-D,R-F all
|
||||
DONE; R-E deferred). This file is only the **loose ends**.
|
||||
|
||||
## 1. ✅ RESOLVED — Jetson is online (the tunnel had dropped, not the host)
|
||||
- What looked like the Jetson going offline was the **SSH tunnel** breaking, not the
|
||||
host. The Jetson (`192.168.178.103`) is online. Re-open the tunnel with keepalive:
|
||||
`ssh -o ServerAliveInterval=15 -f -N -L 5433:localhost:5432 alfred@192.168.178.103`.
|
||||
Better still: run long ops in tmux-on-Jetson (item #4) to avoid the tunnel entirely.
|
||||
|
||||
## 2. ✅ DONE — R-F accent re-clean
|
||||
- R-F is functionally DONE (section signal 34/329 → **314/329 = 95 %**). The remaining
|
||||
cosmetic accent/tie artifacts in live `chunks.section` labels were re-cleaned in place
|
||||
once the tunnel was back: applied `codex.quality._clean_title` to every label
|
||||
containing `\` or `~` — **8 labels fixed**, no re-embed. Verified **0 artifacts**
|
||||
remain. R-F fully complete.
|
||||
|
||||
## 3. ✅ READY TO MERGE — all branches integrated + validated
|
||||
All five feature branches are merged (**zero conflicts**) into **`integration/roadmap`**
|
||||
(off `audit/loop-1`) and validated green: **391 tests pass, ruff + ruff-format + mypy
|
||||
clean, tree clean**. This is the single merge candidate for `main`.
|
||||
|
||||
Contents (DQ-1..DQ-5 + R-A..R-F + infra docs):
|
||||
| Source branch | Contents |
|
||||
|---|---|
|
||||
| `audit/loop-1` (base) | DQ-1..DQ-5, **R-A** (GROBID backfill + native arm64 GROBID), this notice |
|
||||
| `feat/section-labels` (incl. `feat/tex-ingest`) | **R-C** (.tex ingest) + DQ-5 arXiv-id fix + gzip fallback + **R-F** (section labels) |
|
||||
| `feat/crossref-references` | **R-B** (Crossref refs) + GROBID-on-demand tweak |
|
||||
| `feat/graph-coverage-warning` | **R-D** (citing-coverage warning) |
|
||||
| `chore/infra-ssh-stability` | infra TODO (#4) |
|
||||
|
||||
**To land on `main`** (user-triggered):
|
||||
```
|
||||
git checkout main && git merge --no-ff integration/roadmap
|
||||
```
|
||||
…or open a PR `integration/roadmap` → `main`. The individual feature branches are
|
||||
preserved unchanged for per-feature review if preferred.
|
||||
|
||||
**Cleanup — branches prunable AFTER the merge lands** (not deleted here; destructive):
|
||||
- Already in `main`: `fix/cli-readme-drift`, `fix/d04-d05-ingest-fixes`.
|
||||
- Folded into `integration/roadmap`: the 5 feature branches + `audit/loop-1`.
|
||||
- Pre-session / stale (verify before pruning): `feat/F-15-literature-graph`,
|
||||
`feat/F-16-chunk-quality`, `fix/audit-wave-1`/`-2`/`-3`, `scratch/spike-embed`.
|
||||
- Working tree is clean — only gitignored `.venv`/caches/`.env`; no stray tracked files.
|
||||
|
||||
## 4. Infra TODO — tmux-on-Jetson for stable long ops (documented, not implemented)
|
||||
- `docs/infra/jetson-ssh-stability.md` (branch `chore/infra-ssh-stability`). The
|
||||
Mac↔Jetson tunnel dropped repeatedly on multi-minute writes all session. Fix: run long
|
||||
ops **on the Jetson in `tmux`** (DB is localhost there → no tunnel; survives SSH drops);
|
||||
`autossh` for interactive tunnels. Open.
|
||||
|
||||
## 5. Known limitations (documented, NOT action items)
|
||||
- `math/0306167` (old AMS-TeX, no `\section{}`) stays `section=body` — inherent.
|
||||
- The 2 edited-volume books yielded 0 GROBID-parsed DOI/arXiv refs in the R-A sweep —
|
||||
inherent (per-chapter refs, no clean end-bibliography).
|
||||
- **DQ-6** (upstream PDF→`.txt` extraction fidelity, done outside codex-py) — flagged
|
||||
earlier, largely **mooted** by R-A/R-C ingesting from PDF/`.tex` directly.
|
||||
@@ -1,6 +1,15 @@
|
||||
# DATA-QUALITY AUDIT — codex knowledge base (handoff for a fresh session)
|
||||
|
||||
**Status:** baseline scan done 2026-06-15; four investigation items open (DQ-1…DQ-4).
|
||||
**Status:** baseline scan done 2026-06-15. **DQ-1 RESOLVED 2026-06-16** (S2 supplement
|
||||
applied to live DB + wired into ingest). **DQ-4 MEASURED 2026-06-16** — retrieval is
|
||||
strong after fixing a P0 `codex search paper` crash. **DQ-2 RESOLVED 2026-06-16**
|
||||
(1911.00966 fully recovered from arXiv; s00454 abstract from S2; book chapter is a
|
||||
documented limit; recovery wired into ingest). **DQ-3 RESOLVED 2026-06-16**
|
||||
(content fidelity clean — full coverage, only OCR junk dropped). **All four
|
||||
assessment items (DQ-1..DQ-4) DONE. DQ-5 RESOLVED 2026-06-17** (DOI id normalized
|
||||
to bare canonical form in `openalex._map_paper`; live re-ingest of
|
||||
`10.1007/s00454-019-00132-8` confirmed idempotent → **unblocks R-A/R-C**).
|
||||
**Roadmap R-A..R-E** added 2026-06-16 (free-source acquisition levers — see Roadmap section).
|
||||
**Author:** Audit-Loop (Opus). **Intended reader:** a *cold* session with no memory
|
||||
of the audit conversation — everything needed to continue is in this file.
|
||||
|
||||
@@ -50,7 +59,35 @@ incomplete, or unrepresentative data. This document is that second axis — a
|
||||
|
||||
## Findings (open investigation items)
|
||||
|
||||
### DQ-1 — Citation coverage: 12 / 29 papers (41 %) have ZERO citations · **HIGH**
|
||||
### DQ-1 — Citation coverage: 12 / 29 papers (41 %) had ZERO citations · **RESOLVED 2026-06-16**
|
||||
|
||||
**Resolution (what was done):**
|
||||
- **Verified upstream, not a bug.** Queried the OpenAlex API directly for all 11
|
||||
zero-citation papers that have an `openalex_id`: every one returns
|
||||
`referenced_works_count = 0` server-side, W-ids all match what we stored. They
|
||||
are `type: preprint` (arXiv) / `article` / `dissertation` — works OpenAlex
|
||||
indexes without a parsed bibliography. Confirmed source-coverage limit.
|
||||
- **Semantic Scholar supplement applied to the live DB** (idempotent backfill,
|
||||
`ON CONFLICT (citing_id,cited_id) DO NOTHING`): **+330 citations (590 → 920)**.
|
||||
Zero-out-edge papers **12 → 2**; in-corpus citing coverage **17/29 → 27/29**;
|
||||
graph 489→704 nodes, 590→920 edges; discovery-lead targets 472→677. 9 of the
|
||||
new edges are in-corpus (e.g. `2305.10988 → 10.1007/s00454-019-00132-8`).
|
||||
- **Wired into ingest** so future re-ingests self-heal: when OpenAlex returns an
|
||||
empty `referenced_works`, `ingest.py` now falls back to
|
||||
`_s2_reference_supplement()` (citing_id rewritten to canonical `papers.id`,
|
||||
DOI cited-ids lowercased). Same helper drove the one-off backfill.
|
||||
- **Bug fixed (would have crashed the batch):** `semanticscholar.fetch_references`
|
||||
used `data.get("data", [])`, but S2 returns HTTP 200 `{"data": null}` for a
|
||||
known paper with no parsed refs (e.g. `depositonce-5415`) → `TypeError`.
|
||||
Changed to `data.get("data") or []`. Removed a now-redundant discarded S2 probe
|
||||
in the OpenAlex-404 arXiv branch. New regression test added.
|
||||
- **Irreducible remainder (documented limit):** the **2 `depositonce` theses**
|
||||
(`10.14279/depositonce-20357`, `-5415`) stay at zero — neither OpenAlex nor S2
|
||||
indexes references for TU-Berlin institutional DOIs. Accept as inherent.
|
||||
- **Files touched (uncommitted in working tree):** `codex/sources/semanticscholar.py`,
|
||||
`codex/ingest.py`, `tests/ingest/test_ingest.py`. Full suite: 331 passed.
|
||||
|
||||
**Original finding (for context):**
|
||||
- **Measured:** 12 papers contribute **no** out-edges; the 590-edge citation graph
|
||||
comes from only ~17 papers. The whole F-15 layer (PageRank / coupling /
|
||||
co-citation / discovery leads) therefore runs on ~59 % of the corpus.
|
||||
@@ -79,7 +116,32 @@ incomplete, or unrepresentative data. This document is that second axis — a
|
||||
supplement, or (b) documented as an inherent OpenAlex-coverage limit with the
|
||||
graph caveated accordingly.
|
||||
|
||||
### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **MED**
|
||||
### DQ-2 — Metadata gaps: 3 no-abstract, 1 fully metadata-less · **RESOLVED 2026-06-16**
|
||||
|
||||
**Resolution (what was done):**
|
||||
- **`1911.00966` fully recovered** from the arXiv Atom API (OpenAlex 404'd on it):
|
||||
title "A discrete version of Liouville's theorem on conformal maps",
|
||||
Pinkall & Springborn, 2019, 384-char abstract → bibkey `PinkallSpringborn2019`
|
||||
auto-generated, abstract embedded (non-zero). It was the worst node in the
|
||||
corpus — now `@cite`-able, in wiki grounding, and its **10 chunks are visible
|
||||
to chunk search** (previously filtered out by `bibkey IS NOT NULL`). Citations
|
||||
were already added in DQ-1 (27 S2 refs).
|
||||
- **`10.1007/s00454-019-00132-8` abstract recovered from S2** (1186 chars) and
|
||||
re-embedded (targeted `UPDATE`, not full re-ingest — see DQ-5).
|
||||
- **`10.1007/978-3-642-17413-1_7` (book chapter): documented inherent limit.**
|
||||
No abstract in OpenAlex, S2, **or Crossref** (verified). Left as the single
|
||||
remaining zero-vector paper; user may add an abstract by hand.
|
||||
- **Corpus now: 1 paper without abstract** (the book chapter), down from 3+1.
|
||||
Zero-vector search pollution (DQ-4 secondary finding) reduced from 2 tail
|
||||
fillers to 1.
|
||||
- **Wired into ingest** so this self-heals: OpenAlex-404 on an arXiv id now
|
||||
recovers metadata via `arxiv.fetch_metadata` (new); an empty abstract is
|
||||
supplemented via `_recover_abstract` → S2 then Crossref (new
|
||||
`codex/sources/crossref.py`). New `semanticscholar.fetch_abstract`.
|
||||
- **Files touched:** `codex/sources/arxiv.py`, `codex/sources/crossref.py` (new),
|
||||
`codex/sources/semanticscholar.py`, `codex/ingest.py`, + tests. Full suite: 342.
|
||||
|
||||
**Original finding (for context):**
|
||||
- **Measured:**
|
||||
- **No abstract (3):** `10.1007/978-3-642-17413-1_7`,
|
||||
`10.1007/s00454-019-00132-8`, `1911.00966`. These get a **zero-vector**
|
||||
@@ -98,7 +160,55 @@ incomplete, or unrepresentative data. This document is that second axis — a
|
||||
- **Acceptance:** every paper has at least a bibkey + non-zero abstract embedding,
|
||||
or the exceptions are documented with rationale.
|
||||
|
||||
### DQ-3 — Content fidelity (chunks vs source `.txt`): NOT YET VERIFIED · **MED**
|
||||
### DQ-3 — Content fidelity (chunks vs source `.txt`): VERIFIED CLEAN · **RESOLVED 2026-06-16**
|
||||
|
||||
**Resolution (read-only; re-ran `chunk_text` + `is_quality_chunk` on all 29
|
||||
source files and compared to stored chunks):**
|
||||
- **No paper is silently gutted.** Coverage (stored chunk chars / source chars)
|
||||
is **109–114% for every paper** — the >100% is exactly the 64/512-word overlap
|
||||
inflation, i.e. full retention. `stored == kept` for all 29, so the live DB
|
||||
matches what the current pipeline produces.
|
||||
- **Only 18 chunks dropped corpus-wide, all `low_alpha`** (0 short, 0 bib),
|
||||
concentrated in `10.1007/0-387-29555-0_13` (6) and `depositonce-5415` (10).
|
||||
Inspected: they are **garbled OCR tables** from scanned-book sources (alpha
|
||||
0.30–0.36, e.g. `"(u: 1) 7 < u D:= ud3424 — else H®: out.v. Family 1…"`) —
|
||||
exactly the artefacts the alpha-ratio gate targets. **No real math content lost**
|
||||
(the handoff's feared failure mode did not occur; math prose stays > 0.40 alpha).
|
||||
- **Head/tail alignment exact** on sampled papers: chunk 0 begins at the source
|
||||
head (title/authors/abstract captured), last chunk ends at the source tail → no
|
||||
front/back truncation.
|
||||
- **Note:** 0 bibliography drops across the corpus → the upstream `.txt` extraction
|
||||
had already stripped reference lists, which is why DQ-1's citation graph relies
|
||||
on the API/GROBID sources rather than text-mined refs.
|
||||
- **Acceptance met** for all 29 papers (not just a sample). No action needed.
|
||||
- **⚠ SCOPE LIMIT — DQ-3 covers `.txt → chunks` ONLY.** It treats each source
|
||||
`.txt` as ground truth and verifies the *ingest pipeline* preserves it. It does
|
||||
**not** measure the layer above — **PDF/source → `.txt`** — which was done by an
|
||||
upstream tool in `ConformalLabpp` (outside codex-py) and is unexamined. There is
|
||||
already evidence that layer is lossy: the 18 dropped chunks were garbled OCR
|
||||
tables *already present in the `.txt`* (i.e. the PDF→txt step garbled the
|
||||
table/formula regions), and the bibliographies were stripped upstream. So
|
||||
"VERIFIED CLEAN" means **the loader faithfully preserves whatever the `.txt`
|
||||
contains, not that no information was lost from the source paper.** See DQ-6.
|
||||
|
||||
### DQ-6 — PDF/source → `.txt` extraction fidelity: NOT MEASURED · **MED, open**
|
||||
- **Open question:** how much did the upstream PDF→`.txt` extraction (done in
|
||||
`ConformalLabpp`, outside codex-py) drop or garble vs the original papers?
|
||||
Math/equations, tables, multi-column layouts, and figure captions are the usual
|
||||
casualties of PDF text extraction — and we have direct evidence of garble
|
||||
(DQ-3's low_alpha drops were already-garbled tables in the `.txt`).
|
||||
- **Checkable:** the 36 original PDFs are present in `…/ConformalLabpp/papers/`
|
||||
(map each paper's `source_path` `.txt` to its sibling `.pdf` by filename stem).
|
||||
- **To investigate:** for a sample (incl. the known-bad scanned book
|
||||
`0-387-29555-0_13` and a born-digital arXiv PDF), extract PDF text independently
|
||||
and compare length / page-coverage to the `.txt`; spot-check whether specific
|
||||
theorem statements & equations survive. Flag papers where the `.txt` is missing
|
||||
large spans.
|
||||
- **Acceptance:** a per-sample source→txt coverage estimate, or a list of papers
|
||||
whose `.txt` is materially degraded (candidates for re-extraction — note roadmap
|
||||
**R-A/R-C** would re-ingest from PDF/`.tex` directly and largely sidestep this).
|
||||
|
||||
**Original open question (for context):**
|
||||
- **Open question:** does the stored chunk set faithfully reconstruct each source
|
||||
file, or did the chunker / `filter_chunks` silently drop material (e.g. an
|
||||
abstract, a section, math-heavy passages)? The whole corpus was ingested from
|
||||
@@ -111,7 +221,40 @@ incomplete, or unrepresentative data. This document is that second axis — a
|
||||
- **Acceptance:** sampled papers retain ≳ the expected fraction of source text;
|
||||
no paper is silently gutted by the quality gate.
|
||||
|
||||
### DQ-4 — Retrieval quality: NOT YET MEASURED · **MED**
|
||||
### DQ-4 — Retrieval quality · **MEASURED 2026-06-16 — good, after fixing a P0 crash**
|
||||
|
||||
**P0 bug found + fixed: `codex search paper` was crash-broken on the live DB.**
|
||||
The CLI passed the query embedding uncast, so pgvector raised
|
||||
`operator does not exist: vector <-> double precision[]` on every paper search.
|
||||
The working call sites (`mcp_server.py`, `wiki.py`) cast `%(emb)s::vector`; the
|
||||
CLI did not. Fixed both `<->` occurrences in `codex/cli.py search_paper`; added a
|
||||
regression guard in `tests/cli/test_cli.py` asserting the cast (mocked-DB unit
|
||||
tests can't catch the type error — this is why it survived the code audit).
|
||||
|
||||
**Relevance verdict (post-fix): strong.** Six domain queries, both surfaces
|
||||
(paper-level abstract search + chunk-level). **rank-1 was the exactly-correct
|
||||
paper in 6/6 queries** for chunk search and 5/6 for paper-level (Laplace–Beltrami
|
||||
put the exact paper at rank-2 behind the closely-related "Polygon Laplacian Made
|
||||
Simple" — acceptable). Examples: "combinatorial Yamabe flow" → `math/0306167`
|
||||
(exact, all 5 top chunks that paper); "variational principle for Delaunay
|
||||
triangulations" → `math/0603097` (exact); "circle packing rigidity" →
|
||||
`2601.22903` (exact). Three of the DQ-1-rescued papers (`2305.10988`,
|
||||
`1005.2698`, `depositonce-5415`) now surface as top hits. The KB is trustworthy
|
||||
for lookups.
|
||||
|
||||
**Secondary finding → reinforces DQ-2: zero-vector abstract pollution.** Papers
|
||||
with no abstract get a zero `abstract_emb`, which sits at constant L2 distance
|
||||
≈1.000 from every query and appears as filler in the paper-level tail whenever
|
||||
< 5 strongly-relevant papers exist. `1911.00966` (fully degraded, DQ-2) showed up
|
||||
at distance 1.000 in 4/6 queries. Harmless at rank>1 with a visible 1.000 score,
|
||||
but noise — and a concrete reason to fix DQ-2. Chunk-level search is immune (it
|
||||
filters `bibkey IS NOT NULL`, and 1911.00966 has no bibkey/chunks).
|
||||
|
||||
**Note:** the MCP `search` docstring claims "hybrid dense + FTS" but the SQL is
|
||||
dense-only; `score = 1.0 - dist` can go negative (bge-m3 L2 ranges 0–2). Ranking
|
||||
is fine (monotonic); only the absolute score label is misleading.
|
||||
|
||||
**Original open question (for context):**
|
||||
- **Open question:** end-to-end, do real queries return *relevant* results? Clean
|
||||
chunks + a working index don't guarantee useful retrieval.
|
||||
- **To investigate next:** run a handful of domain queries through the actual
|
||||
@@ -123,16 +266,288 @@ incomplete, or unrepresentative data. This document is that second axis — a
|
||||
- **Acceptance:** a short relevance table (query → top-5 → on/off-topic) good
|
||||
enough to trust the KB for lookups, or a list of failure modes to fix.
|
||||
|
||||
### DQ-5 — `ingest_paper` is NOT idempotent for DOI papers · **RESOLVED 2026-06-17** (found 2026-06-16)
|
||||
|
||||
**Resolution (what was done):**
|
||||
- **Root-cause fix, minimal + contained:** added `openalex._normalize_doi()` and
|
||||
applied it in `_map_paper` so the DOI branch yields the canonical **bare,
|
||||
lower-cased** form (strips `https://doi.org/` / `http://doi.org/` / `doi:`;
|
||||
lower-cases — DOIs are case-insensitive, matching `ingest._norm_cited_id`). The
|
||||
no-DOI fallback (OpenAlex W-id URL) is left untouched. `_map_paper`→`fetch_paper`
|
||||
→`ingest_paper` is the only consumer chain, so blast radius is one function.
|
||||
- **Checked nothing depended on the URL form.** It was the opposite: the full-URL
|
||||
id silently *defeated* `_s2_id_for(paper.id)` and `_recover_abstract`'s
|
||||
`paper.id.startswith("10.")` (a `https://…` string is neither `10.`- nor
|
||||
`doi:`-prefixed), so S2/Crossref recovery never fired for DOI papers. The fix
|
||||
repairs those paths too.
|
||||
- **Tests (offline, mocked DB):** closed audit **T-3** — the OpenAlex fixture now
|
||||
emits the real URL-form DOI and asserts `paper.id` is the bare form; added
|
||||
`_normalize_doi` unit tests (https/http/`doi:`/upper-case/already-bare) and a
|
||||
W-id-fallback test; added ingest regression `test_ingest_doi_paper_upserts_bare
|
||||
_canonical_id` (runs the real `_map_paper` through ingest, asserts the papers
|
||||
upsert carries the **bare** id into `%(id)s` and uses `ON CONFLICT (id)`).
|
||||
Verified it has teeth (fails on the pre-fix code). Full suite **348 passed**;
|
||||
ruff + mypy --strict clean.
|
||||
- **Live idempotency confirmed (tunnel up):** re-ingested
|
||||
`10.1007/s00454-019-00132-8` against the Jetson DB → no `UniqueViolation`;
|
||||
`IngestResult(paper_id='10.1007/s00454-019-00132-8', citations_upserted=45)`.
|
||||
After-state: **UPDATE in place** — `id` and `openalex_id` unchanged, **`added_at`
|
||||
unchanged** (proves no fresh INSERT), exactly **1** row for the DOI (no URL-form
|
||||
duplicate), citations 45→45, total papers 29→29.
|
||||
- **Unblocks R-A (GROBID re-ingest) and R-C (.tex re-ingest)** — both re-ingest
|
||||
existing papers. **Files touched:** `codex/sources/openalex.py`,
|
||||
`tests/sources/test_openalex.py`, `tests/ingest/test_ingest.py`.
|
||||
- **Note (out of scope, pre-existing):** arXiv papers stored as bare ids
|
||||
(`2305.10988`) are still not round-trip idempotent if OpenAlex returns an
|
||||
arXiv-DOI/W-id instead of the bare arXiv id — a separate canonicalization
|
||||
concern, not this UniqueViolation. The DOI path (this finding) is fixed.
|
||||
|
||||
**Original finding (for context):**
|
||||
- **Symptom:** re-ingesting an existing DOI paper crashes with
|
||||
`UniqueViolation: papers_openalex_id_key`. Surfaced trying to re-ingest
|
||||
`10.1007/s00454-019-00132-8` during the DQ-2 backfill.
|
||||
- **Root cause:** `openalex._map_paper` sets `Paper.id = data["doi"]`, which is the
|
||||
**full URL** `https://doi.org/10.1007/…`. The stored canonical id is the **bare**
|
||||
`10.1007/…` (post the M-1 canonical-id migration). So `INSERT … ON CONFLICT (id)`
|
||||
finds no id match, attempts a fresh INSERT, and trips the *separate* unique
|
||||
constraint on `openalex_id` (already held by the canonical-id row).
|
||||
- **Confirmed:** `openalex.fetch_paper("10.1007/s00454-019-00132-8").id ==
|
||||
"https://doi.org/10.1007/s00454-019-00132-8"` (≠ stored bare id).
|
||||
- **Impact:** any re-ingest of a DOI paper aborts. This **blocks roadmap R-A
|
||||
(GROBID on PDFs) and R-C (.tex ingest)** — both re-ingest existing papers. Also
|
||||
implies *new* DOI ingests store full-URL ids, inconsistent with the bare-id
|
||||
corpus (latent split-identity risk).
|
||||
- **Proposed fix (handle with care — M-1 incident territory):** normalize the DOI
|
||||
in `_map_paper` to the canonical bare, lower-cased form (strip
|
||||
`https://doi.org/`), matching what the M-1 migration produced. Add a re-ingest
|
||||
idempotency test over a DOI paper. Did **not** fix inline this session — the
|
||||
canonical-id derivation is exactly where the M-1 migration incident occurred, so
|
||||
it needs its own focused change + a live re-ingest check.
|
||||
- **Workaround used for DQ-2:** targeted `UPDATE … SET abstract, abstract_emb`
|
||||
instead of full re-ingest.
|
||||
|
||||
---
|
||||
|
||||
## Roadmap — data-acquisition levers (post-audit, all free sources)
|
||||
|
||||
These are *forward-looking acquisition improvements*, distinct from the DQ-1..DQ-4
|
||||
audit findings (which assessed existing data). They came out of a strategy
|
||||
discussion on 2026-06-16: for this Math/CS corpus the valuable data (references,
|
||||
abstracts, full text) is almost entirely **open** (arXiv, OpenAlex, Crossref, S2),
|
||||
so the quality lever is *combining more free sources*, not buying paywall access.
|
||||
Paywall/uni-login was explicitly considered and **deferred** (see R-E). Each item
|
||||
is self-contained so a cold session can pick it up.
|
||||
|
||||
### R-A — Run GROBID reference extraction on PDFs · **CORE DONE 2026-06-17** (full sweep optional)
|
||||
|
||||
**Resolution (what was done) — citing coverage 27/29 → 29/29 (100%); edges 920 → 1022:**
|
||||
- **References-only backfill, not full re-ingest.** Added
|
||||
[scripts/ra_grobid_backfill.py](../../scripts/ra_grobid_backfill.py): per paper,
|
||||
run `grobid.extract_references(pdf)`, normalize cited-ids to the canonical bare
|
||||
form (DOIs de-URL'd + lower-cased; `arXiv:` stripped), INSERT into `citations`
|
||||
(`ON CONFLICT DO NOTHING`). Deliberately avoids `ingest_paper(source_path=pdf)`,
|
||||
whose PDF branch re-runs Nougat + `DELETE FROM chunks` and would overwrite the
|
||||
DQ-3-verified-clean `.txt` chunks (and needs Nougat). Idempotent; dry-run by
|
||||
default; 7 tests; ruff/mypy clean.
|
||||
- **GROBID provisioning:** the pinned `grobid/grobid:0.8.2` is amd64-only, but the
|
||||
Jetson and the dev Mac are both arm64 (this is why GROBID never ran). Ran the
|
||||
lighter CRF image (`grobid/grobid:0.8.2-crf`) on the Mac under Podman+Rosetta
|
||||
(PDFs are local; citations written over the SSH DB tunnel). Made
|
||||
`extract_references` timeout configurable (theses need >60s emulated). A separate
|
||||
task is provisioning a native arm64 GROBID on the Jetson (`docs/infra/grobid-jetson.md`).
|
||||
- **Both `depositonce` theses closed:** `…depositonce-20357` (lutz-2024) **+96**
|
||||
edges (5 in-corpus); `…depositonce-5415` (sechelmann-2016) **+6** edges — its
|
||||
147 MB/173-page scan blew pdfalto's 120s limit, so the 9 bibliography pages were
|
||||
extracted with PyMuPDF first (77 refs, only 6 with a DOI/arXiv). **No paper has
|
||||
zero out-edges now.**
|
||||
- **Optional remaining (secondary acceptance):** sweep the other 27 already-covered
|
||||
papers for *extra* GROBID refs via `python scripts/ra_grobid_backfill.py --write`
|
||||
(the 2 book PDFs need the same bib-page extraction as sechelmann).
|
||||
|
||||
**Original plan (for context):**
|
||||
- **✓ Unblocked (DQ-5 RESOLVED 2026-06-17):** DOI re-ingest is now idempotent, so
|
||||
the GROBID re-ingest no longer aborts on `papers_openalex_id_key`.
|
||||
- **What:** the whole corpus was ingested from `.txt` (`PAPERS_DIR=…/papers/txt`),
|
||||
so the GROBID PDF path never ran. Re-ingest with a PDF `source_path` per paper
|
||||
so `codex.parsing.grobid.extract_references` parses each bibliography.
|
||||
- **Why:** extracts references the APIs lack — a *complement* to DQ-1's S2
|
||||
supplement. Crucially, the **2 `depositonce` theses** that stayed at zero in
|
||||
DQ-1 (no API references anywhere) almost certainly have a references section in
|
||||
their open-access PDFs → GROBID could finally give them out-edges. Closes the
|
||||
last 2/29.
|
||||
- **How:** no new code — ingest already merges + dedups `api_citations +
|
||||
pdf_citations` ([codex/ingest.py:288](../../codex/ingest.py)). GROBID server is
|
||||
already configured (`GROBID_URL=http://192.168.178.103:8070` in
|
||||
`.env.jetson-ingest`). Download free arXiv PDFs, re-ingest with `source_path`.
|
||||
- **Acceptance:** the 2 theses gain references; published papers gain
|
||||
GROBID-parsed refs beyond OpenAlex/S2 coverage; citing coverage 27/29 → 29/29.
|
||||
|
||||
### R-B — Add Crossref as a third citation + abstract source · **DONE 2026-06-17**
|
||||
|
||||
**Resolution (what was done):**
|
||||
- **`crossref.fetch_references(doi)`** added (mirrors `semanticscholar.fetch_references`):
|
||||
GETs `/works/{doi}`, returns one `Citation` per `message.reference[]` entry that
|
||||
carries a registered `DOI` (book/`unstructured`-only refs are skipped — they
|
||||
can't join the graph). `citing_id` is the queried DOI; the ingest supplement
|
||||
rewrites it to `paper.id`.
|
||||
- **Wired as the third leg of the citation fallback chain** in `ingest.py`:
|
||||
OpenAlex-empty → S2 → **Crossref** (`_crossref_reference_supplement`, DOI papers
|
||||
only — Crossref's works endpoint is DOI-keyed). cited DOIs pass through
|
||||
`_norm_cited_id` (bare, lower-cased). Network/parse failures degrade to [].
|
||||
- **Tests:** `fetch_references` unit tests (DOI edges / `unstructured` skipped / no
|
||||
`reference` key / 404) + ingest tests asserting the Crossref leg fires when
|
||||
OpenAlex+S2 are empty (cited DOI normalised) **and** does *not* fire when OpenAlex
|
||||
has citations. Suite **361 green**, ruff/mypy clean. The DQ-2 abstract half
|
||||
(`fetch_abstract`) already shipped earlier.
|
||||
- **Live-validated:** `fetch_references` against the real API returned 33 / 40 / 24
|
||||
DOI-bearing refs for `10.1007/s00454-019-00132-8` / `10.1145/3132705` /
|
||||
`10.1111/cgf.13931` (Springer / ACM / Wiley).
|
||||
- **No live corpus change (by design):** as a *fallback*, Crossref fires only when
|
||||
OpenAlex **and** S2 are both empty. Post-DQ-1/R-A the only such papers were the 2
|
||||
`depositonce` theses, which Crossref doesn't index either — so R-B is a
|
||||
forward-looking third leg for future DOI ingests, not a backfill. (Harvesting
|
||||
Crossref's *unique* refs for already-covered papers would need an always-merged
|
||||
supplement instead — a deliberate scope choice not taken, per the doc's "fallback
|
||||
chain" spec.) **Files:** `codex/sources/crossref.py`, `codex/ingest.py`, + tests.
|
||||
|
||||
**Original plan (for context):**
|
||||
- **What:** new `codex.sources.crossref` client hitting Crossref REST
|
||||
`/works/{doi}` for the `reference` array and `abstract`.
|
||||
- **Why:** Crossref carries **publisher-deposited references** for many DOIs (free,
|
||||
polite pool via `mailto`) — a third leg after OpenAlex/S2. It also recovers
|
||||
abstracts OpenAlex is contractually barred from redistributing (likely the 2
|
||||
Springer DOIs missing abstracts in DQ-2). cited_ids are DOIs → flow through the
|
||||
existing graph resolver unchanged.
|
||||
- **How:** mirror `codex/sources/openalex.py` (tenacity retry, polite pool). Extend
|
||||
the ingest fallback chain to OpenAlex-empty → S2 → Crossref (same
|
||||
`_s2_reference_supplement` shape). Add unit tests with mocked HTTP.
|
||||
- **Acceptance:** Crossref recovers refs and/or an abstract for ≥1 paper where
|
||||
OpenAlex+S2 are empty.
|
||||
|
||||
### R-C — Prefer arXiv `.tex` source over `.txt` for chunk fidelity · **DONE 2026-06-17**
|
||||
|
||||
**Resolution (what was done) — all 13 arXiv papers re-ingested from `.tex`:**
|
||||
- **Multi-file flatten:** `arxiv.fetch_source` now inlines `\input`/`\include`
|
||||
(`tex.flatten_inputs`) so multi-file projects yield the full body, not the
|
||||
primary file's include skeleton (e.g. math/0603097 = 15 files / 12 includes;
|
||||
2305.10988 = 6 / 5). Also handles legacy single-file **bare-gzipped** `.tex`
|
||||
(no tar wrapper) for old math/* papers (commit 121b582).
|
||||
- **Section-aware chunking:** new `tex.chunk_sections` returns `(title, chunk)`
|
||||
pairs (chunks never span a `\section`); the ingest `.tex` path labels the stored
|
||||
`section` from the real heading instead of running `classify_section` on a
|
||||
header-less word-window. Quality-gated as pairs so labels stay aligned.
|
||||
- **Unblocked by the arXiv half of DQ-5** (commit f504ca6): `_canonical_id`
|
||||
strips the `10.48550/arxiv.` DOI prefix so an arXiv paper's id is the bare
|
||||
arXiv id — without it, re-ingest tripped `papers_openalex_id_key` (confirmed
|
||||
live, then fixed; idempotent re-ingest verified).
|
||||
- **Outcome:** `section` column gained signal — ~34/329 arXiv chunks now
|
||||
intro/theorem/proof (was ~all `body`); math/0603097 `{body:31}`→`{body:29,
|
||||
proof:9}`, 2310.17529 gained proof+intro, etc. Math papers with descriptively
|
||||
titled sections stay `body` (classify_section's fixed vocab) — modest but real
|
||||
(R-12). No paper gutted (chunk counts stable/▲); F-16 gate unchanged.
|
||||
- **Files:** `codex/parsing/tex.py`, `codex/sources/arxiv.py`, `codex/ingest.py`,
|
||||
`codex/sources/openalex.py`, `scripts/rc_tex_reingest.py` (driver, dry-run
|
||||
default), + tests. Full suite **370 passed**; ruff/mypy clean. Branch
|
||||
`feat/tex-ingest` (commits 1da81c7, 121b582, f504ca6).
|
||||
|
||||
**Original plan (for context):**
|
||||
- **What:** re-ingest from arXiv LaTeX source through the existing
|
||||
`codex.parsing.tex.latex_to_text` path instead of flattened `.txt`.
|
||||
- **Why:** highest-fidelity input — preserves math, structure, and section
|
||||
boundaries. Directly serves **DQ-3** (content fidelity) and audit **R-12** (the
|
||||
`section` column is low-signal because word-window `.txt` chunks rarely start at
|
||||
a real header; `.tex` headers fix this).
|
||||
- **How:** download free arXiv source tarballs, feed `.tex` to ingest with
|
||||
`source_path`. Handle multi-file projects (main `.tex` + includes).
|
||||
- **Acceptance:** `section` column gains signal (fewer `body`-only); DQ-3 coverage
|
||||
vs source improves; no regression in the F-16 quality gate.
|
||||
|
||||
### R-D — F-15: warn on low *citing-paper* coverage, not just paper count · **DONE 2026-06-17**
|
||||
|
||||
**Resolution (what was done):**
|
||||
- Added `graph.citing_coverage(graph, known_ids)` → `(papers_with_out_edges,
|
||||
total)`, a `graph_min_citing_coverage` setting (default 0.8), and wired both into
|
||||
`codex graph report`: it now prints `Citing coverage: N/M papers with out-edges
|
||||
(P%)` (and in `--json`), and emits a `Warning` when the share is below the
|
||||
threshold ("low citing coverage weakens PageRank/coupling"). Separate from the
|
||||
existing `graph_min_corpus_size` (total-count) warning.
|
||||
- Tests: `citing_coverage` unit tests + CLI tests (line shown; warning fires at
|
||||
low coverage; suppressed at 100%; JSON field). Full suite green; ruff/mypy clean.
|
||||
- **Live:** `codex graph report` on the corpus prints `Citing coverage: 29/29
|
||||
papers with out-edges (100%)` with no warning (post-R-A). Branch
|
||||
`feat/graph-coverage-warning`. Files: `codex/config.py`, `codex/graph.py`,
|
||||
`codex/cli.py`, + tests.
|
||||
|
||||
**Original plan (for context):**
|
||||
- **What:** the open DQ-1 sub-item: `graph_min_corpus_size` only flags total paper
|
||||
count. Add a warning when the share of papers with ≥1 out-edge is low (e.g.
|
||||
< 80%), since that is what actually starves PageRank/coupling.
|
||||
- **Why:** at 17/29 the graph silently ran on 59% of the corpus with no signal.
|
||||
Post-DQ-1 it's 27/29 (93%) — but the guard should catch future regressions.
|
||||
- **Acceptance:** `codex graph report` surfaces citing-coverage % and warns below
|
||||
threshold.
|
||||
|
||||
### R-E — Paywall / institutional access · **DEFERRED — only for a paywall-only expansion**
|
||||
- **Decision (2026-06-16):** a uni login does **not** help the current corpus —
|
||||
every DQ-1/DQ-2 gap is in openly-available works (arXiv preprints, OA theses),
|
||||
and all metadata/citation sources are free APIs. Revisit *only* if the corpus
|
||||
expands to papers that exist solely behind a paywall with no preprint; then
|
||||
institutional access → full-text PDF → GROBID refs (R-A) + clean chunks (R-C).
|
||||
- **ToS caveat:** manual download of individual papers you have legitimate access
|
||||
to is fine; **automated bulk download through a uni proxy (EZproxy/Shibboleth)
|
||||
violates most publisher terms and can get the whole institution's access
|
||||
revoked.** Do not wire a publisher login into the ingest pipeline.
|
||||
|
||||
### R-F — Richer section labels: store real `\section` titles · **DONE 2026-06-17**
|
||||
|
||||
**Resolution (what was done):**
|
||||
- **Why:** R-C labelled `.tex` chunks by mapping the real `\section` heading through
|
||||
`classify_section`'s fixed vocab, so descriptively-titled Math sections collapsed
|
||||
to `body` (only **34/329 = 10 %** non-`body`). A research pass confirmed the
|
||||
`section` column is **write-only** (nothing filters on the vocab — `mcp_server.py`
|
||||
and `wiki.py` don't even SELECT it), so storing free-text titles is safe.
|
||||
- **Change:** new `quality.section_label(title, content)` (+ `_clean_title`) — keep the
|
||||
controlled bucket (intro/theorem/proof/abstract/bibliography) when the heading maps
|
||||
to one, else store the cleaned real title ("the flip algorithm", "main results", …).
|
||||
The `.tex` ingest path uses it; `.pdf`/`.txt`/`run_quality_pass` keep
|
||||
`classify_section` unchanged. `_clean_title` strips LaTeX commands/accents/ties +
|
||||
leading numbering, lower-cases, truncates at a word boundary. ~1 helper + 1 ingest
|
||||
line; no schema change (column already free `TEXT`).
|
||||
- **Result (live):** re-applied to the 13 arXiv papers → non-`body` **34/329 (10 %) →
|
||||
314/329 (95 %)**. Only `math/0306167` (old AMS-TeX, no `\section{}`) stays `body`.
|
||||
- **Tests:** `section_label` / `_clean_title` units + `.tex` ingest stores a
|
||||
descriptive heading verbatim. Full suite 377 passed; ruff/mypy clean. Branch
|
||||
`feat/section-labels` (off `feat/tex-ingest`).
|
||||
- **Accent re-clean — DONE 2026-06-17.** The `_clean_title` polish (`m\"obius`→`mobius`,
|
||||
`~` ties, word-boundary truncation) landed in code, and the existing live labels were
|
||||
re-cleaned in place (apply `_clean_title` to any `chunks.section` containing `\` or
|
||||
`~`; **8 labels** fixed, no re-embed). Verified: **0** labels with LaTeX artifacts
|
||||
remain; non-`body` holds at **314/329 (95 %)**. R-F fully complete.
|
||||
|
||||
---
|
||||
|
||||
## Priority for the next session
|
||||
1. **DQ-1** (biggest lever — 41 % of the corpus is invisible to the graph; the S2
|
||||
supplement is concrete and reuses existing code).
|
||||
2. **DQ-4** (cheap, high-information — tells you if the KB is actually usable).
|
||||
3. **DQ-2**, then **DQ-3**.
|
||||
1. ~~**DQ-1**~~ — **DONE 2026-06-16** (live DB at 920 citations, 27/29 citing
|
||||
coverage; remedy wired into ingest).
|
||||
2. ~~**DQ-4**~~ — **DONE 2026-06-16** (P0 `search paper` crash fixed; relevance
|
||||
verified strong; surfaced the zero-vector pollution that motivates DQ-2).
|
||||
3. ~~**DQ-2**~~ — **DONE 2026-06-16** (1911.00966 fully recovered; s00454 abstract
|
||||
from S2; book chapter documented; recovery wired into ingest).
|
||||
4. ~~**DQ-3**~~ — **DONE 2026-06-16** (content fidelity clean; full coverage, only
|
||||
18 OCR-junk chunks dropped, no math lost). **Assessment axis DQ-1..DQ-4 complete.**
|
||||
5. ~~**DQ-5**~~ — **DONE 2026-06-17** (DOI id normalized to bare form in
|
||||
`openalex._map_paper`; live re-ingest of `10.1007/s00454-019-00132-8` idempotent
|
||||
— UPDATE in place, `added_at` unchanged; suite 348 green). **Unblocks R-A/R-C.**
|
||||
6. **Roadmap levers** (see section above — now all unblocked): **R-A** (GROBID on
|
||||
arXiv PDFs — closes the last 2 zero-citation theses), **R-B** (Crossref refs —
|
||||
abstract half already shipped in DQ-2), **R-C** (`.tex` ingest — serves DQ-3),
|
||||
**R-D** (F-15 coverage warning, quick). **R-E** (paywall) deferred.
|
||||
|
||||
All four are *data* work (queries + maybe a re-ingest/enrichment), not code-audit
|
||||
work — the code is already remediated (see the AUDIT-* docs and PRs #12–#14).
|
||||
DQ-1..DQ-4 are *data*/assessment work; DQ-5 + roadmap R-A..R-D add **new code**
|
||||
(canonical-id fix, re-ingest scripts, Crossref references, a `.tex` path, a graph
|
||||
warning) on top of the already-remediated pipeline (see the AUDIT-* docs and PRs
|
||||
#12–#14). The DQ-1/DQ-2 ingest wiring + Crossref/arXiv abstract recovery already
|
||||
landed this session.
|
||||
|
||||
---
|
||||
|
||||
|
||||
192
docs/infra/grobid-jetson.md
Normal file
192
docs/infra/grobid-jetson.md
Normal file
@@ -0,0 +1,192 @@
|
||||
# GROBID on the Jetson (aarch64) — deploy runbook
|
||||
|
||||
**Status:** ACTIVE
|
||||
**Host:** `alfred@192.168.178.103` — NVIDIA Jetson Orin Nano (aarch64, 8 GB RAM, 8 GB swap, 6 cores)
|
||||
**Service:** GROBID reference extraction for roadmap **R-A** (`scripts/ra_grobid_backfill.py`)
|
||||
**Last updated:** 2026-06-17
|
||||
**Verified:** 2026-06-17 — `grobid/grobid:0.9.0-crf` pulled as native `arm64`, `/api/isalive`→`true`,
|
||||
end-to-end `extract_references` on `lutz-2024-thesis.pdf` → 116 refs (101 with DOI/arXiv) in ~36s.
|
||||
|
||||
---
|
||||
|
||||
## Why this image (`grobid/grobid:0.9.0-crf`)
|
||||
|
||||
GROBID had never come up on the Jetson because `infra/docker-compose.yml` pinned
|
||||
`grobid/grobid:0.8.2`, and **every `0.8.x` tag (and the full `0.9.0`) is published
|
||||
amd64-only**. A Docker tag resolves to a multi-arch *manifest list*; on aarch64
|
||||
the `0.8.2` list has no `linux/arm64` entry, so there is simply nothing to pull or
|
||||
run (hence `curl localhost:8070` → connection refused, nothing listening).
|
||||
|
||||
The fix is to select a tag whose manifest list actually contains a `linux/arm64`
|
||||
build:
|
||||
|
||||
| Candidate | arm64? | Size | Notes |
|
||||
|-------------------------------|:------:|--------:|----------------------------------------|
|
||||
| `grobid/grobid:0.8.2` | ✗ | ~9.5 GB | full DL image, amd64-only (old pin) |
|
||||
| `grobid/grobid:0.9.0` / `-full`| ✗ | ~10 GB | full DL image, amd64-only |
|
||||
| **`grobid/grobid:0.9.0-crf`** | **✓** | ~500 MB | **CRF-only, official, multi-arch** ← |
|
||||
| `lfoppiano/grobid:0.9.0-crf` | ✓ | ~500 MB | same build, community namespace (alt.) |
|
||||
|
||||
We use **`grobid/grobid:0.9.0-crf`**:
|
||||
|
||||
- **CRF-only** is sufficient for reference extraction (`/api/processReferences`)
|
||||
and is ~500 MB vs ~10 GB for the deep-learning image.
|
||||
- **Native arm64** — no qemu emulation (too slow / RAM-heavy on a Jetson).
|
||||
- **Official namespace**, **pinned version** (not `latest-crf`, which is a moving
|
||||
target). arm64 builds only exist from `0.9.0-crf` onward — there is no arm64
|
||||
`0.8.x`, so this is a forced (but compatible) bump from the old `0.8.2` pin. The
|
||||
TEI that `codex/parsing/grobid.py` parses (`biblStruct`, `persName`,
|
||||
`idno[@type='DOI']`, …) is unchanged across the bump.
|
||||
|
||||
> Caveat from upstream: the arm64 image is documented as "tested only on macOS,
|
||||
> not linux/arm64". The container is the *same* `linux/arm64` ELF either way, so
|
||||
> bare-metal Jetson works; the end-to-end check below is what de-risks it.
|
||||
|
||||
---
|
||||
|
||||
## One-time prerequisites
|
||||
|
||||
### 1. Docker permission for `alfred`
|
||||
|
||||
The SSH user `alfred` is in the `sudo` group but **not** the `docker` group, so
|
||||
`docker …` fails with `permission denied … /var/run/docker.sock`. Grant access
|
||||
once (requires the sudo password):
|
||||
|
||||
```bash
|
||||
ssh alfred@192.168.178.103
|
||||
sudo usermod -aG docker alfred # add to docker group (persistent)
|
||||
```
|
||||
|
||||
Then **start a fresh login** so the new group takes effect (group membership is
|
||||
only re-evaluated at login — an existing session/`ssh` won't see it; a *new*
|
||||
`ssh` connection will):
|
||||
|
||||
```bash
|
||||
exit && ssh alfred@192.168.178.103
|
||||
docker ps # should now work without sudo
|
||||
```
|
||||
|
||||
(Alternatively run all `docker` commands below under `sudo`, but group membership
|
||||
is cleaner and lets `ra_grobid_backfill.py` / curl checks be driven over plain SSH.)
|
||||
|
||||
### 2. Docker Compose v2 plugin (arm64)
|
||||
|
||||
This host has Docker Engine but **not** the Compose v2 CLI plugin (on Linux they
|
||||
are separate packages; only Docker Desktop bundles Compose). Without it
|
||||
`docker compose …` fails with `docker: unknown command: docker compose`. Install
|
||||
the arm64 plugin into the user-local plugin dir (no sudo needed):
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.docker/cli-plugins
|
||||
curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-aarch64 \
|
||||
-o ~/.docker/cli-plugins/docker-compose
|
||||
chmod +x ~/.docker/cli-plugins/docker-compose
|
||||
docker compose version # confirm it resolves
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Deploy
|
||||
|
||||
GROBID is **service-scoped** on purpose: Postgres (`papers-db`) is already running
|
||||
on this host, so we bring up only the `grobid` service and never `docker compose up`
|
||||
the whole stack (that would try to start a second `papers-db`).
|
||||
|
||||
```bash
|
||||
# copy this repo's compose file to the Jetson (first time only)
|
||||
scp infra/docker-compose.yml infra/schema.sql alfred@192.168.178.103:~/codex-infra/
|
||||
|
||||
ssh alfred@192.168.178.103
|
||||
cd ~/codex-infra
|
||||
docker compose up -d grobid # pulls grobid/grobid:0.9.0-crf (arm64), starts papers-grobid
|
||||
docker compose logs -f grobid # wait for "Started ... in N seconds" (model load ~20–40s)
|
||||
```
|
||||
|
||||
The `deploy.resources.limits.memory: 4g` cap in the compose file is honored by
|
||||
Compose v2 here (no swarm needed). 3 GB suffices for references; 4 GB leaves
|
||||
headroom alongside Postgres. The `restart: "no"` policy means GROBID does **not**
|
||||
auto-start on boot — see "Run on-demand" below.
|
||||
|
||||
---
|
||||
|
||||
## Run on-demand (stop when idle)
|
||||
|
||||
GROBID is only needed **during reference extraction** (`scripts/ra_grobid_backfill.py`);
|
||||
nothing else in the stack uses it. Idle it still holds ~3.5 GB, so on the 8 GB Jetson
|
||||
keep it stopped and start it only for an R-A run:
|
||||
|
||||
```bash
|
||||
ssh alfred@192.168.178.103 'docker start papers-grobid' # ~30s model load
|
||||
curl -s --retry 30 --retry-delay 2 --retry-connrefused \
|
||||
http://192.168.178.103:8070/api/isalive # wait for: true
|
||||
|
||||
# ... run R-A from the Mac ...
|
||||
PYTHONPATH=. python scripts/ra_grobid_backfill.py --write
|
||||
|
||||
ssh alfred@192.168.178.103 'docker stop papers-grobid' # reclaim ~3.5 GB
|
||||
```
|
||||
|
||||
First-time deploy uses `docker compose up -d grobid` (above); afterwards the
|
||||
container persists, so `docker start/stop papers-grobid` is all you need.
|
||||
|
||||
> Why on-demand rather than a lighter tool: refextract was evaluated head-to-head
|
||||
> on the Jetson and rejected — far worse DOI recall on this math corpus (lutz
|
||||
> thesis: GROBID **101** usable IDs vs refextract **2**), and it was *slower* per
|
||||
> PDF despite a smaller footprint. GROBID stays; running it on-demand removes its
|
||||
> only real downside (idle RAM).
|
||||
|
||||
---
|
||||
|
||||
## Verify `GET /api/isalive` → `true`
|
||||
|
||||
```bash
|
||||
# on the Jetson
|
||||
curl -s http://localhost:8070/api/isalive # => true
|
||||
|
||||
# from the Mac (LAN) — same value the corpus' GROBID_URL points at
|
||||
curl -s http://192.168.178.103:8070/api/isalive # => true
|
||||
```
|
||||
|
||||
Anything other than `true` (timeout / connection refused / `false`) means the
|
||||
container isn't up — check `docker compose logs grobid` and that nothing else
|
||||
holds port 8070.
|
||||
|
||||
---
|
||||
|
||||
## End-to-end reference-extraction check + running R-A
|
||||
|
||||
With `GROBID_URL=http://192.168.178.103:8070` (already set in `.env.jetson-ingest`):
|
||||
|
||||
```bash
|
||||
# one PDF, references only — should print N>0 parsed references
|
||||
PYTHONPATH=. python - <<'PY'
|
||||
from codex.parsing.grobid import extract_references
|
||||
refs = extract_references("/Users/tarikmoussa/Desktop/ConformalLabpp/papers/lutz-2024-thesis.pdf",
|
||||
grobid_url="http://192.168.178.103:8070")
|
||||
print(f"{len(refs)} references; first DOI/arXiv:",
|
||||
next(((r['doi'], r['arxiv_id']) for r in refs if r['doi'] or r['arxiv_id']), None))
|
||||
PY
|
||||
```
|
||||
|
||||
Then run the references-only backfill (DB tunnel up — see
|
||||
`docs/audit/DATA-QUALITY-2026-06-15.md`):
|
||||
|
||||
```bash
|
||||
ssh -N -L 5433:localhost:5432 alfred@192.168.178.103 & # DB tunnel
|
||||
cp .env.jetson-ingest .env
|
||||
PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only # dry-run first
|
||||
PYTHONPATH=. python scripts/ra_grobid_backfill.py --write # apply
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **Container killed mid-request / exit 137** — OOM. The CRF image needs ~3 GB for
|
||||
references; raise the compose `memory` limit or check Postgres isn't starving the
|
||||
box (`free -h`, `docker stats papers-grobid`).
|
||||
- **`isalive` true but `processReferences` 500s** — usually a `pdfalto` failure on a
|
||||
malformed PDF; confirm with a clean PDF (`lutz-2024-thesis.pdf`) and check
|
||||
`docker compose logs grobid`.
|
||||
- **`permission denied … docker.sock`** — the docker-group prerequisite above wasn't
|
||||
applied, or the SSH session predates it (re-login).
|
||||
66
docs/infra/jetson-ssh-stability.md
Normal file
66
docs/infra/jetson-ssh-stability.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Infra TODO — stable long-running ops against the Jetson (tmux + autossh)
|
||||
|
||||
**Status:** OPEN (infra). **Filed:** 2026-06-17. **Priority:** MED — it slows live ops
|
||||
but has a known workaround (short/idempotent ops).
|
||||
|
||||
## Problem (observed)
|
||||
|
||||
Live data ops run on the **Mac** and reach the Jetson Postgres through an SSH
|
||||
port-forward tunnel (`ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103`,
|
||||
`DATABASE_URL` → `localhost:5433`). During the R-A / R-C / R-F work this tunnel
|
||||
**dropped repeatedly mid-run**: any operation longer than a few minutes (corpus
|
||||
re-ingest, the bge-m3 embed loop, the GROBID sweep) had a high chance of failing
|
||||
partway when the tunnel died, leaving the work half-applied. At one point the Jetson
|
||||
went **fully offline** (ping 100 % loss, `ssh:22` timeout) mid-operation.
|
||||
|
||||
`ingest_paper()` and the backfill scripts open one DB connection for the whole run, so
|
||||
a tunnel drop raises `OperationalError: connection refused` and aborts — partial state,
|
||||
manual retry. Adding `ServerAliveInterval` to the tunnel helped only marginally.
|
||||
|
||||
## Recommended fix
|
||||
|
||||
**1. Run long ops *on the Jetson* inside `tmux` (primary fix).**
|
||||
The Jetson is where Postgres (and GROBID) live, so running ingest there means the DB is
|
||||
`localhost` — **no tunnel at all** — and `tmux` keeps the process alive across SSH
|
||||
disconnects (detach / reattach to monitor). This removes the tunnel from the critical
|
||||
path for every long-running write.
|
||||
|
||||
- [ ] Confirm the `codex` env is deployable on the Jetson (Python 3.12+, deps incl. the
|
||||
bge-m3 embedder; the Orin Nano GPU should handle it — GROBID already runs there).
|
||||
If not, stand up a venv / container for it.
|
||||
- [ ] Standard pattern: `ssh alfred@jetson`, then
|
||||
`tmux new -s codex` → `set -a; source .env; set +a` →
|
||||
`PYTHONPATH=. python scripts/<job>.py --write` → detach (`Ctrl-b d`).
|
||||
Reattach later with `tmux attach -t codex`. The job survives the SSH drop.
|
||||
- [ ] Point the Jetson-side `DATABASE_URL` at `localhost:5432` directly (no `:5433`).
|
||||
- [ ] Document this as the default for any multi-minute write (re-ingest, sweeps,
|
||||
embed loops) in `docs/audit/DATA-QUALITY-2026-06-15.md` "How to reach the data".
|
||||
|
||||
**2. `autossh` for when a tunnel *is* needed (secondary).**
|
||||
For interactive/dev use from the Mac (psql, quick probes, read-only queries), replace
|
||||
the plain `ssh -f -N -L` with `autossh -M 0 -f -N -o ServerAliveInterval=15 -o
|
||||
ServerAliveCountMax=3 -L 5433:localhost:5432 alfred@jetson` so a dropped tunnel
|
||||
auto-reconnects. NB: autossh only restarts the *tunnel* — an in-flight transaction
|
||||
still fails, so this is for short/interactive use, not long writes (use tmux-on-Jetson
|
||||
for those).
|
||||
|
||||
**3. (Optional, code) make backfill scripts resilient.**
|
||||
Lower-value once (1)/(2) are in place, but: open a fresh `get_conn()` per paper (not one
|
||||
for the whole run) and/or wrap the per-item write in a small retry, so a transient drop
|
||||
costs one item, not the whole batch. The R-A backfill already does per-paper connections;
|
||||
`rc_tex_reingest.py` opens a connection per paper too — but `ingest_paper` itself holds
|
||||
one connection for its whole body, which is the unit that fails.
|
||||
|
||||
## Why this matters
|
||||
|
||||
The chronic flakiness directly cost time this session and left one cosmetic task
|
||||
unfinished (the R-F accent re-clean — see DATA-QUALITY-2026-06-15.md R-F "Pending"). The
|
||||
**robust pattern that already works** is short, idempotent operations (per-paper, or a
|
||||
single quick `UPDATE`); tmux-on-Jetson makes even the long ops safe.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- A documented, repeatable way to run a full corpus re-ingest that survives an SSH drop
|
||||
(tmux-on-Jetson), verified by detaching/reattaching across a disconnect.
|
||||
- The DATA-QUALITY doc's "How to reach the data" section recommends tmux-on-Jetson for
|
||||
long writes and `autossh` for interactive tunnels.
|
||||
@@ -28,20 +28,34 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
grobid:
|
||||
# CRF-only image is sufficient for reference extraction and is lighter than
|
||||
# the full deeplearning variant. Check https://hub.docker.com/r/grobid/grobid
|
||||
# for the latest 0.8.x tag before deploying.
|
||||
image: grobid/grobid:0.8.2
|
||||
# CRF-only image: sufficient for reference extraction (/api/processReferences)
|
||||
# and ~500MB vs ~10GB for the full deep-learning variant.
|
||||
#
|
||||
# The `-crf` tag is also the ONLY GROBID variant published as a multi-arch
|
||||
# manifest that includes linux/arm64 — the full image (grobid/grobid:0.9.0)
|
||||
# and every 0.8.x tag are amd64-only. So this is what lets GROBID run
|
||||
# *natively* on the aarch64 Jetson (no slow/RAM-heavy qemu emulation). arm64
|
||||
# builds start at 0.9.0-crf; there is no arm64 0.8.x.
|
||||
# Deploy + docker-permission fix: docs/infra/grobid-jetson.md
|
||||
image: grobid/grobid:0.9.0-crf
|
||||
container_name: papers-grobid
|
||||
ports:
|
||||
- "8070:8070"
|
||||
restart: unless-stopped
|
||||
# GROBID is RAM-hungry; uncomment to cap usage on constrained hardware
|
||||
# (e.g. Nvidia Jetson):
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# memory: 4g
|
||||
init: true # reap zombie children (GROBID docs recommend --init)
|
||||
ulimits:
|
||||
core: 0 # disable core dumps (GROBID docs: --ulimit core=0)
|
||||
# On-demand only: GROBID is needed solely during reference extraction
|
||||
# (scripts/ra_grobid_backfill.py), not for normal corpus operations — start it
|
||||
# for an R-A run, stop it after to reclaim ~3.5GB. "no" = no auto-start on boot.
|
||||
# See docs/infra/grobid-jetson.md ("Run on-demand").
|
||||
restart: "no"
|
||||
# GROBID is RAM-hungry; cap usage on the constrained Jetson (Orin Nano, 8GB
|
||||
# RAM shared with Postgres). 3GB suffices for references; 4g leaves headroom.
|
||||
# Honored by `docker compose up` under Compose v2 (no swarm needed).
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4g
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
1
scripts/__init__.py
Normal file
1
scripts/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Operational one-off scripts (importable for tests)."""
|
||||
227
scripts/ra_grobid_backfill.py
Normal file
227
scripts/ra_grobid_backfill.py
Normal file
@@ -0,0 +1,227 @@
|
||||
"""R-A — GROBID reference backfill (references-only, idempotent).
|
||||
|
||||
Adds GROBID-parsed citations to the live corpus **without** re-chunking the body,
|
||||
so the DQ-3-verified-clean ``.txt`` chunks are preserved. Complements the DQ-1
|
||||
Semantic-Scholar supplement with references the APIs lack (notably the two
|
||||
TU-Berlin ``depositonce`` theses, which neither OpenAlex nor S2 indexes).
|
||||
|
||||
Why references-only instead of ``ingest_paper(source_path=pdf)``
|
||||
----------------------------------------------------------------
|
||||
The PDF branch of :func:`codex.ingest.ingest_paper` runs Nougat OCR and then
|
||||
``DELETE FROM chunks`` + re-INSERT — it would replace the clean ``.txt`` chunks
|
||||
(DQ-3) with unmeasured PDF-OCR text and additionally requires the Nougat server.
|
||||
This backfill touches only the ``citations`` table (``ON CONFLICT DO NOTHING``),
|
||||
so it is safe, idempotent, and needs only a reachable GROBID server.
|
||||
|
||||
Prerequisite
|
||||
------------
|
||||
A reachable GROBID server. The corpus' ``GROBID_URL`` points at the Jetson
|
||||
(``http://192.168.178.103:8070``). GROBID now runs natively on that aarch64 host
|
||||
via the multi-arch CRF-only image (``grobid/grobid:0.9.0-crf``) — see
|
||||
``docs/infra/grobid-jetson.md`` for the deploy + docker-permission fix and the
|
||||
``GET /api/isalive`` check. (The old ``grobid/grobid:0.8.2`` pin was amd64-only,
|
||||
which is why GROBID never came up; pass ``--grobid-url`` to override the default.)
|
||||
Use the SSH DB tunnel for ``DATABASE_URL`` as documented in
|
||||
``docs/audit/DATA-QUALITY-2026-06-15.md``.
|
||||
|
||||
Usage
|
||||
-----
|
||||
# dry run (no writes) over the two zero-edge theses
|
||||
PYTHONPATH=. python scripts/ra_grobid_backfill.py --zero-edge-only
|
||||
|
||||
# write every paper's GROBID-parsed refs to the live DB
|
||||
PYTHONPATH=. python scripts/ra_grobid_backfill.py --write
|
||||
|
||||
Default is **dry-run**; pass ``--write`` to mutate the live DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
import psycopg.rows
|
||||
|
||||
from codex.config import get_settings
|
||||
from codex.db import get_conn
|
||||
from codex.models import Citation
|
||||
from codex.parsing.grobid import extract_references
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_PDF_DIR = "/Users/tarikmoussa/Desktop/ConformalLabpp/papers"
|
||||
|
||||
|
||||
def _normalize_cited_id(doi: str = "", arxiv_id: str = "") -> str | None:
|
||||
"""Normalize a GROBID-extracted reference id to the corpus' canonical form.
|
||||
|
||||
DOIs → bare, lower-cased (strip ``https://doi.org/`` / ``http://`` / ``doi:``),
|
||||
matching ``papers.id`` and ``codex.sources.openalex._normalize_doi`` (DQ-5).
|
||||
arXiv ids → strip a leading ``arXiv:`` prefix, leaving the bare id
|
||||
(``2305.10988``, ``math/0603097``). DOI takes precedence when both exist.
|
||||
Kept self-contained rather than reusing ``ingest._norm_cited_id`` because it
|
||||
additionally strips the ``arXiv:`` prefix (which GROBID can emit but that
|
||||
helper leaves untouched), and to avoid importing a private cross-module name.
|
||||
"""
|
||||
doi = (doi or "").strip()
|
||||
if doi:
|
||||
s = doi.lower()
|
||||
for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
|
||||
if s.startswith(prefix):
|
||||
return s[len(prefix) :]
|
||||
return s
|
||||
arx = (arxiv_id or "").strip()
|
||||
if arx:
|
||||
return arx[len("arxiv:") :].strip() if arx.lower().startswith("arxiv:") else arx
|
||||
return None
|
||||
|
||||
|
||||
def build_grobid_citations(
|
||||
paper_id: str,
|
||||
pdf_path: str | Path,
|
||||
grobid_url: str | None = None,
|
||||
timeout: float = 60.0,
|
||||
) -> list[Citation]:
|
||||
"""Extract a paper's references via GROBID and return normalized Citations.
|
||||
|
||||
``citing_id`` is the canonical ``paper_id``; ``cited_id`` is normalized.
|
||||
Self-citations and within-paper duplicates are dropped. Refs with neither a
|
||||
DOI nor an arXiv id are skipped (they cannot join the graph).
|
||||
"""
|
||||
refs = extract_references(str(pdf_path), grobid_url=grobid_url, timeout=timeout)
|
||||
seen: set[str] = set()
|
||||
citations: list[Citation] = []
|
||||
for ref in refs:
|
||||
cited = _normalize_cited_id(ref.get("doi", ""), ref.get("arxiv_id", ""))
|
||||
if not cited or cited == paper_id or cited in seen:
|
||||
continue
|
||||
seen.add(cited)
|
||||
citations.append(Citation(citing_id=paper_id, cited_id=cited))
|
||||
return citations
|
||||
|
||||
|
||||
def _coverage(conn: psycopg.Connection[psycopg.rows.DictRow]) -> tuple[int, int, int]:
|
||||
"""Return (n_papers, n_papers_with_out_edges, n_citations) from the live DB."""
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT (SELECT count(*) FROM papers) AS papers,
|
||||
(SELECT count(DISTINCT citing_id) FROM citations) AS citing,
|
||||
(SELECT count(*) FROM citations) AS edges
|
||||
"""
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
return int(row["papers"]), int(row["citing"]), int(row["edges"])
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--pdf-dir", default=DEFAULT_PDF_DIR, help="directory of source PDFs")
|
||||
parser.add_argument("--grobid-url", default=None, help="override Settings().grobid_url")
|
||||
parser.add_argument("--only", default=None, help="restrict to a single paper id")
|
||||
parser.add_argument(
|
||||
"--zero-edge-only",
|
||||
action="store_true",
|
||||
help="only papers with 0 out-edges (the R-A primary targets)",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=None, help="cap number of papers")
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=300.0,
|
||||
help="GROBID HTTP timeout (s); theses can need >60s, esp. emulated",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--write", action="store_true", help="WRITE to the live DB (default: dry-run)"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
grobid_url = args.grobid_url or get_settings().grobid_url
|
||||
pdf_dir = Path(args.pdf_dir)
|
||||
pdfs = {p.stem: p for p in pdf_dir.glob("*.pdf")}
|
||||
|
||||
# Read the target list in a short-lived connection, then release it: the GROBID
|
||||
# extraction below is a slow per-paper network loop and must NOT hold an
|
||||
# idle-in-transaction connection open across it (a mid-loop SSH-tunnel drop would
|
||||
# otherwise abort the whole batch — see docs/infra/jetson-ssh-stability.md).
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT p.id, p.source_path,
|
||||
(SELECT count(*) FROM citations ci WHERE ci.citing_id = p.id) AS out_edges
|
||||
FROM papers p
|
||||
ORDER BY out_edges, p.id
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
targets: list[tuple[str, Path]] = []
|
||||
for r in rows:
|
||||
if args.only and r["id"] != args.only:
|
||||
continue
|
||||
if args.zero_edge_only and r["out_edges"] != 0:
|
||||
continue
|
||||
stem = Path(r["source_path"]).stem if r["source_path"] else None
|
||||
pdf = pdfs.get(stem) if stem else None
|
||||
if pdf is None:
|
||||
logger.warning("no PDF for %s (stem=%s) — skipping", r["id"], stem)
|
||||
continue
|
||||
targets.append((r["id"], pdf))
|
||||
if args.limit is not None:
|
||||
targets = targets[: args.limit]
|
||||
|
||||
logger.info(
|
||||
"GROBID=%s | papers to process=%d | mode=%s",
|
||||
grobid_url,
|
||||
len(targets),
|
||||
"WRITE" if args.write else "DRY-RUN",
|
||||
)
|
||||
|
||||
# GROBID extraction — no DB connection held during this slow network loop.
|
||||
all_citations: list[Citation] = []
|
||||
for paper_id, pdf in targets:
|
||||
try:
|
||||
cits = build_grobid_citations(
|
||||
paper_id, pdf, grobid_url=grobid_url, timeout=args.timeout
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("GROBID extraction failed for %s (%s): %s", paper_id, pdf.name, exc)
|
||||
continue
|
||||
logger.info("%-38s %3d refs (%s)", paper_id, len(cits), pdf.name)
|
||||
all_citations.extend(cits)
|
||||
|
||||
logger.info("total candidate citations: %d", len(all_citations))
|
||||
|
||||
if not args.write:
|
||||
logger.info("DRY-RUN — no rows written. Re-run with --write to apply.")
|
||||
return 0
|
||||
|
||||
# Fresh connection for the write — the read connection was already released.
|
||||
with get_conn() as conn:
|
||||
before = _coverage(conn)
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO citations (citing_id, cited_id, context)
|
||||
VALUES (%s, %s, %s)
|
||||
ON CONFLICT (citing_id, cited_id) DO NOTHING
|
||||
""",
|
||||
[(c.citing_id, c.cited_id, c.context) for c in all_citations],
|
||||
)
|
||||
conn.commit()
|
||||
after = _coverage(conn)
|
||||
logger.info(
|
||||
"coverage papers=%d citing %d->%d edges %d->%d (+%d)",
|
||||
after[0],
|
||||
before[1],
|
||||
after[1],
|
||||
before[2],
|
||||
after[2],
|
||||
after[2] - before[2],
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
131
scripts/rc_tex_reingest.py
Normal file
131
scripts/rc_tex_reingest.py
Normal file
@@ -0,0 +1,131 @@
|
||||
"""R-C — re-ingest arXiv papers from LaTeX source (section-aware, higher fidelity).
|
||||
|
||||
For each arXiv paper, download its source via :func:`codex.sources.arxiv.fetch_source`
|
||||
(multi-file ``\\input``/``\\include`` flattened), write it to a temp ``.tex``, and
|
||||
re-ingest through :func:`codex.ingest.ingest_paper`. The ``.tex`` ingest path is
|
||||
section-aware (:func:`codex.parsing.tex.chunk_sections`), so the stored ``section``
|
||||
column is labelled from real ``\\section`` headings instead of mostly ``body`` —
|
||||
serving DQ-3 (fidelity vs the OCR'd ``.txt``) and audit R-12 (weak ``section`` column).
|
||||
|
||||
Only arXiv papers have downloadable source; DOI/journal papers and books are skipped.
|
||||
Re-ingest REPLACES the paper's chunks (DELETE + re-INSERT) — that is the intended R-C
|
||||
improvement (flattened ``.txt`` → clean ``.tex`` chunks). It is idempotent and
|
||||
**dry-run by default**; pass ``--write`` to mutate the live corpus.
|
||||
|
||||
Usage
|
||||
-----
|
||||
# dry-run: list arXiv papers + source availability (no writes)
|
||||
PYTHONPATH=. python scripts/rc_tex_reingest.py
|
||||
|
||||
# re-ingest every arXiv paper from .tex
|
||||
PYTHONPATH=. python scripts/rc_tex_reingest.py --write
|
||||
|
||||
# one paper
|
||||
PYTHONPATH=. python scripts/rc_tex_reingest.py --only 2305.10988 --write
|
||||
|
||||
Needs ``DATABASE_URL`` (Jetson tunnel) as documented in
|
||||
``docs/audit/DATA-QUALITY-2026-06-15.md``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import tempfile
|
||||
|
||||
import psycopg
|
||||
import psycopg.rows
|
||||
|
||||
from codex.db import get_conn
|
||||
from codex.ingest import ingest_paper
|
||||
from codex.sources import arxiv
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_arxiv_id(paper_id: str) -> bool:
|
||||
"""Return True for arXiv ids (the only papers with downloadable LaTeX source).
|
||||
|
||||
arXiv ids are modern (``2305.10988``) or legacy (``math/0603097``). DOIs start
|
||||
with ``10.``; OpenAlex W-ids start with ``W`` or an ``https://openalex.org/`` URL.
|
||||
"""
|
||||
p = (paper_id or "").strip()
|
||||
# Exclude DOIs (incl. the arXiv DOI 10.48550/arXiv.*), OpenAlex W-ids, and any
|
||||
# http(s) URL form; everything else is treated as a bare arXiv id.
|
||||
return (
|
||||
bool(p)
|
||||
and not p.startswith(("10.", "W"))
|
||||
and not p.lower().startswith(("http://", "https://"))
|
||||
)
|
||||
|
||||
|
||||
def _section_dist(conn: psycopg.Connection[psycopg.rows.DictRow], paper_id: str) -> dict[str, int]:
|
||||
"""Return the section-label distribution for a paper's stored chunks."""
|
||||
rows = conn.execute(
|
||||
"SELECT coalesce(section, '(null)') AS s, count(*) AS n "
|
||||
"FROM chunks WHERE paper_id = %s GROUP BY 1 ORDER BY 2 DESC",
|
||||
(paper_id,),
|
||||
).fetchall()
|
||||
return {r["s"]: int(r["n"]) for r in rows}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--only", default=None, help="restrict to a single arXiv id")
|
||||
parser.add_argument("--limit", type=int, default=None, help="cap number of papers")
|
||||
parser.add_argument(
|
||||
"--write", action="store_true", help="re-ingest into the live DB (default: dry-run)"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute("SELECT id FROM papers ORDER BY id").fetchall()
|
||||
targets = [
|
||||
r["id"]
|
||||
for r in rows
|
||||
if is_arxiv_id(r["id"]) and (args.only is None or r["id"] == args.only)
|
||||
]
|
||||
if args.limit is not None:
|
||||
targets = targets[: args.limit]
|
||||
|
||||
logger.info(
|
||||
"arXiv papers to process=%d | mode=%s",
|
||||
len(targets),
|
||||
"WRITE" if args.write else "DRY-RUN",
|
||||
)
|
||||
|
||||
for pid in targets:
|
||||
try:
|
||||
src = arxiv.fetch_source(pid)
|
||||
except Exception as exc:
|
||||
logger.warning("%s: source fetch failed: %s", pid, exc)
|
||||
continue
|
||||
if not src:
|
||||
logger.warning("%s: no LaTeX source available — skipping", pid)
|
||||
continue
|
||||
if not args.write:
|
||||
logger.info("%s: source OK (%d chars)", pid, len(src))
|
||||
continue
|
||||
|
||||
with get_conn() as conn:
|
||||
before = _section_dist(conn, pid)
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".tex", delete=True) as tf:
|
||||
tf.write(src)
|
||||
tf.flush()
|
||||
res = ingest_paper(pid, source_path=tf.name)
|
||||
with get_conn() as conn:
|
||||
after = _section_dist(conn, pid)
|
||||
logger.info(
|
||||
"%s: chunks %d->%d | section before=%s after=%s",
|
||||
pid,
|
||||
sum(before.values()),
|
||||
res.chunks_upserted,
|
||||
before,
|
||||
after,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -106,6 +106,13 @@ class TestSearch:
|
||||
assert "Test Paper" in result.stdout
|
||||
assert "0.123" in result.stdout
|
||||
|
||||
# Regression guard (DQ-4): the pgvector `<->` operator requires the
|
||||
# query embedding cast to ::vector. Without it, real Postgres raises
|
||||
# "operator does not exist: vector <-> double precision[]". Mocked DBs
|
||||
# don't catch the type error, so assert the cast is in the SQL.
|
||||
executed_sql = mock_conn.execute.call_args_list[0][0][0]
|
||||
assert "<->" in executed_sql and "::vector" in executed_sql
|
||||
|
||||
def test_search_formula_command(self) -> None:
|
||||
"""Test `search formula` command with mocked DB."""
|
||||
with patch("codex.db.get_conn") as mock_get_conn:
|
||||
|
||||
@@ -119,6 +119,36 @@ class TestGraphReport:
|
||||
assert result.exit_code == 0
|
||||
assert "Warning" in result.output or "warning" in result.output
|
||||
|
||||
def test_shows_citing_coverage_line(self):
|
||||
# _sample_graph: A, B, C all cite → 3/3 = 100%, so the line shows but no warning.
|
||||
out = self._run(known_ids=("A", "B", "C")).output
|
||||
assert "Citing coverage" in out
|
||||
assert "weakens" not in out # 100% coverage → no R-D warning
|
||||
|
||||
def test_low_citing_coverage_warning(self):
|
||||
# Only A has an out-edge; B and C are ingested but cite nothing → 1/3 = 33%.
|
||||
g = nx.DiGraph()
|
||||
g.add_edge("A", "X")
|
||||
g.add_nodes_from(["B", "C"])
|
||||
result = self._run(graph=g, known_ids=("A", "B", "C"))
|
||||
assert result.exit_code == 0
|
||||
assert "weakens PageRank" in result.output # the R-D coverage warning
|
||||
|
||||
def test_json_includes_citing_coverage(self):
|
||||
import json
|
||||
|
||||
g = nx.DiGraph()
|
||||
g.add_edge("A", "X")
|
||||
g.add_nodes_from(["B", "C"])
|
||||
conn = _make_conn_with_paper_ids("A", "B", "C")
|
||||
with (
|
||||
patch("codex.db.get_conn", side_effect=_make_conn_cm(conn)),
|
||||
patch("codex.graph.build_citation_graph", return_value=g),
|
||||
):
|
||||
result = runner.invoke(app, ["graph", "report", "--json"])
|
||||
data = json.loads(result.output)
|
||||
assert data["citing_coverage"] == {"citing": 1, "papers": 3, "fraction": 0.3333}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# codex graph related
|
||||
|
||||
@@ -10,6 +10,7 @@ import pytest
|
||||
from codex.graph import (
|
||||
build_citation_graph,
|
||||
citation_pagerank,
|
||||
citing_coverage,
|
||||
dangling_citations,
|
||||
find_co_cited,
|
||||
find_related,
|
||||
@@ -287,3 +288,24 @@ class TestDanglingCitations:
|
||||
g = build_citation_graph(_make_conn(rows))
|
||||
dangling = dangling_citations(g, set())
|
||||
assert set(dangling) == {"A", "B"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# citing_coverage (R-D)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCitingCoverage:
|
||||
def test_counts_papers_with_out_edges(self):
|
||||
# _small_graph: A→.., B→.., C→D have out-edges; D, E have none.
|
||||
g = _small_graph()
|
||||
n_citing, n_papers = citing_coverage(g, {"A", "B", "C", "D", "E"})
|
||||
assert (n_citing, n_papers) == (3, 5)
|
||||
|
||||
def test_paper_absent_from_graph_counts_as_no_out_edges(self):
|
||||
# A cites; Z was ingested but has no chunks/edges, so it isn't a graph node.
|
||||
g = _small_graph()
|
||||
assert citing_coverage(g, {"A", "Z"}) == (1, 2)
|
||||
|
||||
def test_empty(self):
|
||||
assert citing_coverage(nx.DiGraph(), set()) == (0, 0)
|
||||
|
||||
@@ -10,6 +10,7 @@ from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import psycopg
|
||||
import pytest
|
||||
@@ -107,6 +108,60 @@ def test_ingest_paper_basic() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_citation_fallback_to_crossref() -> None:
|
||||
"""R-B: OpenAlex + S2 both empty → Crossref references supply the edges.
|
||||
|
||||
Crossref is the third leg of the fallback chain. Its cited DOIs are
|
||||
case-normalised and citing_id is rewritten to the canonical paper.id.
|
||||
"""
|
||||
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
# Upper-case suffix proves the cited DOI is normalised to bare lower-case.
|
||||
crossref_refs = [Citation(citing_id=paper.id, cited_id="10.1090/S0002-9947-04-03545-7")]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]), # OpenAlex empty
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]), # S2 empty
|
||||
patch("codex.ingest.crossref.fetch_references", return_value=crossref_refs) as mock_cr,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper(paper.id)
|
||||
|
||||
# Crossref consulted with the bare DOI; one edge upserted.
|
||||
mock_cr.assert_called_once_with(paper.id)
|
||||
assert result.citations_upserted == 1
|
||||
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
||||
cit_rows = mock_cursor.executemany.call_args_list[0][0][1]
|
||||
assert cit_rows[0][0] == paper.id
|
||||
assert cit_rows[0][1] == "10.1090/s0002-9947-04-03545-7"
|
||||
|
||||
|
||||
def test_ingest_crossref_not_called_when_openalex_has_citations() -> None:
|
||||
"""R-B: the Crossref leg fires only when OpenAlex+S2 are empty (it's a fallback)."""
|
||||
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W123")
|
||||
mock_conn = MagicMock()
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch(
|
||||
"codex.ingest.openalex.fetch_citations",
|
||||
return_value=[Citation(citing_id="W123", cited_id="https://openalex.org/W9")],
|
||||
),
|
||||
patch("codex.ingest.crossref.fetch_references") as mock_cr,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
ingest_paper(paper.id)
|
||||
|
||||
mock_cr.assert_not_called()
|
||||
|
||||
|
||||
def test_ingest_canonicalises_paper_id_to_caller_arg() -> None:
|
||||
"""papers.id is the caller-supplied id, not OpenAlex's URL-form id (audit C-7).
|
||||
|
||||
@@ -144,7 +199,8 @@ def test_ingest_canonicalises_paper_id_to_caller_arg() -> None:
|
||||
|
||||
|
||||
def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
"""``.tex`` source → latex_to_text, chunk_text, embed, chunks inserted in DB."""
|
||||
"""``.tex`` source → section-aware chunking; the stored ``section`` column is
|
||||
labelled from the real ``\\section`` heading, not the chunk body (R-C / R-12)."""
|
||||
tex_file = tmp_path / "paper.tex"
|
||||
tex_file.write_text(r"\section{Introduction} Hello world. This is a test paper.")
|
||||
|
||||
@@ -154,16 +210,20 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
fake_chunks = ["chunk one text here.", "chunk two text here."]
|
||||
# (section title, chunk content) pairs from the section-aware chunker.
|
||||
section_pairs = [
|
||||
("Introduction", "Body of the introduction goes here."),
|
||||
("Proof of the Main Theorem", "Body of the proof goes here."),
|
||||
("3.2 Discrete Conformal Maps", "Body about discrete conformal maps."),
|
||||
]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
patch("codex.parsing.tex.latex_to_text", return_value="Hello world. Intro text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
patch("codex.parsing.tex.chunk_sections", return_value=section_pairs),
|
||||
patch("codex.ingest.is_quality_chunk", return_value=True),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(tex_file))
|
||||
|
||||
@@ -178,7 +238,16 @@ def test_ingest_paper_source_tex(tmp_path: Any) -> None:
|
||||
chunk_sql: str = chunk_insert_call[0][0]
|
||||
assert "INSERT INTO chunks" in chunk_sql
|
||||
|
||||
assert result.chunks_upserted == len(fake_chunks)
|
||||
# R-F: a bare canonical heading → bucket ("Introduction"→intro); any other
|
||||
# heading is stored as its cleaned real title ("proof of the main theorem",
|
||||
# "discrete conformal maps") instead of being collapsed/mislabelled.
|
||||
chunk_rows = chunk_insert_call[0][1]
|
||||
assert [row[4] for row in chunk_rows] == [
|
||||
"intro",
|
||||
"proof of the main theorem",
|
||||
"discrete conformal maps",
|
||||
]
|
||||
assert result.chunks_upserted == len(section_pairs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -206,12 +275,15 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
# OpenAlex empty → ingest now consults the S2 reference supplement (DQ-1);
|
||||
# mock it empty so only the GROBID refs contribute to the count.
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=fake_chunks),
|
||||
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
||||
patch("codex.ingest.filter_chunks", side_effect=lambda chunks, settings: chunks),
|
||||
patch("codex.ingest.is_quality_chunk", return_value=True),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
||||
|
||||
@@ -227,6 +299,73 @@ def test_ingest_paper_source_pdf(tmp_path: Any) -> None:
|
||||
assert mock_cursor.executemany.call_count >= 2
|
||||
|
||||
|
||||
def test_ingest_pdf_grobid_citations_normalize_doi(tmp_path: Any) -> None:
|
||||
"""R-A guard: GROBID extracts the DOI verbatim from the PDF reference text, so
|
||||
it may be mixed-case, a ``https://doi.org/…`` URL, or ``doi:``-prefixed. Each
|
||||
must be normalized to the bare, lower-cased canonical form before the Citation
|
||||
is built — otherwise the same reference splits into case-/URL-variant graph
|
||||
nodes that never join the bare-lowercase ``papers.id``. arXiv-only refs (no
|
||||
DOI) are left untouched. Mirrors the S2-supplement normalization (DQ-1)."""
|
||||
pdf_file = tmp_path / "paper.pdf"
|
||||
pdf_file.write_bytes(b"%PDF-1.4 fake content")
|
||||
|
||||
paper = _make_paper()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
|
||||
# Faithful to GROBID's output shape: all five keys always present.
|
||||
return {"title": "", "authors": "", "year": "", "doi": doi, "arxiv_id": arxiv_id}
|
||||
|
||||
grobid_refs = [
|
||||
_ref(doi="10.1007/S00454-019-00132-8"), # mixed-case bare DOI
|
||||
_ref(doi="https://doi.org/10.1145/AbC.123"), # URL-form DOI
|
||||
_ref(doi="doi:10.1/MixedCase"), # doi:-prefixed DOI
|
||||
_ref(arxiv_id="2101.00001"), # arXiv-only ref → untouched
|
||||
]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
# OpenAlex empty → S2 supplement consulted (DQ-1); empty so only GROBID
|
||||
# refs contribute to the citation rows under inspection.
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
patch("codex.parsing.nougat.pdf_to_markdown", return_value="pdf markdown text."),
|
||||
patch("codex.parsing.tex.chunk_text", return_value=["pdf chunk one."]),
|
||||
patch("codex.parsing.grobid.extract_references", return_value=grobid_refs),
|
||||
patch("codex.ingest.is_quality_chunk", return_value=True),
|
||||
):
|
||||
result = ingest_paper(paper.id, source_path=str(pdf_file))
|
||||
|
||||
assert result.citations_upserted == 4
|
||||
|
||||
# Locate the citations INSERT among the cursor's executemany calls (chunks
|
||||
# are also inserted via executemany, so filter by SQL rather than by index).
|
||||
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
||||
citation_calls = [
|
||||
c for c in mock_cursor.executemany.call_args_list if "INSERT INTO citations" in c[0][0]
|
||||
]
|
||||
assert len(citation_calls) == 1
|
||||
inserted_rows: list[tuple[str, str, Any]] = citation_calls[0][0][1]
|
||||
|
||||
# Every edge hangs off the canonical paper.id (citing side).
|
||||
assert all(row[0] == paper.id for row in inserted_rows)
|
||||
|
||||
cited_ids = {row[1] for row in inserted_rows}
|
||||
assert cited_ids == {
|
||||
"10.1007/s00454-019-00132-8", # mixed-case → lower-cased
|
||||
"10.1145/abc.123", # URL prefix stripped + lower-cased
|
||||
"10.1/mixedcase", # doi: prefix stripped + lower-cased
|
||||
"2101.00001", # arXiv id untouched
|
||||
}
|
||||
# No raw URL-form / prefixed DOI leaked into the citation graph.
|
||||
assert not any(cid.startswith(("http", "doi:")) for cid in cited_ids)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. test_ingest_paper_not_found
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -279,7 +418,7 @@ def test_ingest_paper_idempotent() -> None:
|
||||
|
||||
|
||||
def test_ingest_paper_arxiv_s2_fallback() -> None:
|
||||
"""If OpenAlex returns None for an arXiv ID, S2 fallback produces a stub Paper."""
|
||||
"""OpenAlex None + arXiv has nothing → stub Paper; S2 references still consulted."""
|
||||
paper_id = "2301.07041"
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
@@ -288,17 +427,121 @@ def test_ingest_paper_arxiv_s2_fallback() -> None:
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=None),
|
||||
# arXiv + abstract sources also empty → genuine stub fallback path.
|
||||
patch("codex.ingest.arxiv.fetch_metadata", return_value=None) as mock_arxiv,
|
||||
patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
|
||||
patch("codex.ingest.crossref.fetch_abstract", return_value=None),
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]) as mock_s2,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper(paper_id)
|
||||
|
||||
# S2 was consulted as fallback
|
||||
# arXiv metadata recovery attempted; S2 references consulted as fallback.
|
||||
mock_arxiv.assert_called_once_with(paper_id)
|
||||
mock_s2.assert_called()
|
||||
assert result.paper_id == paper_id
|
||||
|
||||
|
||||
def test_ingest_paper_openalex_404_recovers_from_arxiv() -> None:
|
||||
"""DQ-2: OpenAlex 404 on an arXiv id → authoritative metadata from the arXiv API,
|
||||
bibkey auto-generated, abstract embedded (no zero vector)."""
|
||||
recovered = Paper(
|
||||
id="1911.00966",
|
||||
title="A discrete version of Liouville's theorem on conformal maps",
|
||||
authors=["Ulrich Pinkall", "Boris Springborn"],
|
||||
year=2019,
|
||||
abstract="Liouville's theorem says that in dimension greater than two...",
|
||||
)
|
||||
fake_emb = _fake_embedder()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=None),
|
||||
patch("codex.ingest.arxiv.fetch_metadata", return_value=recovered) as mock_arxiv,
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=fake_emb),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper("1911.00966")
|
||||
|
||||
mock_arxiv.assert_called_once_with("1911.00966")
|
||||
assert result.paper_id == "1911.00966"
|
||||
params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
||||
assert params["title"].startswith("A discrete version")
|
||||
assert params["bibkey"] == "PinkallSpringborn2019" # generated from recovered authors+year
|
||||
# abstract present → real embedding computed, not a zero vector
|
||||
fake_emb.encode_dense.assert_called_once()
|
||||
|
||||
|
||||
def test_ingest_paper_empty_abstract_recovered_from_s2() -> None:
|
||||
"""DQ-2: OpenAlex has the paper but no abstract → supplemented from S2 before embedding."""
|
||||
paper = _make_paper(abstract=None) # has openalex_id, no abstract
|
||||
fake_emb = _fake_embedder()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
||||
patch(
|
||||
"codex.ingest.semanticscholar.fetch_abstract",
|
||||
return_value="Recovered abstract text.",
|
||||
) as mock_abs,
|
||||
patch("codex.ingest.get_embedder", return_value=fake_emb),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
ingest_paper(paper.id)
|
||||
|
||||
mock_abs.assert_called_once()
|
||||
params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
||||
assert params["abstract"] == "Recovered abstract text."
|
||||
fake_emb.encode_dense.assert_called_once_with(["Recovered abstract text."])
|
||||
|
||||
|
||||
def test_ingest_paper_openalex_empty_uses_s2_supplement() -> None:
|
||||
"""DQ-1: OpenAlex returns an empty reference list → ingest supplements from S2,
|
||||
rewriting citing_id to paper.id and lowercasing DOI cited-ids."""
|
||||
paper = _make_paper() # openalex_id="W123", id="2301.07041"
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
# S2 keys citing_id by the id form passed in; ingest must canonicalise it.
|
||||
s2_refs = [
|
||||
Citation(citing_id="arXiv:2301.07041", cited_id="10.1/UPPER", context="ctx"),
|
||||
Citation(citing_id="arXiv:2301.07041", cited_id="2202.00002"),
|
||||
]
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=s2_refs) as mock_s2,
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper(paper.id)
|
||||
|
||||
# S2 consulted with a namespaced id (a bare arXiv id 404s on S2).
|
||||
mock_s2.assert_called_once_with("arXiv:2301.07041")
|
||||
assert result.citations_upserted == 2
|
||||
|
||||
mock_cursor = mock_conn.cursor.return_value.__enter__.return_value
|
||||
inserted_rows: list[tuple[str, str, Any]] = mock_cursor.executemany.call_args_list[0][0][1]
|
||||
# citing_id rewritten to paper.id, not the S2 id form.
|
||||
assert all(row[0] == paper.id for row in inserted_rows)
|
||||
cited = {row[1] for row in inserted_rows}
|
||||
assert "10.1/upper" in cited # DOI lowercased
|
||||
assert "2202.00002" in cited # arXiv id untouched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. test_ingest_paper_no_abstract_uses_zero_vector
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -317,6 +560,9 @@ def test_ingest_paper_no_abstract_uses_zero_vector() -> None:
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
# No abstract recoverable from any source (DQ-2) → genuine zero-vector case.
|
||||
patch("codex.ingest.semanticscholar.fetch_abstract", return_value=None),
|
||||
patch("codex.ingest.crossref.fetch_abstract", return_value=None),
|
||||
patch("codex.ingest.get_embedder", return_value=fake_emb),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
@@ -526,6 +772,93 @@ def test_ingest_sets_bibkey_when_none(tmp_path: Any) -> None:
|
||||
assert upsert_params["bibkey"] == "AliceBob2023"
|
||||
|
||||
|
||||
def test_ingest_doi_paper_upserts_bare_canonical_id() -> None:
|
||||
"""DQ-5 regression: re-ingesting an existing DOI paper must be idempotent.
|
||||
|
||||
OpenAlex returns the ``doi`` as a full URL (and possibly mixed-case), but the
|
||||
stored canonical ``papers.id`` is the BARE, lower-cased DOI (M-1 migration).
|
||||
The papers upsert must therefore carry the BARE id into ``%(id)s`` so
|
||||
``ON CONFLICT (id)`` matches the existing row and UPDATEs it — rather than
|
||||
attempting a fresh INSERT that trips the separate ``papers_openalex_id_key``
|
||||
unique constraint (the UniqueViolation crash).
|
||||
|
||||
The real ``_map_paper``/``fetch_paper`` run here (only the HTTP ``_get``, the
|
||||
embedder and the DB are mocked), so this exercises the actual normalization
|
||||
through the ingest path. The DB is a MagicMock, so it cannot raise the real
|
||||
constraint error; the live re-ingest of 10.1007/s00454-019-00132-8 is the
|
||||
end-to-end UniqueViolation proof (see DATA-QUALITY-2026-06-15.md, DQ-5).
|
||||
"""
|
||||
bare = "10.1007/s00454-019-00132-8"
|
||||
work = {
|
||||
"id": "https://openalex.org/W2899262408",
|
||||
# URL form + an upper-case suffix char — the worst case for ON CONFLICT.
|
||||
"doi": "https://doi.org/10.1007/S00454-019-00132-8",
|
||||
"title": "A Test Paper",
|
||||
"publication_year": 2019,
|
||||
"authorships": [{"author": {"display_name": "Alice Smith"}}],
|
||||
"abstract_inverted_index": {"Hello": [0], "world": [1]},
|
||||
# Non-empty so fetch_citations yields edges and the S2 fallback is skipped.
|
||||
"referenced_works": ["https://openalex.org/W1"],
|
||||
}
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"codex.ingest.openalex._get",
|
||||
return_value=httpx.Response(200, json=work),
|
||||
),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
result = ingest_paper(bare)
|
||||
|
||||
# The IngestResult and the upsert must use the bare canonical id, not the URL.
|
||||
assert result.paper_id == bare
|
||||
|
||||
upsert_sql: str = mock_conn.execute.call_args_list[0][0][0]
|
||||
upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
||||
assert "INSERT INTO papers" in upsert_sql
|
||||
assert "ON CONFLICT (id)" in upsert_sql
|
||||
assert upsert_params["id"] == bare, (
|
||||
f"upsert id must be the bare canonical DOI '{bare}', "
|
||||
f"not the OpenAlex URL form '{upsert_params['id']}'"
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_normalizes_non_bare_caller_id() -> None:
|
||||
"""Review #1: a non-bare CALLER id (URL-form / mixed-case DOI) is normalized to the
|
||||
bare canonical form before being pinned to papers.id (the C-7 pin runs it through
|
||||
_norm_cited_id), so a sloppy caller cannot store a URL-form papers.id that would
|
||||
defeat ON CONFLICT (id) idempotency or the ``paper.id.startswith('10.')`` gates."""
|
||||
paper = _make_paper(paper_id="10.1007/s00454-019-00132-8", openalex_id="W1")
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = MagicMock()
|
||||
mock_conn.executemany = MagicMock()
|
||||
mock_conn.commit = MagicMock()
|
||||
|
||||
with (
|
||||
patch("codex.ingest.openalex.fetch_paper", return_value=paper),
|
||||
patch("codex.ingest.openalex.fetch_citations", return_value=[]),
|
||||
patch("codex.ingest.semanticscholar.fetch_references", return_value=[]),
|
||||
patch("codex.ingest.crossref.fetch_references", return_value=[]),
|
||||
patch("codex.ingest.get_embedder", return_value=_fake_embedder()),
|
||||
patch("codex.ingest.get_conn", side_effect=_make_conn_cm(mock_conn)),
|
||||
):
|
||||
# Caller passes the URL form with an upper-case suffix char (worst case).
|
||||
result = ingest_paper("https://doi.org/10.1007/S00454-019-00132-8")
|
||||
|
||||
bare = "10.1007/s00454-019-00132-8"
|
||||
assert result.paper_id == bare
|
||||
upsert_params: dict[str, Any] = mock_conn.execute.call_args_list[0][0][1]
|
||||
assert upsert_params["id"] == bare, (
|
||||
f"non-bare caller must be normalized to '{bare}', got '{upsert_params['id']}'"
|
||||
)
|
||||
|
||||
|
||||
def test_ingest_disambiguates_bibkey_on_unique_violation() -> None:
|
||||
"""A papers.bibkey UNIQUE collision is retried with an a/b/c suffix (audit C-15)."""
|
||||
paper = _make_paper() # bibkey auto-generates to "AliceBob2023"
|
||||
|
||||
@@ -4,7 +4,13 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from codex.parsing.tex import chunk_text, extract_sections, latex_to_text
|
||||
from codex.parsing.tex import (
|
||||
chunk_sections,
|
||||
chunk_text,
|
||||
extract_sections,
|
||||
flatten_inputs,
|
||||
latex_to_text,
|
||||
)
|
||||
|
||||
|
||||
class TestExtractSections:
|
||||
@@ -110,3 +116,62 @@ class TestLatexToText:
|
||||
result = latex_to_text(latex)
|
||||
assert r"\cite" not in result
|
||||
assert "%" not in result
|
||||
|
||||
|
||||
class TestFlattenInputs:
|
||||
def test_inlines_input_and_include(self) -> None:
|
||||
files = {
|
||||
"main.tex": r"Start \input{sec/intro} mid \include{proof} end",
|
||||
"sec/intro.tex": "INTRO BODY",
|
||||
"proof.tex": "PROOF BODY",
|
||||
}
|
||||
out = flatten_inputs(files["main.tex"], files)
|
||||
assert "INTRO BODY" in out
|
||||
assert "PROOF BODY" in out
|
||||
assert r"\input" not in out and r"\include" not in out
|
||||
|
||||
def test_resolves_without_tex_suffix(self) -> None:
|
||||
# \input{intro} must resolve the archive member "intro.tex".
|
||||
files = {"main.tex": r"\input{intro}", "intro.tex": "BODY"}
|
||||
assert "BODY" in flatten_inputs(files["main.tex"], files)
|
||||
|
||||
def test_recursive_includes(self) -> None:
|
||||
files = {"a.tex": r"X \input{b}", "b.tex": r"Y \input{c}", "c.tex": "Z"}
|
||||
out = flatten_inputs(files["a.tex"], files)
|
||||
assert "X" in out and "Y" in out and "Z" in out
|
||||
|
||||
def test_unresolved_input_dropped(self) -> None:
|
||||
# A reference with no matching file is removed, not left as a directive.
|
||||
out = flatten_inputs(r"A \input{missing} B", {})
|
||||
assert r"\input" not in out
|
||||
assert "A" in out and "B" in out
|
||||
|
||||
def test_input_cycle_does_not_leak_directive(self) -> None:
|
||||
# a -> b -> a: the depth cap must strip the unresolved directive, not leak it.
|
||||
files = {"a.tex": r"AA \input{b}", "b.tex": r"BB \input{a}"}
|
||||
out = flatten_inputs(files["a.tex"], files)
|
||||
assert r"\input" not in out
|
||||
assert "AA" in out and "BB" in out
|
||||
|
||||
def test_dot_slash_prefix_resolves_without_overstrip(self) -> None:
|
||||
# "./intro" resolves intro.tex; lstrip must not eat a real leading dot/run.
|
||||
files = {"main.tex": r"\input{./intro}", "intro.tex": "BODY"}
|
||||
assert "BODY" in flatten_inputs(files["main.tex"], files)
|
||||
|
||||
|
||||
class TestChunkSections:
|
||||
def test_pairs_carry_section_titles(self) -> None:
|
||||
latex = (
|
||||
r"\section{Introduction}" + "\nIntro body text here.\n"
|
||||
r"\section{Proof}" + "\nProof body text here.\n"
|
||||
)
|
||||
pairs = chunk_sections(latex)
|
||||
titles = [t for t, _ in pairs]
|
||||
assert "Introduction" in titles
|
||||
assert "Proof" in titles
|
||||
# No chunk spans a section boundary: each chunk belongs to one title.
|
||||
assert all(isinstance(c, str) and c for _, c in pairs)
|
||||
|
||||
def test_fallback_titleless_when_no_sections(self) -> None:
|
||||
pairs = chunk_sections("Plain text with no section commands at all here.")
|
||||
assert pairs and all(title is None for title, _ in pairs)
|
||||
|
||||
@@ -6,10 +6,12 @@ from unittest.mock import MagicMock
|
||||
|
||||
from codex.quality import (
|
||||
_bib_score,
|
||||
_clean_title,
|
||||
classify_section,
|
||||
filter_chunks,
|
||||
is_quality_chunk,
|
||||
run_quality_pass,
|
||||
section_label,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -243,3 +245,58 @@ class TestRunQualityPass:
|
||||
kwargs = update_calls[0][0][1]
|
||||
assert kwargs["section"] == "abstract"
|
||||
assert kwargs["id"] == 5
|
||||
|
||||
|
||||
class TestCleanTitle:
|
||||
def test_strips_leading_numbering(self):
|
||||
assert _clean_title("3.2 Variational Principles") == "variational principles"
|
||||
|
||||
def test_strips_latex_and_braces(self):
|
||||
assert _clean_title(r"The \emph{Discrete} Laplacian") == "the discrete laplacian"
|
||||
|
||||
def test_collapses_and_lowercases(self):
|
||||
assert _clean_title(" Rigidity Results ") == "rigidity results"
|
||||
|
||||
def test_strips_accent_macros(self):
|
||||
assert _clean_title(r"M\"obius Invariance") == "mobius invariance"
|
||||
|
||||
def test_strips_ties_and_accents(self):
|
||||
assert _clean_title(r"Colin~de~Verdi\`ere") == "colin de verdiere"
|
||||
|
||||
def test_truncates_at_word_boundary(self):
|
||||
long = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu omega"
|
||||
out = _clean_title(long)
|
||||
assert len(out) <= 60
|
||||
assert not out.endswith(" ")
|
||||
assert long.lower().startswith(out) # whole words from the start, no partial tail
|
||||
|
||||
|
||||
class TestSectionLabel:
|
||||
def test_bare_canonical_heading_maps_to_bucket(self):
|
||||
# Only an EXACT canonical heading collapses to its controlled bucket.
|
||||
assert section_label("Introduction", "body text") == "intro"
|
||||
assert section_label("Proof", "body text") == "proof"
|
||||
assert section_label("References", "body text") == "bibliography"
|
||||
assert section_label("Abstract", "body text") == "abstract"
|
||||
|
||||
def test_descriptive_title_stored_verbatim(self):
|
||||
# R-F: a heading that merely STARTS with a keyword keeps its real title,
|
||||
# not the bucket — fixes classify_section prefix-match over-bucketing.
|
||||
assert section_label("Preliminaries", "body text") == "preliminaries"
|
||||
assert section_label("3.1 Discrete Conformal Maps", "x") == "discrete conformal maps"
|
||||
assert section_label("Proof of Theorem 3", "x") == "proof of theorem 3"
|
||||
assert section_label("Abstract Nonsense and Categories", "x") == (
|
||||
"abstract nonsense and categories"
|
||||
)
|
||||
assert section_label("Introduction to Operator Algebras", "x") == (
|
||||
"introduction to operator algebras"
|
||||
)
|
||||
|
||||
def test_none_or_blank_title_falls_back_to_content(self):
|
||||
# .pdf/.txt path: unchanged content-based classification.
|
||||
assert section_label(None, "Theorem 1. Let X be ...") == "theorem"
|
||||
assert section_label("", "ordinary body prose here") == "body"
|
||||
|
||||
def test_title_cleaning_to_empty_falls_back_to_content(self):
|
||||
# A pure-numbering heading cleans to "" → classify by content instead.
|
||||
assert section_label("3.2", "Lemma 2. Suppose ...") == "theorem"
|
||||
|
||||
0
tests/scripts/__init__.py
Normal file
0
tests/scripts/__init__.py
Normal file
112
tests/scripts/test_ra_grobid_backfill.py
Normal file
112
tests/scripts/test_ra_grobid_backfill.py
Normal file
@@ -0,0 +1,112 @@
|
||||
"""Tests for scripts.ra_grobid_backfill — the R-A GROBID references backfill.
|
||||
|
||||
Only the pure, offline logic is covered (id normalization + Citation assembly);
|
||||
``extract_references`` is mocked so no GROBID server or DB is required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from codex.models import Citation
|
||||
from scripts.ra_grobid_backfill import _normalize_cited_id, build_grobid_citations, main
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _normalize_cited_id
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_normalize_doi_url_to_bare_lowercase() -> None:
|
||||
assert _normalize_cited_id(doi="https://doi.org/10.1007/S00454-019-00132-8") == (
|
||||
"10.1007/s00454-019-00132-8"
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_doi_prefixes() -> None:
|
||||
assert _normalize_cited_id(doi="http://doi.org/10.1/X") == "10.1/x"
|
||||
assert _normalize_cited_id(doi="doi:10.1/X") == "10.1/x"
|
||||
assert _normalize_cited_id(doi="10.1/x") == "10.1/x"
|
||||
|
||||
|
||||
def test_normalize_arxiv_prefix_stripped() -> None:
|
||||
assert _normalize_cited_id(arxiv_id="arXiv:2305.10988") == "2305.10988"
|
||||
assert _normalize_cited_id(arxiv_id="2305.10988") == "2305.10988"
|
||||
# Legacy ids keep their slash.
|
||||
assert _normalize_cited_id(arxiv_id="math/0603097") == "math/0603097"
|
||||
|
||||
|
||||
def test_normalize_doi_takes_precedence_over_arxiv() -> None:
|
||||
assert _normalize_cited_id(doi="10.5/y", arxiv_id="2305.10988") == "10.5/y"
|
||||
|
||||
|
||||
def test_normalize_empty_returns_none() -> None:
|
||||
assert _normalize_cited_id() is None
|
||||
assert _normalize_cited_id(doi=" ", arxiv_id="") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_grobid_citations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ref(doi: str = "", arxiv_id: str = "") -> dict[str, str]:
|
||||
return {"title": "t", "authors": "a", "year": "2020", "doi": doi, "arxiv_id": arxiv_id}
|
||||
|
||||
|
||||
def test_build_citations_normalizes_dedups_and_drops_self() -> None:
|
||||
paper_id = "10.1007/s00454-019-00132-8"
|
||||
refs = [
|
||||
_ref(doi="https://doi.org/10.1/A"), # → 10.1/a
|
||||
_ref(doi="DOI:10.1/a"), # duplicate of the above after normalization
|
||||
_ref(arxiv_id="arXiv:2305.10988"), # → 2305.10988
|
||||
_ref(doi="", arxiv_id=""), # no id → skipped
|
||||
_ref(doi="https://doi.org/10.1007/S00454-019-00132-8"), # self-cite → dropped
|
||||
]
|
||||
with patch("scripts.ra_grobid_backfill.extract_references", return_value=refs) as mock_ex:
|
||||
cits = build_grobid_citations(paper_id, "some.pdf", grobid_url="http://grobid")
|
||||
|
||||
mock_ex.assert_called_once_with("some.pdf", grobid_url="http://grobid", timeout=60.0)
|
||||
assert all(isinstance(c, Citation) for c in cits)
|
||||
assert all(c.citing_id == paper_id for c in cits)
|
||||
assert [c.cited_id for c in cits] == ["10.1/a", "2305.10988"]
|
||||
|
||||
|
||||
def test_build_citations_empty_when_no_refs() -> None:
|
||||
with patch("scripts.ra_grobid_backfill.extract_references", return_value=[]):
|
||||
assert build_grobid_citations("10.1/x", "x.pdf") == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# main — connection scope (review finding #4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_main_uses_separate_connections_for_read_and_write(tmp_path: Any) -> None:
|
||||
"""Review #4: the read connection is released BEFORE the slow GROBID loop, and a
|
||||
fresh connection is opened for the write — so no idle-in-transaction connection
|
||||
spans the per-paper network loop (which would abort on an SSH-tunnel drop)."""
|
||||
(tmp_path / "paper.pdf").write_bytes(b"%PDF-1.4")
|
||||
conns: list[MagicMock] = []
|
||||
|
||||
@contextmanager
|
||||
def fake_get_conn() -> Any:
|
||||
c = MagicMock()
|
||||
c.execute.return_value.fetchall.return_value = [
|
||||
{"id": "10.1/x", "source_path": "/d/paper.txt", "out_edges": 0}
|
||||
]
|
||||
c.execute.return_value.fetchone.return_value = {"papers": 1, "citing": 1, "edges": 1}
|
||||
conns.append(c)
|
||||
yield c
|
||||
|
||||
with (
|
||||
patch("scripts.ra_grobid_backfill.get_conn", side_effect=fake_get_conn),
|
||||
patch("scripts.ra_grobid_backfill.build_grobid_citations", return_value=[]),
|
||||
):
|
||||
rc = main(["--pdf-dir", str(tmp_path), "--write"])
|
||||
|
||||
assert rc == 0
|
||||
# Two DISTINCT get_conn() context managers (read, then write) — not one held
|
||||
# open across the GROBID loop.
|
||||
assert len(conns) == 2
|
||||
21
tests/scripts/test_rc_tex_reingest.py
Normal file
21
tests/scripts/test_rc_tex_reingest.py
Normal file
@@ -0,0 +1,21 @@
|
||||
"""Tests for scripts.rc_tex_reingest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from scripts.rc_tex_reingest import is_arxiv_id
|
||||
|
||||
|
||||
def test_is_arxiv_id_accepts_modern_and_legacy() -> None:
|
||||
assert is_arxiv_id("2305.10988")
|
||||
assert is_arxiv_id("math/0603097")
|
||||
assert is_arxiv_id("1005.2698")
|
||||
|
||||
|
||||
def test_is_arxiv_id_rejects_dois_and_wids() -> None:
|
||||
assert not is_arxiv_id("10.1007/s00454-019-00132-8")
|
||||
assert not is_arxiv_id("10.14279/depositonce-20357")
|
||||
assert not is_arxiv_id("W2971636899")
|
||||
assert not is_arxiv_id("https://openalex.org/W2971636899")
|
||||
assert not is_arxiv_id("http://openalex.org/W2971636899")
|
||||
assert not is_arxiv_id("10.48550/arXiv.2305.10988") # arXiv DOI form, not bare id
|
||||
assert not is_arxiv_id("")
|
||||
@@ -53,6 +53,94 @@ def test_fetch_source_returns_latex(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assert "Hello world" in result
|
||||
|
||||
|
||||
def test_fetch_source_flattens_multifile(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Multi-file projects: \\input/\\include directives are inlined so the body
|
||||
from the included files is returned, not just the primary skeleton (R-C)."""
|
||||
main = (
|
||||
b"\\documentclass{article}\\begin{document}"
|
||||
b"\\input{sections/intro}\\include{proof}\\end{document}"
|
||||
)
|
||||
tar_bytes = _make_tar_gz(
|
||||
{
|
||||
"main.tex": main,
|
||||
"sections/intro.tex": b"INTRODUCTION BODY TEXT",
|
||||
"proof.tex": b"PROOF BODY TEXT",
|
||||
}
|
||||
)
|
||||
|
||||
def mock_get(
|
||||
url: str,
|
||||
*,
|
||||
timeout: int = 60,
|
||||
follow_redirects: bool = True,
|
||||
) -> httpx.Response:
|
||||
return httpx.Response(200, content=tar_bytes)
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
|
||||
result = arxiv.fetch_source("2301.07041")
|
||||
|
||||
assert result is not None
|
||||
assert "INTRODUCTION BODY TEXT" in result # \input was inlined
|
||||
assert "PROOF BODY TEXT" in result # \include was inlined
|
||||
assert "\\input" not in result and "\\include" not in result
|
||||
|
||||
|
||||
def test_fetch_source_bare_gzip_single_file(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Legacy arXiv submissions are served as a single bare gzipped .tex (no tar
|
||||
wrapper); fetch_source must gunzip the body directly rather than fail."""
|
||||
import gzip
|
||||
|
||||
latex = b"\\documentstyle[a4]{article}\n\\section{Intro}\nOld single-file paper body."
|
||||
gz_bytes = gzip.compress(latex)
|
||||
|
||||
def mock_get(
|
||||
url: str,
|
||||
*,
|
||||
timeout: int = 60,
|
||||
follow_redirects: bool = True,
|
||||
) -> httpx.Response:
|
||||
return httpx.Response(200, content=gz_bytes)
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
|
||||
result = arxiv.fetch_source("math/0001176")
|
||||
|
||||
assert result is not None
|
||||
assert "Old single-file paper body." in result
|
||||
|
||||
|
||||
def test_fetch_source_resolves_non_tex_include(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""\\input of a non-.tex body member (e.g. a .def file) is inlined, not dropped (R-C)."""
|
||||
main = b"\\documentclass{article}\\begin{document}\\input{body.def}\\end{document}"
|
||||
tar_bytes = _make_tar_gz({"main.tex": main, "body.def": b"NON TEX BODY TEXT"})
|
||||
|
||||
def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response:
|
||||
return httpx.Response(200, content=tar_bytes)
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
result = arxiv.fetch_source("2301.07041")
|
||||
assert result is not None
|
||||
assert "NON TEX BODY TEXT" in result
|
||||
|
||||
|
||||
def test_fetch_source_ignores_commented_documentclass(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A commented %\\documentclass must not select a file as the primary document."""
|
||||
decoy = b"% \\documentclass{article}\n\\section{Stub}\nDECOY ONLY"
|
||||
real = b"\\documentclass{book}\\begin{document}REAL BODY HERE\\end{document}"
|
||||
# decoy sorts/inserts first; without the fix it would win on the substring match.
|
||||
tar_bytes = _make_tar_gz({"aaa_decoy.tex": decoy, "real.tex": real})
|
||||
|
||||
def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response:
|
||||
return httpx.Response(200, content=tar_bytes)
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
result = arxiv.fetch_source("2301.07041")
|
||||
assert result is not None
|
||||
assert "REAL BODY HERE" in result
|
||||
assert "DECOY ONLY" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_source — 404 → None
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -138,3 +226,63 @@ def test_fetch_pdf_url_pure_computation() -> None:
|
||||
|
||||
url2 = arxiv.fetch_pdf_url("1234.56789")
|
||||
assert url2 == "https://arxiv.org/pdf/1234.56789.pdf"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_metadata — parse the Atom feed into a Paper (DQ-2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ATOM_FEED = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<title>A discrete version of Liouville's theorem on conformal maps</title>
|
||||
<summary>Liouville's theorem says that in dimension greater than two,
|
||||
all conformal maps are Moebius transformations.</summary>
|
||||
<published>2019-11-03T00:00:00Z</published>
|
||||
<author><name>Ulrich Pinkall</name></author>
|
||||
<author><name>Boris Springborn</name></author>
|
||||
</entry>
|
||||
</feed>"""
|
||||
|
||||
_ATOM_EMPTY = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom"><title>ArXiv Query</title></feed>"""
|
||||
|
||||
|
||||
def test_fetch_metadata_parses_entry(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A valid Atom feed yields a populated Paper."""
|
||||
|
||||
def mock_get(url: str, **kwargs: object) -> httpx.Response:
|
||||
return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
|
||||
paper = arxiv.fetch_metadata("1911.00966")
|
||||
assert paper is not None
|
||||
assert paper.id == "1911.00966"
|
||||
assert paper.title.startswith("A discrete version of Liouville")
|
||||
assert paper.authors == ["Ulrich Pinkall", "Boris Springborn"]
|
||||
assert paper.year == 2019
|
||||
assert paper.abstract and "conformal maps" in paper.abstract
|
||||
assert paper.openalex_id is None
|
||||
|
||||
|
||||
def test_fetch_metadata_no_entry_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A feed with no <entry> (id not found) returns None."""
|
||||
|
||||
def mock_get(url: str, **kwargs: object) -> httpx.Response:
|
||||
return httpx.Response(200, text=_ATOM_EMPTY, request=httpx.Request("GET", url))
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
assert arxiv.fetch_metadata("0000.00000") is None
|
||||
|
||||
|
||||
def test_fetch_metadata_strips_arxiv_prefix(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""An ``arXiv:`` prefix is stripped so Paper.id is the bare id."""
|
||||
|
||||
def mock_get(url: str, **kwargs: object) -> httpx.Response:
|
||||
return httpx.Response(200, text=_ATOM_FEED, request=httpx.Request("GET", url))
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
paper = arxiv.fetch_metadata("arXiv:1911.00966")
|
||||
assert paper is not None
|
||||
assert paper.id == "1911.00966"
|
||||
|
||||
104
tests/sources/test_crossref.py
Normal file
104
tests/sources/test_crossref.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""Tests for codex.sources.crossref."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from codex.models import Citation
|
||||
from codex.sources import crossref
|
||||
|
||||
|
||||
def test_fetch_abstract_strips_jats(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A JATS-tagged abstract is returned as plain text with tags removed."""
|
||||
body = {
|
||||
"message": {
|
||||
"abstract": "<jats:p>We prove a <jats:italic>theorem</jats:italic> about "
|
||||
"polyhedra.</jats:p>"
|
||||
}
|
||||
}
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
monkeypatch.setattr(crossref, "_get", mock_get)
|
||||
|
||||
result = crossref.fetch_abstract("10.1007/s00454-019-00132-8")
|
||||
assert result == "We prove a theorem about polyhedra."
|
||||
|
||||
|
||||
def test_fetch_abstract_absent_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A work with no abstract field (e.g. a book chapter) returns None."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json={"message": {"title": ["Some Chapter"]}})
|
||||
|
||||
monkeypatch.setattr(crossref, "_get", mock_get)
|
||||
assert crossref.fetch_abstract("10.1007/978-3-642-17413-1_7") is None
|
||||
|
||||
|
||||
def test_fetch_abstract_404_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A 404 returns None rather than raising."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
raise httpx.HTTPStatusError(
|
||||
"404",
|
||||
request=httpx.Request("GET", url),
|
||||
response=httpx.Response(404, request=httpx.Request("GET", url)),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(crossref, "_get", mock_get)
|
||||
assert crossref.fetch_abstract("10.0000/nonexistent") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_references (roadmap R-B)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_fetch_references_returns_doi_edges(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Only references carrying a DOI become Citations; citing_id is the queried DOI."""
|
||||
body = {
|
||||
"message": {
|
||||
"reference": [
|
||||
{"key": "ref1", "DOI": "10.1090/s0002-9947-04-03545-7"},
|
||||
{"key": "ref2", "unstructured": "A book with no DOI, 1992."}, # skipped
|
||||
{"key": "ref3", "DOI": "10.1007/bf02392449", "article-title": "X"},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
monkeypatch.setattr(crossref, "_get", mock_get)
|
||||
refs = crossref.fetch_references("10.1007/s00454-019-00132-8")
|
||||
|
||||
assert refs == [
|
||||
Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1090/s0002-9947-04-03545-7"),
|
||||
Citation(citing_id="10.1007/s00454-019-00132-8", cited_id="10.1007/bf02392449"),
|
||||
]
|
||||
|
||||
|
||||
def test_fetch_references_no_reference_key_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A work with no deposited reference list returns []."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json={"message": {"title": ["No refs deposited"]}})
|
||||
|
||||
monkeypatch.setattr(crossref, "_get", mock_get)
|
||||
assert crossref.fetch_references("10.1007/978-3-642-17413-1_7") == []
|
||||
|
||||
|
||||
def test_fetch_references_404_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A 404 returns [] rather than raising."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
raise httpx.HTTPStatusError(
|
||||
"404",
|
||||
request=httpx.Request("GET", url),
|
||||
response=httpx.Response(404, request=httpx.Request("GET", url)),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(crossref, "_get", mock_get)
|
||||
assert crossref.fetch_references("10.0000/nonexistent") == []
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
|
||||
from codex.models import Citation, Paper
|
||||
from codex.sources import openalex
|
||||
from codex.sources.openalex import _resolve_id
|
||||
from codex.sources.openalex import _canonical_id, _normalize_doi, _resolve_id
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -15,7 +15,8 @@ from codex.sources.openalex import _resolve_id
|
||||
|
||||
_SAMPLE_WORK = {
|
||||
"id": "https://openalex.org/W2741809807",
|
||||
# Real OpenAlex returns the DOI as a full URL, not a bare DOI (audit T-3).
|
||||
# Real OpenAlex returns the DOI as a full URL, not a bare DOI (audit T-3 / DQ-5
|
||||
# — the old bare fixture hid the URL-form behaviour both fixes address).
|
||||
"doi": "https://doi.org/10.1145/3592430",
|
||||
"title": "Conformal Prediction: A Review",
|
||||
"publication_year": 2023,
|
||||
@@ -68,6 +69,87 @@ def test_resolve_prefixed_unchanged() -> None:
|
||||
assert _resolve_id("doi:10.1145/x") == "doi:10.1145/x"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _normalize_doi — canonical bare, lower-cased form (DQ-5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_normalize_doi_strips_https_url() -> None:
|
||||
# The exact form OpenAlex returns in the `doi` field.
|
||||
assert _normalize_doi("https://doi.org/10.1007/s00454-019-00132-8") == (
|
||||
"10.1007/s00454-019-00132-8"
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_doi_strips_http_and_doi_prefix() -> None:
|
||||
assert _normalize_doi("http://doi.org/10.1145/3592430") == "10.1145/3592430"
|
||||
assert _normalize_doi("doi:10.1145/3592430") == "10.1145/3592430"
|
||||
|
||||
|
||||
def test_normalize_doi_lowercases() -> None:
|
||||
# DOIs are case-insensitive; the canonical id (and _norm_cited_id) is lower-case.
|
||||
assert _normalize_doi("https://doi.org/10.1007/S00454-019-00132-8") == (
|
||||
"10.1007/s00454-019-00132-8"
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_doi_already_bare_is_idempotent() -> None:
|
||||
assert _normalize_doi("10.1007/s00454-019-00132-8") == "10.1007/s00454-019-00132-8"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _canonical_id — arXiv DOI → bare arXiv id (arXiv half of DQ-5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_canonical_id_strips_arxiv_doi_modern() -> None:
|
||||
# arXiv works carry the arXiv DOI; the canonical id is the bare arXiv id.
|
||||
assert _canonical_id("https://doi.org/10.48550/arXiv.2305.10988") == "2305.10988"
|
||||
|
||||
|
||||
def test_canonical_id_strips_arxiv_doi_legacy() -> None:
|
||||
assert _canonical_id("https://doi.org/10.48550/arXiv.math/0603097") == "math/0603097"
|
||||
|
||||
|
||||
def test_canonical_id_leaves_regular_doi_bare() -> None:
|
||||
# A normal journal DOI is only normalized (bare + lower-case), not stripped.
|
||||
assert _canonical_id("https://doi.org/10.1007/S00454-019-00132-8") == (
|
||||
"10.1007/s00454-019-00132-8"
|
||||
)
|
||||
|
||||
|
||||
def test_fetch_paper_arxiv_doi_yields_bare_id(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""An arXiv work maps to the BARE arXiv id (not the arXiv DOI), so re-ingesting
|
||||
by arXiv id hits ON CONFLICT (id) instead of tripping papers_openalex_id_key."""
|
||||
work = {**_SAMPLE_WORK, "doi": "https://doi.org/10.48550/arXiv.2305.10988"}
|
||||
|
||||
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json=work)
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", mock_get)
|
||||
paper = openalex.fetch_paper("2305.10988")
|
||||
|
||||
assert paper is not None
|
||||
assert paper.id == "2305.10988"
|
||||
|
||||
|
||||
def test_map_paper_falls_back_to_openalex_id_without_doi(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A work with no DOI keeps the OpenAlex W-id (URL form) as id, unchanged —
|
||||
normalization only rewrites the DOI branch."""
|
||||
work = {k: v for k, v in _SAMPLE_WORK.items() if k != "doi"}
|
||||
|
||||
def mock_get(url: str, params: dict[str, str] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json=work)
|
||||
|
||||
monkeypatch.setattr(openalex, "_get", mock_get)
|
||||
paper = openalex.fetch_paper("W2741809807")
|
||||
|
||||
assert paper is not None
|
||||
assert paper.id == "https://openalex.org/W2741809807"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_paper — success
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -86,15 +168,17 @@ def test_fetch_paper_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
assert paper is not None
|
||||
assert isinstance(paper, Paper)
|
||||
# DQ-5: openalex._canonical_id normalizes OpenAlex's URL-form doi to the BARE
|
||||
# lower-cased DOI at the source layer; ingest additionally pins papers.id to the
|
||||
# caller's id (audit C-7). Both converge on the bare canonical id — which is what
|
||||
# makes the ingest ON CONFLICT (id) upsert idempotent. (T-3: the fixture is
|
||||
# URL-form so this mapping can't silently regress.)
|
||||
assert paper.id == "10.1145/3592430"
|
||||
assert paper.title == "Conformal Prediction: A Review"
|
||||
assert paper.year == 2023
|
||||
assert paper.authors == ["Alice Smith", "Bob Jones"]
|
||||
assert paper.abstract == "Conformal prediction is great"
|
||||
assert paper.openalex_id == "https://openalex.org/W2741809807"
|
||||
# fetch_paper maps id from the (URL-form) doi; ingest later canonicalises
|
||||
# papers.id to the caller's bare id (audit C-7). Pin the real mapping here so
|
||||
# the fixture cannot silently misrepresent OpenAlex again (audit T-3).
|
||||
assert paper.id == "https://doi.org/10.1145/3592430"
|
||||
|
||||
|
||||
def test_fetch_paper_arxiv_id_prefixed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
@@ -124,3 +124,46 @@ def test_fetch_recommendations_404_returns_empty(
|
||||
|
||||
result = semanticscholar.fetch_recommendations("nonexistent")
|
||||
assert result == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_references — {"data": null} guard (DQ-1 regression)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_fetch_references_null_data_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""S2 returns HTTP 200 with ``{"data": null}`` for a known paper that has no
|
||||
parsed references; this must yield [] rather than raising TypeError."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json={"data": None, "citingPaperInfo": {}})
|
||||
|
||||
monkeypatch.setattr(semanticscholar, "_get", mock_get)
|
||||
assert semanticscholar.fetch_references("DOI:10.14279/depositonce-5415") == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_abstract (DQ-2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_fetch_abstract_returns_text(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""fetch_abstract returns the abstract string when present."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json={"abstract": "We study ideal hyperbolic polyhedra."})
|
||||
|
||||
monkeypatch.setattr(semanticscholar, "_get", mock_get)
|
||||
assert semanticscholar.fetch_abstract("DOI:10.1007/s00454-019-00132-8") == (
|
||||
"We study ideal hyperbolic polyhedra."
|
||||
)
|
||||
|
||||
|
||||
def test_fetch_abstract_null_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A null/absent abstract yields None."""
|
||||
|
||||
def mock_get(url: str, params: dict[str, object] | None = None) -> httpx.Response:
|
||||
return httpx.Response(200, json={"abstract": None})
|
||||
|
||||
monkeypatch.setattr(semanticscholar, "_get", mock_get)
|
||||
assert semanticscholar.fetch_abstract("arXiv:1234.5678") is None
|
||||
|
||||
Reference in New Issue
Block a user