fix(review): address PR #16 code-review findings

#1 ingest: normalize the pinned caller id (_norm_cited_id) so a non-bare DOI caller cannot store a URL-form papers.id that defeats DQ-5 idempotency / the startswith(10.) recovery gates. #2 quality.section_label: collapse to a controlled bucket only for an EXACT canonical heading; descriptive titles ('Abstract Nonsense...') keep their real title instead of being mislabelled. #4 ra_grobid_backfill: release the read connection before the slow GROBID network loop, fresh connection for the write (no idle-in-transaction across the loop over the flaky tunnel). #5/#10 tex: flatten_inputs strips unresolved input/include at the depth cap (no literal leak on cycles); _norm_texkey strips only a single leading ./ . #6/#7 arxiv.fetch_source: keep non-.tex members resolvable for input; pick primary on an UN-commented documentclass line. #13 is_arxiv_id: also exclude http:// and arXiv-DOI forms. Tests added/updated for each. Left as deliberate decisions: #3 (pre-section text drop is pre-existing in extract_sections; abstract stored separately), #8 (script normalizer is intentionally self-contained, already documented), #9/#11/#12 (no current trigger / tightening the gzip heuristic would reject valid old LaTeX like documentstyle / title truncation is by design on a write-only column). ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-26 22:50:23 +02:00
parent 9e5184385a
commit b2f01ce010
11 changed files with 190 additions and 75 deletions
--- a/codex/ingest.py
+++ b/codex/ingest.py
@@ -207,12 +207,14 @@ def ingest_paper(
        raise ValueError(f"Paper not found: {paper_id}")

    # Canonical identity is the caller-supplied id — what the CLI / ingest scripts
-    # use and what papers.id is contracted to hold (arXiv id or DOI). OpenAlex's
-    # fetch_paper fills paper.id with the URL-form doi/id (audit C-7); since DQ-5
-    # `openalex._canonical_id` already normalizes that to the bare form, this pin
-    # is belt-and-suspenders and keeps re-ingest idempotent regardless. The
-    # OpenAlex work id is retained as openalex_id and in paper_identifiers.
-    paper.id = paper_id.strip()
+    # use and what papers.id is contracted to hold (arXiv id or DOI). Pin it to the
+    # caller's id (audit C-7), but normalize it the same way cited-ids are (strip a
+    # https://doi.org/ or doi: prefix, lower-case DOIs) so a non-bare caller cannot
+    # store a URL-form papers.id that would defeat DQ-5's ON CONFLICT (id) idempotency
+    # or the `paper.id.startswith("10.")` recovery gates below. arXiv/W ids pass
+    # through unchanged. The OpenAlex work id is retained as openalex_id / in
+    # paper_identifiers.
+    paper.id = _norm_cited_id(paper_id.strip())

    # DQ-2: OpenAlex often has the paper but no abstract (it cannot redistribute
    # some publishers' abstracts). Supplement from S2, then Crossref (uses the
--- a/codex/parsing/tex.py
+++ b/codex/parsing/tex.py
@@ -170,7 +170,9 @@ def _norm_texkey(name: str) -> str:
    ``.tex`` so ``\\input{sections/intro}`` matches the archive member
    ``sections/intro.tex``.
    """
-    n = name.strip().lstrip("./")
+    n = name.strip()
+    if n.startswith("./"):  # a single leading "./" only — don't eat "../" or ".hidden"
+        n = n[2:]
    return n[:-4] if n.endswith(".tex") else n


@@ -188,7 +190,9 @@ def flatten_inputs(primary: str, files: dict[str, str]) -> str:

    def _expand(text: str, depth: int) -> str:
        if depth > 10:
-            return text
+            # Cap reached (likely an \input cycle): strip any still-unresolved
+            # directives rather than leak a literal "\input{…}" into the parsed text.
+            return _INPUT_RE.sub("", text)

        def _repl(match: re.Match[str]) -> str:
            content = norm.get(_norm_texkey(match.group(1)))
--- a/codex/quality.py
+++ b/codex/quality.py
@@ -127,14 +127,34 @@ def _clean_title(title: str) -> str:
    return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t


+# A heading collapses to a controlled bucket only when it IS one of these canonical
+# section words — NOT when it merely starts with one. Prefix-matching via
+# classify_section mislabels real sections ("Abstract Nonsense and Categories" →
+# abstract, "References Architecture" → bibliography); an exact map avoids that while
+# keeping cross-source consistency for the bare headings.
+_SECTION_TITLE_BUCKETS = {
+    "abstract": "abstract",
+    "zusammenfassung": "abstract",
+    "introduction": "intro",
+    "einleitung": "intro",
+    "proof": "proof",
+    "beweis": "proof",
+    "references": "bibliography",
+    "bibliography": "bibliography",
+    "bibliographie": "bibliography",
+    "literatur": "bibliography",
+}
+
+
 def section_label(title: str | None, content: str) -> str:
    """Label a chunk's section, preferring the real ``\\section`` heading (R-F).

-    For section-aware ``.tex`` ingest *title* is the real heading: if it maps to a
-    controlled bucket via :func:`classify_section` (intro / theorem / proof /
-    abstract / bibliography) that bucket is kept (cross-source consistency);
-    otherwise the cleaned title itself is stored (e.g. ``"preliminaries"``,
-    ``"rigidity"``) — far more signal than collapsing everything to ``body``.
+    For section-aware ``.tex`` ingest *title* is the real heading: a bare canonical
+    heading ("Introduction", "References", "Proof") maps to its controlled bucket
+    (cross-source consistency); any other heading is stored as its cleaned real
+    title (e.g. ``"preliminaries"``, ``"introduction to operator algebras"``) — far
+    more signal than collapsing to ``body`` AND without the prefix-match mislabels
+    that ``classify_section`` would produce on descriptive titles.

    When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass``
    paths) this falls back to content-based :func:`classify_section` — unchanged
@@ -146,8 +166,7 @@ def section_label(title: str | None, content: str) -> str:
    cleaned = _clean_title(title)
    if not cleaned:
        return classify_section(content)
-    bucket = classify_section(cleaned)
-    return bucket if bucket != "body" else cleaned
+    return _SECTION_TITLE_BUCKETS.get(cleaned, cleaned)


 # ---------------------------------------------------------------------------
--- a/codex/sources/arxiv.py
+++ b/codex/sources/arxiv.py
@@ -97,6 +97,16 @@ def fetch_metadata(arxiv_id: str) -> Paper | None:
    )


+def _has_documentclass(content: str) -> bool:
+    """True if *content* has an UN-commented ``\\documentclass`` line.
+
+    A plain ``"\\documentclass" in content`` substring test also matches a
+    commented-out ``%\\documentclass`` (common in arXiv preambles); requiring the
+    command to start its line avoids selecting the wrong primary file.
+    """
+    return any(line.lstrip().startswith("\\documentclass") for line in content.splitlines())
+
+
 def fetch_source(arxiv_id: str) -> str | None:
    """Download and extract the primary LaTeX source for an arXiv paper.

@@ -128,29 +138,38 @@ def fetch_source(arxiv_id: str) -> str | None:
    from codex.parsing.tex import flatten_inputs

    raw = response.content
+    # Image/font/binary members are never \input'd as text; skip them so the source
+    # map stays text-only (decode-with-replace would otherwise store garbled bytes).
+    skip_ext = (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".pdf", ".eps", ".ps", ".ttf", ".otf")
    try:
        with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf:
            files: dict[str, str] = {}
            primary_name: str | None = None
            for member in tf.getmembers():
-                if not member.name.endswith(".tex"):
+                if not member.isfile() or member.name.lower().endswith(skip_ext):
                    continue
                f = tf.extractfile(member)
                if f is None:
                    continue
-                content = f.read().decode("utf-8", errors="replace")
-                files[member.name] = content
-                # Prefer the \documentclass file as the primary document.
-                if primary_name is None and "\\documentclass" in content:
+                # Keep ALL text members resolvable so \input of a non-.tex include
+                # (e.g. a .def or extension-less body file) is not silently dropped (R-C).
+                files[member.name] = f.read().decode("utf-8", errors="replace")
+                # Primary doc = a .tex with an UN-commented \documentclass.
+                if (
+                    primary_name is None
+                    and member.name.endswith(".tex")
+                    and _has_documentclass(files[member.name])
+                ):
                    primary_name = member.name

-            if not files:
+            tex_files = {k: v for k, v in files.items() if k.endswith(".tex")}
+            if not tex_files:
                logger.debug("No .tex files found in arXiv source for %s", arxiv_id)
                return None

            # No \documentclass anywhere → fall back to the largest .tex.
            if primary_name is None:
-                primary_name = max(files, key=lambda k: len(files[k]))
+                primary_name = max(tex_files, key=lambda k: len(tex_files[k]))

            # Inline \input/\include so multi-file projects yield the full body,
            # not just the primary file's skeleton of include directives (R-C).