fix(review): address PR #16 code-review findings
#1 ingest: normalize the pinned caller id (_norm_cited_id) so a non-bare DOI caller cannot store a URL-form papers.id that defeats DQ-5 idempotency / the startswith(10.) recovery gates. #2 quality.section_label: collapse to a controlled bucket only for an EXACT canonical heading; descriptive titles ('Abstract Nonsense...') keep their real title instead of being mislabelled. #4 ra_grobid_backfill: release the read connection before the slow GROBID network loop, fresh connection for the write (no idle-in-transaction across the loop over the flaky tunnel). #5/#10 tex: flatten_inputs strips unresolved input/include at the depth cap (no literal leak on cycles); _norm_texkey strips only a single leading ./ . #6/#7 arxiv.fetch_source: keep non-.tex members resolvable for input; pick primary on an UN-commented documentclass line. #13 is_arxiv_id: also exclude http:// and arXiv-DOI forms. Tests added/updated for each. Left as deliberate decisions: #3 (pre-section text drop is pre-existing in extract_sections; abstract stored separately), #8 (script normalizer is intentionally self-contained, already documented), #9/#11/#12 (no current trigger / tightening the gzip heuristic would reject valid old LaTeX like documentstyle / title truncation is by design on a write-only column). ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -127,14 +127,34 @@ def _clean_title(title: str) -> str:
|
||||
return t[:60].rsplit(" ", 1)[0] if len(t) > 60 else t
|
||||
|
||||
|
||||
# A heading collapses to a controlled bucket only when it IS one of these canonical
|
||||
# section words — NOT when it merely starts with one. Prefix-matching via
|
||||
# classify_section mislabels real sections ("Abstract Nonsense and Categories" →
|
||||
# abstract, "References Architecture" → bibliography); an exact map avoids that while
|
||||
# keeping cross-source consistency for the bare headings.
|
||||
_SECTION_TITLE_BUCKETS = {
|
||||
"abstract": "abstract",
|
||||
"zusammenfassung": "abstract",
|
||||
"introduction": "intro",
|
||||
"einleitung": "intro",
|
||||
"proof": "proof",
|
||||
"beweis": "proof",
|
||||
"references": "bibliography",
|
||||
"bibliography": "bibliography",
|
||||
"bibliographie": "bibliography",
|
||||
"literatur": "bibliography",
|
||||
}
|
||||
|
||||
|
||||
def section_label(title: str | None, content: str) -> str:
|
||||
"""Label a chunk's section, preferring the real ``\\section`` heading (R-F).
|
||||
|
||||
For section-aware ``.tex`` ingest *title* is the real heading: if it maps to a
|
||||
controlled bucket via :func:`classify_section` (intro / theorem / proof /
|
||||
abstract / bibliography) that bucket is kept (cross-source consistency);
|
||||
otherwise the cleaned title itself is stored (e.g. ``"preliminaries"``,
|
||||
``"rigidity"``) — far more signal than collapsing everything to ``body``.
|
||||
For section-aware ``.tex`` ingest *title* is the real heading: a bare canonical
|
||||
heading ("Introduction", "References", "Proof") maps to its controlled bucket
|
||||
(cross-source consistency); any other heading is stored as its cleaned real
|
||||
title (e.g. ``"preliminaries"``, ``"introduction to operator algebras"``) — far
|
||||
more signal than collapsing to ``body`` AND without the prefix-match mislabels
|
||||
that ``classify_section`` would produce on descriptive titles.
|
||||
|
||||
When *title* is None / blank (the ``.pdf`` / ``.txt`` / ``run_quality_pass``
|
||||
paths) this falls back to content-based :func:`classify_section` — unchanged
|
||||
@@ -146,8 +166,7 @@ def section_label(title: str | None, content: str) -> str:
|
||||
cleaned = _clean_title(title)
|
||||
if not cleaned:
|
||||
return classify_section(content)
|
||||
bucket = classify_section(cleaned)
|
||||
return bucket if bucket != "body" else cleaned
|
||||
return _SECTION_TITLE_BUCKETS.get(cleaned, cleaned)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user