fix(review): address PR #16 code-review findings
#1 ingest: normalize the pinned caller id (_norm_cited_id) so a non-bare DOI caller cannot store a URL-form papers.id that defeats DQ-5 idempotency / the startswith(10.) recovery gates. #2 quality.section_label: collapse to a controlled bucket only for an EXACT canonical heading; descriptive titles ('Abstract Nonsense...') keep their real title instead of being mislabelled. #4 ra_grobid_backfill: release the read connection before the slow GROBID network loop, fresh connection for the write (no idle-in-transaction across the loop over the flaky tunnel). #5/#10 tex: flatten_inputs strips unresolved input/include at the depth cap (no literal leak on cycles); _norm_texkey strips only a single leading ./ . #6/#7 arxiv.fetch_source: keep non-.tex members resolvable for input; pick primary on an UN-commented documentclass line. #13 is_arxiv_id: also exclude http:// and arXiv-DOI forms. Tests added/updated for each. Left as deliberate decisions: #3 (pre-section text drop is pre-existing in extract_sections; abstract stored separately), #8 (script normalizer is intentionally self-contained, already documented), #9/#11/#12 (no current trigger / tightening the gzip heuristic would reject valid old LaTeX like documentstyle / title truncation is by design on a write-only column). ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -110,6 +110,37 @@ def test_fetch_source_bare_gzip_single_file(monkeypatch: pytest.MonkeyPatch) ->
|
||||
assert "Old single-file paper body." in result
|
||||
|
||||
|
||||
def test_fetch_source_resolves_non_tex_include(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""\\input of a non-.tex body member (e.g. a .def file) is inlined, not dropped (R-C)."""
|
||||
main = b"\\documentclass{article}\\begin{document}\\input{body.def}\\end{document}"
|
||||
tar_bytes = _make_tar_gz({"main.tex": main, "body.def": b"NON TEX BODY TEXT"})
|
||||
|
||||
def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response:
|
||||
return httpx.Response(200, content=tar_bytes)
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
result = arxiv.fetch_source("2301.07041")
|
||||
assert result is not None
|
||||
assert "NON TEX BODY TEXT" in result
|
||||
|
||||
|
||||
def test_fetch_source_ignores_commented_documentclass(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A commented %\\documentclass must not select a file as the primary document."""
|
||||
decoy = b"% \\documentclass{article}\n\\section{Stub}\nDECOY ONLY"
|
||||
real = b"\\documentclass{book}\\begin{document}REAL BODY HERE\\end{document}"
|
||||
# decoy sorts/inserts first; without the fix it would win on the substring match.
|
||||
tar_bytes = _make_tar_gz({"aaa_decoy.tex": decoy, "real.tex": real})
|
||||
|
||||
def mock_get(url: str, *, timeout: int = 60, follow_redirects: bool = True) -> httpx.Response:
|
||||
return httpx.Response(200, content=tar_bytes)
|
||||
|
||||
monkeypatch.setattr(httpx, "get", mock_get)
|
||||
result = arxiv.fetch_source("2301.07041")
|
||||
assert result is not None
|
||||
assert "REAL BODY HERE" in result
|
||||
assert "DECOY ONLY" not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fetch_source — 404 → None
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user