chore(ops): ignore .env.* profiles + add batch ingest script
- .gitignore: ignore .env.* (keep .env.example) so local DB-credential profiles like .env.jetson-ingest can never be staged (audit S-1). - ingest_all.sh: batch-ingest helper writing to the Jetson DB via SSH tunnel. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -18,6 +18,8 @@ build/
|
||||
|
||||
# Environment secrets — NEVER commit
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Type-checker and linter caches
|
||||
.mypy_cache/
|
||||
|
||||
116
ingest_all.sh
Executable file
116
ingest_all.sh
Executable file
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
# Batch ingest — writes directly to Jetson PostgreSQL via SSH tunnel.
|
||||
#
|
||||
# Prerequisites:
|
||||
# ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103
|
||||
# uv sync (done once)
|
||||
#
|
||||
# Usage: bash ingest_all.sh 2>&1 | tee ingest_all.log
|
||||
set -euo pipefail
|
||||
|
||||
PAPERS_DIR="/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt"
|
||||
CODEX_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ENV_FILE="$CODEX_DIR/.env.jetson-ingest"
|
||||
TUNNEL_PORT=5433
|
||||
|
||||
# ── 1. Verify SSH tunnel is active ───────────────────────────────────────────
|
||||
if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then
|
||||
echo "ERROR: SSH tunnel not active on port $TUNNEL_PORT"
|
||||
echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ SSH tunnel active on port $TUNNEL_PORT"
|
||||
|
||||
# ── 1b. Clear macOS UF_HIDDEN flag on editable-install .pth files ─────────────
|
||||
# Python 3.13 skips .pth files with this flag set (new in 3.13); uv sets it.
|
||||
# --no-sync below prevents uv from re-syncing (which would re-set the flag).
|
||||
SP="$CODEX_DIR/.venv/lib/python3.13/site-packages"
|
||||
for pth in "$SP"/_editable_impl_codex*.pth; do
|
||||
[[ -f "$pth" ]] && chflags nohidden "$pth" 2>/dev/null || true
|
||||
done
|
||||
echo "✓ Cleared hidden flags on editable .pth files"
|
||||
|
||||
# ── 2. Load env (DATABASE_URL → Jetson via tunnel) ───────────────────────────
|
||||
# pydantic-settings reads DATABASE_URL directly from the environment.
|
||||
# DOTENV_PATH is not supported; export each var explicitly.
|
||||
set -a; source "$ENV_FILE"; set +a
|
||||
export DATABASE_URL GROBID_URL OLLAMA_BASE_URL EMBEDDING_MODEL EMBEDDING_DIM OPENALEX_MAILTO
|
||||
echo "✓ DATABASE_URL: $DATABASE_URL"
|
||||
|
||||
# ── 3. Paper ID → source file mapping ────────────────────────────────────────
|
||||
# Format: "PAPER_ID|filename.txt"
|
||||
# SKIP: farkas-kra-1992-riemann-surfaces.txt (textbook, no OpenAlex entry)
|
||||
declare -a PAPERS=(
|
||||
# arXiv papers
|
||||
"math/0603097|springborn-2008-weighted-delaunay-hyperideal.txt"
|
||||
"1005.2698|bobenko-pinkall-springborn-2015-discrete-conformal-maps.txt"
|
||||
"math/0306167|luo-2004-combinatorial-yamabe-flow.txt"
|
||||
"math/0203250|bobenko-springborn-2004-circle-patterns.txt"
|
||||
"math/0503219|bobenko-springborn-2007-discrete-laplace-beltrami.txt"
|
||||
"2310.17529|bobenko-lutz-2025-non-euclidean-dce.txt"
|
||||
"2305.10988|bobenko-lutz-2024-decorated-conformal-maps.txt"
|
||||
"2206.13461|lutz-2023-canonical-tessellations.txt"
|
||||
"2601.22903|bowers-bowers-lutz-2026-koebe-rigidity.txt"
|
||||
"1911.00966|pinkall-springborn-2021-liouville.txt"
|
||||
"1505.01341|born-bucking-springborn-2015-quasiconformal.txt"
|
||||
"0906.1560|glickenstein-2011-discrete-conformal-variations.txt"
|
||||
"math/0001176|rivin-schlenker-2000-schlafli-formula-arxiv-preprint.txt"
|
||||
# DOI papers
|
||||
"10.1007/s00454-019-00132-8|springborn-2020-ideal-hyperbolic-polyhedra.txt"
|
||||
"10.1145/1964921.1964997|alexa-wardetzky-2011-discrete-laplacians-polygonal.txt"
|
||||
"10.1111/cgf.13931|bunge-herholz-kazhdan-botsch-2020-polygon-laplacian.txt"
|
||||
"10.1145/3450626.3459763|gillespie-springborn-crane-2021-discrete-conformal-equivalence.txt"
|
||||
"10.1145/3306346.3323042|sharp-soliman-crane-2019-navigating-intrinsic-triangulations.txt"
|
||||
"10.1145/3197517.3201367|soliman-slepcv-crane-2018-optimal-cone-singularities.txt"
|
||||
"10.1145/3132705|sawhney-crane-2017-boundary-first-flattening.txt"
|
||||
"10.1145/2767000|knoppel-crane-pinkall-schroder-2015-stripe-patterns.txt"
|
||||
"10.5555/1070432.1070581|erickson-whittlesey-2005-greedy-homotopy-homology.txt"
|
||||
"10.1145/1185657.1185665|desbrun-kanso-tong-2006-discrete-differential-forms.txt"
|
||||
"10.1080/10586458.1993.10504266|pinkall-polthier-1993-computing-discrete-minimal-surfaces.txt"
|
||||
"10.1007/s11040-021-09394-2|bobenko-bucking-2021-period-matrices.txt"
|
||||
"10.14279/depositonce-20357|lutz-2024-thesis.txt"
|
||||
"10.14279/depositonce-5415|sechelmann-2016-thesis.txt"
|
||||
# Book chapters (txt contains full book; ID = cited chapter)
|
||||
"10.1007/0-387-29555-0_13|precopa-molnar-eds-2006-non-euclidean-geometries-book.txt"
|
||||
"10.1007/978-3-642-17413-1_7|bobenko-klein-eds-2011-computational-approach-riemann-surfaces-book.txt"
|
||||
)
|
||||
|
||||
# ── 4. Ingest loop ────────────────────────────────────────────────────────────
|
||||
OK=0; FAIL=0; SKIP=0
|
||||
FAILED_IDS=()
|
||||
|
||||
cd "$CODEX_DIR"
|
||||
|
||||
for entry in "${PAPERS[@]}"; do
|
||||
PAPER_ID="${entry%%|*}"
|
||||
FILENAME="${entry##*|}"
|
||||
SOURCE="$PAPERS_DIR/$FILENAME"
|
||||
|
||||
if [[ ! -f "$SOURCE" ]]; then
|
||||
echo "SKIP (file missing): $FILENAME"
|
||||
(( SKIP++ )) || true
|
||||
continue
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Ingesting: $PAPER_ID"
|
||||
echo " source: $FILENAME"
|
||||
|
||||
if PYTHONPATH="$CODEX_DIR" "$CODEX_DIR/.venv/bin/codex" ingest "$PAPER_ID" --source "$SOURCE"; then
|
||||
(( OK++ )) || true
|
||||
else
|
||||
echo "FAILED: $PAPER_ID"
|
||||
FAILED_IDS+=("$PAPER_ID")
|
||||
(( FAIL++ )) || true
|
||||
fi
|
||||
done
|
||||
|
||||
# ── 5. Summary ────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "════════════════════════════════════════"
|
||||
echo "Ingest complete: $OK OK, $FAIL FAILED, $SKIP SKIPPED"
|
||||
if [[ ${#FAILED_IDS[@]} -gt 0 ]]; then
|
||||
echo "Failed IDs:"
|
||||
for id in "${FAILED_IDS[@]}"; do echo " $id"; done
|
||||
fi
|
||||
echo "════════════════════════════════════════"
|
||||
Reference in New Issue
Block a user