chore(ops): ignore .env.* profiles + add batch ingest script

- .gitignore: ignore .env.* (keep .env.example) so local DB-credential
  profiles like .env.jetson-ingest can never be staged (audit S-1).
- ingest_all.sh: batch-ingest helper writing to the Jetson DB via SSH tunnel.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
2026-06-15 08:43:53 +02:00
parent 9726042964
commit 0c89eebeb8
2 changed files with 118 additions and 0 deletions

2
.gitignore vendored
View File

@@ -18,6 +18,8 @@ build/
# Environment secrets — NEVER commit # Environment secrets — NEVER commit
.env .env
.env.*
!.env.example
# Type-checker and linter caches # Type-checker and linter caches
.mypy_cache/ .mypy_cache/

116
ingest_all.sh Executable file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# Batch ingest — writes directly to Jetson PostgreSQL via SSH tunnel.
#
# Prerequisites:
# ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103
# uv sync (done once)
#
# Usage: bash ingest_all.sh 2>&1 | tee ingest_all.log
set -euo pipefail
PAPERS_DIR="/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt"
CODEX_DIR="$(cd "$(dirname "$0")" && pwd)"
ENV_FILE="$CODEX_DIR/.env.jetson-ingest"
TUNNEL_PORT=5433
# ── 1. Verify SSH tunnel is active ───────────────────────────────────────────
if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then
echo "ERROR: SSH tunnel not active on port $TUNNEL_PORT"
echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103"
exit 1
fi
echo "✓ SSH tunnel active on port $TUNNEL_PORT"
# ── 1b. Clear macOS UF_HIDDEN flag on editable-install .pth files ─────────────
# Python 3.13 skips .pth files with this flag set (new in 3.13); uv sets it.
# --no-sync below prevents uv from re-syncing (which would re-set the flag).
SP="$CODEX_DIR/.venv/lib/python3.13/site-packages"
for pth in "$SP"/_editable_impl_codex*.pth; do
[[ -f "$pth" ]] && chflags nohidden "$pth" 2>/dev/null || true
done
echo "✓ Cleared hidden flags on editable .pth files"
# ── 2. Load env (DATABASE_URL → Jetson via tunnel) ───────────────────────────
# pydantic-settings reads DATABASE_URL directly from the environment.
# DOTENV_PATH is not supported; export each var explicitly.
set -a; source "$ENV_FILE"; set +a
export DATABASE_URL GROBID_URL OLLAMA_BASE_URL EMBEDDING_MODEL EMBEDDING_DIM OPENALEX_MAILTO
echo "✓ DATABASE_URL: $DATABASE_URL"
# ── 3. Paper ID → source file mapping ────────────────────────────────────────
# Format: "PAPER_ID|filename.txt"
# SKIP: farkas-kra-1992-riemann-surfaces.txt (textbook, no OpenAlex entry)
declare -a PAPERS=(
# arXiv papers
"math/0603097|springborn-2008-weighted-delaunay-hyperideal.txt"
"1005.2698|bobenko-pinkall-springborn-2015-discrete-conformal-maps.txt"
"math/0306167|luo-2004-combinatorial-yamabe-flow.txt"
"math/0203250|bobenko-springborn-2004-circle-patterns.txt"
"math/0503219|bobenko-springborn-2007-discrete-laplace-beltrami.txt"
"2310.17529|bobenko-lutz-2025-non-euclidean-dce.txt"
"2305.10988|bobenko-lutz-2024-decorated-conformal-maps.txt"
"2206.13461|lutz-2023-canonical-tessellations.txt"
"2601.22903|bowers-bowers-lutz-2026-koebe-rigidity.txt"
"1911.00966|pinkall-springborn-2021-liouville.txt"
"1505.01341|born-bucking-springborn-2015-quasiconformal.txt"
"0906.1560|glickenstein-2011-discrete-conformal-variations.txt"
"math/0001176|rivin-schlenker-2000-schlafli-formula-arxiv-preprint.txt"
# DOI papers
"10.1007/s00454-019-00132-8|springborn-2020-ideal-hyperbolic-polyhedra.txt"
"10.1145/1964921.1964997|alexa-wardetzky-2011-discrete-laplacians-polygonal.txt"
"10.1111/cgf.13931|bunge-herholz-kazhdan-botsch-2020-polygon-laplacian.txt"
"10.1145/3450626.3459763|gillespie-springborn-crane-2021-discrete-conformal-equivalence.txt"
"10.1145/3306346.3323042|sharp-soliman-crane-2019-navigating-intrinsic-triangulations.txt"
"10.1145/3197517.3201367|soliman-slepcv-crane-2018-optimal-cone-singularities.txt"
"10.1145/3132705|sawhney-crane-2017-boundary-first-flattening.txt"
"10.1145/2767000|knoppel-crane-pinkall-schroder-2015-stripe-patterns.txt"
"10.5555/1070432.1070581|erickson-whittlesey-2005-greedy-homotopy-homology.txt"
"10.1145/1185657.1185665|desbrun-kanso-tong-2006-discrete-differential-forms.txt"
"10.1080/10586458.1993.10504266|pinkall-polthier-1993-computing-discrete-minimal-surfaces.txt"
"10.1007/s11040-021-09394-2|bobenko-bucking-2021-period-matrices.txt"
"10.14279/depositonce-20357|lutz-2024-thesis.txt"
"10.14279/depositonce-5415|sechelmann-2016-thesis.txt"
# Book chapters (txt contains full book; ID = cited chapter)
"10.1007/0-387-29555-0_13|precopa-molnar-eds-2006-non-euclidean-geometries-book.txt"
"10.1007/978-3-642-17413-1_7|bobenko-klein-eds-2011-computational-approach-riemann-surfaces-book.txt"
)
# ── 4. Ingest loop ────────────────────────────────────────────────────────────
OK=0; FAIL=0; SKIP=0
FAILED_IDS=()
cd "$CODEX_DIR"
for entry in "${PAPERS[@]}"; do
PAPER_ID="${entry%%|*}"
FILENAME="${entry##*|}"
SOURCE="$PAPERS_DIR/$FILENAME"
if [[ ! -f "$SOURCE" ]]; then
echo "SKIP (file missing): $FILENAME"
(( SKIP++ )) || true
continue
fi
echo ""
echo "── Ingesting: $PAPER_ID"
echo " source: $FILENAME"
if PYTHONPATH="$CODEX_DIR" "$CODEX_DIR/.venv/bin/codex" ingest "$PAPER_ID" --source "$SOURCE"; then
(( OK++ )) || true
else
echo "FAILED: $PAPER_ID"
FAILED_IDS+=("$PAPER_ID")
(( FAIL++ )) || true
fi
done
# ── 5. Summary ────────────────────────────────────────────────────────────────
echo ""
echo "════════════════════════════════════════"
echo "Ingest complete: $OK OK, $FAIL FAILED, $SKIP SKIPPED"
if [[ ${#FAILED_IDS[@]} -gt 0 ]]; then
echo "Failed IDs:"
for id in "${FAILED_IDS[@]}"; do echo " $id"; done
fi
echo "════════════════════════════════════════"