From 0c89eebeb892c714b5effe51cfc3205fa71a0f65 Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 15 Jun 2026 08:43:53 +0200 Subject: [PATCH] chore(ops): ignore .env.* profiles + add batch ingest script - .gitignore: ignore .env.* (keep .env.example) so local DB-credential profiles like .env.jetson-ingest can never be staged (audit S-1). - ingest_all.sh: batch-ingest helper writing to the Jetson DB via SSH tunnel. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + ingest_all.sh | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100755 ingest_all.sh diff --git a/.gitignore b/.gitignore index 9a69510..278f7af 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ build/ # Environment secrets — NEVER commit .env +.env.* +!.env.example # Type-checker and linter caches .mypy_cache/ diff --git a/ingest_all.sh b/ingest_all.sh new file mode 100755 index 0000000..7cb8586 --- /dev/null +++ b/ingest_all.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Batch ingest — writes directly to Jetson PostgreSQL via SSH tunnel. +# +# Prerequisites: +# ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103 +# uv sync (done once) +# +# Usage: bash ingest_all.sh 2>&1 | tee ingest_all.log +set -euo pipefail + +PAPERS_DIR="/Users/tarikmoussa/Desktop/ConformalLabpp/papers/txt" +CODEX_DIR="$(cd "$(dirname "$0")" && pwd)" +ENV_FILE="$CODEX_DIR/.env.jetson-ingest" +TUNNEL_PORT=5433 + +# ── 1. Verify SSH tunnel is active ─────────────────────────────────────────── +if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then + echo "ERROR: SSH tunnel not active on port $TUNNEL_PORT" + echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103" + exit 1 +fi +echo "✓ SSH tunnel active on port $TUNNEL_PORT" + +# ── 1b. Clear macOS UF_HIDDEN flag on editable-install .pth files ───────────── +# Python 3.13 skips .pth files with this flag set (new in 3.13); uv sets it. +# --no-sync below prevents uv from re-syncing (which would re-set the flag). +SP="$CODEX_DIR/.venv/lib/python3.13/site-packages" +for pth in "$SP"/_editable_impl_codex*.pth; do + [[ -f "$pth" ]] && chflags nohidden "$pth" 2>/dev/null || true +done +echo "✓ Cleared hidden flags on editable .pth files" + +# ── 2. Load env (DATABASE_URL → Jetson via tunnel) ─────────────────────────── +# pydantic-settings reads DATABASE_URL directly from the environment. +# DOTENV_PATH is not supported; export each var explicitly. +set -a; source "$ENV_FILE"; set +a +export DATABASE_URL GROBID_URL OLLAMA_BASE_URL EMBEDDING_MODEL EMBEDDING_DIM OPENALEX_MAILTO +echo "✓ DATABASE_URL: $DATABASE_URL" + +# ── 3. Paper ID → source file mapping ──────────────────────────────────────── +# Format: "PAPER_ID|filename.txt" +# SKIP: farkas-kra-1992-riemann-surfaces.txt (textbook, no OpenAlex entry) +declare -a PAPERS=( + # arXiv papers + "math/0603097|springborn-2008-weighted-delaunay-hyperideal.txt" + "1005.2698|bobenko-pinkall-springborn-2015-discrete-conformal-maps.txt" + "math/0306167|luo-2004-combinatorial-yamabe-flow.txt" + "math/0203250|bobenko-springborn-2004-circle-patterns.txt" + "math/0503219|bobenko-springborn-2007-discrete-laplace-beltrami.txt" + "2310.17529|bobenko-lutz-2025-non-euclidean-dce.txt" + "2305.10988|bobenko-lutz-2024-decorated-conformal-maps.txt" + "2206.13461|lutz-2023-canonical-tessellations.txt" + "2601.22903|bowers-bowers-lutz-2026-koebe-rigidity.txt" + "1911.00966|pinkall-springborn-2021-liouville.txt" + "1505.01341|born-bucking-springborn-2015-quasiconformal.txt" + "0906.1560|glickenstein-2011-discrete-conformal-variations.txt" + "math/0001176|rivin-schlenker-2000-schlafli-formula-arxiv-preprint.txt" + # DOI papers + "10.1007/s00454-019-00132-8|springborn-2020-ideal-hyperbolic-polyhedra.txt" + "10.1145/1964921.1964997|alexa-wardetzky-2011-discrete-laplacians-polygonal.txt" + "10.1111/cgf.13931|bunge-herholz-kazhdan-botsch-2020-polygon-laplacian.txt" + "10.1145/3450626.3459763|gillespie-springborn-crane-2021-discrete-conformal-equivalence.txt" + "10.1145/3306346.3323042|sharp-soliman-crane-2019-navigating-intrinsic-triangulations.txt" + "10.1145/3197517.3201367|soliman-slepcv-crane-2018-optimal-cone-singularities.txt" + "10.1145/3132705|sawhney-crane-2017-boundary-first-flattening.txt" + "10.1145/2767000|knoppel-crane-pinkall-schroder-2015-stripe-patterns.txt" + "10.5555/1070432.1070581|erickson-whittlesey-2005-greedy-homotopy-homology.txt" + "10.1145/1185657.1185665|desbrun-kanso-tong-2006-discrete-differential-forms.txt" + "10.1080/10586458.1993.10504266|pinkall-polthier-1993-computing-discrete-minimal-surfaces.txt" + "10.1007/s11040-021-09394-2|bobenko-bucking-2021-period-matrices.txt" + "10.14279/depositonce-20357|lutz-2024-thesis.txt" + "10.14279/depositonce-5415|sechelmann-2016-thesis.txt" + # Book chapters (txt contains full book; ID = cited chapter) + "10.1007/0-387-29555-0_13|precopa-molnar-eds-2006-non-euclidean-geometries-book.txt" + "10.1007/978-3-642-17413-1_7|bobenko-klein-eds-2011-computational-approach-riemann-surfaces-book.txt" +) + +# ── 4. Ingest loop ──────────────────────────────────────────────────────────── +OK=0; FAIL=0; SKIP=0 +FAILED_IDS=() + +cd "$CODEX_DIR" + +for entry in "${PAPERS[@]}"; do + PAPER_ID="${entry%%|*}" + FILENAME="${entry##*|}" + SOURCE="$PAPERS_DIR/$FILENAME" + + if [[ ! -f "$SOURCE" ]]; then + echo "SKIP (file missing): $FILENAME" + (( SKIP++ )) || true + continue + fi + + echo "" + echo "── Ingesting: $PAPER_ID" + echo " source: $FILENAME" + + if PYTHONPATH="$CODEX_DIR" "$CODEX_DIR/.venv/bin/codex" ingest "$PAPER_ID" --source "$SOURCE"; then + (( OK++ )) || true + else + echo "FAILED: $PAPER_ID" + FAILED_IDS+=("$PAPER_ID") + (( FAIL++ )) || true + fi +done + +# ── 5. Summary ──────────────────────────────────────────────────────────────── +echo "" +echo "════════════════════════════════════════" +echo "Ingest complete: $OK OK, $FAIL FAILED, $SKIP SKIPPED" +if [[ ${#FAILED_IDS[@]} -gt 0 ]]; then + echo "Failed IDs:" + for id in "${FAILED_IDS[@]}"; do echo " $id"; done +fi +echo "════════════════════════════════════════"