From 1ff8052b6998bfe0189c48d50ddac265100a0a9b Mon Sep 17 00:00:00 2001 From: Tarik Moussa Date: Mon, 15 Jun 2026 12:17:17 +0200 Subject: [PATCH] chore(infra): add canonical-id re-ingest migration helper (audit C-7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One-time, guarded migration for the C-7 id-format change: a plain re-ingest would duplicate every paper (new bare-id row beside the old URL-id PK), so this wipes (TRUNCATE papers CASCADE) and rebuilds via ingest_all.sh. Safety: requires the SSH tunnel, prints a BEFORE snapshot, gates the TRUNCATE behind an explicit 'MIGRATE' confirmation, then prints AFTER verification — C-7 (url_form_ids should be 0) and C-1 via the real resolver-based discovery_leads() (ingested papers leaked should be 0). Uses .venv psycopg (psql is not installed); DATABASE_URL is sourced, never echoed. Joins PR #13 (Wave 2). Read-only verification SQL validated against the live DB. Co-Authored-By: Claude Fable 5 --- infra/reingest_canonical_ids.sh | 110 ++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 infra/reingest_canonical_ids.sh diff --git a/infra/reingest_canonical_ids.sh b/infra/reingest_canonical_ids.sh new file mode 100755 index 0000000..f106cdc --- /dev/null +++ b/infra/reingest_canonical_ids.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# One-time migration to canonical paper IDs (audit C-7). +# +# Before the C-7 fix, ingest stored OpenAlex's URL-form doi/id as papers.id +# (e.g. "https://doi.org/10.48550/arxiv.math/0603097"). After the fix, ingest +# keys papers.id on the bare caller id ("math/0603097"). The existing rows keep +# their old URL-form primary keys, so a plain re-ingest would DUPLICATE every +# paper (new bare-id row alongside the old URL-id row). This script wipes the +# corpus and rebuilds it on canonical ids. +# +# DESTRUCTIVE: `TRUNCATE papers CASCADE` removes papers + chunks + citations + +# formulas + figures + paper_identifiers (code_links.paper_id is set NULL). All +# content is rebuilt by ingest_all.sh, which re-fetches OpenAlex and re-embeds +# (minutes for ~36 papers). +# +# Prerequisites: SSH tunnel on :5433 (see ingest_all.sh) and .env.jetson-ingest. +# Usage: bash infra/reingest_canonical_ids.sh +set -euo pipefail + +CODEX_DIR="$(cd "$(dirname "$0")/.." && pwd)" +ENV_FILE="$CODEX_DIR/.env.jetson-ingest" +PY="$CODEX_DIR/.venv/bin/python" +TUNNEL_PORT=5433 + +# ── 1. Preconditions ───────────────────────────────────────────────────────── +if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then + echo "ERROR: SSH tunnel not active on :$TUNNEL_PORT" + echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103" + exit 1 +fi +[[ -f "$ENV_FILE" ]] || { echo "ERROR: $ENV_FILE missing"; exit 1; } +[[ -x "$PY" ]] || { echo "ERROR: venv python not found at $PY (run uv sync)"; exit 1; } +# Load DATABASE_URL into the environment (not echoed — keeps the password out of logs). +set -a; source "$ENV_FILE"; set +a + +# run_sql : execute one statement via psycopg, print any result rows. +# Reads DATABASE_URL from the environment; suppresses the DSN on error. +run_sql() { + "$PY" - "$1" <<'PYEOF' +import os, sys, psycopg +from psycopg.rows import dict_row +try: + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c: + cur = c.execute(sys.argv[1]) + if cur.description: + for row in cur.fetchall(): + print(" " + " ".join(f"{k}={v!r}" for k, v in row.items())) + c.commit() +except Exception as e: + print(f" DB ERROR: {type(e).__name__} (details suppressed to avoid DSN leak)") + sys.exit(1) +PYEOF +} + +# ── 2. BEFORE snapshot ─────────────────────────────────────────────────────── +echo "── BEFORE migration ──────────────────────────────────────────" +run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers" +echo " sample ids:" +run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3" + +# ── 3. Confirmation gate (destructive) ─────────────────────────────────────── +echo "" +echo "This TRUNCATEs papers CASCADE (papers/chunks/citations/formulas/figures/" +echo "paper_identifiers) and re-ingests via ingest_all.sh. The DB content is" +echo "rebuilt from scratch." +read -r -p "Type 'MIGRATE' to proceed: " confirm +[[ "$confirm" == "MIGRATE" ]] || { echo "Aborted — nothing changed."; exit 1; } + +# ── 4. Wipe ────────────────────────────────────────────────────────────────── +echo "── TRUNCATE papers CASCADE ───────────────────────────────────" +run_sql "TRUNCATE papers CASCADE" +echo " wiped." + +# ── 5. Re-ingest on canonical ids ──────────────────────────────────────────── +echo "── Re-ingest (ingest_all.sh) ─────────────────────────────────" +bash "$CODEX_DIR/ingest_all.sh" + +# ── 6. AFTER snapshot + live verification ──────────────────────────────────── +echo "── AFTER migration ───────────────────────────────────────────" +echo " C-7 — url_form_ids should now be 0; sample ids should be bare:" +run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers" +run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3" + +# C-1 must be verified through the REAL resolver-based discovery_leads(), not a +# raw cited_id check: cited_id/openalex_id stay in OpenAlex form, so the raw +# "cited_id NOT IN papers.id" count is non-zero by design — the resolver is what +# excludes ingested papers. Check that no ingested paper leaks into the leads. +echo " C-1 — real discovery_leads() must contain no already-ingested paper:" +PYTHONPATH="$CODEX_DIR" "$PY" - <<'PYEOF' +import os, psycopg +from psycopg.rows import dict_row +from codex.discover import discovery_leads +try: + leads = discovery_leads(limit=100000) + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c: + ingested = {r["id"] for r in c.execute("SELECT id FROM papers").fetchall()} + ingested |= { + r["openalex_id"] + for r in c.execute("SELECT openalex_id FROM papers WHERE openalex_id IS NOT NULL").fetchall() + } + leaked = sum(1 for lead in leads if lead["cited_id"] in ingested) + print(f" leads={len(leads)} ingested-papers-leaked-into-leads={leaked}") +except Exception as e: + print(f" DB ERROR: {type(e).__name__} (details suppressed)") +PYEOF + +echo "" +echo "✓ Migration complete." +echo " Expected: url_form_ids=0 (C-7 verified) and leaked=0 (C-1 verified) above." +echo " Next: re-run 'codex graph report' and refresh the ADR-F15 spike table (audit D-1)."