codex-py/infra/reingest_canonical_ids.sh

#!/usr/bin/env bash
# One-time migration to canonical paper IDs (audit C-7).
#
# Before the C-7 fix, ingest stored OpenAlex's URL-form doi/id as papers.id
# (e.g. "https://doi.org/10.48550/arxiv.math/0603097"). After the fix, ingest
# keys papers.id on the bare caller id ("math/0603097"). The existing rows keep
# their old URL-form primary keys, so a plain re-ingest would DUPLICATE every
# paper (new bare-id row alongside the old URL-id row). This script wipes the
# corpus and rebuilds it on canonical ids.
#
# DESTRUCTIVE: `TRUNCATE papers CASCADE` removes papers + chunks + citations +
# formulas + figures + paper_identifiers (code_links.paper_id is set NULL). All
# content is rebuilt by ingest_all.sh, which re-fetches OpenAlex and re-embeds
# (minutes for ~36 papers).
#
# Prerequisites: SSH tunnel on :5433 (see ingest_all.sh) and .env.jetson-ingest.
# Usage: bash infra/reingest_canonical_ids.sh
set -euo pipefail

CODEX_DIR="$(cd "$(dirname "$0")/.." && pwd)"
ENV_FILE="$CODEX_DIR/.env.jetson-ingest"
PY="$CODEX_DIR/.venv/bin/python"
TUNNEL_PORT=5433

# ── 1. Preconditions ─────────────────────────────────────────────────────────
if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then
  echo "ERROR: SSH tunnel not active on :$TUNNEL_PORT"
  echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103"
  exit 1
fi
[[ -f "$ENV_FILE" ]] || { echo "ERROR: $ENV_FILE missing"; exit 1; }
[[ -x "$PY" ]] || { echo "ERROR: venv python not found at $PY (run uv sync)"; exit 1; }
# Load DATABASE_URL into the environment (not echoed — keeps the password out of logs).
set -a; source "$ENV_FILE"; set +a

# run_sql <SQL>: execute one statement via psycopg, print any result rows.
# Reads DATABASE_URL from the environment; suppresses the DSN on error.
run_sql() {
  "$PY" - "$1" <<'PYEOF'
import os, sys, psycopg
from psycopg.rows import dict_row
try:
    with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c:
        cur = c.execute(sys.argv[1])
        if cur.description:
            for row in cur.fetchall():
                print("   " + "  ".join(f"{k}={v!r}" for k, v in row.items()))
        c.commit()
except Exception as e:
    print(f"   DB ERROR: {type(e).__name__} (details suppressed to avoid DSN leak)")
    sys.exit(1)
PYEOF
}

# ── 2. BEFORE snapshot ───────────────────────────────────────────────────────
echo "── BEFORE migration ──────────────────────────────────────────"
run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers"
echo "   sample ids:"
run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3"

# ── 2b. Preflight: verify the schema is re-ingestable BEFORE wiping anything ──
# ingest writes chunks.section (F-16). On a live DB that predates it the column
# is missing, and the app user cannot ALTER tables owned by 'postgres' (audit
# M-1). Check FIRST so we never TRUNCATE and then fail mid-re-ingest.
echo "── Preflight: schema readiness ───────────────────────────────"
SECTION_OK="$("$PY" - <<'PYEOF'
import os, psycopg
try:
    with psycopg.connect(os.environ["DATABASE_URL"], connect_timeout=10) as c:
        cols = [r[0] for r in c.execute(
            "SELECT column_name FROM information_schema.columns WHERE table_name='chunks'").fetchall()]
        print("yes" if "section" in cols else "no")
except Exception:
    print("error")
PYEOF
)"
if [[ "$SECTION_OK" != "yes" ]]; then
  echo "ABORT — schema not ready: chunks.section is missing (audit M-1)."
  echo "The app user cannot add it (core tables are owned by 'postgres'). Sync the"
  echo "schema with a privileged role, then re-run this script. Either:"
  echo ""
  echo "  (a) MIGRATION_DATABASE_URL=postgresql://postgres:PASS@HOST:PORT/papers \\"
  echo "        \"$CODEX_DIR/.venv/bin/codex\" migrate     # applies schema.sql idempotently"
  echo "  (b) sudo -u postgres psql -d papers \\"
  echo "        -c \"ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;\""
  echo ""
  echo "Nothing was changed — no TRUNCATE was issued."
  exit 1
fi
echo "   chunks.section present — schema OK."

# ── 3. Confirmation gate (destructive) ───────────────────────────────────────
echo ""
echo "This TRUNCATEs papers CASCADE (papers/chunks/citations/formulas/figures/"
echo "paper_identifiers) and re-ingests via ingest_all.sh. The DB content is"
echo "rebuilt from scratch."
read -r -p "Type 'MIGRATE' to proceed: " confirm
[[ "$confirm" == "MIGRATE" ]] || { echo "Aborted — nothing changed."; exit 1; }

# ── 4. Wipe ──────────────────────────────────────────────────────────────────
echo "── TRUNCATE papers CASCADE ───────────────────────────────────"
run_sql "TRUNCATE papers CASCADE"
echo "   wiped."

# ── 5. Re-ingest on canonical ids ────────────────────────────────────────────
echo "── Re-ingest (ingest_all.sh) ─────────────────────────────────"
bash "$CODEX_DIR/ingest_all.sh"

# ── 6. AFTER snapshot + live verification ────────────────────────────────────
echo "── AFTER migration ───────────────────────────────────────────"
echo "   C-7 — url_form_ids should now be 0; sample ids should be bare:"
run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers"
run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3"

# C-1 must be verified through the REAL resolver-based discovery_leads(), not a
# raw cited_id check: cited_id/openalex_id stay in OpenAlex form, so the raw
# "cited_id NOT IN papers.id" count is non-zero by design — the resolver is what
# excludes ingested papers. Check that no ingested paper leaks into the leads.
echo "   C-1 — real discovery_leads() must contain no already-ingested paper:"
PYTHONPATH="$CODEX_DIR" "$PY" - <<'PYEOF'
import os, psycopg
from psycopg.rows import dict_row
from codex.discover import discovery_leads
try:
    leads = discovery_leads(limit=100000)
    with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c:
        ingested = {r["id"] for r in c.execute("SELECT id FROM papers").fetchall()}
        ingested |= {
            r["openalex_id"]
            for r in c.execute("SELECT openalex_id FROM papers WHERE openalex_id IS NOT NULL").fetchall()
        }
    leaked = sum(1 for lead in leads if lead["cited_id"] in ingested)
    print(f"   leads={len(leads)}  ingested-papers-leaked-into-leads={leaked}")
except Exception as e:
    print(f"   DB ERROR: {type(e).__name__} (details suppressed)")
PYEOF

echo ""
echo "✓ Migration complete."
echo "  Expected: url_form_ids=0 (C-7 verified) and leaked=0 (C-1 verified) above."
echo "  Next: re-run 'codex graph report' and refresh the ADR-F15 spike table (audit D-1)."