merge: reconcile data-quality roadmap with audit-remediation main (#12-15)
Resolved 4 conflicts: cli.py (R-D citing-coverage warning + main's small-corpus JSON/hub-title changes both kept); ingest.py (C-7 paper.id=caller-id pin first, then DQ-2 abstract recovery); test_ingest.py + test_openalex.py (kept both branches' tests). C-7/DQ-5 overlap (both canonicalise papers.id): kept both layers — openalex._canonical_id normalizes fetch_paper's id (DQ-5); ingest pins papers.id to the caller's bare id (C-7); they converge on the bare canonical id. Fixed the auto-merged test_fetch_paper_success that had contradictory bare/URL assertions. 402 tests pass; ruff + mypy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Tarik Moussa
141
infra/reingest_canonical_ids.sh
Executable file
141
infra/reingest_canonical_ids.sh
Executable file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-time migration to canonical paper IDs (audit C-7).
|
||||
#
|
||||
# Before the C-7 fix, ingest stored OpenAlex's URL-form doi/id as papers.id
|
||||
# (e.g. "https://doi.org/10.48550/arxiv.math/0603097"). After the fix, ingest
|
||||
# keys papers.id on the bare caller id ("math/0603097"). The existing rows keep
|
||||
# their old URL-form primary keys, so a plain re-ingest would DUPLICATE every
|
||||
# paper (new bare-id row alongside the old URL-id row). This script wipes the
|
||||
# corpus and rebuilds it on canonical ids.
|
||||
#
|
||||
# DESTRUCTIVE: `TRUNCATE papers CASCADE` removes papers + chunks + citations +
|
||||
# formulas + figures + paper_identifiers (code_links.paper_id is set NULL). All
|
||||
# content is rebuilt by ingest_all.sh, which re-fetches OpenAlex and re-embeds
|
||||
# (minutes for ~36 papers).
|
||||
#
|
||||
# Prerequisites: SSH tunnel on :5433 (see ingest_all.sh) and .env.jetson-ingest.
|
||||
# Usage: bash infra/reingest_canonical_ids.sh
|
||||
set -euo pipefail
|
||||
|
||||
CODEX_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
ENV_FILE="$CODEX_DIR/.env.jetson-ingest"
|
||||
PY="$CODEX_DIR/.venv/bin/python"
|
||||
TUNNEL_PORT=5433
|
||||
|
||||
# ── 1. Preconditions ─────────────────────────────────────────────────────────
|
||||
if ! nc -z localhost "$TUNNEL_PORT" 2>/dev/null; then
|
||||
echo "ERROR: SSH tunnel not active on :$TUNNEL_PORT"
|
||||
echo "Run: ssh -f -N -L 5433:localhost:5432 alfred@192.168.178.103"
|
||||
exit 1
|
||||
fi
|
||||
[[ -f "$ENV_FILE" ]] || { echo "ERROR: $ENV_FILE missing"; exit 1; }
|
||||
[[ -x "$PY" ]] || { echo "ERROR: venv python not found at $PY (run uv sync)"; exit 1; }
|
||||
# Load DATABASE_URL into the environment (not echoed — keeps the password out of logs).
|
||||
set -a; source "$ENV_FILE"; set +a
|
||||
|
||||
# run_sql <SQL>: execute one statement via psycopg, print any result rows.
|
||||
# Reads DATABASE_URL from the environment; suppresses the DSN on error.
|
||||
run_sql() {
|
||||
"$PY" - "$1" <<'PYEOF'
|
||||
import os, sys, psycopg
|
||||
from psycopg.rows import dict_row
|
||||
try:
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c:
|
||||
cur = c.execute(sys.argv[1])
|
||||
if cur.description:
|
||||
for row in cur.fetchall():
|
||||
print(" " + " ".join(f"{k}={v!r}" for k, v in row.items()))
|
||||
c.commit()
|
||||
except Exception as e:
|
||||
print(f" DB ERROR: {type(e).__name__} (details suppressed to avoid DSN leak)")
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# ── 2. BEFORE snapshot ───────────────────────────────────────────────────────
|
||||
echo "── BEFORE migration ──────────────────────────────────────────"
|
||||
run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers"
|
||||
echo " sample ids:"
|
||||
run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3"
|
||||
|
||||
# ── 2b. Preflight: verify the schema is re-ingestable BEFORE wiping anything ──
|
||||
# ingest writes chunks.section (F-16). On a live DB that predates it the column
|
||||
# is missing, and the app user cannot ALTER tables owned by 'postgres' (audit
|
||||
# M-1). Check FIRST so we never TRUNCATE and then fail mid-re-ingest.
|
||||
echo "── Preflight: schema readiness ───────────────────────────────"
|
||||
SECTION_OK="$("$PY" - <<'PYEOF'
|
||||
import os, psycopg
|
||||
try:
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], connect_timeout=10) as c:
|
||||
cols = [r[0] for r in c.execute(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='chunks'").fetchall()]
|
||||
print("yes" if "section" in cols else "no")
|
||||
except Exception:
|
||||
print("error")
|
||||
PYEOF
|
||||
)"
|
||||
if [[ "$SECTION_OK" != "yes" ]]; then
|
||||
echo "ABORT — schema not ready: chunks.section is missing (audit M-1)."
|
||||
echo "The app user cannot add it (core tables are owned by 'postgres'). Sync the"
|
||||
echo "schema with a privileged role, then re-run this script. Either:"
|
||||
echo ""
|
||||
echo " (a) MIGRATION_DATABASE_URL=postgresql://postgres:PASS@HOST:PORT/papers \\"
|
||||
echo " \"$CODEX_DIR/.venv/bin/codex\" migrate # applies schema.sql idempotently"
|
||||
echo " (b) sudo -u postgres psql -d papers \\"
|
||||
echo " -c \"ALTER TABLE chunks ADD COLUMN IF NOT EXISTS section TEXT;\""
|
||||
echo ""
|
||||
echo "Nothing was changed — no TRUNCATE was issued."
|
||||
exit 1
|
||||
fi
|
||||
echo " chunks.section present — schema OK."
|
||||
|
||||
# ── 3. Confirmation gate (destructive) ───────────────────────────────────────
|
||||
echo ""
|
||||
echo "This TRUNCATEs papers CASCADE (papers/chunks/citations/formulas/figures/"
|
||||
echo "paper_identifiers) and re-ingests via ingest_all.sh. The DB content is"
|
||||
echo "rebuilt from scratch."
|
||||
read -r -p "Type 'MIGRATE' to proceed: " confirm
|
||||
[[ "$confirm" == "MIGRATE" ]] || { echo "Aborted — nothing changed."; exit 1; }
|
||||
|
||||
# ── 4. Wipe ──────────────────────────────────────────────────────────────────
|
||||
echo "── TRUNCATE papers CASCADE ───────────────────────────────────"
|
||||
run_sql "TRUNCATE papers CASCADE"
|
||||
echo " wiped."
|
||||
|
||||
# ── 5. Re-ingest on canonical ids ────────────────────────────────────────────
|
||||
echo "── Re-ingest (ingest_all.sh) ─────────────────────────────────"
|
||||
bash "$CODEX_DIR/ingest_all.sh"
|
||||
|
||||
# ── 6. AFTER snapshot + live verification ────────────────────────────────────
|
||||
echo "── AFTER migration ───────────────────────────────────────────"
|
||||
echo " C-7 — url_form_ids should now be 0; sample ids should be bare:"
|
||||
run_sql "SELECT count(*) AS papers, count(*) FILTER (WHERE starts_with(id, 'https://')) AS url_form_ids FROM papers"
|
||||
run_sql "SELECT id FROM papers ORDER BY added_at LIMIT 3"
|
||||
|
||||
# C-1 must be verified through the REAL resolver-based discovery_leads(), not a
|
||||
# raw cited_id check: cited_id/openalex_id stay in OpenAlex form, so the raw
|
||||
# "cited_id NOT IN papers.id" count is non-zero by design — the resolver is what
|
||||
# excludes ingested papers. Check that no ingested paper leaks into the leads.
|
||||
echo " C-1 — real discovery_leads() must contain no already-ingested paper:"
|
||||
PYTHONPATH="$CODEX_DIR" "$PY" - <<'PYEOF'
|
||||
import os, psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from codex.discover import discovery_leads
|
||||
try:
|
||||
leads = discovery_leads(limit=100000)
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=10) as c:
|
||||
ingested = {r["id"] for r in c.execute("SELECT id FROM papers").fetchall()}
|
||||
ingested |= {
|
||||
r["openalex_id"]
|
||||
for r in c.execute("SELECT openalex_id FROM papers WHERE openalex_id IS NOT NULL").fetchall()
|
||||
}
|
||||
leaked = sum(1 for lead in leads if lead["cited_id"] in ingested)
|
||||
print(f" leads={len(leads)} ingested-papers-leaked-into-leads={leaked}")
|
||||
except Exception as e:
|
||||
print(f" DB ERROR: {type(e).__name__} (details suppressed)")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "✓ Migration complete."
|
||||
echo " Expected: url_form_ids=0 (C-7 verified) and leaked=0 (C-1 verified) above."
|
||||
echo " Next: re-run 'codex graph report' and refresh the ADR-F15 spike table (audit D-1)."
|
||||
@@ -13,7 +13,7 @@ CREATE EXTENSION IF NOT EXISTS vector;
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 1: Papers + full-text chunks
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE papers (
|
||||
CREATE TABLE IF NOT EXISTS papers (
|
||||
id TEXT PRIMARY KEY, -- canonical ID: arXiv ID or DOI
|
||||
openalex_id TEXT UNIQUE, -- e.g. W2741809807
|
||||
bibkey TEXT UNIQUE, -- BibTeX key -> .bib + Doxygen @cite
|
||||
@@ -27,10 +27,10 @@ CREATE TABLE papers (
|
||||
);
|
||||
|
||||
-- HNSW index for fast approximate nearest-neighbour search on abstracts.
|
||||
CREATE INDEX papers_abstract_emb_idx
|
||||
CREATE INDEX IF NOT EXISTS papers_abstract_emb_idx
|
||||
ON papers USING hnsw (abstract_emb vector_cosine_ops);
|
||||
|
||||
CREATE TABLE chunks (
|
||||
CREATE TABLE IF NOT EXISTS chunks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
paper_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
||||
ord INT NOT NULL, -- position within the paper
|
||||
@@ -38,13 +38,13 @@ CREATE TABLE chunks (
|
||||
embedding vector(1024)
|
||||
);
|
||||
|
||||
CREATE INDEX chunks_emb_idx
|
||||
CREATE INDEX IF NOT EXISTS chunks_emb_idx
|
||||
ON chunks USING hnsw (embedding vector_cosine_ops);
|
||||
CREATE INDEX chunks_paper_idx ON chunks (paper_id);
|
||||
CREATE INDEX IF NOT EXISTS chunks_paper_idx ON chunks (paper_id);
|
||||
|
||||
-- Sparse / keyword hits for exact mathematical terminology (hybrid search).
|
||||
-- Dense (above) + full-text (below) combined = robust against math terms.
|
||||
CREATE INDEX chunks_fts_idx
|
||||
CREATE INDEX IF NOT EXISTS chunks_fts_idx
|
||||
ON chunks USING gin (to_tsvector('english', content));
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
@@ -53,19 +53,19 @@ CREATE INDEX chunks_fts_idx
|
||||
-- edges to not-yet-ingested papers are intentionally preserved —
|
||||
-- those are your discovery leads.
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE citations (
|
||||
CREATE TABLE IF NOT EXISTS citations (
|
||||
citing_id TEXT REFERENCES papers(id) ON DELETE CASCADE,
|
||||
cited_id TEXT NOT NULL, -- arXiv/DOI/OpenAlex ID of the target
|
||||
context TEXT, -- optional: citation context (S2)
|
||||
PRIMARY KEY (citing_id, cited_id)
|
||||
);
|
||||
|
||||
CREATE INDEX citations_cited_idx ON citations (cited_id);
|
||||
CREATE INDEX IF NOT EXISTS citations_cited_idx ON citations (cited_id);
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Layer 3: Provenance (the "cleanly couple" goal)
|
||||
-- ---------------------------------------------------------------------
|
||||
CREATE TABLE code_links (
|
||||
CREATE TABLE IF NOT EXISTS code_links (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
symbol TEXT NOT NULL, -- e.g. 'dec::hodge_star' or 'file.cpp:120'
|
||||
paper_id TEXT REFERENCES papers(id) ON DELETE SET NULL,
|
||||
@@ -74,8 +74,8 @@ CREATE TABLE code_links (
|
||||
added_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX code_links_symbol_idx ON code_links (symbol);
|
||||
CREATE INDEX code_links_paper_idx ON code_links (paper_id);
|
||||
CREATE INDEX IF NOT EXISTS code_links_symbol_idx ON code_links (symbol);
|
||||
CREATE INDEX IF NOT EXISTS code_links_paper_idx ON code_links (paper_id);
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Example query: discovery leads
|
||||
|
||||
Reference in New Issue
Block a user