ConformalLabpp/scripts/check-test-counts.sh

#!/usr/bin/env bash
# scripts/check-test-counts.sh
#
# CI gate that verifies the totals in `doc/api/tests.md` match the
# actual `ctest` output.  Catches the canonical test-count document
# going stale even if no other doc hardcodes the number.
#
# Usage:
#   bash scripts/check-test-counts.sh
#
# Exit codes:
#   0  totals match
#   1  totals diverge — prints diff and which file to fix
#   2  prerequisites missing (cmake not found, tests.md missing, …)
#
# This script is meant to be cheap enough to run on every PR (~30 s on
# a typical CI runner).  It re-uses the existing build-cgal/ directory
# if one is present; otherwise it builds a throwaway build-counts/.

set -euo pipefail

cd "$(dirname "${BASH_SOURCE[0]}")/.."   # repo root

TESTS_MD="doc/api/tests.md"

if [ ! -f "$TESTS_MD" ]; then
    echo "FAIL: $TESTS_MD not found" >&2
    exit 2
fi
command -v cmake >/dev/null || { echo "FAIL: cmake not in PATH" >&2; exit 2; }
command -v ctest >/dev/null || { echo "FAIL: ctest not in PATH" >&2; exit 2; }

# ── Build ───────────────────────────────────────────────────────────────────
# Priority order:
#   1. `BUILD_DIR` env var (CI sets it to the dir cpp-tests.yml just built).
#   2. existing build-cgal/ if it has WITH_CGAL_TESTS=ON.
#   3. throwaway build-counts/ — full build from scratch.
if [ -n "${BUILD_DIR:-}" ]; then
    if [ ! -f "$BUILD_DIR/CMakeCache.txt" ]; then
        echo "FAIL: BUILD_DIR=$BUILD_DIR has no CMakeCache.txt" >&2
        exit 2
    fi
    if ! grep -q "WITH_CGAL_TESTS:.*=ON\|WITH_CGAL:.*=ON" "$BUILD_DIR/CMakeCache.txt"; then
        echo "FAIL: BUILD_DIR=$BUILD_DIR was not configured with WITH_CGAL_TESTS=ON" >&2
        exit 2
    fi
elif [ -f build-cgal/CMakeCache.txt ] && grep -q "WITH_CGAL_TESTS:.*=ON\|WITH_CGAL:.*=ON" build-cgal/CMakeCache.txt; then
    BUILD_DIR=build-cgal
    cmake --build "$BUILD_DIR" -j"$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 2)" >/dev/null
else
    BUILD_DIR=build-counts
    cmake -S code -B "$BUILD_DIR" -DWITH_CGAL_TESTS=ON -DCMAKE_BUILD_TYPE=Release >/dev/null
    cmake --build "$BUILD_DIR" -j"$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 2)" >/dev/null
fi

# ── Get actual counts ───────────────────────────────────────────────────────
cd "$BUILD_DIR"
actual_cgal=$(ctest -R "^cgal" 2>&1 | grep "tests passed" | sed -E 's/.*out of ([0-9]+).*/\1/')
actual_fast=$(ctest -E "^cgal" 2>&1 | grep "tests passed" | sed -E 's/.*out of ([0-9]+).*/\1/')
cd - >/dev/null

# ── Get claimed counts from tests.md ────────────────────────────────────────
# Expected format (in this order):
#   **Total: <N1> tests, 0 skipped.**   (non-CGAL section)
#   ...
#   **Total: <N2> tests, 0 skipped.**   (CGAL section)
claimed_fast=$(grep -E "\*\*Total: [0-9]+ tests, 0 skipped\.\*\*" "$TESTS_MD" \
    | head -n 1 | sed -E 's/.*Total: ([0-9]+) tests.*/\1/')
claimed_cgal=$(grep -E "\*\*Total: [0-9]+ tests, 0 skipped\.\*\*" "$TESTS_MD" \
    | tail -n 1 | sed -E 's/.*Total: ([0-9]+) tests.*/\1/')

# ── Compare ────────────────────────────────────────────────────────────────
ok=1
if [ "${actual_fast}" != "${claimed_fast}" ]; then
    echo "MISMATCH: non-CGAL — $TESTS_MD claims ${claimed_fast}, ctest reports ${actual_fast}"
    ok=0
fi
if [ "${actual_cgal}" != "${claimed_cgal}" ]; then
    echo "MISMATCH: CGAL — $TESTS_MD claims ${claimed_cgal}, ctest reports ${actual_cgal}"
    ok=0
fi

if [ "$ok" -eq 0 ]; then
    cat <<EOF

FAIL: $TESTS_MD totals out of sync with actual test counts.

Recovery:
  1. Edit $TESTS_MD: update the two
     '**Total: N tests, 0 skipped.**' lines.
  2. Add/remove suite rows in the per-suite tables if test
     suites were added/removed.
  3. Re-run this script to confirm.

EOF
    exit 1
fi

echo "OK: test counts in $TESTS_MD match ctest output"
echo "    non-CGAL: ${actual_fast}  CGAL: ${actual_cgal}  total: $((actual_fast + actual_cgal))"
exit 0