#!/bin/bash set -euo pipefail # ── PostgreSQL Backup Restore Verification Script ── # Restores the latest backup to an isolated test database, verifies data # integrity (table existence, row counts, key checksums), and reports pass/fail. # # SAFETY: Never touches the production database. Creates a temporary # "goodgo_verify_" database and drops it on exit. # # Usage: # ./pg-verify-backup.sh # Verify latest backup # ./pg-verify-backup.sh # Verify specific backup # SKIP_CLEANUP=1 ./pg-verify-backup.sh # Keep test DB for inspection # # Environment variables: # BACKUP_DIR — Directory containing backups (default: /backups) # PGHOST — PostgreSQL host (default: postgres) # PGPORT — PostgreSQL port (default: 5432) # PGUSER — PostgreSQL user (default: goodgo) # PGPASSWORD — PostgreSQL password (from environment) # PGDATABASE — Source/production database name (default: goodgo) # SKIP_CLEANUP — Set to 1 to keep test database after verification # REPORT_FILE — Path for JSON report output (optional) # # Exit codes: # 0 — All checks passed # 1 — One or more checks failed # 2 — Setup error (no backups found, restore failed, etc.) # ── Configuration ── BACKUP_DIR="${BACKUP_DIR:-/backups}" PGHOST="${PGHOST:-postgres}" PGPORT="${PGPORT:-5432}" PGUSER="${PGUSER:-goodgo}" PGDATABASE="${PGDATABASE:-goodgo}" SKIP_CLEANUP="${SKIP_CLEANUP:-0}" REPORT_FILE="${REPORT_FILE:-}" TIMESTAMP=$(date +%Y%m%d_%H%M%S) VERIFY_DB="goodgo_verify_${TIMESTAMP}" PASSED=0 FAILED=0 WARNINGS=0 RESULTS=() # ── Color output (if terminal) ── if [ -t 1 ]; then GREEN='\033[0;32m' RED='\033[0;31m' YELLOW='\033[0;33m' BLUE='\033[0;34m' NC='\033[0m' else GREEN='' RED='' YELLOW='' BLUE='' NC='' fi log_pass() { PASSED=$((PASSED + 1)) RESULTS+=("{\"check\":\"$1\",\"status\":\"pass\",\"detail\":\"$2\"}") echo -e "${GREEN}[PASS]${NC} $1: $2" } log_fail() { FAILED=$((FAILED + 1)) RESULTS+=("{\"check\":\"$1\",\"status\":\"fail\",\"detail\":\"$2\"}") echo -e "${RED}[FAIL]${NC} $1: $2" } log_warn() { WARNINGS=$((WARNINGS + 1)) RESULTS+=("{\"check\":\"$1\",\"status\":\"warn\",\"detail\":\"$2\"}") echo -e "${YELLOW}[WARN]${NC} $1: $2" } log_info() { echo -e "${BLUE}[INFO]${NC} $1" } # ── Cleanup function ── cleanup() { local exit_code=$? if [ "${SKIP_CLEANUP}" = "1" ]; then log_info "SKIP_CLEANUP=1 — keeping test database '${VERIFY_DB}' for inspection" log_info "Drop manually: psql -h ${PGHOST} -U ${PGUSER} -d postgres -c 'DROP DATABASE IF EXISTS \"${VERIFY_DB}\";'" else log_info "Cleaning up test database '${VERIFY_DB}'..." psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d postgres -c \ "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${VERIFY_DB}' AND pid <> pg_backend_pid();" \ > /dev/null 2>&1 || true psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d postgres -c \ "DROP DATABASE IF EXISTS \"${VERIFY_DB}\";" \ > /dev/null 2>&1 || true log_info "Test database dropped." fi return $exit_code } trap cleanup EXIT # ── Determine backup file ── BACKUP_FILE="${1:-}" if [ -z "${BACKUP_FILE}" ]; then log_info "Finding latest backup in ${BACKUP_DIR}..." BACKUP_FILE=$(ls -t "${BACKUP_DIR}"/goodgo_*.sql.gz 2>/dev/null | head -n1 || true) if [ -z "${BACKUP_FILE}" ]; then echo -e "${RED}[ERROR]${NC} No backup files found in ${BACKUP_DIR}" exit 2 fi fi if [ ! -f "${BACKUP_FILE}" ]; then echo -e "${RED}[ERROR]${NC} Backup file not found: ${BACKUP_FILE}" exit 2 fi BACKUP_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1) BACKUP_MTIME=$(stat -c '%Y' "${BACKUP_FILE}" 2>/dev/null || stat -f '%m' "${BACKUP_FILE}" 2>/dev/null || echo "unknown") echo "" echo "================================================================" echo " GoodGo Backup Restore Verification" echo "================================================================" echo " Backup file : ${BACKUP_FILE}" echo " Backup size : ${BACKUP_SIZE}" echo " Source DB : ${PGHOST}:${PGPORT}/${PGDATABASE}" echo " Test DB : ${VERIFY_DB}" echo " Started at : $(date -Iseconds)" echo "================================================================" echo "" # ── Step 1: Create isolated test database ── log_info "Step 1/5: Creating isolated test database '${VERIFY_DB}'..." psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d postgres -c \ "CREATE DATABASE \"${VERIFY_DB}\";" > /dev/null 2>&1 if ! psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d postgres -c \ "SELECT 1 FROM pg_database WHERE datname = '${VERIFY_DB}';" 2>/dev/null | grep -q 1; then echo -e "${RED}[ERROR]${NC} Failed to create test database" exit 2 fi log_pass "Database creation" "Test database '${VERIFY_DB}' created" # ── Step 2: Restore backup into test database ── log_info "Step 2/5: Restoring backup to test database..." RESTORE_START=$(date +%s) # Enable PostGIS extension before restore (required for geometry columns) psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" -c \ "CREATE EXTENSION IF NOT EXISTS postgis;" > /dev/null 2>&1 RESTORE_OUTPUT=$(pg_restore \ -h "${PGHOST}" \ -p "${PGPORT}" \ -U "${PGUSER}" \ -d "${VERIFY_DB}" \ --no-owner \ --no-privileges \ --clean \ --if-exists \ "${BACKUP_FILE}" 2>&1) || true RESTORE_END=$(date +%s) RESTORE_DURATION=$((RESTORE_END - RESTORE_START)) # Check if any critical errors occurred (ignore warnings about objects not existing) RESTORE_ERRORS=$(echo "${RESTORE_OUTPUT}" | grep -ci "ERROR" || true) if [ "${RESTORE_ERRORS}" -gt 0 ]; then # Filter out harmless "does not exist" errors from --clean --if-exists CRITICAL_ERRORS=$(echo "${RESTORE_OUTPUT}" | grep -i "ERROR" | grep -cv "does not exist" || true) if [ "${CRITICAL_ERRORS}" -gt 0 ]; then log_fail "Restore" "pg_restore completed with ${CRITICAL_ERRORS} critical error(s) in ${RESTORE_DURATION}s" echo "${RESTORE_OUTPUT}" | grep -i "ERROR" | grep -v "does not exist" | head -5 else log_pass "Restore" "pg_restore completed in ${RESTORE_DURATION}s (non-critical warnings only)" fi else log_pass "Restore" "pg_restore completed cleanly in ${RESTORE_DURATION}s" fi # ── Step 3: Verify table existence ── log_info "Step 3/5: Verifying table existence..." # Expected tables from Prisma schema (22 models + Prisma migration tracking) EXPECTED_TABLES=( "User" "RefreshToken" "OAuthAccount" "Agent" "Property" "PropertyMedia" "Listing" "SavedSearch" "Transaction" "Inquiry" "Lead" "Payment" "Plan" "Subscription" "UsageRecord" "Valuation" "MarketIndex" "NotificationLog" "NotificationPreference" "AdminAuditLog" "Review" "_prisma_migrations" ) ACTUAL_TABLES=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" \ -t -A -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename;" 2>/dev/null) MISSING_TABLES=() for table in "${EXPECTED_TABLES[@]}"; do if echo "${ACTUAL_TABLES}" | grep -qx "${table}"; then : # Table exists else MISSING_TABLES+=("${table}") fi done ACTUAL_COUNT=$(echo "${ACTUAL_TABLES}" | grep -c . || true) EXPECTED_COUNT=${#EXPECTED_TABLES[@]} if [ ${#MISSING_TABLES[@]} -eq 0 ]; then log_pass "Table existence" "All ${EXPECTED_COUNT} expected tables present (${ACTUAL_COUNT} total)" else log_fail "Table existence" "Missing ${#MISSING_TABLES[@]} table(s): ${MISSING_TABLES[*]}" fi # ── Step 4: Row count comparison ── log_info "Step 4/5: Comparing row counts between source and restored databases..." ROW_MISMATCHES=0 ROW_REPORT="" for table in "${EXPECTED_TABLES[@]}"; do if [ "${table}" = "_prisma_migrations" ]; then continue # Skip migration tracking table for row comparison fi # Skip missing tables if [[ " ${MISSING_TABLES[*]} " =~ " ${table} " ]]; then continue fi SOURCE_COUNT=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" \ -t -A -c "SELECT count(*) FROM \"${table}\";" 2>/dev/null || echo "ERROR") VERIFY_COUNT=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" \ -t -A -c "SELECT count(*) FROM \"${table}\";" 2>/dev/null || echo "ERROR") if [ "${SOURCE_COUNT}" = "ERROR" ] || [ "${VERIFY_COUNT}" = "ERROR" ]; then log_warn "Row count: ${table}" "Could not query row count" continue fi ROW_REPORT="${ROW_REPORT}\n ${table}: source=${SOURCE_COUNT} restored=${VERIFY_COUNT}" if [ "${SOURCE_COUNT}" != "${VERIFY_COUNT}" ]; then ROW_MISMATCHES=$((ROW_MISMATCHES + 1)) log_fail "Row count: ${table}" "Mismatch — source=${SOURCE_COUNT} restored=${VERIFY_COUNT}" fi done if [ "${ROW_MISMATCHES}" -eq 0 ]; then log_pass "Row counts" "All tables match between source and restored database" else log_fail "Row counts" "${ROW_MISMATCHES} table(s) have row count mismatches" fi # ── Step 5: Key data checksums ── log_info "Step 5/5: Verifying data checksums on critical tables..." # Checksum key tables by hashing sorted IDs + key fields verify_checksum() { local table="$1" local query="$2" local label="$3" # Skip if table is missing if [[ " ${MISSING_TABLES[*]} " =~ " ${table} " ]]; then log_warn "Checksum: ${label}" "Table '${table}' missing, skipped" return fi SOURCE_HASH=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" \ -t -A -c "${query}" 2>/dev/null || echo "ERROR") VERIFY_HASH=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" \ -t -A -c "${query}" 2>/dev/null || echo "ERROR") if [ "${SOURCE_HASH}" = "ERROR" ] || [ "${VERIFY_HASH}" = "ERROR" ]; then log_warn "Checksum: ${label}" "Could not compute checksum" return fi if [ "${SOURCE_HASH}" = "${VERIFY_HASH}" ]; then log_pass "Checksum: ${label}" "Hashes match (${SOURCE_HASH:0:16}...)" else log_fail "Checksum: ${label}" "Hash mismatch — source=${SOURCE_HASH:0:16}... restored=${VERIFY_HASH:0:16}..." fi } # Checksum queries for critical tables verify_checksum "User" \ "SELECT md5(string_agg(id || email || phone || \"fullName\" || role::text || \"kycStatus\"::text, '|' ORDER BY id)) FROM \"User\";" \ "User identities" verify_checksum "Property" \ "SELECT md5(string_agg(id || title || district || city || \"propertyType\"::text || \"areaM2\"::text, '|' ORDER BY id)) FROM \"Property\";" \ "Property records" verify_checksum "Listing" \ "SELECT md5(string_agg(id || \"propertyId\" || \"sellerId\" || status::text || \"priceVND\"::text || \"transactionType\"::text, '|' ORDER BY id)) FROM \"Listing\";" \ "Listing records" verify_checksum "Payment" \ "SELECT md5(string_agg(id || \"userId\" || provider::text || type::text || \"amountVND\"::text || status::text, '|' ORDER BY id)) FROM \"Payment\";" \ "Payment records" verify_checksum "Subscription" \ "SELECT md5(string_agg(id || \"userId\" || \"planId\" || status::text, '|' ORDER BY id)) FROM \"Subscription\";" \ "Subscription records" verify_checksum "Transaction" \ "SELECT md5(string_agg(id || \"listingId\" || \"buyerId\" || status::text, '|' ORDER BY id)) FROM \"Transaction\";" \ "Transaction records" verify_checksum "Plan" \ "SELECT md5(string_agg(id || tier::text || name || \"priceMonthlyVND\"::text, '|' ORDER BY id)) FROM \"Plan\";" \ "Plan records" verify_checksum "_prisma_migrations" \ "SELECT md5(string_agg(id || migration_name || checksum, '|' ORDER BY started_at)) FROM \"_prisma_migrations\";" \ "Migration history" # ── PostGIS extension check ── POSTGIS_CHECK=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" \ -t -A -c "SELECT extname FROM pg_extension WHERE extname = 'postgis';" 2>/dev/null || echo "") if [ "${POSTGIS_CHECK}" = "postgis" ]; then log_pass "PostGIS extension" "PostGIS is available in restored database" else log_fail "PostGIS extension" "PostGIS extension not found in restored database" fi # ── Index verification ── SOURCE_INDEX_COUNT=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" \ -t -A -c "SELECT count(*) FROM pg_indexes WHERE schemaname = 'public';" 2>/dev/null || echo "0") VERIFY_INDEX_COUNT=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" \ -t -A -c "SELECT count(*) FROM pg_indexes WHERE schemaname = 'public';" 2>/dev/null || echo "0") if [ "${SOURCE_INDEX_COUNT}" = "${VERIFY_INDEX_COUNT}" ]; then log_pass "Index count" "All ${SOURCE_INDEX_COUNT} indexes restored" else log_warn "Index count" "Source has ${SOURCE_INDEX_COUNT} indexes, restored has ${VERIFY_INDEX_COUNT}" fi # ── Enum type verification ── SOURCE_ENUM_COUNT=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" \ -t -A -c "SELECT count(*) FROM pg_type WHERE typtype = 'e';" 2>/dev/null || echo "0") VERIFY_ENUM_COUNT=$(psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${VERIFY_DB}" \ -t -A -c "SELECT count(*) FROM pg_type WHERE typtype = 'e';" 2>/dev/null || echo "0") if [ "${SOURCE_ENUM_COUNT}" = "${VERIFY_ENUM_COUNT}" ]; then log_pass "Enum types" "All ${SOURCE_ENUM_COUNT} enum types restored" else log_fail "Enum types" "Source has ${SOURCE_ENUM_COUNT} enums, restored has ${VERIFY_ENUM_COUNT}" fi # ── Summary ── echo "" echo "================================================================" echo " Verification Summary" echo "================================================================" echo " Backup file : ${BACKUP_FILE}" echo " Restore time : ${RESTORE_DURATION}s" TOTAL=$((PASSED + FAILED)) echo " Checks : ${TOTAL} total, ${PASSED} passed, ${FAILED} failed, ${WARNINGS} warnings" echo " Finished at : $(date -Iseconds)" if [ "${FAILED}" -eq 0 ]; then echo -e " Result : ${GREEN}ALL CHECKS PASSED${NC}" echo "================================================================" else echo -e " Result : ${RED}${FAILED} CHECK(S) FAILED${NC}" echo "================================================================" fi echo "" # ── Write JSON report (if requested) ── if [ -n "${REPORT_FILE}" ]; then RESULTS_JSON=$(printf '%s,' "${RESULTS[@]}") RESULTS_JSON="[${RESULTS_JSON%,}]" cat > "${REPORT_FILE}" << JSONEOF { "timestamp": "$(date -Iseconds)", "backupFile": "${BACKUP_FILE}", "backupSize": "${BACKUP_SIZE}", "testDatabase": "${VERIFY_DB}", "restoreDurationSeconds": ${RESTORE_DURATION}, "passed": ${PASSED}, "failed": ${FAILED}, "warnings": ${WARNINGS}, "result": "$([ ${FAILED} -eq 0 ] && echo "pass" || echo "fail")", "checks": ${RESULTS_JSON} } JSONEOF log_info "JSON report written to ${REPORT_FILE}" fi # Exit with appropriate code if [ "${FAILED}" -gt 0 ]; then exit 1 fi exit 0