feat(ops): add automated backup restore verification

Adds pg-verify-backup.sh that restores the latest backup to an isolated
test database and verifies integrity (table existence, row counts, key
checksums, PostGIS extension, indexes, enum types). Reports pass/fail
with optional JSON output.

- Cron schedule: daily at 04:00 UTC (2h after backup)
- On-demand: docker compose run --rm pg-verify-backup
- CI: weekly GitHub Actions workflow with artifact upload

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Ho Ngoc Hai
2026-04-10 23:19:43 +07:00
parent 90839cf542
commit 8cdfe17205
3 changed files with 556 additions and 1 deletions

View File

@@ -111,7 +111,7 @@ services:
- -c
- |
apt-get update -qq && apt-get install -y -qq cron > /dev/null 2>&1
echo "0 2 * * * PGHOST=postgres PGPORT=5432 PGUSER=${DB_USER:-goodgo} PGDATABASE=${DB_NAME:-goodgo} PGPASSWORD=${DB_PASSWORD:-goodgo_secret} BACKUP_DIR=/backups RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-7} /scripts/pg-backup.sh >> /var/log/pg-backup.log 2>&1" | crontab -
(echo "0 2 * * * PGHOST=postgres PGPORT=5432 PGUSER=${DB_USER:-goodgo} PGDATABASE=${DB_NAME:-goodgo} PGPASSWORD=${DB_PASSWORD:-goodgo_secret} BACKUP_DIR=/backups RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-7} /scripts/pg-backup.sh >> /var/log/pg-backup.log 2>&1"; echo "0 4 * * * PGHOST=postgres PGPORT=5432 PGUSER=${DB_USER:-goodgo} PGDATABASE=${DB_NAME:-goodgo} PGPASSWORD=${DB_PASSWORD:-goodgo_secret} BACKUP_DIR=/backups REPORT_FILE=/backups/verify-latest.json /scripts/pg-verify-backup.sh >> /var/log/pg-verify-backup.log 2>&1") | crontab -
/scripts/pg-backup.sh
cron -f
environment:
@@ -131,6 +131,34 @@ services:
networks:
- goodgo-net
# ── Backup Verification (on-demand) ──
# Run manually: docker compose run --rm pg-verify-backup
pg-verify-backup:
image: postgis/postgis:16-3.4
container_name: goodgo-pg-verify-backup
profiles:
- tools
entrypoint: /bin/bash
command:
- -c
- /scripts/pg-verify-backup.sh
environment:
PGHOST: postgres
PGPORT: '5432'
PGUSER: ${DB_USER:-goodgo}
PGDATABASE: ${DB_NAME:-goodgo}
PGPASSWORD: ${DB_PASSWORD:-goodgo_secret}
BACKUP_DIR: /backups
REPORT_FILE: /backups/verify-report.json
volumes:
- ./scripts/backup:/scripts:ro
- pg_backups:/backups
depends_on:
postgres:
condition: service_healthy
networks:
- goodgo-net
# ── Log Aggregation ──
loki:
image: grafana/loki:3.0.0