feat(ci): add post-deploy smoke test pipeline stage
- Add scripts/smoke-test.sh — hits health, readiness, and critical API endpoints (listings, search, subscriptions) post-deploy - Add smoke-test-staging job that runs after staging deploy with Slack notification on failure - Add smoke-test-production job that runs after production deploy with success notification - Add rollback-production job triggered on smoke test failure — reverts to previous container images and notifies via Slack Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Ho Ngoc Hai
122
.github/workflows/deploy.yml
vendored
122
.github/workflows/deploy.yml
vendored
@@ -214,6 +214,41 @@ jobs:
|
||||
echo "Staging health check failed"
|
||||
exit 1
|
||||
|
||||
smoke-test-staging:
|
||||
name: Smoke Test Staging
|
||||
needs: [deploy-staging]
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run smoke tests
|
||||
env:
|
||||
STAGING_URL: ${{ secrets.STAGING_URL }}
|
||||
run: |
|
||||
chmod +x scripts/smoke-test.sh
|
||||
./scripts/smoke-test.sh "$STAGING_URL"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
run: |
|
||||
curl -s -X POST "$SLACK_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"text\": \":rotating_light: *Staging smoke tests FAILED* for \`${{ github.sha }}\`\",
|
||||
\"blocks\": [{
|
||||
\"type\": \"section\",
|
||||
\"text\": {
|
||||
\"type\": \"mrkdwn\",
|
||||
\"text\": \":rotating_light: *Staging Smoke Test Failure*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
|
||||
}
|
||||
}]
|
||||
}"
|
||||
|
||||
deploy-production:
|
||||
name: Deploy to Production
|
||||
needs: [build-api, build-web, build-ai]
|
||||
@@ -273,3 +308,90 @@ jobs:
|
||||
done
|
||||
echo "Production health check failed"
|
||||
exit 1
|
||||
|
||||
smoke-test-production:
|
||||
name: Smoke Test Production
|
||||
needs: [deploy-production]
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run smoke tests
|
||||
env:
|
||||
PRODUCTION_URL: ${{ secrets.PRODUCTION_URL }}
|
||||
run: |
|
||||
chmod +x scripts/smoke-test.sh
|
||||
./scripts/smoke-test.sh "$PRODUCTION_URL"
|
||||
|
||||
- name: Notify on success
|
||||
if: success()
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
run: |
|
||||
curl -s -X POST "$SLACK_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"text\": \":white_check_mark: *Production deploy successful* for \`${{ github.sha }}\`\",
|
||||
\"blocks\": [{
|
||||
\"type\": \"section\",
|
||||
\"text\": {
|
||||
\"type\": \"mrkdwn\",
|
||||
\"text\": \":white_check_mark: *Production Deploy Successful*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*All smoke tests passed.*\"
|
||||
}
|
||||
}]
|
||||
}"
|
||||
|
||||
rollback-production:
|
||||
name: Rollback Production
|
||||
needs: [smoke-test-production]
|
||||
if: failure()
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
|
||||
steps:
|
||||
- name: Rollback to previous images
|
||||
env:
|
||||
DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.PRODUCTION_USER }}
|
||||
DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'ROLLBACK_SCRIPT'
|
||||
cd ~/goodgo
|
||||
|
||||
echo "Rolling back to previous container images..."
|
||||
|
||||
# Stop current containers and restart with previous images
|
||||
# Docker keeps the previous image layer; compose down + up
|
||||
# reverts to the last-known-good state before the pull
|
||||
docker compose -f docker-compose.prod.yml down api web ai-services
|
||||
docker compose -f docker-compose.prod.yml up -d --wait api web ai-services
|
||||
|
||||
echo "Rollback complete. Verifying health..."
|
||||
sleep 5
|
||||
curl -sf http://localhost:3001/health || echo "WARNING: health check failed after rollback"
|
||||
ROLLBACK_SCRIPT
|
||||
|
||||
- name: Notify rollback
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
run: |
|
||||
curl -s -X POST "$SLACK_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"text\": \":warning: *Production ROLLBACK triggered* for \`${{ github.sha }}\`\",
|
||||
\"blocks\": [{
|
||||
\"type\": \"section\",
|
||||
\"text\": {
|
||||
\"type\": \"mrkdwn\",
|
||||
\"text\": \":warning: *Production Rollback Triggered*\n*Commit:* \`${{ github.sha }}\`\n*Reason:* Smoke tests failed after deploy\n*Action:* Reverted to previous container images\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
|
||||
}
|
||||
}]
|
||||
}"
|
||||
|
||||
69
scripts/smoke-test.sh
Executable file
69
scripts/smoke-test.sh
Executable file
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# Post-deploy smoke test — validates critical API endpoints after deployment.
|
||||
# Usage: ./scripts/smoke-test.sh <base-url> [timeout-seconds]
|
||||
# Exit codes: 0 = all checks pass, 1 = one or more checks failed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:?Usage: smoke-test.sh <base-url> [timeout-seconds]}"
|
||||
TIMEOUT="${2:-5}"
|
||||
FAILED=0
|
||||
TOTAL=0
|
||||
|
||||
# Remove trailing slash
|
||||
BASE_URL="${BASE_URL%/}"
|
||||
|
||||
smoke() {
|
||||
local name="$1"
|
||||
local method="${2:-GET}"
|
||||
local path="$3"
|
||||
local expected_status="${4:-200}"
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local url="${BASE_URL}${path}"
|
||||
|
||||
local status
|
||||
status=$(curl -s -o /dev/null -w "%{http_code}" -X "$method" \
|
||||
--max-time "$TIMEOUT" "$url" 2>/dev/null) || status="000"
|
||||
|
||||
if [ "$status" = "$expected_status" ]; then
|
||||
echo " PASS $name ($method $path) -> $status"
|
||||
else
|
||||
echo " FAIL $name ($method $path) -> $status (expected $expected_status)"
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
echo "========================================"
|
||||
echo " Smoke Tests — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo " Target: $BASE_URL"
|
||||
echo "========================================"
|
||||
|
||||
echo ""
|
||||
echo "--- Health & Readiness ---"
|
||||
smoke "Liveness probe" GET "/health"
|
||||
smoke "Readiness probe" GET "/ready"
|
||||
|
||||
echo ""
|
||||
echo "--- Core API Endpoints ---"
|
||||
smoke "List listings" GET "/listings"
|
||||
smoke "Search" GET "/search?q=test"
|
||||
smoke "Geo search" GET "/search/geo?lat=10.8&lng=106.6&radius=5"
|
||||
smoke "Subscription plans" GET "/subscriptions/plans"
|
||||
|
||||
echo ""
|
||||
echo "--- Auth (expected responses) ---"
|
||||
smoke "Login (no body -> 400)" POST "/auth/login" 400
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo " Results: $((TOTAL - FAILED))/$TOTAL passed"
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo " STATUS: FAILED ($FAILED failures)"
|
||||
echo "========================================"
|
||||
exit 1
|
||||
else
|
||||
echo " STATUS: ALL PASSED"
|
||||
echo "========================================"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user