feat(ci): add post-deploy smoke test pipeline stage

- Add scripts/smoke-test.sh — hits health, readiness, and critical API
  endpoints (listings, search, subscriptions) post-deploy
- Add smoke-test-staging job that runs after staging deploy with Slack
  notification on failure
- Add smoke-test-production job that runs after production deploy with
  success notification
- Add rollback-production job triggered on smoke test failure — reverts
  to previous container images and notifies via Slack

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Ho Ngoc Hai
2026-04-09 09:09:09 +07:00
parent b23be886b1
commit e6d38c796f
2 changed files with 191 additions and 0 deletions

View File

@@ -214,6 +214,41 @@ jobs:
echo "Staging health check failed"
exit 1
smoke-test-staging:
name: Smoke Test Staging
needs: [deploy-staging]
runs-on: ubuntu-latest
environment: staging
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Run smoke tests
env:
STAGING_URL: ${{ secrets.STAGING_URL }}
run: |
chmod +x scripts/smoke-test.sh
./scripts/smoke-test.sh "$STAGING_URL"
- name: Notify on failure
if: failure()
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
run: |
curl -s -X POST "$SLACK_WEBHOOK" \
-H "Content-Type: application/json" \
-d "{
\"text\": \":rotating_light: *Staging smoke tests FAILED* for \`${{ github.sha }}\`\",
\"blocks\": [{
\"type\": \"section\",
\"text\": {
\"type\": \"mrkdwn\",
\"text\": \":rotating_light: *Staging Smoke Test Failure*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
}
}]
}"
deploy-production:
name: Deploy to Production
needs: [build-api, build-web, build-ai]
@@ -273,3 +308,90 @@ jobs:
done
echo "Production health check failed"
exit 1
smoke-test-production:
name: Smoke Test Production
needs: [deploy-production]
runs-on: ubuntu-latest
environment: production
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Run smoke tests
env:
PRODUCTION_URL: ${{ secrets.PRODUCTION_URL }}
run: |
chmod +x scripts/smoke-test.sh
./scripts/smoke-test.sh "$PRODUCTION_URL"
- name: Notify on success
if: success()
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
run: |
curl -s -X POST "$SLACK_WEBHOOK" \
-H "Content-Type: application/json" \
-d "{
\"text\": \":white_check_mark: *Production deploy successful* for \`${{ github.sha }}\`\",
\"blocks\": [{
\"type\": \"section\",
\"text\": {
\"type\": \"mrkdwn\",
\"text\": \":white_check_mark: *Production Deploy Successful*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*All smoke tests passed.*\"
}
}]
}"
rollback-production:
name: Rollback Production
needs: [smoke-test-production]
if: failure()
runs-on: ubuntu-latest
environment: production
steps:
- name: Rollback to previous images
env:
DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }}
DEPLOY_USER: ${{ secrets.PRODUCTION_USER }}
DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
run: |
mkdir -p ~/.ssh
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'ROLLBACK_SCRIPT'
cd ~/goodgo
echo "Rolling back to previous container images..."
# Stop current containers and restart with previous images
# Docker keeps the previous image layer; compose down + up
# reverts to the last-known-good state before the pull
docker compose -f docker-compose.prod.yml down api web ai-services
docker compose -f docker-compose.prod.yml up -d --wait api web ai-services
echo "Rollback complete. Verifying health..."
sleep 5
curl -sf http://localhost:3001/health || echo "WARNING: health check failed after rollback"
ROLLBACK_SCRIPT
- name: Notify rollback
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
run: |
curl -s -X POST "$SLACK_WEBHOOK" \
-H "Content-Type: application/json" \
-d "{
\"text\": \":warning: *Production ROLLBACK triggered* for \`${{ github.sha }}\`\",
\"blocks\": [{
\"type\": \"section\",
\"text\": {
\"type\": \"mrkdwn\",
\"text\": \":warning: *Production Rollback Triggered*\n*Commit:* \`${{ github.sha }}\`\n*Reason:* Smoke tests failed after deploy\n*Action:* Reverted to previous container images\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
}
}]
}"