diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5490902..2a15b20 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -214,6 +214,41 @@ jobs: echo "Staging health check failed" exit 1 + smoke-test-staging: + name: Smoke Test Staging + needs: [deploy-staging] + runs-on: ubuntu-latest + environment: staging + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run smoke tests + env: + STAGING_URL: ${{ secrets.STAGING_URL }} + run: | + chmod +x scripts/smoke-test.sh + ./scripts/smoke-test.sh "$STAGING_URL" + + - name: Notify on failure + if: failure() + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + curl -s -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "{ + \"text\": \":rotating_light: *Staging smoke tests FAILED* for \`${{ github.sha }}\`\", + \"blocks\": [{ + \"type\": \"section\", + \"text\": { + \"type\": \"mrkdwn\", + \"text\": \":rotating_light: *Staging Smoke Test Failure*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\" + } + }] + }" + deploy-production: name: Deploy to Production needs: [build-api, build-web, build-ai] @@ -273,3 +308,90 @@ jobs: done echo "Production health check failed" exit 1 + + smoke-test-production: + name: Smoke Test Production + needs: [deploy-production] + runs-on: ubuntu-latest + environment: production + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run smoke tests + env: + PRODUCTION_URL: ${{ secrets.PRODUCTION_URL }} + run: | + chmod +x scripts/smoke-test.sh + ./scripts/smoke-test.sh "$PRODUCTION_URL" + + - name: Notify on success + if: success() + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + curl -s -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "{ + \"text\": \":white_check_mark: *Production deploy successful* for \`${{ github.sha }}\`\", + \"blocks\": [{ + \"type\": \"section\", + \"text\": { + \"type\": \"mrkdwn\", + \"text\": \":white_check_mark: *Production Deploy Successful*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*All smoke tests passed.*\" + } + }] + }" + + rollback-production: + name: Rollback Production + needs: [smoke-test-production] + if: failure() + runs-on: ubuntu-latest + environment: production + + steps: + - name: Rollback to previous images + env: + DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }} + DEPLOY_USER: ${{ secrets.PRODUCTION_USER }} + DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }} + run: | + mkdir -p ~/.ssh + echo "$DEPLOY_KEY" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null + + ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'ROLLBACK_SCRIPT' + cd ~/goodgo + + echo "Rolling back to previous container images..." + + # Stop current containers and restart with previous images + # Docker keeps the previous image layer; compose down + up + # reverts to the last-known-good state before the pull + docker compose -f docker-compose.prod.yml down api web ai-services + docker compose -f docker-compose.prod.yml up -d --wait api web ai-services + + echo "Rollback complete. Verifying health..." + sleep 5 + curl -sf http://localhost:3001/health || echo "WARNING: health check failed after rollback" + ROLLBACK_SCRIPT + + - name: Notify rollback + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + curl -s -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "{ + \"text\": \":warning: *Production ROLLBACK triggered* for \`${{ github.sha }}\`\", + \"blocks\": [{ + \"type\": \"section\", + \"text\": { + \"type\": \"mrkdwn\", + \"text\": \":warning: *Production Rollback Triggered*\n*Commit:* \`${{ github.sha }}\`\n*Reason:* Smoke tests failed after deploy\n*Action:* Reverted to previous container images\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\" + } + }] + }" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh new file mode 100755 index 0000000..de926ee --- /dev/null +++ b/scripts/smoke-test.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Post-deploy smoke test — validates critical API endpoints after deployment. +# Usage: ./scripts/smoke-test.sh [timeout-seconds] +# Exit codes: 0 = all checks pass, 1 = one or more checks failed + +set -euo pipefail + +BASE_URL="${1:?Usage: smoke-test.sh [timeout-seconds]}" +TIMEOUT="${2:-5}" +FAILED=0 +TOTAL=0 + +# Remove trailing slash +BASE_URL="${BASE_URL%/}" + +smoke() { + local name="$1" + local method="${2:-GET}" + local path="$3" + local expected_status="${4:-200}" + + TOTAL=$((TOTAL + 1)) + local url="${BASE_URL}${path}" + + local status + status=$(curl -s -o /dev/null -w "%{http_code}" -X "$method" \ + --max-time "$TIMEOUT" "$url" 2>/dev/null) || status="000" + + if [ "$status" = "$expected_status" ]; then + echo " PASS $name ($method $path) -> $status" + else + echo " FAIL $name ($method $path) -> $status (expected $expected_status)" + FAILED=$((FAILED + 1)) + fi +} + +echo "========================================" +echo " Smoke Tests — $(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo " Target: $BASE_URL" +echo "========================================" + +echo "" +echo "--- Health & Readiness ---" +smoke "Liveness probe" GET "/health" +smoke "Readiness probe" GET "/ready" + +echo "" +echo "--- Core API Endpoints ---" +smoke "List listings" GET "/listings" +smoke "Search" GET "/search?q=test" +smoke "Geo search" GET "/search/geo?lat=10.8&lng=106.6&radius=5" +smoke "Subscription plans" GET "/subscriptions/plans" + +echo "" +echo "--- Auth (expected responses) ---" +smoke "Login (no body -> 400)" POST "/auth/login" 400 + +echo "" +echo "========================================" +echo " Results: $((TOTAL - FAILED))/$TOTAL passed" +if [ "$FAILED" -gt 0 ]; then + echo " STATUS: FAILED ($FAILED failures)" + echo "========================================" + exit 1 +else + echo " STATUS: ALL PASSED" + echo "========================================" + exit 0 +fi