name: Deploy

on:
  push:
    branches: [master, develop]
  workflow_dispatch:
    inputs:
      environment:
        description: Target environment
        required: true
        default: staging
        type: choice
        options:
          - staging
          - production

concurrency:
  group: deploy-${{ github.ref_name == 'develop' && 'staging' || inputs.environment || 'staging' }}
  cancel-in-progress: false

env:
  REGISTRY: ghcr.io
  REGISTRY_URL: ghcr.io/${{ github.repository_owner }}

jobs:
  build-api:
    name: Build API Image
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY_URL }}/goodgo-api
          tags: |
            type=sha,prefix=
            type=ref,event=branch
            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
            type=raw,value=staging-latest,enable=${{ github.ref == 'refs/heads/develop' }}

      - name: Build and push API image
        uses: docker/build-push-action@v6
        with:
          context: .
          file: apps/api/Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha,scope=api
          cache-to: type=gha,mode=max,scope=api

  build-web:
    name: Build Web Image
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY_URL }}/goodgo-web
          tags: |
            type=sha,prefix=
            type=ref,event=branch
            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
            type=raw,value=staging-latest,enable=${{ github.ref == 'refs/heads/develop' }}

      - name: Build and push Web image
        uses: docker/build-push-action@v6
        with:
          context: .
          file: apps/web/Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha,scope=web
          cache-to: type=gha,mode=max,scope=web

  build-ai:
    name: Build AI Services Image
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY_URL }}/goodgo-ai-services
          tags: |
            type=sha,prefix=
            type=ref,event=branch
            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
            type=raw,value=staging-latest,enable=${{ github.ref == 'refs/heads/develop' }}

      - name: Build and push AI Services image
        uses: docker/build-push-action@v6
        with:
          context: ./libs/ai-services
          file: libs/ai-services/Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha,scope=ai
          cache-to: type=gha,mode=max,scope=ai

  deploy-staging:
    name: Deploy to Staging
    needs: [build-api, build-web, build-ai]
    if: >-
      github.ref == 'refs/heads/develop' ||
      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
      (github.event_name == 'workflow_dispatch' && inputs.environment == 'staging')
    runs-on: ubuntu-latest
    environment: staging

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Record pre-deploy image tags
        id: pre-deploy
        env:
          DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
          DEPLOY_USER: ${{ secrets.STAGING_USER }}
          DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh
          echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null

          # Capture current image digests for rollback
          PREV_API=$(ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \
            "docker inspect --format='{{.Image}}' goodgo-api 2>/dev/null" || echo "none")
          PREV_WEB=$(ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \
            "docker inspect --format='{{.Image}}' goodgo-web 2>/dev/null" || echo "none")
          PREV_AI=$(ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \
            "docker inspect --format='{{.Image}}' goodgo-ai-services 2>/dev/null" || echo "none")

          echo "prev_api=$PREV_API" >> "$GITHUB_OUTPUT"
          echo "prev_web=$PREV_WEB" >> "$GITHUB_OUTPUT"
          echo "prev_ai=$PREV_AI" >> "$GITHUB_OUTPUT"

      - name: Deploy to staging
        env:
          DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
          DEPLOY_USER: ${{ secrets.STAGING_USER }}
          DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
          IMAGE_TAG: ${{ github.sha }}
        run: |
          # Copy production compose, monitoring, and infra configs
          scp -i ~/.ssh/deploy_key docker-compose.prod.yml "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/"
          scp -i ~/.ssh/deploy_key -r monitoring/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/monitoring/"
          scp -i ~/.ssh/deploy_key -r infra/pgbouncer/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/infra/pgbouncer/"
          scp -i ~/.ssh/deploy_key -r scripts/backup/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/scripts/backup/"

          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << DEPLOY_SCRIPT
            cd ~/goodgo
            export IMAGE_TAG="${IMAGE_TAG}"
            export REGISTRY_URL="${REGISTRY_URL}"

            # Login to GHCR
            echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin

            # Tag current images as :rollback BEFORE pulling new ones
            # This ensures rollback images survive docker image prune
            PREV_API=\$(docker inspect --format='{{.Config.Image}}' goodgo-api 2>/dev/null || echo "none")
            PREV_WEB=\$(docker inspect --format='{{.Config.Image}}' goodgo-web 2>/dev/null || echo "none")
            PREV_AI=\$(docker inspect --format='{{.Config.Image}}' goodgo-ai-services 2>/dev/null || echo "none")

            [ "\$PREV_API" != "none" ] && docker tag "\$PREV_API" goodgo-api:rollback 2>/dev/null || true
            [ "\$PREV_WEB" != "none" ] && docker tag "\$PREV_WEB" goodgo-web:rollback 2>/dev/null || true
            [ "\$PREV_AI" != "none" ] && docker tag "\$PREV_AI" goodgo-ai-services:rollback 2>/dev/null || true

            # Pull new images
            docker compose -f docker-compose.prod.yml pull api web ai-services

            # Rolling update — zero downtime
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait api
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait web
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait ai-services

            # Run database migrations
            docker compose -f docker-compose.prod.yml exec -T api npx prisma migrate deploy

            # NOTE: docker image prune is NOT run here — it runs after smoke tests pass
          DEPLOY_SCRIPT

      - name: Sync Nginx configs
        env:
          DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
          DEPLOY_USER: ${{ secrets.STAGING_USER }}
          DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
        run: |
          scp -i ~/.ssh/deploy_key infra/nginx/*.conf \
            "$DEPLOY_USER@$DEPLOY_HOST:/tmp/goodgo-nginx/"
          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'NGINX_SCRIPT'
            sudo mkdir -p /tmp/goodgo-nginx
            sudo cp /tmp/goodgo-nginx/*.conf /etc/nginx/sites-available/ 2>/dev/null || true
            for conf in /etc/nginx/sites-available/*goodgo*; do
              [ -f "$conf" ] && sudo ln -sf "$conf" /etc/nginx/sites-enabled/
            done
            sudo nginx -t && sudo systemctl reload nginx
          NGINX_SCRIPT

      - name: Verify staging deployment
        env:
          STAGING_URL: ${{ secrets.STAGING_URL }}
        run: |
          for i in $(seq 1 10); do
            if curl -sf "$STAGING_URL/health" > /dev/null 2>&1; then
              echo "Staging deployment verified successfully"
              exit 0
            fi
            echo "Waiting for staging to be ready... ($i/10)"
            sleep 10
          done
          echo "Staging health check failed"
          exit 1

    outputs:
      prev_api: ${{ steps.pre-deploy.outputs.prev_api }}
      prev_web: ${{ steps.pre-deploy.outputs.prev_web }}
      prev_ai: ${{ steps.pre-deploy.outputs.prev_ai }}

  smoke-test-staging:
    name: Smoke Test Staging
    needs: [deploy-staging]
    runs-on: ubuntu-latest
    environment: staging

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Run smoke tests
        env:
          STAGING_URL: ${{ secrets.STAGING_URL }}
        run: |
          chmod +x scripts/smoke-test.sh
          ./scripts/smoke-test.sh "$STAGING_URL"

      - name: Cleanup old images after successful smoke tests
        if: success()
        env:
          DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
          DEPLOY_USER: ${{ secrets.STAGING_USER }}
          DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh
          echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null

          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'CLEANUP_SCRIPT'
            cd ~/goodgo
            # Remove rollback tags — no longer needed after successful smoke tests
            docker rmi goodgo-api:rollback goodgo-web:rollback goodgo-ai-services:rollback 2>/dev/null || true
            docker image prune -f
          CLEANUP_SCRIPT

      - name: Notify on success
        if: success()
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          curl -s -X POST "$SLACK_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "{
              \"text\": \":white_check_mark: *Staging deploy successful* for \`${{ github.sha }}\`\",
              \"blocks\": [{
                \"type\": \"section\",
                \"text\": {
                  \"type\": \"mrkdwn\",
                  \"text\": \":white_check_mark: *Staging Deploy Successful*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*All smoke tests passed.*\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
                }
              }]
            }"

      - name: Notify on failure
        if: failure()
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          curl -s -X POST "$SLACK_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "{
              \"text\": \":rotating_light: *Staging smoke tests FAILED* for \`${{ github.sha }}\`\",
              \"blocks\": [{
                \"type\": \"section\",
                \"text\": {
                  \"type\": \"mrkdwn\",
                  \"text\": \":rotating_light: *Staging Smoke Test Failure*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Action:* Automatic rollback initiated\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
                }
              }]
            }"

  rollback-staging:
    name: Rollback Staging
    needs: [deploy-staging, smoke-test-staging]
    if: failure()
    runs-on: ubuntu-latest
    environment: staging

    steps:
      - name: Rollback to previous images
        env:
          DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
          DEPLOY_USER: ${{ secrets.STAGING_USER }}
          DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
          REGISTRY_URL: ${{ env.REGISTRY_URL }}
          IMAGE_TAG: ${{ github.sha }}
        run: |
          mkdir -p ~/.ssh
          echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null

          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << ROLLBACK_SCRIPT
            cd ~/goodgo

            echo "Rolling back staging using :rollback tagged images..."

            REGISTRY_URL="${REGISTRY_URL}"
            IMAGE_TAG="${IMAGE_TAG}"

            # Stop current containers
            docker compose -f docker-compose.prod.yml stop api web ai-services

            # Retag :rollback images to match compose image template so compose uses them
            for svc in goodgo-api goodgo-web goodgo-ai-services; do
              if docker image inspect "\${svc}:rollback" > /dev/null 2>&1; then
                echo "Restoring \${svc} from :rollback tag"
                docker tag "\${svc}:rollback" "\${REGISTRY_URL}/\${svc}:\${IMAGE_TAG}"
              else
                echo "WARNING: No rollback image for \${svc}"
              fi
            done

            # Restart with rollback images (now tagged to match compose template)
            export IMAGE_TAG REGISTRY_URL
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait api web ai-services

            echo "Rollback complete. Verifying health..."
            sleep 5
            curl -sf http://localhost:3001/health || echo "WARNING: health check failed after rollback"
          ROLLBACK_SCRIPT

      - name: Notify rollback
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          curl -s -X POST "$SLACK_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "{
              \"text\": \":warning: *Staging ROLLBACK triggered* for \`${{ github.sha }}\`\",
              \"blocks\": [{
                \"type\": \"section\",
                \"text\": {
                  \"type\": \"mrkdwn\",
                  \"text\": \":warning: *Staging Rollback Triggered*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Reason:* Smoke tests failed after deploy\n*Action:* Reverted to previous container images using :rollback tags\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
                }
              }]
            }"

  deploy-production:
    name: Deploy to Production
    needs: [build-api, build-web, build-ai]
    if: inputs.environment == 'production'
    runs-on: ubuntu-latest
    environment: production

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Deploy to production
        env:
          DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }}
          DEPLOY_USER: ${{ secrets.PRODUCTION_USER }}
          DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
          IMAGE_TAG: ${{ github.sha }}
        run: |
          mkdir -p ~/.ssh
          echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null

          # Copy configs
          scp -i ~/.ssh/deploy_key docker-compose.prod.yml "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/"
          scp -i ~/.ssh/deploy_key -r monitoring/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/monitoring/"
          scp -i ~/.ssh/deploy_key -r infra/pgbouncer/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/infra/pgbouncer/"
          scp -i ~/.ssh/deploy_key -r scripts/backup/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/scripts/backup/"

          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << DEPLOY_SCRIPT
            cd ~/goodgo
            export IMAGE_TAG="${IMAGE_TAG}"
            export REGISTRY_URL="${REGISTRY_URL}"

            echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin

            # Tag current images as :rollback BEFORE pulling new ones
            PREV_API=\$(docker inspect --format='{{.Config.Image}}' goodgo-api 2>/dev/null || echo "none")
            PREV_WEB=\$(docker inspect --format='{{.Config.Image}}' goodgo-web 2>/dev/null || echo "none")
            PREV_AI=\$(docker inspect --format='{{.Config.Image}}' goodgo-ai-services 2>/dev/null || echo "none")

            [ "\$PREV_API" != "none" ] && docker tag "\$PREV_API" goodgo-api:rollback 2>/dev/null || true
            [ "\$PREV_WEB" != "none" ] && docker tag "\$PREV_WEB" goodgo-web:rollback 2>/dev/null || true
            [ "\$PREV_AI" != "none" ] && docker tag "\$PREV_AI" goodgo-ai-services:rollback 2>/dev/null || true

            docker compose -f docker-compose.prod.yml pull api web ai-services

            # Rolling update with health checks
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait api
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait web
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait ai-services

            docker compose -f docker-compose.prod.yml exec -T api npx prisma migrate deploy

            # NOTE: docker image prune is NOT run here — it runs after smoke tests pass
          DEPLOY_SCRIPT

      - name: Sync Nginx configs (production)
        env:
          DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }}
          DEPLOY_USER: ${{ secrets.PRODUCTION_USER }}
          DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
        run: |
          scp -i ~/.ssh/deploy_key infra/nginx/*.conf \
            "$DEPLOY_USER@$DEPLOY_HOST:/tmp/goodgo-nginx/"
          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'NGINX_SCRIPT'
            sudo cp /tmp/goodgo-nginx/*.conf /etc/nginx/sites-available/ 2>/dev/null || true
            for conf in /etc/nginx/sites-available/*goodgo*; do
              [ -f "$conf" ] && sudo ln -sf "$conf" /etc/nginx/sites-enabled/
            done
            sudo nginx -t && sudo systemctl reload nginx
          NGINX_SCRIPT

      - name: Verify production deployment
        env:
          PRODUCTION_URL: ${{ secrets.PRODUCTION_URL }}
        run: |
          for i in $(seq 1 10); do
            if curl -sf "$PRODUCTION_URL/health" > /dev/null 2>&1; then
              echo "Production deployment verified successfully"
              exit 0
            fi
            echo "Waiting for production to be ready... ($i/10)"
            sleep 10
          done
          echo "Production health check failed"
          exit 1

  smoke-test-production:
    name: Smoke Test Production
    needs: [deploy-production]
    runs-on: ubuntu-latest
    environment: production

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Run smoke tests
        env:
          PRODUCTION_URL: ${{ secrets.PRODUCTION_URL }}
        run: |
          chmod +x scripts/smoke-test.sh
          ./scripts/smoke-test.sh "$PRODUCTION_URL"

      - name: Cleanup old images after successful smoke tests
        if: success()
        env:
          DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }}
          DEPLOY_USER: ${{ secrets.PRODUCTION_USER }}
          DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh
          echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null

          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'CLEANUP_SCRIPT'
            cd ~/goodgo
            # Remove rollback tags — no longer needed after successful smoke tests
            docker rmi goodgo-api:rollback goodgo-web:rollback goodgo-ai-services:rollback 2>/dev/null || true
            docker image prune -f
          CLEANUP_SCRIPT

      - name: Notify on success
        if: success()
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          curl -s -X POST "$SLACK_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "{
              \"text\": \":white_check_mark: *Production deploy successful* for \`${{ github.sha }}\`\",
              \"blocks\": [{
                \"type\": \"section\",
                \"text\": {
                  \"type\": \"mrkdwn\",
                  \"text\": \":white_check_mark: *Production Deploy Successful*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*All smoke tests passed.*\"
                }
              }]
            }"

  rollback-production:
    name: Rollback Production
    needs: [smoke-test-production]
    if: failure()
    runs-on: ubuntu-latest
    environment: production

    steps:
      - name: Rollback to previous images
        env:
          DEPLOY_HOST: ${{ secrets.PRODUCTION_HOST }}
          DEPLOY_USER: ${{ secrets.PRODUCTION_USER }}
          DEPLOY_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
          REGISTRY_URL: ${{ env.REGISTRY_URL }}
          IMAGE_TAG: ${{ github.sha }}
        run: |
          mkdir -p ~/.ssh
          echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null

          ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << ROLLBACK_SCRIPT
            cd ~/goodgo

            echo "Rolling back production using :rollback tagged images..."

            REGISTRY_URL="${REGISTRY_URL}"
            IMAGE_TAG="${IMAGE_TAG}"

            # Stop current containers
            docker compose -f docker-compose.prod.yml stop api web ai-services

            # Retag :rollback images to match compose image template so compose uses them
            for svc in goodgo-api goodgo-web goodgo-ai-services; do
              if docker image inspect "\${svc}:rollback" > /dev/null 2>&1; then
                echo "Restoring \${svc} from :rollback tag"
                docker tag "\${svc}:rollback" "\${REGISTRY_URL}/\${svc}:\${IMAGE_TAG}"
              else
                echo "WARNING: No rollback image for \${svc}"
              fi
            done

            # Restart with rollback images (now tagged to match compose template)
            export IMAGE_TAG REGISTRY_URL
            docker compose -f docker-compose.prod.yml up -d --no-deps --wait api web ai-services

            echo "Rollback complete. Verifying health..."
            sleep 5
            curl -sf http://localhost:3001/health || echo "WARNING: health check failed after rollback"
          ROLLBACK_SCRIPT

      - name: Notify rollback
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          curl -s -X POST "$SLACK_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "{
              \"text\": \":warning: *Production ROLLBACK triggered* for \`${{ github.sha }}\`\",
              \"blocks\": [{
                \"type\": \"section\",
                \"text\": {
                  \"type\": \"mrkdwn\",
                  \"text\": \":warning: *Production Rollback Triggered*\n*Commit:* \`${{ github.sha }}\`\n*Reason:* Smoke tests failed after deploy\n*Action:* Reverted to previous container images using :rollback tags\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
                }
              }]
            }"