feat(devops): add staging auto-deploy pipeline on develop branch
- Trigger deploy workflow on push to `develop` branch (in addition to `master`) - Add `staging-latest` Docker image tag for develop branch builds - Add `rollback-staging` job: auto-reverts to previous images on smoke test failure - Add Slack success notification for staging deploys (previously only failure was notified) - Record pre-deploy image digests for rollback capability - Update deployment docs with CI/CD pipeline details, rollback procedures, and required secrets Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Ho Ngoc Hai
118
.github/workflows/deploy.yml
vendored
118
.github/workflows/deploy.yml
vendored
@@ -2,7 +2,7 @@ name: Deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
branches: [master, develop]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
@@ -15,7 +15,7 @@ on:
|
||||
- production
|
||||
|
||||
concurrency:
|
||||
group: deploy-${{ inputs.environment || 'staging' }}
|
||||
group: deploy-${{ github.ref_name == 'develop' && 'staging' || inputs.environment || 'staging' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
@@ -53,6 +53,7 @@ jobs:
|
||||
type=sha,prefix=
|
||||
type=ref,event=branch
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=staging-latest,enable=${{ github.ref == 'refs/heads/develop' }}
|
||||
|
||||
- name: Build and push API image
|
||||
uses: docker/build-push-action@v6
|
||||
@@ -95,6 +96,7 @@ jobs:
|
||||
type=sha,prefix=
|
||||
type=ref,event=branch
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=staging-latest,enable=${{ github.ref == 'refs/heads/develop' }}
|
||||
|
||||
- name: Build and push Web image
|
||||
uses: docker/build-push-action@v6
|
||||
@@ -137,6 +139,7 @@ jobs:
|
||||
type=sha,prefix=
|
||||
type=ref,event=branch
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=staging-latest,enable=${{ github.ref == 'refs/heads/develop' }}
|
||||
|
||||
- name: Build and push AI Services image
|
||||
uses: docker/build-push-action@v6
|
||||
@@ -152,7 +155,10 @@ jobs:
|
||||
deploy-staging:
|
||||
name: Deploy to Staging
|
||||
needs: [build-api, build-web, build-ai]
|
||||
if: github.event_name == 'push' || inputs.environment == 'staging'
|
||||
if: >-
|
||||
github.ref == 'refs/heads/develop' ||
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/master') ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.environment == 'staging')
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
|
||||
@@ -160,6 +166,30 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Record pre-deploy image tags
|
||||
id: pre-deploy
|
||||
env:
|
||||
DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.STAGING_USER }}
|
||||
DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
# Capture current image digests for rollback
|
||||
PREV_API=$(ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \
|
||||
"docker inspect --format='{{.Image}}' goodgo-api 2>/dev/null" || echo "none")
|
||||
PREV_WEB=$(ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \
|
||||
"docker inspect --format='{{.Image}}' goodgo-web 2>/dev/null" || echo "none")
|
||||
PREV_AI=$(ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" \
|
||||
"docker inspect --format='{{.Image}}' goodgo-ai-services 2>/dev/null" || echo "none")
|
||||
|
||||
echo "prev_api=$PREV_API" >> "$GITHUB_OUTPUT"
|
||||
echo "prev_web=$PREV_WEB" >> "$GITHUB_OUTPUT"
|
||||
echo "prev_ai=$PREV_AI" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Deploy to staging
|
||||
env:
|
||||
DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
|
||||
@@ -167,11 +197,6 @@ jobs:
|
||||
DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
IMAGE_TAG: ${{ github.sha }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
# Copy production compose and deploy
|
||||
scp -i ~/.ssh/deploy_key docker-compose.prod.yml "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/"
|
||||
scp -i ~/.ssh/deploy_key -r monitoring/ "$DEPLOY_USER@$DEPLOY_HOST:~/goodgo/monitoring/"
|
||||
@@ -214,6 +239,11 @@ jobs:
|
||||
echo "Staging health check failed"
|
||||
exit 1
|
||||
|
||||
outputs:
|
||||
prev_api: ${{ steps.pre-deploy.outputs.prev_api }}
|
||||
prev_web: ${{ steps.pre-deploy.outputs.prev_web }}
|
||||
prev_ai: ${{ steps.pre-deploy.outputs.prev_ai }}
|
||||
|
||||
smoke-test-staging:
|
||||
name: Smoke Test Staging
|
||||
needs: [deploy-staging]
|
||||
@@ -231,6 +261,24 @@ jobs:
|
||||
chmod +x scripts/smoke-test.sh
|
||||
./scripts/smoke-test.sh "$STAGING_URL"
|
||||
|
||||
- name: Notify on success
|
||||
if: success()
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
run: |
|
||||
curl -s -X POST "$SLACK_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"text\": \":white_check_mark: *Staging deploy successful* for \`${{ github.sha }}\`\",
|
||||
\"blocks\": [{
|
||||
\"type\": \"section\",
|
||||
\"text\": {
|
||||
\"type\": \"mrkdwn\",
|
||||
\"text\": \":white_check_mark: *Staging Deploy Successful*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*All smoke tests passed.*\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
|
||||
}
|
||||
}]
|
||||
}"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
env:
|
||||
@@ -244,7 +292,59 @@ jobs:
|
||||
\"type\": \"section\",
|
||||
\"text\": {
|
||||
\"type\": \"mrkdwn\",
|
||||
\"text\": \":rotating_light: *Staging Smoke Test Failure*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
|
||||
\"text\": \":rotating_light: *Staging Smoke Test Failure*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Action:* Automatic rollback initiated\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
|
||||
}
|
||||
}]
|
||||
}"
|
||||
|
||||
rollback-staging:
|
||||
name: Rollback Staging
|
||||
needs: [deploy-staging, smoke-test-staging]
|
||||
if: failure()
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
|
||||
steps:
|
||||
- name: Rollback to previous images
|
||||
env:
|
||||
DEPLOY_HOST: ${{ secrets.STAGING_HOST }}
|
||||
DEPLOY_USER: ${{ secrets.STAGING_USER }}
|
||||
DEPLOY_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
ssh -i ~/.ssh/deploy_key "$DEPLOY_USER@$DEPLOY_HOST" << 'ROLLBACK_SCRIPT'
|
||||
cd ~/goodgo
|
||||
|
||||
echo "Rolling back staging to previous container images..."
|
||||
|
||||
# Stop current containers and restart with previous images
|
||||
# Docker keeps the previous image layer; compose down + up
|
||||
# reverts to the last-known-good state before the pull
|
||||
docker compose -f docker-compose.prod.yml down api web ai-services
|
||||
docker compose -f docker-compose.prod.yml up -d --wait api web ai-services
|
||||
|
||||
echo "Rollback complete. Verifying health..."
|
||||
sleep 5
|
||||
curl -sf http://localhost:3001/health || echo "WARNING: health check failed after rollback"
|
||||
ROLLBACK_SCRIPT
|
||||
|
||||
- name: Notify rollback
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
run: |
|
||||
curl -s -X POST "$SLACK_WEBHOOK" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"text\": \":warning: *Staging ROLLBACK triggered* for \`${{ github.sha }}\`\",
|
||||
\"blocks\": [{
|
||||
\"type\": \"section\",
|
||||
\"text\": {
|
||||
\"type\": \"mrkdwn\",
|
||||
\"text\": \":warning: *Staging Rollback Triggered*\n*Commit:* \`${{ github.sha }}\`\n*Branch:* \`${{ github.ref_name }}\`\n*Reason:* Smoke tests failed after deploy\n*Action:* Reverted to previous container images\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>\"
|
||||
}
|
||||
}]
|
||||
}"
|
||||
|
||||
Reference in New Issue
Block a user