groups: - name: slo:availability:recording interval: 30s rules: - record: slo:http_requests:rate5m expr: sum(rate(http_requests_total{job="goodgo-api"}[5m])) by (route, method) - record: slo:http_errors:rate5m expr: sum(rate(http_requests_total{job="goodgo-api", status_code=~"5.."}[5m])) by (route, method) - record: slo:error_ratio:rate5m expr: slo:http_errors:rate5m / slo:http_requests:rate5m - record: slo:http_requests:rate30m expr: sum(rate(http_requests_total{job="goodgo-api"}[30m])) by (route, method) - record: slo:http_errors:rate30m expr: sum(rate(http_requests_total{job="goodgo-api", status_code=~"5.."}[30m])) by (route, method) - record: slo:error_ratio:rate30m expr: slo:http_errors:rate30m / slo:http_requests:rate30m - record: slo:http_requests:rate1h expr: sum(rate(http_requests_total{job="goodgo-api"}[1h])) by (route, method) - record: slo:http_errors:rate1h expr: sum(rate(http_requests_total{job="goodgo-api", status_code=~"5.."}[1h])) by (route, method) - record: slo:error_ratio:rate1h expr: slo:http_errors:rate1h / slo:http_requests:rate1h - record: slo:http_requests:rate6h expr: sum(rate(http_requests_total{job="goodgo-api"}[6h])) by (route, method) - record: slo:http_errors:rate6h expr: sum(rate(http_requests_total{job="goodgo-api", status_code=~"5.."}[6h])) by (route, method) - record: slo:error_ratio:rate6h expr: slo:http_errors:rate6h / slo:http_requests:rate6h - record: slo:http_requests:rate1d expr: sum(rate(http_requests_total{job="goodgo-api"}[1d])) by (route, method) - record: slo:http_errors:rate1d expr: sum(rate(http_requests_total{job="goodgo-api", status_code=~"5.."}[1d])) by (route, method) - record: slo:error_ratio:rate1d expr: slo:http_errors:rate1d / slo:http_requests:rate1d - record: slo:http_requests:rate3d expr: sum(rate(http_requests_total{job="goodgo-api"}[3d])) by (route, method) - record: slo:http_errors:rate3d expr: sum(rate(http_requests_total{job="goodgo-api", status_code=~"5.."}[3d])) by (route, method) - record: slo:error_ratio:rate3d expr: slo:http_errors:rate3d / slo:http_requests:rate3d - name: slo:latency:recording interval: 30s rules: - record: slo:latency_good:rate5m expr: > sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/listings", le="0.5"}[5m])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/listings/:id", le="0.25"}[5m])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/payments/create", le="1"}[5m])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/auth/login", le="0.5"}[5m])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/search", le="0.5"}[5m])) by (route, method) - record: slo:latency_total:rate5m expr: sum(rate(goodgo_api_request_duration_seconds_count{job="goodgo-api", route=~"/api/listings|/api/listings/:id|/api/payments/create|/api/auth/login|/api/search"}[5m])) by (route, method) - record: slo:latency_good_ratio:rate5m expr: slo:latency_good:rate5m / slo:latency_total:rate5m - record: slo:latency_good:rate1h expr: > sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/listings", le="0.5"}[1h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/listings/:id", le="0.25"}[1h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/payments/create", le="1"}[1h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/auth/login", le="0.5"}[1h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/search", le="0.5"}[1h])) by (route, method) - record: slo:latency_total:rate1h expr: sum(rate(goodgo_api_request_duration_seconds_count{job="goodgo-api", route=~"/api/listings|/api/listings/:id|/api/payments/create|/api/auth/login|/api/search"}[1h])) by (route, method) - record: slo:latency_good_ratio:rate1h expr: slo:latency_good:rate1h / slo:latency_total:rate1h - record: slo:latency_good:rate6h expr: > sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/listings", le="0.5"}[6h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/listings/:id", le="0.25"}[6h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/payments/create", le="1"}[6h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/auth/login", le="0.5"}[6h])) by (route, method) or sum(rate(goodgo_api_request_duration_seconds_bucket{job="goodgo-api", route="/api/search", le="0.5"}[6h])) by (route, method) - record: slo:latency_total:rate6h expr: sum(rate(goodgo_api_request_duration_seconds_count{job="goodgo-api", route=~"/api/listings|/api/listings/:id|/api/payments/create|/api/auth/login|/api/search"}[6h])) by (route, method) - record: slo:latency_good_ratio:rate6h expr: slo:latency_good:rate6h / slo:latency_total:rate6h - name: slo:availability:burn_rate_alerts rules: - alert: SloAvailFastBurn expr: > (slo:error_ratio:rate1h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 0.0144 and slo:error_ratio:rate5m{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 0.0144) and slo:http_requests:rate1h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 1 for: 2m labels: {severity: critical, team: sre, slo_type: availability, burn_window: fast, slo_target: "99.9", environment: staging} annotations: summary: "SLO FAST BURN: {{ $labels.method }} {{ $labels.route }} availability (14.4x)" description: "Error ratio {{ $value | printf \"%.4f\" }} exceeds 14.4x burn threshold 0.0144." - alert: SloAvailFastBurnPayments expr: > (slo:error_ratio:rate1h{route="/api/payments/create"} > 0.0072 and slo:error_ratio:rate5m{route="/api/payments/create"} > 0.0072) and slo:http_requests:rate1h{route="/api/payments/create"} > 1 for: 2m labels: {severity: critical, team: sre, slo_type: availability, burn_window: fast, slo_target: "99.95", environment: staging} annotations: summary: "SLO FAST BURN: payments availability (14.4x)" description: "Payments error ratio {{ $value | printf \"%.4f\" }} exceeds threshold 0.0072." - alert: SloAvailSlowBurn expr: > (slo:error_ratio:rate6h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 0.006 and slo:error_ratio:rate30m{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 0.006) and slo:http_requests:rate6h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 1 for: 5m labels: {severity: warning, team: sre, slo_type: availability, burn_window: slow, slo_target: "99.9", environment: staging} annotations: summary: "SLO SLOW BURN: {{ $labels.method }} {{ $labels.route }} availability (6x)" description: "6h error ratio {{ $value | printf \"%.4f\" }} exceeds 6x threshold 0.006." - alert: SloAvailSlowBurnPayments expr: > (slo:error_ratio:rate6h{route="/api/payments/create"} > 0.003 and slo:error_ratio:rate30m{route="/api/payments/create"} > 0.003) and slo:http_requests:rate6h{route="/api/payments/create"} > 1 for: 5m labels: {severity: warning, team: sre, slo_type: availability, burn_window: slow, slo_target: "99.95", environment: staging} annotations: summary: "SLO SLOW BURN: payments availability (6x)" description: "Payments 6h error ratio {{ $value | printf \"%.4f\" }} exceeds threshold 0.003." - name: slo:latency:burn_rate_alerts rules: - alert: SloLatencyFastBurn expr: > (slo:latency_good_ratio:rate1h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} < 0.9856 and slo:latency_good_ratio:rate5m{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} < 0.9856) and slo:latency_total:rate1h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 1 for: 2m labels: {severity: critical, team: sre, slo_type: latency, burn_window: fast, environment: staging} annotations: summary: "SLO LATENCY FAST BURN: {{ $labels.method }} {{ $labels.route }} (14.4x)" description: "Good ratio {{ $value | printf \"%.4f\" }} below 0.9856 threshold." - alert: SloLatencyFastBurnPayments expr: > (slo:latency_good_ratio:rate1h{route="/api/payments/create"} < 0.9928 and slo:latency_good_ratio:rate5m{route="/api/payments/create"} < 0.9928) and slo:latency_total:rate1h{route="/api/payments/create"} > 1 for: 2m labels: {severity: critical, team: sre, slo_type: latency, burn_window: fast, environment: staging} annotations: summary: "SLO LATENCY FAST BURN: payments (14.4x)" description: "Payments good ratio {{ $value | printf \"%.4f\" }} below 0.9928." - alert: SloLatencySlowBurn expr: > (slo:latency_good_ratio:rate6h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} < 0.994 and slo:latency_good_ratio:rate5m{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} < 0.994) and slo:latency_total:rate6h{route=~"/api/listings|/api/listings/:id|/api/auth/login|/api/search"} > 1 for: 5m labels: {severity: warning, team: sre, slo_type: latency, burn_window: slow, environment: staging} annotations: summary: "SLO latency slow burn: {{ $labels.method }} {{ $labels.route }} (6x)" description: "6h good ratio {{ $value | printf \"%.4f\" }} below 0.994."