Files
Ho Ngoc Hai 775eb7b374 feat(ops): add database backup strategy and log aggregation stack
- Add pg-backup container with daily automated pg_dump (02:00 UTC) and 7-day retention
- Add backup/restore scripts with documented recovery procedure
- Add Loki + Promtail for centralized log aggregation from all Docker containers
- Add Loki as Grafana datasource with correlation ID derived fields
- Add Grafana logs dashboard with volume, error rate, HTTP request, and log viewer panels
- Configure Promtail to parse Pino structured JSON logs with level/context labels
- Enhance LoggerService with string-level formatter and service base field
- Configure 15-day log retention in Loki

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-04-08 04:04:32 +07:00

138 lines
4.3 KiB
JSON

{
"annotations": { "list": [] },
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"title": "Log Volume by Level",
"type": "timeseries",
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
"datasource": { "type": "loki", "uid": "loki" },
"targets": [
{
"expr": "sum by (level) (count_over_time({compose_service=~\"$service\"} | json [$__interval]))",
"legendFormat": "{{ level }}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "bars",
"stacking": { "mode": "normal" },
"fillOpacity": 80
},
"color": { "mode": "palette-classic" }
},
"overrides": [
{ "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] },
{ "matcher": { "id": "byName", "options": "fatal" }, "properties": [{ "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } }] },
{ "matcher": { "id": "byName", "options": "warn" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] },
{ "matcher": { "id": "byName", "options": "info" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] },
{ "matcher": { "id": "byName", "options": "debug" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }
]
}
},
{
"title": "Error Rate",
"type": "stat",
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 6 },
"datasource": { "type": "loki", "uid": "loki" },
"targets": [
{
"expr": "sum(count_over_time({compose_service=~\"$service\"} | json | level = `error` or level = `fatal` [$__range]))",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 10 },
{ "color": "red", "value": 50 }
]
}
}
}
},
{
"title": "Errors by Service",
"type": "piechart",
"gridPos": { "h": 8, "w": 6, "x": 6, "y": 6 },
"datasource": { "type": "loki", "uid": "loki" },
"targets": [
{
"expr": "sum by (compose_service) (count_over_time({compose_service=~\".+\"} | json | level = `error` or level = `fatal` [$__range]))",
"legendFormat": "{{ compose_service }}",
"refId": "A"
}
]
},
{
"title": "HTTP Request Logs (4xx/5xx)",
"type": "table",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"datasource": { "type": "loki", "uid": "loki" },
"targets": [
{
"expr": "{compose_service=~\"$service\", component=\"http\"} | json | statusCode >= 400 | line_format \"{{.method}} {{.url}} {{.statusCode}} {{.duration}}ms\"",
"refId": "A"
}
]
},
{
"title": "All Logs",
"type": "logs",
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 14 },
"datasource": { "type": "loki", "uid": "loki" },
"targets": [
{
"expr": "{compose_service=~\"$service\"} | json |= `$search`",
"refId": "A"
}
],
"options": {
"showTime": true,
"showLabels": true,
"showCommonLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": true,
"enableLogDetails": true,
"sortOrder": "Descending"
}
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": ["goodgo", "logs", "loki"],
"templating": {
"list": [
{
"name": "service",
"type": "query",
"datasource": { "type": "loki", "uid": "loki" },
"query": "label_values(compose_service)",
"includeAll": true,
"allValue": ".+",
"current": { "text": "All", "value": "$__all" },
"refresh": 2,
"multi": true
},
{
"name": "search",
"type": "textbox",
"current": { "text": "", "value": "" },
"label": "Search"
}
]
},
"time": { "from": "now-1h", "to": "now" },
"title": "GoodGo - Logs",
"uid": "goodgo-logs"
}