- Add pg-backup container with daily automated pg_dump (02:00 UTC) and 7-day retention - Add backup/restore scripts with documented recovery procedure - Add Loki + Promtail for centralized log aggregation from all Docker containers - Add Loki as Grafana datasource with correlation ID derived fields - Add Grafana logs dashboard with volume, error rate, HTTP request, and log viewer panels - Configure Promtail to parse Pino structured JSON logs with level/context labels - Enhance LoggerService with string-level formatter and service base field - Configure 15-day log retention in Loki Co-Authored-By: Paperclip <noreply@paperclip.ing>
138 lines
4.3 KiB
JSON
138 lines
4.3 KiB
JSON
{
|
|
"annotations": { "list": [] },
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"title": "Log Volume by Level",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (level) (count_over_time({compose_service=~\"$service\"} | json [$__interval]))",
|
|
"legendFormat": "{{ level }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"stacking": { "mode": "normal" },
|
|
"fillOpacity": 80
|
|
},
|
|
"color": { "mode": "palette-classic" }
|
|
},
|
|
"overrides": [
|
|
{ "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] },
|
|
{ "matcher": { "id": "byName", "options": "fatal" }, "properties": [{ "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } }] },
|
|
{ "matcher": { "id": "byName", "options": "warn" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] },
|
|
{ "matcher": { "id": "byName", "options": "info" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] },
|
|
{ "matcher": { "id": "byName", "options": "debug" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"title": "Error Rate",
|
|
"type": "stat",
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 6 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"targets": [
|
|
{
|
|
"expr": "sum(count_over_time({compose_service=~\"$service\"} | json | level = `error` or level = `fatal` [$__range]))",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 10 },
|
|
{ "color": "red", "value": 50 }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Errors by Service",
|
|
"type": "piechart",
|
|
"gridPos": { "h": 8, "w": 6, "x": 6, "y": 6 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (compose_service) (count_over_time({compose_service=~\".+\"} | json | level = `error` or level = `fatal` [$__range]))",
|
|
"legendFormat": "{{ compose_service }}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "HTTP Request Logs (4xx/5xx)",
|
|
"type": "table",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"targets": [
|
|
{
|
|
"expr": "{compose_service=~\"$service\", component=\"http\"} | json | statusCode >= 400 | line_format \"{{.method}} {{.url}} {{.statusCode}} {{.duration}}ms\"",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "All Logs",
|
|
"type": "logs",
|
|
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 14 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"targets": [
|
|
{
|
|
"expr": "{compose_service=~\"$service\"} | json |= `$search`",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": true,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": true,
|
|
"enableLogDetails": true,
|
|
"sortOrder": "Descending"
|
|
}
|
|
}
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 39,
|
|
"tags": ["goodgo", "logs", "loki"],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "service",
|
|
"type": "query",
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"query": "label_values(compose_service)",
|
|
"includeAll": true,
|
|
"allValue": ".+",
|
|
"current": { "text": "All", "value": "$__all" },
|
|
"refresh": 2,
|
|
"multi": true
|
|
},
|
|
{
|
|
"name": "search",
|
|
"type": "textbox",
|
|
"current": { "text": "", "value": "" },
|
|
"label": "Search"
|
|
}
|
|
]
|
|
},
|
|
"time": { "from": "now-1h", "to": "now" },
|
|
"title": "GoodGo - Logs",
|
|
"uid": "goodgo-logs"
|
|
}
|