From 4d0b41e63b96fcf338026c8cac12322f37588565 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 23 Sep 2024 16:54:42 +0200 Subject: [PATCH] deployment: add panel and alerts for displying go scheduler latency (#7078) The panel and alerting rule should help to understand whether VM component doesn't have enough CPU resources or gets throttled. The alert is applicable for all VM components. The panel was added to vmalert, vmagent, vmsingle, vm clusert and victorialogs dashes. ------------------- This alerting rule should have help us identify resource shortage for sandbox vmagent - see [this link](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/?g0.range_input=23d13h25m25s424ms&g0.end_input=2024-09-23T14%3A11%3A00&g0.relative_time=none&g0.tab=0&g0.expr=histogram_quantile%280.99%2C+sum%28rate%28go_sched_latencies_seconds_bucket%7Bjob%3D%22vmagent-monitoring-vmagent%22%7D%5B5m%5D%29%29+by+%28le%2C+job%2C+instance%29%29+%3E+0.1) for example. We weren't aware of resource shortage, because VM metrics assumed this vmagent had 1vCPU while in fact its limit was 0.2vCPU. Signed-off-by: hagen1778 --- dashboards/victorialogs.json | 2992 ++++++++++---------- dashboards/victoriametrics-cluster.json | 309 +- dashboards/victoriametrics.json | 207 +- dashboards/vm/victoriametrics-cluster.json | 309 +- dashboards/vm/victoriametrics.json | 207 +- dashboards/vm/vmagent.json | 257 +- dashboards/vm/vmalert.json | 142 +- dashboards/vmagent.json | 257 +- dashboards/vmalert.json | 142 +- deployment/docker/alerts-health.yml | 12 + docs/changelog/CHANGELOG.md | 2 + 11 files changed, 2872 insertions(+), 1964 deletions(-) diff --git a/dashboards/victorialogs.json b/dashboards/victorialogs.json index 60f282df5..68c0d630a 100644 --- a/dashboards/victorialogs.json +++ b/dashboards/victorialogs.json @@ -1117,1450 +1117,1558 @@ "y": 22 }, "id": 28, - "panels": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "requested from system", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "heap inuse", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "stack inuse", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "resident", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "resident anonymous", + "refId": "E" + } + ], + "title": "Memory usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RSS anonymous memory % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU % usage ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C4162A", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n/\nprocess_max_fds{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open FDs ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU cores used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "exemplar": false, + "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Limit", + "refId": "B" + } + ], + "title": "CPU ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of bytes read/write from the storage layer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write", + "refId": "B" + } + ], + "title": "Disk writes/reads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Threads ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "read calls" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 56, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "read calls", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write calls", + "range": true, + "refId": "B" + } + ], + "title": "Disk write/read calls ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 58, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections rate ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP connections ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + } + ], "title": "Resource usage", "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 38, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "RSS memory % usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 40, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "requested from system", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}) + sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "heap inuse", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "stack inuse", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "resident", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": false, - "expr": "sum(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"})", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "resident anonymous", - "refId": "E" - } - ], - "title": "Memory usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 42, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "RSS anonymous memory % usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 44, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU % usage ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 46, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n/\nprocess_max_fds{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open FDs ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 48, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU cores used", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "exemplar": false, - "expr": "process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "CPU ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 50, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(go_goroutines{job=~\"$job\", instance=~\"$instance\"}) by(instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Goroutines ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of bytes read/write from the storage layer.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "read" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 52, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "read", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "write", - "refId": "B" - } - ], - "title": "Disk writes/reads ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 54, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(process_num_threads{job=~\"$job\", instance=~\"$instance\"}) by(instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Threads ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "read calls" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 56, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "read calls", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "write calls", - "range": true, - "refId": "B" - } - ], - "title": "Disk write/read calls ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 58, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "TCP connections rate ($instance)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 60, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$ds" - }, - "editorMode": "code", - "expr": "sum(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}) by(instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "TCP connections ($instance)", - "type": "timeseries" } ], "refresh": "", diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json index 1c57d7d5e..00104d37a 100644 --- a/dashboards/victoriametrics-cluster.json +++ b/dashboards/victoriametrics-cluster.json @@ -1659,7 +1659,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1667,8 +1668,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1679,7 +1679,6 @@ "y": 3 }, "id": 66, - "links": [], "options": { "legend": { "calcs": [ @@ -2268,8 +2267,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [ { @@ -2293,7 +2291,6 @@ "y": 19 }, "id": 204, - "links": [], "options": { "legend": { "calcs": [ @@ -2399,7 +2396,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2407,8 +2405,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2419,7 +2416,6 @@ "y": 27 }, "id": 68, - "links": [], "options": { "legend": { "calcs": [ @@ -2506,7 +2502,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2514,8 +2511,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2613,7 +2609,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2621,8 +2618,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2633,7 +2629,6 @@ "y": 35 }, "id": 70, - "links": [], "options": { "legend": { "calcs": [ @@ -2720,7 +2715,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2728,8 +2724,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2828,7 +2823,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2836,8 +2832,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2848,7 +2843,6 @@ "y": 43 }, "id": 210, - "links": [], "options": { "legend": { "calcs": [ @@ -2884,6 +2878,113 @@ ], "title": "CPU spent on GC ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 213, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" } ], "title": "Resource usage ($job)", @@ -2913,7 +3014,7 @@ "h": 2, "w": 24, "x": 0, - "y": 4 + "y": 12 }, "id": 211, "links": [], @@ -3011,7 +3112,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 31 }, "id": 102, "options": { @@ -3125,7 +3226,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 31 }, "id": 108, "options": { @@ -3226,7 +3327,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 39 }, "id": 142, "links": [ @@ -3338,7 +3439,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "id": 107, "options": { @@ -3439,7 +3540,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 47 }, "id": 170, "links": [], @@ -3545,7 +3646,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 47 }, "id": 116, "links": [], @@ -3647,7 +3748,7 @@ "h": 9, "w": 12, "x": 0, - "y": 47 + "y": 55 }, "id": 144, "options": { @@ -3750,7 +3851,7 @@ "h": 9, "w": 12, "x": 12, - "y": 47 + "y": 55 }, "id": 58, "links": [], @@ -3856,7 +3957,7 @@ "h": 6, "w": 24, "x": 0, - "y": 56 + "y": 64 }, "id": 183, "options": { @@ -3984,7 +4085,7 @@ "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 70 }, "id": 135, "options": { @@ -4106,7 +4207,7 @@ "h": 9, "w": 12, "x": 0, - "y": 37 + "y": 45 }, "id": 76, "links": [], @@ -4222,7 +4323,7 @@ "h": 9, "w": 12, "x": 12, - "y": 37 + "y": 45 }, "id": 86, "links": [], @@ -4347,7 +4448,7 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 54 }, "id": 80, "links": [], @@ -4452,7 +4553,7 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 54 }, "id": 78, "links": [], @@ -4568,7 +4669,7 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 62 }, "id": 82, "options": { @@ -4675,7 +4776,7 @@ "h": 8, "w": 12, "x": 12, - "y": 54 + "y": 62 }, "id": 74, "options": { @@ -4779,8 +4880,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4796,7 +4896,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 14 }, "id": 100, "options": { @@ -4892,8 +4992,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4909,7 +5008,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 14 }, "id": 113, "options": { @@ -5006,8 +5105,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5023,7 +5121,7 @@ "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 22 }, "id": 151, "options": { @@ -5153,8 +5251,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5170,7 +5267,7 @@ "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 22 }, "id": 167, "options": { @@ -5292,8 +5389,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5325,7 +5421,7 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 29 }, "id": 212, "links": [ @@ -5434,8 +5530,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5467,7 +5562,7 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 29 }, "id": 133, "options": { @@ -5572,8 +5667,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5589,7 +5683,7 @@ "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 37 }, "id": 54, "options": { @@ -5678,8 +5772,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5695,7 +5788,7 @@ "h": 8, "w": 12, "x": 12, - "y": 29 + "y": 37 }, "id": 55, "options": { @@ -5789,8 +5882,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5806,7 +5898,7 @@ "h": 8, "w": 12, "x": 0, - "y": 37 + "y": 45 }, "id": 20, "options": { @@ -5924,8 +6016,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5941,7 +6032,7 @@ "h": 8, "w": 12, "x": 12, - "y": 37 + "y": 45 }, "id": 22, "options": { @@ -6038,8 +6129,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6051,7 +6141,7 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 53 }, "id": 202, "options": { @@ -6155,8 +6245,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6189,7 +6278,7 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 53 }, "id": 14, "options": { @@ -6293,8 +6382,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6323,7 +6411,7 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 61 }, "id": 206, "options": { @@ -6424,8 +6512,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6441,7 +6528,7 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 61 }, "id": 141, "links": [ @@ -6564,7 +6651,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 58 }, "id": 92, "links": [], @@ -6690,7 +6777,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 58 }, "id": 95, "links": [], @@ -6812,7 +6899,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 66 }, "id": 163, "links": [], @@ -6956,7 +7043,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 66 }, "id": 165, "links": [], @@ -7096,7 +7183,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 74 }, "id": 178, "links": [], @@ -7203,7 +7290,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 74 }, "id": 180, "links": [], @@ -7310,7 +7397,7 @@ "h": 8, "w": 12, "x": 0, - "y": 74 + "y": 82 }, "id": 179, "links": [], @@ -7417,7 +7504,7 @@ "h": 8, "w": 12, "x": 12, - "y": 74 + "y": 82 }, "id": 181, "links": [], @@ -7535,7 +7622,7 @@ "h": 8, "w": 12, "x": 0, - "y": 82 + "y": 90 }, "id": 93, "links": [], @@ -7665,7 +7752,7 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 90 }, "id": 207, "links": [], @@ -7801,7 +7888,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 97, "links": [], @@ -7927,7 +8014,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 99, "links": [], @@ -8051,7 +8138,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 185, "links": [], @@ -8195,7 +8282,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 187, "links": [], @@ -8335,7 +8422,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 75 }, "id": 139, "links": [], @@ -8442,7 +8529,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 114, "links": [], @@ -8557,7 +8644,7 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 83 }, "id": 208, "links": [], @@ -8687,7 +8774,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 83 }, "id": 209, "links": [], @@ -8806,7 +8893,7 @@ "h": 8, "w": 12, "x": 12, - "y": 83 + "y": 91 }, "id": 88, "links": [], @@ -8871,7 +8958,7 @@ "h": 2, "w": 24, "x": 0, - "y": 100 + "y": 108 }, "id": 198, "options": { @@ -8947,7 +9034,7 @@ "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 110 }, "id": 189, "links": [], @@ -9048,7 +9135,7 @@ "h": 8, "w": 12, "x": 12, - "y": 102 + "y": 110 }, "id": 190, "links": [], @@ -9149,7 +9236,7 @@ "h": 7, "w": 12, "x": 0, - "y": 110 + "y": 118 }, "id": 192, "links": [], @@ -9256,7 +9343,7 @@ "h": 7, "w": 12, "x": 12, - "y": 110 + "y": 118 }, "id": 196, "links": [], @@ -9358,7 +9445,7 @@ "h": 8, "w": 12, "x": 0, - "y": 117 + "y": 125 }, "id": 200, "links": [], @@ -9459,7 +9546,7 @@ "h": 8, "w": 12, "x": 12, - "y": 117 + "y": 125 }, "id": 201, "links": [], @@ -9578,7 +9665,7 @@ "h": 8, "w": 12, "x": 0, - "y": 125 + "y": 133 }, "id": 203, "links": [], diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json index 3703a5558..c189aa5b2 100644 --- a/dashboards/victoriametrics.json +++ b/dashboards/victoriametrics.json @@ -1569,7 +1569,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 3 }, "id": 112, "links": [], @@ -1677,10 +1677,9 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 3 }, "id": 44, - "links": [], "options": { "legend": { "calcs": [ @@ -1845,7 +1844,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 11 }, "id": 123, "links": [], @@ -1952,10 +1951,9 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 11 }, "id": 114, - "links": [], "options": { "legend": { "calcs": [ @@ -2046,7 +2044,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2054,8 +2053,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [ { @@ -2079,10 +2077,9 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 19 }, "id": 75, - "links": [], "options": { "legend": { "calcs": [ @@ -2172,7 +2169,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2180,8 +2178,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [ { @@ -2205,10 +2202,9 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 19 }, "id": 57, - "links": [], "options": { "legend": { "calcs": [ @@ -2313,7 +2309,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2321,8 +2318,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2330,10 +2326,9 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 27 }, "id": 47, - "links": [], "options": { "legend": { "calcs": [ @@ -2420,7 +2415,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2428,8 +2424,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -2450,10 +2445,9 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 27 }, "id": 76, - "links": [], "options": { "legend": { "calcs": [ @@ -2558,7 +2552,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2566,8 +2561,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2575,10 +2569,9 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 35 }, "id": 48, - "links": [], "options": { "legend": { "calcs": [ @@ -2665,7 +2658,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2673,8 +2667,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [ { @@ -2695,10 +2688,9 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 35 }, "id": 124, - "links": [], "options": { "legend": { "calcs": [ @@ -2803,7 +2795,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2811,8 +2804,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2820,10 +2812,9 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 43 }, "id": 49, - "links": [], "options": { "legend": { "calcs": [ @@ -2912,7 +2903,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2920,8 +2912,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2929,10 +2920,9 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 43 }, "id": 37, - "links": [], "options": { "legend": { "calcs": [ @@ -3020,7 +3010,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3028,8 +3019,7 @@ } ] }, - "unit": "decbits", - "unitScale": true + "unit": "decbits" }, "overrides": [ { @@ -3050,10 +3040,9 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 51 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3157,7 +3146,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3165,8 +3155,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -3174,10 +3163,9 @@ "h": 8, "w": 12, "x": 12, - "y": 79 + "y": 51 }, "id": 125, - "links": [], "options": { "legend": { "calcs": [ @@ -3213,6 +3201,113 @@ ], "title": "CPU spent on GC", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" } ], "targets": [ diff --git a/dashboards/vm/victoriametrics-cluster.json b/dashboards/vm/victoriametrics-cluster.json index dabd1bcff..3df5c1a0f 100644 --- a/dashboards/vm/victoriametrics-cluster.json +++ b/dashboards/vm/victoriametrics-cluster.json @@ -1660,7 +1660,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1668,8 +1669,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -1680,7 +1680,6 @@ "y": 3 }, "id": 66, - "links": [], "options": { "legend": { "calcs": [ @@ -2269,8 +2268,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [ { @@ -2294,7 +2292,6 @@ "y": 19 }, "id": 204, - "links": [], "options": { "legend": { "calcs": [ @@ -2400,7 +2397,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2408,8 +2406,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2420,7 +2417,6 @@ "y": 27 }, "id": 68, - "links": [], "options": { "legend": { "calcs": [ @@ -2507,7 +2503,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2515,8 +2512,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2614,7 +2610,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2622,8 +2619,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2634,7 +2630,6 @@ "y": 35 }, "id": 70, - "links": [], "options": { "legend": { "calcs": [ @@ -2721,7 +2716,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2729,8 +2725,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2829,7 +2824,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2837,8 +2833,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -2849,7 +2844,6 @@ "y": 43 }, "id": 210, - "links": [], "options": { "legend": { "calcs": [ @@ -2885,6 +2879,113 @@ ], "title": "CPU spent on GC ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 213, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" } ], "title": "Resource usage ($job)", @@ -2914,7 +3015,7 @@ "h": 2, "w": 24, "x": 0, - "y": 4 + "y": 12 }, "id": 211, "links": [], @@ -3012,7 +3113,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 31 }, "id": 102, "options": { @@ -3126,7 +3227,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 31 }, "id": 108, "options": { @@ -3227,7 +3328,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 39 }, "id": 142, "links": [ @@ -3339,7 +3440,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "id": 107, "options": { @@ -3440,7 +3541,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 47 }, "id": 170, "links": [], @@ -3546,7 +3647,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 47 }, "id": 116, "links": [], @@ -3648,7 +3749,7 @@ "h": 9, "w": 12, "x": 0, - "y": 47 + "y": 55 }, "id": 144, "options": { @@ -3751,7 +3852,7 @@ "h": 9, "w": 12, "x": 12, - "y": 47 + "y": 55 }, "id": 58, "links": [], @@ -3857,7 +3958,7 @@ "h": 6, "w": 24, "x": 0, - "y": 56 + "y": 64 }, "id": 183, "options": { @@ -3985,7 +4086,7 @@ "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 70 }, "id": 135, "options": { @@ -4107,7 +4208,7 @@ "h": 9, "w": 12, "x": 0, - "y": 37 + "y": 45 }, "id": 76, "links": [], @@ -4223,7 +4324,7 @@ "h": 9, "w": 12, "x": 12, - "y": 37 + "y": 45 }, "id": 86, "links": [], @@ -4348,7 +4449,7 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 54 }, "id": 80, "links": [], @@ -4453,7 +4554,7 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 54 }, "id": 78, "links": [], @@ -4569,7 +4670,7 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 62 }, "id": 82, "options": { @@ -4676,7 +4777,7 @@ "h": 8, "w": 12, "x": 12, - "y": 54 + "y": 62 }, "id": 74, "options": { @@ -4780,8 +4881,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4797,7 +4897,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 14 }, "id": 100, "options": { @@ -4893,8 +4993,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4910,7 +5009,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 14 }, "id": 113, "options": { @@ -5007,8 +5106,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5024,7 +5122,7 @@ "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 22 }, "id": 151, "options": { @@ -5154,8 +5252,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5171,7 +5268,7 @@ "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 22 }, "id": 167, "options": { @@ -5293,8 +5390,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5326,7 +5422,7 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 29 }, "id": 212, "links": [ @@ -5435,8 +5531,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5468,7 +5563,7 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 29 }, "id": 133, "options": { @@ -5573,8 +5668,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5590,7 +5684,7 @@ "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 37 }, "id": 54, "options": { @@ -5679,8 +5773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5696,7 +5789,7 @@ "h": 8, "w": 12, "x": 12, - "y": 29 + "y": 37 }, "id": 55, "options": { @@ -5790,8 +5883,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5807,7 +5899,7 @@ "h": 8, "w": 12, "x": 0, - "y": 37 + "y": 45 }, "id": 20, "options": { @@ -5925,8 +6017,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5942,7 +6033,7 @@ "h": 8, "w": 12, "x": 12, - "y": 37 + "y": 45 }, "id": 22, "options": { @@ -6039,8 +6130,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6052,7 +6142,7 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 53 }, "id": 202, "options": { @@ -6156,8 +6246,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6190,7 +6279,7 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 53 }, "id": 14, "options": { @@ -6294,8 +6383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6324,7 +6412,7 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 61 }, "id": 206, "options": { @@ -6425,8 +6513,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6442,7 +6529,7 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 61 }, "id": 141, "links": [ @@ -6565,7 +6652,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 58 }, "id": 92, "links": [], @@ -6691,7 +6778,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 58 }, "id": 95, "links": [], @@ -6813,7 +6900,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 66 }, "id": 163, "links": [], @@ -6957,7 +7044,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 66 }, "id": 165, "links": [], @@ -7097,7 +7184,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 74 }, "id": 178, "links": [], @@ -7204,7 +7291,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 74 }, "id": 180, "links": [], @@ -7311,7 +7398,7 @@ "h": 8, "w": 12, "x": 0, - "y": 74 + "y": 82 }, "id": 179, "links": [], @@ -7418,7 +7505,7 @@ "h": 8, "w": 12, "x": 12, - "y": 74 + "y": 82 }, "id": 181, "links": [], @@ -7536,7 +7623,7 @@ "h": 8, "w": 12, "x": 0, - "y": 82 + "y": 90 }, "id": 93, "links": [], @@ -7666,7 +7753,7 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 90 }, "id": 207, "links": [], @@ -7802,7 +7889,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 97, "links": [], @@ -7928,7 +8015,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 99, "links": [], @@ -8052,7 +8139,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 185, "links": [], @@ -8196,7 +8283,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 187, "links": [], @@ -8336,7 +8423,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 75 }, "id": 139, "links": [], @@ -8443,7 +8530,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 114, "links": [], @@ -8558,7 +8645,7 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 83 }, "id": 208, "links": [], @@ -8688,7 +8775,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 83 }, "id": 209, "links": [], @@ -8807,7 +8894,7 @@ "h": 8, "w": 12, "x": 12, - "y": 83 + "y": 91 }, "id": 88, "links": [], @@ -8872,7 +8959,7 @@ "h": 2, "w": 24, "x": 0, - "y": 100 + "y": 108 }, "id": 198, "options": { @@ -8948,7 +9035,7 @@ "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 110 }, "id": 189, "links": [], @@ -9049,7 +9136,7 @@ "h": 8, "w": 12, "x": 12, - "y": 102 + "y": 110 }, "id": 190, "links": [], @@ -9150,7 +9237,7 @@ "h": 7, "w": 12, "x": 0, - "y": 110 + "y": 118 }, "id": 192, "links": [], @@ -9257,7 +9344,7 @@ "h": 7, "w": 12, "x": 12, - "y": 110 + "y": 118 }, "id": 196, "links": [], @@ -9359,7 +9446,7 @@ "h": 8, "w": 12, "x": 0, - "y": 117 + "y": 125 }, "id": 200, "links": [], @@ -9460,7 +9547,7 @@ "h": 8, "w": 12, "x": 12, - "y": 117 + "y": 125 }, "id": 201, "links": [], @@ -9579,7 +9666,7 @@ "h": 8, "w": 12, "x": 0, - "y": 125 + "y": 133 }, "id": 203, "links": [], diff --git a/dashboards/vm/victoriametrics.json b/dashboards/vm/victoriametrics.json index c324fafdd..39cc5f80e 100644 --- a/dashboards/vm/victoriametrics.json +++ b/dashboards/vm/victoriametrics.json @@ -1570,7 +1570,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 3 }, "id": 112, "links": [], @@ -1678,10 +1678,9 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 3 }, "id": 44, - "links": [], "options": { "legend": { "calcs": [ @@ -1846,7 +1845,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 11 }, "id": 123, "links": [], @@ -1953,10 +1952,9 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 11 }, "id": 114, - "links": [], "options": { "legend": { "calcs": [ @@ -2047,7 +2045,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2055,8 +2054,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [ { @@ -2080,10 +2078,9 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 19 }, "id": 75, - "links": [], "options": { "legend": { "calcs": [ @@ -2173,7 +2170,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2181,8 +2179,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [ { @@ -2206,10 +2203,9 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 19 }, "id": 57, - "links": [], "options": { "legend": { "calcs": [ @@ -2314,7 +2310,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2322,8 +2319,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2331,10 +2327,9 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 27 }, "id": 47, - "links": [], "options": { "legend": { "calcs": [ @@ -2421,7 +2416,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2429,8 +2425,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [ { @@ -2451,10 +2446,9 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 27 }, "id": 76, - "links": [], "options": { "legend": { "calcs": [ @@ -2559,7 +2553,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2567,8 +2562,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2576,10 +2570,9 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 35 }, "id": 48, - "links": [], "options": { "legend": { "calcs": [ @@ -2666,7 +2659,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2674,8 +2668,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [ { @@ -2696,10 +2689,9 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 35 }, "id": 124, - "links": [], "options": { "legend": { "calcs": [ @@ -2804,7 +2796,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2812,8 +2805,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2821,10 +2813,9 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 43 }, "id": 49, - "links": [], "options": { "legend": { "calcs": [ @@ -2913,7 +2904,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2921,8 +2913,7 @@ } ] }, - "unit": "short", - "unitScale": true + "unit": "short" }, "overrides": [] }, @@ -2930,10 +2921,9 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 43 }, "id": 37, - "links": [], "options": { "legend": { "calcs": [ @@ -3021,7 +3011,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3029,8 +3020,7 @@ } ] }, - "unit": "decbits", - "unitScale": true + "unit": "decbits" }, "overrides": [ { @@ -3051,10 +3041,9 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 51 }, "id": 127, - "links": [], "options": { "legend": { "calcs": [ @@ -3158,7 +3147,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3166,8 +3156,7 @@ } ] }, - "unit": "percentunit", - "unitScale": true + "unit": "percentunit" }, "overrides": [] }, @@ -3175,10 +3164,9 @@ "h": 8, "w": 12, "x": 12, - "y": 79 + "y": 51 }, "id": 125, - "links": [], "options": { "legend": { "calcs": [ @@ -3214,6 +3202,113 @@ ], "title": "CPU spent on GC", "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" } ], "targets": [ diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json index 244e83f09..cbac922ee 100644 --- a/dashboards/vm/vmagent.json +++ b/dashboards/vm/vmagent.json @@ -1635,7 +1635,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 35 }, "id": 109, "options": { @@ -1733,7 +1733,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1749,7 +1750,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 35 }, "id": 111, "options": { @@ -1844,7 +1845,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1873,7 +1875,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 43 }, "id": 81, "options": { @@ -1979,7 +1981,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2008,7 +2011,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 43 }, "id": 7, "options": { @@ -2110,7 +2113,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2126,7 +2130,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 51 }, "id": 83, "options": { @@ -2215,7 +2219,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2231,7 +2236,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 51 }, "id": 39, "options": { @@ -2321,7 +2326,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2337,7 +2343,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 59 }, "id": 135, "options": { @@ -2376,6 +2382,113 @@ "title": "CPU spent on GC ($instance)", "type": "timeseries" }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + }, { "datasource": { "type": "victoriametrics-datasource", @@ -2426,7 +2539,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2442,7 +2556,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 67 }, "id": 41, "options": { @@ -2571,7 +2685,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 92, "options": { @@ -2675,7 +2789,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 95, "options": { @@ -2782,7 +2896,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 98, "options": { @@ -2889,7 +3003,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 99, "options": { @@ -2993,7 +3107,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 79, "options": { @@ -3097,7 +3211,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 18, "links": [ @@ -3207,7 +3321,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "id": 127, "options": { @@ -3309,7 +3423,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "id": 50, "options": { @@ -3409,7 +3523,7 @@ "h": 7, "w": 24, "x": 0, - "y": 44 + "y": 52 }, "id": 129, "options": { @@ -3548,8 +3662,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3565,7 +3678,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 45 }, "id": 48, "options": { @@ -3655,8 +3768,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3672,7 +3784,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 45 }, "id": 76, "options": { @@ -3760,8 +3872,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3777,7 +3888,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "id": 132, "options": { @@ -3867,8 +3978,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3884,7 +3994,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 52 }, "id": 133, "options": { @@ -3973,8 +4083,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3990,7 +4099,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 20, "options": { @@ -4078,8 +4187,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4095,7 +4203,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 126, "options": { @@ -4182,8 +4290,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4199,7 +4306,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 46, "options": { @@ -4286,8 +4393,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4303,7 +4409,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 148, "options": { @@ -4390,8 +4496,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4407,7 +4512,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 31, "options": { @@ -4586,7 +4691,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 22 }, "id": 73, "options": { @@ -4703,7 +4808,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 14 }, "id": 131, "options": { @@ -4807,7 +4912,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 22 }, "id": 130, "options": { @@ -4924,7 +5029,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 22 }, "id": 77, "options": { @@ -5058,7 +5163,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 15 }, "id": 146, "options": { @@ -5160,7 +5265,7 @@ "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 15 }, "id": 143, "options": { @@ -5256,7 +5361,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 23 }, "id": 147, "options": { @@ -5359,7 +5464,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 23 }, "id": 139, "options": { @@ -5470,7 +5575,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 31 }, "id": 142, "options": { @@ -5567,7 +5672,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 31 }, "id": 137, "options": { @@ -5680,7 +5785,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "id": 141, "options": { @@ -5811,7 +5916,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 16 }, "id": 60, "options": { @@ -5915,7 +6020,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 16 }, "id": 66, "options": { @@ -6019,7 +6124,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "id": 61, "options": { @@ -6123,7 +6228,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "id": 65, "options": { @@ -6226,7 +6331,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 32 }, "id": 88, "options": { @@ -6325,7 +6430,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 32 }, "id": 84, "options": { @@ -6428,7 +6533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 40 }, "id": 90, "options": { @@ -6494,7 +6599,7 @@ "h": 2, "w": 24, "x": 0, - "y": 17 + "y": 25 }, "id": 115, "options": { @@ -6572,7 +6677,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 119, "options": { @@ -6676,7 +6781,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 117, "options": { @@ -6782,7 +6887,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 125, "links": [ @@ -6904,7 +7009,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 123, "options": { @@ -7034,7 +7139,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 43 }, "id": 121, "options": { @@ -7233,4 +7338,4 @@ "uid": "G7Z9GzMGz_vm", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/dashboards/vm/vmalert.json b/dashboards/vm/vmalert.json index 3acf3edb7..3d71e5c18 100644 --- a/dashboards/vm/vmalert.json +++ b/dashboards/vm/vmalert.json @@ -1532,7 +1532,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1540,8 +1541,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -2167,6 +2167,113 @@ ], "title": "CPU spent on GC ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" } ], "targets": [ @@ -2257,7 +2364,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "id": 14, "options": { @@ -2359,7 +2466,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "id": 13, "options": { @@ -2461,7 +2568,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "id": 20, "options": { @@ -2564,7 +2671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 52 }, "id": 32, "options": { @@ -2663,7 +2770,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 60 }, "id": 26, "options": { @@ -2790,7 +2897,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 31, "options": { @@ -2892,7 +2999,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 33, "options": { @@ -2993,7 +3100,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 30, "options": { @@ -3114,7 +3221,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 25 }, "id": 52, "options": { @@ -3206,7 +3313,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 25 }, "id": 53, "options": { @@ -3303,7 +3410,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "id": 54, "options": { @@ -3390,8 +3497,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3407,9 +3513,9 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 52 }, - "id": 55, + "id": 60, "options": { "legend": { "calcs": [ diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index cd26420fc..aa664f4b4 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -1634,7 +1634,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 35 }, "id": 109, "options": { @@ -1732,7 +1732,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1748,7 +1749,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 35 }, "id": 111, "options": { @@ -1843,7 +1844,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1872,7 +1874,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 43 }, "id": 81, "options": { @@ -1978,7 +1980,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2007,7 +2010,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 43 }, "id": 7, "options": { @@ -2109,7 +2112,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2125,7 +2129,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 51 }, "id": 83, "options": { @@ -2214,7 +2218,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2230,7 +2235,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 51 }, "id": 39, "options": { @@ -2320,7 +2325,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2336,7 +2342,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 59 }, "id": 135, "options": { @@ -2375,6 +2381,113 @@ "title": "CPU spent on GC ($instance)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -2425,7 +2538,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2441,7 +2555,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 67 }, "id": 41, "options": { @@ -2570,7 +2684,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 20 }, "id": 92, "options": { @@ -2674,7 +2788,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 20 }, "id": 95, "options": { @@ -2781,7 +2895,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 28 }, "id": 98, "options": { @@ -2888,7 +3002,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 28 }, "id": 99, "options": { @@ -2992,7 +3106,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "id": 79, "options": { @@ -3096,7 +3210,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "id": 18, "links": [ @@ -3206,7 +3320,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "id": 127, "options": { @@ -3308,7 +3422,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "id": 50, "options": { @@ -3408,7 +3522,7 @@ "h": 7, "w": 24, "x": 0, - "y": 44 + "y": 52 }, "id": 129, "options": { @@ -3547,8 +3661,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3564,7 +3677,7 @@ "h": 7, "w": 12, "x": 0, - "y": 37 + "y": 45 }, "id": 48, "options": { @@ -3654,8 +3767,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3671,7 +3783,7 @@ "h": 7, "w": 12, "x": 12, - "y": 37 + "y": 45 }, "id": 76, "options": { @@ -3759,8 +3871,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3776,7 +3887,7 @@ "h": 7, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "id": 132, "options": { @@ -3866,8 +3977,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3883,7 +3993,7 @@ "h": 7, "w": 12, "x": 12, - "y": 44 + "y": 52 }, "id": 133, "options": { @@ -3972,8 +4082,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3989,7 +4098,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 20, "options": { @@ -4077,8 +4186,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4094,7 +4202,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 126, "options": { @@ -4181,8 +4289,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4198,7 +4305,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 46, "options": { @@ -4285,8 +4392,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4302,7 +4408,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 148, "options": { @@ -4389,8 +4495,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4406,7 +4511,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 31, "options": { @@ -4585,7 +4690,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 22 }, "id": 73, "options": { @@ -4702,7 +4807,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 14 }, "id": 131, "options": { @@ -4806,7 +4911,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 22 }, "id": 130, "options": { @@ -4923,7 +5028,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 22 }, "id": 77, "options": { @@ -5057,7 +5162,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 15 }, "id": 146, "options": { @@ -5159,7 +5264,7 @@ "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 15 }, "id": 143, "options": { @@ -5255,7 +5360,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 23 }, "id": 147, "options": { @@ -5358,7 +5463,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 23 }, "id": 139, "options": { @@ -5469,7 +5574,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 31 }, "id": 142, "options": { @@ -5566,7 +5671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 31 }, "id": 137, "options": { @@ -5679,7 +5784,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "id": 141, "options": { @@ -5810,7 +5915,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 16 }, "id": 60, "options": { @@ -5914,7 +6019,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 16 }, "id": 66, "options": { @@ -6018,7 +6123,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "id": 61, "options": { @@ -6122,7 +6227,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "id": 65, "options": { @@ -6225,7 +6330,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 32 }, "id": 88, "options": { @@ -6324,7 +6429,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 32 }, "id": 84, "options": { @@ -6427,7 +6532,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 40 }, "id": 90, "options": { @@ -6493,7 +6598,7 @@ "h": 2, "w": 24, "x": 0, - "y": 17 + "y": 25 }, "id": 115, "options": { @@ -6571,7 +6676,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 119, "options": { @@ -6675,7 +6780,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 117, "options": { @@ -6781,7 +6886,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 125, "links": [ @@ -6903,7 +7008,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 123, "options": { @@ -7033,7 +7138,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 43 }, "id": 121, "options": { @@ -7232,4 +7337,4 @@ "uid": "G7Z9GzMGz", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/dashboards/vmalert.json b/dashboards/vmalert.json index d1d44ecf2..405f438b0 100644 --- a/dashboards/vmalert.json +++ b/dashboards/vmalert.json @@ -1531,7 +1531,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1539,8 +1540,7 @@ } ] }, - "unit": "bytes", - "unitScale": true + "unit": "bytes" }, "overrides": [] }, @@ -2166,6 +2166,113 @@ ], "title": "CPU spent on GC ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Go scheduling latency", + "type": "timeseries" } ], "targets": [ @@ -2256,7 +2363,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "id": 14, "options": { @@ -2358,7 +2465,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "id": 13, "options": { @@ -2460,7 +2567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "id": 20, "options": { @@ -2563,7 +2670,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 52 }, "id": 32, "options": { @@ -2662,7 +2769,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 60 }, "id": 26, "options": { @@ -2789,7 +2896,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 31, "options": { @@ -2891,7 +2998,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 33, "options": { @@ -2992,7 +3099,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 30, "options": { @@ -3113,7 +3220,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 25 }, "id": 52, "options": { @@ -3205,7 +3312,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 25 }, "id": 53, "options": { @@ -3302,7 +3409,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "id": 54, "options": { @@ -3389,8 +3496,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3406,9 +3512,9 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 52 }, - "id": 55, + "id": 60, "options": { "legend": { "calcs": [ diff --git a/deployment/docker/alerts-health.yml b/deployment/docker/alerts-health.yml index 65d7fa41e..66043210a 100644 --- a/deployment/docker/alerts-health.yml +++ b/deployment/docker/alerts-health.yml @@ -58,6 +58,18 @@ groups: Too high CPU usage may be a sign of insufficient resources and make process unstable. Consider to either increase available CPU resources or decrease the load on the process. + - alert: TooHighGoroutineSchedulingLatency + expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job="vmagent-monitoring-vmagent"}[5m])) by (le, job, instance)) > 0.1 + for: 15m + labels: + severity: critical + annotations: + summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m" + description: > + Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of + insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise, + the service could work unreliably with delays in processing. + - alert: TooManyLogs expr: sum(increase(vm_log_messages_total{level="error"}[5m])) without (app_version, location) > 0 for: 15m diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 47188d071..a0595d63b 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): support parsing `vm_access` claims in string format. This is useful for cases when identity provider does not support mapping claims to JSON format. * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add new metrics for data ingestion: `vm_rows_received_by_storage_total`, `vm_rows_ignored_total{reason="nan_value"}`, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6663) for details. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): change request method for `/query_range` and `/query` calls from `GET` to `POST`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6288). +* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for VM single-node, cluster, vmalert, vmagent, VictoriaLogs: add `Go scheduling latency` panel to show the 99th quantile of Go goroutines scheduling. This panel should help identifying insufficient CPU resources for the service. It is especially useful if CPU gets throttled, which now should be visible on this panel. +* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): add alerting rule to track the Go scheduling latency for goroutines. It should notify users if VM component doesn't have enough CPU to run or gets throttled. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink` in `Host:Port` format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6912). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): properly consume messages [from kafka](https://docs.victoriametrics.com/vmagent/#kafka-integration). Previously vmagent could skip some messages during start-up.