From 12acea2584e6a459975fc15e4ebb973dbef96156 Mon Sep 17 00:00:00 2001
From: hagen1778 <roman@victoriametrics.com>
Date: Thu, 4 Jan 2024 10:37:13 +0100
Subject: [PATCH] dashboards: update cluster dashboard
* add panels for detailed visualization of traffic usage between vmstorage, vminsert, vmselect
components and their clients. New panels are available in the rows dedicated to specific components.
* update "Slow Queries" panel to show percentage of the slow queries to the total number of read queries
served by vmselect. The percentage value should make it more clear for users whether there is a service degradation.
Signed-off-by: hagen1778 <roman@victoriametrics.com>
(cherry picked from commit 463455665ba778135b4da5f556bf844c9a00c3d1)
---
dashboards/victoriametrics-cluster.json | 1146 ++++++++++++++------
dashboards/vm/victoriametrics-cluster.json | 1146 ++++++++++++++------
docs/CHANGELOG.md | 2 +
3 files changed, 1592 insertions(+), 702 deletions(-)
diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json
index 6abd17a678..f553cbfc60 100644
--- a/dashboards/victoriametrics-cluster.json
+++ b/dashboards/victoriametrics-cluster.json
@@ -1660,7 +1660,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 38
+ "y": 14
},
"id": 66,
"links": [],
@@ -1772,7 +1772,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 38
+ "y": 14
},
"id": 138,
"links": [],
@@ -1866,8 +1866,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1883,7 +1882,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 46
+ "y": 22
},
"id": 64,
"links": [],
@@ -1973,8 +1972,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2003,7 +2001,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 46
+ "y": 22
},
"id": 122,
"links": [],
@@ -2111,8 +2109,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2144,7 +2141,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 54
+ "y": 30
},
"id": 117,
"links": [],
@@ -2234,8 +2231,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2264,7 +2260,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 54
+ "y": 30
},
"id": 204,
"links": [],
@@ -2371,8 +2367,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2388,7 +2383,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 62
+ "y": 38
},
"id": 68,
"links": [],
@@ -2476,8 +2471,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2493,7 +2487,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 62
+ "y": 38
},
"id": 119,
"options": {
@@ -2581,8 +2575,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2598,7 +2591,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 70
+ "y": 46
},
"id": 70,
"links": [],
@@ -2686,8 +2679,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2703,7 +2695,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 70
+ "y": 46
},
"id": 120,
"options": {
@@ -2809,7 +2801,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -2842,7 +2835,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 31
+ "y": 15
},
"id": 102,
"options": {
@@ -2940,7 +2933,8 @@
"mode": "absolute",
"steps": [
{
- "color": "transparent"
+ "color": "transparent",
+ "value": null
},
{
"color": "red",
@@ -2956,7 +2950,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 31
+ "y": 15
},
"id": 108,
"options": {
@@ -3041,7 +3035,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3057,7 +3052,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 39
+ "y": 23
},
"id": 142,
"links": [
@@ -3109,7 +3104,7 @@
"type": "prometheus",
"uid": "$ds"
},
- "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
+ "description": "Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.\n\nThe less value is better.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3120,6 +3115,7 @@
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
+ "axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
@@ -3152,7 +3148,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3160,7 +3157,7 @@
}
]
},
- "unit": "short"
+ "unit": "percentunit"
},
"overrides": []
},
@@ -3168,7 +3165,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 39
+ "y": 23
},
"id": 107,
"options": {
@@ -3194,13 +3191,15 @@
"type": "prometheus",
"uid": "$ds"
},
- "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))",
+ "editorMode": "code",
+ "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))",
"interval": "",
- "legendFormat": "slow queries rate",
+ "legendFormat": "slow queries %",
+ "range": true,
"refId": "A"
}
],
- "title": "Slow queries rate ($instance)",
+ "title": "Slow queries % ($instance)",
"type": "timeseries"
},
{
@@ -3251,7 +3250,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3267,7 +3267,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 47
+ "y": 31
},
"id": 170,
"links": [],
@@ -3357,7 +3357,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3373,7 +3374,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 47
+ "y": 31
},
"id": 116,
"links": [],
@@ -3459,7 +3460,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3475,7 +3477,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 55
+ "y": 39
},
"id": 144,
"options": {
@@ -3562,7 +3564,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3578,7 +3581,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 55
+ "y": 39
},
"id": 58,
"links": [],
@@ -3642,7 +3645,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3679,10 +3683,10 @@
]
},
"gridPos": {
- "h": 7,
+ "h": 6,
"w": 24,
"x": 0,
- "y": 64
+ "y": 48
},
"id": 183,
"options": {
@@ -3701,7 +3705,7 @@
}
]
},
- "pluginVersion": "9.1.0",
+ "pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@@ -3747,6 +3751,109 @@
}
],
"type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "id": 135,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job_storage\", instance=~\"$instance\"}[1h])) by (reason)",
+ "interval": "",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rows ignored for last 1h ($instance)",
+ "type": "timeseries"
}
],
"title": "Troubleshooting",
@@ -3830,7 +3937,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 21
+ "y": 29
},
"id": 76,
"links": [],
@@ -3946,7 +4053,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 21
+ "y": 29
},
"id": 86,
"links": [],
@@ -4071,7 +4178,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 30
+ "y": 38
},
"id": 80,
"links": [],
@@ -4176,7 +4283,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 30
+ "y": 38
},
"id": 78,
"links": [],
@@ -4292,7 +4399,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 38
+ "y": 46
},
"id": 82,
"options": {
@@ -4399,7 +4506,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 38
+ "y": 46
},
"id": 74,
"options": {
@@ -4501,7 +4608,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -4612,7 +4720,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -4724,7 +4833,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -4869,7 +4979,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5006,7 +5117,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5109,7 +5221,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5251,7 +5364,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5354,7 +5468,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5462,7 +5577,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5595,7 +5711,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5707,7 +5824,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
}
]
},
@@ -5822,7 +5940,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5916,7 +6035,7 @@
"type": "prometheus",
"uid": "$ds"
},
- "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
+ "description": "Shows network usage by vmstorage services.\n* Writes show traffic sent to vmselects.\n* Reads show traffic received from vminserts.",
"fieldConfig": {
"defaults": {
"color": {
@@ -5952,12 +6071,14 @@
"mode": "off"
}
},
+ "links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5965,23 +6086,36 @@
}
]
},
- "unit": "short"
+ "unit": "bps"
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 24,
"x": 0,
"y": 53
},
- "id": 135,
+ "id": 206,
+ "links": [],
"options": {
"legend": {
"calcs": [
"mean",
- "lastNotNull",
- "max"
+ "lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
@@ -5991,7 +6125,7 @@
},
"tooltip": {
"mode": "multi",
- "sort": "none"
+ "sort": "desc"
}
},
"pluginVersion": "9.1.0",
@@ -6002,15 +6136,29 @@
"uid": "$ds"
},
"editorMode": "code",
- "exemplar": true,
- "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job_storage\", instance=~\"$instance\"}[1h])) by (reason)",
- "interval": "",
- "legendFormat": "__auto",
+ "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Rows ignored for last 1h ($instance)",
+ "title": "Network usage ($instance)",
"type": "timeseries"
}
],
@@ -6079,7 +6227,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6095,7 +6244,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 98
+ "y": 42
},
"id": 92,
"links": [],
@@ -6185,7 +6334,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6221,7 +6371,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 98
+ "y": 42
},
"id": 95,
"links": [],
@@ -6327,7 +6477,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6343,7 +6494,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 106
+ "y": 50
},
"id": 163,
"links": [],
@@ -6471,7 +6622,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6487,7 +6639,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 106
+ "y": 50
},
"id": 165,
"links": [],
@@ -6611,7 +6763,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6627,7 +6780,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 114
+ "y": 58
},
"id": 178,
"links": [],
@@ -6718,7 +6871,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6734,7 +6888,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 114
+ "y": 58
},
"id": 180,
"links": [],
@@ -6825,7 +6979,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6841,7 +6996,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 122
+ "y": 66
},
"id": 179,
"links": [],
@@ -6932,7 +7087,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6948,7 +7104,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 122
+ "y": 66
},
"id": 181,
"links": [],
@@ -6995,7 +7151,7 @@
"type": "prometheus",
"uid": "$ds"
},
- "description": "",
+ "description": "Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7037,7 +7193,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7064,9 +7221,9 @@
},
"gridPos": {
"h": 8,
- "w": 24,
+ "w": 12,
"x": 0,
- "y": 130
+ "y": 74
},
"id": 93,
"links": [],
@@ -7117,7 +7274,138 @@
"refId": "B"
}
],
- "title": "Network usage ($instance)",
+ "title": "Network usage: clients ($instance)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows network usage between vmselects and vmstorages.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 74
+ },
+ "id": 207,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Network usage: vmstorage ($instance)",
"type": "timeseries"
}
],
@@ -7186,7 +7474,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7202,7 +7491,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 24
+ "y": 43
},
"id": 97,
"links": [],
@@ -7292,7 +7581,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7328,7 +7618,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 24
+ "y": 43
},
"id": 99,
"links": [],
@@ -7436,7 +7726,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7452,7 +7743,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 32
+ "y": 51
},
"id": 185,
"links": [],
@@ -7580,7 +7871,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7596,7 +7888,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 32
+ "y": 51
},
"id": 187,
"links": [],
@@ -7671,219 +7963,6 @@
"title": "Memory usage % ($instance)",
"type": "timeseries"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 40
- },
- "id": 90,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "read",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Network usage ($instance)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "decimals": 2,
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 40
- },
- "id": 88,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "max",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Rows per insert ($instance)",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -7933,7 +8012,8 @@
"mode": "absolute",
"steps": [
{
- "color": "transparent"
+ "color": "transparent",
+ "value": null
},
{
"color": "red",
@@ -7949,7 +8029,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 48
+ "y": 59
},
"id": 139,
"links": [],
@@ -8040,7 +8120,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -8056,7 +8137,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 48
+ "y": 59
},
"id": 114,
"links": [],
@@ -8094,6 +8175,376 @@
],
"title": "Storage reachability ($instance)",
"type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 67
+ },
+ "id": 208,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Network usage: clients ($instance)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "Shows network usage between vminserts and vmstorages.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 67
+ },
+ "id": 209,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Network usage: vmstorage ($instance)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 2,
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 75
+ },
+ "id": 88,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rows per insert ($instance)",
+ "type": "timeseries"
}
],
"title": "vminsert ($instance)",
@@ -8118,7 +8569,7 @@
"h": 2,
"w": 24,
"x": 0,
- "y": 84
+ "y": 92
},
"id": 198,
"options": {
@@ -8182,8 +8633,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8195,7 +8645,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 86
+ "y": 94
},
"id": 189,
"links": [],
@@ -8284,8 +8734,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8297,7 +8746,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 86
+ "y": 94
},
"id": 190,
"links": [],
@@ -8386,8 +8835,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8399,7 +8847,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 94
+ "y": 102
},
"id": 192,
"links": [],
@@ -8490,8 +8938,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -8507,7 +8954,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 94
+ "y": 102
},
"id": 196,
"links": [],
@@ -8597,8 +9044,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8610,7 +9056,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 101
+ "y": 109
},
"id": 200,
"links": [],
@@ -8699,8 +9145,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8712,7 +9157,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 101
+ "y": 109
},
"id": 201,
"links": [],
@@ -8815,8 +9260,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -8832,7 +9276,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 109
+ "y": 117
},
"id": 203,
"links": [],
diff --git a/dashboards/vm/victoriametrics-cluster.json b/dashboards/vm/victoriametrics-cluster.json
index 6ff054d0fb..854a233b23 100644
--- a/dashboards/vm/victoriametrics-cluster.json
+++ b/dashboards/vm/victoriametrics-cluster.json
@@ -1661,7 +1661,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 38
+ "y": 14
},
"id": 66,
"links": [],
@@ -1773,7 +1773,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 38
+ "y": 14
},
"id": 138,
"links": [],
@@ -1867,8 +1867,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1884,7 +1883,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 46
+ "y": 22
},
"id": 64,
"links": [],
@@ -1974,8 +1973,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2004,7 +2002,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 46
+ "y": 22
},
"id": 122,
"links": [],
@@ -2112,8 +2110,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2145,7 +2142,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 54
+ "y": 30
},
"id": 117,
"links": [],
@@ -2235,8 +2232,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2265,7 +2261,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 54
+ "y": 30
},
"id": 204,
"links": [],
@@ -2372,8 +2368,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2389,7 +2384,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 62
+ "y": 38
},
"id": 68,
"links": [],
@@ -2477,8 +2472,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2494,7 +2488,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 62
+ "y": 38
},
"id": 119,
"options": {
@@ -2582,8 +2576,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2599,7 +2592,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 70
+ "y": 46
},
"id": 70,
"links": [],
@@ -2687,8 +2680,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2704,7 +2696,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 70
+ "y": 46
},
"id": 120,
"options": {
@@ -2810,7 +2802,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -2843,7 +2836,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 31
+ "y": 15
},
"id": 102,
"options": {
@@ -2941,7 +2934,8 @@
"mode": "absolute",
"steps": [
{
- "color": "transparent"
+ "color": "transparent",
+ "value": null
},
{
"color": "red",
@@ -2957,7 +2951,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 31
+ "y": 15
},
"id": 108,
"options": {
@@ -3042,7 +3036,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3058,7 +3053,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 39
+ "y": 23
},
"id": 142,
"links": [
@@ -3110,7 +3105,7 @@
"type": "victoriametrics-datasource",
"uid": "$ds"
},
- "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.",
+ "description": "Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.\n\nThe less value is better.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3121,6 +3116,7 @@
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
+ "axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
@@ -3153,7 +3149,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3161,7 +3158,7 @@
}
]
},
- "unit": "short"
+ "unit": "percentunit"
},
"overrides": []
},
@@ -3169,7 +3166,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 39
+ "y": 23
},
"id": 107,
"options": {
@@ -3195,13 +3192,15 @@
"type": "victoriametrics-datasource",
"uid": "$ds"
},
- "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))",
+ "editorMode": "code",
+ "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))",
"interval": "",
- "legendFormat": "slow queries rate",
+ "legendFormat": "slow queries %",
+ "range": true,
"refId": "A"
}
],
- "title": "Slow queries rate ($instance)",
+ "title": "Slow queries % ($instance)",
"type": "timeseries"
},
{
@@ -3252,7 +3251,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3268,7 +3268,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 47
+ "y": 31
},
"id": 170,
"links": [],
@@ -3358,7 +3358,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3374,7 +3375,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 47
+ "y": 31
},
"id": 116,
"links": [],
@@ -3460,7 +3461,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3476,7 +3478,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 55
+ "y": 39
},
"id": 144,
"options": {
@@ -3563,7 +3565,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3579,7 +3582,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 55
+ "y": 39
},
"id": 58,
"links": [],
@@ -3643,7 +3646,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -3680,10 +3684,10 @@
]
},
"gridPos": {
- "h": 7,
+ "h": 6,
"w": 24,
"x": 0,
- "y": 64
+ "y": 48
},
"id": 183,
"options": {
@@ -3702,7 +3706,7 @@
}
]
},
- "pluginVersion": "9.1.0",
+ "pluginVersion": "9.2.7",
"targets": [
{
"datasource": {
@@ -3748,6 +3752,109 @@
}
],
"type": "table"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "id": 135,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job_storage\", instance=~\"$instance\"}[1h])) by (reason)",
+ "interval": "",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rows ignored for last 1h ($instance)",
+ "type": "timeseries"
}
],
"title": "Troubleshooting",
@@ -3831,7 +3938,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 21
+ "y": 29
},
"id": 76,
"links": [],
@@ -3947,7 +4054,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 21
+ "y": 29
},
"id": 86,
"links": [],
@@ -4072,7 +4179,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 30
+ "y": 38
},
"id": 80,
"links": [],
@@ -4177,7 +4284,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 30
+ "y": 38
},
"id": 78,
"links": [],
@@ -4293,7 +4400,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 38
+ "y": 46
},
"id": 82,
"options": {
@@ -4400,7 +4507,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 38
+ "y": 46
},
"id": 74,
"options": {
@@ -4502,7 +4609,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -4613,7 +4721,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -4725,7 +4834,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -4870,7 +4980,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5007,7 +5118,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5110,7 +5222,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5252,7 +5365,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5355,7 +5469,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5463,7 +5578,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5596,7 +5712,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5708,7 +5825,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
}
]
},
@@ -5823,7 +5941,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5917,7 +6036,7 @@
"type": "victoriametrics-datasource",
"uid": "$ds"
},
- "description": "Shows how many rows were ignored on insertion due to corrupted or out of retention timestamps.",
+ "description": "Shows network usage by vmstorage services.\n* Writes show traffic sent to vmselects.\n* Reads show traffic received from vminserts.",
"fieldConfig": {
"defaults": {
"color": {
@@ -5953,12 +6072,14 @@
"mode": "off"
}
},
+ "links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -5966,23 +6087,36 @@
}
]
},
- "unit": "short"
+ "unit": "bps"
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 24,
"x": 0,
"y": 53
},
- "id": 135,
+ "id": 206,
+ "links": [],
"options": {
"legend": {
"calcs": [
"mean",
- "lastNotNull",
- "max"
+ "lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
@@ -5992,7 +6126,7 @@
},
"tooltip": {
"mode": "multi",
- "sort": "none"
+ "sort": "desc"
}
},
"pluginVersion": "9.1.0",
@@ -6003,15 +6137,29 @@
"uid": "$ds"
},
"editorMode": "code",
- "exemplar": true,
- "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job_storage\", instance=~\"$instance\"}[1h])) by (reason)",
- "interval": "",
- "legendFormat": "__auto",
+ "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Rows ignored for last 1h ($instance)",
+ "title": "Network usage ($instance)",
"type": "timeseries"
}
],
@@ -6080,7 +6228,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6096,7 +6245,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 98
+ "y": 42
},
"id": 92,
"links": [],
@@ -6186,7 +6335,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6222,7 +6372,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 98
+ "y": 42
},
"id": 95,
"links": [],
@@ -6328,7 +6478,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6344,7 +6495,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 106
+ "y": 50
},
"id": 163,
"links": [],
@@ -6472,7 +6623,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6488,7 +6640,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 106
+ "y": 50
},
"id": 165,
"links": [],
@@ -6612,7 +6764,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6628,7 +6781,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 114
+ "y": 58
},
"id": 178,
"links": [],
@@ -6719,7 +6872,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6735,7 +6889,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 114
+ "y": 58
},
"id": 180,
"links": [],
@@ -6826,7 +6980,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6842,7 +6997,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 122
+ "y": 66
},
"id": 179,
"links": [],
@@ -6933,7 +7088,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -6949,7 +7105,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 122
+ "y": 66
},
"id": 181,
"links": [],
@@ -6996,7 +7152,7 @@
"type": "victoriametrics-datasource",
"uid": "$ds"
},
- "description": "",
+ "description": "Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7038,7 +7194,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7065,9 +7222,9 @@
},
"gridPos": {
"h": 8,
- "w": 24,
+ "w": 12,
"x": 0,
- "y": 130
+ "y": 74
},
"id": 93,
"links": [],
@@ -7118,7 +7275,138 @@
"refId": "B"
}
],
- "title": "Network usage ($instance)",
+ "title": "Network usage: clients ($instance)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "description": "Shows network usage between vmselects and vmstorages.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 74
+ },
+ "id": 207,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Network usage: vmstorage ($instance)",
"type": "timeseries"
}
],
@@ -7187,7 +7475,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7203,7 +7492,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 24
+ "y": 43
},
"id": 97,
"links": [],
@@ -7293,7 +7582,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7329,7 +7619,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 24
+ "y": 43
},
"id": 99,
"links": [],
@@ -7437,7 +7727,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7453,7 +7744,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 32
+ "y": 51
},
"id": 185,
"links": [],
@@ -7581,7 +7872,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -7597,7 +7889,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 32
+ "y": 51
},
"id": 187,
"links": [],
@@ -7672,219 +7964,6 @@
"title": "Memory usage % ($instance)",
"type": "timeseries"
},
- {
- "datasource": {
- "type": "victoriametrics-datasource",
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "normal"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 40
- },
- "id": 90,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.0",
- "targets": [
- {
- "datasource": {
- "type": "victoriametrics-datasource",
- "uid": "$ds"
- },
- "editorMode": "code",
- "exemplar": true,
- "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "read",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Network usage ($instance)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "victoriametrics-datasource",
- "uid": "$ds"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "decimals": 2,
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 40
- },
- "id": 88,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "9.1.0",
- "targets": [
- {
- "datasource": {
- "type": "victoriametrics-datasource",
- "uid": "$ds"
- },
- "editorMode": "code",
- "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "max",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Rows per insert ($instance)",
- "type": "timeseries"
- },
{
"datasource": {
"type": "victoriametrics-datasource",
@@ -7934,7 +8013,8 @@
"mode": "absolute",
"steps": [
{
- "color": "transparent"
+ "color": "transparent",
+ "value": null
},
{
"color": "red",
@@ -7950,7 +8030,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 48
+ "y": 59
},
"id": 139,
"links": [],
@@ -8041,7 +8121,8 @@
"mode": "absolute",
"steps": [
{
- "color": "green"
+ "color": "green",
+ "value": null
},
{
"color": "red",
@@ -8057,7 +8138,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 48
+ "y": 59
},
"id": 114,
"links": [],
@@ -8095,6 +8176,376 @@
],
"title": "Storage reachability ($instance)",
"type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "description": "Shows network usage between vminserts and clients, such as vmagent, VictoriaMetrics, or any other client pushing metrics to vminsert.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 67
+ },
+ "id": 208,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Network usage: clients ($instance)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "description": "Shows network usage between vminserts and vmstorages.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "/read.*/"
+ },
+ "properties": [
+ {
+ "id": "custom.transform",
+ "value": "negative-Y"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 67
+ },
+ "id": 209,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": "write ",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Network usage: vmstorage ($instance)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 2,
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 75
+ },
+ "id": 88,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "victoriametrics-datasource",
+ "uid": "$ds"
+ },
+ "editorMode": "code",
+ "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "max",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rows per insert ($instance)",
+ "type": "timeseries"
}
],
"title": "vminsert ($instance)",
@@ -8119,7 +8570,7 @@
"h": 2,
"w": 24,
"x": 0,
- "y": 84
+ "y": 92
},
"id": 198,
"options": {
@@ -8183,8 +8634,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8196,7 +8646,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 86
+ "y": 94
},
"id": 189,
"links": [],
@@ -8285,8 +8735,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8298,7 +8747,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 86
+ "y": 94
},
"id": 190,
"links": [],
@@ -8387,8 +8836,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8400,7 +8848,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 94
+ "y": 102
},
"id": 192,
"links": [],
@@ -8491,8 +8939,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -8508,7 +8955,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 94
+ "y": 102
},
"id": 196,
"links": [],
@@ -8598,8 +9045,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8611,7 +9057,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 101
+ "y": 109
},
"id": 200,
"links": [],
@@ -8700,8 +9146,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -8713,7 +9158,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 101
+ "y": 109
},
"id": 201,
"links": [],
@@ -8816,8 +9261,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -8833,7 +9277,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 109
+ "y": 117
},
"id": 203,
"links": [],
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 611d8f6c6e..ce1cbc9337 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -39,6 +39,8 @@ The sandbox cluster installation is running under the constant load generated by
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
* FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics).
* FEATURE: all VictoriaMetrics components: add `-metrics.exposeMetadata` command-line flag, which allows displaying `TYPE` and `HELP` metadata at `/metrics` page exposed at `-httpListenAddr`. This may be needed when the `/metrics` page is scraped by collector, which requires the `TYPE` and `HELP` metadata such as [Google Cloud Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type).
+* FEATURE: dashboards/cluster: add panels for detailed visualization of traffic usage between vmstorage, vminsert, vmselect components and their clients. New panels are available in the rows dedicated to specific components.
+* FEATURE: dashboards/cluster: update "Slow Queries" panel to show percentage of the slow queries to the total number of read queries served by vmselect. The percentage value should make it more clear for users whether there is a service degradation.
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly return full results when `-search.skipSlowReplicas` command-line flag is passed to `vmselect` and when [vmstorage groups](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmstorage-groups-at-vmselect) are in use. Previously partial results could be returned in this case.
* BUGFIX: `vminsert`: properly accept samples via [OpenTelemetry data ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) when these samples have no [resource attributes](https://opentelemetry.io/docs/instrumentation/go/resources/). Previously such samples were silently skipped.