github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

deployment: add panel and alerts for displying go scheduler latency (#7078)

Browse source 
The panel and alerting rule should help to understand whether VM
component doesn't have enough CPU resources or gets throttled. The alert
is applicable for all VM components.
The panel was added to vmalert, vmagent, vmsingle, vm clusert and
victorialogs dashes.

-------------------

This alerting rule should have help us identify resource shortage for
sandbox vmagent - see [this
link](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/?g0.range_input=23d13h25m25s424ms&g0.end_input=2024-09-23T14%3A11%3A00&g0.relative_time=none&g0.tab=0&g0.expr=histogram_quantile%280.99%2C+sum%28rate%28go_sched_latencies_seconds_bucket%7Bjob%3D%22vmagent-monitoring-vmagent%22%7D%5B5m%5D%29%29+by+%28le%2C+job%2C+instance%29%29+%3E+0.1)
for example. We weren't aware of resource shortage, because VM metrics
assumed this vmagent had 1vCPU while in fact its limit was 0.2vCPU.

Signed-off-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
Roman Khavronenko 2024-09-23 16:54:42 +02:00 committed by GitHub
parent 109772bdc4
commit 4d0b41e63b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 2872 additions and 1964 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
dashboards/victorialogs.json
View file

@ -1117,10 +1117,7 @@
"y": 22
},
"id": 28,
"panels": [],
"title": "Resource usage",
"type": "row"
},
"panels": [
{
"datasource": {
"type": "prometheus",
@ -1188,7 +1185,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 3
},
"id": 38,
"options": {
@ -1295,7 +1292,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 23
"y": 3
},
"id": 40,
"options": {
@ -1452,7 +1449,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 31
"y": 11
},
"id": 42,
"options": {
@ -1558,7 +1555,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 11
},
"id": 44,
"options": {
@ -1684,7 +1681,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 39
"y": 19
},
"id": 46,
"options": {
@ -1809,7 +1806,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 39
"y": 19
},
"id": 48,
"options": {
@ -1931,7 +1928,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 47
"y": 27
},
"id": 50,
"options": {
@ -2050,7 +2047,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 47
"y": 27
},
"id": 52,
"options": {
@ -2170,7 +2167,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 55
"y": 35
},
"id": 54,
"options": {
@ -2289,7 +2286,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 55
"y": 35
},
"id": 56,
"options": {
@ -2413,7 +2410,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 63
"y": 43
},
"id": 58,
"options": {
@ -2521,7 +2518,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 63
"y": 43
},
"id": 60,
"options": {
@ -2561,6 +2558,117 @@
],
"title": "TCP connections ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 51
},
"id": 61,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"title": "Resource usage",
"type": "row"
}
],
"refresh": "",

309
dashboards/victoriametrics-cluster.json
View file

@ -1659,7 +1659,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1667,8 +1668,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": []
},
@ -1679,7 +1679,6 @@
"y": 3
},
"id": 66,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2268,8 +2267,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": [
{
@ -2293,7 +2291,6 @@
"y": 19
},
"id": 204,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2399,7 +2396,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2407,8 +2405,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2419,7 +2416,6 @@
"y": 27
},
"id": 68,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2506,7 +2502,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2514,8 +2511,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2613,7 +2609,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2621,8 +2618,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2633,7 +2629,6 @@
"y": 35
},
"id": 70,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2720,7 +2715,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2728,8 +2724,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2828,7 +2823,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2836,8 +2832,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": []
},
@ -2848,7 +2843,6 @@
"y": 43
},
"id": 210,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2884,6 +2878,113 @@
],
"title": "CPU spent on GC ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 43
},
"id": 213,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"title": "Resource usage ($job)",
@ -2913,7 +3014,7 @@
"h": 2,
"w": 24,
"x": 0,
"y": 4
"y": 12
},
"id": 211,
"links": [],
@ -3011,7 +3112,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 31
},
"id": 102,
"options": {
@ -3125,7 +3226,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 23
"y": 31
},
"id": 108,
"options": {
@ -3226,7 +3327,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 31
"y": 39
},
"id": 142,
"links": [
@ -3338,7 +3439,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 39
},
"id": 107,
"options": {
@ -3439,7 +3540,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 39
"y": 47
},
"id": 170,
"links": [],
@ -3545,7 +3646,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 39
"y": 47
},
"id": 116,
"links": [],
@ -3647,7 +3748,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 47
"y": 55
},
"id": 144,
"options": {
@ -3750,7 +3851,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 47
"y": 55
},
"id": 58,
"links": [],
@ -3856,7 +3957,7 @@
"h": 6,
"w": 24,
"x": 0,
"y": 56
"y": 64
},
"id": 183,
"options": {
@ -3984,7 +4085,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 62
"y": 70
},
"id": 135,
"options": {
@ -4106,7 +4207,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"id": 76,
"links": [],
@ -4222,7 +4323,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 37
"y": 45
},
"id": 86,
"links": [],
@ -4347,7 +4448,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 46
"y": 54
},
"id": 80,
"links": [],
@ -4452,7 +4553,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 46
"y": 54
},
"id": 78,
"links": [],
@ -4568,7 +4669,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 54
"y": 62
},
"id": 82,
"options": {
@ -4675,7 +4776,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 54
"y": 62
},
"id": 74,
"options": {
@ -4779,8 +4880,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4796,7 +4896,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 6
"y": 14
},
"id": 100,
"options": {
@ -4892,8 +4992,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4909,7 +5008,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 6
"y": 14
},
"id": 113,
"options": {
@ -5006,8 +5105,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5023,7 +5121,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 14
"y": 22
},
"id": 151,
"options": {
@ -5153,8 +5251,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5170,7 +5267,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 14
"y": 22
},
"id": 167,
"options": {
@ -5292,8 +5389,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5325,7 +5421,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 21
"y": 29
},
"id": 212,
"links": [
@ -5434,8 +5530,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5467,7 +5562,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 21
"y": 29
},
"id": 133,
"options": {
@ -5572,8 +5667,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5589,7 +5683,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 29
"y": 37
},
"id": 54,
"options": {
@ -5678,8 +5772,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5695,7 +5788,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 29
"y": 37
},
"id": 55,
"options": {
@ -5789,8 +5882,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5806,7 +5898,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"id": 20,
"options": {
@ -5924,8 +6016,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5941,7 +6032,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 37
"y": 45
},
"id": 22,
"options": {
@ -6038,8 +6129,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
}
]
},
@ -6051,7 +6141,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 45
"y": 53
},
"id": 202,
"options": {
@ -6155,8 +6245,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6189,7 +6278,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 45
"y": 53
},
"id": 14,
"options": {
@ -6293,8 +6382,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6323,7 +6411,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 53
"y": 61
},
"id": 206,
"options": {
@ -6424,8 +6512,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6441,7 +6528,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 53
"y": 61
},
"id": 141,
"links": [
@ -6564,7 +6651,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 50
"y": 58
},
"id": 92,
"links": [],
@ -6690,7 +6777,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 50
"y": 58
},
"id": 95,
"links": [],
@ -6812,7 +6899,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 58
"y": 66
},
"id": 163,
"links": [],
@ -6956,7 +7043,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 58
"y": 66
},
"id": 165,
"links": [],
@ -7096,7 +7183,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 66
"y": 74
},
"id": 178,
"links": [],
@ -7203,7 +7290,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 66
"y": 74
},
"id": 180,
"links": [],
@ -7310,7 +7397,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 74
"y": 82
},
"id": 179,
"links": [],
@ -7417,7 +7504,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 74
"y": 82
},
"id": 181,
"links": [],
@ -7535,7 +7622,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 82
"y": 90
},
"id": 93,
"links": [],
@ -7665,7 +7752,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 82
"y": 90
},
"id": 207,
"links": [],
@ -7801,7 +7888,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 51
"y": 59
},
"id": 97,
"links": [],
@ -7927,7 +8014,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 51
"y": 59
},
"id": 99,
"links": [],
@ -8051,7 +8138,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 59
"y": 67
},
"id": 185,
"links": [],
@ -8195,7 +8282,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 59
"y": 67
},
"id": 187,
"links": [],
@ -8335,7 +8422,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 67
"y": 75
},
"id": 139,
"links": [],
@ -8442,7 +8529,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 67
"y": 75
},
"id": 114,
"links": [],
@ -8557,7 +8644,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 75
"y": 83
},
"id": 208,
"links": [],
@ -8687,7 +8774,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 75
"y": 83
},
"id": 209,
"links": [],
@ -8806,7 +8893,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 83
"y": 91
},
"id": 88,
"links": [],
@ -8871,7 +8958,7 @@
"h": 2,
"w": 24,
"x": 0,
"y": 100
"y": 108
},
"id": 198,
"options": {
@ -8947,7 +9034,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 102
"y": 110
},
"id": 189,
"links": [],
@ -9048,7 +9135,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 102
"y": 110
},
"id": 190,
"links": [],
@ -9149,7 +9236,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 110
"y": 118
},
"id": 192,
"links": [],
@ -9256,7 +9343,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 110
"y": 118
},
"id": 196,
"links": [],
@ -9358,7 +9445,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 117
"y": 125
},
"id": 200,
"links": [],
@ -9459,7 +9546,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 117
"y": 125
},
"id": 201,
"links": [],
@ -9578,7 +9665,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 125
"y": 133
},
"id": 203,
"links": [],

207
dashboards/victoriametrics.json
View file

@ -1569,7 +1569,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 31
"y": 3
},
"id": 112,
"links": [],
@ -1677,10 +1677,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 3
},
"id": 44,
"links": [],
"options": {
"legend": {
"calcs": [
@ -1845,7 +1844,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 39
"y": 11
},
"id": 123,
"links": [],
@ -1952,10 +1951,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 39
"y": 11
},
"id": 114,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2046,7 +2044,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2054,8 +2053,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": [
{
@ -2079,10 +2077,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 47
"y": 19
},
"id": 75,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2172,7 +2169,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2180,8 +2178,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": [
{
@ -2205,10 +2202,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 47
"y": 19
},
"id": 57,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2313,7 +2309,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2321,8 +2318,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2330,10 +2326,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 55
"y": 27
},
"id": 47,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2420,7 +2415,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2428,8 +2424,7 @@
}
]
},
"unit": "bytes",
"unitScale": true
"unit": "bytes"
},
"overrides": [
{
@ -2450,10 +2445,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 55
"y": 27
},
"id": 76,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2558,7 +2552,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2566,8 +2561,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2575,10 +2569,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 63
"y": 35
},
"id": 48,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2665,7 +2658,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2673,8 +2667,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": [
{
@ -2695,10 +2688,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 63
"y": 35
},
"id": 124,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2803,7 +2795,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2811,8 +2804,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2820,10 +2812,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 71
"y": 43
},
"id": 49,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2912,7 +2903,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2920,8 +2912,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2929,10 +2920,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 71
"y": 43
},
"id": 37,
"links": [],
"options": {
"legend": {
"calcs": [
@ -3020,7 +3010,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3028,8 +3019,7 @@
}
]
},
"unit": "decbits",
"unitScale": true
"unit": "decbits"
},
"overrides": [
{
@ -3050,10 +3040,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 79
"y": 51
},
"id": 127,
"links": [],
"options": {
"legend": {
"calcs": [
@ -3157,7 +3146,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3165,8 +3155,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": []
},
@ -3174,10 +3163,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 79
"y": 51
},
"id": 125,
"links": [],
"options": {
"legend": {
"calcs": [
@ -3213,6 +3201,113 @@
],
"title": "CPU spent on GC",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 59
},
"id": 128,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"targets": [

309
dashboards/vm/victoriametrics-cluster.json
View file

@ -1660,7 +1660,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1668,8 +1669,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": []
},
@ -1680,7 +1680,6 @@
"y": 3
},
"id": 66,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2269,8 +2268,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": [
{
@ -2294,7 +2292,6 @@
"y": 19
},
"id": 204,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2400,7 +2397,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2408,8 +2406,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2420,7 +2417,6 @@
"y": 27
},
"id": 68,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2507,7 +2503,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2515,8 +2512,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2614,7 +2610,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2622,8 +2619,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2634,7 +2630,6 @@
"y": 35
},
"id": 70,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2721,7 +2716,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2729,8 +2725,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2829,7 +2824,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2837,8 +2833,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": []
},
@ -2849,7 +2844,6 @@
"y": 43
},
"id": 210,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2885,6 +2879,113 @@
],
"title": "CPU spent on GC ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 43
},
"id": 213,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"title": "Resource usage ($job)",
@ -2914,7 +3015,7 @@
"h": 2,
"w": 24,
"x": 0,
"y": 4
"y": 12
},
"id": 211,
"links": [],
@ -3012,7 +3113,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 31
},
"id": 102,
"options": {
@ -3126,7 +3227,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 23
"y": 31
},
"id": 108,
"options": {
@ -3227,7 +3328,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 31
"y": 39
},
"id": 142,
"links": [
@ -3339,7 +3440,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 39
},
"id": 107,
"options": {
@ -3440,7 +3541,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 39
"y": 47
},
"id": 170,
"links": [],
@ -3546,7 +3647,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 39
"y": 47
},
"id": 116,
"links": [],
@ -3648,7 +3749,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 47
"y": 55
},
"id": 144,
"options": {
@ -3751,7 +3852,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 47
"y": 55
},
"id": 58,
"links": [],
@ -3857,7 +3958,7 @@
"h": 6,
"w": 24,
"x": 0,
"y": 56
"y": 64
},
"id": 183,
"options": {
@ -3985,7 +4086,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 62
"y": 70
},
"id": 135,
"options": {
@ -4107,7 +4208,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"id": 76,
"links": [],
@ -4223,7 +4324,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 37
"y": 45
},
"id": 86,
"links": [],
@ -4348,7 +4449,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 46
"y": 54
},
"id": 80,
"links": [],
@ -4453,7 +4554,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 46
"y": 54
},
"id": 78,
"links": [],
@ -4569,7 +4670,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 54
"y": 62
},
"id": 82,
"options": {
@ -4676,7 +4777,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 54
"y": 62
},
"id": 74,
"options": {
@ -4780,8 +4881,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4797,7 +4897,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 6
"y": 14
},
"id": 100,
"options": {
@ -4893,8 +4993,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4910,7 +5009,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 6
"y": 14
},
"id": 113,
"options": {
@ -5007,8 +5106,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5024,7 +5122,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 14
"y": 22
},
"id": 151,
"options": {
@ -5154,8 +5252,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5171,7 +5268,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 14
"y": 22
},
"id": 167,
"options": {
@ -5293,8 +5390,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5326,7 +5422,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 21
"y": 29
},
"id": 212,
"links": [
@ -5435,8 +5531,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5468,7 +5563,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 21
"y": 29
},
"id": 133,
"options": {
@ -5573,8 +5668,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5590,7 +5684,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 29
"y": 37
},
"id": 54,
"options": {
@ -5679,8 +5773,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5696,7 +5789,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 29
"y": 37
},
"id": 55,
"options": {
@ -5790,8 +5883,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5807,7 +5899,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"id": 20,
"options": {
@ -5925,8 +6017,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5942,7 +6033,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 37
"y": 45
},
"id": 22,
"options": {
@ -6039,8 +6130,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
}
]
},
@ -6052,7 +6142,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 45
"y": 53
},
"id": 202,
"options": {
@ -6156,8 +6246,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6190,7 +6279,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 45
"y": 53
},
"id": 14,
"options": {
@ -6294,8 +6383,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6324,7 +6412,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 53
"y": 61
},
"id": 206,
"options": {
@ -6425,8 +6513,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6442,7 +6529,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 53
"y": 61
},
"id": 141,
"links": [
@ -6565,7 +6652,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 50
"y": 58
},
"id": 92,
"links": [],
@ -6691,7 +6778,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 50
"y": 58
},
"id": 95,
"links": [],
@ -6813,7 +6900,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 58
"y": 66
},
"id": 163,
"links": [],
@ -6957,7 +7044,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 58
"y": 66
},
"id": 165,
"links": [],
@ -7097,7 +7184,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 66
"y": 74
},
"id": 178,
"links": [],
@ -7204,7 +7291,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 66
"y": 74
},
"id": 180,
"links": [],
@ -7311,7 +7398,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 74
"y": 82
},
"id": 179,
"links": [],
@ -7418,7 +7505,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 74
"y": 82
},
"id": 181,
"links": [],
@ -7536,7 +7623,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 82
"y": 90
},
"id": 93,
"links": [],
@ -7666,7 +7753,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 82
"y": 90
},
"id": 207,
"links": [],
@ -7802,7 +7889,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 51
"y": 59
},
"id": 97,
"links": [],
@ -7928,7 +8015,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 51
"y": 59
},
"id": 99,
"links": [],
@ -8052,7 +8139,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 59
"y": 67
},
"id": 185,
"links": [],
@ -8196,7 +8283,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 59
"y": 67
},
"id": 187,
"links": [],
@ -8336,7 +8423,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 67
"y": 75
},
"id": 139,
"links": [],
@ -8443,7 +8530,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 67
"y": 75
},
"id": 114,
"links": [],
@ -8558,7 +8645,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 75
"y": 83
},
"id": 208,
"links": [],
@ -8688,7 +8775,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 75
"y": 83
},
"id": 209,
"links": [],
@ -8807,7 +8894,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 83
"y": 91
},
"id": 88,
"links": [],
@ -8872,7 +8959,7 @@
"h": 2,
"w": 24,
"x": 0,
"y": 100
"y": 108
},
"id": 198,
"options": {
@ -8948,7 +9035,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 102
"y": 110
},
"id": 189,
"links": [],
@ -9049,7 +9136,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 102
"y": 110
},
"id": 190,
"links": [],
@ -9150,7 +9237,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 110
"y": 118
},
"id": 192,
"links": [],
@ -9257,7 +9344,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 110
"y": 118
},
"id": 196,
"links": [],
@ -9359,7 +9446,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 117
"y": 125
},
"id": 200,
"links": [],
@ -9460,7 +9547,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 117
"y": 125
},
"id": 201,
"links": [],
@ -9579,7 +9666,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 125
"y": 133
},
"id": 203,
"links": [],

207
dashboards/vm/victoriametrics.json
View file

@ -1570,7 +1570,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 31
"y": 3
},
"id": 112,
"links": [],
@ -1678,10 +1678,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 3
},
"id": 44,
"links": [],
"options": {
"legend": {
"calcs": [
@ -1846,7 +1845,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 39
"y": 11
},
"id": 123,
"links": [],
@ -1953,10 +1952,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 39
"y": 11
},
"id": 114,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2047,7 +2045,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2055,8 +2054,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": [
{
@ -2080,10 +2078,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 47
"y": 19
},
"id": 75,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2173,7 +2170,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2181,8 +2179,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": [
{
@ -2206,10 +2203,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 47
"y": 19
},
"id": 57,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2314,7 +2310,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2322,8 +2319,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2331,10 +2327,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 55
"y": 27
},
"id": 47,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2421,7 +2416,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2429,8 +2425,7 @@
}
]
},
"unit": "bytes",
"unitScale": true
"unit": "bytes"
},
"overrides": [
{
@ -2451,10 +2446,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 55
"y": 27
},
"id": 76,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2559,7 +2553,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2567,8 +2562,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2576,10 +2570,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 63
"y": 35
},
"id": 48,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2666,7 +2659,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2674,8 +2668,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": [
{
@ -2696,10 +2689,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 63
"y": 35
},
"id": 124,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2804,7 +2796,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2812,8 +2805,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2821,10 +2813,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 71
"y": 43
},
"id": 49,
"links": [],
"options": {
"legend": {
"calcs": [
@ -2913,7 +2904,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2921,8 +2913,7 @@
}
]
},
"unit": "short",
"unitScale": true
"unit": "short"
},
"overrides": []
},
@ -2930,10 +2921,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 71
"y": 43
},
"id": 37,
"links": [],
"options": {
"legend": {
"calcs": [
@ -3021,7 +3011,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3029,8 +3020,7 @@
}
]
},
"unit": "decbits",
"unitScale": true
"unit": "decbits"
},
"overrides": [
{
@ -3051,10 +3041,9 @@
"h": 8,
"w": 12,
"x": 0,
"y": 79
"y": 51
},
"id": 127,
"links": [],
"options": {
"legend": {
"calcs": [
@ -3158,7 +3147,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3166,8 +3156,7 @@
}
]
},
"unit": "percentunit",
"unitScale": true
"unit": "percentunit"
},
"overrides": []
},
@ -3175,10 +3164,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 79
"y": 51
},
"id": 125,
"links": [],
"options": {
"legend": {
"calcs": [
@ -3214,6 +3202,113 @@
],
"title": "CPU spent on GC",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 59
},
"id": 128,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"targets": [

255
dashboards/vm/vmagent.json
View file

@ -1635,7 +1635,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 11
"y": 35
},
"id": 109,
"options": {
@ -1733,7 +1733,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1749,7 +1750,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 11
"y": 35
},
"id": 111,
"options": {
@ -1844,7 +1845,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1873,7 +1875,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 19
"y": 43
},
"id": 81,
"options": {
@ -1979,7 +1981,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2008,7 +2011,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 19
"y": 43
},
"id": 7,
"options": {
@ -2110,7 +2113,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2126,7 +2130,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 27
"y": 51
},
"id": 83,
"options": {
@ -2215,7 +2219,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2231,7 +2236,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 27
"y": 51
},
"id": 39,
"options": {
@ -2321,7 +2326,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2337,7 +2343,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 59
},
"id": 135,
"options": {
@ -2376,6 +2382,113 @@
"title": "CPU spent on GC ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 59
},
"id": 149,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
@ -2426,7 +2539,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2442,7 +2556,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 35
"y": 67
},
"id": 41,
"options": {
@ -2571,7 +2685,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 12
"y": 20
},
"id": 92,
"options": {
@ -2675,7 +2789,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 12
"y": 20
},
"id": 95,
"options": {
@ -2782,7 +2896,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 20
"y": 28
},
"id": 98,
"options": {
@ -2889,7 +3003,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 20
"y": 28
},
"id": 99,
"options": {
@ -2993,7 +3107,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 28
"y": 36
},
"id": 79,
"options": {
@ -3097,7 +3211,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 28
"y": 36
},
"id": 18,
"links": [
@ -3207,7 +3321,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 36
"y": 44
},
"id": 127,
"options": {
@ -3309,7 +3423,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 36
"y": 44
},
"id": 50,
"options": {
@ -3409,7 +3523,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 44
"y": 52
},
"id": 129,
"options": {
@ -3548,8 +3662,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3565,7 +3678,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"id": 48,
"options": {
@ -3655,8 +3768,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3672,7 +3784,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 37
"y": 45
},
"id": 76,
"options": {
@ -3760,8 +3872,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3777,7 +3888,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 44
"y": 52
},
"id": 132,
"options": {
@ -3867,8 +3978,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3884,7 +3994,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 44
"y": 52
},
"id": 133,
"options": {
@ -3973,8 +4083,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3990,7 +4099,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 51
"y": 59
},
"id": 20,
"options": {
@ -4078,8 +4187,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4095,7 +4203,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 51
"y": 59
},
"id": 126,
"options": {
@ -4182,8 +4290,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4199,7 +4306,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 59
"y": 67
},
"id": 46,
"options": {
@ -4286,8 +4393,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4303,7 +4409,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 59
"y": 67
},
"id": 148,
"options": {
@ -4390,8 +4496,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4407,7 +4512,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 67
"y": 75
},
"id": 31,
"options": {
@ -4586,7 +4691,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 14
"y": 22
},
"id": 73,
"options": {
@ -4703,7 +4808,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 6
"y": 14
},
"id": 131,
"options": {
@ -4807,7 +4912,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 14
"y": 22
},
"id": 130,
"options": {
@ -4924,7 +5029,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 14
"y": 22
},
"id": 77,
"options": {
@ -5058,7 +5163,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 7
"y": 15
},
"id": 146,
"options": {
@ -5160,7 +5265,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 7
"y": 15
},
"id": 143,
"options": {
@ -5256,7 +5361,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 15
"y": 23
},
"id": 147,
"options": {
@ -5359,7 +5464,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 15
"y": 23
},
"id": 139,
"options": {
@ -5470,7 +5575,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 31
},
"id": 142,
"options": {
@ -5567,7 +5672,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 23
"y": 31
},
"id": 137,
"options": {
@ -5680,7 +5785,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 39
},
"id": 141,
"options": {
@ -5811,7 +5916,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 8
"y": 16
},
"id": 60,
"options": {
@ -5915,7 +6020,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 8
"y": 16
},
"id": 66,
"options": {
@ -6019,7 +6124,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 16
"y": 24
},
"id": 61,
"options": {
@ -6123,7 +6228,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 16
"y": 24
},
"id": 65,
"options": {
@ -6226,7 +6331,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 24
"y": 32
},
"id": 88,
"options": {
@ -6325,7 +6430,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 24
"y": 32
},
"id": 84,
"options": {
@ -6428,7 +6533,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 32
"y": 40
},
"id": 90,
"options": {
@ -6494,7 +6599,7 @@
"h": 2,
"w": 24,
"x": 0,
"y": 17
"y": 25
},
"id": 115,
"options": {
@ -6572,7 +6677,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 19
"y": 27
},
"id": 119,
"options": {
@ -6676,7 +6781,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 19
"y": 27
},
"id": 117,
"options": {
@ -6782,7 +6887,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 27
"y": 35
},
"id": 125,
"links": [
@ -6904,7 +7009,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 27
"y": 35
},
"id": 123,
"options": {
@ -7034,7 +7139,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 43
},
"id": 121,
"options": {

142
dashboards/vm/vmalert.json
View file

@ -1532,7 +1532,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1540,8 +1541,7 @@
}
]
},
"unit": "bytes",
"unitScale": true
"unit": "bytes"
},
"overrides": []
},
@ -2167,6 +2167,113 @@
],
"title": "CPU spent on GC ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"id": 61,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"targets": [
@ -2257,7 +2364,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 36
"y": 44
},
"id": 14,
"options": {
@ -2359,7 +2466,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 36
"y": 44
},
"id": 13,
"options": {
@ -2461,7 +2568,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 44
"y": 52
},
"id": 20,
"options": {
@ -2564,7 +2671,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 44
"y": 52
},
"id": 32,
"options": {
@ -2663,7 +2770,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 52
"y": 60
},
"id": 26,
"options": {
@ -2790,7 +2897,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 51
"y": 59
},
"id": 31,
"options": {
@ -2892,7 +2999,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 51
"y": 59
},
"id": 33,
"options": {
@ -2993,7 +3100,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 59
"y": 67
},
"id": 30,
"options": {
@ -3114,7 +3221,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 17
"y": 25
},
"id": 52,
"options": {
@ -3206,7 +3313,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 17
"y": 25
},
"id": 53,
"options": {
@ -3303,7 +3410,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 44
"y": 52
},
"id": 54,
"options": {
@ -3390,8 +3497,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3407,9 +3513,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 44
"y": 52
},
"id": 55,
"id": 60,
"options": {
"legend": {
"calcs": [

255
dashboards/vmagent.json
View file

@ -1634,7 +1634,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 11
"y": 35
},
"id": 109,
"options": {
@ -1732,7 +1732,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1748,7 +1749,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 11
"y": 35
},
"id": 111,
"options": {
@ -1843,7 +1844,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1872,7 +1874,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 19
"y": 43
},
"id": 81,
"options": {
@ -1978,7 +1980,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2007,7 +2010,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 19
"y": 43
},
"id": 7,
"options": {
@ -2109,7 +2112,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2125,7 +2129,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 27
"y": 51
},
"id": 83,
"options": {
@ -2214,7 +2218,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2230,7 +2235,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 27
"y": 51
},
"id": 39,
"options": {
@ -2320,7 +2325,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2336,7 +2342,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 59
},
"id": 135,
"options": {
@ -2375,6 +2381,113 @@
"title": "CPU spent on GC ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 59
},
"id": 149,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
@ -2425,7 +2538,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2441,7 +2555,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 35
"y": 67
},
"id": 41,
"options": {
@ -2570,7 +2684,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 12
"y": 20
},
"id": 92,
"options": {
@ -2674,7 +2788,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 12
"y": 20
},
"id": 95,
"options": {
@ -2781,7 +2895,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 20
"y": 28
},
"id": 98,
"options": {
@ -2888,7 +3002,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 20
"y": 28
},
"id": 99,
"options": {
@ -2992,7 +3106,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 28
"y": 36
},
"id": 79,
"options": {
@ -3096,7 +3210,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 28
"y": 36
},
"id": 18,
"links": [
@ -3206,7 +3320,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 36
"y": 44
},
"id": 127,
"options": {
@ -3308,7 +3422,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 36
"y": 44
},
"id": 50,
"options": {
@ -3408,7 +3522,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 44
"y": 52
},
"id": 129,
"options": {
@ -3547,8 +3661,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3564,7 +3677,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"id": 48,
"options": {
@ -3654,8 +3767,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3671,7 +3783,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 37
"y": 45
},
"id": 76,
"options": {
@ -3759,8 +3871,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3776,7 +3887,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 44
"y": 52
},
"id": 132,
"options": {
@ -3866,8 +3977,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3883,7 +3993,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 44
"y": 52
},
"id": 133,
"options": {
@ -3972,8 +4082,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3989,7 +4098,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 51
"y": 59
},
"id": 20,
"options": {
@ -4077,8 +4186,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4094,7 +4202,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 51
"y": 59
},
"id": 126,
"options": {
@ -4181,8 +4289,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4198,7 +4305,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 59
"y": 67
},
"id": 46,
"options": {
@ -4285,8 +4392,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4302,7 +4408,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 59
"y": 67
},
"id": 148,
"options": {
@ -4389,8 +4495,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4406,7 +4511,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 67
"y": 75
},
"id": 31,
"options": {
@ -4585,7 +4690,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 14
"y": 22
},
"id": 73,
"options": {
@ -4702,7 +4807,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 6
"y": 14
},
"id": 131,
"options": {
@ -4806,7 +4911,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 14
"y": 22
},
"id": 130,
"options": {
@ -4923,7 +5028,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 14
"y": 22
},
"id": 77,
"options": {
@ -5057,7 +5162,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 7
"y": 15
},
"id": 146,
"options": {
@ -5159,7 +5264,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 7
"y": 15
},
"id": 143,
"options": {
@ -5255,7 +5360,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 15
"y": 23
},
"id": 147,
"options": {
@ -5358,7 +5463,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 15
"y": 23
},
"id": 139,
"options": {
@ -5469,7 +5574,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 23
"y": 31
},
"id": 142,
"options": {
@ -5566,7 +5671,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 23
"y": 31
},
"id": 137,
"options": {
@ -5679,7 +5784,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 31
"y": 39
},
"id": 141,
"options": {
@ -5810,7 +5915,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 8
"y": 16
},
"id": 60,
"options": {
@ -5914,7 +6019,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 8
"y": 16
},
"id": 66,
"options": {
@ -6018,7 +6123,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 16
"y": 24
},
"id": 61,
"options": {
@ -6122,7 +6227,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 16
"y": 24
},
"id": 65,
"options": {
@ -6225,7 +6330,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 24
"y": 32
},
"id": 88,
"options": {
@ -6324,7 +6429,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 24
"y": 32
},
"id": 84,
"options": {
@ -6427,7 +6532,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 32
"y": 40
},
"id": 90,
"options": {
@ -6493,7 +6598,7 @@
"h": 2,
"w": 24,
"x": 0,
"y": 17
"y": 25
},
"id": 115,
"options": {
@ -6571,7 +6676,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 19
"y": 27
},
"id": 119,
"options": {
@ -6675,7 +6780,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 19
"y": 27
},
"id": 117,
"options": {
@ -6781,7 +6886,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 27
"y": 35
},
"id": 125,
"links": [
@ -6903,7 +7008,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 27
"y": 35
},
"id": 123,
"options": {
@ -7033,7 +7138,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 43
},
"id": 121,
"options": {

142
dashboards/vmalert.json
View file

@ -1531,7 +1531,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -1539,8 +1540,7 @@
}
]
},
"unit": "bytes",
"unitScale": true
"unit": "bytes"
},
"overrides": []
},
@ -2166,6 +2166,113 @@
],
"title": "CPU spent on GC ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"decimals": 0,
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"id": 61,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Go scheduling latency",
"type": "timeseries"
}
],
"targets": [
@ -2256,7 +2363,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 36
"y": 44
},
"id": 14,
"options": {
@ -2358,7 +2465,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 36
"y": 44
},
"id": 13,
"options": {
@ -2460,7 +2567,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 44
"y": 52
},
"id": 20,
"options": {
@ -2563,7 +2670,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 44
"y": 52
},
"id": 32,
"options": {
@ -2662,7 +2769,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 52
"y": 60
},
"id": 26,
"options": {
@ -2789,7 +2896,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 51
"y": 59
},
"id": 31,
"options": {
@ -2891,7 +2998,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 51
"y": 59
},
"id": 33,
"options": {
@ -2992,7 +3099,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 59
"y": 67
},
"id": 30,
"options": {
@ -3113,7 +3220,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 17
"y": 25
},
"id": 52,
"options": {
@ -3205,7 +3312,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 17
"y": 25
},
"id": 53,
"options": {
@ -3302,7 +3409,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 44
"y": 52
},
"id": 54,
"options": {
@ -3389,8 +3496,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3406,9 +3512,9 @@
"h": 8,
"w": 12,
"x": 12,
"y": 44
"y": 52
},
"id": 55,
"id": 60,
"options": {
"legend": {
"calcs": [

12
deployment/docker/alerts-health.yml
View file

@ -58,6 +58,18 @@ groups:
Too high CPU usage may be a sign of insufficient resources and make process unstable.
Consider to either increase available CPU resources or decrease the load on the process.
- alert: TooHighGoroutineSchedulingLatency
expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job="vmagent-monitoring-vmagent"}[5m])) by (le, job, instance)) > 0.1
for: 15m
labels:
severity: critical
annotations:
summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m"
description: >
Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of
insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise,
the service could work unreliably with delays in processing.
- alert: TooManyLogs
expr: sum(increase(vm_log_messages_total{level="error"}[5m])) without (app_version, location) > 0
for: 15m

2
docs/changelog/CHANGELOG.md
View file

@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): support parsing `vm_access` claims in string format. This is useful for cases when identity provider does not support mapping claims to JSON format.
* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add new metrics for data ingestion: `vm_rows_received_by_storage_total`, `vm_rows_ignored_total{reason="nan_value"}`, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6663) for details.
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): change request method for `/query_range` and `/query` calls from `GET` to `POST`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6288).
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards) for VM single-node, cluster, vmalert, vmagent, VictoriaLogs: add `Go scheduling latency` panel to show the 99th quantile of Go goroutines scheduling. This panel should help identifying insufficient CPU resources for the service. It is especially useful if CPU gets throttled, which now should be visible on this panel.
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): add alerting rule to track the Go scheduling latency for goroutines. It should notify users if VM component doesn't have enough CPU to run or gets throttled.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink` in `Host:Port` format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6912).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): properly consume messages [from kafka](https://docs.victoriametrics.com/vmagent/#kafka-integration). Previously vmagent could skip some messages during start-up.