dashboards: add Scrape duration 0.99 quantile panel

The new panel will show the 99th quantile of scrape duration in seconds.
This should help identifying vmagent instances that experiences too high scraping durations.

Signed-off-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
hagen1778 2024-07-30 14:57:17 +02:00
parent d09182da11
commit d225a2eb56
No known key found for this signature in database
GPG key ID: 3BF75F3741CA9640
3 changed files with 289 additions and 96 deletions

View file

@ -3548,7 +3548,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3564,7 +3565,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 13
"y": 37
},
"id": 48,
"options": {
@ -3654,7 +3655,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3670,7 +3672,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 13
"y": 37
},
"id": 76,
"options": {
@ -3758,7 +3760,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3774,7 +3777,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 20
"y": 44
},
"id": 132,
"options": {
@ -3864,7 +3867,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3880,7 +3884,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 20
"y": 44
},
"id": 133,
"options": {
@ -3969,7 +3973,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3985,7 +3990,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 27
"y": 51
},
"id": 20,
"options": {
@ -4073,7 +4078,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4089,7 +4095,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 27
"y": 51
},
"id": 126,
"options": {
@ -4176,7 +4182,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4192,7 +4199,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 59
},
"id": 46,
"options": {
@ -4230,6 +4237,110 @@
"title": "Scrape response size 0.99 quantile ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 59
},
"id": 148,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "victoriametrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)",
"format": "time_series",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Scrape duration 0.99 quantile ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-datasource",
@ -4279,7 +4390,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4295,7 +4407,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 35
"y": 67
},
"id": 31,
"options": {
@ -4575,8 +4687,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4680,8 +4791,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4798,8 +4908,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4934,8 +5043,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5037,8 +5145,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5134,8 +5241,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5238,8 +5344,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5349,8 +5454,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5447,8 +5551,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5545,8 +5648,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5693,8 +5795,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5798,8 +5899,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5903,8 +6003,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6008,8 +6107,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6112,8 +6210,7 @@
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
"color": "transparent"
},
{
"color": "red",
@ -6315,8 +6412,7 @@
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
"color": "transparent"
},
{
"color": "red",

View file

@ -3547,7 +3547,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3563,7 +3564,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 13
"y": 37
},
"id": 48,
"options": {
@ -3653,7 +3654,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3669,7 +3671,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 13
"y": 37
},
"id": 76,
"options": {
@ -3757,7 +3759,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3773,7 +3776,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 20
"y": 44
},
"id": 132,
"options": {
@ -3863,7 +3866,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3879,7 +3883,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 20
"y": 44
},
"id": 133,
"options": {
@ -3968,7 +3972,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3984,7 +3989,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 27
"y": 51
},
"id": 20,
"options": {
@ -4072,7 +4077,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4088,7 +4094,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 27
"y": 51
},
"id": 126,
"options": {
@ -4175,7 +4181,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4191,7 +4198,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 35
"y": 59
},
"id": 46,
"options": {
@ -4229,6 +4236,110 @@
"title": "Scrape response size 0.99 quantile ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 59
},
"id": 148,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.2.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)",
"format": "time_series",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Scrape duration 0.99 quantile ($instance)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
@ -4278,7 +4389,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -4294,7 +4406,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 35
"y": 67
},
"id": 31,
"options": {
@ -4574,8 +4686,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4679,8 +4790,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4797,8 +4907,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -4933,8 +5042,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5036,8 +5144,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5133,8 +5240,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5237,8 +5343,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5348,8 +5453,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5446,8 +5550,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5544,8 +5647,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5692,8 +5794,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5797,8 +5898,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -5902,8 +6002,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6007,8 +6106,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -6111,8 +6209,7 @@
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
"color": "transparent"
},
{
"color": "red",
@ -6314,8 +6411,7 @@
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
"color": "transparent"
},
{
"color": "red",

View file

@ -35,6 +35,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* FEATURE: [vmauth](./vmauth.md): add `keep_original_host` option, which can be used for proxying the original `Host` header from client request to the backend. By default the backend host is used as `Host` header when proxying requests to the configured backends. See [these docs](./vmauth.md#host-http-header).
* FEATURE: [vmauth](./vmauth.md) now returns HTTP 502 status code when all upstream backends are not available. Previously, it returned HTTP 503 status code. This change aligns vmauth behavior with other well-known reverse-proxies behavior.
* FEATURE: [vmagent dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmagent.json): add `Scrape duration 0.99 quantile` panel to show the 99th quantile of scrape duration in seconds. This should help identifying vmagent instances that experiences too high scraping durations.
* BUGFIX: all VictoriaMetrics components: validate files specified via `-tlsKeyFile` and `-tlsCertFile` cmd-line flags on the process start-up. Previously, validation happened on the first connection accepted by HTTP server. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6608) for the details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6621).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly proxy requests to backend urls ending with `/` if the original request path equals to `/`. Previously the trailing `/` at the backend path was incorrectly removed. For example, if the request to `http://vmauth/` is configured to be proxied to `url_prefix=http://backend/foo/`, then it was proxied to `http://backend/foo`, while it should go to `http://backend/foo/`.