From d225a2eb5682afdd637c4318f39e1eb95242bb3e Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Tue, 30 Jul 2024 14:57:17 +0200 Subject: [PATCH] dashboards: add `Scrape duration 0.99 quantile` panel The new panel will show the 99th quantile of scrape duration in seconds. This should help identifying vmagent instances that experiences too high scraping durations. Signed-off-by: hagen1778 --- dashboards/vm/vmagent.json | 192 +++++++++++++++++++++++++++---------- dashboards/vmagent.json | 192 +++++++++++++++++++++++++++---------- docs/CHANGELOG.md | 1 + 3 files changed, 289 insertions(+), 96 deletions(-) diff --git a/dashboards/vm/vmagent.json b/dashboards/vm/vmagent.json index 9afcc9109..c64192512 100644 --- a/dashboards/vm/vmagent.json +++ b/dashboards/vm/vmagent.json @@ -3548,7 +3548,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3564,7 +3565,7 @@ "h": 7, "w": 12, "x": 0, - "y": 13 + "y": 37 }, "id": 48, "options": { @@ -3654,7 +3655,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3670,7 +3672,7 @@ "h": 7, "w": 12, "x": 12, - "y": 13 + "y": 37 }, "id": 76, "options": { @@ -3758,7 +3760,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3774,7 +3777,7 @@ "h": 7, "w": 12, "x": 0, - "y": 20 + "y": 44 }, "id": 132, "options": { @@ -3864,7 +3867,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3880,7 +3884,7 @@ "h": 7, "w": 12, "x": 12, - "y": 20 + "y": 44 }, "id": 133, "options": { @@ -3969,7 +3973,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3985,7 +3990,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 51 }, "id": 20, "options": { @@ -4073,7 +4078,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4089,7 +4095,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 51 }, "id": 126, "options": { @@ -4176,7 +4182,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4192,7 +4199,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 59 }, "id": 46, "options": { @@ -4230,6 +4237,110 @@ "title": "Scrape response size 0.99 quantile ($instance)", "type": "timeseries" }, + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "victoriametrics-datasource", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", + "format": "time_series", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape duration 0.99 quantile ($instance)", + "type": "timeseries" + }, { "datasource": { "type": "victoriametrics-datasource", @@ -4279,7 +4390,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4295,7 +4407,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 67 }, "id": 31, "options": { @@ -4575,8 +4687,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4680,8 +4791,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4798,8 +4908,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4934,8 +5043,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5037,8 +5145,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5134,8 +5241,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5238,8 +5344,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5349,8 +5454,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5447,8 +5551,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5545,8 +5648,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5693,8 +5795,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5798,8 +5899,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5903,8 +6003,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6008,8 +6107,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6112,8 +6210,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -6315,8 +6412,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index 557cdc55b..0debca079 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -3547,7 +3547,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3563,7 +3564,7 @@ "h": 7, "w": 12, "x": 0, - "y": 13 + "y": 37 }, "id": 48, "options": { @@ -3653,7 +3654,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3669,7 +3671,7 @@ "h": 7, "w": 12, "x": 12, - "y": 13 + "y": 37 }, "id": 76, "options": { @@ -3757,7 +3759,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3773,7 +3776,7 @@ "h": 7, "w": 12, "x": 0, - "y": 20 + "y": 44 }, "id": 132, "options": { @@ -3863,7 +3866,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3879,7 +3883,7 @@ "h": 7, "w": 12, "x": 12, - "y": 20 + "y": 44 }, "id": 133, "options": { @@ -3968,7 +3972,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3984,7 +3989,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 51 }, "id": 20, "options": { @@ -4072,7 +4077,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4088,7 +4094,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 51 }, "id": 126, "options": { @@ -4175,7 +4181,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4191,7 +4198,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 59 }, "id": 46, "options": { @@ -4229,6 +4236,110 @@ "title": "Scrape response size 0.99 quantile ($instance)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)", + "format": "time_series", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Scrape duration 0.99 quantile ($instance)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -4278,7 +4389,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4294,7 +4406,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 67 }, "id": 31, "options": { @@ -4574,8 +4686,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4679,8 +4790,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4797,8 +4907,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4933,8 +5042,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5036,8 +5144,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5133,8 +5240,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5237,8 +5343,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5348,8 +5453,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5446,8 +5550,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5544,8 +5647,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5692,8 +5794,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5797,8 +5898,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5902,8 +6002,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6007,8 +6106,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6111,8 +6209,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -6314,8 +6411,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c3c33cd1a..6afaa6899 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmauth](./vmauth.md): add `keep_original_host` option, which can be used for proxying the original `Host` header from client request to the backend. By default the backend host is used as `Host` header when proxying requests to the configured backends. See [these docs](./vmauth.md#host-http-header). * FEATURE: [vmauth](./vmauth.md) now returns HTTP 502 status code when all upstream backends are not available. Previously, it returned HTTP 503 status code. This change aligns vmauth behavior with other well-known reverse-proxies behavior. +* FEATURE: [vmagent dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmagent.json): add `Scrape duration 0.99 quantile` panel to show the 99th quantile of scrape duration in seconds. This should help identifying vmagent instances that experiences too high scraping durations. * BUGFIX: all VictoriaMetrics components: validate files specified via `-tlsKeyFile` and `-tlsCertFile` cmd-line flags on the process start-up. Previously, validation happened on the first connection accepted by HTTP server. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6608) for the details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6621). * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly proxy requests to backend urls ending with `/` if the original request path equals to `/`. Previously the trailing `/` at the backend path was incorrectly removed. For example, if the request to `http://vmauth/` is configured to be proxied to `url_prefix=http://backend/foo/`, then it was proxied to `http://backend/foo`, while it should go to `http://backend/foo/`.