From ac411be904a4a5f3467d73e7f4edac6812982751 Mon Sep 17 00:00:00 2001 From: Loki's Wager <32408858+LokiWager@users.noreply.github.com> Date: Tue, 21 Jun 2022 20:56:41 +0800 Subject: [PATCH 1/6] BugFix part_header.go (#2763) https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2757 Co-authored-by: haotingyi --- lib/mergeset/part_header.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/mergeset/part_header.go b/lib/mergeset/part_header.go index de9a23866..a4d7007f7 100644 --- a/lib/mergeset/part_header.go +++ b/lib/mergeset/part_header.go @@ -136,7 +136,6 @@ func (ph *partHeader) ParseFromPath(partPath string) error { if ph.itemsCount != phj.ItemsCount { return fmt.Errorf("invalid ItemsCount in %q; got %d; want %d", metadataPath, phj.ItemsCount, ph.itemsCount) } - ph.blocksCount = phj.BlocksCount if ph.blocksCount != phj.BlocksCount { return fmt.Errorf("invalid BlocksCount in %q; got %d; want %d", metadataPath, phj.BlocksCount, ph.blocksCount) } From f456e486b7c2cc847ddc5b4c6673df312f27f816 Mon Sep 17 00:00:00 2001 From: Denys Holius <5650611+denisgolius@users.noreply.github.com> Date: Tue, 21 Jun 2022 17:20:08 +0300 Subject: [PATCH 2/6] url-examples: added curl output after deleting metrics (#2764) docs: add more details to url-examples for series deleting --- docs/url-examples.md | 48 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/url-examples.md b/docs/url-examples.md index f6350fc52..84fde9a78 100644 --- a/docs/url-examples.md +++ b/docs/url-examples.md @@ -12,7 +12,29 @@ Single:
```console -curl 'http://:8428/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +curl -v 'http://:8428/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +``` + +
+ +The expected output should return [HTTP Status 204](https://datatracker.ietf.org/doc/html/rfc7231#page-53) and will look like: + +
+ +```console +* Trying 127.0.0.1:8428... +* Connected to 127.0.0.1 (127.0.0.1) port 8428 (#0) +> GET /api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total HTTP/1.1 +> Host: 127.0.0.1:8428 +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 204 No Content +< X-Server-Hostname: eba075fb0e1a +< Date: Tue, 21 Jun 2022 07:33:35 GMT +< +* Connection #0 to host 127.0.0.1 left intact ```
@@ -21,7 +43,29 @@ Cluster:
```console -curl 'http://:8481/delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +curl -v'http://:8481/delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +``` + +
+ +The expected output should return [HTTP Status 204](https://datatracker.ietf.org/doc/html/rfc7231#page-53) and will look like: + +
+ +```console +* Trying 127.0.0.1:8481... +* Connected to 127.0.0.1 (127.0.0.1) port 8481 (#0) +> GET /delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total HTTP/1.1 +> Host: 127.0.0.1:8481 +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 204 No Content +< X-Server-Hostname: 101ed7a45c94 +< Date: Tue, 21 Jun 2022 07:21:36 GMT +< +* Connection #0 to host 127.0.0.1 left intact ```
From e6ed92529b7bdda80f1fc0028da4120ca1a9803b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 21 Jun 2022 20:23:30 +0300 Subject: [PATCH 3/6] all: remove explicit "xxhash" name when importing github.com/cespare/xxhash/v2 package This package already has the same name, so there is no need in explicit name --- app/vmagent/remotewrite/remotewrite.go | 2 +- app/vmselect/promql/aggr.go | 2 +- lib/blockcache/blockcache.go | 2 +- lib/lrucache/lrucache.go | 2 +- lib/promauth/config.go | 2 +- lib/promrelabel/relabel.go | 2 +- lib/promscrape/config.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index e810bd2ed..ff77572df 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -22,7 +22,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) var ( diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index c4927a16c..353c0226b 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -11,7 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" "github.com/VictoriaMetrics/metricsql" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) var aggrFuncs = map[string]aggrFunc{ diff --git a/lib/blockcache/blockcache.go b/lib/blockcache/blockcache.go index b4911ebef..ce2aaab0e 100644 --- a/lib/blockcache/blockcache.go +++ b/lib/blockcache/blockcache.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) // Cache caches Block entries. diff --git a/lib/lrucache/lrucache.go b/lib/lrucache/lrucache.go index c52e20850..0c5acbcff 100644 --- a/lib/lrucache/lrucache.go +++ b/lib/lrucache/lrucache.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) // Cache caches Entry entries. diff --git a/lib/promauth/config.go b/lib/promauth/config.go index 5f48bf66f..3f7ee1178 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -15,7 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" ) diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 5fa938104..d9c2852d7 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) // parsedRelabelConfig contains parsed `relabel_config`. diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index acba6d7b5..92be6081d 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -35,7 +35,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" "github.com/VictoriaMetrics/metrics" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" "gopkg.in/yaml.v2" ) From 197d3cdd7488dfff1f1030409bec9ed91fc602e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91=E5=8E=9F=E7=94=9F=E9=A9=BF=E7=AB=99?= <33771248+CloudCourierStation@users.noreply.github.com> Date: Wed, 22 Jun 2022 13:43:41 +0800 Subject: [PATCH 4/6] docs: supplement vmalert downsampling docs (#2765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 吴典秋 --- app/vmalert/README.md | 12 ++++++++++++ docs/vmalert.md | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index d6ab0de9f..1e637a70c 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -417,6 +417,18 @@ or reducing resolution) and push results to "cold" cluster. -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus # vminsert addr to persist recording rules results ``` +`vmalert` passes the configuration [recording rules](https://docs.victoriametrics.com/vmalert.html#recording-rules). These rules execute arbitrary MetricsQL expression and write the received result back to remote write destination. + +For example, for `downsampling` of the total number of requests within five minutes, the configuration file can be as follows + +``` +groups: + - name: downsampling + rules: + - record: requests:avg5m + expr: avg_over_time(my_requests_total[5m]) +``` + vmalert multi cluster Please note, [replay](#rules-backfilling) feature may be used for transforming historical data. diff --git a/docs/vmalert.md b/docs/vmalert.md index 7a562e27c..6f0196e2e 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -421,6 +421,18 @@ or reducing resolution) and push results to "cold" cluster. -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus # vminsert addr to persist recording rules results ``` +`vmalert` passes the configuration [recording rules](https://docs.victoriametrics.com/vmalert.html#recording-rules). These rules execute arbitrary MetricsQL expression and write the received result back to remote write destination. + +For example, for `downsampling` of the total number of requests within five minutes, the configuration file can be as follows + +``` +groups: + - name: downsampling + rules: + - record: requests:avg5m + expr: avg_over_time(my_requests_total[5m]) +``` + vmalert multi cluster Please note, [replay](#rules-backfilling) feature may be used for transforming historical data. From 75dd7542e57ebabbfcb146812d5e6e9a0d83572a Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Wed, 22 Jun 2022 08:53:54 +0200 Subject: [PATCH 5/6] docs: follow-up for 197d3cdd7488dfff1f1030409bec9ed91fc602e0 (#2766) Signed-off-by: hagen1778 --- app/vmalert/README.md | 42 ++++++++++++++++++++++++++++++------------ docs/vmalert.md | 42 ++++++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 1e637a70c..52e2a6d0f 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -402,6 +402,36 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) #### Downsampling and aggregation via vmalert +`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries +via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`. +This ability allows to aggregate data. For example, the following rule will calculate the average value for +metric `http_requests` on the `5m` interval: + +```yaml + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Every time this rule will be evaluated, `vmalert` will backfill its results as a new time series `http_requests:avg5m` +to the configured `-remoteWrite.url`. + +`vmalert` executes rules with specified interval (configured via flag `-evaluationInterval` +or as [group's](#groups) `interval` param). The interval helps to control "resolution" of the produced series. +This ability allows to downsample data. For example, the following config will execute the rule only once every `5m`: + +```yaml +groups: + - name: my_group + interval: 5m + rules: + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows +reading data from one data source and backfilling results to another. This helps to build a system +for aggregating and downsampling the data. + The following example shows how to build a topology where `vmalert` will process data from one cluster and write results into another. Such clusters may be called as "hot" (low retention, high-speed disks, used for operative monitoring) and "cold" (long term retention, @@ -417,18 +447,6 @@ or reducing resolution) and push results to "cold" cluster. -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus # vminsert addr to persist recording rules results ``` -`vmalert` passes the configuration [recording rules](https://docs.victoriametrics.com/vmalert.html#recording-rules). These rules execute arbitrary MetricsQL expression and write the received result back to remote write destination. - -For example, for `downsampling` of the total number of requests within five minutes, the configuration file can be as follows - -``` -groups: - - name: downsampling - rules: - - record: requests:avg5m - expr: avg_over_time(my_requests_total[5m]) -``` - vmalert multi cluster Please note, [replay](#rules-backfilling) feature may be used for transforming historical data. diff --git a/docs/vmalert.md b/docs/vmalert.md index 6f0196e2e..75166636d 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -406,6 +406,36 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) #### Downsampling and aggregation via vmalert +`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries +via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`. +This ability allows to aggregate data. For example, the following rule will calculate the average value for +metric `http_requests` on the `5m` interval: + +```yaml + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Every time this rule will be evaluated, `vmalert` will backfill its results as a new time series `http_requests:avg5m` +to the configured `-remoteWrite.url`. + +`vmalert` executes rules with specified interval (configured via flag `-evaluationInterval` +or as [group's](#groups) `interval` param). The interval helps to control "resolution" of the produced series. +This ability allows to downsample data. For example, the following config will execute the rule only once every `5m`: + +```yaml +groups: + - name: my_group + interval: 5m + rules: + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows +reading data from one data source and backfilling results to another. This helps to build a system +for aggregating and downsampling the data. + The following example shows how to build a topology where `vmalert` will process data from one cluster and write results into another. Such clusters may be called as "hot" (low retention, high-speed disks, used for operative monitoring) and "cold" (long term retention, @@ -421,18 +451,6 @@ or reducing resolution) and push results to "cold" cluster. -remoteWrite.url=http://aggregated-cluster-vminsert:8480/insert/0/prometheus # vminsert addr to persist recording rules results ``` -`vmalert` passes the configuration [recording rules](https://docs.victoriametrics.com/vmalert.html#recording-rules). These rules execute arbitrary MetricsQL expression and write the received result back to remote write destination. - -For example, for `downsampling` of the total number of requests within five minutes, the configuration file can be as follows - -``` -groups: - - name: downsampling - rules: - - record: requests:avg5m - expr: avg_over_time(my_requests_total[5m]) -``` - vmalert multi cluster Please note, [replay](#rules-backfilling) feature may be used for transforming historical data. From 7bf75c7e61b7bb29f9ed5ae4bd387e4c563ff4c5 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 22 Jun 2022 13:14:47 +0300 Subject: [PATCH 6/6] app/vmselect: typo fix in the exported metric name: vm_http_request_total -> vm_http_requests_total --- app/vmselect/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 498f2e402..8bc0ff2e6 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -549,7 +549,7 @@ var ( graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`) graphiteTagsDelSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`) - graphiteFunctionsRequests = metrics.NewCounter(`vm_http_request_total{path="/functions"}`) + graphiteFunctionsRequests = metrics.NewCounter(`vm_http_requests_total{path="/functions"}`) rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)