diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index e810bd2ed..ff77572df 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -22,7 +22,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) var ( diff --git a/app/vmalert/README.md b/app/vmalert/README.md index d6ab0de9f..52e2a6d0f 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -402,6 +402,36 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) #### Downsampling and aggregation via vmalert +`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries +via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`. +This ability allows to aggregate data. For example, the following rule will calculate the average value for +metric `http_requests` on the `5m` interval: + +```yaml + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Every time this rule will be evaluated, `vmalert` will backfill its results as a new time series `http_requests:avg5m` +to the configured `-remoteWrite.url`. + +`vmalert` executes rules with specified interval (configured via flag `-evaluationInterval` +or as [group's](#groups) `interval` param). The interval helps to control "resolution" of the produced series. +This ability allows to downsample data. For example, the following config will execute the rule only once every `5m`: + +```yaml +groups: + - name: my_group + interval: 5m + rules: + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows +reading data from one data source and backfilling results to another. This helps to build a system +for aggregating and downsampling the data. + The following example shows how to build a topology where `vmalert` will process data from one cluster and write results into another. Such clusters may be called as "hot" (low retention, high-speed disks, used for operative monitoring) and "cold" (long term retention, diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 498f2e402..8bc0ff2e6 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -549,7 +549,7 @@ var ( graphiteTagsDelSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/delSeries"}`) graphiteTagsDelSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/delSeries"}`) - graphiteFunctionsRequests = metrics.NewCounter(`vm_http_request_total{path="/functions"}`) + graphiteFunctionsRequests = metrics.NewCounter(`vm_http_requests_total{path="/functions"}`) rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`) diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index c4927a16c..353c0226b 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -11,7 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" "github.com/VictoriaMetrics/metricsql" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) var aggrFuncs = map[string]aggrFunc{ diff --git a/docs/url-examples.md b/docs/url-examples.md index f6350fc52..84fde9a78 100644 --- a/docs/url-examples.md +++ b/docs/url-examples.md @@ -12,7 +12,29 @@ Single:
```console -curl 'http://:8428/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +curl -v 'http://:8428/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +``` + +
+ +The expected output should return [HTTP Status 204](https://datatracker.ietf.org/doc/html/rfc7231#page-53) and will look like: + +
+ +```console +* Trying 127.0.0.1:8428... +* Connected to 127.0.0.1 (127.0.0.1) port 8428 (#0) +> GET /api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total HTTP/1.1 +> Host: 127.0.0.1:8428 +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 204 No Content +< X-Server-Hostname: eba075fb0e1a +< Date: Tue, 21 Jun 2022 07:33:35 GMT +< +* Connection #0 to host 127.0.0.1 left intact ```
@@ -21,7 +43,29 @@ Cluster:
```console -curl 'http://:8481/delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +curl -v'http://:8481/delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total' +``` + +
+ +The expected output should return [HTTP Status 204](https://datatracker.ietf.org/doc/html/rfc7231#page-53) and will look like: + +
+ +```console +* Trying 127.0.0.1:8481... +* Connected to 127.0.0.1 (127.0.0.1) port 8481 (#0) +> GET /delete/0/prometheus/api/v1/admin/tsdb/delete_series?match[]=vm_http_request_errors_total HTTP/1.1 +> Host: 127.0.0.1:8481 +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 204 No Content +< X-Server-Hostname: 101ed7a45c94 +< Date: Tue, 21 Jun 2022 07:21:36 GMT +< +* Connection #0 to host 127.0.0.1 left intact ```
diff --git a/docs/vmalert.md b/docs/vmalert.md index 7a562e27c..75166636d 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -406,6 +406,36 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) #### Downsampling and aggregation via vmalert +`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries +via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`. +This ability allows to aggregate data. For example, the following rule will calculate the average value for +metric `http_requests` on the `5m` interval: + +```yaml + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Every time this rule will be evaluated, `vmalert` will backfill its results as a new time series `http_requests:avg5m` +to the configured `-remoteWrite.url`. + +`vmalert` executes rules with specified interval (configured via flag `-evaluationInterval` +or as [group's](#groups) `interval` param). The interval helps to control "resolution" of the produced series. +This ability allows to downsample data. For example, the following config will execute the rule only once every `5m`: + +```yaml +groups: + - name: my_group + interval: 5m + rules: + - record: http_requests:avg5m + expr: avg_over_time(http_requests[5m]) +``` + +Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows +reading data from one data source and backfilling results to another. This helps to build a system +for aggregating and downsampling the data. + The following example shows how to build a topology where `vmalert` will process data from one cluster and write results into another. Such clusters may be called as "hot" (low retention, high-speed disks, used for operative monitoring) and "cold" (long term retention, diff --git a/lib/blockcache/blockcache.go b/lib/blockcache/blockcache.go index b4911ebef..ce2aaab0e 100644 --- a/lib/blockcache/blockcache.go +++ b/lib/blockcache/blockcache.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) // Cache caches Block entries. diff --git a/lib/lrucache/lrucache.go b/lib/lrucache/lrucache.go index c52e20850..0c5acbcff 100644 --- a/lib/lrucache/lrucache.go +++ b/lib/lrucache/lrucache.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) // Cache caches Entry entries. diff --git a/lib/mergeset/part_header.go b/lib/mergeset/part_header.go index de9a23866..a4d7007f7 100644 --- a/lib/mergeset/part_header.go +++ b/lib/mergeset/part_header.go @@ -136,7 +136,6 @@ func (ph *partHeader) ParseFromPath(partPath string) error { if ph.itemsCount != phj.ItemsCount { return fmt.Errorf("invalid ItemsCount in %q; got %d; want %d", metadataPath, phj.ItemsCount, ph.itemsCount) } - ph.blocksCount = phj.BlocksCount if ph.blocksCount != phj.BlocksCount { return fmt.Errorf("invalid BlocksCount in %q; got %d; want %d", metadataPath, phj.BlocksCount, ph.blocksCount) } diff --git a/lib/promauth/config.go b/lib/promauth/config.go index 5f48bf66f..3f7ee1178 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -15,7 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" ) diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 5fa938104..d9c2852d7 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" ) // parsedRelabelConfig contains parsed `relabel_config`. diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index acba6d7b5..92be6081d 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -35,7 +35,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" "github.com/VictoriaMetrics/metrics" - xxhash "github.com/cespare/xxhash/v2" + "github.com/cespare/xxhash/v2" "gopkg.in/yaml.v2" )