From 2951dd0a570477bff620c68f7631c345033da222 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 17 Sep 2021 13:31:48 +0300 Subject: [PATCH] app/vmselect/promql: add histogram_quantiles("phiLabel", phi1, ..., phiN, buckets) function This function calculates multiple quantiles over the given buckets at once Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1573 --- app/vmselect/promql/exec_test.go | 41 +++++ app/vmselect/promql/transform.go | 144 +++++++++++------- docs/CHANGELOG.md | 3 +- docs/MetricsQL.md | 8 +- go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/transform.go | 107 ++++++------- vendor/modules.txt | 2 +- 8 files changed, 198 insertions(+), 113 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 8c9f22cdd..8c7081635 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -3428,6 +3428,46 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`histogram_quantiles()`, func(t *testing.T) { + t.Parallel() + q := `sort_by_label(histogram_quantiles("phi", 0.2, 0.3, + label_set(0, "foo", "bar", "le", "10") + or label_set(100, "foo", "bar", "le", "30") + or label_set(300, "foo", "bar", "le", "+Inf") + ), "phi")` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{22, 22, 22, 22, 22, 22}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{ + { + Key: []byte("foo"), + Value: []byte("bar"), + }, + { + Key: []byte("phi"), + Value: []byte("0.2"), + }, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{28, 28, 28, 28, 28, 28}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{ + { + Key: []byte("foo"), + Value: []byte("bar"), + }, + { + Key: []byte("phi"), + Value: []byte("0.3"), + }, + } + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`histogram_share(normal-bucket-count)`, func(t *testing.T) { t.Parallel() q := `histogram_share(35, @@ -6980,6 +7020,7 @@ func TestExecError(t *testing.T) { f(`timestamp()`) f(`vector()`) f(`histogram_quantile()`) + f(`histogram_quantiles()`) f(`sum()`) f(`count_values()`) f(`quantile()`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index ddf138285..cbebf9bd5 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -74,59 +74,60 @@ var transformFuncs = map[string]transformFunc{ "year": newTransformFuncDateTime(transformYear), // New funcs - "label_set": transformLabelSet, - "label_map": transformLabelMap, - "label_uppercase": transformLabelUppercase, - "label_lowercase": transformLabelLowercase, - "label_del": transformLabelDel, - "label_keep": transformLabelKeep, - "label_copy": transformLabelCopy, - "label_move": transformLabelMove, - "label_transform": transformLabelTransform, - "label_value": transformLabelValue, - "label_match": transformLabelMatch, - "label_mismatch": transformLabelMismatch, - "union": transformUnion, - "": transformUnion, // empty func is a synonym to union - "keep_last_value": transformKeepLastValue, - "keep_next_value": transformKeepNextValue, - "interpolate": transformInterpolate, - "start": newTransformFuncZeroArgs(transformStart), - "end": newTransformFuncZeroArgs(transformEnd), - "step": newTransformFuncZeroArgs(transformStep), - "running_sum": newTransformFuncRunning(runningSum), - "running_max": newTransformFuncRunning(runningMax), - "running_min": newTransformFuncRunning(runningMin), - "running_avg": newTransformFuncRunning(runningAvg), - "range_sum": newTransformFuncRange(runningSum), - "range_max": newTransformFuncRange(runningMax), - "range_min": newTransformFuncRange(runningMin), - "range_avg": newTransformFuncRange(runningAvg), - "range_first": transformRangeFirst, - "range_last": transformRangeLast, - "range_quantile": transformRangeQuantile, - "smooth_exponential": transformSmoothExponential, - "remove_resets": transformRemoveResets, - "rand": newTransformRand(newRandFloat64), - "rand_normal": newTransformRand(newRandNormFloat64), - "rand_exponential": newTransformRand(newRandExpFloat64), - "pi": transformPi, - "sin": newTransformFuncOneArg(transformSin), - "cos": newTransformFuncOneArg(transformCos), - "asin": newTransformFuncOneArg(transformAsin), - "acos": newTransformFuncOneArg(transformAcos), - "prometheus_buckets": transformPrometheusBuckets, - "buckets_limit": transformBucketsLimit, - "histogram_share": transformHistogramShare, - "histogram_avg": transformHistogramAvg, - "histogram_stdvar": transformHistogramStdvar, - "histogram_stddev": transformHistogramStddev, - "sort_by_label": newTransformFuncSortByLabel(false), - "sort_by_label_desc": newTransformFuncSortByLabel(true), - "timezone_offset": transformTimezoneOffset, - "bitmap_and": newTransformBitmap(bitmapAnd), - "bitmap_or": newTransformBitmap(bitmapOr), - "bitmap_xor": newTransformBitmap(bitmapXor), + "label_set": transformLabelSet, + "label_map": transformLabelMap, + "label_uppercase": transformLabelUppercase, + "label_lowercase": transformLabelLowercase, + "label_del": transformLabelDel, + "label_keep": transformLabelKeep, + "label_copy": transformLabelCopy, + "label_move": transformLabelMove, + "label_transform": transformLabelTransform, + "label_value": transformLabelValue, + "label_match": transformLabelMatch, + "label_mismatch": transformLabelMismatch, + "union": transformUnion, + "": transformUnion, // empty func is a synonym to union + "keep_last_value": transformKeepLastValue, + "keep_next_value": transformKeepNextValue, + "interpolate": transformInterpolate, + "start": newTransformFuncZeroArgs(transformStart), + "end": newTransformFuncZeroArgs(transformEnd), + "step": newTransformFuncZeroArgs(transformStep), + "running_sum": newTransformFuncRunning(runningSum), + "running_max": newTransformFuncRunning(runningMax), + "running_min": newTransformFuncRunning(runningMin), + "running_avg": newTransformFuncRunning(runningAvg), + "range_sum": newTransformFuncRange(runningSum), + "range_max": newTransformFuncRange(runningMax), + "range_min": newTransformFuncRange(runningMin), + "range_avg": newTransformFuncRange(runningAvg), + "range_first": transformRangeFirst, + "range_last": transformRangeLast, + "range_quantile": transformRangeQuantile, + "smooth_exponential": transformSmoothExponential, + "remove_resets": transformRemoveResets, + "rand": newTransformRand(newRandFloat64), + "rand_normal": newTransformRand(newRandNormFloat64), + "rand_exponential": newTransformRand(newRandExpFloat64), + "pi": transformPi, + "sin": newTransformFuncOneArg(transformSin), + "cos": newTransformFuncOneArg(transformCos), + "asin": newTransformFuncOneArg(transformAsin), + "acos": newTransformFuncOneArg(transformAcos), + "prometheus_buckets": transformPrometheusBuckets, + "buckets_limit": transformBucketsLimit, + "histogram_share": transformHistogramShare, + "histogram_avg": transformHistogramAvg, + "histogram_stdvar": transformHistogramStdvar, + "histogram_stddev": transformHistogramStddev, + "sort_by_label": newTransformFuncSortByLabel(false), + "sort_by_label_desc": newTransformFuncSortByLabel(true), + "timezone_offset": transformTimezoneOffset, + "bitmap_and": newTransformBitmap(bitmapAnd), + "bitmap_or": newTransformBitmap(bitmapOr), + "bitmap_xor": newTransformBitmap(bitmapXor), + "histogram_quantiles": transformHistogramQuantiles, } func getTransformFunc(s string) transformFunc { @@ -789,6 +790,43 @@ func stdvarForLeTimeseries(i int, xss []leTimeseries) float64 { return stdvar } +func transformHistogramQuantiles(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if len(args) < 3 { + return nil, fmt.Errorf("unexpected number of args: %d; expecting at least 3 args", len(args)) + } + dstLabel, err := getString(args[0], 0) + if err != nil { + return nil, fmt.Errorf("cannot obtain dstLabel: %w", err) + } + phiArgs := args[1 : len(args)-1] + tssOrig := args[len(args)-1] + // Calculate quantile individually per each phi. + var rvs []*timeseries + for _, phiArg := range phiArgs { + phiStr := fmt.Sprintf("%g", phiArg[0].Values[0]) + tss := copyTimeseries(tssOrig) + tfaTmp := &transformFuncArg{ + ec: tfa.ec, + fe: tfa.fe, + args: [][]*timeseries{ + phiArg, + tss, + }, + } + tssTmp, err := transformHistogramQuantile(tfaTmp) + if err != nil { + return nil, fmt.Errorf("cannot calculate quantile %s: %w", phiStr, err) + } + for _, ts := range tssTmp { + ts.MetricName.RemoveTag(dstLabel) + ts.MetricName.AddTag(dstLabel, phiStr) + } + rvs = append(rvs, tssTmp...) + } + return rvs, nil +} + func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if len(args) < 2 || len(args) > 3 { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9173e406a..547a07759 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -26,6 +26,7 @@ sort: 15 * FEATURE: vminsert: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8400,vmstorage2:8400`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup). * FEATURE: add [mad(q)](https://docs.victoriametrics.com/MetricsQL.html#mad) function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). It calculates [Median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) for groups of points with identical timestamps across multiple time series. * FEATURE: add [outliers_mad(tolerance, q)](https://docs.victoriametrics.com/MetricsQL.html#outliers_mad) function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). It returns time series with peaks outside the [Median absolute deviation](https://en.wikipedia.org/wiki/Median_absolute_deviation) multiplied by `tolerance`. +* FEATURE: add `histogram_quantiles("phiLabel", phi1, ..., phiN, buckets)` function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). It calculates the given `phi*`-quantiles over the given `buckets` and returns time series per each quantile with the corresponding `{phiLabel="phi*"}` label. * FEATURE: [enterprise](https://victoriametrics.com/enterprise.html): do not ask for `-eula` flag if `-version` flag is passed to enteprise app. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1621). * BUGFIX: properly handle queries with multiple filters matching empty labels such as `metric{label1=~"foo|",label2="bar|"}`. This filter must match the following series: `metric`, `metric{label1="foo"}`, `metric{label2="bar"}` and `metric{label1="foo",label2="bar"}`. Previously it was matching only `metric{label1="foo",label2="bar"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1601). @@ -50,7 +51,7 @@ sort: 15 * FEATURE: update Go builder from v1.16.7 to v1.17.0. This improves data ingestion and query performance by up to 5% according to benchmarks. See [the release post for Go1.17](https://go.dev/blog/go1.17). * FEATURE: vmagent: expose `promscrape_discovery_http_errors_total` metric, which can be used for monitoring the number of failed discovery attempts per each `http_sd` config. * FEATURE: do not reset response cache when a sample with old timestamp is ingested into VictoriaMetrics if `-search.disableAutoCacheReset` command-line option is set. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1570). -* FEATURE: add `quantiles("quantileLabel", phi1, ..., phiN, q)` aggregate function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which calculates the given `phi*` quantiles over time series returned by `q`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1573). +* FEATURE: add `quantiles("phiLabel", phi1, ..., phiN, q)` aggregate function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which calculates the given `phi*` quantiles over time series returned by `q`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1573). * BUGFIX: rename `sign` function to `sgn` in order to be consistent with PromQL. See [this pull request from Prometheus](https://github.com/prometheus/prometheus/pull/8457). * BUGFIX: vmagent: add `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) in order to be consistent with Prometheus. Previously this role was supported with incorrect name: `role: endpointslices`. Now both `endpointslice` and `endpointslices` are supported. See [the corresponding code in Prometheus](https://github.com/prometheus/prometheus/blob/2ec6c7dbb82b72834021e01f1773eb90a67a371f/discovery/kubernetes/kubernetes.go#L99). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 97a5a7871..affe22655 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -413,7 +413,11 @@ See also [implicit query conversions](#implicit-query-conversions). #### histogram_quantile -`histogram_quantile(phi, buckets)` calculates `phi`-quantile over the given [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). For example, `histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket[5m]) by (le))` would return median request duration for all the requests during the last 5 minutes. It accepts optional third arg - `boundsLabel`. In this case it returns `lower` and `upper` bounds for the estimated percentile. See [this issue for details](https://github.com/prometheus/prometheus/issues/5706). This function is supported by PromQL (except of the `boundLabel` arg). See also [histogram_share](#histogram_share). +`histogram_quantile(phi, buckets)` calculates `phi`-quantile over the given [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). For example, `histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket[5m]) by (le))` would return median request duration for all the requests during the last 5 minutes. It accepts optional third arg - `boundsLabel`. In this case it returns `lower` and `upper` bounds for the estimated percentile. See [this issue for details](https://github.com/prometheus/prometheus/issues/5706). This function is supported by PromQL (except of the `boundLabel` arg). See also [histogram_quantiles](#histogram_quantiles) and [histogram_share](#histogram_share). + +#### histogram_quantiles + +`histogram_quantiles("phiLabel", phi1, ..., phiN, buckets)` calculates the given `phi*`-quantiles over the given [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). Each calculated quantile is returned in a separate time series with the corresponding `{phiLabel="phi*"}` label. See also [histogram_quantile](#histogram_quantile). #### histogram_share @@ -782,7 +786,7 @@ See also [implicit query conversions](#implicit-query-conversions). #### quantiles -`quantiles("quantileLabel", phi1, ..., phiN, q)` calculates `phi*`-quantiles for all the time series returned by `q` and return them in time series with `{quantileLabel="phi*"}` label. The aggregate is calculated individually per each group of points with the same timestamp. See also [quantile](#quantile). +`quantiles("phiLabel", phi1, ..., phiN, q)` calculates `phi*`-quantiles for all the time series returned by `q` and return them in time series with `{phiLabel="phi*"}` label. The aggregate is calculated individually per each group of points with the same timestamp. See also [quantile](#quantile). #### stddev diff --git a/go.mod b/go.mod index 9511e16ee..7b9779d74 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.0 - github.com/VictoriaMetrics/metricsql v0.22.0 + github.com/VictoriaMetrics/metricsql v0.23.0 github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go v1.40.43 github.com/cespare/xxhash/v2 v2.1.2 diff --git a/go.sum b/go.sum index a4c0550ba..5af185252 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.18.0 h1:vov5NxDHRSXFbdiH4dYLYEjKLoAXXSQ7hcnG8TSD9JQ= github.com/VictoriaMetrics/metrics v1.18.0/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metricsql v0.22.0 h1:QWXvNfaFGeEQvh7Cjjpfp/7Un1N+W+ODQOiM+numyN0= -github.com/VictoriaMetrics/metricsql v0.22.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.23.0 h1:NWqoCrL2kz864OlaDBEU7c2fA7AjKfFq/nXM6di4xz8= +github.com/VictoriaMetrics/metricsql v0.23.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index 801a00feb..131e5a6a4 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -39,59 +39,60 @@ var transformFuncs = map[string]bool{ "year": true, // New funcs from MetricsQL - "label_set": true, - "label_map": true, - "label_uppercase": true, - "label_lowercase": true, - "label_del": true, - "label_keep": true, - "label_copy": true, - "label_move": true, - "label_transform": true, - "label_value": true, - "label_match": true, - "label_mismatch": true, - "union": true, - "": true, // empty func is a synonym to union - "keep_last_value": true, - "keep_next_value": true, - "interpolate": true, - "start": true, - "end": true, - "step": true, - "running_sum": true, - "running_max": true, - "running_min": true, - "running_avg": true, - "range_sum": true, - "range_max": true, - "range_min": true, - "range_avg": true, - "range_first": true, - "range_last": true, - "range_quantile": true, - "smooth_exponential": true, - "remove_resets": true, - "rand": true, - "rand_normal": true, - "rand_exponential": true, - "pi": true, - "sin": true, - "cos": true, - "asin": true, - "acos": true, - "prometheus_buckets": true, - "buckets_limit": true, - "histogram_share": true, - "histogram_avg": true, - "histogram_stdvar": true, - "histogram_stddev": true, - "sort_by_label": true, - "sort_by_label_desc": true, - "timezone_offset": true, - "bitmap_and": true, - "bitmap_or": true, - "bitmap_xor": true, + "label_set": true, + "label_map": true, + "label_uppercase": true, + "label_lowercase": true, + "label_del": true, + "label_keep": true, + "label_copy": true, + "label_move": true, + "label_transform": true, + "label_value": true, + "label_match": true, + "label_mismatch": true, + "union": true, + "": true, // empty func is a synonym to union + "keep_last_value": true, + "keep_next_value": true, + "interpolate": true, + "start": true, + "end": true, + "step": true, + "running_sum": true, + "running_max": true, + "running_min": true, + "running_avg": true, + "range_sum": true, + "range_max": true, + "range_min": true, + "range_avg": true, + "range_first": true, + "range_last": true, + "range_quantile": true, + "smooth_exponential": true, + "remove_resets": true, + "rand": true, + "rand_normal": true, + "rand_exponential": true, + "pi": true, + "sin": true, + "cos": true, + "asin": true, + "acos": true, + "prometheus_buckets": true, + "buckets_limit": true, + "histogram_share": true, + "histogram_avg": true, + "histogram_stdvar": true, + "histogram_stddev": true, + "sort_by_label": true, + "sort_by_label_desc": true, + "timezone_offset": true, + "bitmap_and": true, + "bitmap_or": true, + "bitmap_xor": true, + "histogram_quantiles": true, } // IsTransformFunc returns whether funcName is known transform function. diff --git a/vendor/modules.txt b/vendor/modules.txt index 5e93f56ae..659da9db3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -22,7 +22,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.18.0 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.22.0 +# github.com/VictoriaMetrics/metricsql v0.23.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop