diff --git a/app/vmselect/promql/aggr.go b/app/vmselect/promql/aggr.go index 2555ad75f..627eaef90 100644 --- a/app/vmselect/promql/aggr.go +++ b/app/vmselect/promql/aggr.go @@ -48,6 +48,7 @@ var aggrFuncs = map[string]aggrFunc{ "outliersk": aggrFuncOutliersK, "mode": newAggrFunc(aggrFuncMode), "zscore": aggrFuncZScore, + "quantiles": aggrFuncQuantiles, } type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error) @@ -883,6 +884,42 @@ func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) { return aggrFuncExt(afe, args[1], &afa.ae.Modifier, afa.ae.Limit, true) } +func aggrFuncQuantiles(afa *aggrFuncArg) ([]*timeseries, error) { + args := afa.args + if len(args) < 3 { + return nil, fmt.Errorf("unexpected number of args: %d; expecting at least 3 args", len(args)) + } + dstLabel, err := getString(args[0], 0) + if err != nil { + return nil, fmt.Errorf("cannot obtain dstLabel: %w", err) + } + phiArgs := args[1 : len(args)-1] + argOrig := args[len(args)-1] + var rvs []*timeseries + for i, phiArg := range phiArgs { + phis, err := getScalar(phiArg, i+1) + if err != nil { + return nil, err + } + if len(phis) == 0 { + logger.Panicf("BUG: expecting at least a single sample") + } + phi := phis[0] + afe := newAggrQuantileFunc(phis) + arg := copyTimeseries(argOrig) + tss, err := aggrFuncExt(afe, arg, &afa.ae.Modifier, afa.ae.Limit, false) + if err != nil { + return nil, fmt.Errorf("cannot calculate quantile %g: %w", phi, err) + } + for _, ts := range tss { + ts.MetricName.RemoveTag(dstLabel) + ts.MetricName.AddTag(dstLabel, fmt.Sprintf("%g", phi)) + } + rvs = append(rvs, tss...) + } + return rvs, nil +} + func aggrFuncQuantile(afa *aggrFuncArg) ([]*timeseries, error) { args := afa.args if err := expectTransformArgsNum(args, 2); err != nil { diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index a29dde170..a818c5c0f 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -5461,6 +5461,30 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`quantiles("phi", 0.2, 0.5)`, func(t *testing.T) { + t.Parallel() + q := `sort(quantiles("phi", 0.2, 0.5, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{6.666666666666667, 8, 9.333333333333334, 10, 10, 10}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{{ + Key: []byte("phi"), + Value: []byte("0.2"), + }} + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{10, 10, 10, 10.666666666666666, 12, 13.333333333333334}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{{ + Key: []byte("phi"), + Value: []byte("0.5"), + }} + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) t.Run(`median()`, func(t *testing.T) { t.Parallel() q := `median(label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss"))` @@ -6961,6 +6985,7 @@ func TestExecError(t *testing.T) { f(`bitmap_and()`) f(`bitmap_or()`) f(`bitmap_xor()`) + f(`quantiles()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 1968a3c45..ddf138285 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -2016,6 +2016,17 @@ func transformEnd(tfa *transformFuncArg) float64 { return float64(tfa.ec.End) / 1e3 } +// copyTimeseries returns a copy of tss. +func copyTimeseries(tss []*timeseries) []*timeseries { + rvs := make([]*timeseries, len(tss)) + for i, src := range tss { + var dst timeseries + dst.CopyFromShallowTimestamps(src) + rvs[i] = &dst + } + return rvs +} + // copyTimeseriesMetricNames returns a copy of tss with real copy of MetricNames, // but with shallow copy of Timestamps and Values if makeCopy is set. // @@ -2033,7 +2044,7 @@ func copyTimeseriesMetricNames(tss []*timeseries, makeCopy bool) []*timeseries { return rvs } -// copyShallow returns a copy of arg with shallow copies of MetricNames, +// copyTimeseriesShallow returns a copy of arg with shallow copies of MetricNames, // Timestamps and Values. func copyTimeseriesShallow(arg []*timeseries) []*timeseries { rvs := make([]*timeseries, len(arg)) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6b0480590..ff4226f27 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,7 @@ sort: 15 * FEATURE: update Go builder from v1.16.7 to v1.17.0. This improves data ingestion and query performance by up to 5% according to benchmarks. See [the release post for Go1.17](https://go.dev/blog/go1.17). * FEATURE: vmagent: expose `promscrape_discovery_http_errors_total` metric, which can be used for monitoring the number of failed discovery attempts per each `http_sd` config. * FEATURE: do not reset response cache when a sample with old timestamp is ingested into VictoriaMetrics if `-search.disableAutoCacheReset` command-line option is set. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1570). +* FEATURE: add `quantiles("quantileLabel", phi1, ..., phiN, q)` aggregate function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which calculates the given `phi*` quantiles over time series returned by `q`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1573). * BUGFIX: rename `sign` function to `sgn` in order to be consistent with PromQL. See [this pull request from Prometheus](https://github.com/prometheus/prometheus/pull/8457). * BUGFIX: improve the detection of the needed free space for background merge operation. This should prevent from possible out of disk space crashes during big merges. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1560). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 1584f3f2e..a94bc85af 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -770,7 +770,11 @@ See also [implicit query conversions](#implicit-query-conversions). #### quantile -`quantile(phi, q) by (group_labels)` calculates `phi`-quantile per each `group_labels` for all the time series returned by `q`. The aggregate is calculated individually per each group of points with the same timestamp. This function is supported by PromQL. +`quantile(phi, q) by (group_labels)` calculates `phi`-quantile per each `group_labels` for all the time series returned by `q`. The aggregate is calculated individually per each group of points with the same timestamp. This function is supported by PromQL. See also [quantiles](#quantiles). + +#### quantiles + +`quantiles("quantileLabel", phi1, ..., phiN, q)` calculates `phi*`-quantiles for all the time series returned by `q` and return them in time series with `{quantileLabel="phi*"}` label. The aggregate is calculated individually per each group of points with the same timestamp. See also [quantile](#quantile). #### stddev diff --git a/go.mod b/go.mod index e664ac3ae..209ccdbd6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.16 github.com/VictoriaMetrics/metrics v1.17.3 - github.com/VictoriaMetrics/metricsql v0.20.0 + github.com/VictoriaMetrics/metricsql v0.21.0 github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go v1.40.30 github.com/cespare/xxhash/v2 v2.1.2 diff --git a/go.sum b/go.sum index 3bb6d04e8..1db47c336 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.16/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKC github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.17.3 h1:QPUakR6JRy8BhL2C2kOgYKLuoPDwtJQ+7iKIZSjt1A4= github.com/VictoriaMetrics/metrics v1.17.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.20.0 h1:1rL/naP4+PXdY3Hg5Oj8+Ql+NZCIHzl/OjTmOdg3+mM= -github.com/VictoriaMetrics/metricsql v0.20.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.21.0 h1:wA/IVfRFQaThy4bM1kAmPiCR0BkWv4tEXD9lBF+GPdU= +github.com/VictoriaMetrics/metricsql v0.21.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/aggr.go b/vendor/github.com/VictoriaMetrics/metricsql/aggr.go index 197227106..27821de5f 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/aggr.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/aggr.go @@ -38,6 +38,7 @@ var aggrFuncs = map[string]bool{ "outliersk": true, "mode": true, "zscore": true, + "quantiles": true, } func isAggrFunc(s string) bool { diff --git a/vendor/modules.txt b/vendor/modules.txt index 2b0219f3f..26f836040 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,7 +21,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.17.3 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.20.0 +# github.com/VictoriaMetrics/metricsql v0.21.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop