diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index d2806a1ea..18c3db1ad 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -6466,6 +6466,39 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r1, r2, r3} f(q, resultExpected) }) + t.Run(`rollup_scrape_interval()`, func(t *testing.T) { + t.Parallel() + q := `sort_by_label(rollup_scrape_interval(1[5m:10s]), "rollup")` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{10, 10, 10, 10, 10, 10}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{{ + Key: []byte("rollup"), + Value: []byte("avg"), + }} + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{10, 10, 10, 10, 10, 10}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{{ + Key: []byte("rollup"), + Value: []byte("max"), + }} + r3 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{10, 10, 10, 10, 10, 10}, + Timestamps: timestampsExpected, + } + r3.MetricName.Tags = []storage.Tag{{ + Key: []byte("rollup"), + Value: []byte("min"), + }} + resultExpected := []netstorage.Result{r1, r2, r3} + f(q, resultExpected) + }) t.Run(`rollup()`, func(t *testing.T) { t.Parallel() q := `sort(rollup(time()[:50s]))` diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 9203e6519..b14cebaf6 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -45,44 +45,45 @@ var rollupFuncs = map[string]newRollupFunc{ "last_over_time": newRollupFuncOneArg(rollupLast), // Additional rollup funcs. - "default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func - "range_over_time": newRollupFuncOneArg(rollupRange), - "sum2_over_time": newRollupFuncOneArg(rollupSum2), - "geomean_over_time": newRollupFuncOneArg(rollupGeomean), - "first_over_time": newRollupFuncOneArg(rollupFirst), - "distinct_over_time": newRollupFuncOneArg(rollupDistinct), - "increases_over_time": newRollupFuncOneArg(rollupIncreases), - "decreases_over_time": newRollupFuncOneArg(rollupDecreases), - "increase_pure": newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets - "integrate": newRollupFuncOneArg(rollupIntegrate), - "ideriv": newRollupFuncOneArg(rollupIderiv), - "lifetime": newRollupFuncOneArg(rollupLifetime), - "lag": newRollupFuncOneArg(rollupLag), - "scrape_interval": newRollupFuncOneArg(rollupScrapeInterval), - "tmin_over_time": newRollupFuncOneArg(rollupTmin), - "tmax_over_time": newRollupFuncOneArg(rollupTmax), - "tfirst_over_time": newRollupFuncOneArg(rollupTfirst), - "tlast_over_time": newRollupFuncOneArg(rollupTlast), - "share_le_over_time": newRollupShareLE, - "share_gt_over_time": newRollupShareGT, - "count_le_over_time": newRollupCountLE, - "count_gt_over_time": newRollupCountGT, - "count_eq_over_time": newRollupCountEQ, - "count_ne_over_time": newRollupCountNE, - "histogram_over_time": newRollupFuncOneArg(rollupHistogram), - "rollup": newRollupFuncOneArg(rollupFake), - "rollup_rate": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets - "rollup_deriv": newRollupFuncOneArg(rollupFake), - "rollup_delta": newRollupFuncOneArg(rollupFake), - "rollup_increase": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets - "rollup_candlestick": newRollupFuncOneArg(rollupFake), - "aggr_over_time": newRollupFuncTwoArgs(rollupFake), - "hoeffding_bound_upper": newRollupHoeffdingBoundUpper, - "hoeffding_bound_lower": newRollupHoeffdingBoundLower, - "ascent_over_time": newRollupFuncOneArg(rollupAscentOverTime), - "descent_over_time": newRollupFuncOneArg(rollupDescentOverTime), - "zscore_over_time": newRollupFuncOneArg(rollupZScoreOverTime), - "quantiles_over_time": newRollupQuantiles, + "default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func + "range_over_time": newRollupFuncOneArg(rollupRange), + "sum2_over_time": newRollupFuncOneArg(rollupSum2), + "geomean_over_time": newRollupFuncOneArg(rollupGeomean), + "first_over_time": newRollupFuncOneArg(rollupFirst), + "distinct_over_time": newRollupFuncOneArg(rollupDistinct), + "increases_over_time": newRollupFuncOneArg(rollupIncreases), + "decreases_over_time": newRollupFuncOneArg(rollupDecreases), + "increase_pure": newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets + "integrate": newRollupFuncOneArg(rollupIntegrate), + "ideriv": newRollupFuncOneArg(rollupIderiv), + "lifetime": newRollupFuncOneArg(rollupLifetime), + "lag": newRollupFuncOneArg(rollupLag), + "scrape_interval": newRollupFuncOneArg(rollupScrapeInterval), + "tmin_over_time": newRollupFuncOneArg(rollupTmin), + "tmax_over_time": newRollupFuncOneArg(rollupTmax), + "tfirst_over_time": newRollupFuncOneArg(rollupTfirst), + "tlast_over_time": newRollupFuncOneArg(rollupTlast), + "share_le_over_time": newRollupShareLE, + "share_gt_over_time": newRollupShareGT, + "count_le_over_time": newRollupCountLE, + "count_gt_over_time": newRollupCountGT, + "count_eq_over_time": newRollupCountEQ, + "count_ne_over_time": newRollupCountNE, + "histogram_over_time": newRollupFuncOneArg(rollupHistogram), + "rollup": newRollupFuncOneArg(rollupFake), + "rollup_rate": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets + "rollup_deriv": newRollupFuncOneArg(rollupFake), + "rollup_delta": newRollupFuncOneArg(rollupFake), + "rollup_increase": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets + "rollup_candlestick": newRollupFuncOneArg(rollupFake), + "rollup_scrape_interval": newRollupFuncOneArg(rollupFake), + "aggr_over_time": newRollupFuncTwoArgs(rollupFake), + "hoeffding_bound_upper": newRollupHoeffdingBoundUpper, + "hoeffding_bound_lower": newRollupHoeffdingBoundLower, + "ascent_over_time": newRollupFuncOneArg(rollupAscentOverTime), + "descent_over_time": newRollupFuncOneArg(rollupDescentOverTime), + "zscore_over_time": newRollupFuncOneArg(rollupZScoreOverTime), + "quantiles_over_time": newRollupQuantiles, // `timestamp` function must return timestamp for the last datapoint on the current window // in order to properly handle offset and timestamps unaligned to the current step. @@ -319,6 +320,24 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en rcs = append(rcs, newRollupConfig(rollupClose, "close")) rcs = append(rcs, newRollupConfig(rollupLow, "low")) rcs = append(rcs, newRollupConfig(rollupHigh, "high")) + case "rollup_scrape_interval": + preFuncPrev := preFunc + preFunc = func(values []float64, timestamps []int64) { + preFuncPrev(values, timestamps) + // Calculate intervals in seconds between samples. + tsSecsPrev := nan + for i, ts := range timestamps { + tsSecs := float64(ts) / 1000 + values[i] = tsSecs - tsSecsPrev + tsSecsPrev = tsSecs + } + if len(values) > 1 { + // Overwrite the first NaN interval with the second interval, + // So min, max and avg rollups could be calculated properly, since they don't expect to receive NaNs. + values[0] = values[1] + } + } + rcs = appendRollupConfigs(rcs) case "aggr_over_time": aggrFuncNames, err := getRollupAggrFuncNames(expr) if err != nil { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5797e8db7..835604d26 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ sort: 15 * FEATURE: vmagent [enterprise](https://victoriametrics.com/enterprise.html): add support for data reading from [Apache Kafka](https://kafka.apache.org/). * FEATURE: calculate quantiles in the same way as Prometheus does in such functions as [quantile_over_time](https://docs.victoriametrics.com/MetricsQL.html#quantile_over_time) and [quantile](https://docs.victoriametrics.com/MetricsQL.html#quantile). Previously results from VictoriaMetrics could be slightly different than results from Prometheus. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1625) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1612) issues. +* FEATURE: add `rollup_scrape_interval(m[d])` function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which returns `min`, `max` and `avg` values for the interval between samples for `m` on the given lookbehind window `d`. * BUGFIX: align behavior of the queries `a or on (labels) b`, `a and on (labels) b` and `a unless on (labels) b` where `b` has multiple time series with the given `labels` to Prometheus behavior. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1643). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index fac354a79..db135d71c 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -274,9 +274,13 @@ See also [implicit query conversions](#implicit-query-conversions). `rollup_rate(series_selector[d])` calculates per-second change rates for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. +#### rollup_scrape_interval + +`rollup_scrape_interval(series_selector[d])` calculates the interval in seconds between adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated interval. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [scrape_interval](#scrape_interval). + #### scrape_interval -`scrape_interval(series_selector[d])` calculates the average interval in seconds between raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. +`scrape_interval(series_selector[d])` calculates the average interval in seconds between raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [rollup_scrape_interval](#rollup_scrape_interval). #### share_gt_over_time diff --git a/go.mod b/go.mod index 157b520be..d2318d75a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.0 - github.com/VictoriaMetrics/metricsql v0.24.0 + github.com/VictoriaMetrics/metricsql v0.25.0 github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go v1.40.49 github.com/cespare/xxhash/v2 v2.1.2 @@ -30,7 +30,6 @@ require ( github.com/valyala/fastrand v1.1.0 github.com/valyala/fasttemplate v1.2.1 github.com/valyala/gozstd v1.13.0 - github.com/valyala/histogram v1.2.0 github.com/valyala/quicktemplate v1.7.0 golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f diff --git a/go.sum b/go.sum index 2a9c9e0bb..7efbddfb9 100644 --- a/go.sum +++ b/go.sum @@ -110,8 +110,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.18.0 h1:vov5NxDHRSXFbdiH4dYLYEjKLoAXXSQ7hcnG8TSD9JQ= github.com/VictoriaMetrics/metrics v1.18.0/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metricsql v0.24.0 h1:1SOiuEaSgfS2CiQyCAlYQs3WPHzXNMPUSXtE1Zx6qDw= -github.com/VictoriaMetrics/metricsql v0.24.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.25.0 h1:7bjs82RYWpO/myqfXLZIIn7VMWwWd9qPgiBhsIG7c8s= +github.com/VictoriaMetrics/metricsql v0.25.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index 68b1c8929..dc7f8c429 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -31,44 +31,45 @@ var rollupFuncs = map[string]bool{ "last_over_time": true, // Additional rollup funcs. - "default_rollup": true, - "range_over_time": true, - "sum2_over_time": true, - "geomean_over_time": true, - "first_over_time": true, - "distinct_over_time": true, - "increases_over_time": true, - "decreases_over_time": true, - "increase_pure": true, - "integrate": true, - "ideriv": true, - "lifetime": true, - "lag": true, - "scrape_interval": true, - "tmin_over_time": true, - "tmax_over_time": true, - "tfirst_over_time": true, - "tlast_over_time": true, - "share_le_over_time": true, - "share_gt_over_time": true, - "count_le_over_time": true, - "count_gt_over_time": true, - "count_eq_over_time": true, - "count_ne_over_time": true, - "histogram_over_time": true, - "rollup": true, - "rollup_rate": true, - "rollup_deriv": true, - "rollup_delta": true, - "rollup_increase": true, - "rollup_candlestick": true, - "aggr_over_time": true, - "hoeffding_bound_upper": true, - "hoeffding_bound_lower": true, - "ascent_over_time": true, - "descent_over_time": true, - "zscore_over_time": true, - "quantiles_over_time": true, + "default_rollup": true, + "range_over_time": true, + "sum2_over_time": true, + "geomean_over_time": true, + "first_over_time": true, + "distinct_over_time": true, + "increases_over_time": true, + "decreases_over_time": true, + "increase_pure": true, + "integrate": true, + "ideriv": true, + "lifetime": true, + "lag": true, + "scrape_interval": true, + "tmin_over_time": true, + "tmax_over_time": true, + "tfirst_over_time": true, + "tlast_over_time": true, + "share_le_over_time": true, + "share_gt_over_time": true, + "count_le_over_time": true, + "count_gt_over_time": true, + "count_eq_over_time": true, + "count_ne_over_time": true, + "histogram_over_time": true, + "rollup": true, + "rollup_rate": true, + "rollup_deriv": true, + "rollup_delta": true, + "rollup_increase": true, + "rollup_candlestick": true, + "rollup_scrape_interval": true, + "aggr_over_time": true, + "hoeffding_bound_upper": true, + "hoeffding_bound_lower": true, + "ascent_over_time": true, + "descent_over_time": true, + "zscore_over_time": true, + "quantiles_over_time": true, // `timestamp` func has been moved here because it must work properly with offsets and samples unaligned to the current step. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 for details. diff --git a/vendor/modules.txt b/vendor/modules.txt index 6f87f5c7b..0876e2ec9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -22,7 +22,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.18.0 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.24.0 +# github.com/VictoriaMetrics/metricsql v0.25.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop @@ -214,7 +214,6 @@ github.com/valyala/fasttemplate ## explicit github.com/valyala/gozstd # github.com/valyala/histogram v1.2.0 -## explicit github.com/valyala/histogram # github.com/valyala/quicktemplate v1.7.0 ## explicit