From 3419ac1d36b54ca394926a36b7e72077998ed73c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 8 Nov 2021 13:14:06 +0200 Subject: [PATCH] app/vmselect/promql: add `duration_over_time(m[d], max_interval)` function This function calculates the actual lifetime of the time series on the given lookbehdind window `d` Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1780 --- app/vmselect/promql/exec_test.go | 12 +++++++ app/vmselect/promql/rollup.go | 32 +++++++++++++++++++ docs/CHANGELOG.md | 1 + docs/MetricsQL.md | 8 +++-- go.mod | 2 +- go.sum | 4 +-- .../VictoriaMetrics/metricsql/rollup.go | 1 + vendor/modules.txt | 2 +- 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 007a2bcf1..0c58c84b1 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -5030,6 +5030,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`duration_over_time`, func(t *testing.T) { + t.Parallel() + q := `duration_over_time((time()<1200)[600s:10s], 20s)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{590, 580, 380, 180, nan, nan}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`share_gt_over_time`, func(t *testing.T) { t.Parallel() q := `share_gt_over_time(rand(0)[200s:10s], 0.7)` @@ -7440,6 +7451,7 @@ func TestExecError(t *testing.T) { f(`prometheus_buckets()`) f(`buckets_limit()`) f(`buckets_limit(1)`) + f(`duration_over_time()`) f(`share_le_over_time()`) f(`share_gt_over_time()`) f(`count_le_over_time()`) diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index b80c953a2..0ae73cb35 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -63,6 +63,7 @@ var rollupFuncs = map[string]newRollupFunc{ "tmax_over_time": newRollupFuncOneArg(rollupTmax), "tfirst_over_time": newRollupFuncOneArg(rollupTfirst), "tlast_over_time": newRollupFuncOneArg(rollupTlast), + "duration_over_time": newRollupDurationOverTime, "share_le_over_time": newRollupShareLE, "share_gt_over_time": newRollupShareGT, "count_le_over_time": newRollupCountLE, @@ -879,6 +880,37 @@ func linearRegression(rfa *rollupFuncArg) (float64, float64) { return v, k } +func newRollupDurationOverTime(args []interface{}) (rollupFunc, error) { + if err := expectRollupArgsNum(args, 2); err != nil { + return nil, err + } + dMaxs, err := getScalar(args[1], 1) + if err != nil { + return nil, err + } + rf := func(rfa *rollupFuncArg) float64 { + // There is no need in handling NaNs here, since they must be cleaned up + // before calling rollup funcs. + values := rfa.values + if len(values) == 0 { + return nan + } + timestamps := rfa.timestamps + tPrev := timestamps[0] + dSum := int64(0) + dMax := int64(dMaxs[rfa.idx] * 1000) + for _, t := range timestamps { + d := t - tPrev + if d <= dMax { + dSum += d + } + tPrev = t + } + return float64(dSum / 1000) + } + return rf, nil +} + func newRollupShareLE(args []interface{}) (rollupFunc, error) { return newRollupShareFilter(args, countFilterLE) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 84ec55b38..f64e15df9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,7 @@ sort: 15 * FEATURE: vmagent: add `-remoteWrite.maxRowsPerBlock` command-line flag for tuning the number of samples to send to remote storage per each block. Bigger values may improve data ingestion performance at the cost of higher memory usage. * FEATURE: vmagent: distribute Kafka messages among all the partitions when [writing data to Kafka](https://docs.victoriametrics.com/vmagent.html#writing-metrics-to-kafka). * FEATURE: add [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function for extracting the given groups from Graphite metric names. +* FEATURE: add [duration_over_time](https://docs.victoriametrics.com/MetricsQL.html#duration_over_time) function for calculating the actual lifetime of the time series with possible gaps. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1780). * FEATURE: add [limit_offset](https://docs.victoriametrics.com/MetricsQL.html#limit_offset) function, which can be used for implementing simple paging over big number of time series. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1778). * BUGFIX: vmagent: reduce the increased memory usage when scraping targets with big number of metrics which periodically change. The memory usage has been increased in v1.68.0 after vmagent started generating staleness markers in [stream parse mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1745). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 8014d098c..42bc57d56 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -142,6 +142,10 @@ See also [implicit query conversions](#implicit-query-conversions). `distinct_over_time(series_selector[d])` returns the number of distinct raw sample values on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. +#### duration_over_time + +`duration_over_time(series_selector[d], max_interval)` returns the duration in seconds when time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) were present over the given lookbehind window `d`. It is expected that intervals between adjacent samples per each series don't exceed the `max_interval`. Otherwise such intervals are considered as gaps and aren't counted. See also [lifetime](#lifetime) and [lag](#lag). + #### first_over_time `first_over_time(series_selector[d])` returns the first raw sample value on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). See also [last_over_time](#last_over_time) and [tfirst_over_time](#tfirst_over_time). @@ -196,7 +200,7 @@ See also [implicit query conversions](#implicit-query-conversions). #### lag -`lag(series_selector[d])` returns the duration in seconds between the last sample on the given lookbehind window `d` and the timestamp of the current point. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [lifetime](#lifetime). +`lag(series_selector[d])` returns the duration in seconds between the last sample on the given lookbehind window `d` and the timestamp of the current point. It is calculated independently per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [lifetime](#lifetime) and [duration_over_time](#duration_over_time). #### last_over_time @@ -204,7 +208,7 @@ See also [implicit query conversions](#implicit-query-conversions). #### lifetime -`lifetime(series_selector[d])` returns the duration in seconds between the last and the first sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [lag](#lag). +`lifetime(series_selector[d])` returns the duration in seconds between the last and the first sample on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [duration_over_time](#duration_over_time) and [lag](#lag). #### max_over_time diff --git a/go.mod b/go.mod index b7fc41bfa..6d0280b83 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.18.1 - github.com/VictoriaMetrics/metricsql v0.29.0 + github.com/VictoriaMetrics/metricsql v0.30.0 github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go v1.41.19 github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect diff --git a/go.sum b/go.sum index dbd43e894..dd1de7652 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= -github.com/VictoriaMetrics/metricsql v0.29.0 h1:bQRxsHT6rWZgyMZqocV0lAM5GjpuPvAAW9eFXYR3iBY= -github.com/VictoriaMetrics/metricsql v0.29.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.30.0 h1:MGLqMcnVU2PTg+eOhqxpizQni6YwsYPbfv7rsHs+kU0= +github.com/VictoriaMetrics/metricsql v0.30.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index dc7f8c429..86d3c2b5d 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -49,6 +49,7 @@ var rollupFuncs = map[string]bool{ "tmax_over_time": true, "tfirst_over_time": true, "tlast_over_time": true, + "duration_over_time": true, "share_le_over_time": true, "share_gt_over_time": true, "count_le_over_time": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 70662e244..e8b1de40f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,7 +21,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.18.1 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.29.0 +# github.com/VictoriaMetrics/metricsql v0.30.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop