mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
app/vmselect/promql: add rollup_scrape_interval(m[d]) function
It calculates the min, max and avg scrape intervals for m over the given lookbehind window d
This commit is contained in:
Aliaksandr Valialkin
parent
c4c77aa2dd
commit
2efe0acfc9
8 changed files with 139 additions and 83 deletions
|
|
@ -6466,6 +6466,39 @@ func TestExecSuccess(t *testing.T) {
|
|||
resultExpected := []netstorage.Result{r1, r2, r3}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`rollup_scrape_interval()`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort_by_label(rollup_scrape_interval(1[5m:10s]), "rollup")`
|
||||
r1 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{10, 10, 10, 10, 10, 10},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r1.MetricName.Tags = []storage.Tag{{
|
||||
Key: []byte("rollup"),
|
||||
Value: []byte("avg"),
|
||||
}}
|
||||
r2 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{10, 10, 10, 10, 10, 10},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r2.MetricName.Tags = []storage.Tag{{
|
||||
Key: []byte("rollup"),
|
||||
Value: []byte("max"),
|
||||
}}
|
||||
r3 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{10, 10, 10, 10, 10, 10},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r3.MetricName.Tags = []storage.Tag{{
|
||||
Key: []byte("rollup"),
|
||||
Value: []byte("min"),
|
||||
}}
|
||||
resultExpected := []netstorage.Result{r1, r2, r3}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`rollup()`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(rollup(time()[:50s]))`
|
||||
|
|
|
|||
|
|
@ -45,44 +45,45 @@ var rollupFuncs = map[string]newRollupFunc{
|
|||
"last_over_time": newRollupFuncOneArg(rollupLast),
|
||||
|
||||
// Additional rollup funcs.
|
||||
"default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func
|
||||
"range_over_time": newRollupFuncOneArg(rollupRange),
|
||||
"sum2_over_time": newRollupFuncOneArg(rollupSum2),
|
||||
"geomean_over_time": newRollupFuncOneArg(rollupGeomean),
|
||||
"first_over_time": newRollupFuncOneArg(rollupFirst),
|
||||
"distinct_over_time": newRollupFuncOneArg(rollupDistinct),
|
||||
"increases_over_time": newRollupFuncOneArg(rollupIncreases),
|
||||
"decreases_over_time": newRollupFuncOneArg(rollupDecreases),
|
||||
"increase_pure": newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets
|
||||
"integrate": newRollupFuncOneArg(rollupIntegrate),
|
||||
"ideriv": newRollupFuncOneArg(rollupIderiv),
|
||||
"lifetime": newRollupFuncOneArg(rollupLifetime),
|
||||
"lag": newRollupFuncOneArg(rollupLag),
|
||||
"scrape_interval": newRollupFuncOneArg(rollupScrapeInterval),
|
||||
"tmin_over_time": newRollupFuncOneArg(rollupTmin),
|
||||
"tmax_over_time": newRollupFuncOneArg(rollupTmax),
|
||||
"tfirst_over_time": newRollupFuncOneArg(rollupTfirst),
|
||||
"tlast_over_time": newRollupFuncOneArg(rollupTlast),
|
||||
"share_le_over_time": newRollupShareLE,
|
||||
"share_gt_over_time": newRollupShareGT,
|
||||
"count_le_over_time": newRollupCountLE,
|
||||
"count_gt_over_time": newRollupCountGT,
|
||||
"count_eq_over_time": newRollupCountEQ,
|
||||
"count_ne_over_time": newRollupCountNE,
|
||||
"histogram_over_time": newRollupFuncOneArg(rollupHistogram),
|
||||
"rollup": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_rate": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
|
||||
"rollup_deriv": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_delta": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_increase": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
|
||||
"rollup_candlestick": newRollupFuncOneArg(rollupFake),
|
||||
"aggr_over_time": newRollupFuncTwoArgs(rollupFake),
|
||||
"hoeffding_bound_upper": newRollupHoeffdingBoundUpper,
|
||||
"hoeffding_bound_lower": newRollupHoeffdingBoundLower,
|
||||
"ascent_over_time": newRollupFuncOneArg(rollupAscentOverTime),
|
||||
"descent_over_time": newRollupFuncOneArg(rollupDescentOverTime),
|
||||
"zscore_over_time": newRollupFuncOneArg(rollupZScoreOverTime),
|
||||
"quantiles_over_time": newRollupQuantiles,
|
||||
"default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func
|
||||
"range_over_time": newRollupFuncOneArg(rollupRange),
|
||||
"sum2_over_time": newRollupFuncOneArg(rollupSum2),
|
||||
"geomean_over_time": newRollupFuncOneArg(rollupGeomean),
|
||||
"first_over_time": newRollupFuncOneArg(rollupFirst),
|
||||
"distinct_over_time": newRollupFuncOneArg(rollupDistinct),
|
||||
"increases_over_time": newRollupFuncOneArg(rollupIncreases),
|
||||
"decreases_over_time": newRollupFuncOneArg(rollupDecreases),
|
||||
"increase_pure": newRollupFuncOneArg(rollupIncreasePure), // + rollupFuncsRemoveCounterResets
|
||||
"integrate": newRollupFuncOneArg(rollupIntegrate),
|
||||
"ideriv": newRollupFuncOneArg(rollupIderiv),
|
||||
"lifetime": newRollupFuncOneArg(rollupLifetime),
|
||||
"lag": newRollupFuncOneArg(rollupLag),
|
||||
"scrape_interval": newRollupFuncOneArg(rollupScrapeInterval),
|
||||
"tmin_over_time": newRollupFuncOneArg(rollupTmin),
|
||||
"tmax_over_time": newRollupFuncOneArg(rollupTmax),
|
||||
"tfirst_over_time": newRollupFuncOneArg(rollupTfirst),
|
||||
"tlast_over_time": newRollupFuncOneArg(rollupTlast),
|
||||
"share_le_over_time": newRollupShareLE,
|
||||
"share_gt_over_time": newRollupShareGT,
|
||||
"count_le_over_time": newRollupCountLE,
|
||||
"count_gt_over_time": newRollupCountGT,
|
||||
"count_eq_over_time": newRollupCountEQ,
|
||||
"count_ne_over_time": newRollupCountNE,
|
||||
"histogram_over_time": newRollupFuncOneArg(rollupHistogram),
|
||||
"rollup": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_rate": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
|
||||
"rollup_deriv": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_delta": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_increase": newRollupFuncOneArg(rollupFake), // + rollupFuncsRemoveCounterResets
|
||||
"rollup_candlestick": newRollupFuncOneArg(rollupFake),
|
||||
"rollup_scrape_interval": newRollupFuncOneArg(rollupFake),
|
||||
"aggr_over_time": newRollupFuncTwoArgs(rollupFake),
|
||||
"hoeffding_bound_upper": newRollupHoeffdingBoundUpper,
|
||||
"hoeffding_bound_lower": newRollupHoeffdingBoundLower,
|
||||
"ascent_over_time": newRollupFuncOneArg(rollupAscentOverTime),
|
||||
"descent_over_time": newRollupFuncOneArg(rollupDescentOverTime),
|
||||
"zscore_over_time": newRollupFuncOneArg(rollupZScoreOverTime),
|
||||
"quantiles_over_time": newRollupQuantiles,
|
||||
|
||||
// `timestamp` function must return timestamp for the last datapoint on the current window
|
||||
// in order to properly handle offset and timestamps unaligned to the current step.
|
||||
|
|
@ -319,6 +320,24 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en
|
|||
rcs = append(rcs, newRollupConfig(rollupClose, "close"))
|
||||
rcs = append(rcs, newRollupConfig(rollupLow, "low"))
|
||||
rcs = append(rcs, newRollupConfig(rollupHigh, "high"))
|
||||
case "rollup_scrape_interval":
|
||||
preFuncPrev := preFunc
|
||||
preFunc = func(values []float64, timestamps []int64) {
|
||||
preFuncPrev(values, timestamps)
|
||||
// Calculate intervals in seconds between samples.
|
||||
tsSecsPrev := nan
|
||||
for i, ts := range timestamps {
|
||||
tsSecs := float64(ts) / 1000
|
||||
values[i] = tsSecs - tsSecsPrev
|
||||
tsSecsPrev = tsSecs
|
||||
}
|
||||
if len(values) > 1 {
|
||||
// Overwrite the first NaN interval with the second interval,
|
||||
// So min, max and avg rollups could be calculated properly, since they don't expect to receive NaNs.
|
||||
values[0] = values[1]
|
||||
}
|
||||
}
|
||||
rcs = appendRollupConfigs(rcs)
|
||||
case "aggr_over_time":
|
||||
aggrFuncNames, err := getRollupAggrFuncNames(expr)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ sort: 15
|
|||
|
||||
* FEATURE: vmagent [enterprise](https://victoriametrics.com/enterprise.html): add support for data reading from [Apache Kafka](https://kafka.apache.org/).
|
||||
* FEATURE: calculate quantiles in the same way as Prometheus does in such functions as [quantile_over_time](https://docs.victoriametrics.com/MetricsQL.html#quantile_over_time) and [quantile](https://docs.victoriametrics.com/MetricsQL.html#quantile). Previously results from VictoriaMetrics could be slightly different than results from Prometheus. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1625) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1612) issues.
|
||||
* FEATURE: add `rollup_scrape_interval(m[d])` function to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which returns `min`, `max` and `avg` values for the interval between samples for `m` on the given lookbehind window `d`.
|
||||
|
||||
* BUGFIX: align behavior of the queries `a or on (labels) b`, `a and on (labels) b` and `a unless on (labels) b` where `b` has multiple time series with the given `labels` to Prometheus behavior. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1643).
|
||||
|
||||
|
|
|
|||
|
|
@ -274,9 +274,13 @@ See also [implicit query conversions](#implicit-query-conversions).
|
|||
|
||||
`rollup_rate(series_selector[d])` calculates per-second change rates for adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups.
|
||||
|
||||
#### rollup_scrape_interval
|
||||
|
||||
`rollup_scrape_interval(series_selector[d])` calculates the interval in seconds between adjancent raw samples on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated interval. The calculations are perfomed individually per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [scrape_interval](#scrape_interval).
|
||||
|
||||
#### scrape_interval
|
||||
|
||||
`scrape_interval(series_selector[d])` calculates the average interval in seconds between raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups.
|
||||
`scrape_interval(series_selector[d])` calculates the average interval in seconds between raw samples on the given lookbehind window `d` per each time series returned from the given [series_selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). Metric names are stripped from the resulting rollups. See also [rollup_scrape_interval](#rollup_scrape_interval).
|
||||
|
||||
#### share_gt_over_time
|
||||
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -9,7 +9,7 @@ require (
|
|||
// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
|
||||
github.com/VictoriaMetrics/fasthttp v1.1.0
|
||||
github.com/VictoriaMetrics/metrics v1.18.0
|
||||
github.com/VictoriaMetrics/metricsql v0.24.0
|
||||
github.com/VictoriaMetrics/metricsql v0.25.0
|
||||
github.com/VividCortex/ewma v1.2.0 // indirect
|
||||
github.com/aws/aws-sdk-go v1.40.49
|
||||
github.com/cespare/xxhash/v2 v2.1.2
|
||||
|
|
@ -30,7 +30,6 @@ require (
|
|||
github.com/valyala/fastrand v1.1.0
|
||||
github.com/valyala/fasttemplate v1.2.1
|
||||
github.com/valyala/gozstd v1.13.0
|
||||
github.com/valyala/histogram v1.2.0
|
||||
github.com/valyala/quicktemplate v1.7.0
|
||||
golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167
|
||||
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -110,8 +110,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR
|
|||
github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
|
||||
github.com/VictoriaMetrics/metrics v1.18.0 h1:vov5NxDHRSXFbdiH4dYLYEjKLoAXXSQ7hcnG8TSD9JQ=
|
||||
github.com/VictoriaMetrics/metrics v1.18.0/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA=
|
||||
github.com/VictoriaMetrics/metricsql v0.24.0 h1:1SOiuEaSgfS2CiQyCAlYQs3WPHzXNMPUSXtE1Zx6qDw=
|
||||
github.com/VictoriaMetrics/metricsql v0.24.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
|
||||
github.com/VictoriaMetrics/metricsql v0.25.0 h1:7bjs82RYWpO/myqfXLZIIn7VMWwWd9qPgiBhsIG7c8s=
|
||||
github.com/VictoriaMetrics/metricsql v0.25.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
|
||||
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
|
||||
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
|
||||
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
|
||||
|
|
|
|||
77
vendor/github.com/VictoriaMetrics/metricsql/rollup.go
generated
vendored
77
vendor/github.com/VictoriaMetrics/metricsql/rollup.go
generated
vendored
|
|
@ -31,44 +31,45 @@ var rollupFuncs = map[string]bool{
|
|||
"last_over_time": true,
|
||||
|
||||
// Additional rollup funcs.
|
||||
"default_rollup": true,
|
||||
"range_over_time": true,
|
||||
"sum2_over_time": true,
|
||||
"geomean_over_time": true,
|
||||
"first_over_time": true,
|
||||
"distinct_over_time": true,
|
||||
"increases_over_time": true,
|
||||
"decreases_over_time": true,
|
||||
"increase_pure": true,
|
||||
"integrate": true,
|
||||
"ideriv": true,
|
||||
"lifetime": true,
|
||||
"lag": true,
|
||||
"scrape_interval": true,
|
||||
"tmin_over_time": true,
|
||||
"tmax_over_time": true,
|
||||
"tfirst_over_time": true,
|
||||
"tlast_over_time": true,
|
||||
"share_le_over_time": true,
|
||||
"share_gt_over_time": true,
|
||||
"count_le_over_time": true,
|
||||
"count_gt_over_time": true,
|
||||
"count_eq_over_time": true,
|
||||
"count_ne_over_time": true,
|
||||
"histogram_over_time": true,
|
||||
"rollup": true,
|
||||
"rollup_rate": true,
|
||||
"rollup_deriv": true,
|
||||
"rollup_delta": true,
|
||||
"rollup_increase": true,
|
||||
"rollup_candlestick": true,
|
||||
"aggr_over_time": true,
|
||||
"hoeffding_bound_upper": true,
|
||||
"hoeffding_bound_lower": true,
|
||||
"ascent_over_time": true,
|
||||
"descent_over_time": true,
|
||||
"zscore_over_time": true,
|
||||
"quantiles_over_time": true,
|
||||
"default_rollup": true,
|
||||
"range_over_time": true,
|
||||
"sum2_over_time": true,
|
||||
"geomean_over_time": true,
|
||||
"first_over_time": true,
|
||||
"distinct_over_time": true,
|
||||
"increases_over_time": true,
|
||||
"decreases_over_time": true,
|
||||
"increase_pure": true,
|
||||
"integrate": true,
|
||||
"ideriv": true,
|
||||
"lifetime": true,
|
||||
"lag": true,
|
||||
"scrape_interval": true,
|
||||
"tmin_over_time": true,
|
||||
"tmax_over_time": true,
|
||||
"tfirst_over_time": true,
|
||||
"tlast_over_time": true,
|
||||
"share_le_over_time": true,
|
||||
"share_gt_over_time": true,
|
||||
"count_le_over_time": true,
|
||||
"count_gt_over_time": true,
|
||||
"count_eq_over_time": true,
|
||||
"count_ne_over_time": true,
|
||||
"histogram_over_time": true,
|
||||
"rollup": true,
|
||||
"rollup_rate": true,
|
||||
"rollup_deriv": true,
|
||||
"rollup_delta": true,
|
||||
"rollup_increase": true,
|
||||
"rollup_candlestick": true,
|
||||
"rollup_scrape_interval": true,
|
||||
"aggr_over_time": true,
|
||||
"hoeffding_bound_upper": true,
|
||||
"hoeffding_bound_lower": true,
|
||||
"ascent_over_time": true,
|
||||
"descent_over_time": true,
|
||||
"zscore_over_time": true,
|
||||
"quantiles_over_time": true,
|
||||
|
||||
// `timestamp` func has been moved here because it must work properly with offsets and samples unaligned to the current step.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 for details.
|
||||
|
|
|
|||
3
vendor/modules.txt
vendored
3
vendor/modules.txt
vendored
|
|
@ -22,7 +22,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
|
|||
# github.com/VictoriaMetrics/metrics v1.18.0
|
||||
## explicit
|
||||
github.com/VictoriaMetrics/metrics
|
||||
# github.com/VictoriaMetrics/metricsql v0.24.0
|
||||
# github.com/VictoriaMetrics/metricsql v0.25.0
|
||||
## explicit
|
||||
github.com/VictoriaMetrics/metricsql
|
||||
github.com/VictoriaMetrics/metricsql/binaryop
|
||||
|
|
@ -214,7 +214,6 @@ github.com/valyala/fasttemplate
|
|||
## explicit
|
||||
github.com/valyala/gozstd
|
||||
# github.com/valyala/histogram v1.2.0
|
||||
## explicit
|
||||
github.com/valyala/histogram
|
||||
# github.com/valyala/quicktemplate v1.7.0
|
||||
## explicit
|
||||
|
|
|
|||
Loading…
Reference in a new issue