diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 9e7830381..8f1bcc715 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -132,6 +132,72 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run("bitmap_and(0xB3, 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_and(0xB3, 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{17, 17, 17, 17, 17, 17}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_and(time(), 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_and(time(), 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0, 16, 16, 0, 0, 16}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_or(0xA2, 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_or(0xA2, 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{179, 179, 179, 179, 179, 179}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_or(time(), 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_or(time(), 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1017, 1201, 1401, 1617, 1817, 2001}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_xor(0xB3, 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_xor(0xB3, 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{162, 162, 162, 162, 162, 162}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_xor(time(), 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_xor(time(), 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1017, 1185, 1385, 1617, 1817, 1985}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run("timezone_offset(UTC)", func(t *testing.T) { t.Parallel() q := `timezone_offset("UTC")` @@ -6891,6 +6957,10 @@ func TestExecError(t *testing.T) { f(`count_gt_over_time()`) f(`count_eq_over_time()`) f(`count_ne_over_time()`) + f(`timezone_offset()`) + f(`bitmap_and()`) + f(`bitmap_or()`) + f(`bitmap_xor()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 9ef2fefd7..9a0840673 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -124,6 +124,9 @@ var transformFuncs = map[string]transformFunc{ "sort_by_label": newTransformFuncSortByLabel(false), "sort_by_label_desc": newTransformFuncSortByLabel(true), "timezone_offset": transformTimezoneOffset, + "bitmap_and": newTransformBitmap(bitmapAnd), + "bitmap_or": newTransformBitmap(bitmapOr), + "bitmap_xor": newTransformBitmap(bitmapXor), } func getTransformFunc(s string) transformFunc { @@ -1914,6 +1917,37 @@ func transformPi(tfa *transformFuncArg) ([]*timeseries, error) { return evalNumber(tfa.ec, math.Pi), nil } +func bitmapAnd(a, b uint64) uint64 { + return a & b +} + +func bitmapOr(a, b uint64) uint64 { + return a | b +} + +func bitmapXor(a, b uint64) uint64 { + return a ^ b +} + +func newTransformBitmap(bitmapFunc func(a, b uint64) uint64) func(tfa *transformFuncArg) ([]*timeseries, error) { + return func(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if err := expectTransformArgsNum(args, 2); err != nil { + return nil, err + } + ns, err := getScalar(args[1], 1) + if err != nil { + return nil, err + } + tf := func(values []float64) { + for i, v := range values { + values[i] = float64(bitmapFunc(uint64(v), uint64(ns[i]))) + } + } + return doTransformValues(args[0], tf, tfa.fe) + } +} + func transformTimezoneOffset(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if err := expectTransformArgsNum(args, 1); err != nil { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d8547c7aa..f6446634c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,7 @@ sort: 15 ## tip +* FEATURE: add `bitmap_and(q, mask)`, `bitmap_or(q, mask)` and `bitmak_xor(q, mask)` functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). These functions allow performing bitwise operations over data points in time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1541). * FEATURE: vmalert: add `-remoteWrite.disablePathAppend` command-line flag, which can be used when custom `-remoteWrite.url` must be specified. For example, `./vmalert -disablePathAppend -remoteWrite.url='http://foo.bar/a/b/c?d=e'` would write data to `http://foo.bar/a/b/c?d=e` instead of `http://foo.bar/a/b/c?d=e/api/v1/write`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1536). * BUGFIX: vmagent: stop scrapers for deleted targets before starting scrapers for added targets. This should prevent from possible time series overlap when old targets are substituted by new targets (for example, during new deployment in Kubernetes). The overlap could lead to incorrect query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1509). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index b966db012..436994be8 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -160,3 +160,6 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g - `zscore(q) by (group)` - returns independent [z-score](https://en.wikipedia.org/wiki/Standard_score) values for every point in every `group` of `q`. Useful for detecting anomalies in the group of related time series. - `timezone_offset("tz")` - returns offset in seconds for the given timezone `tz` relative to UTC. This can be useful when combining with datetime-related functions. For example, `day_of_week(time()+timezone_offset("America/Los_Angeles"))` would return weekdays for `America/Los_Angeles` time zone. Special `Local` time zone can be used for returning an offset for the time zone set on the host where VictoriaMetrics runs. See [the list of supported timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). +- `bitmap_and(q, mask)` - calculates bitwise `v & mask` for every `v` point returned from `q`. +- `bitmap_or(q, mask)` - calculates bitwise `v | mask` for every `v` point returned from `q`. +- `bitmap_xor(q, mask)` - calculates bitwise `v ^ mask` for every `v` point returned from `q`. diff --git a/go.mod b/go.mod index 3cef4652a..9acf67183 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.16 github.com/VictoriaMetrics/metrics v1.17.3 - github.com/VictoriaMetrics/metricsql v0.17.0 + github.com/VictoriaMetrics/metricsql v0.18.0 github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go v1.40.22 github.com/cespare/xxhash/v2 v2.1.1 diff --git a/go.sum b/go.sum index 414fdf0c1..f0cebea77 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.16/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKC github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.17.3 h1:QPUakR6JRy8BhL2C2kOgYKLuoPDwtJQ+7iKIZSjt1A4= github.com/VictoriaMetrics/metrics v1.17.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.17.0 h1:OdOkx5GK2p1+EnHn/+zpLWW6ze96L1AmMV5X1HOzyfY= -github.com/VictoriaMetrics/metricsql v0.17.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.18.0 h1:BLNCjJPK305rfD4fjpWu7GV9snFj+zlhY8ispLB6rDs= +github.com/VictoriaMetrics/metricsql v0.18.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index bfeb53401..81f0afb34 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -89,6 +89,9 @@ var transformFuncs = map[string]bool{ "sort_by_label": true, "sort_by_label_desc": true, "timezone_offset": true, + "bitmap_and": true, + "bitmap_or": true, + "bitmap_xor": true, } // IsTransformFunc returns whether funcName is known transform function. diff --git a/vendor/modules.txt b/vendor/modules.txt index 722a43fec..c54343cb5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,7 +21,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.17.3 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.17.0 +# github.com/VictoriaMetrics/metricsql v0.18.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop