app/vmselect/promql: add bitmap_and(), bitmap_or() and bitmap_xor() functions to MetricsQL

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1541

app/vmselect/promql/exec_test.go

@@ -132,6 +132,72 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("bitmap_and(0xB3, 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(0xB3, 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{17, 17, 17, 17, 17, 17},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_and(time(), 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(time(), 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 16, 16, 0, 0, 16},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_or(0xA2, 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_or(0xA2, 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{179, 179, 179, 179, 179, 179},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_or(time(), 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_or(time(), 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1017, 1201, 1401, 1617, 1817, 2001},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_xor(0xB3, 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_xor(0xB3, 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{162, 162, 162, 162, 162, 162},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("bitmap_xor(time(), 0x11)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_xor(time(), 0x11)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1017, 1185, 1385, 1617, 1817, 1985},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("timezone_offset(UTC)", func(t *testing.T) {
 		t.Parallel()
 		q := `timezone_offset("UTC")`
@@ -6891,6 +6957,10 @@ func TestExecError(t *testing.T) {
 	f(`count_gt_over_time()`)
 	f(`count_eq_over_time()`)
 	f(`count_ne_over_time()`)
+	f(`timezone_offset()`)
+	f(`bitmap_and()`)
+	f(`bitmap_or()`)
+	f(`bitmap_xor()`)
 
 	// Invalid argument type
 	f(`median_over_time({}, 2)`)

app/vmselect/promql/transform.go

@@ -124,6 +124,9 @@ var transformFuncs = map[string]transformFunc{
 	"sort_by_label":      newTransformFuncSortByLabel(false),
 	"sort_by_label_desc": newTransformFuncSortByLabel(true),
 	"timezone_offset":    transformTimezoneOffset,
+	"bitmap_and":         newTransformBitmap(bitmapAnd),
+	"bitmap_or":          newTransformBitmap(bitmapOr),
+	"bitmap_xor":         newTransformBitmap(bitmapXor),
 }
 
 func getTransformFunc(s string) transformFunc {
@@ -1914,6 +1917,37 @@ func transformPi(tfa *transformFuncArg) ([]*timeseries, error) {
 	return evalNumber(tfa.ec, math.Pi), nil
 }
 
+func bitmapAnd(a, b uint64) uint64 {
+	return a & b
+}
+
+func bitmapOr(a, b uint64) uint64 {
+	return a | b
+}
+
+func bitmapXor(a, b uint64) uint64 {
+	return a ^ b
+}
+
+func newTransformBitmap(bitmapFunc func(a, b uint64) uint64) func(tfa *transformFuncArg) ([]*timeseries, error) {
+	return func(tfa *transformFuncArg) ([]*timeseries, error) {
+		args := tfa.args
+		if err := expectTransformArgsNum(args, 2); err != nil {
+			return nil, err
+		}
+		ns, err := getScalar(args[1], 1)
+		if err != nil {
+			return nil, err
+		}
+		tf := func(values []float64) {
+			for i, v := range values {
+				values[i] = float64(bitmapFunc(uint64(v), uint64(ns[i])))
+			}
+		}
+		return doTransformValues(args[0], tf, tfa.fe)
+	}
+}
+
 func transformTimezoneOffset(tfa *transformFuncArg) ([]*timeseries, error) {
 	args := tfa.args
 	if err := expectTransformArgsNum(args, 1); err != nil {

docs/CHANGELOG.md

@@ -6,6 +6,7 @@ sort: 15
 
 ## tip
 
+* FEATURE: add `bitmap_and(q, mask)`, `bitmap_or(q, mask)` and `bitmak_xor(q, mask)` functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). These functions allow performing bitwise operations over data points in time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1541).
 * FEATURE: vmalert: add `-remoteWrite.disablePathAppend` command-line flag, which can be used when custom `-remoteWrite.url` must be specified. For example, `./vmalert -disablePathAppend -remoteWrite.url='http://foo.bar/a/b/c?d=e'` would write data to `http://foo.bar/a/b/c?d=e` instead of `http://foo.bar/a/b/c?d=e/api/v1/write`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1536).
 
 * BUGFIX: vmagent: stop scrapers for deleted targets before starting scrapers for added targets. This should prevent from possible time series overlap when old targets are substituted by new targets (for example, during new deployment in Kubernetes). The overlap could lead to incorrect query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1509).

docs/MetricsQL.md

@@ -160,3 +160,6 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g
 - `zscore(q) by (group)` - returns independent [z-score](https://en.wikipedia.org/wiki/Standard_score) values for every point in every `group` of `q`.
   Useful for detecting anomalies in the group of related time series.
 - `timezone_offset("tz")` - returns offset in seconds for the given timezone `tz` relative to UTC. This can be useful when combining with datetime-related functions. For example, `day_of_week(time()+timezone_offset("America/Los_Angeles"))` would return weekdays for `America/Los_Angeles` time zone. Special `Local` time zone can be used for returning an offset for the time zone set on the host where VictoriaMetrics runs. See [the list of supported timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones).
+- `bitmap_and(q, mask)` - calculates bitwise `v & mask` for every `v` point returned from `q`.
+- `bitmap_or(q, mask)` - calculates bitwise `v | mask` for every `v` point returned from `q`.
+- `bitmap_xor(q, mask)` - calculates bitwise `v ^ mask` for every `v` point returned from `q`.

go.mod

@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.16
 	github.com/VictoriaMetrics/metrics v1.17.3
-	github.com/VictoriaMetrics/metricsql v0.17.0
+	github.com/VictoriaMetrics/metricsql v0.18.0
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/aws/aws-sdk-go v1.40.22
 	github.com/cespare/xxhash/v2 v2.1.1

go.sum

@@ -109,8 +109,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.16/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKC
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.17.3 h1:QPUakR6JRy8BhL2C2kOgYKLuoPDwtJQ+7iKIZSjt1A4=
 github.com/VictoriaMetrics/metrics v1.17.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.17.0 h1:OdOkx5GK2p1+EnHn/+zpLWW6ze96L1AmMV5X1HOzyfY=
+github.com/VictoriaMetrics/metricsql v0.18.0 h1:BLNCjJPK305rfD4fjpWu7GV9snFj+zlhY8ispLB6rDs=
-github.com/VictoriaMetrics/metricsql v0.17.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.18.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=

vendor/github.com/VictoriaMetrics/metricsql/transform.go

@@ -89,6 +89,9 @@ var transformFuncs = map[string]bool{
 	"sort_by_label":      true,
 	"sort_by_label_desc": true,
 	"timezone_offset":    true,
+	"bitmap_and":         true,
+	"bitmap_or":          true,
+	"bitmap_xor":         true,
 }
 
 // IsTransformFunc returns whether funcName is known transform function.

vendor/modules.txt

@@ -21,7 +21,7 @@ github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.17.3
 ## explicit
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.17.0
+# github.com/VictoriaMetrics/metricsql v0.18.0
 ## explicit
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop