Merge branch 'master' into master

Makefile

@@ -467,9 +467,9 @@ benchmark-pure:
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	go get -u -d ./lib/...
-	go get -u -d ./app/...
-	go mod tidy -compat=1.22
+	go get -u ./lib/...
+	go get -u ./app/...
+	go mod tidy -compat=1.23
 	go mod vendor
 
 app-local:
@@ -495,7 +495,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

app/victoria-logs/multiarch/Dockerfile

@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 9428
 ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/multiarch/Dockerfile

@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY victoria-metrics-linux-${TARGETARCH}-prod ./victoria-metrics-prod

app/vlselect/logsql/logsql.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -44,6 +45,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 	if step <= 0 {
 		httpserver.Errorf(w, r, "'step' must be bigger than zero")
+		return
 	}
 
 	// Obtain offset
@@ -380,6 +382,8 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 }
 
 // ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#live-tailing
 func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	liveTailRequests.Inc()
 	defer liveTailRequests.Dec()
@@ -560,9 +564,212 @@ func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
 	return tailRows, nil
 }
 
+// ProcessStatsQueryRangeRequest handles /select/logsql/stats_query_range request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats
+func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	// Obtain step
+	stepStr := r.FormValue("step")
+	if stepStr == "" {
+		stepStr = "1d"
+	}
+	step, err := promutils.ParseDuration(stepStr)
+	if err != nil {
+		err = fmt.Errorf("cannot parse 'step' arg: %s", err)
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+	if step <= 0 {
+		err := fmt.Errorf("'step' must be bigger than zero")
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	// Obtain `by(...)` fields from the last `| stats` pipe in q.
+	// Add `_time:step` to the `by(...)` list.
+	byFields, err := q.GetStatsByFieldsAddGroupingByTime(int64(step))
+	if err != nil {
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	q.Optimize()
+
+	m := make(map[string]*statsSeries)
+	var mLock sync.Mutex
+
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		clonedColumnNames := make([]string, len(columns))
+		for i, c := range columns {
+			clonedColumnNames[i] = strings.Clone(c.Name)
+		}
+		for i := range timestamps {
+			timestamp := q.GetTimestamp()
+			labels := make([]logstorage.Field, 0, len(byFields))
+			for j, c := range columns {
+				if c.Name == "_time" {
+					nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
+					if ok {
+						timestamp = nsec
+						continue
+					}
+				}
+				if slices.Contains(byFields, c.Name) {
+					labels = append(labels, logstorage.Field{
+						Name:  clonedColumnNames[j],
+						Value: strings.Clone(c.Values[i]),
+					})
+				}
+			}
+
+			var dst []byte
+			for j, c := range columns {
+				if !slices.Contains(byFields, c.Name) {
+					name := clonedColumnNames[j]
+					dst = dst[:0]
+					dst = append(dst, name...)
+					dst = logstorage.MarshalFieldsToJSON(dst, labels)
+					key := string(dst)
+					p := statsPoint{
+						Timestamp: timestamp,
+						Value:     strings.Clone(c.Values[i]),
+					}
+
+					mLock.Lock()
+					ss := m[key]
+					if ss == nil {
+						ss = &statsSeries{
+							key:    key,
+							Name:   name,
+							Labels: labels,
+						}
+						m[key] = ss
+					}
+					ss.Points = append(ss.Points, p)
+					mLock.Unlock()
+				}
+			}
+		}
+	}
+
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
+		err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	// Sort the collected stats by time
+	rows := make([]*statsSeries, 0, len(m))
+	for _, ss := range m {
+		points := ss.Points
+		sort.Slice(points, func(i, j int) bool {
+			return points[i].Timestamp < points[j].Timestamp
+		})
+		rows = append(rows, ss)
+	}
+	sort.Slice(rows, func(i, j int) bool {
+		return rows[i].key < rows[j].key
+	})
+
+	w.Header().Set("Content-Type", "application/json")
+	WriteStatsQueryRangeResponse(w, rows)
+}
+
+type statsSeries struct {
+	key string
+
+	Name   string
+	Labels []logstorage.Field
+	Points []statsPoint
+}
+
+type statsPoint struct {
+	Timestamp int64
+	Value     string
+}
+
+// ProcessStatsQueryRequest handles /select/logsql/stats_query request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats
+func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	// Obtain `by(...)` fields from the last `| stats` pipe in q.
+	byFields, err := q.GetStatsByFields()
+	if err != nil {
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	q.Optimize()
+
+	var rows []statsRow
+	var rowsLock sync.Mutex
+
+	timestamp := q.GetTimestamp()
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		clonedColumnNames := make([]string, len(columns))
+		for i, c := range columns {
+			clonedColumnNames[i] = strings.Clone(c.Name)
+		}
+		for i := range timestamps {
+			labels := make([]logstorage.Field, 0, len(byFields))
+			for j, c := range columns {
+				if slices.Contains(byFields, c.Name) {
+					labels = append(labels, logstorage.Field{
+						Name:  clonedColumnNames[j],
+						Value: strings.Clone(c.Values[i]),
+					})
+				}
+			}
+
+			for j, c := range columns {
+				if !slices.Contains(byFields, c.Name) {
+					r := statsRow{
+						Name:      clonedColumnNames[j],
+						Labels:    labels,
+						Timestamp: timestamp,
+						Value:     strings.Clone(c.Values[i]),
+					}
+
+					rowsLock.Lock()
+					rows = append(rows, r)
+					rowsLock.Unlock()
+				}
+			}
+		}
+	}
+
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
+		err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
+		httpserver.SendPrometheusError(w, r, err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	WriteStatsQueryResponse(w, rows)
+}
+
+type statsRow struct {
+	Name      string
+	Labels    []logstorage.Field
+	Timestamp int64
+	Value     string
+}
+
 // ProcessQueryRequest handles /select/logsql/query request.
 //
-// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-logs
 func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	q, tenantIDs, err := parseCommonArgs(r)
 	if err != nil {
@@ -637,6 +844,7 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
 	q.Optimize()
+
 	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 	if err != nil {
 		return nil, err
@@ -647,7 +855,7 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 		return rows, nil
 	}
 
-	// Slow path - search for the time range containing up to limitUpper rows.
+	// Slow path - adjust time range for selecting up to limitUpper rows
 	start, end := q.GetFilterTimeRange()
 	d := end/2 - start/2
 	start += d
@@ -661,18 +869,44 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 			return nil, err
 		}
 
-		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
+		if d == 0 || start >= end {
+			// The [start ... end] time range equals one nanosecond.
+			// Just return up to limit rows.
+			if len(rows) > limit {
+				rows = rows[:limit]
+			}
+			return rows, nil
+		}
+
+		dLastBit := d & 1
+		d /= 2
+
+		if len(rows) >= limitUpper {
+			// The number of found rows on the [start ... end] time range exceeds limitUpper,
+			// so reduce the time range to [start+d ... end].
+			start += d
+			continue
+		}
+		if len(rows) >= limit {
+			// The number of found rows is in the range [limit ... limitUpper).
+			// This means that found rows contains the needed limit rows with the biggest timestamps.
 			rows = getLastNRows(rows, limit)
 			return rows, nil
 		}
 
-		lastBit := d & 1
-		d /= 2
-		if len(rows) > limit {
-			start += d
-		} else {
-			start -= d + lastBit
+		// The number of found rows on [start ... end] time range is below the limit.
+		// This means the time range doesn't cover the needed logs, so it must be extended.
+
+		if len(rows) == 0 {
+			// The [start ... end] time range doesn't contain any rows, so change it to [start-d ... start).
+			end = start - 1
+			start -= d + dLastBit
+			continue
 		}
+
+		// The number of found rows on [start ... end] time range is bigger than 0 but smaller than limit.
+		// Increase the time range to [start-d ... end].
+		start -= d + dLastBit
 	}
 }
 
@@ -693,20 +927,25 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 	var rows []row
 	var rowsLock sync.Mutex
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		rowsLock.Lock()
-		defer rowsLock.Unlock()
+		clonedColumnNames := make([]string, len(columns))
+		for i, c := range columns {
+			clonedColumnNames[i] = strings.Clone(c.Name)
+		}
 
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
 				f := &fields[j]
-				f.Name = strings.Clone(columns[j].Name)
+				f.Name = clonedColumnNames[j]
 				f.Value = strings.Clone(columns[j].Values[i])
 			}
+
+			rowsLock.Lock()
 			rows = append(rows, row{
 				timestamp: timestamp,
 				fields:    fields,
 			})
+			rowsLock.Unlock()
 		}
 
 		if len(rows) >= limit {
@@ -728,9 +967,23 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	}
 	tenantIDs := []logstorage.TenantID{tenantID}
 
+	// Parse optional time arg
+	timestamp, okTime, err := getTimeNsec(r, "time")
+	if err != nil {
+		return nil, nil, err
+	}
+	if !okTime {
+		// If time arg is missing, then evaluate query at the current timestamp
+		timestamp = time.Now().UnixNano()
+	}
+
+	// decrease timestamp by one nanosecond in order to avoid capturing logs belonging
+	// to the first nanosecond at the next period of time (month, week, day, hour, etc.)
+	timestamp--
+
 	// Parse query
 	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQuery(qStr)
+	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
 	if err != nil {
 		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
 	}

app/vlselect/logsql/stats_query_range_response.qtpl

@@ -0,0 +1,52 @@
+{% stripspace %}
+
+// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
+{% func StatsQueryRangeResponse(rows []*statsSeries) %}
+{
+	"status":"success",
+	"data":{
+		"resultType":"matrix",
+		"result":[
+			{% if len(rows) > 0 %}
+				{%= formatStatsSeries(rows[0]) %}
+				{% code rows = rows[1:] %}
+				{% for i := range rows %}
+					,{%= formatStatsSeries(rows[i]) %}
+				{% endfor %}
+			{% endif %}
+		]
+	}
+}
+{% endfunc %}
+
+{% func formatStatsSeries(ss *statsSeries) %}
+{
+	"metric":{
+		"__name__":{%q= ss.Name %}
+		{% if len(ss.Labels) > 0 %}
+			{% for _, label := range ss.Labels %}
+				,{%q= label.Name %}:{%q= label.Value %}
+			{% endfor %}
+		{% endif %}
+	},
+	"values":[
+		{% code points := ss.Points %}
+		{% if len(points) > 0 %}
+			{%= formatStatsPoint(&points[0]) %}
+			{% code points = points[1:] %}
+			{% for i := range points %}
+				,{%= formatStatsPoint(&points[i]) %}
+			{% endfor %}
+		{% endif %}
+	]
+}
+{% endfunc %}
+
+{% func formatStatsPoint(p *statsPoint) %}
+[
+	{%f= float64(p.Timestamp)/1e9 %},
+	{%q= p.Value %}
+]
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/stats_query_range_response.qtpl.go

@@ -0,0 +1,188 @@
+// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:4
+package logsql
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:4
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:4
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:4
+func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:4
+	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:10
+	if len(rows) > 0 {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:11
+		streamformatStatsSeries(qw422016, rows[0])
+//line app/vlselect/logsql/stats_query_range_response.qtpl:12
+		rows = rows[1:]
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:13
+		for i := range rows {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:13
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:14
+			streamformatStatsSeries(qw422016, rows[i])
+//line app/vlselect/logsql/stats_query_range_response.qtpl:15
+		}
+//line app/vlselect/logsql/stats_query_range_response.qtpl:16
+	}
+//line app/vlselect/logsql/stats_query_range_response.qtpl:16
+	qw422016.N().S(`]}}`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	StreamStatsQueryRangeResponse(qw422016, rows)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+func StatsQueryRangeResponse(rows []*statsSeries) string {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	WriteStatsQueryRangeResponse(qb422016, rows)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+	return qs422016
+//line app/vlselect/logsql/stats_query_range_response.qtpl:20
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:22
+func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:22
+	qw422016.N().S(`{"metric":{"__name__":`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:25
+	qw422016.N().Q(ss.Name)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:26
+	if len(ss.Labels) > 0 {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:27
+		for _, label := range ss.Labels {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:27
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:28
+			qw422016.N().Q(label.Name)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:28
+			qw422016.N().S(`:`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:28
+			qw422016.N().Q(label.Value)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:29
+		}
+//line app/vlselect/logsql/stats_query_range_response.qtpl:30
+	}
+//line app/vlselect/logsql/stats_query_range_response.qtpl:30
+	qw422016.N().S(`},"values":[`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:33
+	points := ss.Points
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:34
+	if len(points) > 0 {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:35
+		streamformatStatsPoint(qw422016, &points[0])
+//line app/vlselect/logsql/stats_query_range_response.qtpl:36
+		points = points[1:]
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:37
+		for i := range points {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:37
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:38
+			streamformatStatsPoint(qw422016, &points[i])
+//line app/vlselect/logsql/stats_query_range_response.qtpl:39
+		}
+//line app/vlselect/logsql/stats_query_range_response.qtpl:40
+	}
+//line app/vlselect/logsql/stats_query_range_response.qtpl:40
+	qw422016.N().S(`]}`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	streamformatStatsSeries(qw422016, ss)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+func formatStatsSeries(ss *statsSeries) string {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	writeformatStatsSeries(qb422016, ss)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+	return qs422016
+//line app/vlselect/logsql/stats_query_range_response.qtpl:43
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:45
+func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:45
+	qw422016.N().S(`[`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:47
+	qw422016.N().F(float64(p.Timestamp) / 1e9)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:47
+	qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:48
+	qw422016.N().Q(p.Value)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:48
+	qw422016.N().S(`]`)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	streamformatStatsPoint(qw422016, p)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+}
+
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+func formatStatsPoint(p *statsPoint) string {
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	writeformatStatsPoint(qb422016, p)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+	return qs422016
+//line app/vlselect/logsql/stats_query_range_response.qtpl:50
+}

app/vlselect/logsql/stats_query_response.qtpl

@@ -0,0 +1,36 @@
+{% stripspace %}
+
+// StatsQueryResponse generates response for /select/logsql/stats_query
+{% func StatsQueryResponse(rows []statsRow) %}
+{
+	"status":"success",
+	"data":{
+		"resultType":"vector",
+		"result":[
+			{% if len(rows) > 0 %}
+				{%= formatStatsRow(&rows[0]) %}
+				{% code rows = rows[1:] %}
+				{% for i := range rows %}
+					,{%= formatStatsRow(&rows[i]) %}
+				{% endfor %}
+			{% endif %}
+		]
+	}
+}
+{% endfunc %}
+
+{% func formatStatsRow(r *statsRow) %}
+{
+	"metric":{
+		"__name__":{%q= r.Name %}
+		{% if len(r.Labels) > 0 %}
+			{% for _, label := range r.Labels %}
+				,{%q= label.Name %}:{%q= label.Value %}
+			{% endfor %}
+		{% endif %}
+	},
+	"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/stats_query_response.qtpl.go

@@ -0,0 +1,133 @@
+// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// StatsQueryResponse generates response for /select/logsql/stats_query
+
+//line app/vlselect/logsql/stats_query_response.qtpl:4
+package logsql
+
+//line app/vlselect/logsql/stats_query_response.qtpl:4
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/stats_query_response.qtpl:4
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/stats_query_response.qtpl:4
+func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
+//line app/vlselect/logsql/stats_query_response.qtpl:4
+	qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
+//line app/vlselect/logsql/stats_query_response.qtpl:10
+	if len(rows) > 0 {
+//line app/vlselect/logsql/stats_query_response.qtpl:11
+		streamformatStatsRow(qw422016, &rows[0])
+//line app/vlselect/logsql/stats_query_response.qtpl:12
+		rows = rows[1:]
+
+//line app/vlselect/logsql/stats_query_response.qtpl:13
+		for i := range rows {
+//line app/vlselect/logsql/stats_query_response.qtpl:13
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_response.qtpl:14
+			streamformatStatsRow(qw422016, &rows[i])
+//line app/vlselect/logsql/stats_query_response.qtpl:15
+		}
+//line app/vlselect/logsql/stats_query_response.qtpl:16
+	}
+//line app/vlselect/logsql/stats_query_response.qtpl:16
+	qw422016.N().S(`]}}`)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+}
+
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	StreamStatsQueryResponse(qw422016, rows)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+}
+
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+func StatsQueryResponse(rows []statsRow) string {
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	WriteStatsQueryResponse(qb422016, rows)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+	return qs422016
+//line app/vlselect/logsql/stats_query_response.qtpl:20
+}
+
+//line app/vlselect/logsql/stats_query_response.qtpl:22
+func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
+//line app/vlselect/logsql/stats_query_response.qtpl:22
+	qw422016.N().S(`{"metric":{"__name__":`)
+//line app/vlselect/logsql/stats_query_response.qtpl:25
+	qw422016.N().Q(r.Name)
+//line app/vlselect/logsql/stats_query_response.qtpl:26
+	if len(r.Labels) > 0 {
+//line app/vlselect/logsql/stats_query_response.qtpl:27
+		for _, label := range r.Labels {
+//line app/vlselect/logsql/stats_query_response.qtpl:27
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_response.qtpl:28
+			qw422016.N().Q(label.Name)
+//line app/vlselect/logsql/stats_query_response.qtpl:28
+			qw422016.N().S(`:`)
+//line app/vlselect/logsql/stats_query_response.qtpl:28
+			qw422016.N().Q(label.Value)
+//line app/vlselect/logsql/stats_query_response.qtpl:29
+		}
+//line app/vlselect/logsql/stats_query_response.qtpl:30
+	}
+//line app/vlselect/logsql/stats_query_response.qtpl:30
+	qw422016.N().S(`},"value":[`)
+//line app/vlselect/logsql/stats_query_response.qtpl:32
+	qw422016.N().F(float64(r.Timestamp) / 1e9)
+//line app/vlselect/logsql/stats_query_response.qtpl:32
+	qw422016.N().S(`,`)
+//line app/vlselect/logsql/stats_query_response.qtpl:32
+	qw422016.N().Q(r.Value)
+//line app/vlselect/logsql/stats_query_response.qtpl:32
+	qw422016.N().S(`]}`)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+}
+
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	streamformatStatsRow(qw422016, r)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+}
+
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+func formatStatsRow(r *statsRow) string {
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	writeformatStatsRow(qb422016, r)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+	return qs422016
+//line app/vlselect/logsql/stats_query_response.qtpl:34
+}

app/vlselect/main.go

@@ -193,6 +193,14 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 		logsqlQueryRequests.Inc()
 		logsql.ProcessQueryRequest(ctx, w, r)
 		return true
+	case "/select/logsql/stats_query":
+		logsqlStatsQueryRequests.Inc()
+		logsql.ProcessStatsQueryRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stats_query_range":
+		logsqlStatsQueryRangeRequests.Inc()
+		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
+		return true
 	case "/select/logsql/stream_field_names":
 		logsqlStreamFieldNamesRequests.Inc()
 		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
@@ -232,6 +240,8 @@ var (
 	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
 	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
 	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlStatsQueryRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
+	logsqlStatsQueryRangeRequests   = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
 	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
 	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)

app/vmagent/multiarch/Dockerfile

@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmalert-tool/multiarch/Dockerfile

@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8429
 ENTRYPOINT ["/vmalert-tool-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod

app/vmalert/multiarch/Dockerfile

@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8880
 ENTRYPOINT ["/vmalert-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmalert-linux-${TARGETARCH}-prod ./vmalert-prod

app/vmalert/notifier/alert.go

@@ -187,7 +187,7 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
 	return nil
 }
 
-func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
+func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
 	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
 		labels = append(labels, prompbmarshal.Label{

app/vmalert/notifier/alert_test.go

@@ -187,7 +187,7 @@ func TestAlert_toPromLabels(t *testing.T) {
 	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
 		t.Helper()
 		a := Alert{Labels: labels}
-		got := a.toPromLabels(relabel)
+		got := a.applyRelabelingIfNeeded(relabel)
 		if !reflect.DeepEqual(got, exp) {
 			t.Fatalf("expected to have: \n%v;\ngot:\n%v",
 				exp, got)

app/vmalert/notifier/alertmanager_request.qtpl

@@ -8,6 +8,8 @@
 {% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
 [
 {% for i, alert := range alerts %}
+{% code lbls := alert.applyRelabelingIfNeeded(relabelCfg) %}
+{% if len(lbls) == 0 %} {% continue %} {% endif %}
 {
     "startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
     "generatorURL": {%q= generatorURL(alert) %},
@@ -15,7 +17,6 @@
     "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
     {% endif %}
     "labels": {
-        {% code lbls := alert.toPromLabels(relabelCfg) %}
         {% code ll := len(lbls) %}
         {% for idx, l := range lbls %}
             {%q= l.Name %}:{%q= l.Value %}{% if idx != ll-1 %}, {% endif %}

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -30,111 +30,117 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun
 	qw422016.N().S(`[`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:10
 	for i, alert := range alerts {
-//line app/vmalert/notifier/alertmanager_request.qtpl:10
-		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
-		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:12
-		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
-		qw422016.N().Q(generatorURL(alert))
-//line app/vmalert/notifier/alertmanager_request.qtpl:13
-		qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-		if !alert.End.IsZero() {
-//line app/vmalert/notifier/alertmanager_request.qtpl:14
-			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:15
-			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/notifier/alertmanager_request.qtpl:15
-			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:16
-		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().S(`"labels": {`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:18
-		lbls := alert.toPromLabels(relabelCfg)
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
+		lbls := alert.applyRelabelingIfNeeded(relabelCfg)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		if len(lbls) == 0 {
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+			continue
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		qw422016.N().S(`{"startsAt":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
+		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
+		qw422016.N().S(`,"generatorURL":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
+		qw422016.N().Q(generatorURL(alert))
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
+		qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+		if !alert.End.IsZero() {
+//line app/vmalert/notifier/alertmanager_request.qtpl:16
+			qw422016.N().S(`"endsAt":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
+			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		qw422016.N().S(`"labels": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
 		ll := len(lbls)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:20
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
 		for idx, l := range lbls {
-//line app/vmalert/notifier/alertmanager_request.qtpl:21
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 			qw422016.N().Q(l.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:21
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:21
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 			qw422016.N().Q(l.Value)
-//line app/vmalert/notifier/alertmanager_request.qtpl:21
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 			if idx != ll-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:21
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:21
+//line app/vmalert/notifier/alertmanager_request.qtpl:22
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:22
+//line app/vmalert/notifier/alertmanager_request.qtpl:23
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:22
+//line app/vmalert/notifier/alertmanager_request.qtpl:23
 		qw422016.N().S(`},"annotations": {`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:25
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
 		c := len(alert.Annotations)
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:26
-		for k, v := range alert.Annotations {
 //line app/vmalert/notifier/alertmanager_request.qtpl:27
+		for k, v := range alert.Annotations {
+//line app/vmalert/notifier/alertmanager_request.qtpl:28
 			c = c - 1
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:28
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 			qw422016.N().Q(k)
-//line app/vmalert/notifier/alertmanager_request.qtpl:28
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 			qw422016.N().S(`:`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:28
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 			qw422016.N().Q(v)
-//line app/vmalert/notifier/alertmanager_request.qtpl:28
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 			if c > 0 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:28
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 				qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:28
+//line app/vmalert/notifier/alertmanager_request.qtpl:29
 			}
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:29
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
 		qw422016.N().S(`}}`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 		if i != len(alerts)-1 {
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 			qw422016.N().S(`,`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:32
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
 		}
-//line app/vmalert/notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:34
 	}
-//line app/vmalert/notifier/alertmanager_request.qtpl:33
+//line app/vmalert/notifier/alertmanager_request.qtpl:34
 	qw422016.N().S(`]`)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 }
 
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	qs422016 := string(qb422016.B)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 	return qs422016
-//line app/vmalert/notifier/alertmanager_request.qtpl:35
+//line app/vmalert/notifier/alertmanager_request.qtpl:36
 }

app/vmalert/notifier/alertmanager_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 func TestAlertManager_Addr(t *testing.T) {
@@ -76,12 +77,33 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].EndAt.IsZero() {
 				t.Fatalf("expected non-zero end time")
 			}
+			if len(a[0].Labels) != 1 {
+				t.Fatalf("expected 1 labels got %d", len(a[0].Labels))
+			}
+			if len(a[0].Annotations) != 2 {
+				t.Fatalf("expected 2 annotations got %d", len(a[0].Annotations))
+			}
 			if r.Header.Get(headerKey) != "bar" {
 				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
 			}
 		case 3:
-			if r.Header.Get(headerKey) != headerValue {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			var a []struct {
+				Labels map[string]string `json:"labels"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
+				t.Fatalf("can not unmarshal data into alert %s", err)
+			}
+			if len(a) != 1 {
+				t.Fatalf("expected 1 alert in array got %d", len(a))
+			}
+			if len(a[0].Labels) != 3 {
+				t.Fatalf("expected 3 labels got %d", len(a[0].Labels))
+			}
+			if a[0].Labels["env"] != "prod" {
+				t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"])
+			}
+			if r.Header.Get(headerKey) != "bar" {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, "bar", r.Header.Get(headerKey))
 			}
 		}
 	})
@@ -95,31 +117,58 @@ func TestAlertManager_Send(t *testing.T) {
 		},
 		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
+	parsedConfigs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- action: drop
+  if: '{tenant="0"}'
+  regex: ".*"
+- target_label: "env"
+  replacement: "prod"
+  if: '{tenant="1"}'
+`))
+	if err != nil {
+		t.Fatalf("unexpected error when parse relabeling config: %s", err)
+	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
-	}, aCfg, nil, 0)
+	}, aCfg, parsedConfigs, 0)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
+
+	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
-	if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
+
+	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
 		t.Fatalf("expected wrong http code error got nil")
 	}
+
 	if err := am.Send(context.Background(), []Alert{{
 		GroupID:     0,
 		Name:        "alert0",
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
-		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+		Labels:      map[string]string{"alertname": "alert0"},
+		Annotations: map[string]string{"a": "b", "c": "d"},
 	}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
-	if c != 2 {
-		t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
-	}
-	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
+
+	if err := am.Send(context.Background(), []Alert{
+		// drop tenant0 alert message during relabeling
+		{
+			Name:   "alert1",
+			Labels: map[string]string{"rule": "test", "tenant": "0"},
+		},
+		{
+			Name:   "alert2",
+			Labels: map[string]string{"rule": "test", "tenant": "1"},
+		},
+	}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
+
+	if c != 3 {
+		t.Fatalf("expected 3 calls(count from zero) to server got %d", c)
+	}
 }

app/vmauth/multiarch/Dockerfile

@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8427
 ENTRYPOINT ["/vmauth-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmauth-linux-${TARGETARCH}-prod ./vmauth-prod

app/vmbackup/multiarch/Dockerfile

@@ -7,5 +7,5 @@ RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmbackup-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmbackup-linux-${TARGETARCH}-prod ./vmbackup-prod

app/vmctl/multiarch/Dockerfile

@@ -7,5 +7,5 @@ RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmctl-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmctl-linux-${TARGETARCH}-prod ./vmctl-prod

app/vmrestore/multiarch/Dockerfile

@@ -7,5 +7,5 @@ RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmrestore-prod"]
-ARG TARGETARCH=non-existing
+ARG TARGETARCH
 COPY vmrestore-linux-${TARGETARCH}-prod ./vmrestore-prod

app/vmselect/main.go

@@ -2,7 +2,6 @@ package vmselect
 
 import (
 	"embed"
-	"errors"
 	"flag"
 	"fmt"
 	"net/http"
@@ -187,7 +186,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.EnableCORS(w, r)
 			if err := prometheus.LabelValuesHandler(qt, startTime, labelName, w, r); err != nil {
 				labelValuesErrors.Inc()
-				sendPrometheusError(w, r, err)
+				httpserver.SendPrometheusError(w, r, err)
 				return true
 			}
 			return true
@@ -210,7 +209,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.QueryHandler(qt, startTime, w, r); err != nil {
 			queryErrors.Inc()
-			sendPrometheusError(w, r, err)
+			httpserver.SendPrometheusError(w, r, err)
 			return true
 		}
 		return true
@@ -219,7 +218,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.QueryRangeHandler(qt, startTime, w, r); err != nil {
 			queryRangeErrors.Inc()
-			sendPrometheusError(w, r, err)
+			httpserver.SendPrometheusError(w, r, err)
 			return true
 		}
 		return true
@@ -228,7 +227,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.SeriesHandler(qt, startTime, w, r); err != nil {
 			seriesErrors.Inc()
-			sendPrometheusError(w, r, err)
+			httpserver.SendPrometheusError(w, r, err)
 			return true
 		}
 		return true
@@ -237,7 +236,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.SeriesCountHandler(startTime, w, r); err != nil {
 			seriesCountErrors.Inc()
-			sendPrometheusError(w, r, err)
+			httpserver.SendPrometheusError(w, r, err)
 			return true
 		}
 		return true
@@ -246,7 +245,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.LabelsHandler(qt, startTime, w, r); err != nil {
 			labelsErrors.Inc()
-			sendPrometheusError(w, r, err)
+			httpserver.SendPrometheusError(w, r, err)
 			return true
 		}
 		return true
@@ -255,7 +254,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.TSDBStatusHandler(qt, startTime, w, r); err != nil {
 			statusTSDBErrors.Inc()
-			sendPrometheusError(w, r, err)
+			httpserver.SendPrometheusError(w, r, err)
 			return true
 		}
 		return true
@@ -498,7 +497,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		httpserver.EnableCORS(w, r)
 		if err := prometheus.QueryStatsHandler(w, r); err != nil {
 			topQueriesErrors.Inc()
-			sendPrometheusError(w, r, fmt.Errorf("cannot query status endpoint: %w", err))
+			httpserver.SendPrometheusError(w, r, fmt.Errorf("cannot query status endpoint: %w", err))
 			return true
 		}
 		return true
@@ -575,24 +574,6 @@ func isGraphiteTagsPath(path string) bool {
 	}
 }
 
-func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
-	logger.WarnfSkipframes(1, "error in %q: %s", httpserver.GetRequestURI(r), err)
-
-	w.Header().Set("Content-Type", "application/json")
-	statusCode := http.StatusUnprocessableEntity
-	var esc *httpserver.ErrorWithStatusCode
-	if errors.As(err, &esc) {
-		statusCode = esc.StatusCode
-	}
-	w.WriteHeader(statusCode)
-
-	var ure *promql.UserReadableError
-	if errors.As(err, &ure) {
-		err = ure
-	}
-	prometheus.WriteErrorResponse(w, statusCode, err)
-}
-
 var (
 	requestDuration = metrics.NewHistogram(`vmselect_request_duration_seconds`)
 

app/vmselect/prometheus/error_response.qtpl.go

@@ -1,61 +0,0 @@
-// Code generated by qtc from "error_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// ErrorResponse generates error response for /api/v1/query.See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview
-
-//line app/vmselect/prometheus/error_response.qtpl:4
-package prometheus
-
-//line app/vmselect/prometheus/error_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vmselect/prometheus/error_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vmselect/prometheus/error_response.qtpl:4
-func StreamErrorResponse(qw422016 *qt422016.Writer, statusCode int, err error) {
-//line app/vmselect/prometheus/error_response.qtpl:4
-	qw422016.N().S(`{"status":"error","errorType":"`)
-//line app/vmselect/prometheus/error_response.qtpl:7
-	qw422016.N().D(statusCode)
-//line app/vmselect/prometheus/error_response.qtpl:7
-	qw422016.N().S(`","error":`)
-//line app/vmselect/prometheus/error_response.qtpl:8
-	qw422016.N().Q(err.Error())
-//line app/vmselect/prometheus/error_response.qtpl:8
-	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/error_response.qtpl:10
-}
-
-//line app/vmselect/prometheus/error_response.qtpl:10
-func WriteErrorResponse(qq422016 qtio422016.Writer, statusCode int, err error) {
-//line app/vmselect/prometheus/error_response.qtpl:10
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/error_response.qtpl:10
-	StreamErrorResponse(qw422016, statusCode, err)
-//line app/vmselect/prometheus/error_response.qtpl:10
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/error_response.qtpl:10
-}
-
-//line app/vmselect/prometheus/error_response.qtpl:10
-func ErrorResponse(statusCode int, err error) string {
-//line app/vmselect/prometheus/error_response.qtpl:10
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/error_response.qtpl:10
-	WriteErrorResponse(qb422016, statusCode, err)
-//line app/vmselect/prometheus/error_response.qtpl:10
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/error_response.qtpl:10
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/error_response.qtpl:10
-	return qs422016
-//line app/vmselect/prometheus/error_response.qtpl:10
-}

app/vmselect/promql/eval.go

@@ -19,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
@@ -354,7 +355,7 @@ func evalExprInternal(qt *querytracer.Tracer, ec *EvalConfig, e metricsql.Expr)
 func evalTransformFunc(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.FuncExpr) ([]*timeseries, error) {
 	tf := getTransformFunc(fe.Name)
 	if tf == nil {
-		return nil, &UserReadableError{
+		return nil, &httpserver.UserReadableError{
 			Err: fmt.Errorf(`unknown func %q`, fe.Name),
 		}
 	}
@@ -376,7 +377,7 @@ func evalTransformFunc(qt *querytracer.Tracer, ec *EvalConfig, fe *metricsql.Fun
 	}
 	rv, err := tf(tfa)
 	if err != nil {
-		return nil, &UserReadableError{
+		return nil, &httpserver.UserReadableError{
 			Err: fmt.Errorf(`cannot evaluate %q: %w`, fe.AppendString(nil), err),
 		}
 	}
@@ -407,7 +408,7 @@ func evalAggrFunc(qt *querytracer.Tracer, ec *EvalConfig, ae *metricsql.AggrFunc
 	}
 	af := getAggrFunc(ae.Name)
 	if af == nil {
-		return nil, &UserReadableError{
+		return nil, &httpserver.UserReadableError{
 			Err: fmt.Errorf(`unknown func %q`, ae.Name),
 		}
 	}
@@ -802,12 +803,12 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 	}
 	tssAt, err := evalExpr(qt, ec, re.At)
 	if err != nil {
-		return nil, &UserReadableError{
+		return nil, &httpserver.UserReadableError{
 			Err: fmt.Errorf("cannot evaluate `@` modifier: %w", err),
 		}
 	}
 	if len(tssAt) != 1 {
-		return nil, &UserReadableError{
+		return nil, &httpserver.UserReadableError{
 			Err: fmt.Errorf("`@` modifier must return a single series; it returns %d series instead", len(tssAt)),
 		}
 	}
@@ -869,7 +870,7 @@ func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName st
 		rvs, err = evalRollupFuncWithSubquery(qt, ecNew, funcName, rf, expr, re)
 	}
 	if err != nil {
-		return nil, &UserReadableError{
+		return nil, &httpserver.UserReadableError{
 			Err: err,
 		}
 	}
@@ -1601,7 +1602,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	if ec.Start == ec.End {
 		rvs, err := evalInstantRollup(qt, ec, funcName, rf, expr, me, iafc, window)
 		if err != nil {
-			err = &UserReadableError{
+			err = &httpserver.UserReadableError{
 				Err: err,
 			}
 			return nil, err
@@ -1612,7 +1613,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	evalWithConfig := func(ec *EvalConfig) ([]*timeseries, error) {
 		tss, err := evalRollupFuncNoCache(qt, ec, funcName, rf, expr, me, iafc, window, pointsPerSeries)
 		if err != nil {
-			err = &UserReadableError{
+			err = &httpserver.UserReadableError{
 				Err: err,
 			}
 			return nil, err

app/vmselect/promql/exec.go

@@ -35,24 +35,6 @@ var (
 		"Such conversion can be disabled using -search.disableImplicitConversion.")
 )
 
-// UserReadableError is a type of error which supposed to be returned to the user without additional context.
-type UserReadableError struct {
-	// Err is the error which needs to be returned to the user.
-	Err error
-}
-
-// Unwrap returns ure.Err.
-//
-// This is used by standard errors package. See https://golang.org/pkg/errors
-func (ure *UserReadableError) Unwrap() error {
-	return ure.Err
-}
-
-// Error satisfies Error interface
-func (ure *UserReadableError) Error() string {
-	return ure.Err.Error()
-}
-
 // Exec executes q for the given ec.
 func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result, error) {
 	if querystats.Enabled() {

app/vmstorage/main.go

@@ -521,12 +521,21 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/inmemory"}`, idbm.InmemorySizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_data_size_bytes{type="indexdb/file"}`, idbm.FileSizeBytes)
 
+	metrics.WriteCounterUint64(w, `vm_rows_received_by_storage_total`, m.RowsReceivedTotal)
 	metrics.WriteCounterUint64(w, `vm_rows_added_to_storage_total`, m.RowsAddedTotal)
 	metrics.WriteCounterUint64(w, `vm_deduplicated_samples_total{type="merge"}`, m.DedupsDuringMerge)
 	metrics.WriteGaugeUint64(w, `vm_snapshots`, m.SnapshotsCount)
 
+	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="nan_value"}`, m.NaNValueRows)
 	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="big_timestamp"}`, m.TooBigTimestampRows)
 	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="small_timestamp"}`, m.TooSmallTimestampRows)
+	metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, m.InvalidRawMetricNames)
+	if *maxHourlySeries > 0 {
+		metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, m.HourlySeriesLimitRowsDropped)
+	}
+	if *maxDailySeries > 0 {
+		metrics.WriteCounterUint64(w, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`, m.DailySeriesLimitRowsDropped)
+	}
 
 	metrics.WriteCounterUint64(w, `vm_timeseries_repopulated_total`, m.TimeseriesRepopulated)
 	metrics.WriteCounterUint64(w, `vm_timeseries_precreated_total`, m.TimeseriesPreCreated)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.23.0 AS build-web-stage
+FROM golang:1.23.1 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.20.2
+FROM alpine:3.20.3
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

deployment/docker/Makefile

@@ -2,11 +2,11 @@
 
 DOCKER_NAMESPACE ?= victoriametrics
 
-ROOT_IMAGE ?= alpine:3.20.2
+ROOT_IMAGE ?= alpine:3.20.3
 ROOT_IMAGE_SCRATCH ?= scratch
-CERTS_IMAGE := alpine:3.20.2
+CERTS_IMAGE := alpine:3.20.3
 
-GO_BUILDER_IMAGE := golang:1.23.0-alpine
+GO_BUILDER_IMAGE := golang:1.23.1-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 DOCKER ?= docker

deployment/docker/docker-compose-victorialogs.yml

@@ -40,7 +40,7 @@ services:
 #  storing logs and serving read queries.
   victorialogs:
     container_name: victorialogs
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     command:
       - "--storageDataPath=/vlogs"
       - "--httpListenAddr=:9428"

deployment/docker/victorialogs/filebeat-syslog/compose.yml

@@ -11,7 +11,7 @@ services:
       - "5140:5140"
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-filebeat-syslog-vl:/vlogs
     ports:

deployment/docker/victorialogs/filebeat/compose.yml

@@ -20,7 +20,7 @@ services:
       - -beat.uri=http://filebeat-victorialogs:5066
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-filebeat-docker-vl:/vlogs
     ports:

deployment/docker/victorialogs/fluentbit-ha-single-node/README.md

@@ -0,0 +1,88 @@
+# Docker compose Fluentbit integration with VictoriaLogs for docker. High-Availability example 
+
+The folder contains the example of integration of [fluentbit](https://docs.fluentbit.io/manual) with VictoriaLogs Single-Nodes(s) and [vmauth](https://docs.victoriametrics.com/vmauth/) for achieving High Availability. 
+
+Check [this documentation](https://docs.victoriametrics.com/victorialogs/#high-availability) with a description of the architecture and components.
+
+To spin-up environment  run the following command:
+
+```shell
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+
+```shell
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* fluentbit - fluentbit is configured to collect logs from the `docker`, you can find configuration in the `fluent-bit.conf`. It writes data in VictoriaLogs
+* VictoriaLogs - the two instances of log database, they accept the data from `fluentbit` by json line protocol
+* vmauth - load balancer for proxying requests to one of VictoriaLogs
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:8427/select/vmui/`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+
+the example of fluentbit configuration(`filebeat.yml`)
+
+```text
+[INPUT]
+    name             tail
+    path             /var/lib/docker/containers/**/*.log
+    path_key         path
+    multiline.parser docker, cri
+    Parser           docker
+    Docker_Mode      On
+
+[INPUT]
+    Name     syslog
+    Listen   0.0.0.0
+    Port     5140
+    Parser   syslog-rfc3164
+    Mode     tcp
+
+[SERVICE]
+    Flush        1
+    Parsers_File parsers.conf
+
+[OUTPUT]
+    Name http
+    Match *
+    host victorialogs-2
+    port 9428
+    compress gzip
+    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
+    format json_lines
+    json_date_format iso8601
+    header AccountID 0
+    header ProjectID 0
+
+[OUTPUT]
+    Name http
+    Match *
+    host victorialogs-1
+    port 9428
+    compress gzip
+    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
+    format json_lines
+    json_date_format iso8601
+    header AccountID 0
+    header ProjectID 0
+```
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
+
+The example of vmauth configuration (`auth.yml`)
+
+```yaml
+unauthorized_user:
+  url_prefix:
+    - http://victorialogs-1:9428
+    - http://victorialogs-2:9428
+```

deployment/docker/victorialogs/fluentbit-ha-single-node/auth.yml

@@ -0,0 +1,6 @@
+# balance load among victorialogs instances
+# see https://docs.victoriametrics.com/vmauth/#load-balancing
+unauthorized_user:
+  url_prefix:
+    - http://victorialogs-1:9428
+    - http://victorialogs-2:9428

deployment/docker/victorialogs/fluentbit-ha-single-node/compose.yml

@@ -0,0 +1,38 @@
+services:
+  fluentbit:
+    image: cr.fluentbit.io/fluent/fluent-bit:3.0.7
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+    depends_on: [victorialogs-1,victorialogs-2]
+    ports:
+      - "5140:5140"
+
+  victorialogs-1:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-fluentbit-vl-ha-single-1:/vlogs
+    command:
+      - -storageDataPath=/vlogs
+  victorialogs-2:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-fluentbit-vl-ha-single-2:/vlogs
+    command:
+      - -storageDataPath=/vlogs
+  vmauth:
+    container_name: vmauth
+    image: victoriametrics/vmauth:v1.103.0
+    depends_on:
+      - "victorialogs-1"
+      - "victorialogs-2"
+    volumes:
+      - ./auth.yml:/etc/auth.yml
+    command:
+      - '--auth.config=/etc/auth.yml'
+    ports:
+      - 8427:8427
+    restart: always
+volumes:
+  victorialogs-fluentbit-vl-ha-single-1:
+  victorialogs-fluentbit-vl-ha-single-2:

deployment/docker/victorialogs/fluentbit-ha-single-node/fluent-bit.conf

@@ -0,0 +1,42 @@
+[INPUT]
+    name             tail
+    path             /var/lib/docker/containers/**/*.log
+    path_key         path
+    multiline.parser docker, cri
+    Parser           docker
+    Docker_Mode      On
+
+[INPUT]
+    Name     syslog
+    Listen   0.0.0.0
+    Port     5140
+    Parser   syslog-rfc3164
+    Mode     tcp
+
+[SERVICE]
+    Flush        1
+    Parsers_File parsers.conf
+
+[OUTPUT]
+    Name http
+    Match *
+    host victorialogs-2
+    port 9428
+    compress gzip
+    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
+    format json_lines
+    json_date_format iso8601
+    header AccountID 0
+    header ProjectID 0
+
+[OUTPUT]
+    Name http
+    Match *
+    host victorialogs-1
+    port 9428
+    compress gzip
+    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
+    format json_lines
+    json_date_format iso8601
+    header AccountID 0
+    header ProjectID 0

deployment/docker/victorialogs/fluentbit/compose.yml

@@ -9,7 +9,7 @@ services:
       - "5140:5140"
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-fluentbit-vl:/vlogs
     ports:

deployment/docker/victorialogs/logstash-ha-single-node/README.md

@@ -0,0 +1,71 @@
+# Docker compose Logstash integration with VictoriaLogs for docker. High-Availability example
+
+The folder contains the example of integration of [logstash](https://www.elastic.co/logstash) with VictoriaLogs Single-Node(s) and [vmauth](https://docs.victoriametrics.com/vmauth/) for achieving High Availability.
+
+Check [this documentation](https://docs.victoriametrics.com/victorialogs/#high-availability) with a description of the architecture and components.
+
+To spin-up environment  run the following command:
+
+```shell
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+
+```shell
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* logstash - logstash is configured to read docker log files, you can find configuration in the `pipeline.conf`. It writes data in two instances of VictoriaLogs
+* VictoriaLogs - the two instances of log database, they accept the data from `fluentbit` by json line protocol
+* vmauth - load balancer for proxying requests to one of VictoriaLogs
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:8427/select/vmui/`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+
+Here is an example of logstash configuration(`pipeline.conf`):
+
+```text
+input {
+  file {
+    path => "/var/lib/docker/containers/*/*.log"
+    start_position => "beginning"
+    type => "docker"
+    sincedb_path => "/dev/null"
+    codec => "json"
+    add_field => {
+      "path" => "%{[@metadata][path]}"
+    }
+  }
+}
+
+output {
+  http {
+    url => "http://victorialogs-1:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time"
+    format => "json"
+    http_method => "post"
+  }
+  http {
+    url => "http://victorialogs-2:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time"
+    format => "json"
+    http_method => "post"
+  }
+}
+```
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
+
+The example of vmauth configuration (`auth.yml`)
+
+```yaml
+unauthorized_user:
+  url_prefix:
+    - http://victorialogs-1:9428
+    - http://victorialogs-2:9428
+```

deployment/docker/victorialogs/logstash-ha-single-node/auth.yml

@@ -0,0 +1,6 @@
+# balance load among victorialogs instances
+# see https://docs.victoriametrics.com/vmauth/#load-balancing
+unauthorized_user:
+  url_prefix:
+    - http://victorialogs-1:9428
+    - http://victorialogs-2:9428

deployment/docker/victorialogs/logstash-ha-single-node/compose.yml

@@ -0,0 +1,41 @@
+services:
+  logstash:
+    image: docker.elastic.co/logstash/logstash:8.8.1
+    user: root
+    volumes:
+      - ./pipeline.conf:/usr/share/logstash/pipeline/logstash.conf:ro
+      - ./logstash.yml:/usr/share/logstash/config/logstash.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+    depends_on: [victorialogs-1,victorialogs-2]
+    ports:
+      - "5140:5140"
+
+  victorialogs-1:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-logstash-vl-ha-single-1:/vlogs
+    command:
+      - -storageDataPath=/vlogs
+  victorialogs-2:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-logstash-vl-ha-single-2:/vlogs
+    command:
+      - -storageDataPath=/vlogs
+  vmauth:
+    container_name: vmauth
+    image: victoriametrics/vmauth:v1.103.0
+    depends_on:
+      - "victorialogs-1"
+      - "victorialogs-2"
+    volumes:
+      - ./auth.yml:/etc/auth.yml
+    command:
+      - '--auth.config=/etc/auth.yml'
+    ports:
+      - 8427:8427
+    restart: always
+
+volumes:
+  victorialogs-logstash-vl-ha-single-1:
+  victorialogs-logstash-vl-ha-single-2:

deployment/docker/victorialogs/logstash-ha-single-node/logstash.yml

@@ -0,0 +1,2 @@
+http.host: 0.0.0.0
+xpack.monitoring.enabled: false

deployment/docker/victorialogs/logstash-ha-single-node/pipeline.conf

@@ -0,0 +1,25 @@
+input {
+  file {
+    path => "/var/lib/docker/containers/*/*.log"
+    start_position => "beginning"
+    type => "docker"
+    sincedb_path => "/dev/null"
+    codec => "json"
+    add_field => {
+      "path" => "%{[@metadata][path]}"
+    }
+  }
+}
+
+output {
+  http {
+    url => "http://victorialogs-1:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time"
+    format => "json"
+    http_method => "post"
+  }
+  http {
+    url => "http://victorialogs-2:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time"
+    format => "json"
+    http_method => "post"
+  }
+}

deployment/docker/victorialogs/logstash/compose.yml

@@ -12,7 +12,7 @@ services:
       - "5140:5140"
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-logstash-vl:/vlogs
     ports:

deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/compose.yml

@@ -12,7 +12,7 @@ services:
         condition: service_healthy
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-vector-docker-vl:/vlogs
     ports:

deployment/docker/victorialogs/opentelemetry-collector/loki/compose.yml

@@ -12,7 +12,7 @@ services:
         condition: service_healthy
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-vector-docker-vl:/loki
     ports:

deployment/docker/victorialogs/opentelemetry-collector/syslog/compose.yml

@@ -12,7 +12,7 @@ services:
         condition: service_healthy
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-vector-docker-vl:/syslog
     ports:

deployment/docker/victorialogs/promtail/compose.yml

@@ -10,7 +10,7 @@ services:
       - "5140:5140"
 
   vlogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-promtail-docker:/vlogs
     ports:

deployment/docker/victorialogs/telegraf-docker/compose.yml

@@ -18,7 +18,7 @@ services:
         condition: service_healthy
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-vector-docker-vl:/vlogs
     ports:

deployment/docker/victorialogs/vector-ha-single-node/README.md

@@ -0,0 +1,104 @@
+# Docker compose Vector integration with VictoriaLogs for docker. High-Availability example
+
+The folder contains the example of integration of [vector](https://vector.dev/docs/) with VictoriaLogs Single-Node(s) and [vmauth](https://docs.victoriametrics.com/vmauth/) for achieving High Availability.
+
+Check [this documentation](https://docs.victoriametrics.com/victorialogs/#high-availability) with a description of the architecture and components.
+
+
+To spin-up environment  run the following command:
+
+```shell
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+
+```shell
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.yaml`. It writes data in two instances of VictoriaLogs
+* VictoriaLogs - the two instances of log database, they accept the data from `vector` by json line protocol
+* vmauth - load balancer for proxying requests to one of VictoriaLogs
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:8427/select/vmui/`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+
+the example of vector configuration(`vector.yaml`)
+
+```yaml
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  docker:
+    type: docker_logs
+transforms:
+  msg_parser:
+    type: remap
+    inputs:
+      - docker
+    source: |
+      if exists(.message) {
+        .log, err = parse_json(.message)
+        if err == null {
+          del(.message)
+        }
+      }
+sinks:
+  console_out:
+    type: console
+    inputs:
+      - msg_parser
+    encoding:
+      codec: json
+  vlogs_http_1:
+    type: http
+    inputs:
+      - msg_parser
+    uri: http://victorialogs-1:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    encoding:
+      codec: json
+    framing:
+      method: newline_delimited
+    compression: gzip
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        AccountID: '0'
+        ProjectID: '0'
+  vlogs_http_2:
+    type: http
+    inputs:
+      - msg_parser
+    uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    encoding:
+      codec: json
+    framing:
+      method: newline_delimited
+    compression: gzip
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        AccountID: '0'
+        ProjectID: '0'
+```
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.
+
+The example of vmauth configuration (`auth.yml`)
+
+```yaml
+unauthorized_user:
+  url_prefix:
+    - http://victorialogs-1:9428
+    - http://victorialogs-2:9428
+```

deployment/docker/victorialogs/vector-ha-single-node/auth.yml

@@ -0,0 +1,6 @@
+# balance load among victorialogs instances
+# see https://docs.victoriametrics.com/vmauth/#load-balancing
+unauthorized_user:
+  url_prefix:
+    - http://victorialogs-1:9428
+    - http://victorialogs-2:9428

deployment/docker/victorialogs/vector-ha-single-node/compose.yml

@@ -0,0 +1,48 @@
+services:
+  vector:
+    image: docker.io/timberio/vector:0.40.1-distroless-static
+    restart: on-failure
+    volumes:
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+      - type: bind
+        source: /var/lib/docker
+        target: /var/lib/docker
+      - ./vector.yaml:/etc/vector/vector.yaml:ro
+    user: root
+    ports:
+      - '8686:8686'
+    depends_on: [victorialogs-1,victorialogs-2]
+
+  victorialogs-1:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl-ha-single-1:/vlogs
+    command:
+      - -storageDataPath=/vlogs
+      - -loggerFormat=json
+  victorialogs-2:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl-ha-single-2:/vlogs
+    command:
+      - -storageDataPath=/vlogs
+      - -loggerFormat=json
+  vmauth:
+    container_name: vmauth
+    image: victoriametrics/vmauth:v1.103.0
+    depends_on:
+      - "victorialogs-1"
+      - "victorialogs-2"
+    volumes:
+      - ./auth.yml:/etc/auth.yml
+    command:
+      - '--auth.config=/etc/auth.yml'
+    ports:
+      - 8427:8427
+    restart: always
+
+volumes:
+  victorialogs-vector-docker-vl-ha-single-1:
+  victorialogs-vector-docker-vl-ha-single-2:

deployment/docker/victorialogs/vector-ha-single-node/vector.yaml

@@ -0,0 +1,58 @@
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  docker:
+    type: docker_logs
+transforms:
+  msg_parser:
+    type: remap
+    inputs:
+      - docker
+    source: |
+      if exists(.message) {
+        .log, err = parse_json(.message)
+        if err == null {
+          del(.message)
+        }
+      }
+sinks:
+  console_out:
+    type: console
+    inputs:
+      - msg_parser
+    encoding:
+      codec: json
+  vlogs_http_1:
+    type: http
+    inputs:
+      - msg_parser
+    uri: http://victorialogs-1:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    encoding:
+      codec: json
+    framing:
+      method: newline_delimited
+    compression: gzip
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        AccountID: '0'
+        ProjectID: '0'
+  vlogs_http_2:
+    type: http
+    inputs:
+      - msg_parser
+    uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    encoding:
+      codec: json
+    framing:
+      method: newline_delimited
+    compression: gzip
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        AccountID: '0'
+        ProjectID: '0'
+

deployment/docker/victorialogs/vector/compose.yml

@@ -20,7 +20,7 @@ services:
         condition: service_healthy
 
   victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - victorialogs-vector-docker-vl:/vlogs
     ports:

deployment/logs-benchmark/docker-compose-elk.yml

@@ -18,7 +18,7 @@ services:
       - vlogs
 
   generator:
-    image: golang:1.23.0-alpine
+    image: golang:1.23.1-alpine
     restart: always
     working_dir: /go/src/app
     volumes:

deployment/logs-benchmark/docker-compose-loki.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   generator:
-    image: golang:1.23.0-alpine
+    image: golang:1.23.1-alpine
     restart: always
     working_dir: /go/src/app
     volumes:

deployment/logs-benchmark/docker-compose.yml

@@ -3,7 +3,7 @@ version: '3'
 services:
   # Run `make package-victoria-logs` to build victoria-logs image
   vlogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
     volumes:
       - vlogs:/vlogs
     ports:

docs/VictoriaLogs/CHANGELOG.md

@@ -15,6 +15,13 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 ## tip
 
+## [v0.29.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.29.0-victorialogs)
+
+Released at 2024-09-08
+
+* FEATURE: add [`/select/logsql/stats_query` HTTP API](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats), which is going to be used by [vmalert](https://docs.victoriametrics.com/vmalert/) for executing alerting and recording rules against VictoriaLogs. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6942) for details.
+* FEATURE: add [`/select/logsql/stats_query_range` HTTP API](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats), which is going to be used by [VictoriaLogs plugin for Grafana](https://docs.victoriametrics.com/victorialogs/victorialogs-datasource/) for building time series panels. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6943) for details.
+* FEATURE: optimize [multi-exact queries](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter) with many phrases to search. For example, `ip:in(path:="/foo/bar" | keep ip)` when there are many unique values for `ip` field among log entries with `/foo/bar` path.
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add support for displaying the top 5 log streams in the hits graph. The remaining log streams are grouped into an "other" label. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6545).
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the ability to customize the graph display with options for bar, line, stepped line, and points.
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add fields for setting AccountID and ProjectID. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6631).
@@ -29,7 +36,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 * BUGFIX: properly handle Logstash requests for Elasticsearch configuration when using `outputs.elasticsearch` in Logstash pipelines. Previously, the requests could be rejected with `400 Bad Request` response. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4750).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `not found index.js` error when loading vmui in VictoriaLogs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6764). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6770).
-* BUGFIX: fix filtering when logical operators `AND` and `OR` are used in query expression. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
+* BUGFIX: properly execute queries with `OR` [filters](https://docs.victoriametrics.com/victorialogs/logsql/#logical-filter) for distinct [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). For example, `field1:foo OR field2:bar`. Previously logs matching these filters may be skipped during querying. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
 
 ## [v0.28.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.28.0-victorialogs)
 

docs/VictoriaLogs/LogsQL.md

@@ -1799,7 +1799,7 @@ The following mathematical operations are supported by `math` pipe:
 - `arg1 % arg2` - returns the remainder of the division of `arg1` by `arg2`
 - `arg1 ^ arg2` - returns the power of `arg1` by `arg2`
 - `arg1 & arg2` - returns bitwise `and` for `arg1` and `arg2`. It is expected that `arg1` and `arg2` are in the range `[0 .. 2^53-1]`
-- `arg1 | arg2` - returns bitwise `or` for `arg1` and `arg2`. It is expected that `arg1` and `arg2` are in the range `[0 .. 2^53-1]`
+- `arg1 or arg2` - returns bitwise `or` for `arg1` and `arg2`. It is expected that `arg1` and `arg2` are in the range `[0 .. 2^53-1]`
 - `arg1 xor arg2` - returns bitwise `xor` for `arg1` and `arg2`. It is expected that `arg1` and `arg2` are in the range `[0 .. 2^53-1]`
 - `arg1 default arg2` - returns `arg2` if `arg1` is non-[numeric](#numeric-values) or equals to `NaN`
 - `abs(arg)` - returns an absolute value for the given `arg`

docs/VictoriaLogs/QuickStart.md

@@ -33,8 +33,8 @@ Just download archive for the needed Operating system and architecture, unpack i
 For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
 
 ```sh
-curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.28.0-victorialogs/victoria-logs-linux-amd64-v0.28.0-victorialogs.tar.gz
-tar xzf victoria-logs-linux-amd64-v0.28.0-victorialogs.tar.gz
+curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.29.0-victorialogs/victoria-logs-linux-amd64-v0.29.0-victorialogs.tar.gz
+tar xzf victoria-logs-linux-amd64-v0.29.0-victorialogs.tar.gz
 ./victoria-logs-prod
 ```
 
@@ -58,7 +58,7 @@ Here is the command to run VictoriaLogs in a Docker container:
 
 ```sh
 docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
-  docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+  docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
 ```
 
 See also:
@@ -139,10 +139,10 @@ See also:
 
 Here are a Docker-compose demos, which start VictoriaLogs and push logs to it via various log collectors:
 
-- [Filebeat demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat-docker)
-- [Fluentbit demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit-docker)
+- [Filebeat demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat)
+- [Fluentbit demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit)
 - [Logstash demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/logstash)
-- [Vector demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector-docker)
+- [Vector demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector)
 - [Promtail demo](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/promtail)
 
 You can use [this Helm chart](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-logs-single/README.md)

docs/VictoriaLogs/README.md

@@ -129,6 +129,26 @@ For example, the following command starts VictoriaLogs, which stores the data at
 
 VictoriaLogs automatically creates the `-storageDataPath` directory on the first run if it is missing.
 
+## High Availability
+
+### High Availability (HA) Setup with VictoriaLogs Single-Node Instances
+
+This schema outlines how to configure a High Availability (HA) setup using VictoriaLogs Single-Node instances. The setup consists of the following components:
+
+- **Log Collector**: The log collector should support multiplexing incoming data to multiple outputs (destinations). Popular log collectors like [Fluent Bit](https://docs.fluentbit.io/manual/concepts/data-pipeline/router), [Logstash](https://www.elastic.co/guide/en/logstash/current/output-plugins.html), [Fluentd](https://docs.fluentd.org/output/copy), and [Vector](https://vector.dev/docs/setup/configuration/sinks/) already offer this capability. Refer to their documentation for configuration details.
+
+- **VictoriaLogs Single-Node Instances**: Use two or more instances to achieve HA.
+
+- **[vmauth](https://docs.victoriametrics.com/vmauth/#load-balancing) or Load Balancer**: Used for reading data from one of the replicas to ensure balanced and redundant access.
+
+![VictoriaLogs Single-Node Instance High-Availability schema](ha-victorialogs-single-node.webp)
+
+Here are the working example of HA configuration for VictoriaLogs using Docker Compose:
+
+- [Fluent Bit + VictoriaLogs Single-Node + vmauth](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit-ha-single-node)
+- [Logstash + VictoriaLogs Single-Node + vmauth](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/logstash-ha-single-node)
+- [Vector + VictoriaLogs Single-Node + vmauth](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector-ha-single-node)
+
 ## Backup and restore
 
 VictoriaLogs currently does not have a snapshot feature and a tool like vmbackup as VictoriaMetrics does.

docs/VictoriaLogs/Roadmap.md

@@ -21,11 +21,11 @@ See [these docs](https://docs.victoriametrics.com/victorialogs/) for details.
 The following functionality is planned in the future versions of VictoriaLogs:
 
 - Support for [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/) from popular log collectors and formats:
-  - [ ] [OpenTelemetry for logs](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4839)
+  - [x] [OpenTelemetry for logs](https://docs.victoriametrics.com/victorialogs/data-ingestion/opentelemetry/)
   - [ ] Fluentd
   - [ ] [Journald](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4618) (systemd)
   - [ ] [Datadog protocol for logs](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6632)
-  - [ ] [Telegraf http output](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5310)
+  - [x] [Telegraf](https://docs.victoriametrics.com/victorialogs/data-ingestion/telegraf/)
 - [ ] Integration with Grafana. Partially done, check the [documentation](https://docs.victoriametrics.com/victorialogs/victorialogs-datasource/) and [datasource repository](https://github.com/VictoriaMetrics/victorialogs-datasource).
 - [ ] Ability to make instant snapshots and backups in the way [similar to VictoriaMetrics](https://docs.victoriametrics.com/#how-to-work-with-snapshots).
 - [ ] Cluster version of VictoriaLogs.

docs/VictoriaLogs/data-ingestion/Filebeat.md

@@ -120,4 +120,4 @@ See also:
 - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).
 - [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
 - [Filebeat `output.elasticsearch` docs](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html).
-- [Docker-compose demo for Filebeat integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat-docker).
+- [Docker-compose demo for Filebeat integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/filebeat).

docs/VictoriaLogs/data-ingestion/Fluentbit.md

@@ -145,4 +145,4 @@ See also:
 - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).
 - [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
 - [Fluentbit HTTP output config docs](https://docs.fluentbit.io/manual/pipeline/outputs/http).
-- [Docker-compose demo for Fluentbit integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit-docker).
+- [Docker-compose demo for Fluentbit integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentbit).

docs/VictoriaLogs/data-ingestion/Vector.md

@@ -213,4 +213,4 @@ See also:
 - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).
 - [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
 - [Elasticsearch output docs for Vector](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/).
-- [Docker-compose demo for Filebeat integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector-docker).
+- [Docker-compose demo for Filebeat integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector).

docs/VictoriaLogs/querying/README.md

@@ -13,6 +13,8 @@ VictoriaLogs provides the following HTTP endpoints:
 - [`/select/logsql/query`](#querying-logs) for querying logs.
 - [`/select/logsql/tail`](#live-tailing) for live tailing of query results.
 - [`/select/logsql/hits`](#querying-hits-stats) for querying log hits stats over the given time range.
+- [`/select/logsql/stats_query`](#querying-log-stats) for querying log stats at the given time.
+- [`/select/logsql/stats_query_range`](#querying-log-range-stats) for querying log stats over the given time range.
 - [`/select/logsql/stream_ids`](#querying-stream_ids) for querying `_stream_id` values of [log streams](#https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
 - [`/select/logsql/streams`](#querying-streams) for querying [log streams](#https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
 - [`/select/logsql/stream_field_names`](#querying-stream-field-names) for querying [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) field names.
@@ -105,6 +107,8 @@ See also:
 
 - [Live tailing](#live-tailing)
 - [Querying hits stats](#querying-hits-stats)
+- [Querying log stats](#querying-log-stats)
+- [Querying log range stats](#querying-log-range-stats)
 - [Querying streams](#querying-streams)
 - [Querying stream field names](#querying-stream-field-names)
 - [Querying stream field values](#querying-stream-field-values)
@@ -273,9 +277,175 @@ curl http://localhost:9428/select/logsql/hits -H 'AccountID: 12' -H 'ProjectID:
 See also:
 
 - [Querying logs](#querying-logs)
+- [Querying log stats](#querying-log-stats)
+- [Querying log range stats](#querying-log-range-stats)
 - [Querying streams](#querying-streams)
 - [HTTP API](#http-api)
 
+### Querying log stats
+
+VictoriaLogs provides `/select/logsql/stats_query?query=<query>&time=<t>` HTTP endpoint, which returns log stats
+for the given [`query`](https://docs.victoriametrics.com/victorialogs/logsql/) at the given timestamp `t`
+in the format compatible with [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries).
+
+The `<query>` must contain [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe). The calculated stats is converted into metrics
+with labels from `by(...)` clause of the `| stats by(...)` pipe.
+
+The `<t>` arg can contain values in [any supported format](https://docs.victoriametrics.com/#timestamp-formats).
+If `<t>` is missing, then it equals to the current time.
+
+For example, the following command returns the number of logs per each `level` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
+across logs over `2024-01-01` day by UTC:
+
+```sh
+curl http://localhost:9428/select/logsql/stats_query -d 'query=_time:1d | stats by (level) count(*)' -d 'time=2024-01-02'
+```
+
+Below is an example JSON output returned from this endpoint:
+
+```json
+{
+  "status": "success",
+  "data": {
+    "resultType": "vector",
+    "result": [
+      {
+        "metric": {
+          "__name__": "count(*)",
+          "level": "info"
+        },
+        "value": [
+          1704153600,
+          "20395342"
+        ]
+      },
+      {
+        "metric": {
+          "__name__": "count(*)",
+          "level": "warn"
+        },
+        "value": [
+          1704153600,
+          "1239222"
+        ]
+      },
+      {
+        "metric": {
+          "__name__": "count(*)",
+          "level": "error"
+        },
+        "value": [
+          1704153600,
+          "832"
+        ]
+      },
+    ]
+  }
+}
+```
+
+The `/select/logsql/stats_query` API is useful for generating Prometheus-compatible alerts and calculating recording rules results.
+
+See also:
+
+- [Querying log range stats](#querying-log-range-stats)
+- [Querying logs](#querying-logs)
+- [Querying hits stats](#querying-hits-stats)
+- [HTTP API](#http-api)
+
+### Querying log range stats
+
+VictoriaLogs provides `/select/logsql/stats_query_range?query=<query>&start=<start>&end=<end>&step=<step>` HTTP endpoint, which returns log stats
+for the given [`query`](https://docs.victoriametrics.com/victorialogs/logsql/) on the given `[start ... end]` time range with the given `step` interval.
+The stats is returned in the format compatible with [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries).
+
+The `<query>` must contain [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe). The calculated stats is converted into metrics
+with labels from `by(...)` clause of the `| stats by(...)` pipe.
+
+The `<start>` and `<end>` args can contain values in [any supported format](https://docs.victoriametrics.com/#timestamp-formats).
+If `<start>` is missing, then it equals to the minimum timestamp across logs stored in VictoriaLogs.
+If `<end>` is missing, then it equals to the maximum timestamp across logs stored in VictoriaLogs.
+
+The `<step>` arg can contain values in [the format specified here](https://docs.victoriametrics.com/victorialogs/logsql/#stats-by-time-buckets).
+If `<step>` is missing, then it equals to `1d` (one day).
+
+For example, the following command returns the number of logs per each `level` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
+across logs over `2024-01-01` day by UTC with 6-hour granularity:
+
+```sh
+curl http://localhost:9428/select/logsql/stats_query_range -d 'query=* | stats by (level) count(*)' -d 'start=2024-01-01' -d 'end=2024-01-02' -d 'step=6h'
+```
+
+Below is an example JSON output returned from this endpoint:
+
+```json
+{
+  "status": "success",
+  "data": {
+    "resultType": "matrix",
+    "result": [
+      {
+        "metric": {
+          "__name__": "count(*)",
+          "level": "info"
+        },
+        "values": [
+          [
+            1704067200,
+            "103125"
+          ],
+          [
+            1704088800,
+            "102500"
+          ],
+          [
+            1704110400,
+            "103125"
+          ],
+          [
+            1704132000,
+            "102500"
+          ]
+        ]
+      },
+      {
+        "metric": {
+          "__name__": "count(*)",
+          "level": "error"
+        },
+        "values": [
+          [
+            1704067200,
+            "31"
+          ],
+          [
+            1704088800,
+            "25"
+          ],
+          [
+            1704110400,
+            "31"
+          ],
+          [
+            1704132000,
+            "125"
+          ]
+        ]
+      }
+    ]
+  }
+}
+```
+
+The `/select/logsql/stats_query_range` API is useful for generating Prometheus-compatible graphs in Grafana.
+
+See also:
+
+- [Querying log stats](#querying-log-stats)
+- [Querying logs](#querying-logs)
+- [Querying hits stats](#querying-hits-stats)
+- [HTTP API](#http-api)
+
 ### Querying stream_ids
 
 VictoriaLogs provides `/select/logsql/stream_ids?query=<query>&start=<start>&end=<end>` HTTP endpoint, which returns `_stream_id` values

docs/VictoriaLogs/victorialogs-datasource.md

@@ -95,7 +95,7 @@ docker-compose -f docker-compose.yaml up
 
 After Grafana starts successfully, datasource should be available in the datasources tab
 
-<img src="provision_datasources.png" width="800" alt="Configuration">
+<img src="provision_datasources.webp" width="800" alt="Configuration">
 
 ### Install in Kubernetes
 

docs/changelog/CHANGELOG.md

@@ -20,11 +20,22 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 **Update note 1: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flag. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details.**
 
+* SECURITY: upgrade Go builder from Go1.23.0 to Go1.23.1. See the list of issues addressed in [Go1.23.1](https://github.com/golang/go/issues?q=milestone%3AGo1.23.1+label%3ACherryPickApproved).
+* SECURITY: upgrade base docker image (Alpine) from 3.20.2 to 3.20.3. See [alpine 3.20.3 release notes](https://alpinelinux.org/posts/Alpine-3.17.10-3.18.9-3.19.4-3.20.3-released.html).
+
 * FEATURE [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flags are set. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details.
 * FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag.
+* FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): allow disabling `Bearer` prefix enforcement for authentication header. This is useful for cases when identity token is used instead of access token.
+* FEATURE: [vmgateway](https://docs.victoriametrics.com/vmgateway/): support parting `vm_access` claims in string format. This is useful for cases when identity provider does not support mapping claims to JSON format.
+* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Adds new metrics for data ingestion: `vm_rows_received_by_storage_total`, `vm_rows_ignored_total{reason="nan_value"}`, `vm_rows_ignored_total{reason="invalid_raw_metric_name"}`, `vm_rows_ignored_total{reason="hourly_limit_exceeded"}`, `vm_rows_ignored_total{reason="daily_limit_exceeded"}`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6663) for details.
 
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink` in `Host:Port` format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6912).
 * BUGFIX: [vmagent dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmagent.json): fix legend captions for stream aggregation related panels. Before they were displaying wrong label names.
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway/): add missing `datadog`, `newrelic`, `opentelemetry` and `pushgateway` routes to the `JWT` authorization routes. Allows prefixed (`promtheus/graphite`) routes for query requests.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric `vm_object_references{type="indexdb"}`. Previously, it was overcounted.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ingest stale NaN samples. Previously it could be dropped if series didn't exist at storage node. See this issue [https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069] for details.
+
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): do not send notifications without labels to Alertmanager. Such notifications are rejected by Alertmanager anyway. Before, vmalert could send alert notifications even if no label-value pairs left after applying `alert_relabel_configs` from [notifier config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file).
 
 ## [v1.103.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0)
 

docs/vmagent.md

@@ -199,7 +199,7 @@ to the same `-remoteWrite.url`. In this case you can specify comma-separated lis
 command-line flag. For example, `-remoteWrite.shardByURL.labels=instance,__name__` would shard metrics with the same name and `instance`
 label to the same `-remoteWrite.url`.
 
-Sometimes is may be needed ignoring some labels when sharding samples across multiple `-remoteWrite.url` backends.
+Sometimes, it may be necessary to ignore some labels when sharding samples across multiple `-remoteWrite.url` backends.
 For example, if all the [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) with the same set of labels
 except of `instance` and `pod` labels must be routed to the same backend. In this case the list of ignored labels must be passed to
 `-remoteWrite.shardByURL.ignoreLabels` command-line flag: `-remoteWrite.shardByURL.ignoreLabels=instance,pod`.

docs/vmalert.md

@@ -91,6 +91,8 @@ Then configure `vmalert` accordingly:
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
+> _To validate the syntax of configured rules simply run vmalert with `-rule` and `-dryRun` cmd-line flags._
+
 Note there's a separate `-remoteWrite.url` command-line flag to allow writing results of
 alerting/recording rules into a different storage than the initial data that's
 queried. This allows using `vmalert` to aggregate data from a short-term,
@@ -196,6 +198,7 @@ headers:
 # Optional list of HTTP headers in form `header-name: value`
 # applied for all alert notifications sent to notifiers 
 # generated by rules of this group.
+# It has higher priority over headers defined in notifier config.
 # For example:
 #  notifier_headers:
 #    - "TenantID: foo"

go.mod

@@ -1,68 +1,69 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
-go 1.23.0
+go 1.23.1
 
 require (
 	cloud.google.com/go/storage v1.43.0
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
 	github.com/VictoriaMetrics/easyproto v0.1.4
 	github.com/VictoriaMetrics/fastcache v1.12.2
 	github.com/VictoriaMetrics/metrics v1.35.1
 	github.com/VictoriaMetrics/metricsql v0.77.0
-	github.com/aws/aws-sdk-go-v2 v1.30.3
-	github.com/aws/aws-sdk-go-v2/config v1.27.27
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.9
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
+	github.com/aws/aws-sdk-go-v2 v1.30.5
+	github.com/aws/aws-sdk-go-v2/config v1.27.33
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
 	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cheggaaa/pb/v3 v3.1.5
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
 	github.com/googleapis/gax-go/v2 v2.13.0
-	github.com/influxdata/influxdb v1.11.5
+	github.com/influxdata/influxdb v1.11.6
 	github.com/klauspost/compress v1.17.9
-	github.com/prometheus/prometheus v0.53.1
-	github.com/urfave/cli/v2 v2.27.3
+	github.com/prometheus/prometheus v0.54.1
+	github.com/urfave/cli/v2 v2.27.4
 	github.com/valyala/fastjson v1.6.4
 	github.com/valyala/fastrand v1.1.0
 	github.com/valyala/fasttemplate v1.2.2
 	github.com/valyala/gozstd v1.21.1
 	github.com/valyala/histogram v1.2.0
 	github.com/valyala/quicktemplate v1.8.0
-	golang.org/x/oauth2 v0.21.0
-	golang.org/x/sys v0.23.0
-	google.golang.org/api v0.189.0
+	golang.org/x/net v0.29.0
+	golang.org/x/oauth2 v0.23.0
+	golang.org/x/sys v0.25.0
+	google.golang.org/api v0.196.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	cloud.google.com/go v0.115.0 // indirect
-	cloud.google.com/go/auth v0.7.2 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
+	cloud.google.com/go v0.115.1 // indirect
+	cloud.google.com/go/auth v0.9.3 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.5.0 // indirect
-	cloud.google.com/go/iam v1.1.12 // indirect
+	cloud.google.com/go/iam v1.2.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
-	github.com/aws/aws-sdk-go v1.55.3 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
-	github.com/aws/smithy-go v1.20.3 // indirect
+	github.com/aws/aws-sdk-go v1.55.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
@@ -79,7 +80,7 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
@@ -95,9 +96,9 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_golang v1.20.3 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.59.1 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
@@ -106,30 +107,29 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.12.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.105.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
-	go.opentelemetry.io/otel v1.28.0 // indirect
-	go.opentelemetry.io/otel/metric v1.28.0 // indirect
-	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.14.1 // indirect
+	go.opentelemetry.io/collector/semconv v0.108.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
+	go.opentelemetry.io/otel v1.29.0 // indirect
+	go.opentelemetry.io/otel/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.26.0 // indirect
-	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto v0.0.0-20240725223205-93522f1f2a9f // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect
-	google.golang.org/grpc v1.65.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
+	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/grpc v1.66.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apimachinery v0.30.3 // indirect
-	k8s.io/client-go v0.30.3 // indirect
+	k8s.io/apimachinery v0.31.0 // indirect
+	k8s.io/client-go v0.31.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
+	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
 )

go.sum

@@ -13,12 +13,12 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14=
-cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU=
-cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE=
-cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs=
-cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI=
-cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I=
+cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ=
+cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc=
+cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U=
+cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk=
+cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
+cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -29,10 +29,10 @@ cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJ
 cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw=
-cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg=
-cloud.google.com/go/longrunning v0.5.10 h1:eB/BniENNRKhjz/xgiillrdcH3G74TGSl3BXinGlI7E=
-cloud.google.com/go/longrunning v0.5.10/go.mod h1:tljz5guTr5oc/qhlUjBlk7UAIFMOGuPNxkNDZXlLics=
+cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8=
+cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q=
+cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI=
+cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -45,8 +45,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
 cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
@@ -74,8 +74,6 @@ github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkT
 github.com/VictoriaMetrics/metrics v1.34.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
 github.com/VictoriaMetrics/metrics v1.35.1 h1:o84wtBKQbzLdDy14XeskkCZih6anG+veZ1SwJHFGwrU=
 github.com/VictoriaMetrics/metrics v1.35.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
-github.com/VictoriaMetrics/metricsql v0.76.0 h1:hl7vqJqyH2d8zKImzalkFrkFiD5q4ACF8gl3s86DqKA=
-github.com/VictoriaMetrics/metricsql v0.76.0/go.mod h1:1g4hdCwlbJZ851PU9VN65xy9Rdlzupo6fx3SNZ8Z64U=
 github.com/VictoriaMetrics/metricsql v0.77.0 h1:eD+1RuIBQmbSPdl8ItbghxLifE+gexJxQBWKSJYwhBE=
 github.com/VictoriaMetrics/metricsql v0.77.0/go.mod h1:1g4hdCwlbJZ851PU9VN65xy9Rdlzupo6fx3SNZ8Z64U=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
@@ -92,46 +90,46 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E=
-github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
-github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
-github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
-github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.9 h1:TC2vjvaAv1VNl9A0rm+SeuBjrzXnrlwk6Yop+gKRi38=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.9/go.mod h1:WPv2FRnkIOoDv/8j2gSUsI4qDc7392w5anFB/I89GZ8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
-github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
-github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
-github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
+github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
+github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
+github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU=
+github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18 h1:9DIp7vhmOPmueCDwpXa45bEbLHHTt1kcxChdTJWWxvI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18/go.mod h1:aJv/Fwz8r56ozwYFRC4bzoeL1L17GYQYemfblOBux1M=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -162,12 +160,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
-github.com/digitalocean/godo v1.117.0 h1:WVlTe09melDYTd7VCVyvHcNWbgB+uI1O115+5LOtdSw=
-github.com/digitalocean/godo v1.117.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo=
+github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4=
+github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo=
-github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v27.0.3+incompatible h1:aBGI9TeQ4MPlhquTQKq9XbK79rKFVwXNUAYz9aXyEBE=
+github.com/docker/docker v27.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -179,8 +177,8 @@ github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI=
-github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0=
+github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155 h1:IgJPqnrlY2Mr4pYB6oaMKvFvwJ9H+X6CCY5x1vCTcpc=
+github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
 github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
@@ -192,6 +190,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -296,51 +296,51 @@ github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
-github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0=
+github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
 github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
-github.com/gophercloud/gophercloud v1.12.0 h1:Jrz16vPAL93l80q16fp8NplrTCp93y7rZh2P3Q4Yq7g=
-github.com/gophercloud/gophercloud v1.12.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
+github.com/gophercloud/gophercloud v1.13.0 h1:8iY9d1DAbzMW6Vok1AxbbK5ZaUjzMp0tdyt4fX9IeJ0=
+github.com/gophercloud/gophercloud v1.13.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
 github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
-github.com/hashicorp/consul/api v1.29.1 h1:UEwOjYJrd3lG1x5w7HxDRMGiAUPrb3f103EoeKuuEcc=
-github.com/hashicorp/consul/api v1.29.1/go.mod h1:lumfRkY/coLuqMICkI7Fh3ylMG31mQSRZyef2c5YvJI=
+github.com/hashicorp/consul/api v1.29.2 h1:aYyRn8EdE2mSfG14S1+L9Qkjtz8RzmaWh6AcNGRNwPw=
+github.com/hashicorp/consul/api v1.29.2/go.mod h1:0YObcaLNDSbtlgzIRtmRXI1ZkeuK0trCBxwZQ4MYnIk=
 github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=
 github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
-github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
-github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
 github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d h1:KHq+mAzWSkumj4PDoXc5VZbycPGcmYu8tohgVLQ6SIc=
-github.com/hashicorp/nomad/api v0.0.0-20240604134157-e73d8bb1140d/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
+github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 h1:fgVfQ4AC1avVOnu2cfms8VAiD8lUq3vWI8mTocOXN/w=
+github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
-github.com/hetznercloud/hcloud-go/v2 v2.9.0 h1:s0N6R7Zoi2DPfMtUF5o9VeUBzTtHVY6MIkHOQnfu/AY=
-github.com/hetznercloud/hcloud-go/v2 v2.9.0/go.mod h1:qtW/TuU7Bs16ibXl/ktJarWqU2LwHr7eGlwoilHxtgg=
+github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I=
+github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
-github.com/influxdata/influxdb v1.11.5 h1:+em5VOl6lhAZubXj5o6SobCwvrRs3XDlBx/MUI4schI=
-github.com/influxdata/influxdb v1.11.5/go.mod h1:k8sWREQl1/9t46VrkrH5adUM4UNGIt206ipO3plbkw8=
+github.com/influxdata/influxdb v1.11.6 h1:zS5MRY+RQ5/XFTer5R8xQRnY17JYSbacvO6OaP164wU=
+github.com/influxdata/influxdb v1.11.6/go.mod h1:F10NoQb9qa04lME3pTPWQrYt4JZ/ke1Eei+1ttgHHrg=
 github.com/ionos-cloud/sdk-go/v6 v6.1.11 h1:J/uRN4UWO3wCyGOeDdMKv8LWRzKu6UIkLEaes38Kzh8=
 github.com/ionos-cloud/sdk-go/v6 v6.1.11/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -378,8 +378,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/linode/linodego v1.35.0 h1:rIhUeCHBLEDlkoRnOTwzSGzljQ3ksXwLxacmXnrV+Do=
-github.com/linode/linodego v1.35.0/go.mod h1:JxuhOEAMfSxun6RU5/MgTKH2GGTmFrhKRj3wL1NFin0=
+github.com/linode/linodego v1.37.0 h1:B/2Spzv9jYXzKA+p+GD8fVCNJ7Wuw6P91ZDD9eCkkso=
+github.com/linode/linodego v1.37.0/go.mod h1:L7GXKFD3PoN2xSEtFc04wIXP5WK65O10jYQx0PQISWQ=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -390,8 +390,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
-github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
+github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
+github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -416,14 +416,16 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/ovh/go-ovh v1.5.1 h1:P8O+7H+NQuFK9P/j4sFW5C0fvSS2DnHYGPwdVCp45wI=
-github.com/ovh/go-ovh v1.5.1/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
+github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI=
+github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -431,8 +433,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
+github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -443,8 +445,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
-github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0=
+github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
 github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -453,8 +455,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/prometheus/prometheus v0.53.1 h1:B0xu4VuVTKYrIuBMn/4YSUoIPYxs956qsOfcS4rqCuA=
-github.com/prometheus/prometheus v0.53.1/go.mod h1:RZDkzs+ShMBDkAPQkLEaLBXpjmDcjhNxU2drUVPgKUU=
+github.com/prometheus/prometheus v0.54.1 h1:vKuwQNjnYN2/mDoWfHXDhAsz/68q/dQDb+YbcEqU7MQ=
+github.com/prometheus/prometheus v0.54.1/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -463,8 +465,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27 h1:yGAraK1uUjlhSXgNMIy8o/J4LFNcy7yeipBqt9N9mVg=
-github.com/scaleway/scaleway-sdk-go v1.0.0-beta.27/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29 h1:BkTk4gynLjguayxrYxZoMZjBnAOh7ntQvUkOFmkMqPU=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -486,8 +488,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/urfave/cli/v2 v2.27.3 h1:/POWahRmdh7uztQ3CYnaDddk0Rm90PyOgIxgW2rr41M=
-github.com/urfave/cli/v2 v2.27.3/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
+github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
+github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
@@ -504,6 +506,8 @@ github.com/valyala/quicktemplate v1.8.0 h1:zU0tjbIqTRgKQzFY1L42zq0qR3eh4WoQQdIdq
 github.com/valyala/quicktemplate v1.8.0/go.mod h1:qIqW8/igXt8fdrUln5kOSb+KWMaJ4Y8QUsfd1k6L2jM=
 github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=
 github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
 github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -517,22 +521,22 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/collector/pdata v1.12.0 h1:Xx5VK1p4VO0md8MWm2icwC1MnJ7f8EimKItMWw46BmA=
-go.opentelemetry.io/collector/pdata v1.12.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI=
-go.opentelemetry.io/collector/semconv v0.105.0 h1:8p6dZ3JfxFTjbY38d8xlQGB1TQ3nPUvs+D0RERniZ1g=
-go.opentelemetry.io/collector/semconv v0.105.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
-go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
-go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
-go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
-go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
-go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI=
-go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A=
-go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
-go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
+go.opentelemetry.io/collector/pdata v1.14.1 h1:wXZjtQA7Vy5HFqco+yA95ENyMQU5heBB1IxMHQf6mUk=
+go.opentelemetry.io/collector/pdata v1.14.1/go.mod h1:z1dTjwwtcoXxZx2/nkHysjxMeaxe9pEmYTEr4SMNIx8=
+go.opentelemetry.io/collector/semconv v0.108.1 h1:Txk9tauUnamZaxS5vlf1O0uZ4VD6nioRBR0nX8L/fU4=
+go.opentelemetry.io/collector/semconv v0.108.1/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
+go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
+go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
+go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
+go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
+go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -545,10 +549,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -559,8 +561,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
-golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk=
+golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -581,8 +583,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
-golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -614,18 +616,16 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
-golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
-golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
-golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
+golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -636,8 +636,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -682,29 +680,24 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
-golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
-golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
-golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
+golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
+golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
+golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -747,8 +740,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
-golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -769,8 +762,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI=
-google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8=
+google.golang.org/api v0.196.0 h1:k/RafYqebaIJBO3+SMnfEGtFVlvp5vSgqTUF54UN/zg=
+google.golang.org/api v0.196.0/go.mod h1:g9IL21uGkYgvQ5BZg6BAtoGJQIm8r6EgaAbpNey5wBE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -806,12 +799,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20240725223205-93522f1f2a9f h1:htT2I9bZvGm+110zq8bIErMX+WgBWxCzV3ChwbvnKnc=
-google.golang.org/genproto v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Sk3mLpoDFTAp6R4OvlcUgaG4ISTspKeFsIAXMn9Bm4Y=
-google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f h1:b1Ln/PG8orm0SsBbHZWke8dDp2lrCD4jSmfglFpTZbk=
-google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f h1:RARaIm8pxYuxyNPbBQf5igT7XdOyCNtat1qAT2ZxjU4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
+google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU=
+google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc=
+google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -825,8 +818,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
-google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
+google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c=
+google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -869,18 +862,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ=
-k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04=
-k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
-k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
-k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=
-k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U=
+k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo=
+k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE=
+k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc=
+k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=
+k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
-k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
-k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA=
+k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
@@ -888,5 +881,5 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

lib/httpserver/prometheus.go

@@ -0,0 +1,47 @@
+package httpserver
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// SendPrometheusError sends err to w in Prometheus querying API response format.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview for more details
+func SendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
+	logger.WarnfSkipframes(1, "error in %q: %s", GetRequestURI(r), err)
+
+	w.Header().Set("Content-Type", "application/json")
+	statusCode := http.StatusUnprocessableEntity
+	var esc *ErrorWithStatusCode
+	if errors.As(err, &esc) {
+		statusCode = esc.StatusCode
+	}
+	w.WriteHeader(statusCode)
+
+	var ure *UserReadableError
+	if errors.As(err, &ure) {
+		err = ure
+	}
+	WritePrometheusErrorResponse(w, statusCode, err)
+}
+
+// UserReadableError is a type of error which supposed to be returned to the user without additional context.
+type UserReadableError struct {
+	// Err is the error which needs to be returned to the user.
+	Err error
+}
+
+// Unwrap returns ure.Err.
+//
+// This is used by standard errors package. See https://golang.org/pkg/errors
+func (ure *UserReadableError) Unwrap() error {
+	return ure.Err
+}
+
+// Error satisfies Error interface
+func (ure *UserReadableError) Error() string {
+	return ure.Err.Error()
+}

lib/httpserver/prometheus_error_response.qtpl

@@ -1,7 +1,7 @@
 {% stripspace %}
-ErrorResponse generates error response for /api/v1/query.
+PrometheusErrorResponse generates error response for Prometheus Querying API.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview
-{% func ErrorResponse(statusCode int, err error) %}
+{% func PrometheusErrorResponse(statusCode int, err error) %}
 {
 	"status":"error",
 	"errorType":"{%d statusCode %}",

lib/httpserver/prometheus_error_response.qtpl.go

@@ -0,0 +1,61 @@
+// Code generated by qtc from "prometheus_error_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+// PrometheusErrorResponse generates error response for Prometheus Querying API.See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview
+
+//line lib/httpserver/prometheus_error_response.qtpl:4
+package httpserver
+
+//line lib/httpserver/prometheus_error_response.qtpl:4
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line lib/httpserver/prometheus_error_response.qtpl:4
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line lib/httpserver/prometheus_error_response.qtpl:4
+func StreamPrometheusErrorResponse(qw422016 *qt422016.Writer, statusCode int, err error) {
+//line lib/httpserver/prometheus_error_response.qtpl:4
+	qw422016.N().S(`{"status":"error","errorType":"`)
+//line lib/httpserver/prometheus_error_response.qtpl:7
+	qw422016.N().D(statusCode)
+//line lib/httpserver/prometheus_error_response.qtpl:7
+	qw422016.N().S(`","error":`)
+//line lib/httpserver/prometheus_error_response.qtpl:8
+	qw422016.N().Q(err.Error())
+//line lib/httpserver/prometheus_error_response.qtpl:8
+	qw422016.N().S(`}`)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+}
+
+//line lib/httpserver/prometheus_error_response.qtpl:10
+func WritePrometheusErrorResponse(qq422016 qtio422016.Writer, statusCode int, err error) {
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	StreamPrometheusErrorResponse(qw422016, statusCode, err)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	qt422016.ReleaseWriter(qw422016)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+}
+
+//line lib/httpserver/prometheus_error_response.qtpl:10
+func PrometheusErrorResponse(statusCode int, err error) string {
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	WritePrometheusErrorResponse(qb422016, statusCode, err)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	qs422016 := string(qb422016.B)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/httpserver/prometheus_error_response.qtpl:10
+	return qs422016
+//line lib/httpserver/prometheus_error_response.qtpl:10
+}

lib/logstorage/bloomfilter.go

@@ -72,85 +72,69 @@ func (bf *bloomFilter) mustInit(tokens []string) {
 
 // bloomFilterAdd adds the given tokens to the bloom filter bits
 func bloomFilterAdd(bits []uint64, tokens []string) {
+	hashesCount := len(tokens) * bloomFilterHashesCount
+	a := encoding.GetUint64s(hashesCount)
+	a.A = appendTokensHashes(a.A[:0], tokens)
+
 	maxBits := uint64(len(bits)) * 64
+	for _, h := range a.A {
+		idx := h % maxBits
+		i := idx / 64
+		j := idx % 64
+		mask := uint64(1) << j
+		w := bits[i]
+		if (w & mask) == 0 {
+			bits[i] = w | mask
+		}
+	}
+
+	encoding.PutUint64s(a)
+}
+
+// appendTokensHashes appends hashes for the given tokens to dst and returns the result.
+//
+// the appended hashes can be then passed to bloomFilter.containsAll().
+func appendTokensHashes(dst []uint64, tokens []string) []uint64 {
+	dstLen := len(dst)
+	hashesCount := len(tokens) * bloomFilterHashesCount
+
+	dst = slicesutil.SetLength(dst, dstLen+hashesCount)
+	dst = dst[:dstLen]
+
 	var buf [8]byte
 	hp := (*uint64)(unsafe.Pointer(&buf[0]))
 	for _, token := range tokens {
 		*hp = xxhash.Sum64(bytesutil.ToUnsafeBytes(token))
 		for i := 0; i < bloomFilterHashesCount; i++ {
-			hi := xxhash.Sum64(buf[:])
+			h := xxhash.Sum64(buf[:])
 			(*hp)++
-			idx := hi % maxBits
-			i := idx / 64
-			j := idx % 64
-			mask := uint64(1) << j
-			w := bits[i]
-			if (w & mask) == 0 {
-				bits[i] = w | mask
-			}
+			dst = append(dst, h)
 		}
 	}
+	return dst
 }
 
-// containsAll returns true if bf contains all the given tokens.
-func (bf *bloomFilter) containsAll(tokens []string) bool {
+// containsAll returns true if bf contains all the given tokens hashes generated by appendTokensHashes.
+func (bf *bloomFilter) containsAll(hashes []uint64) bool {
 	bits := bf.bits
 	if len(bits) == 0 {
 		return true
 	}
 	maxBits := uint64(len(bits)) * 64
-	var buf [8]byte
-	hp := (*uint64)(unsafe.Pointer(&buf[0]))
-	for _, token := range tokens {
-		*hp = xxhash.Sum64(bytesutil.ToUnsafeBytes(token))
-		for i := 0; i < bloomFilterHashesCount; i++ {
-			hi := xxhash.Sum64(buf[:])
-			(*hp)++
-			idx := hi % maxBits
-			i := idx / 64
-			j := idx % 64
-			mask := uint64(1) << j
-			w := bits[i]
-			if (w & mask) == 0 {
-				// The token is missing
-				return false
-			}
+	for _, h := range hashes {
+		idx := h % maxBits
+		i := idx / 64
+		j := idx % 64
+		mask := uint64(1) << j
+		w := bits[i]
+		if (w & mask) == 0 {
+			// The token is missing
+			return false
 		}
 	}
 	return true
 }
 
-// containsAny returns true if bf contains at least a single token from the given tokens.
-func (bf *bloomFilter) containsAny(tokens []string) bool {
-	bits := bf.bits
-	if len(bits) == 0 {
-		return true
-	}
-	maxBits := uint64(len(bits)) * 64
-	var buf [8]byte
-	hp := (*uint64)(unsafe.Pointer(&buf[0]))
-nextToken:
-	for _, token := range tokens {
-		*hp = xxhash.Sum64(bytesutil.ToUnsafeBytes(token))
-		for i := 0; i < bloomFilterHashesCount; i++ {
-			hi := xxhash.Sum64(buf[:])
-			(*hp)++
-			idx := hi % maxBits
-			i := idx / 64
-			j := idx % 64
-			mask := uint64(1) << j
-			w := bits[i]
-			if (w & mask) == 0 {
-				// The token is missing. Check the next token
-				continue nextToken
-			}
-		}
-		// It is likely the token exists in the bloom filter
-		return true
-	}
-	return false
-}
-
 func getBloomFilter() *bloomFilter {
 	v := bloomFilterPool.Get()
 	if v == nil {

lib/logstorage/bloomfilter_test.go

@@ -14,13 +14,9 @@ func TestBloomFilter(t *testing.T) {
 		if err := bf.unmarshal(data); err != nil {
 			t.Fatalf("unexpected error when unmarshaling bloom filter: %s", err)
 		}
-		for _, token := range tokens {
-			if !bf.containsAny([]string{token}) {
-				t.Fatalf("bloomFilterContains must return true for the added token %q", token)
-			}
-		}
-		if !bf.containsAll(tokens) {
-			t.Fatalf("bloomFilterContains must return true for the added tokens")
+		tokensHashes := appendTokensHashes(nil, tokens)
+		if !bf.containsAll(tokensHashes) {
+			t.Fatalf("containsAll must return true for the added tokens")
 		}
 	}
 	f(nil)
@@ -72,7 +68,8 @@ func TestBloomFilterFalsePositive(t *testing.T) {
 	falsePositives := 0
 	for i := range tokens {
 		token := fmt.Sprintf("non-existing-token_%d", i)
-		if bf.containsAny([]string{token}) {
+		tokensHashes := appendTokensHashes(nil, []string{token})
+		if bf.containsAll(tokensHashes) {
 			falsePositives++
 		}
 	}

lib/logstorage/filter_and.go

@@ -18,8 +18,9 @@ type filterAnd struct {
 }
 
 type fieldTokens struct {
-	field  string
-	tokens []string
+	field        string
+	tokens       []string
+	tokensHashes []uint64
 }
 
 func (fa *filterAnd) String() string {
@@ -76,16 +77,16 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool {
 		return true
 	}
 
-	for _, fieldTokens := range byFieldTokens {
-		fieldName := fieldTokens.field
-		tokens := fieldTokens.tokens
+	for _, ft := range byFieldTokens {
+		fieldName := ft.field
+		tokens := ft.tokens
 
 		v := bs.csh.getConstColumnValue(fieldName)
 		if v != "" {
-			if !matchStringByAllTokens(v, tokens) {
-				return false
+			if matchStringByAllTokens(v, tokens) {
+				continue
 			}
-			continue
+			return false
 		}
 
 		ch := bs.csh.getColumnHeader(fieldName)
@@ -94,12 +95,12 @@ func (fa *filterAnd) matchBloomFilters(bs *blockSearch) bool {
 		}
 
 		if ch.valueType == valueTypeDict {
-			if !matchDictValuesByAllTokens(ch.valuesDict.values, tokens) {
-				return false
+			if matchDictValuesByAllTokens(ch.valuesDict.values, tokens) {
+				continue
 			}
-			continue
+			return false
 		}
-		if !matchBloomFilterAllTokens(bs, ch, tokens) {
+		if !matchBloomFilterAllTokens(bs, ch, ft.tokensHashes) {
 			return false
 		}
 	}
@@ -112,12 +113,12 @@ func (fa *filterAnd) getByFieldTokens() []fieldTokens {
 	return fa.byFieldTokens
 }
 
-// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554
-// and filter shouldn't return or filter which result in
-// bloom filter execute error interception.
-// detail see: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556#issuecomment-2323643507
 func (fa *filterAnd) initByFieldTokens() {
-	m := make(map[string]map[string]struct{})
+	fa.byFieldTokens = getCommonTokensForAndFilters(fa.filters)
+}
+
+func getCommonTokensForAndFilters(filters []filter) []fieldTokens {
+	m := make(map[string][]string)
 	var fieldNames []string
 
 	mergeFieldTokens := func(fieldName string, tokens []string) {
@@ -126,18 +127,13 @@ func (fa *filterAnd) initByFieldTokens() {
 		}
 
 		fieldName = getCanonicalColumnName(fieldName)
-		mTokens, ok := m[fieldName]
-		if !ok {
+		if _, ok := m[fieldName]; !ok {
 			fieldNames = append(fieldNames, fieldName)
-			mTokens = make(map[string]struct{})
-			m[fieldName] = mTokens
-		}
-		for _, token := range tokens {
-			mTokens[token] = struct{}{}
 		}
+		m[fieldName] = append(m[fieldName], tokens...)
 	}
 
-	for _, f := range fa.filters {
+	for _, f := range filters {
 		switch t := f.(type) {
 		case *filterExact:
 			tokens := t.getTokens()
@@ -157,24 +153,35 @@ func (fa *filterAnd) initByFieldTokens() {
 		case *filterSequence:
 			tokens := t.getTokens()
 			mergeFieldTokens(t.fieldName, tokens)
+		case *filterOr:
+			bfts := t.getByFieldTokens()
+			for _, bft := range bfts {
+				mergeFieldTokens(bft.field, bft.tokens)
+			}
 		}
 	}
 
 	var byFieldTokens []fieldTokens
 	for _, fieldName := range fieldNames {
 		mTokens := m[fieldName]
+		seenTokens := make(map[string]struct{})
 		tokens := make([]string, 0, len(mTokens))
-		for token := range mTokens {
+		for _, token := range mTokens {
+			if _, ok := seenTokens[token]; ok {
+				continue
+			}
+			seenTokens[token] = struct{}{}
 			tokens = append(tokens, token)
 		}
 
 		byFieldTokens = append(byFieldTokens, fieldTokens{
-			field:  fieldName,
-			tokens: tokens,
+			field:        fieldName,
+			tokens:       tokens,
+			tokensHashes: appendTokensHashes(nil, tokens),
 		})
 	}
 
-	fa.byFieldTokens = byFieldTokens
+	return byFieldTokens
 }
 
 func matchStringByAllTokens(v string, tokens []string) bool {

lib/logstorage/filter_and_test.go

@@ -1,6 +1,7 @@
 package logstorage
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -25,115 +26,142 @@ func TestFilterAnd(t *testing.T) {
 		},
 	}
 
-	//non-empty intersection
-	fa := &filterAnd{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "a",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "abc",
-			},
-		},
+	f := func(qStr string, expectedRowIdxs []int) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("unexpected error in ParseQuery: %s", err)
+		}
+		testFilterMatchForColumns(t, columns, q.f, "foo", expectedRowIdxs)
 	}
-	testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6})
+
+	// non-empty intersection
+	f(`foo:a AND foo:abc*`, []int{2, 6})
 
 	// reverse non-empty intersection
-	fa = &filterAnd{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "abc",
-			},
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "a",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fa, "foo", []int{2, 6})
+	f(`foo:abc* AND foo:a`, []int{2, 6})
 
 	// the first filter mismatch
-	fa = &filterAnd{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "bc",
-			},
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "a",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fa, "foo", nil)
+	f(`foo:bc* AND foo:a`, nil)
 
 	// the last filter mismatch
-	fa = &filterAnd{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "abc",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "foo",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fa, "foo", nil)
+	f(`foo:abc AND foo:foo*`, nil)
 
 	// empty intersection
-	fa = &filterAnd{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "foo",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "abc",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fa, "foo", nil)
+	f(`foo:foo AND foo:abc*`, nil)
+	f(`foo:abc* AND foo:foo`, nil)
 
-	// reverse empty intersection
-	fa = &filterAnd{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "abc",
-			},
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "foo",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fa, "foo", nil)
+	// empty value
+	f(`foo:"" AND bar:""`, []int{5})
 
-	fa = &filterAnd{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "a foo",
-			},
-			&filterOr{
-				filters: []filter{
-					&filterExact{
-						fieldName: "foo",
-						value:     "a foobar",
-					},
-					&filterExact{
-						fieldName: "boo",
-						value:     "bbbbbbb",
-					},
-				},
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fa, "foo", []int{1})
+	// non-existing field with empty value
+	f(`foo:foo* AND bar:""`, []int{0, 1, 3, 4, 6})
+	f(`bar:"" AND foo:foo*`, []int{0, 1, 3, 4, 6})
+
+	// non-existing field with non-empty value
+	f(`foo:foo* AND bar:*`, nil)
+	f(`bar:* AND foo:foo*`, nil)
+
+	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554
+	f(`foo:"a foo"* AND (foo:="a foobar" OR boo:bbbbbbb)`, []int{1})
+
+	f(`foo:"a foo"* AND (foo:"abcd foobar" OR foo:foobar)`, []int{1, 6})
+	f(`(foo:foo* OR bar:baz) AND (bar:x OR foo:a)`, []int{0, 1, 3, 4, 6})
+	f(`(foo:foo* OR bar:baz) AND (bar:x OR foo:xyz)`, nil)
+	f(`(foo:foo* OR bar:baz) AND (bar:* OR foo:xyz)`, nil)
+	f(`(foo:foo* OR bar:baz) AND (bar:"" OR foo:xyz)`, []int{0, 1, 3, 4, 6})
+
+	// negative filters
+	f(`foo:foo* AND !foo:~bar`, []int{0})
+	f(`foo:foo* AND foo:!~bar`, []int{0})
+}
+
+func TestGetCommonTokensForAndFilters(t *testing.T) {
+	f := func(qStr string, tokensExpected []fieldTokens) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("unexpected error in ParseQuery: %s", err)
+		}
+		fa, ok := q.f.(*filterAnd)
+		if !ok {
+			t.Fatalf("unexpected filter type: %T; want *filterAnd", q.f)
+		}
+		tokens := getCommonTokensForAndFilters(fa.filters)
+
+		if len(tokens) != len(tokensExpected) {
+			t.Fatalf("unexpected len(tokens); got %d; want %d\ntokens\n%#v\ntokensExpected\n%#v", len(tokens), len(tokensExpected), tokens, tokensExpected)
+		}
+		for i, ft := range tokens {
+			ftExpected := tokensExpected[i]
+			if ft.field != ftExpected.field {
+				t.Fatalf("unexpected field; got %q; want %q\ntokens\n%q\ntokensExpected\n%q", ft.field, ftExpected.field, ft.tokens, ftExpected.tokens)
+			}
+			if !reflect.DeepEqual(ft.tokens, ftExpected.tokens) {
+				t.Fatalf("unexpected tokens for field %q; got %q; want %q", ft.field, ft.tokens, ftExpected.tokens)
+			}
+		}
+	}
+
+	f(`foo AND bar`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo", "bar"},
+		},
+	})
+
+	f(`="foo bar" AND ="a foo"* AND "bar foo" AND "foo bar"* AND ~"foo qwe bar.+" AND seq(x, bar, "foo qwe")`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo", "bar", "a", "qwe", "x"},
+		},
+	})
+
+	// extract common tokens from OR filters
+	f(`foo AND (bar OR ~"x bar baz")`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo", "bar"},
+		},
+	})
+
+	// star matches any non-empty token, so it is skipped
+	f(`foo bar *`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo", "bar"},
+		},
+	})
+	f(`* *`, nil)
+
+	// empty filter must be skipped
+	f(`foo "" bar`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo", "bar"},
+		},
+	})
+	f(`"" ""`, nil)
+
+	// unknown filters must be skipped
+	f(`_time:5m !foo "bar baz" x`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"bar", "baz", "x"},
+		},
+	})
+
+	// distinct field names
+	f(`foo:x bar:"a bc" (foo:y OR (bar:qwe AND foo:"z y a"))`, []fieldTokens{
+		{
+			field:  "foo",
+			tokens: []string{"x", "y"},
+		},
+		{
+			field:  "bar",
+			tokens: []string{"a", "bc"},
+		},
+	})
 }

lib/logstorage/filter_any_case_phrase.go

@@ -24,11 +24,9 @@ type filterAnyCasePhrase struct {
 	phraseUppercaseOnce sync.Once
 	phraseUppercase     string
 
-	tokensOnce sync.Once
-	tokens     []string
-
-	tokensUppercaseOnce sync.Once
-	tokensUppercase     []string
+	tokensOnce            sync.Once
+	tokensHashes          []uint64
+	tokensHashesUppercase []uint64
 }
 
 func (fp *filterAnyCasePhrase) String() string {
@@ -39,27 +37,25 @@ func (fp *filterAnyCasePhrase) updateNeededFields(neededFields fieldsSet) {
 	neededFields.add(fp.fieldName)
 }
 
-func (fp *filterAnyCasePhrase) getTokens() []string {
+func (fp *filterAnyCasePhrase) getTokensHashes() []uint64 {
 	fp.tokensOnce.Do(fp.initTokens)
-	return fp.tokens
+	return fp.tokensHashes
+}
+
+func (fp *filterAnyCasePhrase) getTokensHashesUppercase() []uint64 {
+	fp.tokensOnce.Do(fp.initTokens)
+	return fp.tokensHashesUppercase
 }
 
 func (fp *filterAnyCasePhrase) initTokens() {
-	fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
-}
+	tokens := tokenizeStrings(nil, []string{fp.phrase})
+	fp.tokensHashes = appendTokensHashes(nil, tokens)
 
-func (fp *filterAnyCasePhrase) getTokensUppercase() []string {
-	fp.tokensUppercaseOnce.Do(fp.initTokensUppercase)
-	return fp.tokensUppercase
-}
-
-func (fp *filterAnyCasePhrase) initTokensUppercase() {
-	tokens := fp.getTokens()
 	tokensUppercase := make([]string, len(tokens))
 	for i, token := range tokens {
 		tokensUppercase[i] = strings.ToUpper(token)
 	}
-	fp.tokensUppercase = tokensUppercase
+	fp.tokensHashesUppercase = appendTokensHashes(nil, tokensUppercase)
 }
 
 func (fp *filterAnyCasePhrase) getPhraseLowercase() string {
@@ -109,7 +105,7 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fp.getTokens()
+	tokens := fp.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -130,7 +126,7 @@ func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens)
 	case valueTypeTimestampISO8601:
 		phraseUppercase := fp.getPhraseUppercase()
-		tokensUppercase := fp.getTokensUppercase()
+		tokensUppercase := fp.getTokensHashesUppercase()
 		matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokensUppercase)
 	default:
 		logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)

lib/logstorage/filter_any_case_prefix.go

@@ -25,11 +25,9 @@ type filterAnyCasePrefix struct {
 	prefixUppercaseOnce sync.Once
 	prefixUppercase     string
 
-	tokensOnce sync.Once
-	tokens     []string
-
-	tokensUppercaseOnce sync.Once
-	tokensUppercase     []string
+	tokensOnce            sync.Once
+	tokensHashes          []uint64
+	tokensUppercaseHashes []uint64
 }
 
 func (fp *filterAnyCasePrefix) String() string {
@@ -43,27 +41,25 @@ func (fp *filterAnyCasePrefix) updateNeededFields(neededFields fieldsSet) {
 	neededFields.add(fp.fieldName)
 }
 
-func (fp *filterAnyCasePrefix) getTokens() []string {
+func (fp *filterAnyCasePrefix) getTokensHashes() []uint64 {
 	fp.tokensOnce.Do(fp.initTokens)
-	return fp.tokens
+	return fp.tokensHashes
+}
+
+func (fp *filterAnyCasePrefix) getTokensUppercaseHashes() []uint64 {
+	fp.tokensOnce.Do(fp.initTokens)
+	return fp.tokensUppercaseHashes
 }
 
 func (fp *filterAnyCasePrefix) initTokens() {
-	fp.tokens = getTokensSkipLast(fp.prefix)
-}
+	tokens := getTokensSkipLast(fp.prefix)
+	fp.tokensHashes = appendTokensHashes(nil, tokens)
 
-func (fp *filterAnyCasePrefix) getTokensUppercase() []string {
-	fp.tokensUppercaseOnce.Do(fp.initTokensUppercase)
-	return fp.tokensUppercase
-}
-
-func (fp *filterAnyCasePrefix) initTokensUppercase() {
-	tokens := fp.getTokens()
 	tokensUppercase := make([]string, len(tokens))
 	for i, token := range tokens {
 		tokensUppercase[i] = strings.ToUpper(token)
 	}
-	fp.tokensUppercase = tokensUppercase
+	fp.tokensUppercaseHashes = appendTokensHashes(nil, tokensUppercase)
 }
 
 func (fp *filterAnyCasePrefix) getPrefixLowercase() string {
@@ -110,7 +106,7 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fp.getTokens()
+	tokens := fp.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -131,7 +127,7 @@ func (fp *filterAnyCasePrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		matchIPv4ByPrefix(bs, ch, bm, prefixLowercase, tokens)
 	case valueTypeTimestampISO8601:
 		prefixUppercase := fp.getPrefixUppercase()
-		tokensUppercase := fp.getTokensUppercase()
+		tokensUppercase := fp.getTokensUppercaseHashes()
 		matchTimestampISO8601ByPrefix(bs, ch, bm, prefixUppercase, tokensUppercase)
 	default:
 		logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)

lib/logstorage/filter_exact.go

@@ -16,8 +16,9 @@ type filterExact struct {
 	fieldName string
 	value     string
 
-	tokensOnce sync.Once
-	tokens     []string
+	tokensOnce   sync.Once
+	tokens       []string
+	tokensHashes []uint64
 }
 
 func (fe *filterExact) String() string {
@@ -33,8 +34,14 @@ func (fe *filterExact) getTokens() []string {
 	return fe.tokens
 }
 
+func (fe *filterExact) getTokensHashes() []uint64 {
+	fe.tokensOnce.Do(fe.initTokens)
+	return fe.tokensHashes
+}
+
 func (fe *filterExact) initTokens() {
 	fe.tokens = tokenizeStrings(nil, []string{fe.value})
+	fe.tokensHashes = appendTokensHashes(nil, fe.tokens)
 }
 
 func (fe *filterExact) applyToBlockResult(br *blockResult, bm *bitmap) {
@@ -186,7 +193,7 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fe.getTokens()
+	tokens := fe.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -212,7 +219,7 @@ func (fe *filterExact) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchTimestampISO8601ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
+func matchTimestampISO8601ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []uint64) {
 	n, ok := tryParseTimestampISO8601(value)
 	if !ok || n < int64(ch.minValue) || n > int64(ch.maxValue) {
 		bm.resetBits()
@@ -224,7 +231,7 @@ func matchTimestampISO8601ByExactValue(bs *blockSearch, ch *columnHeader, bm *bi
 	bbPool.Put(bb)
 }
 
-func matchIPv4ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
+func matchIPv4ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []uint64) {
 	n, ok := tryParseIPv4(value)
 	if !ok || uint64(n) < ch.minValue || uint64(n) > ch.maxValue {
 		bm.resetBits()
@@ -236,7 +243,7 @@ func matchIPv4ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value
 	bbPool.Put(bb)
 }
 
-func matchFloat64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
+func matchFloat64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []uint64) {
 	f, ok := tryParseFloat64(value)
 	if !ok || f < math.Float64frombits(ch.minValue) || f > math.Float64frombits(ch.maxValue) {
 		bm.resetBits()
@@ -262,7 +269,7 @@ func matchValuesDictByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap,
 	bbPool.Put(bb)
 }
 
-func matchStringByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []string) {
+func matchStringByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, value string, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -272,7 +279,7 @@ func matchStringByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, valu
 	})
 }
 
-func matchUint8ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchUint8ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	n, ok := tryParseUint64(phrase)
 	if !ok || n < ch.minValue || n > ch.maxValue {
 		bm.resetBits()
@@ -284,7 +291,7 @@ func matchUint8ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phras
 	bbPool.Put(bb)
 }
 
-func matchUint16ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchUint16ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	n, ok := tryParseUint64(phrase)
 	if !ok || n < ch.minValue || n > ch.maxValue {
 		bm.resetBits()
@@ -296,7 +303,7 @@ func matchUint16ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phra
 	bbPool.Put(bb)
 }
 
-func matchUint32ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchUint32ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	n, ok := tryParseUint64(phrase)
 	if !ok || n < ch.minValue || n > ch.maxValue {
 		bm.resetBits()
@@ -308,7 +315,7 @@ func matchUint32ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phra
 	bbPool.Put(bb)
 }
 
-func matchUint64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchUint64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	n, ok := tryParseUint64(phrase)
 	if !ok || n < ch.minValue || n > ch.maxValue {
 		bm.resetBits()
@@ -320,7 +327,7 @@ func matchUint64ByExactValue(bs *blockSearch, ch *columnHeader, bm *bitmap, phra
 	bbPool.Put(bb)
 }
 
-func matchBinaryValue(bs *blockSearch, ch *columnHeader, bm *bitmap, binValue []byte, tokens []string) {
+func matchBinaryValue(bs *blockSearch, ch *columnHeader, bm *bitmap, binValue []byte, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return

lib/logstorage/filter_exact_prefix.go

@@ -15,8 +15,9 @@ type filterExactPrefix struct {
 	fieldName string
 	prefix    string
 
-	tokensOnce sync.Once
-	tokens     []string
+	tokensOnce   sync.Once
+	tokens       []string
+	tokensHashes []uint64
 }
 
 func (fep *filterExactPrefix) String() string {
@@ -32,8 +33,14 @@ func (fep *filterExactPrefix) getTokens() []string {
 	return fep.tokens
 }
 
+func (fep *filterExactPrefix) getTokensHashes() []uint64 {
+	fep.tokensOnce.Do(fep.initTokens)
+	return fep.tokensHashes
+}
+
 func (fep *filterExactPrefix) initTokens() {
 	fep.tokens = getTokensSkipLast(fep.prefix)
+	fep.tokensHashes = appendTokensHashes(nil, fep.tokens)
 }
 
 func (fep *filterExactPrefix) applyToBlockResult(br *blockResult, bm *bitmap) {
@@ -62,7 +69,7 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fep.getTokens()
+	tokens := fep.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -88,7 +95,7 @@ func (fep *filterExactPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if prefix == "" {
 		return
 	}
@@ -105,11 +112,11 @@ func matchTimestampISO8601ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *b
 	bbPool.Put(bb)
 }
 
-func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if prefix == "" {
 		return
 	}
-	if prefix < "0" || prefix > "9" || len(tokens) > 3 || !matchBloomFilterAllTokens(bs, ch, tokens) {
+	if prefix < "0" || prefix > "9" || len(tokens) > 3*bloomFilterHashesCount || !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
 	}
@@ -122,12 +129,12 @@ func matchIPv4ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefi
 	bbPool.Put(bb)
 }
 
-func matchFloat64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchFloat64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if prefix == "" {
 		// An empty prefix matches all the values
 		return
 	}
-	if len(tokens) > 2 || !matchBloomFilterAllTokens(bs, ch, tokens) {
+	if len(tokens) > 2*bloomFilterHashesCount || !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
 	}
@@ -153,7 +160,7 @@ func matchValuesDictByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap,
 	bbPool.Put(bb)
 }
 
-func matchStringByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchStringByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -163,7 +170,7 @@ func matchStringByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pre
 	})
 }
 
-func matchUint8ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchUint8ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
 		return
 	}
@@ -176,7 +183,7 @@ func matchUint8ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pref
 	bbPool.Put(bb)
 }
 
-func matchUint16ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchUint16ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
 		return
 	}
@@ -189,7 +196,7 @@ func matchUint16ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pre
 	bbPool.Put(bb)
 }
 
-func matchUint32ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchUint32ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
 		return
 	}
@@ -202,7 +209,7 @@ func matchUint32ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pre
 	bbPool.Put(bb)
 }
 
-func matchUint64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchUint64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if !matchMinMaxExactPrefix(ch, bm, prefix, tokens) {
 		return
 	}
@@ -215,7 +222,7 @@ func matchUint64ByExactPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pre
 	bbPool.Put(bb)
 }
 
-func matchMinMaxExactPrefix(ch *columnHeader, bm *bitmap, prefix string, tokens []string) bool {
+func matchMinMaxExactPrefix(ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) bool {
 	if prefix == "" {
 		// An empty prefix matches all the values
 		return false

lib/logstorage/filter_in.go

@@ -28,9 +28,9 @@ type filterIn struct {
 	// qFieldName must be set to field name for obtaining values from if q is non-nil.
 	qFieldName string
 
-	tokensOnce   sync.Once
-	commonTokens []string
-	tokenSets    [][]string
+	tokensOnce         sync.Once
+	commonTokensHashes []uint64
+	tokenSetsHashes    [][]uint64
 
 	stringValuesOnce sync.Once
 	stringValues     map[string]struct{}
@@ -76,16 +76,21 @@ func (fi *filterIn) updateNeededFields(neededFields fieldsSet) {
 	neededFields.add(fi.fieldName)
 }
 
-func (fi *filterIn) getTokens() ([]string, [][]string) {
+func (fi *filterIn) getTokensHashes() ([]uint64, [][]uint64) {
 	fi.tokensOnce.Do(fi.initTokens)
-	return fi.commonTokens, fi.tokenSets
+	return fi.commonTokensHashes, fi.tokenSetsHashes
 }
 
 func (fi *filterIn) initTokens() {
 	commonTokens, tokenSets := getCommonTokensAndTokenSets(fi.values)
 
-	fi.commonTokens = commonTokens
-	fi.tokenSets = tokenSets
+	fi.commonTokensHashes = appendTokensHashes(nil, commonTokens)
+
+	tokenSetsHashes := make([][]uint64, len(tokenSets))
+	for i, tokens := range tokenSets {
+		tokenSetsHashes[i] = appendTokensHashes(nil, tokens)
+	}
+	fi.tokenSetsHashes = tokenSetsHashes
 }
 
 func (fi *filterIn) getStringValues() map[string]struct{} {
@@ -374,7 +379,7 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	commonTokens, tokenSets := fi.getTokens()
+	commonTokens, tokenSets := fi.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -409,7 +414,7 @@ func (fi *filterIn) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}, commonTokens []string, tokenSets [][]string) {
+func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[string]struct{}, commonTokens []uint64, tokenSets [][]uint64) {
 	if len(values) == 0 {
 		bm.resetBits()
 		return
@@ -424,7 +429,7 @@ func matchAnyValue(bs *blockSearch, ch *columnHeader, bm *bitmap, values map[str
 	})
 }
 
-func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, commonTokens []string, tokenSets [][]string) bool {
+func matchBloomFilterAnyTokenSet(bs *blockSearch, ch *columnHeader, commonTokens []uint64, tokenSets [][]uint64) bool {
 	if len(commonTokens) > 0 {
 		if !matchBloomFilterAllTokens(bs, ch, commonTokens) {
 			return false
@@ -511,6 +516,9 @@ func getCommonTokens(tokenSets [][]string) []string {
 			}
 		}
 	}
+	if len(m) == 0 {
+		return nil
+	}
 
 	tokens := make([]string, 0, len(m))
 	for token := range m {

lib/logstorage/filter_or.go

@@ -89,9 +89,9 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool {
 		return true
 	}
 
-	for _, fieldTokens := range byFieldTokens {
-		fieldName := fieldTokens.field
-		tokens := fieldTokens.tokens
+	for _, ft := range byFieldTokens {
+		fieldName := ft.field
+		tokens := ft.tokens
 
 		v := bs.csh.getConstColumnValue(fieldName)
 		if v != "" {
@@ -112,7 +112,7 @@ func (fo *filterOr) matchBloomFilters(bs *blockSearch) bool {
 			}
 			continue
 		}
-		if matchBloomFilterAllTokens(bs, ch, tokens) {
+		if matchBloomFilterAllTokens(bs, ch, ft.tokensHashes) {
 			return true
 		}
 	}
@@ -126,6 +126,10 @@ func (fo *filterOr) getByFieldTokens() []fieldTokens {
 }
 
 func (fo *filterOr) initByFieldTokens() {
+	fo.byFieldTokens = getCommonTokensForOrFilters(fo.filters)
+}
+
+func getCommonTokensForOrFilters(filters []filter) []fieldTokens {
 	m := make(map[string][][]string)
 	var fieldNames []string
 
@@ -141,7 +145,7 @@ func (fo *filterOr) initByFieldTokens() {
 		m[fieldName] = append(m[fieldName], tokens)
 	}
 
-	for _, f := range fo.filters {
+	for _, f := range filters {
 		switch t := f.(type) {
 		case *filterExact:
 			tokens := t.getTokens()
@@ -166,19 +170,30 @@ func (fo *filterOr) initByFieldTokens() {
 			for _, bft := range bfts {
 				mergeFieldTokens(bft.field, bft.tokens)
 			}
+		default:
+			// Cannot extract tokens from this filter. This means that it is impossible to extract common tokens from OR filters.
+			return nil
 		}
 	}
 
 	var byFieldTokens []fieldTokens
 	for _, fieldName := range fieldNames {
-		commonTokens := getCommonTokens(m[fieldName])
-		if len(commonTokens) > 0 {
-			byFieldTokens = append(byFieldTokens, fieldTokens{
-				field:  fieldName,
-				tokens: commonTokens,
-			})
+		tokenss := m[fieldName]
+		if len(tokenss) != len(filters) {
+			// The filter for the given fieldName is missing in some OR filters,
+			// so it is impossible to extract common tokens from these filters.
+			continue
 		}
+		commonTokens := getCommonTokens(tokenss)
+		if len(commonTokens) == 0 {
+			continue
+		}
+		byFieldTokens = append(byFieldTokens, fieldTokens{
+			field:        fieldName,
+			tokens:       commonTokens,
+			tokensHashes: appendTokensHashes(nil, commonTokens),
+		})
 	}
 
-	fo.byFieldTokens = byFieldTokens
+	return byFieldTokens
 }

lib/logstorage/filter_or_test.go

@@ -1,6 +1,7 @@
 package logstorage
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -25,108 +26,145 @@ func TestFilterOr(t *testing.T) {
 		},
 	}
 
-	// non-empty union
-	fo := &filterOr{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "23",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "abc",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fo, "foo", []int{2, 6, 9})
+	f := func(qStr string, expectedRowIdxs []int) {
+		t.Helper()
 
-	// reverse non-empty union
-	fo = &filterOr{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "abc",
-			},
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "23",
-			},
-		},
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("unexpected error in ParseQuery: %s", err)
+		}
+		testFilterMatchForColumns(t, columns, q.f, "foo", expectedRowIdxs)
 	}
-	testFilterMatchForColumns(t, columns, fo, "foo", []int{2, 6, 9})
+
+	// non-empty union
+	f(`foo:23 OR foo:abc*`, []int{2, 6, 9})
+	f(`foo:abc* OR foo:23`, []int{2, 6, 9})
 
 	// first empty result, second non-empty result
-	fo = &filterOr{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "xabc",
-			},
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "23",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fo, "foo", []int{9})
+	f(`foo:xabc* OR foo:23`, []int{9})
 
 	// first non-empty result, second empty result
-	fo = &filterOr{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "23",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "xabc",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fo, "foo", []int{9})
+	f(`foo:23 OR foo:xabc*`, []int{9})
 
 	// first match all
-	fo = &filterOr{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "a",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "23",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fo, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
+	f(`foo:a OR foo:23`, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
 
 	// second match all
-	fo = &filterOr{
-		filters: []filter{
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "23",
-			},
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "a",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fo, "foo", []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
+	f(`foo:23 OR foo:a`, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
 
 	// both empty results
-	fo = &filterOr{
-		filters: []filter{
-			&filterPhrase{
-				fieldName: "foo",
-				phrase:    "x23",
-			},
-			&filterPrefix{
-				fieldName: "foo",
-				prefix:    "xabc",
-			},
-		},
-	}
-	testFilterMatchForColumns(t, columns, fo, "foo", nil)
+	f(`foo:x23 OR foo:xabc`, nil)
+
+	// non-existing column (last)
+	f(`foo:23 OR bar:xabc*`, []int{9})
+
+	// non-existing column (first)
+	f(`bar:xabc* OR foo:23`, []int{9})
+
+	f(`(foo:23 AND bar:"") OR (foo:foo AND bar:*)`, []int{9})
+	f(`(foo:23 AND bar:"") OR (foo:foo AND bar:"")`, []int{0, 9})
+	f(`(foo:23 AND bar:"") OR (foo:foo AND baz:"")`, []int{0, 9})
+	f(`(foo:23 AND bar:abc) OR (foo:foo AND bar:"")`, []int{0})
+	f(`(foo:23 AND bar:abc) OR (foo:foo AND bar:*)`, nil)
+
+	// negative filter
+	f(`foo:baz or !foo:~foo`, []int{2, 3, 5, 7, 8, 9})
+	f(`foo:baz or foo:!~foo`, []int{2, 3, 5, 7, 8, 9})
+}
+
+func TestGetCommonTokensForOrFilters(t *testing.T) {
+	f := func(qStr string, tokensExpected []fieldTokens) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("unexpected error in ParseQuery: %s", err)
+		}
+		fo, ok := q.f.(*filterOr)
+		if !ok {
+			t.Fatalf("unexpected filter type: %T; want *filterOr", q.f)
+		}
+		tokens := getCommonTokensForOrFilters(fo.filters)
+
+		if len(tokens) != len(tokensExpected) {
+			t.Fatalf("unexpected len(tokens); got %d; want %d\ntokens\n%#v\ntokensExpected\n%#v", len(tokens), len(tokensExpected), tokens, tokensExpected)
+		}
+		for i, ft := range tokens {
+			ftExpected := tokensExpected[i]
+			if ft.field != ftExpected.field {
+				t.Fatalf("unexpected field; got %q; want %q\ntokens\n%q\ntokensExpected\n%q", ft.field, ftExpected.field, ft.tokens, ftExpected.tokens)
+			}
+			if !reflect.DeepEqual(ft.tokens, ftExpected.tokens) {
+				t.Fatalf("unexpected tokens for field %q; got %q; want %q", ft.field, ft.tokens, ftExpected.tokens)
+			}
+		}
+	}
+
+	// no common tokens
+	f(`foo OR bar`, nil)
+
+	// star filter matches non-empty value; it is skipped, since one of OR filters may contain an empty filter
+	f(`* OR foo`, nil)
+	f(`foo or *`, nil)
+	f(`* or *`, nil)
+	f(`"" or * or foo`, nil)
+
+	// empty filter suppresses all the common tokens.
+	f(`"" or foo or "foo bar"`, nil)
+	f(`foo or ""`, nil)
+	f(`"" or ""`, nil)
+
+	// common foo token
+	f(`foo OR "foo bar" OR ="a foo" OR ="foo bar"* OR "bar foo "* OR seq("bar foo", "baz") OR ~"a.+ foo bar"`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo"},
+		},
+	})
+
+	// regexp ending on foo token doesn't contain foo token, since it may match foobar.
+	f(`foo OR "foo bar" OR ="a foo" OR ="foo bar"* OR "bar foo "* OR seq("bar foo", "baz") OR ~"a.+ foo"`, nil)
+
+	// regexp starting from foo token doesn't contain foo token, since it may match barfoo.
+	f(`foo OR "foo bar" OR ="a foo" OR ="foo bar"* OR "bar foo "* OR seq("bar foo", "baz") OR ~"foo bar"`, nil)
+
+	// prefix filter ending on foo doesn't contain foo token, since it may match foobar.
+	f(`foo OR "foo bar" OR ="a foo" OR ="foo bar"* OR "bar foo"* OR seq("bar foo", "baz") OR ~"a.+ foo bar"`, nil)
+
+	// bar and foo are common tokens
+	f(`"bar foo baz" OR (foo AND "bar x" AND foobar)`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"bar", "foo"},
+		},
+	})
+
+	// bar and foo are common tokens, x:foobar should be ignored, since it doesn't present in every OR filter
+	f(`"bar foo baz" OR (foo AND "bar x" AND x:foobar)`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"bar", "foo"},
+		},
+	})
+
+	// common tokens for distinct fields
+	f(`(foo AND x:a) OR (x:"b a c" AND ~"aaa foo ")`, []fieldTokens{
+		{
+			field:  "_msg",
+			tokens: []string{"foo"},
+		},
+		{
+			field:  "x",
+			tokens: []string{"a"},
+		},
+	})
+
+	// zero common tokens for distinct fields
+	f(`(foo AND x:a) OR (x:"b c" AND ~"aaa foo" AND y:z)`, nil)
+
+	// negative filter removes all the matching
+	f(`foo OR !"foo bar"`, nil)
+
+	// time filter removes all the matching
+	f(`_time:5m or foo`, nil)
 }

lib/logstorage/filter_phrase.go

@@ -24,8 +24,9 @@ type filterPhrase struct {
 	fieldName string
 	phrase    string
 
-	tokensOnce sync.Once
-	tokens     []string
+	tokensOnce   sync.Once
+	tokens       []string
+	tokensHashes []uint64
 }
 
 func (fp *filterPhrase) String() string {
@@ -41,8 +42,14 @@ func (fp *filterPhrase) getTokens() []string {
 	return fp.tokens
 }
 
+func (fp *filterPhrase) getTokensHashes() []uint64 {
+	fp.tokensOnce.Do(fp.initTokens)
+	return fp.tokensHashes
+}
+
 func (fp *filterPhrase) initTokens() {
 	fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
+	fp.tokensHashes = appendTokensHashes(nil, fp.tokens)
 }
 
 func (fp *filterPhrase) applyToBlockResult(br *blockResult, bm *bitmap) {
@@ -73,7 +80,7 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fp.getTokens()
+	tokens := fp.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -99,7 +106,7 @@ func (fp *filterPhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchTimestampISO8601ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchTimestampISO8601ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	_, ok := tryParseTimestampISO8601(phrase)
 	if ok {
 		// Fast path - the phrase contains complete timestamp, so we can use exact search
@@ -121,7 +128,7 @@ func matchTimestampISO8601ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap
 	bbPool.Put(bb)
 }
 
-func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	_, ok := tryParseIPv4(phrase)
 	if ok {
 		// Fast path - phrase contains the full IP address, so we can use exact matching
@@ -145,7 +152,7 @@ func matchIPv4ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase str
 	bbPool.Put(bb)
 }
 
-func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchFloat64ByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	// The phrase may contain a part of the floating-point number.
 	// For example, `foo:"123"` must match `123`, `123.456` and `-0.123`.
 	// This means we cannot search in binary representation of floating-point numbers.
@@ -187,7 +194,7 @@ func matchValuesDictByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phra
 	bbPool.Put(bb)
 }
 
-func matchStringByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []string) {
+func matchStringByPhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase string, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -288,7 +295,7 @@ func visitValues(bs *blockSearch, ch *columnHeader, bm *bitmap, f func(value str
 	})
 }
 
-func matchBloomFilterAllTokens(bs *blockSearch, ch *columnHeader, tokens []string) bool {
+func matchBloomFilterAllTokens(bs *blockSearch, ch *columnHeader, tokens []uint64) bool {
 	if len(tokens) == 0 {
 		return true
 	}

lib/logstorage/filter_prefix.go

@@ -19,8 +19,9 @@ type filterPrefix struct {
 	fieldName string
 	prefix    string
 
-	tokensOnce sync.Once
-	tokens     []string
+	tokensOnce   sync.Once
+	tokens       []string
+	tokensHashes []uint64
 }
 
 func (fp *filterPrefix) String() string {
@@ -39,8 +40,14 @@ func (fp *filterPrefix) getTokens() []string {
 	return fp.tokens
 }
 
+func (fp *filterPrefix) getTokensHashes() []uint64 {
+	fp.tokensOnce.Do(fp.initTokens)
+	return fp.tokensHashes
+}
+
 func (fp *filterPrefix) initTokens() {
 	fp.tokens = getTokensSkipLast(fp.prefix)
+	fp.tokensHashes = appendTokensHashes(nil, fp.tokens)
 }
 
 func (fp *filterPrefix) applyToBlockResult(bs *blockResult, bm *bitmap) {
@@ -68,7 +75,7 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fp.getTokens()
+	tokens := fp.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -94,7 +101,7 @@ func (fp *filterPrefix) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchTimestampISO8601ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchTimestampISO8601ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if prefix == "" {
 		// Fast path - all the timestamp values match an empty prefix aka `*`
 		return
@@ -115,7 +122,7 @@ func matchTimestampISO8601ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap
 	bbPool.Put(bb)
 }
 
-func matchIPv4ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchIPv4ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if prefix == "" {
 		// Fast path - all the ipv4 values match an empty prefix aka `*`
 		return
@@ -136,7 +143,7 @@ func matchIPv4ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix str
 	bbPool.Put(bb)
 }
 
-func matchFloat64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchFloat64ByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if prefix == "" {
 		// Fast path - all the float64 values match an empty prefix aka `*`
 		return
@@ -177,7 +184,7 @@ func matchValuesDictByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, pref
 	bbPool.Put(bb)
 }
 
-func matchStringByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []string) {
+func matchStringByPrefix(bs *blockSearch, ch *columnHeader, bm *bitmap, prefix string, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return

lib/logstorage/filter_regexp.go

@@ -16,8 +16,9 @@ type filterRegexp struct {
 	fieldName string
 	re        *regexutil.Regex
 
-	tokens     []string
-	tokensOnce sync.Once
+	tokensOnce   sync.Once
+	tokens       []string
+	tokensHashes []uint64
 }
 
 func (fr *filterRegexp) String() string {
@@ -33,12 +34,18 @@ func (fr *filterRegexp) getTokens() []string {
 	return fr.tokens
 }
 
+func (fr *filterRegexp) getTokensHashes() []uint64 {
+	fr.tokensOnce.Do(fr.initTokens)
+	return fr.tokensHashes
+}
+
 func (fr *filterRegexp) initTokens() {
 	literals := fr.re.GetLiterals()
 	for i, literal := range literals {
 		literals[i] = skipFirstLastToken(literal)
 	}
 	fr.tokens = tokenizeStrings(nil, literals)
+	fr.tokensHashes = appendTokensHashes(nil, fr.tokens)
 }
 
 func skipFirstLastToken(s string) string {
@@ -89,7 +96,7 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fr.getTokens()
+	tokens := fr.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -115,7 +122,7 @@ func (fr *filterRegexp) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -128,7 +135,7 @@ func matchTimestampISO8601ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap
 	bbPool.Put(bb)
 }
 
-func matchIPv4ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchIPv4ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -141,7 +148,7 @@ func matchIPv4ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexu
 	bbPool.Put(bb)
 }
 
-func matchFloat64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchFloat64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -167,7 +174,7 @@ func matchValuesDictByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *
 	bbPool.Put(bb)
 }
 
-func matchStringByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchStringByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -177,7 +184,7 @@ func matchStringByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *rege
 	})
 }
 
-func matchUint8ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchUint8ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -190,7 +197,7 @@ func matchUint8ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regex
 	bbPool.Put(bb)
 }
 
-func matchUint16ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchUint16ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -203,7 +210,7 @@ func matchUint16ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *rege
 	bbPool.Put(bb)
 }
 
-func matchUint32ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchUint32ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -216,7 +223,7 @@ func matchUint32ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *rege
 	bbPool.Put(bb)
 }
 
-func matchUint64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []string) {
+func matchUint64ByRegexp(bs *blockSearch, ch *columnHeader, bm *bitmap, re *regexutil.Regex, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return

lib/logstorage/filter_sequence.go

@@ -15,8 +15,9 @@ type filterSequence struct {
 	fieldName string
 	phrases   []string
 
-	tokensOnce sync.Once
-	tokens     []string
+	tokensOnce   sync.Once
+	tokens       []string
+	tokensHashes []uint64
 
 	nonEmptyPhrasesOnce sync.Once
 	nonEmptyPhrases     []string
@@ -40,10 +41,15 @@ func (fs *filterSequence) getTokens() []string {
 	return fs.tokens
 }
 
+func (fs *filterSequence) getTokensHashes() []uint64 {
+	fs.tokensOnce.Do(fs.initTokens)
+	return fs.tokensHashes
+}
+
 func (fs *filterSequence) initTokens() {
 	phrases := fs.getNonEmptyPhrases()
-	tokens := tokenizeStrings(nil, phrases)
-	fs.tokens = tokens
+	fs.tokens = tokenizeStrings(nil, phrases)
+	fs.tokensHashes = appendTokensHashes(nil, fs.tokens)
 }
 
 func (fs *filterSequence) getNonEmptyPhrases() []string {
@@ -100,7 +106,7 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 		return
 	}
 
-	tokens := fs.getTokens()
+	tokens := fs.getTokensHashes()
 
 	switch ch.valueType {
 	case valueTypeString:
@@ -126,7 +132,7 @@ func (fs *filterSequence) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
 	}
 }
 
-func matchTimestampISO8601BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchTimestampISO8601BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if len(phrases) == 1 {
 		matchTimestampISO8601ByPhrase(bs, ch, bm, phrases[0], tokens)
 		return
@@ -145,7 +151,7 @@ func matchTimestampISO8601BySequence(bs *blockSearch, ch *columnHeader, bm *bitm
 	bbPool.Put(bb)
 }
 
-func matchIPv4BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchIPv4BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if len(phrases) == 1 {
 		matchIPv4ByPhrase(bs, ch, bm, phrases[0], tokens)
 		return
@@ -166,7 +172,7 @@ func matchIPv4BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases,
 	bbPool.Put(bb)
 }
 
-func matchFloat64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchFloat64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -197,7 +203,7 @@ func matchValuesDictBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, ph
 	bbPool.Put(bb)
 }
 
-func matchStringBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []string) {
+func matchStringBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if !matchBloomFilterAllTokens(bs, ch, tokens) {
 		bm.resetBits()
 		return
@@ -207,7 +213,7 @@ func matchStringBySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
 	})
 }
 
-func matchUint8BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchUint8BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if len(phrases) > 1 {
 		bm.resetBits()
 		return
@@ -215,7 +221,7 @@ func matchUint8BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases
 	matchUint8ByExactValue(bs, ch, bm, phrases[0], tokens)
 }
 
-func matchUint16BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchUint16BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if len(phrases) > 1 {
 		bm.resetBits()
 		return
@@ -223,7 +229,7 @@ func matchUint16BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
 	matchUint16ByExactValue(bs, ch, bm, phrases[0], tokens)
 }
 
-func matchUint32BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchUint32BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if len(phrases) > 1 {
 		bm.resetBits()
 		return
@@ -231,7 +237,7 @@ func matchUint32BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrase
 	matchUint32ByExactValue(bs, ch, bm, phrases[0], tokens)
 }
 
-func matchUint64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases, tokens []string) {
+func matchUint64BySequence(bs *blockSearch, ch *columnHeader, bm *bitmap, phrases []string, tokens []uint64) {
 	if len(phrases) > 1 {
 		bm.resetBits()
 		return

lib/logstorage/parser.go

@@ -3,6 +3,7 @@ package logstorage
 import (
 	"fmt"
 	"math"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -57,10 +58,15 @@ func (lex *lexer) restoreState(ls *lexerState) {
 //
 // The lex.token points to the first token in s.
 func newLexer(s string) *lexer {
+	timestamp := time.Now().UnixNano()
+	return newLexerAtTimestamp(s, timestamp)
+}
+
+func newLexerAtTimestamp(s string, timestamp int64) *lexer {
 	lex := &lexer{
 		s:                s,
 		sOrig:            s,
-		currentTimestamp: time.Now().UnixNano(),
+		currentTimestamp: timestamp,
 	}
 	lex.nextToken()
 	return lex
@@ -221,6 +227,9 @@ type Query struct {
 	f filter
 
 	pipes []pipe
+
+	// timestamp is the timestamp context used for parsing the query.
+	timestamp int64
 }
 
 // String returns string representation for q.
@@ -329,7 +338,8 @@ func (q *Query) AddCountByTimePipe(step, off int64, fields []string) {
 // Clone returns a copy of q.
 func (q *Query) Clone() *Query {
 	qStr := q.String()
-	qCopy, err := ParseQuery(qStr)
+	timestamp := q.GetTimestamp()
+	qCopy, err := ParseQueryAtTimestamp(qStr, timestamp)
 	if err != nil {
 		logger.Panicf("BUG: cannot parse %q: %s", qStr, err)
 	}
@@ -344,6 +354,7 @@ func (q *Query) CanReturnLastNResults() bool {
 			*pipeFieldValues,
 			*pipeLimit,
 			*pipeOffset,
+			*pipeTop,
 			*pipeSort,
 			*pipeStats,
 			*pipeUniq:
@@ -445,6 +456,115 @@ func (q *Query) Optimize() {
 	}
 }
 
+// GetStatsByFields returns `by (...)` fields from the last `stats` pipe at q.
+func (q *Query) GetStatsByFields() ([]string, error) {
+	return q.GetStatsByFieldsAddGroupingByTime(0)
+}
+
+// GetStatsByFieldsAddGroupingByTime returns `by (...)` fields from the last `stats` pipe at q.
+//
+// if step > 0, then _time:step is added to the last `stats by (...)` pipe at q.
+func (q *Query) GetStatsByFieldsAddGroupingByTime(step int64) ([]string, error) {
+	pipes := q.pipes
+
+	idx := getLastPipeStatsIdx(pipes)
+	if idx < 0 {
+		return nil, fmt.Errorf("missing `| stats ...` pipe in the query [%s]", q)
+	}
+
+	ps := pipes[idx].(*pipeStats)
+
+	// add _time:step to ps.byFields if it doesn't contain it yet.
+	ps.byFields = addByTimeField(ps.byFields, step)
+
+	// extract by(...) field names from stats pipe
+	byFields := ps.byFields
+	fields := make([]string, len(byFields))
+	for i, f := range byFields {
+		fields[i] = f.name
+	}
+
+	// verify that all the pipes after the idx do not add new fields
+	for i := idx + 1; i < len(pipes); i++ {
+		p := pipes[i]
+		switch t := p.(type) {
+		case *pipeSort, *pipeOffset, *pipeLimit, *pipeFilter:
+			// These pipes do not change the set of fields.
+		case *pipeMath:
+			// Allow pipeMath, since it adds additional metrics to the given set of fields.
+		case *pipeFields:
+			// `| fields ...` pipe must contain all the by(...) fields, otherwise it breaks output.
+			for _, f := range fields {
+				if !slices.Contains(t.fields, f) {
+					return nil, fmt.Errorf("missing %q field at %q pipe in the query [%s]", f, p, q)
+				}
+			}
+		case *pipeDelete:
+			// Disallow deleting by(...) fields, since this breaks output.
+			for _, f := range t.fields {
+				if slices.Contains(fields, f) {
+					return nil, fmt.Errorf("the %q field cannot be deleted via %q in the query [%s]", f, p, q)
+				}
+			}
+		case *pipeCopy:
+			// Disallow copying by(...) fields, since this breaks output.
+			for _, f := range t.srcFields {
+				if slices.Contains(fields, f) {
+					return nil, fmt.Errorf("the %q field cannot be copied via %q in the query [%s]", f, p, q)
+				}
+			}
+		case *pipeRename:
+			// Update by(...) fields with dst fields
+			for i, f := range t.srcFields {
+				if n := slices.Index(fields, f); n >= 0 {
+					fields[n] = t.dstFields[i]
+				}
+			}
+		default:
+			return nil, fmt.Errorf("the %q pipe cannot be put after %q pipe in the query [%s]", p, ps, q)
+		}
+	}
+
+	return fields, nil
+}
+
+func getLastPipeStatsIdx(pipes []pipe) int {
+	for i := len(pipes) - 1; i >= 0; i-- {
+		if _, ok := pipes[i].(*pipeStats); ok {
+			return i
+		}
+	}
+	return -1
+}
+
+func addByTimeField(byFields []*byStatsField, step int64) []*byStatsField {
+	if step <= 0 {
+		return byFields
+	}
+	stepStr := fmt.Sprintf("%d", step)
+	dstFields := make([]*byStatsField, 0, len(byFields)+1)
+	hasByTime := false
+	for _, f := range byFields {
+		if f.name == "_time" {
+			f = &byStatsField{
+				name:          "_time",
+				bucketSizeStr: stepStr,
+				bucketSize:    float64(step),
+			}
+			hasByTime = true
+		}
+		dstFields = append(dstFields, f)
+	}
+	if !hasByTime {
+		dstFields = append(dstFields, &byStatsField{
+			name:          "_time",
+			bucketSizeStr: stepStr,
+			bucketSize:    float64(step),
+		})
+	}
+	return dstFields
+}
+
 func removeStarFilters(f filter) filter {
 	visitFunc := func(f filter) bool {
 		fp, ok := f.(*filterPrefix)
@@ -584,7 +704,15 @@ func (q *Query) getNeededColumns() ([]string, []string) {
 
 // ParseQuery parses s.
 func ParseQuery(s string) (*Query, error) {
-	lex := newLexer(s)
+	timestamp := time.Now().UnixNano()
+	return ParseQueryAtTimestamp(s, timestamp)
+}
+
+// ParseQueryAtTimestamp parses s in the context of the given timestamp.
+//
+// E.g. _time:duration filters are ajusted according to the provided timestamp as _time:[timestamp-duration, duration].
+func ParseQueryAtTimestamp(s string, timestamp int64) (*Query, error) {
+	lex := newLexerAtTimestamp(s, timestamp)
 
 	// Verify the first token doesn't match pipe names.
 	firstToken := strings.ToLower(lex.rawToken)
@@ -600,9 +728,15 @@ func ParseQuery(s string) (*Query, error) {
 	if !lex.isEnd() {
 		return nil, fmt.Errorf("unexpected unparsed tail after [%s]; context: [%s]; tail: [%s]", q, lex.context(), lex.s)
 	}
+	q.timestamp = timestamp
 	return q, nil
 }
 
+// GetTimestamp returns timestamp context for the given q, which was passed to ParseQueryAtTimestamp().
+func (q *Query) GetTimestamp() int64 {
+	return q.timestamp
+}
+
 func parseQuery(lex *lexer) (*Query, error) {
 	f, err := parseFilter(lex)
 	if err != nil {
@@ -1644,8 +1778,8 @@ func getDayRangeArg(lex *lexer) (int64, string, error) {
 	if offset < 0 {
 		offset = 0
 	}
-	if offset >= nsecPerDay {
-		offset = nsecPerDay - 1
+	if offset >= nsecsPerDay {
+		offset = nsecsPerDay - 1
 	}
 	return offset, argStr, nil
 }

lib/logstorage/parser_test.go

@@ -2032,6 +2032,7 @@ func TestQueryCanReturnLastNResults(t *testing.T) {
 	f("* | uniq (x)", false)
 	f("* | field_names", false)
 	f("* | field_values x", false)
+	f("* | top 5 by (x)", false)
 
 }
 
@@ -2100,3 +2101,145 @@ func TestQueryDropAllPipes(t *testing.T) {
 	f(`foo or bar and baz | top 5 by (x)`, `foo or bar baz`)
 	f(`foo | filter bar:baz | stats by (x) min(y)`, `foo bar:baz`)
 }
+
+func TestQueryGetStatsByFieldsAddGroupingByTime_Success(t *testing.T) {
+	f := func(qStr string, step int64, fieldsExpected []string, qExpected string) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("cannot parse [%s]: %s", qStr, err)
+		}
+		fields, err := q.GetStatsByFieldsAddGroupingByTime(step)
+		if err != nil {
+			t.Fatalf("unexpected error in GetStatsByFieldsAddGroupingByTime(): %s", err)
+		}
+		if !reflect.DeepEqual(fields, fieldsExpected) {
+			t.Fatalf("unexpected byFields;\ngot\n%q\nwant\n%q", fields, fieldsExpected)
+		}
+
+		// Verify the resulting query
+		qResult := q.String()
+		if qResult != qExpected {
+			t.Fatalf("unexpected query\ngot\n%s\nwant\n%s", qResult, qExpected)
+		}
+	}
+
+	f(`* | count()`, nsecsPerHour, []string{"_time"}, `* | stats by (_time:3600000000000) count(*) as "count(*)"`)
+	f(`* | by (level) count() x`, nsecsPerDay, []string{"level", "_time"}, `* | stats by (level, _time:86400000000000) count(*) as x`)
+	f(`* | by (_time:1m) count() x`, nsecsPerDay, []string{"_time"}, `* | stats by (_time:86400000000000) count(*) as x`)
+	f(`* | by (_time:1m offset 30s,level) count() x, count_uniq(z) y`, nsecsPerDay, []string{"_time", "level"}, `* | stats by (_time:86400000000000, level) count(*) as x, count_uniq(z) as y`)
+}
+
+func TestQueryGetStatsByFieldsAddGroupingByTime_Failure(t *testing.T) {
+	f := func(qStr string) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("cannot parse [%s]: %s", qStr, err)
+		}
+		fields, err := q.GetStatsByFieldsAddGroupingByTime(nsecsPerHour)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if fields != nil {
+			t.Fatalf("unexpected non-nil fields: %q", fields)
+		}
+	}
+
+	f(`*`)
+	f(`_time:5m | count() | drop _time`)
+	f(`* | by (x) count() | keep x`)
+}
+
+func TestQueryGetStatsByFields_Success(t *testing.T) {
+	f := func(qStr string, fieldsExpected []string) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("cannot parse [%s]: %s", qStr, err)
+		}
+		fields, err := q.GetStatsByFields()
+		if err != nil {
+			t.Fatalf("unexpected error in GetStatsByFields(): %s", err)
+		}
+		if !reflect.DeepEqual(fields, fieldsExpected) {
+			t.Fatalf("unexpected byFields;\ngot\n%q\nwant\n%q", fields, fieldsExpected)
+		}
+	}
+
+	f(`* | stats count()`, []string{})
+	f(`* | count()`, []string{})
+	f(`* | by (foo) count(), count_uniq(bar)`, []string{"foo"})
+	f(`* | stats by (a, b, cd) min(foo), max(bar)`, []string{"a", "b", "cd"})
+
+	// multiple pipes before stats is ok
+	f(`foo | extract "ip=<ip>," | stats by (host) count_uniq(ip)`, []string{"host"})
+
+	// sort, offset and limit pipes are allowed after stats
+	f(`foo | stats by (x, y) count() rows | sort by (rows) desc | offset 5 | limit 10`, []string{"x", "y"})
+
+	// filter pipe is allowed after stats
+	f(`foo | stats by (x, y) count() rows | filter rows:>100`, []string{"x", "y"})
+
+	// math pipe is allowed after stats
+	f(`foo | stats by (x) count() total, count() if (error) errors | math errors / total`, []string{"x"})
+
+	// keep containing all the by(...) fields
+	f(`foo | stats by (x) count() total | keep x, y`, []string{"x"})
+
+	// drop which doesn't contain by(...) fields
+	f(`foo | stats by (x) count() total | drop y`, []string{"x"})
+
+	// copy which doesn't contain by(...) fields
+	f(`foo | stats by (x) count() total | copy total abc`, []string{"x"})
+
+	// mv by(...) fields
+	f(`foo | stats by (x) count() total | mv x y`, []string{"y"})
+}
+
+func TestQueryGetStatsByFields_Failure(t *testing.T) {
+	f := func(qStr string) {
+		t.Helper()
+
+		q, err := ParseQuery(qStr)
+		if err != nil {
+			t.Fatalf("cannot parse [%s]: %s", qStr, err)
+		}
+		fields, err := q.GetStatsByFields()
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if fields != nil {
+			t.Fatalf("expectig nil fields; got %q", fields)
+		}
+	}
+
+	f(`*`)
+	f(`foo bar`)
+	f(`foo | by (a, b) count() | copy a b`)
+	f(`foo | by (a, b) count() | delete a`)
+	f(`foo | count() | drop_empty_fields`)
+	f(`foo | count() | extract "foo<bar>baz"`)
+	f(`foo | count() | extract_regexp "(?P<ip>([0-9]+[.]){3}[0-9]+)"`)
+	f(`foo | count() | field_names`)
+	f(`foo | count() | field_values abc`)
+	f(`foo | by (x) count() | fields a, b`)
+	f(`foo | count() | format "foo<bar>baz"`)
+	f(`foo | count() | pack_json`)
+	f(`foo | count() | pack_logfmt`)
+	f(`foo | rename x y`)
+	f(`foo | count() | replace ("foo", "bar")`)
+	f(`foo | count() | replace_regexp ("foo.+bar", "baz")`)
+	f(`foo | count() | stream_context after 10`)
+	f(`foo | count() | top 5 by (x)`)
+	f(`foo | count() | uniq by (x)`)
+	f(`foo | count() | unpack_json`)
+	f(`foo | count() | unpack_logfmt`)
+	f(`foo | count() | unpack_syslog`)
+	f(`foo | count() | unroll by (x)`)
+
+	f(`* | by (x) count() as rows | math rows * 10, rows / 10 | drop x`)
+}

lib/logstorage/pipe_math.go

@@ -173,7 +173,7 @@ var mathBinaryOps = map[string]mathBinaryOp{
 		priority: 5,
 		f:        mathFuncXor,
 	},
-	"|": {
+	"or": {
 		priority: 6,
 		f:        mathFuncOr,
 	},

lib/logstorage/pipe_math_test.go

@@ -56,7 +56,7 @@ func TestPipeMath(t *testing.T) {
 		'123.45.67.89' + 1000 as ip,
 		time - time % time_step as time_rounded,
 		duration - duration % duration_step as duration_rounded,
-		(ip & ip_mask | 0x1234) xor 5678 as subnet
+		(ip & ip_mask or 0x1234) xor 5678 as subnet
 	`, [][]Field{
 		{
 			{"time_step", "30m"},

lib/logstorage/pipe_stats.go

@@ -728,7 +728,7 @@ var zeroByStatsField = &byStatsField{}
 
 // byStatsField represents 'by (...)' part of the pipeStats.
 //
-// It can have either 'name' representation or 'name:bucket' or 'name:buket offset off' representation,
+// It can have either 'name' representation or 'name:bucket' or 'name:bucket offset off' representation,
 // where `bucket` and `off` can contain duration, size or numeric value for creating different buckets
 // for 'value/bucket'.
 type byStatsField struct {

lib/logstorage/storage.go

@@ -200,8 +200,8 @@ func (ptw *partitionWrapper) decRef() {
 }
 
 func (ptw *partitionWrapper) canAddAllRows(lr *LogRows) bool {
-	minTimestamp := ptw.day * nsecPerDay
-	maxTimestamp := minTimestamp + nsecPerDay - 1
+	minTimestamp := ptw.day * nsecsPerDay
+	maxTimestamp := minTimestamp + nsecsPerDay - 1
 	for _, ts := range lr.timestamps {
 		if ts < minTimestamp || ts > maxTimestamp {
 			return false
@@ -286,7 +286,7 @@ func MustOpenStorage(path string, cfg *StorageConfig) *Storage {
 		if err != nil {
 			logger.Panicf("FATAL: cannot parse partition filename %q at %q; it must be in the form YYYYMMDD: %s", fname, partitionsPath, err)
 		}
-		day := t.UTC().UnixNano() / nsecPerDay
+		day := t.UTC().UnixNano() / nsecsPerDay
 
 		partitionPath := filepath.Join(partitionsPath, fname)
 		pt := mustOpenPartition(s, partitionPath)
@@ -441,11 +441,11 @@ func (s *Storage) watchMaxDiskSpaceUsage() {
 }
 
 func (s *Storage) getMinAllowedDay() int64 {
-	return time.Now().UTC().Add(-s.retention).UnixNano() / nsecPerDay
+	return time.Now().UTC().Add(-s.retention).UnixNano() / nsecsPerDay
 }
 
 func (s *Storage) getMaxAllowedDay() int64 {
-	return time.Now().UTC().Add(s.futureRetention).UnixNano() / nsecPerDay
+	return time.Now().UTC().Add(s.futureRetention).UnixNano() / nsecsPerDay
 }
 
 // MustClose closes s.
@@ -514,11 +514,11 @@ func (s *Storage) MustAddRows(lr *LogRows) {
 	maxAllowedDay := s.getMaxAllowedDay()
 	m := make(map[int64]*LogRows)
 	for i, ts := range lr.timestamps {
-		day := ts / nsecPerDay
+		day := ts / nsecsPerDay
 		if day < minAllowedDay {
 			rf := RowFormatter(lr.rows[i])
 			tsf := TimeFormatter(ts)
-			minAllowedTsf := TimeFormatter(minAllowedDay * nsecPerDay)
+			minAllowedTsf := TimeFormatter(minAllowedDay * nsecsPerDay)
 			tooSmallTimestampLogger.Warnf("skipping log entry with too small timestamp=%s; it must be bigger than %s according "+
 				"to the configured -retentionPeriod=%dd. See https://docs.victoriametrics.com/victorialogs/#retention ; "+
 				"log entry: %s", &tsf, &minAllowedTsf, durationToDays(s.retention), &rf)
@@ -528,7 +528,7 @@ func (s *Storage) MustAddRows(lr *LogRows) {
 		if day > maxAllowedDay {
 			rf := RowFormatter(lr.rows[i])
 			tsf := TimeFormatter(ts)
-			maxAllowedTsf := TimeFormatter(maxAllowedDay * nsecPerDay)
+			maxAllowedTsf := TimeFormatter(maxAllowedDay * nsecsPerDay)
 			tooBigTimestampLogger.Warnf("skipping log entry with too big timestamp=%s; it must be smaller than %s according "+
 				"to the configured -futureRetention=%dd; see https://docs.victoriametrics.com/victorialogs/#retention ; "+
 				"log entry: %s", &tsf, &maxAllowedTsf, durationToDays(s.futureRetention), &rf)
@@ -553,8 +553,6 @@ func (s *Storage) MustAddRows(lr *LogRows) {
 var tooSmallTimestampLogger = logger.WithThrottler("too_small_timestamp", 5*time.Second)
 var tooBigTimestampLogger = logger.WithThrottler("too_big_timestamp", 5*time.Second)
 
-const nsecPerDay = 24 * 3600 * 1e9
-
 // TimeFormatter implements fmt.Stringer for timestamp in nanoseconds
 type TimeFormatter int64
 
@@ -582,7 +580,7 @@ func (s *Storage) getPartitionForDay(day int64) *partitionWrapper {
 	}
 	if ptw == nil {
 		// Missing partition for the given day. Create it.
-		fname := time.Unix(0, day*nsecPerDay).UTC().Format(partitionNameFormat)
+		fname := time.Unix(0, day*nsecsPerDay).UTC().Format(partitionNameFormat)
 		partitionPath := filepath.Join(s.path, partitionsDirname, fname)
 		mustCreatePartition(partitionPath)
 

lib/logstorage/storage_search.go

@@ -673,12 +673,12 @@ func (s *Storage) search(workersCount int, so *genericSearchOptions, stopCh <-ch
 	// Select partitions according to the selected time range
 	s.partitionsLock.Lock()
 	ptws := s.partitions
-	minDay := so.minTimestamp / nsecPerDay
+	minDay := so.minTimestamp / nsecsPerDay
 	n := sort.Search(len(ptws), func(i int) bool {
 		return ptws[i].day >= minDay
 	})
 	ptws = ptws[n:]
-	maxDay := so.maxTimestamp / nsecPerDay
+	maxDay := so.maxTimestamp / nsecsPerDay
 	n = sort.Search(len(ptws), func(i int) bool {
 		return ptws[i].day > maxDay
 	})

lib/logstorage/storage_test.go

@@ -78,10 +78,10 @@ func TestStorageMustAddRows(t *testing.T) {
 	s = MustOpenStorage(path, cfg)
 
 	lr = newTestLogRows(3, 10, 0)
-	now := time.Now().UTC().UnixNano() - int64(len(lr.timestamps)/2)*nsecPerDay
+	now := time.Now().UTC().UnixNano() - int64(len(lr.timestamps)/2)*nsecsPerDay
 	for i := range lr.timestamps {
 		lr.timestamps[i] = now
-		now += nsecPerDay
+		now += nsecsPerDay
 	}
 	totalRowsCount += uint64(len(lr.timestamps))
 	s.MustAddRows(lr)

lib/promscrape/discovery/azure/api.go

@@ -122,7 +122,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 
 	cfg := &apiConfig{
 		c:              c,
-		apiServerHost:  u.Host,
+		apiServerHost:  u.Hostname(),
 		port:           port,
 		resourceGroup:  sdc.ResourceGroup,
 		subscriptionID: sdc.SubscriptionID,

lib/promscrape/discovery/azure/machine.go

@@ -96,8 +96,10 @@ func visitAllAPIObjects(ac *apiConfig, apiURL string, cb func(data json.RawMessa
 			return fmt.Errorf("cannot parse nextLink from response %q: %w", lar.NextLink, err)
 		}
 
-		if nextURL.Host != "" && nextURL.Host != ac.apiServerHost {
-			return fmt.Errorf("unexpected nextLink host %q, expecting %q", nextURL.Host, ac.apiServerHost)
+		// Sometimes Azure will respond a host with a port. Since all possible apiServer defined in cloudEnvironments do not include a port,
+		// it is best to check the host without the port. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6912
+		if nextURL.Host != "" && nextURL.Hostname() != ac.apiServerHost {
+			return fmt.Errorf("unexpected nextLink host %q, expecting %q", nextURL.Hostname(), ac.apiServerHost)
 		}
 
 		nextLinkURI = nextURL.RequestURI()