Merge tag 'v1.106.1' into pmm-6401-read-prometheus-data-files-cpc

Makefile

@@ -527,8 +527,8 @@ test-full:
 test-full-386:
 	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
-integration-test: all
+integration-test: victoria-metrics vmagent vmalert vmauth
-	go test ./apptest/...
+	go test ./apptest/... -skip="^TestCluster.*"
 
 benchmark:
 	go test -bench=. ./lib/...

app/vlinsert/datadog/datadog.go

@@ -0,0 +1,185 @@
+package datadog
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+)
+
+var parserPool fastjson.ParserPool
+
+// RequestHandler processes Datadog insert requests
+func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
+	switch path {
+	case "/api/v1/validate":
+		fmt.Fprintf(w, `{}`)
+		return true
+	case "/api/v2/logs":
+		return datadogLogsIngestion(w, r)
+	default:
+		return false
+	}
+}
+
+func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
+	w.Header().Add("Content-Type", "application/json")
+	startTime := time.Now()
+	v2LogsRequestsTotal.Inc()
+	reader := r.Body
+
+	var ts int64
+	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
+		var err error
+		ts, err = strconv.ParseInt(tsValue, 10, 64)
+		if err != nil {
+			httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err)
+			return true
+		}
+		ts *= 1e6
+	} else {
+		ts = startTime.UnixNano()
+	}
+
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot read gzipped logs request: %s", err)
+			return true
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return true
+	}
+
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
+	}
+
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
+	}
+
+	lmp := cp.NewLogMessageProcessor()
+	n, err := readLogsRequest(ts, data, lmp.AddRow)
+	lmp.MustClose()
+	if n > 0 {
+		rowsIngestedTotal.Add(n)
+	}
+	if err != nil {
+		logger.Warnf("cannot decode log message in /api/v2/logs request: %s, stream fields: %s", err, cp.StreamFields)
+		return true
+	}
+
+	// update v2LogsRequestDuration only for successfully parsed requests
+	// There is no need in updating v2LogsRequestDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	v2LogsRequestDuration.UpdateDuration(startTime)
+	fmt.Fprintf(w, `{}`)
+	return true
+}
+
+var (
+	v2LogsRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
+	rowsIngestedTotal     = metrics.NewCounter(`vl_rows_ingested_total{type="datadog"}`)
+	v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
+)
+
+// readLogsRequest parses data according to DataDog logs format
+// https://docs.datadoghq.com/api/latest/logs/#send-logs
+func readLogsRequest(ts int64, data []byte, processLogMessage func(int64, []logstorage.Field)) (int, error) {
+	p := parserPool.Get()
+	defer parserPool.Put(p)
+	v, err := p.ParseBytes(data)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
+	}
+	records, err := v.Array()
+	if err != nil {
+		return 0, fmt.Errorf("cannot extract array from parsed JSON: %w", err)
+	}
+
+	var fields []logstorage.Field
+	for m, r := range records {
+		o, err := r.Object()
+		if err != nil {
+			return m + 1, fmt.Errorf("could not extract log record: %w", err)
+		}
+		o.Visit(func(k []byte, v *fastjson.Value) {
+			if err != nil {
+				return
+			}
+			val, e := v.StringBytes()
+			if e != nil {
+				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
+				return
+			}
+			switch string(k) {
+			case "message":
+				fields = append(fields, logstorage.Field{
+					Name:  "_msg",
+					Value: bytesutil.ToUnsafeString(val),
+				})
+			case "ddtags":
+				// https://docs.datadoghq.com/getting_started/tagging/
+				var pair []byte
+				idx := 0
+				for idx >= 0 {
+					idx = bytes.IndexByte(val, ',')
+					if idx < 0 {
+						pair = val
+					} else {
+						pair = val[:idx]
+						val = val[idx+1:]
+					}
+					if len(pair) > 0 {
+						n := bytes.IndexByte(pair, ':')
+						if n < 0 {
+							// No tag value.
+							fields = append(fields, logstorage.Field{
+								Name:  bytesutil.ToUnsafeString(pair),
+								Value: "no_label_value",
+							})
+						}
+						fields = append(fields, logstorage.Field{
+							Name:  bytesutil.ToUnsafeString(pair[:n]),
+							Value: bytesutil.ToUnsafeString(pair[n+1:]),
+						})
+					}
+				}
+			default:
+				fields = append(fields, logstorage.Field{
+					Name:  bytesutil.ToUnsafeString(k),
+					Value: bytesutil.ToUnsafeString(val),
+				})
+			}
+		})
+		processLogMessage(ts, fields)
+		fields = fields[:0]
+	}
+	return len(records), nil
+}

app/vlinsert/datadog/datadog_test.go

@@ -0,0 +1,117 @@
+package datadog
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestReadLogsRequestFailure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		ts := time.Now().UnixNano()
+
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
+		}
+
+		rows, err := readLogsRequest(ts, []byte(data), processLogMessage)
+		if err == nil {
+			t.Fatalf("expecting non-empty error")
+		}
+		if rows != 0 {
+			t.Fatalf("unexpected non-zero rows=%d", rows)
+		}
+	}
+	f("foobar")
+	f(`{}`)
+	f(`["create":{}]`)
+	f(`{"create":{}}
+foobar`)
+}
+
+func TestReadLogsRequestSuccess(t *testing.T) {
+	f := func(data string, rowsExpected int, resultExpected string) {
+		t.Helper()
+
+		ts := time.Now().UnixNano()
+		var result string
+		processLogMessage := func(_ int64, fields []logstorage.Field) {
+			a := make([]string, len(fields))
+			for i, f := range fields {
+				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
+			}
+			if len(result) > 0 {
+				result = result + "\n"
+			}
+			s := "{" + strings.Join(a, ",") + "}"
+			result += s
+		}
+
+		// Read the request without compression
+		rows, err := readLogsRequest(ts, []byte(data), processLogMessage)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Verify non-empty data
+	data := `[
+		{
+			"ddsource":"nginx",
+			"ddtags":"tag1:value1,tag2:value2",
+			"hostname":"127.0.0.1",
+			"message":"bar",
+			"service":"test"
+		}, {
+			"ddsource":"nginx",
+			"ddtags":"tag1:value1,tag2:value2",
+			"hostname":"127.0.0.1",
+			"message":"foobar",
+			"service":"test"
+		}, {
+			"ddsource":"nginx",
+			"ddtags":"tag1:value1,tag2:value2",
+			"hostname":"127.0.0.1",
+			"message":"baz",
+			"service":"test"
+		}, {
+			"ddsource":"nginx",
+			"ddtags":"tag1:value1,tag2:value2",
+			"hostname":"127.0.0.1",
+			"message":"xyz",
+			"service":"test"
+		}, {
+			"ddsource": "nginx",
+			"ddtags":"tag1:value1,tag2:value2,",
+			"hostname":"127.0.0.1",
+                        "message":"xyz",
+                        "service":"test"
+                }, {
+			"ddsource":"nginx",
+			"ddtags":",tag1:value1,tag2:value2",
+			"hostname":"127.0.0.1",
+                        "message":"xyz",
+                        "service":"test"
+		}
+	]`
+	rowsExpected := 6
+	resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
+{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
+{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
+{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
+{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
+{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}`
+	f(data, rowsExpected, resultExpected)
+}

app/vlinsert/insertutils/common_params.go

@@ -112,19 +112,24 @@ func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
 }
 
 // GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields []string, extraFields []logstorage.Field) *CommonParams {
 	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
+	if streamFields == nil {
+		streamFields = []string{
+			"hostname",
+			"app_name",
+			"proc_id",
+		}
+	}
 	cp := &CommonParams{
 		TenantID:  tenantID,
 		TimeField: "timestamp",
 		MsgFields: []string{
 			"message",
 		},
-		StreamFields: []string{
+		StreamFields: streamFields,
-			"hostname",
+		IgnoreFields: ignoreFields,
-			"app_name",
+		ExtraFields:  extraFields,
-			"proc_id",
-		},
 	}
 
 	return cp

app/vlinsert/loki/loki_json.go

@@ -8,6 +8,9 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -15,8 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
 )
 
 var parserPool fastjson.ParserPool
@@ -56,7 +57,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 	n, err := parseJSONRequest(data, lmp)
 	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki json request: %s; data=%s", err, data)
 		return
 	}
 
@@ -84,7 +85,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, er
 
 	streamsV := v.Get("streams")
 	if streamsV == nil {
-		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
+		return 0, fmt.Errorf("missing `streams` item in the parsed JSON")
 	}
 	streams, err := streamsV.Array()
 	if err != nil {
@@ -107,9 +108,6 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, er
 			labels = o
 		}
 		labels.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
 			vStr, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
@@ -127,7 +125,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, er
 		// populate messages from `values` array
 		linesV := stream.Get("values")
 		if linesV == nil {
-			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
+			return rowsIngested, fmt.Errorf("missing `values` item in the parsed `stream` object %q", stream)
 		}
 		lines, err := linesV.Array()
 		if err != nil {
@@ -140,8 +138,8 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, er
 			if err != nil {
 				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
 			}
-			if len(lineA) != 2 {
+			if len(lineA) < 2 || len(lineA) > 3 {
-				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
+				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2 or 3", line, len(lineA))
 			}
 
 			// parse timestamp
@@ -167,6 +165,30 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, er
 				Name:  "_msg",
 				Value: bytesutil.ToUnsafeString(msg),
 			})
+
+			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
+			if len(lineA) > 2 {
+				structuredMetadata, err := lineA[2].Object()
+				if err != nil {
+					return rowsIngested, fmt.Errorf("unexpected structured metadata type for %q; want JSON object", lineA[2])
+				}
+
+				structuredMetadata.Visit(func(k []byte, v *fastjson.Value) {
+					vStr, errLocal := v.StringBytes()
+					if errLocal != nil {
+						err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
+						return
+					}
+
+					fields = append(fields, logstorage.Field{
+						Name:  bytesutil.ToUnsafeString(k),
+						Value: bytesutil.ToUnsafeString(vStr),
+					})
+				})
+				if err != nil {
+					return rowsIngested, fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
+				}
+			}
 			lmp.AddRow(ts, fields)
 		}
 		rowsIngested += len(lines)

app/vlinsert/loki/loki_json_test.go

@@ -45,13 +45,19 @@ func TestParseJSONRequest_Failure(t *testing.T) {
 	// Invalid length of `values` individual item
 	f(`{"streams":[{"values":[[]]}]}`)
 	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
+	f(`{"streams":[{"values":[["123","456","789","8123"]]}]}`)
 
 	// Invalid type for timestamp inside `values` individual item
 	f(`{"streams":[{"values":[[123,"456"]}]}`)
 
 	// Invalid type for log message
 	f(`{"streams":[{"values":[["123",1234]]}]}`)
+
+	// invalid structured metadata type
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", ["metadata_1", "md_value"]]]}]}`)
+
+	// structured metadata with unexpected value type
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": 1}]] }]}`)
 }
 
 func TestParseJSONRequest_Success(t *testing.T) {
@@ -116,4 +122,8 @@ func TestParseJSONRequest_Success(t *testing.T) {
 }`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
 {"foo":"bar","a":"b","_msg":"abc"}
 {"x":"y","_msg":"yx"}`)
+
+	// values with metadata
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar","metadata_1":"md_value"}`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 }

app/vlinsert/loki/pb.go

@@ -17,7 +17,7 @@ var mp easyproto.MarshalerPool
 
 // PushRequest represents Loki PushRequest
 //
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
+// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L14
 type PushRequest struct {
 	Streams []Stream
 
@@ -87,7 +87,7 @@ func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []Labe
 
 // Stream represents Loki stream.
 //
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
+// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L23
 type Stream struct {
 	Labels  string
 	Entries []Entry
@@ -139,7 +139,7 @@ func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair,
 
 // Entry represents Loki entry.
 //
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
+// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L38
 type Entry struct {
 	Timestamp          time.Time
 	Line               string
@@ -203,7 +203,7 @@ func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]Label
 
 // LabelPair represents Loki label pair.
 //
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
+// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L33
 type LabelPair struct {
 	Name  string
 	Value string

app/vlinsert/main.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
@@ -25,6 +26,7 @@ func Stop() {
 // RequestHandler handles insert requests for VictoriaLogs
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
+
 	if !strings.HasPrefix(path, "/insert/") {
 		// Skip requests, which do not start with /insert/, since these aren't our requests.
 		return false
@@ -49,6 +51,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case strings.HasPrefix(path, "/journald/"):
 		path = strings.TrimPrefix(path, "/journald")
 		return journald.RequestHandler(path, w, r)
+	case strings.HasPrefix(path, "/datadog/"):
+		path = strings.TrimPrefix(path, "/datadog")
+		return datadog.RequestHandler(path, w, r)
 	default:
 		return false
 	}

app/vlinsert/syslog/syslog.go

@@ -3,11 +3,13 @@ package syslog
 import (
 	"bufio"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"net"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,10 +37,25 @@ var (
 	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
 		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
-	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+	streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
+	streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
+
+	ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
+	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
+
+	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
+	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
+
+	tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
+	tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
 
 	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
 		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
@@ -150,7 +167,7 @@ func runUDPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
 	}
 
-	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
+	tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx)
 	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
 	if err != nil {
 		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
@@ -161,9 +178,27 @@ func runUDPListener(addr string, argIdx int) {
 
 	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
 
+	streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx)
+	streamFields, err := parseFieldsList(streamFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err)
+	}
+
+	ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx)
+	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
+	}
+
+	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
+	extraFields, err := parseExtraFields(extraFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err)
+	}
+
 	doneCh := make(chan struct{})
 	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
 		close(doneCh)
 	}()
 
@@ -193,7 +228,7 @@ func runTCPListener(addr string, argIdx int) {
 		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
 	}
 
-	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
+	tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx)
 	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
 	if err != nil {
 		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
@@ -204,9 +239,27 @@ func runTCPListener(addr string, argIdx int) {
 
 	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
 
+	streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx)
+	streamFields, err := parseFieldsList(streamFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err)
+	}
+
+	ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx)
+	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
+	}
+
+	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
+	extraFields, err := parseExtraFields(extraFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err)
+	}
+
 	doneCh := make(chan struct{})
 	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
 		close(doneCh)
 	}()
 
@@ -228,7 +281,7 @@ func checkCompressMethod(compressMethod, addr, protocol string) {
 	}
 }
 
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -236,7 +289,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
 			var bb bytesutil.ByteBuffer
 			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
 			for {
@@ -270,7 +323,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -300,7 +353,7 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 
 		wg.Add(1)
 		go func() {
-			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
 			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
@@ -531,3 +584,35 @@ var (
 	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
 	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
 )
+
+func parseFieldsList(s string) ([]string, error) {
+	if s == "" {
+		return nil, nil
+	}
+
+	var a []string
+	err := json.Unmarshal([]byte(s), &a)
+	return a, err
+}
+
+func parseExtraFields(s string) ([]logstorage.Field, error) {
+	if s == "" {
+		return nil, nil
+	}
+
+	var m map[string]string
+	if err := json.Unmarshal([]byte(s), &m); err != nil {
+		return nil, err
+	}
+	fields := make([]logstorage.Field, 0, len(m))
+	for k, v := range m {
+		fields = append(fields, logstorage.Field{
+			Name:  k,
+			Value: v,
+		})
+	}
+	sort.Slice(fields, func(i, j int) bool {
+		return fields[i].Name < fields[j].Name
+	})
+	return fields, nil
+}

app/vlogscli/less_wrapper.go

@@ -17,7 +17,7 @@ func isTerminal() bool {
 	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
 }
 
-func readWithLess(r io.Reader) error {
+func readWithLess(r io.Reader, wrapLongLines bool) error {
 	if !isTerminal() {
 		// Just write everything to stdout if no terminal is available.
 		_, err := io.Copy(os.Stdout, r)
@@ -48,7 +48,11 @@ func readWithLess(r io.Reader) error {
 	if err != nil {
 		return fmt.Errorf("cannot find 'less' command: %w", err)
 	}
-	p, err := os.StartProcess(path, []string{"less", "-F", "-X"}, &os.ProcAttr{
+	opts := []string{"less", "-F", "-X"}
+	if !wrapLongLines {
+		opts = append(opts, "-S")
+	}
+	p, err := os.StartProcess(path, opts, &os.ProcAttr{
 		Env:   append(os.Environ(), "LESSCHARSET=utf-8"),
 		Files: []*os.File{pr, os.Stdout, os.Stderr},
 	})

app/vlogscli/main.go

@@ -91,6 +91,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 	}
 
 	outputMode := outputModeJSONMultiline
+	wrapLongLines := false
 	s := ""
 	for {
 		line, err := rl.ReadLine()
@@ -99,7 +100,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			case io.EOF:
 				if s != "" {
 					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode)
+					executeQuery(context.Background(), rl, s, outputMode, wrapLongLines)
 				}
 				return
 			case readline.ErrInterrupt:
@@ -163,6 +164,18 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			s = ""
 			continue
 		}
+		if s == `\wrap_long_lines` {
+			if wrapLongLines {
+				wrapLongLines = false
+				fmt.Fprintf(rl, "wrapping of long lines is disabled\n")
+			} else {
+				wrapLongLines = true
+				fmt.Fprintf(rl, "wrapping of long lines is enabled\n")
+			}
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
 		if line != "" && !strings.HasSuffix(line, ";") {
 			// Assume the query is incomplete and allow the user finishing the query on the next line
 			s += "\n"
@@ -172,7 +185,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 
 		// Execute the query
 		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode)
+		executeQuery(ctx, rl, s, outputMode, wrapLongLines)
 		cancel()
 
 		historyLines = pushToHistory(rl, historyLines, s)
@@ -259,13 +272,14 @@ func printCommandsHelp(w io.Writer) {
 \m - multiline json output mode
 \c - compact output
 \logfmt - logfmt output mode
+\wrap_long_lines - toggles wrapping long lines
 \tail <query> - live tail <query> results
 
 See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
 `)
 }
 
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
+func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, wrapLongLines bool) {
 	if strings.HasPrefix(qStr, `\tail `) {
 		tailQuery(ctx, output, qStr, outputMode)
 		return
@@ -279,7 +293,7 @@ func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode
 		_ = respBody.Close()
 	}()
 
-	if err := readWithLess(respBody); err != nil {
+	if err := readWithLess(respBody, wrapLongLines); err != nil {
 		fmt.Fprintf(output, "error when reading query response: %s\n", err)
 		return
 	}

app/vlselect/logsql/logsql.go

@@ -73,7 +73,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 
 	// Prepare the query for hits count.
-	q.Optimize()
 	q.DropAllPipes()
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
 
@@ -204,7 +203,6 @@ func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *htt
 	}
 
 	// Obtain field names for the given query
-	q.Optimize()
 	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain field names: %s", err)
@@ -244,7 +242,6 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 	}
 
 	// Obtain unique values for the given field
-	q.Optimize()
 	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain values for field %q: %s", fieldName, err)
@@ -267,7 +264,6 @@ func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter,
 	}
 
 	// Obtain stream field names for the given query
-	q.Optimize()
 	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
@@ -306,7 +302,6 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
 	}
 
 	// Obtain stream field values for the given query and the given fieldName
-	q.Optimize()
 	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
@@ -338,7 +333,6 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
 	}
 
 	// Obtain streamIDs for the given query
-	q.Optimize()
 	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
@@ -370,7 +364,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	// Obtain streams for the given query
-	q.Optimize()
 	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
@@ -398,7 +391,6 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 			"see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
 		return
 	}
-	q.Optimize()
 
 	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
 	if err != nil {
@@ -407,13 +399,28 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 	}
 	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
 
+	startOffsetMsecs, err := httputils.GetDuration(r, "start_offset", 5*1000)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	startOffset := startOffsetMsecs * 1e6
+
+	offsetMsecs, err := httputils.GetDuration(r, "offset", 1000)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	offset := offsetMsecs * 1e6
+
 	ctxWithCancel, cancel := context.WithCancel(ctx)
 	tp := newTailProcessor(cancel)
 
 	ticker := time.NewTicker(refreshInterval)
 	defer ticker.Stop()
 
-	end := time.Now().UnixNano()
+	end := time.Now().UnixNano() - offset
+	start := end - startOffset
 	doneCh := ctxWithCancel.Done()
 	flusher, ok := w.(http.Flusher)
 	if !ok {
@@ -421,14 +428,7 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 	}
 	qOrig := q
 	for {
-		start := end - tailOffsetNsecs
+		q = qOrig.CloneWithTimeFilter(end, start, end)
-		end = time.Now().UnixNano()
-
-		q = qOrig.Clone(end)
-		q.AddTimeFilter(start, end)
-		// q.Optimize() call is needed for converting '*' into filterNoop.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
-		q.Optimize()
 		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, tp.writeBlock); err != nil {
 			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
 			return
@@ -447,6 +447,8 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		case <-doneCh:
 			return
 		case <-ticker.C:
+			start = end - tailOffsetNsecs
+			end = time.Now().UnixNano() - offset
 		}
 	}
 }
@@ -605,8 +607,6 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 		return
 	}
 
-	q.Optimize()
-
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
@@ -717,8 +717,6 @@ func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *htt
 		return
 	}
 
-	q.Optimize()
-
 	var rows []statsRow
 	var rowsLock sync.Mutex
 
@@ -818,7 +816,6 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 
 		q.AddPipeLimit(uint64(limit))
 	}
-	q.Optimize()
 
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		if len(columns) == 0 || len(columns[0].Values) == 0 {
@@ -849,7 +846,6 @@ type row struct {
 func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
-	q.Optimize()
 
 	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 	if err != nil {
@@ -869,11 +865,7 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	qOrig := q
 	for {
 		timestamp := qOrig.GetTimestamp()
-		q = qOrig.Clone(timestamp)
+		q = qOrig.CloneWithTimeFilter(timestamp, start, end)
-		q.AddTimeFilter(start, end)
-		// q.Optimize() call is needed for converting '*' into filterNoop.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
-		q.Optimize()
 		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 		if err != nil {
 			return nil, err
@@ -977,15 +969,30 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	}
 	tenantIDs := []logstorage.TenantID{tenantID}
 
+	// Parse optional start and end args
+	start, okStart, err := getTimeNsec(r, "start")
+	if err != nil {
+		return nil, nil, err
+	}
+	end, okEnd, err := getTimeNsec(r, "end")
+	if err != nil {
+		return nil, nil, err
+	}
+
 	// Parse optional time arg
 	timestamp, okTime, err := getTimeNsec(r, "time")
 	if err != nil {
 		return nil, nil, err
 	}
 	if !okTime {
-		// If time arg is missing, then evaluate query at the current timestamp
+		// If time arg is missing, then evaluate query either at the end timestamp (if it is set)
+		// or at the current timestamp (if end query arg isn't set)
+		if okEnd {
+			timestamp = end
+		} else {
 			timestamp = time.Now().UnixNano()
 		}
+	}
 
 	// decrease timestamp by one nanosecond in order to avoid capturing logs belonging
 	// to the first nanosecond at the next period of time (month, week, day, hour, etc.)
@@ -998,16 +1005,8 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
 	}
 
-	// Parse optional start and end args
-	start, okStart, err := getTimeNsec(r, "start")
-	if err != nil {
-		return nil, nil, err
-	}
-	end, okEnd, err := getTimeNsec(r, "end")
-	if err != nil {
-		return nil, nil, err
-	}
 	if okStart || okEnd {
+		// Add _time:[start, end] filter if start or end args were set.
 		if !okStart {
 			start = math.MinInt64
 		}

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
     "main.css": "./static/css/main.faf86aa5.css",
-    "main.js": "./static/js/main.aaafe439.js",
+    "main.js": "./static/js/main.b204330a.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.faf86aa5.css",
-    "static/js/main.aaafe439.js"
+    "static/js/main.b204330a.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.aaafe439.js"></script><link href="./static/css/main.faf86aa5.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.b204330a.js"></script><link href="./static/css/main.faf86aa5.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmalert-tool/unittest/recording.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -48,7 +49,7 @@ Outer:
 		}
 		var expSamples []parsedSample
 		for _, s := range mt.ExpSamples {
-			expLb := datasource.Labels{}
+			expLb := []prompbmarshal.Label{}
 			if s.Labels != "" {
 				metricsqlExpr, err := metricsql.Parse(s.Labels)
 				if err != nil {
@@ -64,7 +65,7 @@ Outer:
 				}
 				if len(metricsqlMetricExpr.LabelFilterss) > 0 {
 					for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-						expLb = append(expLb, datasource.Label{
+						expLb = append(expLb, prompbmarshal.Label{
 							Name:  l.Label,
 							Value: l.Value,
 						})

app/vmalert/config/types.go

@@ -71,7 +71,7 @@ func (t *Type) ValidateExpr(expr string) error {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
-		if _, err := logstorage.ParseStatsQuery(expr); err != nil {
+		if _, err := logstorage.ParseStatsQuery(expr, 0); err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
 	default:

app/vmalert/datasource/client_graphite.go

@@ -46,8 +46,8 @@ const (
 	graphitePrefix = "/graphite"
 )
 
-func (s *Client) setGraphiteReqParams(r *http.Request, query string) {
+func (c *Client) setGraphiteReqParams(r *http.Request, query string) {
-	if s.appendTypePrefix {
+	if c.appendTypePrefix {
 		r.URL.Path += graphitePrefix
 	}
 	r.URL.Path += graphitePath
@@ -58,7 +58,7 @@ func (s *Client) setGraphiteReqParams(r *http.Request, query string) {
 	q.Set("target", query)
 	q.Set("until", "now")
 
-	for k, vs := range s.extraParams {
+	for k, vs := range c.extraParams {
 		if q.Has(k) { // extraParams are prior to params in URL
 			q.Del(k)
 		}

app/vmalert/datasource/client_prom.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 
 	"github.com/valyala/fastjson"
 )
@@ -16,7 +17,8 @@ import (
 var (
 	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' or '/select/logsql/stats_query' path "+
 		"to the configured -datasource.url and -remoteRead.url")
-	disableStepParam = flag.Bool("datasource.disableStepParam", false, "Whether to disable adding 'step' param to the issued instant queries. "+
+	disableStepParam = flag.Bool("datasource.disableStepParam", false, "Whether to disable adding 'step' param in instant queries to the configured -datasource.url and -remoteRead.url. "+
+		"Only valid for prometheus datasource. "+
 		"This might be useful when using vmalert with datasources that do not support 'step' param for instant queries, like Google Managed Prometheus. "+
 		"It is not recommended to enable this flag if you use vmalert with VictoriaMetrics.")
 )
@@ -81,14 +83,14 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 		labels := metric.GetObject()
 
 		r := &pi.ms[i]
-		r.Labels = make([]Label, 0, labels.Len())
+		r.Labels = make([]prompbmarshal.Label, 0, labels.Len())
 		labels.Visit(func(key []byte, v *fastjson.Value) {
 			lv, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("error when parsing label value %q: %s", v, errLocal)
 				return
 			}
-			r.Labels = append(r.Labels, Label{
+			r.Labels = append(r.Labels, prompbmarshal.Label{
 				Name:  string(key),
 				Value: string(lv),
 			})
@@ -218,8 +220,8 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
 	return res, nil
 }
 
-func (s *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+func (c *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
-	if s.appendTypePrefix {
+	if c.appendTypePrefix {
 		r.URL.Path += "/prometheus"
 	}
 	if !*disablePathAppend {
@@ -227,22 +229,22 @@ func (s *Client) setPrometheusInstantReqParams(r *http.Request, query string, ti
 	}
 	q := r.URL.Query()
 	q.Set("time", timestamp.Format(time.RFC3339))
-	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
+	if !*disableStepParam && c.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
 	}
-	if !*disableStepParam && s.queryStep > 0 { // override step with user-specified value
+	if !*disableStepParam && c.queryStep > 0 { // override step with user-specified value
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }
 
-func (s *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
+func (c *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
-	if s.appendTypePrefix {
+	if c.appendTypePrefix {
 		r.URL.Path += "/prometheus"
 	}
 	if !*disablePathAppend {
@@ -251,11 +253,11 @@ func (s *Client) setPrometheusRangeReqParams(r *http.Request, query string, star
 	q := r.URL.Query()
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
-	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+	if c.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }

app/vmalert/datasource/client_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
@@ -144,12 +145,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected := []Metric{
 		{
-			Labels:     []Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -214,7 +215,7 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	exp := []Metric{
 		{
-			Labels:     []Label{{Value: "constantLine(10)", Name: "name"}},
+			Labels:     []prompbmarshal.Label{{Value: "constantLine(10)", Name: "name"}},
 			Timestamps: []int64{1611758403},
 			Values:     []float64{10},
 		},
@@ -236,12 +237,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected = []Metric{
 		{
-			Labels:     []Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
+			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -444,7 +445,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected := Metric{
-		Labels:     []Label{{Value: "vm_rows", Name: "__name__"}},
+		Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{13763},
 	}
@@ -475,7 +476,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected = Metric{
-		Labels:     []Label{{Value: "total", Name: "stats_result"}},
+		Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{10},
 	}

app/vmalert/datasource/client_vlogs.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-func (s *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+func (c *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
 	if !*disablePathAppend {
 		r.URL.Path += "/select/logsql/stats_query"
@@ -16,15 +16,15 @@ func (s *Client) setVLogsInstantReqParams(r *http.Request, query string, timesta
 	q.Set("time", timestamp.Format(time.RFC3339))
 	// set the `start` and `end` params if applyIntervalAsTimeFilter is enabled(time filter is missing in the rule expr),
 	// so the query will be executed in time range [timestamp - evaluationInterval, timestamp].
-	if s.applyIntervalAsTimeFilter && s.evaluationInterval > 0 {
+	if c.applyIntervalAsTimeFilter && c.evaluationInterval > 0 {
-		q.Set("start", timestamp.Add(-s.evaluationInterval).Format(time.RFC3339))
+		q.Set("start", timestamp.Add(-c.evaluationInterval).Format(time.RFC3339))
 		q.Set("end", timestamp.Format(time.RFC3339))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }
 
-func (s *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
+func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
 	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
 	if !*disablePathAppend {
 		r.URL.Path += "/select/logsql/stats_query_range"
@@ -33,11 +33,11 @@ func (s *Client) setVLogsRangeReqParams(r *http.Request, query string, start, en
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
 	// set step as evaluationInterval by default
-	if s.evaluationInterval > 0 {
+	if c.evaluationInterval > 0 {
-		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	s.setReqParams(r, query)
+	c.setReqParams(r, query)
 }
 
 func parseVLogsResponse(req *http.Request, resp *http.Response) (res Result, err error) {

app/vmalert/datasource/datasource.go

@@ -8,6 +8,8 @@ import (
 	"sort"
 	"strconv"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Querier interface wraps Query and QueryRange methods
@@ -55,7 +57,7 @@ type QuerierParams struct {
 
 // Metric is the basic entity which should be return by datasource
 type Metric struct {
-	Labels     []Label
+	Labels     []prompbmarshal.Label
 	Timestamps []int64
 	Values     []float64
 }
@@ -72,22 +74,9 @@ func (m *Metric) SetLabel(key, value string) {
 	m.AddLabel(key, value)
 }
 
-// SetLabels sets the given map as Metric labels
-func (m *Metric) SetLabels(ls map[string]string) {
-	var i int
-	m.Labels = make([]Label, len(ls))
-	for k, v := range ls {
-		m.Labels[i] = Label{
-			Name:  k,
-			Value: v,
-		}
-		i++
-	}
-}
-
 // AddLabel appends the given label to the label set
 func (m *Metric) AddLabel(key, value string) {
-	m.Labels = append(m.Labels, Label{Name: key, Value: value})
+	m.Labels = append(m.Labels, prompbmarshal.Label{Name: key, Value: value})
 }
 
 // DelLabel deletes the given label from the label set
@@ -110,14 +99,8 @@ func (m *Metric) Label(key string) string {
 	return ""
 }
 
-// Label represents metric's label
-type Label struct {
-	Name  string
-	Value string
-}
-
 // Labels is collection of Label
-type Labels []Label
+type Labels []prompbmarshal.Label
 
 func (ls Labels) Len() int           { return len(ls) }
 func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
@@ -172,7 +155,7 @@ func LabelCompare(a, b Labels) int {
 // ConvertToLabels convert map to Labels
 func ConvertToLabels(m map[string]string) (labelset Labels) {
 	for k, v := range m {
-		labelset = append(labelset, Label{
+		labelset = append(labelset, prompbmarshal.Label{
 			Name:  k,
 			Value: v,
 		})

app/vmalert/datasource/init.go

@@ -51,7 +51,7 @@ var (
 	lookBack = flag.Duration("datasource.lookback", 0, `Deprecated: please adjust "-search.latencyOffset" at datasource side `+
 		`or specify "latency_offset" in rule group's params. Lookback defines how far into the past to look when evaluating queries. `+
 		`For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
-	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
+	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries to the configured -datasource.url and -remoteRead.url. Only valid for prometheus datasource. "+
 		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
 		"If set to 0, rule's evaluation interval will be used instead.")
 	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Deprecated: please use "eval_alignment" in rule group instead. `+
@@ -62,8 +62,8 @@ var (
 	idleConnectionTimeout = flag.Duration("datasource.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 	disableKeepAlive      = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
 		`If true, disables HTTP keep-alive and will only use the connection to the server for a single HTTP request.`)
-	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests which limits the number of digits after the decimal point in response values. `+
-		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
+		`Only valid for VictoriaMetrics as the datasource.`)
 )
 
 // InitSecretFlags must be called after flag.Parse and before any logging

app/vmalert/datasource/vm_prom_api_test.go

@@ -3,6 +3,8 @@ package datasource
 import (
 	"reflect"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestPromInstant_UnmarshalPositive(t *testing.T) {
@@ -21,7 +23,7 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 
 	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"42"]}]`, []Metric{
 		{
-			Labels:     []Label{{Name: "__name__", Value: "up"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
@@ -31,17 +33,17 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 {"metric":{"__name__":"foo"},"value":[1583780001,"7"]},
 {"metric":{"__name__":"baz", "instance":"bar"},"value":[1583780002,"8"]}]`, []Metric{
 		{
-			Labels:     []Label{{Name: "__name__", Value: "up"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
 		{
-			Labels:     []Label{{Name: "__name__", Value: "foo"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "foo"}},
 			Timestamps: []int64{1583780001},
 			Values:     []float64{7},
 		},
 		{
-			Labels:     []Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
+			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
 			Timestamps: []int64{1583780002},
 			Values:     []float64{8},
 		},

app/vmalert/notifier/alert.go

@@ -167,14 +167,8 @@ type tplData struct {
 	ExternalURL    string
 }
 
-func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error {
+func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
-	tpl, err := tmpl.Clone()
+	tpl, err := tpl.Parse(text)
-	if err != nil {
-		return fmt.Errorf("error cloning template before parse annotation: %w", err)
-	}
-	// Clone() doesn't copy tpl Options, so we set them manually
-	tpl = tpl.Option("missingkey=zero")
-	tpl, err = tpl.Parse(text)
 	if err != nil {
 		return fmt.Errorf("error parsing annotation template: %w", err)
 	}

app/vmalert/notifier/alert_test.go

@@ -33,7 +33,7 @@ func TestAlertExecTemplate(t *testing.T) {
 		qFn := func(_ string) ([]datasource.Metric, error) {
 			return []datasource.Metric{
 				{
-					Labels: []datasource.Label{
+					Labels: []prompbmarshal.Label{
 						{Name: "foo", Value: "bar"},
 						{Name: "baz", Value: "qux"},
 					},
@@ -41,7 +41,7 @@ func TestAlertExecTemplate(t *testing.T) {
 					Timestamps: []int64{1},
 				},
 				{
-					Labels: []datasource.Label{
+					Labels: []prompbmarshal.Label{
 						{Name: "foo", Value: "garply"},
 						{Name: "baz", Value: "fred"},
 					},

app/vmalert/rule/alerting.go

@@ -14,8 +14,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertingRule is basic alert entity
@@ -454,13 +456,16 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
 	var numActivePending int
+	var tss []prompbmarshal.TimeSeries
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
 		if _, ok := updated[h]; !ok {
 			if a.State == notifier.StatePending {
-				// alert was in Pending state - it is not
+				// alert was in Pending state - it is not active anymore
-				// active anymore
+				// add stale time series
+				tss = append(tss, pendingAlertStaleTimeSeries(a.Labels, ts.Unix(), true)...)
+
 				delete(ar.alerts, h)
 				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
 				continue
@@ -478,6 +483,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
+					// add stale time series
+					tss = append(tss, firingAlertStaleTimeSeries(a.Labels, ts.Unix())...)
+
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
 					continue
 				}
@@ -489,6 +497,10 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.State = notifier.StateFiring
 			a.Start = ts
 			alertsFired.Inc()
+			if ar.For > 0 {
+				// add stale time series
+				tss = append(tss, pendingAlertStaleTimeSeries(a.Labels, ts.Unix(), false)...)
+			}
 			ar.logDebugf(ts, a, "PENDING => FIRING: %s since becoming active at %v", ts.Sub(a.ActiveAt), a.ActiveAt)
 		}
 	}
@@ -497,7 +509,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		curState.Err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
 		return nil, curState.Err
 	}
-	return ar.toTimeSeries(ts.Unix()), nil
+	return append(tss, ar.toTimeSeries(ts.Unix())...), nil
 }
 
 func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
@@ -522,6 +534,7 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 	return ls, as, nil
 }
 
+// toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	for _, a := range ar.alerts {
@@ -601,26 +614,83 @@ func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
+	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
-		labels[k] = v
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// __name__ already been dropped, no need to check duplication
+	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	if ol := promrelabel.GetLabelByName(labels, alertStateLabel); ol != nil {
+		ol.Value = a.State.String()
+	} else {
+		labels = append(labels, prompbmarshal.Label{Name: alertStateLabel, Value: a.State.String()})
 	}
-	labels["__name__"] = alertMetricName
-	labels[alertStateLabel] = a.State.String()
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
-// alertForToTimeSeries returns a timeseries that represents
+// alertForToTimeSeries returns a time series that represents
 // state of active alerts, where value is time when alert become active
 func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
+	var labels []prompbmarshal.Label
 	for k, v := range a.Labels {
-		labels[k] = v
+		labels = append(labels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
 	}
-	labels["__name__"] = alertForStateMetricName
+	// __name__ already been dropped, no need to check duplication
+	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
+// pendingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
+// for alerts which changed their state from Pending to Inactive or Firing.
+func pendingAlertStaleTimeSeries(ls map[string]string, timestamp int64, includeAlertForState bool) []prompbmarshal.TimeSeries {
+	var result []prompbmarshal.TimeSeries
+	var baseLabels []prompbmarshal.Label
+	for k, v := range ls {
+		baseLabels = append(baseLabels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// __name__ already been dropped, no need to check duplication
+	alertsLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StatePending.String()})
+	result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels))
+
+	if includeAlertForState {
+		alertsForStateLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
+		result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsForStateLabels))
+	}
+	return result
+}
+
+// firingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
+// for alerts which changed their state from Firing to Inactive.
+func firingAlertStaleTimeSeries(ls map[string]string, timestamp int64) []prompbmarshal.TimeSeries {
+	var baseLabels []prompbmarshal.Label
+	for k, v := range ls {
+		baseLabels = append(baseLabels, prompbmarshal.Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// __name__ already been dropped, no need to check duplication
+	alertsLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
+	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StateFiring.String()})
+
+	alertsForStateLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
+
+	return []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels),
+		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsForStateLabels),
+	}
+}
+
 // restore restores the value of ActiveAt field for active alerts,
 // based on previously written time series `alertForStateMetricName`.
 // Only rules with For > 0 can be restored.

app/vmalert/rule/alerting_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -28,7 +29,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		rule.alerts[alert.ID] = alert
 		tss := rule.toTimeSeries(timestamp.Unix())
 		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries mismatch: %s", err)
+			t.Fatalf("timeseries mismatch for rule %q: %s", rule.Name, err)
 		}
 	}
 
@@ -36,14 +37,23 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__":      alertMetricName,
+			{
-			alertStateLabel: notifier.StateFiring.String(),
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StateFiring.String(),
+			},
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
+			[]prompbmarshal.Label{
-				"__name__": alertForStateMetricName,
+				{
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
 			}),
 	})
 
@@ -54,18 +64,40 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 			"instance": "bar",
 		},
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
-			"__name__":      alertMetricName,
+			[]prompbmarshal.Label{
-			alertStateLabel: notifier.StateFiring.String(),
+				{
-			"job":           "foo",
+					Name:  "__name__",
-			"instance":      "bar",
+					Value: alertMetricName,
+				},
+				{
+					Name:  alertStateLabel,
+					Value: notifier.StateFiring.String(),
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+				{
+					Name:  "instance",
+					Value: "bar",
+				},
 			}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
+			[]prompbmarshal.Label{
-				"__name__": alertForStateMetricName,
+				{
-				"job":      "foo",
+					Name:  "__name__",
-				"instance": "bar",
+					Value: alertForStateMetricName,
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+				{
+					Name:  "instance",
+					Value: "bar",
+				},
 			}),
 	})
 
@@ -73,18 +105,29 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
 		Labels: map[string]string{
 			alertStateLabel: "foo",
-			"__name__":      "bar",
 		},
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__":      alertMetricName,
+			{
-			alertStateLabel: notifier.StateFiring.String(),
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StateFiring.String(),
+			},
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
+			[]prompbmarshal.Label{
-				"__name__":      alertForStateMetricName,
+				{
-				alertStateLabel: "foo",
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
+				{
+					Name:  alertStateLabel,
+					Value: "foo",
+				},
 			}),
 	})
 
@@ -92,14 +135,23 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__":      alertMetricName,
+			{
-			alertStateLabel: notifier.StateFiring.String(),
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StateFiring.String(),
+			},
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			map[string]string{
+			[]prompbmarshal.Label{
-				"__name__": alertForStateMetricName,
+				{
+					Name:  "__name__",
+					Value: alertForStateMetricName,
+				},
 			}),
 	})
 
@@ -107,12 +159,21 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StatePending,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__":      alertMetricName,
+			{
-			alertStateLabel: notifier.StatePending.String(),
+				Name:  "__name__",
+				Value: alertMetricName,
+			},
+			{
+				Name:  alertStateLabel,
+				Value: notifier.StatePending.String(),
+			},
 		}),
-		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__": alertForStateMetricName,
+			{
+				Name:  "__name__",
+				Value: alertForStateMetricName,
+			},
 		}),
 	})
 }
@@ -124,7 +185,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 		alert  *notifier.Alert
 	}
 
-	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert) {
+	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
+
+	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert, tssExpected map[int][]prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
@@ -134,13 +197,19 @@ func TestAlertingRule_Exec(t *testing.T) {
 			Name: "TestRule_Exec",
 		}
 		rule.GroupID = fakeGroup.ID()
-		ts := time.Now()
 		for i, step := range steps {
 			fq.Reset()
 			fq.Add(step...)
-			if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
+			tss, err := rule.exec(context.TODO(), ts, 0)
+			if err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
+			// check generate time series
+			if _, ok := tssExpected[i]; ok {
+				if err := compareTimeSeries(t, tssExpected[i], tss); err != nil {
+					t.Fatalf("generated time series mismatch for rule %q in step %d: %s", rule.Name, i, err)
+				}
+			}
 
 			// shift the execution timestamp before the next iteration
 			ts = ts.Add(defaultStep)
@@ -174,12 +243,20 @@ func TestAlertingRule_Exec(t *testing.T) {
 		}
 	}
 
-	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil)
+	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil, nil)
 
-	f(newTestAlertingRule("empty labels", 0), [][]datasource.Metric{
+	f(newTestAlertingRule("empty_labels", 0), [][]datasource.Metric{
 		{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 	}, map[int][]testAlert{
 		0: {{alert: &notifier.Alert{State: notifier.StateFiring}}},
+	},
+		map[int][]prompbmarshal.TimeSeries{
+			0: {
+				{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "empty_labels"}, {Name: "alertstate", Value: "firing"}},
+					Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+				{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "empty_labels"}},
+					Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+			},
 		})
 
 	f(newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0), [][]datasource.Metric{
@@ -194,6 +271,25 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
+		2: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive=>firing", 0), [][]datasource.Metric{
@@ -210,7 +306,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRule("multiple-firing", 0), [][]datasource.Metric{
 		{
@@ -224,7 +320,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 		},
-	})
+	}, nil)
 
 	// 1: fire first alert
 	// 2: fire second alert, set first inactive
@@ -233,8 +329,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo1")},
 		{metricWithLabels(t, "name", "foo2")},
-	},
+	}, map[int][]testAlert{
-		map[int][]testAlert{
 		0: {
 			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 		},
@@ -247,13 +342,44 @@ func TestAlertingRule_Exec(t *testing.T) {
 			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
 			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 		},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			// stale time series for foo, `firing -> inactive`
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			// new time series for foo1
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
+		2: {
+			// stale time series for foo1
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			// new time series for foo2
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo2"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo2"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("for-pending", time.Minute), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRule("for-fired", defaultStep), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -261,6 +387,22 @@ func TestAlertingRule_Exec(t *testing.T) {
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			// stale time series for `pending -> firing`
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("for-pending=>empty", time.Second), [][]datasource.Metric{
@@ -272,6 +414,26 @@ func TestAlertingRule_Exec(t *testing.T) {
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		2: {},
+	}, map[int][]prompbmarshal.TimeSeries{
+		0: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
+		},
+		1: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
+		},
+		// stale time series for `pending -> inactive`
+		2: {
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
+				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
+		},
 	})
 
 	f(newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep), [][]datasource.Metric{
@@ -287,7 +449,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>firing", defaultStep, 0, defaultStep, nil), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -300,7 +462,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 0, 2*defaultStep, nil), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -321,7 +483,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		6: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	})
+	}, nil)
 }
 
 func TestAlertingRuleExecRange(t *testing.T) {
@@ -477,7 +639,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
@@ -523,7 +685,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
+			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{
@@ -1047,7 +1209,7 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 
 func TestAlertingRule_ToLabels(t *testing.T) {
 	metric := datasource.Metric{
-		Labels: []datasource.Label{
+		Labels: []prompbmarshal.Label{
 			{Name: "instance", Value: "0.0.0.0:8800"},
 			{Name: "group", Value: "vmalert"},
 			{Name: "alertname", Value: "ConfigurationReloadFailure"},

app/vmalert/rule/group.go

@@ -8,12 +8,9 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
-	"strconv"
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-
 	"github.com/cheggaaa/pb/v3"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -21,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
@@ -353,7 +349,6 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 		Rw:              rw,
 		Notifiers:       nts,
 		notifierHeaders: g.NotifierHeaders,
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 
 	g.infof("started")
@@ -426,8 +421,6 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 				continue
 			}
 
-			// ensure that staleness is tracked for existing rules only
-			e.purgeStaleSeries(g.Rules)
 			e.notifierHeaders = g.NotifierHeaders
 			g.mu.Unlock()
 
@@ -542,7 +535,6 @@ func (g *Group) ExecOnce(ctx context.Context, nts func() []notifier.Notifier, rw
 		Rw:              rw,
 		Notifiers:       nts,
 		notifierHeaders: g.NotifierHeaders,
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 	if len(g.Rules) < 1 {
 		return nil
@@ -633,13 +625,6 @@ type executor struct {
 	notifierHeaders map[string]string
 
 	Rw remotewrite.RWClient
-
-	previouslySentSeriesToRWMu sync.Mutex
-	// previouslySentSeriesToRW stores series sent to RW on previous iteration
-	// map[ruleID]map[ruleLabels][]prompb.Label
-	// where `ruleID` is ID of the Rule within a Group
-	// and `ruleLabels` is []prompb.Label marshalled to a string
-	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
 // execConcurrently executes rules concurrently if concurrency>1
@@ -706,11 +691,6 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		if err := pushToRW(tss); err != nil {
 			return err
 		}
-
-		staleSeries := e.getStaleSeries(r, tss, ts)
-		if err := pushToRW(staleSeries); err != nil {
-			return err
-		}
 	}
 
 	ar, ok := r.(*AlertingRule)
@@ -737,79 +717,3 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	wg.Wait()
 	return errGr.Err()
 }
-
-var bbPool bytesutil.ByteBufferPool
-
-// getStaleSeries checks whether there are stale series from previously sent ones.
-func (e *executor) getStaleSeries(r Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
-	bb := bbPool.Get()
-	defer bbPool.Put(bb)
-
-	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
-	for _, ts := range tss {
-		// convert labels to strings, so we can compare with previously sent series
-		bb.B = labelsToString(bb.B, ts.Labels)
-		ruleLabels[string(bb.B)] = ts.Labels
-		bb.Reset()
-	}
-
-	rID := r.ID()
-	var staleS []prompbmarshal.TimeSeries
-	// check whether there are series which disappeared and need to be marked as stale
-	e.previouslySentSeriesToRWMu.Lock()
-	for key, labels := range e.previouslySentSeriesToRW[rID] {
-		if _, ok := ruleLabels[key]; ok {
-			continue
-		}
-		// previously sent series are missing in current series, so we mark them as stale
-		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
-		staleS = append(staleS, ss)
-	}
-	// set previous series to current
-	e.previouslySentSeriesToRW[rID] = ruleLabels
-	e.previouslySentSeriesToRWMu.Unlock()
-
-	return staleS
-}
-
-// purgeStaleSeries deletes references in tracked
-// previouslySentSeriesToRW list to Rules which aren't present
-// in the given activeRules list. The method is used when the list
-// of loaded rules has changed and executor has to remove
-// references to non-existing rules.
-func (e *executor) purgeStaleSeries(activeRules []Rule) {
-	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
-
-	e.previouslySentSeriesToRWMu.Lock()
-
-	for _, rule := range activeRules {
-		id := rule.ID()
-		prev, ok := e.previouslySentSeriesToRW[id]
-		if ok {
-			// keep previous series for staleness detection
-			newPreviouslySentSeriesToRW[id] = prev
-		}
-	}
-	e.previouslySentSeriesToRW = nil
-	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
-
-	e.previouslySentSeriesToRWMu.Unlock()
-}
-
-func labelsToString(dst []byte, labels []prompbmarshal.Label) []byte {
-	dst = append(dst, '{')
-	for i, label := range labels {
-		if len(label.Name) == 0 {
-			dst = append(dst, "__name__"...)
-		} else {
-			dst = append(dst, label.Name...)
-		}
-		dst = append(dst, '=')
-		dst = strconv.AppendQuote(dst, label.Value)
-		if i < len(labels)-1 {
-			dst = append(dst, ',')
-		}
-	}
-	dst = append(dst, '}')
-	return dst
-}

app/vmalert/rule/group_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -17,8 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -383,153 +380,6 @@ func TestGetResolveDuration(t *testing.T) {
 	f(2*time.Minute, 0, 1*time.Minute, 8*time.Minute)
 }
 
-func TestGetStaleSeries(t *testing.T) {
-	ts := time.Now()
-	e := &executor{
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-	}
-	f := func(r Rule, labels, expLabels [][]prompbmarshal.Label) {
-		t.Helper()
-
-		var tss []prompbmarshal.TimeSeries
-		for _, l := range labels {
-			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
-		}
-		staleS := e.getStaleSeries(r, tss, ts)
-		if staleS == nil && expLabels == nil {
-			return
-		}
-		if len(staleS) != len(expLabels) {
-			t.Fatalf("expected to get %d stale series, got %d",
-				len(expLabels), len(staleS))
-		}
-		for i, exp := range expLabels {
-			got := staleS[i]
-			if !reflect.DeepEqual(exp, got.Labels) {
-				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
-					exp, got.Labels)
-			}
-			if len(got.Samples) != 1 {
-				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
-			}
-			if !decimal.IsStaleNaN(got.Samples[0].Value) {
-				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
-			}
-		}
-	}
-
-	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
-
-	// single series
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		nil,
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
-	f(&AlertingRule{RuleID: 1},
-		nil,
-		nil)
-
-	// multiple series
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{
-			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
-			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
-		},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		nil,
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
-
-	// multiple rules and series
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{
-			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
-			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
-		},
-		nil)
-	f(&AlertingRule{RuleID: 2},
-		[][]prompbmarshal.Label{
-			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
-			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
-		},
-		nil)
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
-	f(&AlertingRule{RuleID: 1},
-		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
-		nil)
-}
-
-func TestPurgeStaleSeries(t *testing.T) {
-	ts := time.Now()
-	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
-	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
-
-	f := func(curRules, newRules, expStaleRules []Rule) {
-		t.Helper()
-		e := &executor{
-			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-		}
-		// seed executor with series for
-		// current rules
-		for _, rule := range curRules {
-			e.getStaleSeries(rule, tss, ts)
-		}
-
-		e.purgeStaleSeries(newRules)
-
-		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
-			t.Fatalf("expected to get %d stale series, got %d",
-				len(expStaleRules), len(e.previouslySentSeriesToRW))
-		}
-
-		for _, exp := range expStaleRules {
-			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
-				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
-			}
-		}
-	}
-
-	f(nil, nil, nil)
-	f(
-		nil,
-		[]Rule{&AlertingRule{RuleID: 1}},
-		nil,
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}},
-		nil,
-		nil,
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}},
-		[]Rule{&AlertingRule{RuleID: 2}},
-		nil,
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 2}},
-	)
-	f(
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
-	)
-}
-
 func TestFaultyNotifier(t *testing.T) {
 	fq := &datasource.FakeQuerier{}
 	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
@@ -581,7 +431,6 @@ func TestFaultyRW(t *testing.T) {
 
 	e := &executor{
 		Rw: &remotewrite.Client{},
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 
 	err := e.exec(context.Background(), r, time.Now(), 0, 10)

app/vmalert/rule/group_timing_test.go

@@ -1,36 +0,0 @@
-package rule
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-)
-
-func BenchmarkGetStaleSeries(b *testing.B) {
-	ts := time.Now()
-	n := 100
-	payload := make([]prompbmarshal.TimeSeries, 0, n)
-	for i := 0; i < n; i++ {
-		s := fmt.Sprintf("%d", i)
-		labels := toPromLabels(b,
-			"__name__", "foo", ""+
-				"instance", s,
-			"job", s,
-			"state", s,
-		)
-		payload = append(payload, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels))
-	}
-
-	e := &executor{
-		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-	}
-	ar := &AlertingRule{RuleID: 1}
-
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		e.getStaleSeries(ar, payload, ts)
-	}
-}

app/vmalert/rule/recording.go

@@ -3,15 +3,17 @@ package rule
 import (
 	"context"
 	"fmt"
-	"sort"
 	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // RecordingRule is a Rule that supposed
@@ -33,6 +35,8 @@ type RecordingRule struct {
 	// during evaluations
 	state *ruleState
 
+	lastEvaluation map[string]struct{}
+
 	metrics *recordingRuleMetrics
 }
 
@@ -112,7 +116,7 @@ func (rr *RecordingRule) execRange(ctx context.Context, start, end time.Time) ([
 	var tss []prompbmarshal.TimeSeries
 	for _, s := range res.Data {
 		ts := rr.toTimeSeries(s)
-		key := stringifyLabels(ts)
+		key := stringifyLabels(ts.Labels)
 		if _, ok := duplicates[key]; ok {
 			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", s.Labels, key, errDuplicate)
 		}
@@ -154,28 +158,47 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 		return nil, curState.Err
 	}
 
-	duplicates := make(map[string]struct{}, len(qMetrics))
+	curEvaluation := make(map[string]struct{}, len(qMetrics))
+	lastEvaluation := rr.lastEvaluation
 	var tss []prompbmarshal.TimeSeries
 	for _, r := range qMetrics {
 		ts := rr.toTimeSeries(r)
-		key := stringifyLabels(ts)
+		key := stringifyLabels(ts.Labels)
-		if _, ok := duplicates[key]; ok {
+		if _, ok := curEvaluation[key]; ok {
 			curState.Err = fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
 			return nil, curState.Err
 		}
-		duplicates[key] = struct{}{}
+		curEvaluation[key] = struct{}{}
+		delete(lastEvaluation, key)
 		tss = append(tss, ts)
 	}
+	// check for stale time series
+	for k := range lastEvaluation {
+		tss = append(tss, prompbmarshal.TimeSeries{
+			Labels: stringToLabels(k),
+			Samples: []prompbmarshal.Sample{
+				{Value: decimal.StaleNaN, Timestamp: ts.UnixNano() / 1e6},
+			}})
+	}
+	rr.lastEvaluation = curEvaluation
 	return tss, nil
 }
 
-func stringifyLabels(ts prompbmarshal.TimeSeries) string {
+func stringToLabels(s string) []prompbmarshal.Label {
-	labels := ts.Labels
+	labels := strings.Split(s, ",")
-	if len(labels) > 1 {
+	rLabels := make([]prompbmarshal.Label, 0, len(labels))
-		sort.Slice(labels, func(i, j int) bool {
+	for i := range labels {
-			return labels[i].Name < labels[j].Name
+		if label := strings.Split(labels[i], "="); len(label) == 2 {
+			rLabels = append(rLabels, prompbmarshal.Label{
+				Name:  label[0],
+				Value: label[1],
 			})
 		}
+	}
+	return rLabels
+}
+
+func stringifyLabels(labels []prompbmarshal.Label) string {
 	b := strings.Builder{}
 	for i, l := range labels {
 		b.WriteString(l.Name)
@@ -189,19 +212,27 @@ func stringifyLabels(ts prompbmarshal.TimeSeries) string {
 }
 
 func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSeries {
-	labels := make(map[string]string)
+	if preN := promrelabel.GetLabelByName(m.Labels, "__name__"); preN != nil {
-	for _, l := range m.Labels {
+		preN.Value = rr.Name
-		labels[l.Name] = l.Value
+	} else {
+		m.Labels = append(m.Labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: rr.Name,
+		})
 	}
-	labels["__name__"] = rr.Name
+	for k := range rr.Labels {
-	// override existing labels with configured ones
+		prevLabel := promrelabel.GetLabelByName(m.Labels, k)
-	for k, v := range rr.Labels {
+		if prevLabel != nil && prevLabel.Value != rr.Labels[k] {
-		if _, ok := labels[k]; ok && labels[k] != v {
+			// Rename the prevLabel to "exported_" + label.Name
-			labels[fmt.Sprintf("exported_%s", k)] = labels[k]
+			prevLabel.Name = fmt.Sprintf("exported_%s", prevLabel.Name)
 		}
-		labels[k] = v
+		m.Labels = append(m.Labels, prompbmarshal.Label{
+			Name:  k,
+			Value: rr.Labels[k],
+		})
 	}
-	return newTimeSeries(m.Values, m.Timestamps, labels)
+	ts := newTimeSeries(m.Values, m.Timestamps, m.Labels)
+	return ts
 }
 
 // updateWith copies all significant fields.
@@ -221,6 +252,9 @@ func setIntervalAsTimeFilter(dType, expr string) bool {
 	if dType != "vlogs" {
 		return false
 	}
-	q, _ := logstorage.ParseStatsQuery(expr)
+	q, err := logstorage.ParseStatsQuery(expr, 0)
-	return !q.ContainAnyTimeFilter()
+	if err != nil {
+		logger.Panicf("BUG: the LogsQL query must be valid here; got error: %s; query=[%s]", err, expr)
+	}
+	return !q.HasGlobalTimeFilter()
 }

app/vmalert/rule/recording_test.go

@@ -9,59 +9,131 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestRecordingRule_Exec(t *testing.T) {
-	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
+	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
+	const defaultStep = 5 * time.Millisecond
+
+	f := func(rule *RecordingRule, steps [][]datasource.Metric, tssExpected [][]prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
-		fq.Add(metrics...)
+		for i, step := range steps {
+			fq.Reset()
+			fq.Add(step...)
 			rule.q = fq
 			rule.state = &ruleState{
 				entries: make([]StateEntry, 10),
 			}
-		tss, err := rule.exec(context.TODO(), time.Now(), 0)
+			tss, err := rule.exec(context.TODO(), ts, 0)
 			if err != nil {
-			t.Fatalf("unexpected RecordingRule.exec error: %s", err)
+				t.Fatalf("fail to test rule %s: unexpected error: %s", rule.Name, err)
-		}
-		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
 			}
+			if err := compareTimeSeries(t, tssExpected[i], tss); err != nil {
+				t.Fatalf("fail to test rule %s: time series mismatch on step %d: %s", rule.Name, i, err)
 			}
 
-	timestamp := time.Now()
+			ts = ts.Add(defaultStep)
+		}
+	}
 
 	f(&RecordingRule{
 		Name: "foo",
-	}, []datasource.Metric{
+	}, [][]datasource.Metric{{
 		metricWithValueAndLabels(t, 10, "__name__", "bar"),
-	}, []prompbmarshal.TimeSeries{
+	}}, [][]prompbmarshal.TimeSeries{{
-		newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{10}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			"__name__": "foo",
+			{
+				Name:  "__name__",
+				Value: "foo",
+			},
 		}),
-	})
+	}})
 
 	f(&RecordingRule{
 		Name: "foobarbaz",
-	}, []datasource.Metric{
+	}, [][]datasource.Metric{
+		{
 			metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
 			metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
-		metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+		},
-	}, []prompbmarshal.TimeSeries{
+		{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			metricWithValueAndLabels(t, 10, "__name__", "foo", "job", "foo"),
-			"__name__": "foobarbaz",
+		},
-			"job":      "foo",
+		{
+			metricWithValueAndLabels(t, 10, "__name__", "foo", "job", "bar"),
+		},
+	}, [][]prompbmarshal.TimeSeries{
+		{
+			newTimeSeries([]float64{1}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
 			}),
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			"__name__": "foobarbaz",
+				{
-			"job":      "bar",
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
 			}),
-		newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
+		},
-			"__name__": "foobarbaz",
+		{
-			"job":      "baz",
+			newTimeSeries([]float64{10}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
 			}),
+			// stale time series
+			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+			}),
+		},
+		{
+			newTimeSeries([]float64{10}, []int64{ts.Add(2 * defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+			}),
+			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(2 * defaultStep).UnixNano()}, []prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "foo",
+				},
+			}),
+		},
 	})
 
 	f(&RecordingRule{
@@ -69,22 +141,44 @@ func TestRecordingRule_Exec(t *testing.T) {
 		Labels: map[string]string{
 			"source": "test",
 		},
-	}, []datasource.Metric{
+	}, [][]datasource.Metric{{
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
-	}, []prompbmarshal.TimeSeries{
+	}}, [][]prompbmarshal.TimeSeries{{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			"__name__": "job:foo",
+			{
-			"job":      "foo",
+				Name:  "__name__",
-			"source":   "test",
+				Value: "job:foo",
+			},
+			{
+				Name:  "job",
+				Value: "foo",
+			},
+			{
+				Name:  "source",
+				Value: "test",
+			},
 		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{ts.UnixNano()},
-			"__name__":        "job:foo",
+			[]prompbmarshal.Label{
-			"job":             "bar",
+				{
-			"source":          "test",
+					Name:  "__name__",
-			"exported_source": "origin",
+					Value: "job:foo",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+				{
+					Name:  "source",
+					Value: "test",
+				},
+				{
+					Name:  "exported_source",
+					Value: "origin",
+				},
 			}),
-	})
+	}})
 }
 
 func TestRecordingRule_ExecRange(t *testing.T) {
@@ -110,8 +204,12 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValuesAndLabels(t, []float64{10, 20, 30}, "__name__", "bar"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
-			"__name__": "foo",
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "foo",
+				},
 			}),
 	})
 
@@ -122,18 +220,36 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__": "foobarbaz",
+			{
-			"job":      "foo",
+				Name:  "__name__",
+				Value: "foobarbaz",
+			},
+			{
+				Name:  "job",
+				Value: "foo",
+			},
 		}),
-		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__": "foobarbaz",
+			{
-			"job":      "bar",
+				Name:  "__name__",
+				Value: "foobarbaz",
+			},
+			{
+				Name:  "job",
+				Value: "bar",
+			},
 		}),
 		newTimeSeries([]float64{4, 5, 6},
-			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, []prompbmarshal.Label{
-				"__name__": "foobarbaz",
+				{
-				"job":      "baz",
+					Name:  "__name__",
+					Value: "foobarbaz",
+				},
+				{
+					Name:  "job",
+					Value: "baz",
+				},
 			}),
 	})
 
@@ -146,15 +262,34 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			"__name__": "job:foo",
+			{
-			"job":      "foo",
+				Name:  "__name__",
-			"source":   "test",
+				Value: "job:foo",
+			},
+			{
+				Name:  "job",
+				Value: "foo",
+			},
+			{
+				Name:  "source",
+				Value: "test",
+			},
 		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
-			"__name__": "job:foo",
+			[]prompbmarshal.Label{
-			"job":      "bar",
+				{
-			"source":   "test",
+					Name:  "__name__",
+					Value: "job:foo",
+				},
+				{
+					Name:  "job",
+					Value: "bar",
+				},
+				{
+					Name:  "source",
+					Value: "test",
+				},
 			}),
 	})
 }

app/vmalert/rule/test_helpers.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -87,7 +88,7 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	}
 	m := datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}
 	for i := 0; i < len(labels); i += 2 {
-		m.Labels = append(m.Labels, datasource.Label{
+		m.Labels = append(m.Labels, prompbmarshal.Label{
 			Name:  labels[i],
 			Value: labels[i+1],
 		})
@@ -95,21 +96,6 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
-func toPromLabels(t testing.TB, labels ...string) []prompbmarshal.Label {
-	t.Helper()
-	if len(labels) == 0 || len(labels)%2 != 0 {
-		t.Fatalf("expected to get even number of labels")
-	}
-	var ls []prompbmarshal.Label
-	for i := 0; i < len(labels); i += 2 {
-		ls = append(ls, prompbmarshal.Label{
-			Name:  labels[i],
-			Value: labels[i+1],
-		})
-	}
-	return ls
-}
-
 func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 	t.Helper()
 	if len(a) != len(b) {
@@ -122,7 +108,7 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 		}
 		for i, exp := range expTS.Samples {
 			got := gotTS.Samples[i]
-			if got.Value != exp.Value {
+			if got.Value != exp.Value && (!decimal.IsStaleNaN(got.Value) || !decimal.IsStaleNaN(exp.Value)) {
 				return fmt.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
 			}
 			// timestamp validation isn't always correct for now.

app/vmalert/rule/utils.go

@@ -9,10 +9,14 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func newTimeSeries(values []float64, timestamps []int64, labels map[string]string) prompbmarshal.TimeSeries {
+// newTimeSeries first sorts given labels, then returns new time series.
+func newTimeSeries(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
+	promrelabel.SortLabels(labels)
 	ts := prompbmarshal.TimeSeries{
+		Labels:  labels,
 		Samples: make([]prompbmarshal.Sample, len(values)),
 	}
 	for i := range values {
@@ -21,34 +25,6 @@ func newTimeSeries(values []float64, timestamps []int64, labels map[string]strin
 			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
 		}
 	}
-	keys := make([]string, 0, len(labels))
-	for k := range labels {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys) // make order deterministic
-	for _, key := range keys {
-		ts.Labels = append(ts.Labels, prompbmarshal.Label{
-			Name:  key,
-			Value: labels[key],
-		})
-	}
-	return ts
-}
-
-// newTimeSeriesPB creates prompbmarshal.TimeSeries with given
-// values, timestamps and labels.
-// It expects that labels are already sorted.
-func newTimeSeriesPB(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
-	ts := prompbmarshal.TimeSeries{
-		Samples: make([]prompbmarshal.Sample, len(values)),
-	}
-	for i := range values {
-		ts.Samples[i] = prompbmarshal.Sample{
-			Value:     values[i],
-			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
-		}
-	}
-	ts.Labels = labels
 	return ts
 }
 

app/vmalert/templates/template.go

@@ -169,6 +169,8 @@ func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tmpl = tmpl.Option("missingkey=zero")
 	return tmpl.Funcs(funcs), nil
 }
 

app/vmauth/auth_config.go

@@ -462,7 +462,6 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 
 	// Slow path - select other backend urls.
 	n := atomicCounter.Add(1) - 1
-
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
@@ -484,7 +483,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		if bu.isBroken() {
 			continue
 		}
-		if n := bu.concurrentRequests.Load(); n < minRequests {
+		if n := bu.concurrentRequests.Load(); n < minRequests || buMin.isBroken() {
 			buMin = bu
 			minRequests = n
 		}
@@ -861,10 +860,6 @@ func (ui *UserInfo) initURLs() error {
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
 	dropSrcPathPrefixParts := 0
 	discoverBackendIPs := *discoverBackendIPsGlobal
-	if ui.URLPrefix != nil {
-		if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
-			return err
-		}
 	if ui.RetryStatusCodes != nil {
 		retryStatusCodes = ui.RetryStatusCodes
 	}
@@ -877,6 +872,11 @@ func (ui *UserInfo) initURLs() error {
 	if ui.DiscoverBackendIPs != nil {
 		discoverBackendIPs = *ui.DiscoverBackendIPs
 	}
+
+	if ui.URLPrefix != nil {
+		if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
+			return err
+		}
 		ui.URLPrefix.retryStatusCodes = retryStatusCodes
 		ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts
 		ui.URLPrefix.discoverBackendIPs = discoverBackendIPs

app/vmauth/auth_config_test.go

@@ -777,6 +777,28 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	fn(7, 7, 7)
 }
 
+func TestBrokenBackend(t *testing.T) {
+	up := mustParseURLs([]string{
+		"http://node1:343",
+		"http://node2:343",
+		"http://node3:343",
+	})
+	up.loadBalancingPolicy = "least_loaded"
+	pbus := up.bus.Load()
+	bus := *pbus
+
+	// explicitly mark one of the backends as broken
+	bus[1].setBroken()
+
+	// broken backend should never return while there are healthy backends
+	for i := 0; i < 1e3; i++ {
+		b := up.getBackendURL()
+		if b.isBroken() {
+			t.Fatalf("unexpected broken backend %q", b.url)
+		}
+	}
+}
+
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {

app/vmauth/main.go

@@ -123,6 +123,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	ui := getUserInfoByAuthTokens(ats)
 	if ui == nil {
+		uu := authConfig.Load().UnauthorizedUser
+		if uu != nil {
+			processUserRequest(w, r, uu)
+			return true
+		}
+
 		invalidAuthTokenRequests.Inc()
 		if *logInvalidAuthTokens {
 			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)

app/vmauth/main_test.go

@@ -90,6 +90,20 @@ User-Agent: vmauth
 X-Forwarded-For: 12.34.56.78, 42.2.3.84`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 
+	// routing of all failed to authorize requests to unauthorized_user (issue #7543)
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo"
+  keep_original_host: true`
+	requestURL = "http://foo:invalid-secret@some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url=http://some-host.com/foo/abc/def`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
 	// keep_original_host
 	cfgStr = `
 unauthorized_user:

app/vmauth/target_url_test.go

@@ -187,6 +187,10 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				RetryStatusCodes:       []int{},
 				DropSrcPathPrefixParts: intp(0),
 			},
+			{
+				SrcPaths:  getRegexs([]string{"/metrics"}),
+				URLPrefix: mustParseURL("http://metrics-server"),
+			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 		HeadersConf: HeadersConf{
@@ -206,6 +210,35 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		"bb: aaa", "x: y", []int{502}, "least_loaded", 2)
 	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
 	f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", "bb: aaa", "x: y", []int{502}, "least_loaded", 2)
+	f(ui, "https://foo-host/metrics", "http://metrics-server", "", "", []int{502}, "least_loaded", 2)
+
+	// Complex routing with `url_map` without global url_prefix
+	ui = &UserInfo{
+		URLMaps: []URLMap{
+			{
+				SrcPaths:               getRegexs([]string{"/api/v1/write"}),
+				URLPrefix:              mustParseURL("http://vminsert/0/prometheus"),
+				RetryStatusCodes:       []int{},
+				DropSrcPathPrefixParts: intp(0),
+			},
+			{
+				SrcPaths:  getRegexs([]string{"/metrics/a/b"}),
+				URLPrefix: mustParseURL("http://metrics-server"),
+			},
+		},
+		HeadersConf: HeadersConf{
+			RequestHeaders: []*Header{
+				mustNewHeader("'bb: aaa'"),
+			},
+			ResponseHeaders: []*Header{
+				mustNewHeader("'x: y'"),
+			},
+		},
+		RetryStatusCodes:       []int{502},
+		DropSrcPathPrefixParts: intp(2),
+	}
+	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
+	f(ui, "https://foo-host/metrics/a/b", "http://metrics-server/b", "", "", []int{502}, "least_loaded", 2)
 
 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{

app/vmctl/flags.go

@@ -616,7 +616,7 @@ var (
 		},
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
-			Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format" +
+			Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format " +
 				"instead of https://docs.victoriametrics.com/#how-to-export-data-in-native-format API." +
 				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",

app/vmctl/influx/influx.go

@@ -51,30 +51,31 @@ type Series struct {
 	Measurement string
 	Field       string
 	LabelPairs  []LabelPair
+
+	// EmptyTags contains tags in measurement whose value must be empty.
+	EmptyTags []string
 }
 
 var valueEscaper = strings.NewReplacer(`\`, `\\`, `'`, `\'`)
 
 func (s Series) fetchQuery(timeFilter string) string {
-	f := &strings.Builder{}
+	conditions := make([]string, 0, len(s.LabelPairs)+len(s.EmptyTags))
-	fmt.Fprintf(f, "select %q from %q", s.Field, s.Measurement)
+	for _, pair := range s.LabelPairs {
-	if len(s.LabelPairs) > 0 || len(timeFilter) > 0 {
+		conditions = append(conditions, fmt.Sprintf("%q::tag='%s'", pair.Name, valueEscaper.Replace(pair.Value)))
-		f.WriteString(" where")
-	}
-	for i, pair := range s.LabelPairs {
-		pairV := valueEscaper.Replace(pair.Value)
-		fmt.Fprintf(f, " %q::tag='%s'", pair.Name, pairV)
-		if i != len(s.LabelPairs)-1 {
-			f.WriteString(" and")
 	}
+	for _, label := range s.EmptyTags {
+		conditions = append(conditions, fmt.Sprintf("%q::tag=''", label))
 	}
 	if len(timeFilter) > 0 {
-		if len(s.LabelPairs) > 0 {
+		conditions = append(conditions, timeFilter)
-			f.WriteString(" and")
 	}
-		fmt.Fprintf(f, " %s", timeFilter)
+
+	q := fmt.Sprintf("select %q from %q", s.Field, s.Measurement)
+	if len(conditions) > 0 {
+		q += fmt.Sprintf(" where %s", strings.Join(conditions, " and "))
 	}
-	return f.String()
+
+	return q
 }
 
 // LabelPair is the key-value record
@@ -118,7 +119,7 @@ func NewClient(cfg Config) (*Client, error) {
 }
 
 // Database returns database name
-func (c Client) Database() string {
+func (c *Client) Database() string {
 	return c.database
 }
 
@@ -140,7 +141,7 @@ func timeFilter(start, end string) string {
 }
 
 // Explore checks the existing data schema in influx
-// by checking available fields and series,
+// by checking available (non-empty) tags, fields and measurements
 // which unique combination represents all possible
 // time series existing in database.
 // The explore required to reduce the load on influx
@@ -150,6 +151,8 @@ func timeFilter(start, end string) string {
 // May contain non-existing time series.
 func (c *Client) Explore() ([]*Series, error) {
 	log.Printf("Exploring scheme for database %q", c.database)
+
+	// {"measurement1": ["value1", "value2"]}
 	mFields, err := c.fieldsByMeasurement()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get field keys: %s", err)
@@ -159,6 +162,12 @@ func (c *Client) Explore() ([]*Series, error) {
 		return nil, fmt.Errorf("found no numeric fields for import in database %q", c.database)
 	}
 
+	// {"measurement1": {"tag1", "tag2"}}
+	measurementTags, err := c.getMeasurementTags()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get tags of measurements: %s", err)
+	}
+
 	series, err := c.getSeries()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get series: %s", err)
@@ -171,11 +180,17 @@ func (c *Client) Explore() ([]*Series, error) {
 			log.Printf("skip measurement %q since it has no fields", s.Measurement)
 			continue
 		}
+		tags, ok := measurementTags[s.Measurement]
+		if !ok {
+			return nil, fmt.Errorf("failed to find tags of measurement %s", s.Measurement)
+		}
+		emptyTags := getEmptyTags(tags, s.LabelPairs)
 		for _, field := range fields {
 			is := &Series{
 				Measurement: s.Measurement,
 				Field:       field,
 				LabelPairs:  s.LabelPairs,
+				EmptyTags:   emptyTags,
 			}
 			iSeries = append(iSeries, is)
 		}
@@ -183,6 +198,22 @@ func (c *Client) Explore() ([]*Series, error) {
 	return iSeries, nil
 }
 
+// getEmptyTags returns tags of a measurement that are missing in a specific series.
+// Tags represent all tags of a measurement. LabelPairs represent tags of a specific series.
+func getEmptyTags(tags map[string]struct{}, LabelPairs []LabelPair) []string {
+	labelMap := make(map[string]struct{})
+	for _, pair := range LabelPairs {
+		labelMap[pair.Name] = struct{}{}
+	}
+	result := make([]string, 0, len(labelMap)-len(LabelPairs))
+	for tag := range tags {
+		if _, ok := labelMap[tag]; !ok {
+			result = append(result, tag)
+		}
+	}
+	return result
+}
+
 // ChunkedResponse is a wrapper over influx.ChunkedResponse.
 // Used for better memory usage control while iterating
 // over huge time series.
@@ -357,6 +388,57 @@ func (c *Client) getSeries() ([]*Series, error) {
 	return result, nil
 }
 
+// getMeasurementTags get the tags for each measurement.
+// tags are placed in a map without values (similar to a set) for quick lookups:
+// {"measurement1": {"tag1", "tag2"}, "measurement2": {"tag3", "tag4"}}
+func (c *Client) getMeasurementTags() (map[string]map[string]struct{}, error) {
+	com := "show tag keys"
+	q := influx.Query{
+		Command:         com,
+		Database:        c.database,
+		RetentionPolicy: c.retention,
+		Chunked:         true,
+		ChunkSize:       c.chunkSize,
+	}
+
+	log.Printf("fetching tag keys: %s", stringify(q))
+	cr, err := c.QueryAsChunk(q)
+	if err != nil {
+		return nil, fmt.Errorf("error while executing query %q: %s", q.Command, err)
+	}
+
+	const tagKey = "tagKey"
+	var tagsCount int
+	result := make(map[string]map[string]struct{})
+	for {
+		resp, err := cr.NextResponse()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		if resp.Error() != nil {
+			return nil, fmt.Errorf("response error for query %q: %s", q.Command, resp.Error())
+		}
+		qValues, err := parseResult(resp.Results[0])
+		if err != nil {
+			return nil, err
+		}
+		for _, qv := range qValues {
+			if result[qv.name] == nil {
+				result[qv.name] = make(map[string]struct{}, len(qv.values[tagKey]))
+			}
+			for _, tk := range qv.values[tagKey] {
+				result[qv.name][tk.(string)] = struct{}{}
+				tagsCount++
+			}
+		}
+	}
+	log.Printf("found %d tag(s) for %d measurements", tagsCount, len(result))
+	return result, nil
+}
+
 func (c *Client) do(q influx.Query) ([]queryValues, error) {
 	res, err := c.Query(q)
 	if err != nil {

app/vmctl/influx/influx_test.go

@@ -73,6 +73,12 @@ func TestFetchQuery(t *testing.T) {
 		Measurement: "cpu",
 		Field:       "value",
 	}, "", `select "value" from "cpu"`)
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value1",
+		EmptyTags:   []string{"e1", "e2", "e3"},
+	}, "", `select "value1" from "cpu" where "e1"::tag='' and "e2"::tag='' and "e3"::tag=''`)
 }
 
 func TestTimeFilter(t *testing.T) {

app/vmctl/main.go

@@ -266,7 +266,7 @@ func main() {
 			},
 			{
 				Name:   "vm-native",
-				Usage:  "Migrate time series between VictoriaMetrics installations via native binary format",
+				Usage:  "Migrate time series between VictoriaMetrics installations",
 				Flags:  mergeFlags(globalFlags, vmNativeFlags),
 				Before: beforeFn,
 				Action: func(c *cli.Context) error {

app/vmselect/graphite/eval_test.go

@@ -2137,6 +2137,25 @@ func TestExecExprSuccess(t *testing.T) {
 		},
 	})
 	f(`removeEmptySeries(removeBelowValue(time('a'),150),1)`, []*series{})
+	// if xFilesFactor is set, a single value in the series needs to be non-null for it to be
+	// considered non-empty
+	f(`removeEmptySeries(removeBelowValue(time('a'),150),0)`, []*series{
+		{
+			Timestamps: []int64{120000, 180000},
+			Values:     []float64{nan, 180},
+			Name:       "removeBelowValue(a,150)",
+			Tags:       map[string]string{"name": "a"},
+		},
+	})
+	f(`removeEmptySeries(removeBelowValue(time('a'),150),-1)`, []*series{
+		{
+			Timestamps: []int64{120000, 180000},
+			Values:     []float64{nan, 180},
+			Name:       "removeBelowValue(a,150)",
+			Tags:       map[string]string{"name": "a"},
+		},
+	})
+
 	f(`round(time('a',17),-1)`, []*series{
 		{
 			Timestamps:     []int64{120000, 137000, 154000, 171000, 188000, 205000},

app/vmselect/graphite/transform.go

@@ -3151,7 +3151,7 @@ func transformRemoveEmptySeries(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSe
 			xff = xFilesFactor
 		}
 		n := aggrCount(s.Values)
-		if n/float64(len(s.Values)) < xff {
+		if n/float64(len(s.Values)) <= xff {
 			return nil, nil
 		}
 		s.expr = fe

app/vmselect/promql/exec.go

@@ -108,7 +108,7 @@ func maySortResults(e metricsql.Expr) bool {
 	switch v := e.(type) {
 	case *metricsql.FuncExpr:
 		switch strings.ToLower(v.Name) {
-		case "sort", "sort_desc",
+		case "sort", "sort_desc", "limit_offset",
 			"sort_by_label", "sort_by_label_desc",
 			"sort_by_label_numeric", "sort_by_label_numeric_desc":
 			// Results already sorted

app/vmselect/promql/exec_test.go

@@ -9274,6 +9274,75 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`limit_offset(5, 0, sort_by_label_numeric_desc(multiple_labels_numbers_special_chars, "foo"))`, func(t *testing.T) {
+		t.Parallel()
+		q := `limit_offset(5, 0, sort_by_label_numeric_desc((
+                        label_set(3, "foo", "1:0:3"),
+			label_set(4, "foo", "5:0:15"),
+			label_set(1, "foo", "1:0:2"),
+			label_set(5, "foo", "7:0:15"),
+			label_set(7, "foo", "3:0:1"),
+			label_set(6, "foo", "1:0:2"),
+			label_set(8, "foo", "9:0:15")
+                ), "foo"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{8, 8, 8, 8, 8, 8},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("9:0:15"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{5, 5, 5, 5, 5, 5},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("7:0:15"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("5:0:15"),
+			},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{7, 7, 7, 7, 7, 7},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("3:0:1"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r5.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("1:0:3"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
+		f(q, resultExpected)
+	})
 	t.Run(`sort_by_label_numeric(alias_numbers_with_special_chars)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_by_label_numeric((

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
     "main.css": "./static/css/main.d781989c.css",
-    "main.js": "./static/js/main.7ec4e6eb.js",
+    "main.js": "./static/js/main.a7037969.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.d781989c.css",
-    "static/js/main.7ec4e6eb.js"
+    "static/js/main.a7037969.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.7ec4e6eb.js"></script><link href="./static/css/main.d781989c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.a7037969.js"></script><link href="./static/css/main.d781989c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.23.1 AS build-web-stage
+FROM golang:1.23.3 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -64,11 +64,15 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
     return groupByMultipleKeys(logs, [groupBy]).map((item) => {
       const streamValue = item.values[0]?.[groupBy] || "";
       const pairs = getStreamPairs(streamValue);
+      // values sorting by time
+      const values = item.values.sort((a,b) => new Date(b._time).getTime() - new Date(a._time).getTime());
       return {
-        ...item,
+        keys: item.keys,
+        keysString: item.keys.join(""),
+        values,
         pairs,
       };
-    });
+    }).sort((a, b) => a.keysString.localeCompare(b.keysString)); // groups sorting
   }, [logs, groupBy]);
 
   const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
@@ -117,7 +121,7 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
         {groupData.map((item, i) => (
           <div
             className="vm-group-logs-section"
-            key={item.keys.join("")}
+            key={item.keysString}
           >
             <Accordion
               key={String(expandGroups[i])}
@@ -129,7 +133,7 @@ const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
                   {item.pairs.map((pair) => (
                     <Tooltip
                       title={copied === pair ? "Copied" : "Copy to clipboard"}
-                      key={`${item.keys.join("")}_${pair}`}
+                      key={`${item.keysString}_${pair}`}
                       placement={"top-center"}
                     >
                       <div

apptest/README.md

@@ -26,7 +26,7 @@ queries to them:
 -   `client.go` - provides helper functions for sending HTTP requests to
     applications.
 
-The integration tests themselves reside in `*_test.go` files. Apart from having
+The integration tests themselves reside in `tests/*_test.go` files. Apart from having
 the `_test` suffix, there are no strict rules of how to name a file, but the
 name should reflect the prevailing purpose of the tests located in that file.
 For example, `sharding_test.go` aims at testing data sharding.
@@ -38,3 +38,10 @@ accounts for that, it builds all application binaries before running the tests.
 But if you want to run the tests without `make`, i.e. by executing
 `go test ./app/apptest`, you will need to build the binaries first (for example,
 by executing `make all`).
+
+Not all binaries can be built from `master` branch, cluster binaries can be built
+only from `cluster` branch. Hence, not all test cases suitable to run in both branches:
+- If test is using binaries from `cluster` branch, then test name should be prefixed 
+  with `TestCluster` word
+- If test is using binaries from `master` branch, then test name should be prefixed
+  with `TestVmsingle` word.

apptest/model.go

@@ -0,0 +1,121 @@
+package apptest
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+)
+
+// PrometheusQuerier contains methods available to Prometheus-like HTTP API for Querying
+type PrometheusQuerier interface {
+	PrometheusAPIV1Query(t *testing.T, query, time, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse
+	PrometheusAPIV1QueryRange(t *testing.T, query, start, end, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse
+	PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse
+}
+
+// PrometheusWriter contains methods available to Prometheus-like HTTP API for Writing new data
+type PrometheusWriter interface {
+	PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
+}
+
+// QueryOpts contains various params used for querying or ingesting data
+type QueryOpts struct {
+	Tenant  string
+	Timeout string
+}
+
+// PrometheusAPIV1QueryResponse is an inmemory representation of the
+// /prometheus/api/v1/query or /prometheus/api/v1/query_range response.
+type PrometheusAPIV1QueryResponse struct {
+	Status string
+	Data   *QueryData
+}
+
+// NewPrometheusAPIV1QueryResponse is a test helper function that creates a new
+// instance of PrometheusAPIV1QueryResponse by unmarshalling a json string.
+func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	res := &PrometheusAPIV1QueryResponse{}
+	if err := json.Unmarshal([]byte(s), res); err != nil {
+		t.Fatalf("could not unmarshal query response: %v", err)
+	}
+	return res
+}
+
+// QueryData holds the query result along with its type.
+type QueryData struct {
+	ResultType string
+	Result     []*QueryResult
+}
+
+// QueryResult holds the metric name (in the form of label name-value
+// collection) and its samples.
+//
+// Sample or Samples field is set for /prometheus/api/v1/query or
+// /prometheus/api/v1/query_range response respectively.
+type QueryResult struct {
+	Metric  map[string]string
+	Sample  *Sample   `json:"value"`
+	Samples []*Sample `json:"values"`
+}
+
+// Sample is a timeseries value at a given timestamp.
+type Sample struct {
+	Timestamp int64
+	Value     float64
+}
+
+// NewSample is a test helper function that creates a new sample out of time in
+// RFC3339 format and a value.
+func NewSample(t *testing.T, timeStr string, value float64) *Sample {
+	parsedTime, err := time.Parse(time.RFC3339, timeStr)
+	if err != nil {
+		t.Fatalf("could not parse RFC3339 time %q: %v", timeStr, err)
+	}
+	return &Sample{parsedTime.Unix(), value}
+}
+
+// UnmarshalJSON populates the sample fields from a JSON string.
+func (s *Sample) UnmarshalJSON(b []byte) error {
+	var (
+		ts int64
+		v  string
+	)
+	raw := []any{&ts, &v}
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+	if got, want := len(raw), 2; got != want {
+		return fmt.Errorf("unexpected number of fields: got %d, want %d (raw sample: %s)", got, want, string(b))
+	}
+	s.Timestamp = ts
+	var err error
+	s.Value, err = strconv.ParseFloat(v, 64)
+	if err != nil {
+		return fmt.Errorf("could not parse sample value %q: %w", v, err)
+	}
+	return nil
+}
+
+// PrometheusAPIV1SeriesResponse is an inmemory representation of the
+// /prometheus/api/v1/series response.
+type PrometheusAPIV1SeriesResponse struct {
+	Status    string
+	IsPartial bool
+	Data      []map[string]string
+}
+
+// NewPrometheusAPIV1SeriesResponse is a test helper function that creates a new
+// instance of PrometheusAPIV1SeriesResponse by unmarshalling a json string.
+func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+
+	res := &PrometheusAPIV1SeriesResponse{}
+	if err := json.Unmarshal([]byte(s), res); err != nil {
+		t.Fatalf("could not unmarshal series response: %v", err)
+	}
+	return res
+}

apptest/testcase.go

@@ -11,11 +11,18 @@ import (
 type TestCase struct {
 	t   *testing.T
 	cli *Client
+
+	startedApps []Stopper
+}
+
+// Stopper is an interface of objects that needs to be stopped via Stop() call
+type Stopper interface {
+	Stop()
 }
 
 // NewTestCase creates a new test case.
 func NewTestCase(t *testing.T) *TestCase {
-	return &TestCase{t, NewClient()}
+	return &TestCase{t, NewClient(), nil}
 }
 
 // Dir returns the directory name that should be used by as the -storageDataDir.
@@ -29,14 +36,73 @@ func (tc *TestCase) Client() *Client {
 	return tc.cli
 }
 
-// Close performs the test case clean up, such as closing all client connections
+// Stop performs the test case clean up, such as closing all client connections
 // and removing the -storageDataDir directory.
 //
 // Note that the -storageDataDir is not removed in case of test case failure to
-// allow for furher manual debugging.
+// allow for further manual debugging.
-func (tc *TestCase) Close() {
+func (tc *TestCase) Stop() {
 	tc.cli.CloseConnections()
+	for _, app := range tc.startedApps {
+		app.Stop()
+	}
 	if !tc.t.Failed() {
 		fs.MustRemoveAll(tc.Dir())
 	}
 }
+
+// MustStartVmsingle is a test helper function that starts an instance of
+// vmsingle and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
+	tc.t.Helper()
+
+	app, err := StartVmsingle(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+// MustStartVmstorage is a test helper function that starts an instance of
+// vmstorage and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstorage {
+	tc.t.Helper()
+
+	app, err := StartVmstorage(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+// MustStartVmselect is a test helper function that starts an instance of
+// vmselect and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmselect(instance string, flags []string) *Vmselect {
+	tc.t.Helper()
+
+	app, err := StartVmselect(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+// MustStartVminsert is a test helper function that starts an instance of
+// vminsert and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVminsert(instance string, flags []string) *Vminsert {
+	tc.t.Helper()
+
+	app, err := StartVminsert(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+func (tc *TestCase) addApp(app Stopper) {
+	tc.startedApps = append(tc.startedApps, app)
+}

apptest/tests/key_concepts_test.go

@@ -0,0 +1,234 @@
+package tests
+
+import (
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+// Data used in examples in
+// https://docs.victoriametrics.com/keyconcepts/#instant-query and
+// https://docs.victoriametrics.com/keyconcepts/#range-query
+var docData = []string{
+	"foo_bar 1.00 1652169600000", // 2022-05-10T08:00:00Z
+	"foo_bar 2.00 1652169660000", // 2022-05-10T08:01:00Z
+	"foo_bar 3.00 1652169720000", // 2022-05-10T08:02:00Z
+	"foo_bar 5.00 1652169840000", // 2022-05-10T08:04:00Z, one point missed
+	"foo_bar 5.50 1652169960000", // 2022-05-10T08:06:00Z, one point missed
+	"foo_bar 5.50 1652170020000", // 2022-05-10T08:07:00Z
+	"foo_bar 4.00 1652170080000", // 2022-05-10T08:08:00Z
+	"foo_bar 3.50 1652170260000", // 2022-05-10T08:11:00Z, two points missed
+	"foo_bar 3.25 1652170320000", // 2022-05-10T08:12:00Z
+	"foo_bar 3.00 1652170380000", // 2022-05-10T08:13:00Z
+	"foo_bar 2.00 1652170440000", // 2022-05-10T08:14:00Z
+	"foo_bar 1.00 1652170500000", // 2022-05-10T08:15:00Z
+	"foo_bar 4.00 1652170560000", // 2022-05-10T08:16:00Z
+}
+
+// TestSingleKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+func TestSingleKeyConceptsQuery(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage",
+		"-retentionPeriod=100y",
+	})
+
+	opts := apptest.QueryOpts{Timeout: "5s"}
+
+	// Insert example data from documentation.
+	vmsingle.PrometheusAPIV1ImportPrometheus(t, docData, opts)
+	vmsingle.ForceFlush(t)
+
+	testInstantQuery(t, vmsingle, opts)
+	testRangeQuery(t, vmsingle, opts)
+	testRangeQueryIsEquivalentToManyInstantQueries(t, vmsingle, opts)
+}
+
+// TestClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
+func TestClusterKeyConceptsQuery(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	// Set up the following cluster configuration:
+	//
+	// - two vmstorage instances
+	// - vminsert points to the two vmstorages, its replication setting
+	//   is off which means it will only shard the incoming data across the two
+	//   vmstorages.
+	// - vmselect points to the two vmstorages and is expected to query both
+	//   vmstorages and build the full result out of the two partial results.
+
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
+		"-retentionPeriod=100y",
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
+		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
+	})
+
+	opts := apptest.QueryOpts{Timeout: "5s", Tenant: "0"}
+
+	// Insert example data from documentation.
+	vminsert.PrometheusAPIV1ImportPrometheus(t, docData, opts)
+	time.Sleep(2 * time.Second)
+
+	vmstorage1.ForceFlush(t)
+	vmstorage2.ForceFlush(t)
+
+	testInstantQuery(t, vmselect, opts)
+	testRangeQuery(t, vmselect, opts)
+	testRangeQueryIsEquivalentToManyInstantQueries(t, vmselect, opts)
+}
+
+// testInstantQuery verifies the statements made in the `Instant query` section
+// of the VictoriaMetrics documentation. See:
+// https://docs.victoriametrics.com/keyconcepts/#instant-query
+func testInstantQuery(t *testing.T, q apptest.PrometheusQuerier, opts apptest.QueryOpts) {
+	// Get the value of the foo_bar time series at 2022-05-10T08:03:00Z with the
+	// step of 5m and timeout 5s. There is no sample at exactly this timestamp.
+	// Therefore, VictoriaMetrics will search for the nearest sample within the
+	// [time-5m..time] interval.
+	got := q.PrometheusAPIV1Query(t, "foo_bar", "2022-05-10T08:03:00.000Z", "5m", opts)
+	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
+	opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+	if diff := cmp.Diff(want, got, opt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Get the value of the foo_bar time series at 2022-05-10T08:18:00Z with the
+	// step of 1m and timeout 5s. There is no sample at this timestamp.
+	// Therefore, VictoriaMetrics will search for the nearest sample within the
+	// [time-1m..time] interval. Since the nearest sample is 2m away and the
+	// step is 1m, then the VictoriaMetrics must return empty response.
+	got = q.PrometheusAPIV1Query(t, "foo_bar", "2022-05-10T08:18:00.000Z", "1m", opts)
+	if len(got.Data.Result) > 0 {
+		t.Errorf("unexpected response: got non-empty result, want empty result:\n%v", got)
+	}
+}
+
+// testRangeQuery verifies the statements made in the `Range query` section of
+// the VictoriaMetrics documentation. See:
+// https://docs.victoriametrics.com/keyconcepts/#range-query
+func testRangeQuery(t *testing.T, q apptest.PrometheusQuerier, opts apptest.QueryOpts) {
+	f := func(start, end, step string, wantSamples []*apptest.Sample) {
+		t.Helper()
+
+		got := q.PrometheusAPIV1QueryRange(t, "foo_bar", start, end, step, opts)
+		want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
+		want.Data.Result[0].Samples = wantSamples
+		opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+		if diff := cmp.Diff(want, got, opt); diff != "" {
+			t.Errorf("unexpected response (-want, +got):\n%s", diff)
+		}
+	}
+
+	// Verify the statement that the query result for
+	// [2022-05-10T07:59:00Z..2022-05-10T08:17:00Z] time range and 1m step will
+	// contain 17 points.
+	f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "1m", []*apptest.Sample{
+		// Sample for 2022-05-10T07:59:00Z is missing because the time series has
+		// samples only starting from 8:00.
+		apptest.NewSample(t, "2022-05-10T08:00:00Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:01:00Z", 2),
+		apptest.NewSample(t, "2022-05-10T08:02:00Z", 3),
+		apptest.NewSample(t, "2022-05-10T08:03:00Z", 3),
+		apptest.NewSample(t, "2022-05-10T08:04:00Z", 5),
+		apptest.NewSample(t, "2022-05-10T08:05:00Z", 5),
+		apptest.NewSample(t, "2022-05-10T08:06:00Z", 5.5),
+		apptest.NewSample(t, "2022-05-10T08:07:00Z", 5.5),
+		apptest.NewSample(t, "2022-05-10T08:08:00Z", 4),
+		apptest.NewSample(t, "2022-05-10T08:09:00Z", 4),
+		// Sample for 2022-05-10T08:10:00Z is missing because there is no sample
+		// within the [8:10 - 1m .. 8:10] interval.
+		apptest.NewSample(t, "2022-05-10T08:11:00Z", 3.5),
+		apptest.NewSample(t, "2022-05-10T08:12:00Z", 3.25),
+		apptest.NewSample(t, "2022-05-10T08:13:00Z", 3),
+		apptest.NewSample(t, "2022-05-10T08:14:00Z", 2),
+		apptest.NewSample(t, "2022-05-10T08:15:00Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:16:00Z", 4),
+		apptest.NewSample(t, "2022-05-10T08:17:00Z", 4),
+	})
+
+	// Verify the statement that a query is executed at start, start+step,
+	// start+2*step, …, step+N*step timestamps, where N is the whole number
+	// of steps that fit between start and end.
+	f("2022-05-10T08:00:01.000Z", "2022-05-10T08:02:00.000Z", "1m", []*apptest.Sample{
+		apptest.NewSample(t, "2022-05-10T08:00:01Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:01:01Z", 2),
+	})
+
+	// Verify the statement that a query is executed at start, start+step,
+	// start+2*step, …, end timestamps, when end = start + N*step.
+	f("2022-05-10T08:00:00.000Z", "2022-05-10T08:02:00.000Z", "1m", []*apptest.Sample{
+		apptest.NewSample(t, "2022-05-10T08:00:00Z", 1),
+		apptest.NewSample(t, "2022-05-10T08:01:00Z", 2),
+		apptest.NewSample(t, "2022-05-10T08:02:00Z", 3),
+	})
+
+	// If the step isn’t set, then it defaults to 5m (5 minutes).
+	f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "", []*apptest.Sample{
+		// Sample for 2022-05-10T07:59:00Z is missing because the time series has
+		// samples only starting from 8:00.
+		apptest.NewSample(t, "2022-05-10T08:04:00Z", 5),
+		apptest.NewSample(t, "2022-05-10T08:09:00Z", 4),
+		apptest.NewSample(t, "2022-05-10T08:14:00Z", 2),
+	})
+}
+
+// testRangeQueryIsEquivalentToManyInstantQueries verifies the statement made in
+// the `Range query` section of the VictoriaMetrics documentation that a range
+// query is actually an instant query executed 1 + (start-end)/step times on the
+// time range from start to end. See:
+// https://docs.victoriametrics.com/keyconcepts/#range-query
+func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q apptest.PrometheusQuerier, opts apptest.QueryOpts) {
+	f := func(timestamp string, want *apptest.Sample) {
+		t.Helper()
+
+		gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", timestamp, "1m", opts)
+		if want == nil {
+			if got, want := len(gotInstant.Data.Result), 0; got != want {
+				t.Errorf("unexpected instant result size: got %d, want %d", got, want)
+			}
+		} else {
+			got := gotInstant.Data.Result[0].Sample
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("unexpected instant sample (-want, +got):\n%s", diff)
+			}
+		}
+	}
+
+	rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", "2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "1m", opts)
+	rangeSamples := rangeRes.Data.Result[0].Samples
+
+	f("2022-05-10T07:59:00.000Z", nil)
+	f("2022-05-10T08:00:00.000Z", rangeSamples[0])
+	f("2022-05-10T08:01:00.000Z", rangeSamples[1])
+	f("2022-05-10T08:02:00.000Z", rangeSamples[2])
+	f("2022-05-10T08:03:00.000Z", rangeSamples[3])
+	f("2022-05-10T08:04:00.000Z", rangeSamples[4])
+	f("2022-05-10T08:05:00.000Z", rangeSamples[5])
+	f("2022-05-10T08:06:00.000Z", rangeSamples[6])
+	f("2022-05-10T08:07:00.000Z", rangeSamples[7])
+	f("2022-05-10T08:08:00.000Z", rangeSamples[8])
+	f("2022-05-10T08:09:00.000Z", rangeSamples[9])
+	f("2022-05-10T08:10:00.000Z", nil)
+	f("2022-05-10T08:11:00.000Z", rangeSamples[10])
+	f("2022-05-10T08:12:00.000Z", rangeSamples[11])
+	f("2022-05-10T08:13:00.000Z", rangeSamples[12])
+	f("2022-05-10T08:14:00.000Z", rangeSamples[13])
+	f("2022-05-10T08:15:00.000Z", rangeSamples[14])
+	f("2022-05-10T08:16:00.000Z", rangeSamples[15])
+	f("2022-05-10T08:17:00.000Z", rangeSamples[16])
+}

apptest/tests/multilevel_test.go

@@ -9,9 +9,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
-func TestMultilevelSelect(t *testing.T) {
+func TestClusterMultilevelSelect(t *testing.T) {
 	tc := apptest.NewTestCase(t)
-	defer tc.Close()
+	defer tc.Stop()
 
 	// Set up the following multi-level cluster configuration:
 	//
@@ -20,24 +20,18 @@ func TestMultilevelSelect(t *testing.T) {
 	// vmisert writes data into vmstorage.
 	// vmselect (L2) reads that data via vmselect (L1).
 
-	cli := tc.Client()
+	vmstorage := tc.MustStartVmstorage("vmstorage", []string{
-
-	vmstorage := apptest.MustStartVmstorage(t, "vmstorage", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage",
-	}, cli)
+	})
-	defer vmstorage.Stop()
+	vminsert := tc.MustStartVminsert("vminsert", []string{
-	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
 		"-storageNode=" + vmstorage.VminsertAddr(),
-	}, cli)
+	})
-	defer vminsert.Stop()
+	vmselectL1 := tc.MustStartVmselect("vmselect-level1", []string{
-	vmselectL1 := apptest.MustStartVmselect(t, "vmselect-level1", []string{
 		"-storageNode=" + vmstorage.VmselectAddr(),
-	}, cli)
+	})
-	defer vmselectL1.Stop()
+	vmselectL2 := tc.MustStartVmselect("vmselect-level2", []string{
-	vmselectL2 := apptest.MustStartVmselect(t, "vmselect-level2", []string{
 		"-storageNode=" + vmselectL1.ClusternativeListenAddr(),
-	}, cli)
+	})
-	defer vmselectL2.Stop()
 
 	// Insert 1000 unique time series.Wait for 2 seconds to let vmstorage
 	// flush pending items so they become searchable.
@@ -47,13 +41,13 @@ func TestMultilevelSelect(t *testing.T) {
 	for i := range numMetrics {
 		records[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
 	}
-	vminsert.PrometheusAPIV1ImportPrometheus(t, "0", records)
+	vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{Tenant: "0"})
 	time.Sleep(2 * time.Second)
 
 	// Retrieve all time series and verify that vmselect (L1) serves the complete
 	// set of time series.
 
-	seriesL1 := vmselectL1.PrometheusAPIV1Series(t, "0", `{__name__=~".*"}`)
+	seriesL1 := vmselectL1.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{Tenant: "0"})
 	if got, want := len(seriesL1.Data), numMetrics; got != want {
 		t.Fatalf("unexpected level-1 series count: got %d, want %d", got, want)
 	}
@@ -61,7 +55,7 @@ func TestMultilevelSelect(t *testing.T) {
 	// Retrieve all time series and verify that vmselect (L2) serves the complete
 	// set of time series.
 
-	seriesL2 := vmselectL2.PrometheusAPIV1Series(t, "0", `{__name__=~".*"}`)
+	seriesL2 := vmselectL2.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{Tenant: "0"})
 	if got, want := len(seriesL2.Data), numMetrics; got != want {
 		t.Fatalf("unexpected level-2 series count: got %d, want %d", got, want)
 	}

apptest/tests/sharding_test.go

@@ -9,9 +9,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
-func TestVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
+func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
 	tc := apptest.NewTestCase(t)
-	defer tc.Close()
+	defer tc.Stop()
 
 	// Set up the following cluster configuration:
 	//
@@ -22,24 +22,18 @@ func TestVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
 	// - vmselect points to the two vmstorages and is expected to query both
 	//   vmstorages and build the full result out of the two partial results.
 
-	cli := tc.Client()
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
-
-	vmstorage1 := apptest.MustStartVmstorage(t, "vmstorage-1", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
-	}, cli)
+	})
-	defer vmstorage1.Stop()
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
-	vmstorage2 := apptest.MustStartVmstorage(t, "vmstorage-2", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
-	}, cli)
+	})
-	defer vmstorage2.Stop()
+	vminsert := tc.MustStartVminsert("vminsert", []string{
-	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
-	}, cli)
+	})
-	defer vminsert.Stop()
+	vmselect := tc.MustStartVmselect("vmselect", []string{
-	vmselect := apptest.MustStartVmselect(t, "vmselect", []string{
 		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
-	}, cli)
+	})
-	defer vmselect.Stop()
 
 	// Insert 1000 unique time series and verify the that inserted data has been
 	// indeed sharded by checking various metrics exposed by vminsert and
@@ -53,7 +47,7 @@ func TestVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
 	for i := range numMetrics {
 		records[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
 	}
-	vminsert.PrometheusAPIV1ImportPrometheus(t, "0", records)
+	vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{Tenant: "0"})
 	time.Sleep(2 * time.Second)
 
 	numMetrics1 := vmstorage1.GetIntMetric(t, "vm_vminsert_metrics_read_total")
@@ -71,7 +65,7 @@ func TestVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
 	// Retrieve all time series and verify that vmselect serves the complete set
 	//of time series.
 
-	series := vmselect.PrometheusAPIV1Series(t, "0", `{__name__=~".*"}`)
+	series := vmselect.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{Tenant: "0"})
 	if got, want := series.Status, "success"; got != want {
 		t.Fatalf("unexpected /ap1/v1/series response status: got %s, want %s", got, want)
 	}

apptest/vminsert.go

@@ -18,19 +18,6 @@ type Vminsert struct {
 	cli            *Client
 }
 
-// MustStartVminsert is a test helper function that starts an instance of
-// vminsert and fails the test if the app fails to start.
-func MustStartVminsert(t *testing.T, instance string, flags []string, cli *Client) *Vminsert {
-	t.Helper()
-
-	app, err := StartVminsert(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
-}
-
 // StartVminsert starts an instance of vminsert with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
@@ -64,10 +51,10 @@ func StartVminsert(instance string, flags []string, cli *Client) (*Vminsert, err
 // /prometheus/api/v1/import/prometheus vminsert endpoint.
 //
 // See https://docs.victoriametrics.com/url-examples/#apiv1importprometheus
-func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, tenant string, records []string) {
+func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
-	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, tenant)
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.Tenant)
 	app.cli.Post(t, url, "text/plain", strings.Join(records, "\n"), http.StatusNoContent)
 }
 

apptest/vmselect.go

@@ -1,7 +1,6 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -20,19 +19,6 @@ type Vmselect struct {
 	cli                     *Client
 }
 
-// MustStartVmselect is a test helper function that starts an instance of
-// vmselect and fails the test if the app fails to start.
-func MustStartVmselect(t *testing.T, instance string, flags []string, cli *Client) *Vmselect {
-	t.Helper()
-
-	app, err := StartVmselect(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
-}
-
 // StartVmselect starts an instance of vmselect with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
@@ -69,30 +55,55 @@ func (app *Vmselect) ClusternativeListenAddr() string {
 	return app.clusternativeListenAddr
 }
 
-// PrometheusAPIV1SeriesResponse is an inmemory representation of the
+// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
-// /prometheus/api/v1/series response.
+// instant query by sending a HTTP POST request to /prometheus/api/v1/query
-type PrometheusAPIV1SeriesResponse struct {
+// vmsingle endpoint.
-	Status    string
+//
-	IsPartial bool
+// See https://docs.victoriametrics.com/url-examples/#apiv1query
-	Data      []map[string]string
+func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query, time, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query", app.httpListenAddr, opts.Tenant)
+	values := url.Values{}
+	values.Add("query", query)
+	values.Add("time", time)
+	values.Add("step", step)
+	values.Add("timeout", opts.Timeout)
+	res := app.cli.PostForm(t, queryURL, values, http.StatusOK)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1QueryRange is a test helper function that performs
+// PromQL/MetricsQL range query by sending a HTTP POST request to
+// /prometheus/api/v1/query_range vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1query_range
+func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query, start, end, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query_range", app.httpListenAddr, opts.Tenant)
+	values := url.Values{}
+	values.Add("query", query)
+	values.Add("start", start)
+	values.Add("end", end)
+	values.Add("step", step)
+	values.Add("timeout", opts.Timeout)
+	res := app.cli.PostForm(t, queryURL, values, http.StatusOK)
+	return NewPrometheusAPIV1QueryResponse(t, res)
 }
 
 // PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
 // and returns the list of time series that match the query.
 //
 // See https://docs.victoriametrics.com/url-examples/#apiv1series
-func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, tenant, matchQuery string) *PrometheusAPIV1SeriesResponse {
+func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
 	t.Helper()
 
-	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, tenant)
+	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, opts.Tenant)
 	values := url.Values{}
 	values.Add("match[]", matchQuery)
-	jsonRes := app.cli.PostForm(t, seriesURL, values, http.StatusOK)
+	res := app.cli.PostForm(t, seriesURL, values, http.StatusOK)
-	var res PrometheusAPIV1SeriesResponse
+	return NewPrometheusAPIV1SeriesResponse(t, res)
-	if err := json.Unmarshal([]byte(jsonRes), &res); err != nil {
-		t.Fatalf("could not unmarshal /api/v1/series response: %v", err)
-	}
-	return &res
 }
 
 // String returns the string representation of the vmselect app state.

apptest/vmsingle.go

@@ -0,0 +1,136 @@
+package apptest
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
+// functions.
+type Vmsingle struct {
+	*app
+	*ServesMetrics
+
+	storageDataPath string
+	httpListenAddr  string
+
+	forceFlushURL                      string
+	prometheusAPIV1ImportPrometheusURL string
+	prometheusAPIV1QueryURL            string
+	prometheusAPIV1QueryRangeURL       string
+	prometheusAPIV1SeriesURL           string
+}
+
+// StartVmsingle starts an instance of vmsingle with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr).
+func StartVmsingle(instance string, flags []string, cli *Client) (*Vmsingle, error) {
+	app, stderrExtracts, err := startApp(instance, "../../bin/victoria-metrics", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
+			"-httpListenAddr":  "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			storageDataPathRE,
+			httpListenAddrRE,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmsingle{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
+			cli:        cli,
+		},
+		storageDataPath: stderrExtracts[0],
+		httpListenAddr:  stderrExtracts[1],
+
+		forceFlushURL:                      fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
+		prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
+		prometheusAPIV1QueryURL:            fmt.Sprintf("http://%s/prometheus/api/v1/query", stderrExtracts[1]),
+		prometheusAPIV1QueryRangeURL:       fmt.Sprintf("http://%s/prometheus/api/v1/query_range", stderrExtracts[1]),
+		prometheusAPIV1SeriesURL:           fmt.Sprintf("http://%s/prometheus/api/v1/series", stderrExtracts[1]),
+	}, nil
+}
+
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
+func (app *Vmsingle) ForceFlush(t *testing.T) {
+	t.Helper()
+
+	app.cli.Get(t, app.forceFlushURL, http.StatusOK)
+}
+
+// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format by sending a HTTP
+// POST request to /prometheus/api/v1/import/prometheus vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1importprometheus
+func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, _ QueryOpts) {
+	t.Helper()
+
+	app.cli.Post(t, app.prometheusAPIV1ImportPrometheusURL, "text/plain", strings.Join(records, "\n"), http.StatusNoContent)
+}
+
+// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// instant query by sending a HTTP POST request to /prometheus/api/v1/query
+// vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1query
+func (app *Vmsingle) PrometheusAPIV1Query(t *testing.T, query, time, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	values := url.Values{}
+	values.Add("query", query)
+	values.Add("time", time)
+	values.Add("step", step)
+	values.Add("timeout", opts.Timeout)
+	res := app.cli.PostForm(t, app.prometheusAPIV1QueryURL, values, http.StatusOK)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1QueryRange is a test helper function that performs
+// PromQL/MetricsQL range query by sending a HTTP POST request to
+// /prometheus/api/v1/query_range vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1query_range
+func (app *Vmsingle) PrometheusAPIV1QueryRange(t *testing.T, query, start, end, step string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	values := url.Values{}
+	values.Add("query", query)
+	values.Add("start", start)
+	values.Add("end", end)
+	values.Add("step", step)
+	values.Add("timeout", opts.Timeout)
+	res := app.cli.PostForm(t, app.prometheusAPIV1QueryRangeURL, values, http.StatusOK)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
+// and returns the list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1series
+func (app *Vmsingle) PrometheusAPIV1Series(t *testing.T, matchQuery string, _ QueryOpts) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+
+	values := url.Values{}
+	values.Add("match[]", matchQuery)
+	res := app.cli.PostForm(t, app.prometheusAPIV1SeriesURL, values, http.StatusOK)
+	return NewPrometheusAPIV1SeriesResponse(t, res)
+}
+
+// String returns the string representation of the vmsingle app state.
+func (app *Vmsingle) String() string {
+	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q}", []any{
+		app.app, app.storageDataPath, app.httpListenAddr}...)
+}

apptest/vmstorage.go

@@ -2,6 +2,7 @@ package apptest
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 	"regexp"
 	"testing"
@@ -18,19 +19,8 @@ type Vmstorage struct {
 	httpListenAddr  string
 	vminsertAddr    string
 	vmselectAddr    string
-}
 
-// MustStartVmstorage is a test helper function that starts an instance of
+	forceFlushURL string
-// vmstorage and fails the test if the app fails to start.
-func MustStartVmstorage(t *testing.T, instance string, flags []string, cli *Client) *Vmstorage {
-	t.Helper()
-
-	app, err := StartVmstorage(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
 }
 
 // StartVmstorage starts an instance of vmstorage with the given flags. It also
@@ -65,6 +55,8 @@ func StartVmstorage(instance string, flags []string, cli *Client) (*Vmstorage, e
 		httpListenAddr:  stderrExtracts[1],
 		vminsertAddr:    stderrExtracts[2],
 		vmselectAddr:    stderrExtracts[3],
+
+		forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
 	}, nil
 }
 
@@ -80,6 +72,14 @@ func (app *Vmstorage) VmselectAddr() string {
 	return app.vmselectAddr
 }
 
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
+func (app *Vmstorage) ForceFlush(t *testing.T) {
+	t.Helper()
+
+	app.cli.Get(t, app.forceFlushURL, http.StatusOK)
+}
+
 // String returns the string representation of the vmstorage app state.
 func (app *Vmstorage) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q vminsertAddr: %q vmselectAddr: %q}", []any{

dashboards/Makefile

@@ -16,3 +16,6 @@ dashboards-sync:
 	SRC=vmalert.json D_UID=LzldHAVnz TITLE="VictoriaMetrics - vmalert" $(MAKE) dashboard-copy
 	SRC=vmauth.json D_UID=nbuo5Mr4k TITLE="VictoriaMetrics - vmauth" $(MAKE) dashboard-copy
 	SRC=operator.json D_UID=1H179hunk TITLE="VictoriaMetrics - operator" $(MAKE) dashboard-copy
+	SRC=backupmanager.json D_UID=gF-lxRdVz TITLE="VictoriaMetrics - backupmanager" $(MAKE) dashboard-copy
+	SRC=clusterbytenant.json D_UID=IZFqd3lMz TITLE="VictoriaMetrics Cluster Per Tenant Statistic" $(MAKE) dashboard-copy
+	SRC=victorialogs.json D_UID=OqPIZTX4z TITLE="VictoriaLogs" $(MAKE) dashboard-copy

dashboards/backupmanager.json

@@ -158,6 +158,7 @@
           "color": {
             "mode": "thresholds"
           },
+          "min": 0,
           "mappings": [
             {
               "options": {
@@ -233,6 +234,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -297,6 +299,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -361,6 +364,7 @@
           "color": {
             "mode": "thresholds"
           },
+          "min": 0,
           "mappings": [
             {
               "options": {
@@ -450,6 +454,7 @@
               "type": "value"
             }
           ],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -523,6 +528,7 @@
               "type": "value"
             }
           ],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [

dashboards/operator.json

@@ -101,6 +101,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -242,6 +243,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -311,6 +313,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -365,7 +368,7 @@
           "refId": "A"
         }
       ],
-      "title": "Electer Leaders",
+      "title": "Elected Leaders",
       "type": "stat"
     },
     {
@@ -382,6 +385,7 @@
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
+            "min": 0,
             "steps": [
               {
                 "color": "green",

dashboards/victorialogs.json

@@ -170,7 +170,7 @@
         "type": "prometheus",
         "uid": "$ds"
       },
-      "description": "Shows the datapoints ingestion rate.",
+      "description": "Shows the logs ingestion rate.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -583,7 +583,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(rate(vl_http_requests_total{job=~\"$job\", instance=~\"$instance\", path!~\".*(/insert|/metrics)\"}[$__rate_interval]))",
+          "expr": "sum(rate(vl_http_requests_total{job=~\"$job\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -682,7 +682,7 @@
         "type": "prometheus",
         "uid": "$ds"
       },
-      "description": "How many datapoints are inserted into storage per second",
+      "description": "How many logs are inserted into storage per second",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -781,7 +781,7 @@
           "refId": "A"
         }
       ],
-      "title": "Datapoints ingestion rate ",
+      "title": "Logs ingestion rate ",
       "type": "timeseries"
     },
     {
@@ -2816,8 +2816,8 @@
         },
         "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)",
         "hide": 0,
-        "includeAll": false,
+        "includeAll": true,
-        "multi": false,
+        "multi": true,
         "name": "instance",
         "options": [],
         "query": {

dashboards/vm/operator.json

@@ -102,6 +102,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -243,6 +244,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -312,6 +314,7 @@
             "mode": "thresholds"
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -366,7 +369,7 @@
           "refId": "A"
         }
       ],
-      "title": "Electer Leaders",
+      "title": "Elected Leaders",
       "type": "stat"
     },
     {
@@ -383,6 +386,7 @@
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
+            "min": 0,
             "steps": [
               {
                 "color": "green",

dashboards/vm/vmagent.json

@@ -142,6 +142,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -211,6 +212,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -281,6 +283,7 @@
         "defaults": {
           "decimals": 1,
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -344,6 +347,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -525,6 +529,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -1115,8 +1120,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1225,8 +1229,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1617,8 +1620,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -1732,8 +1734,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -1844,8 +1845,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -1980,8 +1980,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2112,8 +2111,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2218,8 +2216,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2325,8 +2322,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2432,8 +2428,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2538,8 +2533,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2668,8 +2662,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2773,8 +2766,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2881,8 +2873,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
+                    "color": "transparent"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2989,8 +2980,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
+                    "color": "transparent"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3096,8 +3086,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3203,8 +3192,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3316,8 +3304,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3421,8 +3408,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3498,8 +3484,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3655,8 +3640,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",

dashboards/vm/vmalert.json

@@ -232,6 +232,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -276,7 +277,7 @@
             "uid": "$ds"
           },
           "exemplar": false,
-          "expr": "count(vmalert_alerting_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"})",
+          "expr": "count(vmalert_alerting_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"})",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -294,6 +295,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -338,7 +340,7 @@
             "uid": "$ds"
           },
           "exemplar": false,
-          "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"})",
+          "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"})",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -356,6 +358,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -404,7 +407,7 @@
             "uid": "$ds"
           },
           "exemplar": false,
-          "expr": "(sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) or vector(0)) + \n(sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) or vector(0))",
+          "expr": "(sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) or vector(0)) + \n(sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) or vector(0))",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -422,6 +425,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -910,9 +914,9 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk_max($topk, max(sum(\n    rate(vmalert_iteration_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])\n/\n    rate(vmalert_iteration_duration_seconds_count{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])\n) by(job, instance, group)) \nby(job, group))",
+          "expr": "topk_max($topk, max(sum(\n    rate(vmalert_iteration_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])\n/\n    rate(vmalert_iteration_duration_seconds_count{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])\n) by(job, instance, group, file)) \nby(job, group, file))",
           "interval": "",
-          "legendFormat": "{{group}} ({{job}})",
+          "legendFormat": "({{job}}) {{group}}({{file}})",
           "range": true,
           "refId": "A"
         }
@@ -968,8 +972,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1071,8 +1074,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2090,8 +2092,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2247,8 +2248,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2292,9 +2292,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group) > 0",
+              "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
               "interval": "1m",
-              "legendFormat": "__auto",
+              "legendFormat": "({{job}}) {{group}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -2353,8 +2353,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2517,9 +2516,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk_max($topk, sum(vmalert_alerts_firing{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}) by(job, group, alertname) > 0)",
+              "expr": "topk_max($topk, sum(vmalert_alerts_firing{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}) by(job, group, file, alertname) > 0)",
               "interval": "",
-              "legendFormat": "{{group}}.{{alertname}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{alertname}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -2619,9 +2618,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) by(job, group, alertname) > 0",
+              "expr": "sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) by(job, group, file, alertname) > 0",
               "interval": "",
-              "legendFormat": "{{group}}.{{alertname}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{alertname}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -2721,9 +2720,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(vmalert_alerts_pending{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}) by(job, group, alertname) > 0",
+              "expr": "sum(vmalert_alerts_pending{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}) by(job, group, file, alertname) > 0",
               "interval": "",
-              "legendFormat": "{{group}}.{{alertname}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{alertname}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3050,9 +3049,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk_max($topk, \n  max(\n    sum(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}) by(job, instance, group, recording) > 0\n  ) by(job, group, recording)\n)",
+              "expr": "topk_max($topk, \n  max(\n    sum(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}) by(job, instance, group, file, recording) > 0\n  ) by(job, group, file, recording)\n)",
               "interval": "",
-              "legendFormat": "{{group}}.{{recording}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{recording}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3152,9 +3151,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"} < 1) by(job, group, recording)",
+              "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"} < 1) by(job, group, file, recording)",
               "interval": "",
-              "legendFormat": "{{group}}.{{recording}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{recording}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3251,9 +3250,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) by(job, group, recording) > 0",
+              "expr": "sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) by(job, group, file, recording) > 0",
               "interval": "",
-              "legendFormat": "{{group}}.{{recording}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{recording}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3749,6 +3748,29 @@
         "sort": 0,
         "type": "query"
       },
+      {
+        "allValue": ".*",
+        "current": {},
+        "datasource": {
+          "type": "victoriametrics-datasource",
+          "uid": "$ds"
+        },
+        "definition": "label_values(vmalert_iteration_total{job=~\"$job\", instance=~\"$instance\"},file)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "file",
+        "options": [],
+        "query": {
+          "query": "label_values(vmalert_iteration_total{job=~\"$job\", instance=~\"$instance\"},file)",
+          "refId": "VictoriaMetricsVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
       {
         "allValue": ".*",
         "current": {},

dashboards/vm/vmauth.json

@@ -379,6 +379,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [

dashboards/vmagent.json

@@ -141,6 +141,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -210,6 +211,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -280,6 +282,7 @@
         "defaults": {
           "decimals": 1,
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -343,6 +346,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -524,6 +528,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -1114,8 +1119,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1224,8 +1228,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1616,8 +1619,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -1731,8 +1733,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -1843,8 +1844,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -1979,8 +1979,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2111,8 +2110,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2217,8 +2215,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2324,8 +2321,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2431,8 +2427,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2537,8 +2532,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2667,8 +2661,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2772,8 +2765,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2880,8 +2872,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
+                    "color": "transparent"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2988,8 +2979,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
+                    "color": "transparent"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3095,8 +3085,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3202,8 +3191,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3315,8 +3303,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3420,8 +3407,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3497,8 +3483,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -3654,8 +3639,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",

dashboards/vmalert.json

@@ -231,6 +231,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -275,7 +276,7 @@
             "uid": "$ds"
           },
           "exemplar": false,
-          "expr": "count(vmalert_alerting_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"})",
+          "expr": "count(vmalert_alerting_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"})",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -293,6 +294,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -337,7 +339,7 @@
             "uid": "$ds"
           },
           "exemplar": false,
-          "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"})",
+          "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"})",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -355,6 +357,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -403,7 +406,7 @@
             "uid": "$ds"
           },
           "exemplar": false,
-          "expr": "(sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) or vector(0)) + \n(sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) or vector(0))",
+          "expr": "(sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) or vector(0)) + \n(sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) or vector(0))",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -421,6 +424,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -909,9 +913,9 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "topk_max($topk, max(sum(\n    rate(vmalert_iteration_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])\n/\n    rate(vmalert_iteration_duration_seconds_count{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])\n) by(job, instance, group)) \nby(job, group))",
+          "expr": "topk_max($topk, max(sum(\n    rate(vmalert_iteration_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])\n/\n    rate(vmalert_iteration_duration_seconds_count{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])\n) by(job, instance, group, file)) \nby(job, group, file))",
           "interval": "",
-          "legendFormat": "{{group}} ({{job}})",
+          "legendFormat": "({{job}}) {{group}}({{file}})",
           "range": true,
           "refId": "A"
         }
@@ -967,8 +971,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1070,8 +1073,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2089,8 +2091,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2246,8 +2247,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2291,9 +2291,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group) > 0",
+              "expr": "sum(increase(vmalert_iteration_missed_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
               "interval": "1m",
-              "legendFormat": "__auto",
+              "legendFormat": "({{job}}) {{group}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -2352,8 +2352,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "green"
-                    "value": null
                   },
                   {
                     "color": "red",
@@ -2516,9 +2515,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk_max($topk, sum(vmalert_alerts_firing{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}) by(job, group, alertname) > 0)",
+              "expr": "topk_max($topk, sum(vmalert_alerts_firing{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}) by(job, group, file, alertname) > 0)",
               "interval": "",
-              "legendFormat": "{{group}}.{{alertname}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{alertname}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -2618,9 +2617,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) by(job, group, alertname) > 0",
+              "expr": "sum(increase(vmalert_alerting_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) by(job, group, file, alertname) > 0",
               "interval": "",
-              "legendFormat": "{{group}}.{{alertname}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{alertname}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -2720,9 +2719,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(vmalert_alerts_pending{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}) by(job, group, alertname) > 0",
+              "expr": "sum(vmalert_alerts_pending{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}) by(job, group, file, alertname) > 0",
               "interval": "",
-              "legendFormat": "{{group}}.{{alertname}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{alertname}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3049,9 +3048,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "topk_max($topk, \n  max(\n    sum(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}) by(job, instance, group, recording) > 0\n  ) by(job, group, recording)\n)",
+              "expr": "topk_max($topk, \n  max(\n    sum(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}) by(job, instance, group, file, recording) > 0\n  ) by(job, group, file, recording)\n)",
               "interval": "",
-              "legendFormat": "{{group}}.{{recording}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{recording}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3151,9 +3150,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"} < 1) by(job, group, recording)",
+              "expr": "count(vmalert_recording_rules_last_evaluation_samples{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"} < 1) by(job, group, file, recording)",
               "interval": "",
-              "legendFormat": "{{group}}.{{recording}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{recording}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3250,9 +3249,9 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\"}[$__rate_interval])) by(job, group, recording) > 0",
+              "expr": "sum(increase(vmalert_recording_rules_errors_total{job=~\"$job\", instance=~\"$instance\", group=~\"$group\", file=~\"$file\"}[$__rate_interval])) by(job, group, file, recording) > 0",
               "interval": "",
-              "legendFormat": "{{group}}.{{recording}} ({{job}})",
+              "legendFormat": "({{job}}) {{group}}.{{recording}}({{file}})",
               "range": true,
               "refId": "A"
             }
@@ -3748,6 +3747,29 @@
         "sort": 0,
         "type": "query"
       },
+      {
+        "allValue": ".*",
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds"
+        },
+        "definition": "label_values(vmalert_iteration_total{job=~\"$job\", instance=~\"$instance\"},file)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "file",
+        "options": [],
+        "query": {
+          "query": "label_values(vmalert_iteration_total{job=~\"$job\", instance=~\"$instance\"},file)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
       {
         "allValue": ".*",
         "current": {},

dashboards/vmauth.json

@@ -378,6 +378,7 @@
       "fieldConfig": {
         "defaults": {
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [

deployment/docker/Makefile

@@ -6,7 +6,7 @@ ROOT_IMAGE ?= alpine:3.20.3
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.20.3
 
-GO_BUILDER_IMAGE := golang:1.23.1-alpine
+GO_BUILDER_IMAGE := golang:1.23.3-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 DOCKER ?= docker

deployment/docker/base/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image=non-existing
-ARG root_image==non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

deployment/docker/docker-compose-cluster.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.105.0
+    image: victoriametrics/vmagent:v1.106.0
     depends_on:
       - "vminsert"
     ports:
@@ -39,7 +39,7 @@ services:
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.105.0-cluster
+    image: victoriametrics/vmstorage:v1.106.0-cluster
     ports:
       - 8482
       - 8400
@@ -51,7 +51,7 @@ services:
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.105.0-cluster
+    image: victoriametrics/vmstorage:v1.106.0-cluster
     ports:
       - 8482
       - 8400
@@ -66,7 +66,7 @@ services:
   # pre-process them and distributes across configured vmstorage shards.
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.105.0-cluster
+    image: victoriametrics/vminsert:v1.106.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -81,7 +81,7 @@ services:
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
     container_name: vmselect-1
-    image: victoriametrics/vmselect:v1.105.0-cluster
+    image: victoriametrics/vmselect:v1.106.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -94,7 +94,7 @@ services:
     restart: always
   vmselect-2:
     container_name: vmselect-2
-    image: victoriametrics/vmselect:v1.105.0-cluster
+    image: victoriametrics/vmselect:v1.106.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -112,7 +112,7 @@ services:
   # It can be used as an authentication proxy.
   vmauth:
     container_name: vmauth
-    image: victoriametrics/vmauth:v1.105.0
+    image: victoriametrics/vmauth:v1.106.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -127,7 +127,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.105.0
+    image: victoriametrics/vmalert:v1.106.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/docker-compose-victorialogs.yml

@@ -40,7 +40,7 @@ services:
   #  storing logs and serving read queries.
   victorialogs:
     container_name: victorialogs
-    image: victoriametrics/victoria-logs:v0.40.0-victorialogs
+    image: victoriametrics/victoria-logs:v1.0.0-victorialogs
     command:
       - "--storageDataPath=/vlogs"
       - "--httpListenAddr=:9428"
@@ -55,7 +55,7 @@ services:
   # scraping, storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.105.0
+    image: victoriametrics/victoria-metrics:v1.106.0
     ports:
       - 8428:8428
     volumes:
@@ -74,7 +74,7 @@ services:
   # depending on the requested path.
   vmauth:
     container_name: vmauth
-    image: victoriametrics/vmauth:v1.105.0
+    image: victoriametrics/vmauth:v1.106.0
     depends_on:
       - "victoriametrics"
       - "victorialogs"
@@ -91,7 +91,7 @@ services:
   # vmalert executes alerting and recording rules according to given rule type.
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.105.0
+    image: victoriametrics/vmalert:v1.106.0
     depends_on:
       - "vmauth"
       - "alertmanager"

deployment/docker/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.105.0
+    image: victoriametrics/vmagent:v1.106.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -22,7 +22,7 @@ services:
   # storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.105.0
+    image: victoriametrics/victoria-metrics:v1.106.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -65,7 +65,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.105.0
+    image: victoriametrics/vmalert:v1.106.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-vmalert.yml

@@ -23,9 +23,9 @@ groups:
         labels:
           severity: warning
         annotations:
-          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=13&var-instance={{ $labels.instance }}&var-group={{ $labels.group }}"
+          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=13&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
           summary: "Alerting rules are failing for vmalert instance {{ $labels.instance }}"
-          description: "Alerting rules execution is failing for group \"{{ $labels.group }}\".
+          description: "Alerting rules execution is failing for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
             Check vmalert's logs for detailed error message."
 
       - alert: RecordingRulesError
@@ -34,9 +34,9 @@ groups:
         labels:
           severity: warning
         annotations:
-          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=30&var-instance={{ $labels.instance }}&var-group={{ $labels.group }}"
+          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=30&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
           summary: "Recording rules are failing for vmalert instance {{ $labels.instance }}"
-          description: "Recording rules execution is failing for group \"{{ $labels.group }}\".
+          description: "Recording rules execution is failing for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
             Check vmalert's logs for detailed error message."
 
       - alert: RecordingRulesNoData
@@ -45,9 +45,9 @@ groups:
         labels:
           severity: info
         annotations:
-          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=33&var-group={{ $labels.group }}"
+          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=33&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
           summary: "Recording rule {{ $labels.recording }} ({{ $labels.group }}) produces no data"
-          description: "Recording rule \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\" 
+          description: "Recording rule \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\ in file \"{{ $labels.file }}\" 
             produces 0 samples over the last 30min. It might be caused by a misconfiguration 
             or incorrect query expression."
 
@@ -58,7 +58,7 @@ groups:
           severity: warning
         annotations:
           summary: "vmalert instance {{ $labels.instance }} is missing rules evaluations"
-          description: "vmalert instance {{ $labels.instance }} is missing rules evaluations for group \"{{ $labels.group }}\".
+          description: "vmalert instance {{ $labels.instance }} is missing rules evaluations for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
             The group evaluation time takes longer than the configured evaluation interval. This may result in missed 
             alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency of
             group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/vmalert/#groups. 

deployment/docker/victorialogs/compose-base.yml

@@ -1,7 +1,7 @@
 services:
   # meta service will be ignored by compose
   .victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v0.40.0-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v1.0.0-victorialogs
     command:
       - -storageDataPath=/vlogs
       - -loggerFormat=json
@@ -17,6 +17,13 @@ services:
       timeout: 1s
       retries: 10
 
+  dd-logs:
+    image: docker.io/victoriametrics/vmauth:v1.106.0
+    restart: on-failure
+    volumes:
+      - ./:/etc/vmauth
+    command: -auth.config=/etc/vmauth/vmauth.yaml
+
   victorialogs:
     extends: .victorialogs
     ports:

deployment/docker/victorialogs/datadog-agent/.gitignore

@@ -0,0 +1 @@
+**/logs

deployment/docker/victorialogs/datadog-agent/README.md

@@ -0,0 +1,29 @@
+# Docker compose DataDog Agent integration with VictoriaLogs
+
+The folder contains examples of [DataDog agent](https://docs.datadoghq.com/agent) integration with VictoriaLogs using protocols:
+
+* [datadog](./datadog)
+
+To spin-up environment `cd` to any of listed above directories run the following command:
+```
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+```
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* datadog - Datadog logs collection agent, which is configured to collect and write data to `victorialogs`
+* victorialogs - VictoriaLogs log database, which accepts the data from `datadog`
+* victoriametrics - VictoriaMetrics metrics database, which collects metrics from `victorialogs` and `datadog`
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/datadog-agent/compose-base.yml

@@ -0,0 +1,26 @@
+include:
+ - ../compose-base.yml
+services:
+  agent:
+    image: docker.io/datadog/agent:7.57.2
+    restart: on-failure
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+    environment:
+      DD_API_KEY: test
+      DD_URL: http://victoriametrics:8428/datadog
+      DD_LOGS_CONFIG_LOGS_DD_URL: http://dd-logs:8427
+      DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: true
+      DD_LOGS_ENABLED: true
+      DD_LOGS_CONFIG_USE_HTTP: true
+      DD_PROCESS_CONFIG_PROCESS_COLLECTION_ENABLED: false
+      DD_PROCESS_CONFIG_CONTAINER_COLLECTION_ENABLED: false
+      DD_PROCESS_CONFIG_PROCESS_DISCOVERY_ENABLED: false
+    depends_on:
+      victorialogs:
+        condition: service_healthy
+      victoriametrics:
+        condition: service_healthy

deployment/docker/victorialogs/datadog-agent/datadog/compose.yml

@@ -0,0 +1,3 @@
+include:
+ - ../compose-base.yml
+name: agent-datadog

deployment/docker/victorialogs/fluentbit/README.md

@@ -2,6 +2,7 @@
 
 The folder contains examples of [FluentBit](https://docs.fluentbit.io/manual) integration with VictoriaLogs using protocols:
 
+* [datadog](./datadog)
 * [loki](./loki)
 * [jsonline single node](./jsonline)
 * [jsonline HA setup](./jsonline-ha)
@@ -30,6 +31,7 @@ Querying the data
 * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
 
 FluentBit configuration example can be found below:
+* [datadog](./datadog/fluent-bit.conf)
 * [loki](./loki/fluent-bit.conf)
 * [jsonline single node](./jsonline/fluent-bit.conf)
 * [jsonline HA setup](./jsonline-ha/fluent-bit.conf)

deployment/docker/victorialogs/fluentbit/datadog/compose.yml

@@ -0,0 +1,3 @@
+include:
+ - ../compose-base.yml
+name: fluentbit-datadog

deployment/docker/victorialogs/fluentbit/datadog/fluent-bit.conf

@@ -0,0 +1,31 @@
+[INPUT]
+    name              tail
+    path              /var/lib/docker/containers/**/*.log
+    path_key         path
+    multiline.parser  docker, cri
+    Parser docker
+    Docker_Mode  On
+
+[INPUT]
+    Name     syslog
+    Listen   0.0.0.0
+    Port     5140
+    Parser   syslog-rfc3164
+    Mode     tcp
+
+[SERVICE]
+    Flush        1
+    Parsers_File parsers.conf
+
+[OUTPUT]
+    Name           datadog
+    Match          *
+    Host           dd-logs
+    Port           8427
+    TLS            off
+    compress       gzip
+    apikey         test
+    dd_service     test
+    dd_source      data
+    dd_message_key log
+    dd_tags        env:dev

deployment/docker/victorialogs/fluentd/README.md

@@ -4,6 +4,7 @@ The folder contains examples of [Fluentd](https://www.fluentd.org/) integration
 
 * [loki](./loki)
 * [jsonline](./jsonline)
+* [datadog](./datadog)
 * [elasticsearch](./elasticsearch)
 
 All required plugins, that should be installed in order to support protocols listed above can be found in a [Dockerfile](./Dockerfile)

deployment/docker/victorialogs/fluentd/datadog/compose.yml

@@ -0,0 +1,3 @@
+include:
+ - ../compose-base.yml
+name: fluentd-datadog

deployment/docker/victorialogs/fluentd/datadog/fluent.conf

@@ -0,0 +1,27 @@
+<source>
+  @type tail
+  format none
+  tag docker.testlog
+  path /var/lib/docker/containers/**/*.log
+</source>
+
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<match **>
+  @type datadog
+  api_key test
+  # Optional
+  port 8427
+  use_ssl false
+  host dd-logs
+  include_tag_key true
+  tag_key 'tag'
+  # Optional parameters
+  dd_source 'fluentd'
+  dd_tags 'key1:value1,key2:value2'
+  dd_sourcecategory 'test'
+</match>

deployment/docker/victorialogs/vector/README.md

@@ -6,6 +6,7 @@ The folder contains examples of [Vector](https://vector.dev/docs/) integration w
 * [loki](./loki)
 * [jsonline single node](./jsonline)
 * [jsonline HA setup](./jsonline-ha)
+* [datadog](./datadog)
 
 To spin-up environment `cd` to any of listed above directories run the following command:
 ```
@@ -34,5 +35,6 @@ Vector configuration example can be found below:
 * [loki](./loki/vector.yaml)
 * [jsonline single node](./jsonline/vector.yaml)
 * [jsonline HA setup](./jsonline-ha/vector.yaml)
+* [datadog](./datadog/vector.yaml)
 
 Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/vector/datadog/compose.yml

@@ -0,0 +1,3 @@
+include:
+ - ../compose-base.yml
+name: vector-datadog