Merge branch 'public-single-node' into victorialogs-wip

app/vmalert/rule/group.go

@@ -9,10 +9,11 @@ import (
 	"hash/fnv"
 	"net/url"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+
 	"github.com/cheggaaa/pb/v3"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -724,13 +725,19 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	return errGr.Err()
 }
 
+var bbPool bytesutil.ByteBufferPool
+
 // getStaleSeries checks whether there are stale series from previously sent ones.
 func (e *executor) getStaleSeries(r Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
+	bb := bbPool.Get()
+	defer bbPool.Put(bb)
+
 	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
 	for _, ts := range tss {
-		// convert labels to strings so we can compare with previously sent series
-		key := labelsToString(ts.Labels)
-		ruleLabels[key] = ts.Labels
+		// convert labels to strings, so we can compare with previously sent series
+		bb.B = labelsToString(bb.B, ts.Labels)
+		ruleLabels[string(bb.B)] = ts.Labels
+		bb.Reset()
 	}
 
 	rID := r.ID()
@@ -776,21 +783,20 @@ func (e *executor) purgeStaleSeries(activeRules []Rule) {
 	e.previouslySentSeriesToRWMu.Unlock()
 }
 
-func labelsToString(labels []prompbmarshal.Label) string {
-	var b strings.Builder
-	b.WriteRune('{')
+func labelsToString(dst []byte, labels []prompbmarshal.Label) []byte {
+	dst = append(dst, '{')
 	for i, label := range labels {
 		if len(label.Name) == 0 {
-			b.WriteString("__name__")
+			dst = append(dst, "__name__"...)
 		} else {
-			b.WriteString(label.Name)
+			dst = append(dst, label.Name...)
 		}
-		b.WriteRune('=')
-		b.WriteString(strconv.Quote(label.Value))
+		dst = append(dst, '=')
+		dst = strconv.AppendQuote(dst, label.Value)
 		if i < len(labels)-1 {
-			b.WriteRune(',')
+			dst = append(dst, ',')
 		}
 	}
-	b.WriteRune('}')
-	return b.String()
+	dst = append(dst, '}')
+	return dst
 }

app/vmalert/rule/group_timing_test.go

@@ -0,0 +1,36 @@
+package rule
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func BenchmarkGetStaleSeries(b *testing.B) {
+	ts := time.Now()
+	n := 100
+	payload := make([]prompbmarshal.TimeSeries, n)
+	for i := 0; i < n; i++ {
+		s := fmt.Sprintf("%d", i)
+		labels := toPromLabels(b,
+			"__name__", "foo", ""+
+				"instance", s,
+			"job", s,
+			"state", s,
+		)
+		payload = append(payload, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels))
+	}
+
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	ar := &AlertingRule{RuleID: 1}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		e.getStaleSeries(ar, payload, ts)
+	}
+}

app/vmalert/rule/test_helpers.go

@@ -95,7 +95,7 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
-func toPromLabels(t *testing.T, labels ...string) []prompbmarshal.Label {
+func toPromLabels(t testing.TB, labels ...string) []prompbmarshal.Label {
 	t.Helper()
 	if len(labels) == 0 || len(labels)%2 != 0 {
 		t.Fatalf("expected to get even number of labels")

app/vmauth/auth_config.go

@@ -693,6 +693,14 @@ func loadAuthConfig() (bool, error) {
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
+	if prevAc != nil {
+		// explicilty unregister metrics, since all summary type metrics
+		// are registered at global state of metrics package
+		// and must be removed from it to release memory.
+		// Metrics must be unregistered only after atomic.Value.Store calls above
+		// Otherwise it may lead to metric gaps, since UnregisterAllMetrics is slow operation
+		prevAc.ms.UnregisterAllMetrics()
+	}
 
 	return true, nil
 }

docs/CHANGELOG.md

@@ -50,6 +50,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): set correct suffix `<output>_prometheus` for aggregation outputs [increase_prometheus](https://docs.victoriametrics.com/stream-aggregation/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/stream-aggregation/#total_prometheus). Before, outputs `total` and `total_prometheus` or `increase` and `increase_prometheus` had the same suffix.
 * BUGFIX: properly estimate the needed memory for query execution if it has the format [`aggr_func`](https://docs.victoriametrics.com/metricsql/#aggregate-functions)([`rollup_func[d]`](https://docs.victoriametrics.com/metricsql/#rollup-functions) (for example, `sum(rate(request_duration_seconds_bucket[5m]))`). This should allow performing aggregations over bigger number of time series when VictoriaMetrics runs in environments with small amounts of available memory. The issue has been introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/5138eaeea0791caa34bcfab410e0ca9cd253cd8f) in [v1.83.0](https://docs.victoriametrics.com/changelog_2022/#v1830).
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): correctly apply `-inmemoryDataFlushInterval` when it's set to minimum supported value 1s.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly release memory used for metrics during config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247).
 
 * DEPRECATION: [vmagent](https://docs.victoriametrics.com/vmagent/): removed deprecated `-remoteWrite.multitenantURL` flag from vmagent. This flag was deprecated since [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0). Use `-enableMultitenantHandlers` instead, as it is easier to use and combine with [multitenant URL at vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). See these [docs for details](https://docs.victoriametrics.com/vmagent.html#multitenancy).
 

docs/VictoriaLogs/CHANGELOG.md

@@ -19,11 +19,16 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta
 
 ## tip
 
+## [v0.6.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.1-victorialogs)
+
+Released at 2024-05-14
+
 * FEATURE: use [natural sort order](https://en.wikipedia.org/wiki/Natural_sort_order) when sorting logs via [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe).
 
 * BUGFIX: properly return matching logs in [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) with small number of entries. Previously they could be skipped. The issue has been introduced in [the release v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs).
 * BUGFIX: fix `runtime error: index out of range` panic when using [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) like `_time:1h | sort by (_time)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6258).
 
+
 ## [v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs)
 
 Released at 2024-05-12

lib/streamaggr/streamaggr.go

@@ -871,6 +871,7 @@ func getOutputKey(key string) string {
 	if nSize <= 0 {
 		logger.Panicf("BUG: cannot unmarshal inputKeyLen from uvarint")
 	}
+	src = src[nSize:]
 	outputKey := src[inputKeyLen:]
 	return bytesutil.ToUnsafeString(outputKey)
 }