mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
app/{vminsert,vmagent}: drop series on exceeding -maxLabelsPerTimeseries, -maxLabelValueLen flags and maxLabelNameLen=256 limits
This commit is contained in:
Andrii Chubatiuk
parent
a335ed23c7
commit
a921ad7380
23 changed files with 221 additions and 210 deletions
|
|
@ -10,7 +10,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
)
|
||||
|
|
@ -48,7 +48,7 @@ func selfScraper(scrapeInterval time.Duration) {
|
|||
var bb bytesutil.ByteBuffer
|
||||
var rows prometheus.Rows
|
||||
var mrs []storage.MetricRow
|
||||
var labels []prompb.Label
|
||||
var labels []prompbmarshal.Label
|
||||
t := time.NewTicker(scrapeInterval)
|
||||
f := func(currentTime time.Time, sendStaleMarkers bool) {
|
||||
currentTimestamp := currentTime.UnixNano() / 1e6
|
||||
|
|
@ -99,11 +99,11 @@ func selfScraper(scrapeInterval time.Duration) {
|
|||
}
|
||||
}
|
||||
|
||||
func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
|
||||
func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
|
||||
if len(dst) < cap(dst) {
|
||||
dst = dst[:len(dst)+1]
|
||||
} else {
|
||||
dst = append(dst, prompb.Label{})
|
||||
dst = append(dst, prompbmarshal.Label{})
|
||||
}
|
||||
lb := &dst[len(dst)-1]
|
||||
lb.Name = key
|
||||
|
|
|
|||
|
|
@ -7,13 +7,10 @@ import (
|
|||
"net/url"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
|
|
@ -21,6 +18,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
|
|
@ -29,6 +27,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
storagelimits "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/limits"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
|
|
@ -485,6 +484,14 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
|
|||
rowsCountAfterRelabel := getRowsCount(tssBlock)
|
||||
rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
|
||||
}
|
||||
idxDiff := 0
|
||||
for i, ts := range tssBlock {
|
||||
if storagelimits.ExceedingLabels(ts.Labels) {
|
||||
idxDiff++
|
||||
} else if idxDiff > 0 {
|
||||
tss[i-idxDiff] = tss[i]
|
||||
}
|
||||
}
|
||||
sortLabelsIfNeeded(tssBlock)
|
||||
tssBlock = limitSeriesCardinality(tssBlock)
|
||||
if sas.IsEnabled() {
|
||||
|
|
@ -729,29 +736,14 @@ func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue i
|
|||
select {
|
||||
case <-logSkippedSeriesTicker.C:
|
||||
// Do not use logger.WithThrottler() here, since this will increase CPU usage
|
||||
// because every call to logSkippedSeries will result to a call to labelsToString.
|
||||
logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
|
||||
// because every call to logSkippedSeries will result to a call to prompbmarshal.LabelsToString.
|
||||
logger.Warnf("skip series %s because %s=%d reached", prompbmarshal.LabelsToString(labels), flagName, flagValue)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
var logSkippedSeriesTicker = time.NewTicker(5 * time.Second)
|
||||
|
||||
func labelsToString(labels []prompbmarshal.Label) string {
|
||||
var b []byte
|
||||
b = append(b, '{')
|
||||
for i, label := range labels {
|
||||
b = append(b, label.Name...)
|
||||
b = append(b, '=')
|
||||
b = strconv.AppendQuote(b, label.Value)
|
||||
if i+1 < len(labels) {
|
||||
b = append(b, ',')
|
||||
}
|
||||
}
|
||||
b = append(b, '}')
|
||||
return string(b)
|
||||
}
|
||||
|
||||
var (
|
||||
globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
|
||||
rowsDroppedByGlobalRelabel = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
)
|
||||
|
||||
|
|
@ -214,15 +214,15 @@ func processFlags() {
|
|||
func fillStorage(series []vm.TimeSeries) error {
|
||||
var mrs []storage.MetricRow
|
||||
for _, series := range series {
|
||||
var labels []prompb.Label
|
||||
var labels []prompbmarshal.Label
|
||||
for _, lp := range series.LabelPairs {
|
||||
labels = append(labels, prompb.Label{
|
||||
labels = append(labels, prompbmarshal.Label{
|
||||
Name: lp.Name,
|
||||
Value: lp.Value,
|
||||
})
|
||||
}
|
||||
if series.Name != "" {
|
||||
labels = append(labels, prompb.Label{
|
||||
labels = append(labels, prompbmarshal.Label{
|
||||
Name: "__name__",
|
||||
Value: series.Name,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -8,9 +8,10 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
storagelimits "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/limits"
|
||||
)
|
||||
|
||||
// InsertCtx contains common bits for data points insertion.
|
||||
|
|
@ -30,7 +31,7 @@ type InsertCtx struct {
|
|||
func (ctx *InsertCtx) Reset(rowsLen int) {
|
||||
labels := ctx.Labels
|
||||
for i := range labels {
|
||||
labels[i] = prompb.Label{}
|
||||
labels[i] = prompbmarshal.Label{}
|
||||
}
|
||||
ctx.Labels = labels[:0]
|
||||
|
||||
|
|
@ -51,7 +52,7 @@ func cleanMetricRow(mr *storage.MetricRow) {
|
|||
mr.MetricNameRaw = nil
|
||||
}
|
||||
|
||||
func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
|
||||
func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompbmarshal.Label) []byte {
|
||||
start := len(ctx.metricNamesBuf)
|
||||
ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...)
|
||||
ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels)
|
||||
|
|
@ -60,7 +61,10 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label)
|
|||
}
|
||||
|
||||
// WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
|
||||
func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) error {
|
||||
func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompbmarshal.Label, timestamp int64, value float64) error {
|
||||
if storagelimits.ExceedingLabels(labels) {
|
||||
return nil
|
||||
}
|
||||
metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
|
||||
return ctx.addRow(metricNameRaw, timestamp, value)
|
||||
}
|
||||
|
|
@ -68,7 +72,10 @@ func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, times
|
|||
// WriteDataPointExt writes (timestamp, value) with the given metricNameRaw and labels into ctx buffer.
|
||||
//
|
||||
// It returns metricNameRaw for the given labels if len(metricNameRaw) == 0.
|
||||
func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompb.Label, timestamp int64, value float64) ([]byte, error) {
|
||||
func (ctx *InsertCtx) WriteDataPointExt(metricNameRaw []byte, labels []prompbmarshal.Label, timestamp int64, value float64) ([]byte, error) {
|
||||
if storagelimits.ExceedingLabels(labels) {
|
||||
return metricNameRaw, nil
|
||||
}
|
||||
if len(metricNameRaw) == 0 {
|
||||
metricNameRaw = ctx.marshalMetricNameRaw(nil, labels)
|
||||
}
|
||||
|
|
@ -106,7 +113,7 @@ func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
|
|||
// Do not skip labels with empty name, since they are equal to __name__.
|
||||
return
|
||||
}
|
||||
ctx.Labels = append(ctx.Labels, prompb.Label{
|
||||
ctx.Labels = append(ctx.Labels, prompbmarshal.Label{
|
||||
// Do not copy name and value contents for performance reasons.
|
||||
// This reduces GC overhead on the number of objects and allocations.
|
||||
Name: bytesutil.ToUnsafeString(name),
|
||||
|
|
@ -124,7 +131,7 @@ func (ctx *InsertCtx) AddLabel(name, value string) {
|
|||
// Do not skip labels with empty name, since they are equal to __name__.
|
||||
return
|
||||
}
|
||||
ctx.Labels = append(ctx.Labels, prompb.Label{
|
||||
ctx.Labels = append(ctx.Labels, prompbmarshal.Label{
|
||||
// Do not copy name and value contents for performance reasons.
|
||||
// This reduces GC overhead on the number of objects and allocations.
|
||||
Name: name,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"flag"
|
||||
"sort"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
)
|
||||
|
||||
var sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to storage. `+
|
||||
|
|
@ -19,7 +19,7 @@ func (ctx *InsertCtx) SortLabelsIfNeeded() {
|
|||
}
|
||||
}
|
||||
|
||||
type sortedLabels []prompb.Label
|
||||
type sortedLabels []prompbmarshal.Label
|
||||
|
||||
func (sl *sortedLabels) Len() int { return len(*sl) }
|
||||
func (sl *sortedLabels) Less(i, j int) bool {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
|
||||
|
|
@ -150,7 +149,7 @@ type pushCtx struct {
|
|||
Common common.InsertCtx
|
||||
metricNameBuf []byte
|
||||
metricGroupBuf []byte
|
||||
originLabels []prompb.Label
|
||||
originLabels []prompbmarshal.Label
|
||||
}
|
||||
|
||||
func (ctx *pushCtx) reset() {
|
||||
|
|
@ -160,7 +159,7 @@ func (ctx *pushCtx) reset() {
|
|||
|
||||
originLabels := ctx.originLabels
|
||||
for i := range originLabels {
|
||||
originLabels[i] = prompb.Label{}
|
||||
originLabels[i] = prompbmarshal.Label{}
|
||||
}
|
||||
ctx.originLabels = originLabels[:0]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
|
||||
)
|
||||
|
||||
|
|
@ -67,8 +66,6 @@ var (
|
|||
"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
|
||||
configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
|
||||
reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
|
||||
maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
|
||||
maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -87,8 +84,6 @@ var staticServer = http.FileServer(http.FS(staticFiles))
|
|||
func Init() {
|
||||
relabel.Init()
|
||||
vminsertCommon.InitStreamAggr()
|
||||
storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
|
||||
storage.SetMaxLabelValueLen(*maxLabelValueLen)
|
||||
common.StartUnmarshalWorkers()
|
||||
if len(*graphiteListenAddr) > 0 {
|
||||
graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
|
||||
|
|
@ -439,14 +434,4 @@ var (
|
|||
promscrapeStatusConfigRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/config"}`)
|
||||
|
||||
promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
|
||||
|
||||
_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
|
||||
return float64(storage.MetricsWithDroppedLabels.Load())
|
||||
})
|
||||
_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
|
||||
return float64(storage.TooLongLabelNames.Load())
|
||||
})
|
||||
_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
|
||||
return float64(storage.TooLongLabelValues.Load())
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
|
@ -108,7 +107,7 @@ func (ctx *Ctx) Reset() {
|
|||
// ApplyRelabeling applies relabeling to the given labels and returns the result.
|
||||
//
|
||||
// The returned labels are valid until the next call to ApplyRelabeling.
|
||||
func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
|
||||
func (ctx *Ctx) ApplyRelabeling(labels []prompbmarshal.Label) []prompbmarshal.Label {
|
||||
pcs := pcsGlobal.Load()
|
||||
if pcs.Len() == 0 && !*usePromCompatibleNaming {
|
||||
// There are no relabeling rules.
|
||||
|
|
@ -159,7 +158,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
|
|||
name = ""
|
||||
}
|
||||
value := label.Value
|
||||
dst = append(dst, prompb.Label{
|
||||
dst = append(dst, prompbmarshal.Label{
|
||||
Name: name,
|
||||
Value: value,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
|
@ -95,7 +95,7 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
|
|||
_ = deadline // TODO: use the deadline as in the cluster branch
|
||||
paths := r.Form["path"]
|
||||
var row graphiteparser.Row
|
||||
var labels []prompb.Label
|
||||
var labels []prompbmarshal.Label
|
||||
var b []byte
|
||||
var tagsPool []graphiteparser.Tag
|
||||
mrs := make([]storage.MetricRow, len(paths))
|
||||
|
|
@ -122,12 +122,12 @@ func registerMetrics(startTime time.Time, w http.ResponseWriter, r *http.Request
|
|||
canonicalPaths[i] = string(b)
|
||||
|
||||
// Convert parsed metric and tags to labels.
|
||||
labels = append(labels[:0], prompb.Label{
|
||||
labels = append(labels[:0], prompbmarshal.Label{
|
||||
Name: "__name__",
|
||||
Value: row.Metric,
|
||||
})
|
||||
for _, tag := range row.Tags {
|
||||
labels = append(labels, prompb.Label{
|
||||
labels = append(labels, prompbmarshal.Label{
|
||||
Name: tag.Key,
|
||||
Value: tag.Value,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -3674,12 +3674,12 @@
|
|||
"uid": "$ds"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"expr": "sum(increase(vm_series_dropped_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (reason)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "metrics with dropped labels",
|
||||
"legendFormat": "dropped series, which a hitting {{reason}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
|
|
|
|||
|
|
@ -4190,12 +4190,12 @@
|
|||
"uid": "$ds"
|
||||
},
|
||||
"exemplar": false,
|
||||
"expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance)",
|
||||
"expr": "sum(increase(vm_series_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance,reason)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "{{instance}} - limit exceeded",
|
||||
"legendFormat": "{{instance}} - {{reason}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3675,12 +3675,12 @@
|
|||
"uid": "$ds"
|
||||
},
|
||||
"exemplar": true,
|
||||
"expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"expr": "sum(increase(vm_series_dropped_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (reason)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "metrics with dropped labels",
|
||||
"legendFormat": "dropped series, which a hitting {{reason}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
|
|
|
|||
|
|
@ -4191,12 +4191,12 @@
|
|||
"uid": "$ds"
|
||||
},
|
||||
"exemplar": false,
|
||||
"expr": "sum(increase(vm_metrics_with_dropped_labels_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance)",
|
||||
"expr": "sum(increase(vm_series_dropped_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, reason)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "{{instance}} - limit exceeded",
|
||||
"legendFormat": "{{instance}} - {{reason}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,13 +141,13 @@ groups:
|
|||
See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"
|
||||
|
||||
- alert: LabelsLimitExceededOnIngestion
|
||||
expr: increase(vm_metrics_with_dropped_labels_total[5m]) > 0
|
||||
expr: increase(vm_series_dropped_total{reason="too_many_labels"}[5m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=116&var-instance={{ $labels.instance }}"
|
||||
summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding labels limit"
|
||||
summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding one of labels limits: `{{ $labels.reason}}`"
|
||||
description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
|
||||
This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
|
||||
correctly or that clients which send these metrics aren't misbehaving."
|
||||
|
|
|
|||
|
|
@ -120,10 +120,11 @@ groups:
|
|||
`-maxLabelValueLen` command-line flags.
|
||||
|
||||
- alert: TooLongLabelValues
|
||||
expr: increase(vm_too_long_label_values_total[5m]) > 0
|
||||
expr: increase(vm_series_dropped_total{reason="too_long_label_value"}[5m]) > 0
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}"
|
||||
summary: "VictoriaMetrics truncates too long label values"
|
||||
description: |
|
||||
The maximum length of a label value is limited via `-maxLabelValueLen` cmd-line flag.
|
||||
|
|
@ -132,10 +133,11 @@ groups:
|
|||
either reduce the size of label values or increase `-maxLabelValueLen`.
|
||||
|
||||
- alert: TooLongLabelNames
|
||||
expr: increase(vm_too_long_label_names_total[5m]) > 0
|
||||
expr: increase(vm_series_dropped_total{reason="too_long_label_name"}[5m]) > 0
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}"
|
||||
summary: "VictoriaMetrics truncates too long label names"
|
||||
description: >
|
||||
The maximum length of a label name is limited by 256 bytes.
|
||||
|
|
|
|||
|
|
@ -121,13 +121,13 @@ groups:
|
|||
See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"
|
||||
|
||||
- alert: LabelsLimitExceededOnIngestion
|
||||
expr: increase(vm_metrics_with_dropped_labels_total[5m]) > 0
|
||||
expr: increase(vm_series_dropped_total{reason="too_many_labels"}[5m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=74&var-instance={{ $labels.instance }}"
|
||||
summary: "Metrics ingested in ({{ $labels.instance }}) are exceeding labels limit"
|
||||
summary: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding one of labels limits: `{{ $labels.reason}}`"
|
||||
description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
|
||||
This prevents ingestion of metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
|
||||
This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
|
||||
correctly or that clients which send these metrics aren't misbehaving."
|
||||
|
|
@ -1243,7 +1243,7 @@ Below is the output for `/path/to/vminsert -help`:
|
|||
The maximum size in bytes of a single Prometheus remote_write API request
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
|
||||
-maxLabelValueLen int
|
||||
The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096)
|
||||
The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096)
|
||||
-maxLabelsPerTimeseries int
|
||||
The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
|
||||
-memory.allowedBytes size
|
||||
|
|
|
|||
|
|
@ -2462,9 +2462,10 @@ and [cardinality explorer docs](#cardinality-explorer).
|
|||
* New time series can be logged if `-logNewSeries` command-line flag is passed to VictoriaMetrics.
|
||||
|
||||
* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag
|
||||
and drops superfluous labels. This prevents from ingesting metrics with too many labels.
|
||||
It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
|
||||
and drops series with superfluous labels. This prevents from ingesting metrics with too many labels.
|
||||
It is recommended [monitoring](#monitoring) `vm_series_dropped_total{reason="too_many_labels"}`
|
||||
metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
|
||||
Alternatively you can use [relabeling](https://docs.victoriametrics.com/single-server-victoriametrics/#relabeling) to change metric target labels.
|
||||
|
||||
* If you store Graphite metrics like `foo.bar.baz` in VictoriaMetrics, then `{__graphite__="foo.*.baz"}` filter can be used for selecting such metrics.
|
||||
See [these docs](#selecting-graphite-metrics) for details. You can also query Graphite metrics with [Graphite querying API](#graphite-render-api-usage).
|
||||
|
|
@ -2960,9 +2961,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
|||
The maximum size in bytes of a single Prometheus remote_write API request
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
|
||||
-maxLabelValueLen int
|
||||
The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented (default 4096)
|
||||
The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096)
|
||||
-maxLabelsPerTimeseries int
|
||||
The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
|
||||
The maximum number of labels accepted per time series. Series with superfluous labels are dropped. In this case the `vm_series_dropped_total{reason="too_many_labels"}` metric at /metrics page is incremented (default 30)
|
||||
-memory.allowedBytes size
|
||||
Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
|
||||
|
|
|
|||
|
|
@ -20,12 +20,15 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
|
|||
|
||||
* SECURITY: upgrade Go builder from Go1.23.1 to Go1.23.3. See the list of issues addressed in [Go1.23.2](https://github.com/golang/go/issues?q=milestone%3AGo1.23.2+label%3ACherryPickApproved) and [Go1.23.3](https://github.com/golang/go/issues?q=milestone%3AGo1.23.3+label%3ACherryPickApproved).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent) added `-maxLabelsPerTimeseries` and `-maxLabelValueLen` flags, which limits amount of labels, label name and value length for pushed to/scraped by vmagent data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6928).
|
||||
|
||||
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): drop rows that do not belong to the current series during import. The dropped rows should belong to another series whose tags are a superset of the current series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7301) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7330). Thanks to @dpedu for reporting and cooperating with the test.
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): keep the order of resulting time series when `limit_offset` is applied. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7068).
|
||||
* BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): properly handle xFilesFactor=0 for `transformRemoveEmptySeries` function. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7337) for details.
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly check availability of all the backends before giving up when proxying requests. Previously, vmauth could return an error even if there were healthy backends available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3061) for details.
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly inherit [`drop_src_path_prefix_parts`](https://docs.victoriametrics.com/vmauth/#dropping-request-path-prefix), [`load_balancing_policy`](https://docs.victoriametrics.com/vmauth/#high-availability), [`retry_status_codes`](https://docs.victoriametrics.com/vmauth/#load-balancing) and [`discover_backend_ips`](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) options by `url_map` entries if `url_prefix` option isn't set at the [user config level](https://docs.victoriametrics.com/vmauth/#auth-config). These options were inherited only when the `url_prefix` option was set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7519).
|
||||
* BUGFIX: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards): add `file` label filter to vmalert dashboard panels. Previously, metrics from groups with the same name but different rule files could be mixed in the results.
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) changed a meaning of `-maxLabelsPerTimeseries` and `-maxLabelValueLen`. Previously excessive labels, label names and values were truncated. With this change all series, which are hitting these limits will be dropped. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6928)
|
||||
|
||||
## [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0)
|
||||
|
||||
|
|
|
|||
|
|
@ -1893,6 +1893,10 @@ See the docs at https://docs.victoriametrics.com/vmagent/ .
|
|||
-maxInsertRequestSize size
|
||||
The maximum size in bytes of a single Prometheus remote_write API request
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
|
||||
-maxLabelValueLen int
|
||||
The maximum length of label values in the accepted time series. Series, with longer label values are dropped. In this case the `vm_series_dropped_total{reason="too_long_label_value"}` metric at /metrics page is incremented (default 4096).
|
||||
-maxLabelsPerTimeseries int
|
||||
The maximum number of labels accepted per time series. Series with superfluous labels are dropped. In this case the `vm_series_dropped_total{reason="too_many_labels"}` metric at /metrics page is incremented (default 30)
|
||||
-memory.allowedBytes size
|
||||
Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ package prompbmarshal
|
|||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Sample struct {
|
||||
|
|
@ -124,3 +126,27 @@ func (m *Label) Size() (n int) {
|
|||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// LabelsToString converts labels to Prometheus-compatible string
|
||||
func LabelsToString(labels []Label) string {
|
||||
labelsCopy := append([]Label{}, labels...)
|
||||
sort.Slice(labelsCopy, func(i, j int) bool {
|
||||
return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
|
||||
})
|
||||
var b []byte
|
||||
b = append(b, '{')
|
||||
for i, label := range labelsCopy {
|
||||
if len(label.Name) == 0 {
|
||||
b = append(b, "__name__"...)
|
||||
} else {
|
||||
b = append(b, label.Name...)
|
||||
}
|
||||
b = append(b, '=')
|
||||
b = strconv.AppendQuote(b, label.Value)
|
||||
if i < len(labels)-1 {
|
||||
b = append(b, ',')
|
||||
}
|
||||
}
|
||||
b = append(b, '}')
|
||||
return string(b)
|
||||
}
|
||||
|
|
|
|||
113
lib/storage/limits/limits.go
Normal file
113
lib/storage/limits/limits.go
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
package limits
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
)
|
||||
|
||||
const maxLabelNameLen = 256
|
||||
|
||||
var (
|
||||
maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels per time series to be accepted. Timeseries with superfluous labels are dropped. In this case the vm_series_dropped_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
|
||||
maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Metrics with longer label value are dropped. In this case the vm_series_dropped_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
|
||||
)
|
||||
|
||||
var (
|
||||
droppedSeriesWithTooManyLabelsLogTicker = time.NewTicker(5 * time.Second)
|
||||
droppedSeriesWithTooLongLabelNameLogTicker = time.NewTicker(5 * time.Second)
|
||||
droppedSeriesWithTooLongLabelValueLogTicker = time.NewTicker(5 * time.Second)
|
||||
)
|
||||
|
||||
var (
|
||||
// droppedSeriesWithTooManyLabels is the number of dropped series with too many labels
|
||||
droppedSeriesWithTooManyLabels atomic.Uint64
|
||||
|
||||
// droppedSeriesWithTooLongLabelName is the number of dropped series which contain labels with too long names
|
||||
droppedSeriesWithTooLongLabelName atomic.Uint64
|
||||
|
||||
// droppedSeriesWithTooLongLabelValue is the number of dropped series which contain labels with too long values
|
||||
droppedSeriesWithTooLongLabelValue atomic.Uint64
|
||||
)
|
||||
|
||||
var (
|
||||
_ = metrics.NewGauge(`vm_series_dropped_total{reason="too_many_labels"}`, func() float64 {
|
||||
return float64(droppedSeriesWithTooManyLabels.Load())
|
||||
})
|
||||
_ = metrics.NewGauge(`vm_series_dropped_total{reason="too_long_label_name"}`, func() float64 {
|
||||
return float64(droppedSeriesWithTooLongLabelName.Load())
|
||||
})
|
||||
_ = metrics.NewGauge(`vm_series_dropped_total{reason="too_long_label_value"}`, func() float64 {
|
||||
return float64(droppedSeriesWithTooLongLabelValue.Load())
|
||||
})
|
||||
)
|
||||
|
||||
func trackDroppedSeriesWithTooManyLabels(labels []prompbmarshal.Label) {
|
||||
droppedSeriesWithTooManyLabels.Add(1)
|
||||
select {
|
||||
case <-droppedSeriesWithTooManyLabelsLogTicker.C:
|
||||
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
|
||||
// because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooManyLabels call.
|
||||
logger.Warnf("dropping series with %d labels for %s; either reduce the number of labels for this metric "+
|
||||
"or increase -maxLabelsPerTimeseries=%d command-line flag value",
|
||||
len(labels), prompbmarshal.LabelsToString(labels), *maxLabelsPerTimeseries)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func trackDroppedSeriesWithTooLongLabelValue(l *prompbmarshal.Label, labels []prompbmarshal.Label) {
|
||||
droppedSeriesWithTooLongLabelValue.Add(1)
|
||||
select {
|
||||
case <-droppedSeriesWithTooLongLabelValueLogTicker.C:
|
||||
label := *l
|
||||
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
|
||||
// because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooLongLabelValue call.
|
||||
logger.Warnf("drop series with a value %s for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+
|
||||
"original labels: %s; either reduce the label value length or increase -maxLabelValueLen command-line flag value",
|
||||
label.Value, label.Name, len(label.Value), *maxLabelValueLen, prompbmarshal.LabelsToString(labels))
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func trackDroppedSeriesWithTooLongLabelName(l *prompbmarshal.Label, labels []prompbmarshal.Label) {
|
||||
droppedSeriesWithTooLongLabelName.Add(1)
|
||||
select {
|
||||
case <-droppedSeriesWithTooLongLabelNameLogTicker.C:
|
||||
label := *l
|
||||
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
|
||||
// because prompbmarshal.LabelsToString() will be called with each trackDroppedSeriesWithTooLongLabelName call.
|
||||
logger.Warnf("drop series with a value for label %s because its length=%d exceeds %d; "+
|
||||
"original labels: %s; consider reducing the label name length",
|
||||
label.Name, len(label.Name), maxLabelNameLen, prompbmarshal.LabelsToString(labels))
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// ExceedingLabels checks if passed labels exceed one of the limits:
|
||||
// * Maximum allowed labels limit
|
||||
// * Maximum allowed label name length limit
|
||||
// * Maximum allowed label value length limit
|
||||
//
|
||||
// increments metrics and shows warning in logs
|
||||
func ExceedingLabels(labels []prompbmarshal.Label) bool {
|
||||
if len(labels) > *maxLabelsPerTimeseries {
|
||||
trackDroppedSeriesWithTooManyLabels(labels)
|
||||
return true
|
||||
}
|
||||
for _, l := range labels {
|
||||
if len(l.Name) > maxLabelNameLen {
|
||||
trackDroppedSeriesWithTooLongLabelName(&l, labels)
|
||||
return true
|
||||
}
|
||||
if len(l.Value) > *maxLabelValueLen {
|
||||
trackDroppedSeriesWithTooLongLabelValue(&l, labels)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -5,16 +5,12 @@ import (
|
|||
"fmt"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
)
|
||||
|
||||
|
|
@ -467,63 +463,15 @@ func (mn *MetricName) Unmarshal(src []byte) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// The maximum length of label name.
|
||||
//
|
||||
// Longer names are truncated.
|
||||
const maxLabelNameLen = 256
|
||||
|
||||
// The maximum length of label value.
|
||||
//
|
||||
// Longer values are truncated.
|
||||
var maxLabelValueLen = 1024
|
||||
|
||||
// SetMaxLabelValueLen sets the limit on the label value length.
|
||||
//
|
||||
// This function can be called before using the storage package.
|
||||
//
|
||||
// Label values with longer length are truncated.
|
||||
func SetMaxLabelValueLen(n int) {
|
||||
if n > 0 {
|
||||
maxLabelValueLen = n
|
||||
}
|
||||
}
|
||||
|
||||
// The maximum number of labels per each timeseries.
|
||||
var maxLabelsPerTimeseries = 30
|
||||
|
||||
// SetMaxLabelsPerTimeseries sets the limit on the number of labels
|
||||
// per each time series.
|
||||
//
|
||||
// This function can be called before using the storage package.
|
||||
//
|
||||
// Superfluous labels are dropped.
|
||||
func SetMaxLabelsPerTimeseries(maxLabels int) {
|
||||
if maxLabels > 0 {
|
||||
maxLabelsPerTimeseries = maxLabels
|
||||
}
|
||||
}
|
||||
|
||||
// MarshalMetricNameRaw marshals labels to dst and returns the result.
|
||||
//
|
||||
// The result must be unmarshaled with MetricName.UnmarshalRaw
|
||||
func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
|
||||
func MarshalMetricNameRaw(dst []byte, labels []prompbmarshal.Label) []byte {
|
||||
// Calculate the required space for dst.
|
||||
dstLen := len(dst)
|
||||
dstSize := dstLen
|
||||
for i := range labels {
|
||||
if i >= maxLabelsPerTimeseries {
|
||||
trackDroppedLabels(labels, labels[i:])
|
||||
break
|
||||
}
|
||||
label := &labels[i]
|
||||
if len(label.Name) > maxLabelNameLen {
|
||||
TooLongLabelNames.Add(1)
|
||||
label.Name = label.Name[:maxLabelNameLen]
|
||||
}
|
||||
if len(label.Value) > maxLabelValueLen {
|
||||
trackTruncatedLabels(labels, label)
|
||||
label.Value = label.Value[:maxLabelValueLen]
|
||||
}
|
||||
if len(label.Value) == 0 {
|
||||
// Skip labels without values, since they have no sense in prometheus.
|
||||
continue
|
||||
|
|
@ -539,9 +487,6 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
|
|||
|
||||
// Marshal labels to dst.
|
||||
for i := range labels {
|
||||
if i >= maxLabelsPerTimeseries {
|
||||
break
|
||||
}
|
||||
label := &labels[i]
|
||||
if len(label.Value) == 0 {
|
||||
// Skip labels without values, since they have no sense in prometheus.
|
||||
|
|
@ -553,71 +498,6 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
|
|||
return dst
|
||||
}
|
||||
|
||||
var (
|
||||
// MetricsWithDroppedLabels is the number of metrics with at least a single dropped label
|
||||
MetricsWithDroppedLabels atomic.Uint64
|
||||
|
||||
// TooLongLabelNames is the number of too long label names
|
||||
TooLongLabelNames atomic.Uint64
|
||||
|
||||
// TooLongLabelValues is the number of too long label values
|
||||
TooLongLabelValues atomic.Uint64
|
||||
)
|
||||
|
||||
func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
|
||||
MetricsWithDroppedLabels.Add(1)
|
||||
select {
|
||||
case <-droppedLabelsLogTicker.C:
|
||||
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
|
||||
// because labelsToString() will be called with each trackDroppedLabels call.
|
||||
logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+
|
||||
"or increase -maxLabelsPerTimeseries=%d command-line flag value",
|
||||
len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) {
|
||||
TooLongLabelValues.Add(1)
|
||||
select {
|
||||
case <-truncatedLabelsLogTicker.C:
|
||||
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
|
||||
// because labelsToString() will be called with each trackTruncatedLabels call.
|
||||
logger.Warnf("truncate value for label %s because its length=%d exceeds -maxLabelValueLen=%d; "+
|
||||
"original labels: %s; either reduce the label value length or increase -maxLabelValueLen command-line flag value",
|
||||
truncated.Name, len(truncated.Value), maxLabelValueLen, labelsToString(labels))
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
droppedLabelsLogTicker = time.NewTicker(5 * time.Second)
|
||||
truncatedLabelsLogTicker = time.NewTicker(5 * time.Second)
|
||||
)
|
||||
|
||||
func labelsToString(labels []prompb.Label) string {
|
||||
labelsCopy := append([]prompb.Label{}, labels...)
|
||||
sort.Slice(labelsCopy, func(i, j int) bool {
|
||||
return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
|
||||
})
|
||||
var b []byte
|
||||
b = append(b, '{')
|
||||
for i, label := range labelsCopy {
|
||||
if len(label.Name) == 0 {
|
||||
b = append(b, "__name__"...)
|
||||
} else {
|
||||
b = append(b, label.Name...)
|
||||
}
|
||||
b = append(b, '=')
|
||||
b = strconv.AppendQuote(b, string(label.Value))
|
||||
if i < len(labels)-1 {
|
||||
b = append(b, ',')
|
||||
}
|
||||
}
|
||||
b = append(b, '}')
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// marshalRaw marshals mn to dst and returns the result.
|
||||
//
|
||||
// The results may be unmarshaled with MetricName.UnmarshalRaw.
|
||||
|
|
|
|||
Loading…
Reference in a new issue