github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: log dropped labels if the number of labels in a metric exceeds -maxLabelsPerTimeseries command-line flag value

Browse source 
This should improve debuggability for this case.
This commit is contained in:
Aliaksandr Valialkin 2021-05-01 09:27:53 +03:00
parent 508f500477
commit 58d1e6eeea
2 changed files with 45 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -12,6 +12,7 @@ Thanks to @johnseekins!
* FEATURE: improved new time series registration speed on systems with many CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1244). Thanks to @waldoweng for the idea and [draft implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1243).
* FEATURE: vmagent: list user-visible endpoints at `http://vmagent:8429/`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1251).
* FEATURE: vmalert: use the same technique as Grafana for determining evaluation timestamps for recording rules. This should make consistent graphs for series generated by recording rules compared to graphs generated for queries from recording rules in Grafana. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232).
* FEATURE: log metrics with dropped labels if the number of labels in the ingested metric exceeds `-maxLabelsPerTimeseries`. This should simplify debugging for this case.
* BUGFIX: vmagent: properly update `role: endpoints` and `role: endpointslices` scrape targets if the underlying service changes in `kubernetes_sd_config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240).
* BUGFIX: vmagent: apply `scrape_timeout` on receiving the first response byte from `stream_parse: true` scrape targets. Previously it was applied to receiving and *processing* the full response stream. This could result in false timeout errors when scrape target exposes millions of metrics as described [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047).

45
lib/storage/metric_name.go
View file

@ -4,12 +4,14 @@ import (
"bytes"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
@ -441,7 +443,7 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
dstSize := dstLen
for i := range labels {
if i >= maxLabelsPerTimeseries {
atomic.AddUint64(&MetricsWithDroppedLabels, 1)
trackDroppedLabels(labels, labels[i:])
break
}
label := &labels[i]
@ -493,6 +495,47 @@ var (
TooLongLabelValues uint64
)
func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
atomic.AddUint64(&MetricsWithDroppedLabels, 1)
ct := fasttime.UnixTimestamp()
if ct < atomic.LoadUint64(&droppedLabelsLogNextTimestamp) {
return
}
droppedLabelsLogOnce.Do(func() {
atomic.StoreUint64(&droppedLabelsLogNextTimestamp, ct+5)
logger.Infof("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+
"or increase -maxLabelsPerTimeseries=%d command-line flag value",
len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries)
droppedLabelsLogOnce = &sync.Once{}
})
}
var droppedLabelsLogOnce = &sync.Once{}
var droppedLabelsLogNextTimestamp uint64
func labelsToString(labels []prompb.Label) string {
labelsCopy := append([]prompb.Label{}, labels...)
sort.Slice(labelsCopy, func(i, j int) bool {
return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
})
var b []byte
b = append(b, '{')
for i, label := range labelsCopy {
if len(label.Name) == 0 {
b = append(b, "__name__"...)
} else {
b = append(b, label.Name...)
}
b = append(b, '=')
b = strconv.AppendQuote(b, string(label.Value))
if i < len(labels)-1 {
b = append(b, ',')
}
}
b = append(b, '}')
return string(b)
}
// marshalRaw marshals mn to dst and returns the result.
//
// The results may be unmarshaled with MetricName.unmarshalRaw.