github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: log dropped labels if the number of labels in a metric exceeds -maxLabelsPerTimeseries command-line flag value

Browse source 
This should improve debuggability for this case.
This commit is contained in:
Aliaksandr Valialkin 2021-05-01 09:27:53 +03:00
parent 0a2e746175
commit b43ba6d85f
2 changed files with 45 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/CHANGELOG.md
View file

@ -12,6 +12,7 @@ Thanks to @johnseekins!
* FEATURE: improved new time series registration speed on systems with many CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1244). Thanks to @waldoweng for the idea and [draft implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1243).
* FEATURE: vmagent: list user-visible endpoints at `http://vmagent:8429/`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1251).
* FEATURE: vmalert: use the same technique as Grafana for determining evaluation timestamps for recording rules. This should make consistent graphs for series generated by recording rules compared to graphs generated for queries from recording rules in Grafana. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232).
* FEATURE: log metrics with dropped labels if the number of labels in the ingested metric exceeds `-maxLabelsPerTimeseries`. This should simplify debugging for this case.
* BUGFIX: vmagent: properly update `role: endpoints` and `role: endpointslices` scrape targets if the underlying service changes in `kubernetes_sd_config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240).
* BUGFIX: vmagent: apply `scrape_timeout` on receiving the first response byte from `stream_parse: true` scrape targets. Previously it was applied to receiving and *processing* the full response stream. This could result in false timeout errors when scrape target exposes millions of metrics as described [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047).

45
lib/storage/metric_name.go
View file

@ -4,12 +4,14 @@ import (
"bytes"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
@ -509,7 +511,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
dstSize := dstLen + 8
for i := range labels {
if i >= maxLabelsPerTimeseries {
atomic.AddUint64(&MetricsWithDroppedLabels, 1)
trackDroppedLabels(labels, labels[i:])
break
}
label := &labels[i]
@ -563,6 +565,47 @@ var (
TooLongLabelValues uint64
)
func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
atomic.AddUint64(&MetricsWithDroppedLabels, 1)
ct := fasttime.UnixTimestamp()
if ct < atomic.LoadUint64(&droppedLabelsLogNextTimestamp) {
return
}
droppedLabelsLogOnce.Do(func() {
atomic.StoreUint64(&droppedLabelsLogNextTimestamp, ct+5)
logger.Infof("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+
"or increase -maxLabelsPerTimeseries=%d command-line flag value",
len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries)
droppedLabelsLogOnce = &sync.Once{}
})
}
var droppedLabelsLogOnce = &sync.Once{}
var droppedLabelsLogNextTimestamp uint64
func labelsToString(labels []prompb.Label) string {
labelsCopy := append([]prompb.Label{}, labels...)
sort.Slice(labelsCopy, func(i, j int) bool {
return string(labelsCopy[i].Name) < string(labelsCopy[j].Name)
})
var b []byte
b = append(b, '{')
for i, label := range labelsCopy {
if len(label.Name) == 0 {
b = append(b, "__name__"...)
} else {
b = append(b, label.Name...)
}
b = append(b, '=')
b = strconv.AppendQuote(b, string(label.Value))
if i < len(labels)-1 {
b = append(b, ',')
}
}
b = append(b, '}')
return string(b)
}
// MarshalMetricLabelRaw marshals label to dst.
func MarshalMetricLabelRaw(dst []byte, label *prompb.Label) []byte {
dst = marshalBytesFast(dst, label.Name)