mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-02-09 15:27:11 +00:00
app/vmselect/prometheus: improve performance for /api/v1/labels
and /api/v1/label/<labelName>/values
on time ranges exceeding one day when match[]
query arg is set
This commit is contained in:
parent
6fa806f1ca
commit
27a417bcd3
2 changed files with 60 additions and 37 deletions
|
@ -601,12 +601,25 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
|
||||||
MaxTimestamp: end,
|
MaxTimestamp: end,
|
||||||
TagFilterss: tagFilterss,
|
TagFilterss: tagFilterss,
|
||||||
}
|
}
|
||||||
|
m := make(map[string]struct{})
|
||||||
|
if end-start > 24*3600*1000 {
|
||||||
|
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||||
|
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||||
|
}
|
||||||
|
for _, mn := range mns {
|
||||||
|
labelValue := mn.GetTagValue(labelName)
|
||||||
|
if len(labelValue) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m[string(labelValue)] = struct{}{}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m := make(map[string]struct{})
|
|
||||||
var mLock sync.Mutex
|
var mLock sync.Mutex
|
||||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||||
labelValue := rs.MetricName.GetTagValue(labelName)
|
labelValue := rs.MetricName.GetTagValue(labelName)
|
||||||
|
@ -621,7 +634,7 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error when data fetching: %w", err)
|
return nil, fmt.Errorf("error when data fetching: %w", err)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
labelValues := make([]string, 0, len(m))
|
labelValues := make([]string, 0, len(m))
|
||||||
for labelValue := range m {
|
for labelValue := range m {
|
||||||
labelValues = append(labelValues, labelValue)
|
labelValues = append(labelValues, labelValue)
|
||||||
|
@ -787,19 +800,28 @@ func labelsWithMatches(matches []string, start, end int64, deadline searchutils.
|
||||||
MaxTimestamp: end,
|
MaxTimestamp: end,
|
||||||
TagFilterss: tagFilterss,
|
TagFilterss: tagFilterss,
|
||||||
}
|
}
|
||||||
|
m := make(map[string]struct{})
|
||||||
|
if end-start > 24*3600*1000 {
|
||||||
|
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||||
|
mns, err := netstorage.SearchMetricNames(sq, deadline)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||||
|
}
|
||||||
|
for _, mn := range mns {
|
||||||
|
for _, tag := range mn.Tags {
|
||||||
|
m[string(tag.Key)] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m := make(map[string]struct{})
|
|
||||||
var mLock sync.Mutex
|
var mLock sync.Mutex
|
||||||
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
|
||||||
mLock.Lock()
|
mLock.Lock()
|
||||||
tags := rs.MetricName.Tags
|
for _, tag := range rs.MetricName.Tags {
|
||||||
for i := range tags {
|
m[string(tag.Key)] = struct{}{}
|
||||||
t := &tags[i]
|
|
||||||
m[string(t.Key)] = struct{}{}
|
|
||||||
}
|
}
|
||||||
m["__name__"] = struct{}{}
|
m["__name__"] = struct{}{}
|
||||||
mLock.Unlock()
|
mLock.Unlock()
|
||||||
|
@ -808,6 +830,7 @@ func labelsWithMatches(matches []string, start, end int64, deadline searchutils.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error when data fetching: %w", err)
|
return nil, fmt.Errorf("error when data fetching: %w", err)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
labels := make([]string, 0, len(m))
|
labels := make([]string, 0, len(m))
|
||||||
for label := range m {
|
for label := range m {
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
* FEATURE: vmselect: add `"isPartial":{true|false}` field in JSON output for `/api/v1/*` functions
|
* FEATURE: vmselect: add `"isPartial":{true|false}` field in JSON output for `/api/v1/*` functions
|
||||||
from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/). `"isPartial":true` is set if the response contains partial data
|
from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/). `"isPartial":true` is set if the response contains partial data
|
||||||
because of a part of `vmstorage` nodes were unavailable during query processing.
|
because of a part of `vmstorage` nodes were unavailable during query processing.
|
||||||
* FEATURE: improve performance for `/api/v1/series` on time ranges exceeding one day.
|
* FEATURE: improve performance for `/api/v1/series`, `/api/v1/labels` and `/api/v1/label/<labelName>/values` on time ranges exceeding one day.
|
||||||
* FEATURE: vmagent: reduce memory usage when service discovery detects big number of scrape targets and the set of discovered targets changes over time.
|
* FEATURE: vmagent: reduce memory usage when service discovery detects big number of scrape targets and the set of discovered targets changes over time.
|
||||||
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
|
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
|
||||||
* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.
|
* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.
|
||||||
|
|
Loading…
Reference in a new issue