github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: postpone label filters matching too many time series instead of giving up with error

Browse source 
This should reduce the frequency of the following errors:

    cannot find tag filter matching less than N time series; either increase -search.maxUniqueTimeseries or use more specific tag filters

    more than N time series found on the time range [...]; either increase -search.maxUniqueTimeseries or shrink the time range
This commit is contained in:
Aliaksandr Valialkin 2020-04-24 21:11:46 +03:00
parent 9b386e594f
commit 13b4069c59
1 changed files with 50 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
lib/storage/index_db.go
View file

@ -2425,19 +2425,36 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
})
// Populate metricIDs with the first non-negative filter.
var tfFirst *tagFilter
var tfsPostponed []*tagFilter
var metricIDs *uint64set.Set
maxDateMetrics := maxMetrics * 50
tfsRemainingWithCount := tfsWithCount[:0]
for i := range tfsWithCount {
tf := tfsWithCount[i].tf
if tf.isNegative {
tfsRemainingWithCount = append(tfsRemainingWithCount, tfsWithCount[i])
continue
}
tfFirst = tf
m, err := is.getMetricIDsForDateTagFilter(tf, date, tfs.commonPrefix, tfs.accountID, tfs.projectID, maxDateMetrics)
if err != nil {
return nil, err
}
if m.Len() >= maxDateMetrics {
// Too many time series found by a single tag filter. Postpone applying this filter via metricName match.
tfsPostponed = append(tfsPostponed, tf)
continue
}
metricIDs = m
i++
for i < len(tfsWithCount) {
tfsRemainingWithCount = append(tfsRemainingWithCount, tfsWithCount[i])
i++
}
break
}
var metricIDs *uint64set.Set
maxDateMetrics := maxMetrics * 50
if tfFirst == nil {
// All the filters in tfs are negative. Populate all the metricIDs for the given (date, accountID, projectID),
if metricIDs == nil {
// All the filters in tfs are negative or match too many time series.
// Populate all the metricIDs for the given (date),
// so later they can be filtered out with negative filters.
m, err := is.getMetricIDsForDate(date, tfs.accountID, tfs.projectID, maxDateMetrics)
if err != nil {
@ -2449,52 +2466,34 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
}
return nil, fmt.Errorf("cannot obtain all the metricIDs: %s", err)
}
metricIDs = m
} else {
// Populate metricIDs for the given tfFirst on the given (date, accountId, projectID)
m, err := is.getMetricIDsForDateTagFilter(tfFirst, date, tfs.commonPrefix, tfs.accountID, tfs.projectID, maxDateMetrics)
if err != nil {
return nil, err
if m.Len() >= maxDateMetrics {
// Too many time series found for the given (date). Fall back to global search.
return nil, errFallbackToMetricNameMatch
}
metricIDs = m
}
if metricIDs.Len() >= maxDateMetrics {
// Too many time series found by a single tag filter. Fall back to global search.
return nil, errFallbackToMetricNameMatch
}
// Intersect metricIDs with the rest of filters.
for i := range tfsWithCount {
tfWithCount := &tfsWithCount[i]
tf := tfWithCount.tf
if tf == tfFirst {
continue
}
if n := uint64(metricIDs.Len()); n < 1000 || n < tfWithCount.count/maxIndexScanLoopsPerMetric {
for i := range tfsRemainingWithCount {
tfWithCount := tfsRemainingWithCount[i]
if n := uint64(metricIDs.Len()); n < 1000 || (n < tfWithCount.count/maxIndexScanLoopsPerMetric && n < uint64(maxMetrics)/10) {
// It should be faster performing metricName match on the remaining filters
// instead of scanning big number of entries in the inverted index for these filters.
tfsRemaining := tfsWithCount[i:]
tfsPostponed := make([]*tagFilter, 0, len(tfsRemaining))
for j := range tfsRemaining {
tf := tfsRemaining[j].tf
if tf == tfFirst {
continue
}
tfsPostponed = append(tfsPostponed, tf)
for i < len(tfsRemainingWithCount) {
tfsPostponed = append(tfsPostponed, tfsRemainingWithCount[i].tf)
i++
}
var m uint64set.Set
if err := is.updateMetricIDsByMetricNameMatch(&m, metricIDs, tfsPostponed, tfs.accountID, tfs.projectID); err != nil {
return nil, err
}
return &m, nil
break
}
tf := tfWithCount.tf
m, err := is.getMetricIDsForDateTagFilter(tf, date, tfs.commonPrefix, tfs.accountID, tfs.projectID, maxDateMetrics)
if err != nil {
return nil, err
}
if m.Len() >= maxDateMetrics {
// Too many time series found by a single tag filter. Fall back to global search.
return nil, errFallbackToMetricNameMatch
// Too many time series found by a single tag filter. Postpone applying this filter via metricName match.
tfsPostponed = append(tfsPostponed, tf)
continue
}
if tf.isNegative {
metricIDs.Subtract(m)
@ -2506,6 +2505,19 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
return nil, nil
}
}
if len(tfsPostponed) > 0 {
if n := metricIDs.Len(); n > 50000 && n > maxMetrics/10 {
// It will be slow to perform metricName match on this number of time series.
// Fall back to global search.
return nil, errFallbackToMetricNameMatch
}
// Apply the postponed filters via metricName match.
var m uint64set.Set
if err := is.updateMetricIDsByMetricNameMatch(&m, metricIDs, tfsPostponed, tfs.accountID, tfs.projectID); err != nil {
return nil, err
}
return &m, nil
}
return metricIDs, nil
}