github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-01 14:47:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: remember and skip individual tag filters matching too many metrics

Browse source 
This saves CPU time by skipping useless matching for individual tag filters.
This commit is contained in:
Aliaksandr Valialkin 2019-07-11 14:37:27 +03:00
parent a336bb4e22
commit 43ea4ce428
1 changed files with 53 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
lib/storage/index_db.go
View file

@ -1221,8 +1221,9 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs
func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) {
maxMetrics = is.adjustMaxMetricsAdaptive(maxMetrics) maxMetrics = is.adjustMaxMetricsAdaptive(maxMetrics)
kb := &is.kb kb := &is.kb
kb.B = tfs.marshal(kb.B[:0]) kb.B = append(kb.B[:0], uselessMultiTagFiltersKeyPrefix)
kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics)) kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics))
kb.B = tfs.marshal(kb.B)
if len(is.db.uselessTagFiltersCache.Get(nil, kb.B)) > 0 { if len(is.db.uselessTagFiltersCache.Get(nil, kb.B)) > 0 {
// Skip useless work below, since the tfs doesn't contain tag filters matching less than maxMetrics metrics. // Skip useless work below, since the tfs doesn't contain tag filters matching less than maxMetrics metrics.
return nil, nil, errTooManyMetrics return nil, nil, errTooManyMetrics
@ -1236,6 +1237,7 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters
} }
for { for {
minTf, minMetricIDs, err := is.getTagFilterWithMinMetricIDsCount(tfs, maxAllowedMetrics) minTf, minMetricIDs, err := is.getTagFilterWithMinMetricIDsCount(tfs, maxAllowedMetrics)
if err != errTooManyMetrics {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -1243,13 +1245,15 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters
// Found the tag filter with the minimum number of metrics. // Found the tag filter with the minimum number of metrics.
return minTf, minMetricIDs, nil return minTf, minMetricIDs, nil
} }
}
// Too many metrics matched. // Too many metrics matched.
if maxAllowedMetrics >= maxMetrics { if maxAllowedMetrics >= maxMetrics {
// The tag filter with minimum matching metrics matches at least maxMetrics metrics. // The tag filter with minimum matching metrics matches at least maxMetrics metrics.
kb.B = tfs.marshal(kb.B[:0]) kb.B = append(kb.B[:0], uselessMultiTagFiltersKeyPrefix)
kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics)) kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics))
is.db.uselessTagFiltersCache.Set(kb.B, []byte("1")) kb.B = tfs.marshal(kb.B)
is.db.uselessTagFiltersCache.Set(kb.B, uselessTagFilterCacheValue)
return nil, nil, errTooManyMetrics return nil, nil, errTooManyMetrics
} }
@ -1280,12 +1284,24 @@ func (is *indexSearch) adjustMaxMetricsAdaptive(maxMetrics int) int {
func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) {
var minMetricIDs map[uint64]struct{} var minMetricIDs map[uint64]struct{}
var minTf *tagFilter var minTf *tagFilter
kb := &is.kb
uselessTagFilters := 0
for i := range tfs.tfs { for i := range tfs.tfs {
tf := &tfs.tfs[i] tf := &tfs.tfs[i]
if tf.isNegative { if tf.isNegative {
// Skip negative filters. // Skip negative filters.
continue continue
} }
kb.B = append(kb.B[:0], uselessSingleTagFilterKeyPrefix)
kb.B = encoding.MarshalUint64(kb.B[:0], uint64(maxMetrics))
kb.B = tf.Marshal(kb.B)
if len(is.db.uselessTagFiltersCache.Get(nil, kb.B)) > 0 {
// Skip useless work below, since the tf matches at least maxMetrics metrics.
uselessTagFilters++
continue
}
metricIDs, err := is.getMetricIDsForTagFilter(tf, maxMetrics) metricIDs, err := is.getMetricIDsForTagFilter(tf, maxMetrics)
if err != nil { if err != nil {
if err == errFallbackToMetricNameMatch { if err == errFallbackToMetricNameMatch {
@ -1294,7 +1310,16 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMet
} }
return nil, nil, fmt.Errorf("cannot find MetricIDs for tagFilter %s: %s", tf, err) return nil, nil, fmt.Errorf("cannot find MetricIDs for tagFilter %s: %s", tf, err)
} }
if minTf == nil || len(metricIDs) < len(minMetricIDs) { if len(metricIDs) >= maxMetrics {
// The tf matches at least maxMetrics. Skip it
kb.B = append(kb.B[:0], uselessSingleTagFilterKeyPrefix)
kb.B = encoding.MarshalUint64(kb.B[:0], uint64(maxMetrics))
kb.B = tf.Marshal(kb.B)
is.db.uselessTagFiltersCache.Set(kb.B, uselessTagFilterCacheValue)
uselessTagFilters++
continue
}
minMetricIDs = metricIDs minMetricIDs = metricIDs
minTf = tf minTf = tf
maxMetrics = len(minMetricIDs) maxMetrics = len(minMetricIDs)
@ -1304,10 +1329,13 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMet
break break
} }
} }
}
if minTf != nil { if minTf != nil {
return minTf, minMetricIDs, nil return minTf, minMetricIDs, nil
} }
if uselessTagFilters == len(tfs.tfs) {
// All the tag filters return at least maxMetrics entries.
return nil, nil, errTooManyMetrics
}
// There is no positive filter with small number of matching metrics. // There is no positive filter with small number of matching metrics.
// Create it, so it matches all the MetricIDs for tfs.commonPrefix. // Create it, so it matches all the MetricIDs for tfs.commonPrefix.
@ -1494,6 +1522,13 @@ func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs map[uint64]struct{
return nil return nil
} }
const (
uselessSingleTagFilterKeyPrefix = 0
uselessMultiTagFiltersKeyPrefix = 1
)
var uselessTagFilterCacheValue = []byte("1")
func (is *indexSearch) getMetricIDsForTagFilter(tf *tagFilter, maxMetrics int) (map[uint64]struct{}, error) { func (is *indexSearch) getMetricIDsForTagFilter(tf *tagFilter, maxMetrics int) (map[uint64]struct{}, error) {
if tf.isNegative { if tf.isNegative {
logger.Panicf("BUG: isNegative must be false") logger.Panicf("BUG: isNegative must be false")