github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-09 15:27:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: reduce too big maxMetrics in getTagFilterWithMinMetricIDsCountAdaptive

Browse source 
This should improve performance on inverted index search for big amount of unique time series
when big -search.maxUniqueTimeseries is set.
This commit is contained in:
Aliaksandr Valialkin 2019-06-25 19:55:21 +03:00
parent e8377011ab
commit eb2283a029
1 changed files with 19 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
lib/storage/index_db.go
View file

@ -1206,6 +1206,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs
} }
func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) {
maxMetrics = is.adjustMaxMetricsAdaptive(maxMetrics)
kb := &is.kb kb := &is.kb
kb.B = tfs.marshal(kb.B[:0]) kb.B = tfs.marshal(kb.B[:0])
kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics)) kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics))
@ -1249,6 +1250,23 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters
var errTooManyMetrics = errors.New("all the tag filters match too many metrics") var errTooManyMetrics = errors.New("all the tag filters match too many metrics")
func (is *indexSearch) adjustMaxMetricsAdaptive(maxMetrics int) int {
if is.db.prevHourMetricIDs == nil {
return maxMetrics
}
hmPrev := is.db.prevHourMetricIDs.Load().(*hourMetricIDs)
if hmPrev == nil || !hmPrev.isFull {
return maxMetrics
}
hourMetrics := len(hmPrev.m)
if hourMetrics >= 256 && maxMetrics > hourMetrics/4 {
// It is cheaper to filter on the hour or day metrics if the minimum
// number of matching metrics across tfs exceeds hourMetrics / 4.
return hourMetrics / 4
}
return maxMetrics
}
func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) {
var minMetricIDs map[uint64]struct{} var minMetricIDs map[uint64]struct{}
var minTf *tagFilter var minTf *tagFilter
@ -1417,7 +1435,7 @@ func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs map[uint64]struct{
maxTimeRangeMetrics := 20 * maxMetrics maxTimeRangeMetrics := 20 * maxMetrics
metricIDsForTimeRange, err := is.getMetricIDsForTimeRange(tr, maxTimeRangeMetrics+1) metricIDsForTimeRange, err := is.getMetricIDsForTimeRange(tr, maxTimeRangeMetrics+1)
if err == errMissingMetricIDsForDate { if err == errMissingMetricIDsForDate {
return fmt.Errorf("cannot find tag filter matching less up to %d time series; either increase -search.maxUniqueTimeseries or use more specific tag filters", return fmt.Errorf("cannot find tag filter matching less than %d time series; either increase -search.maxUniqueTimeseries or use more specific tag filters",
maxMetrics) maxMetrics)
} }
if err != nil { if err != nil {