From eb2283a029bef21664f0503b81ca4c891d056da3 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 25 Jun 2019 19:55:21 +0300 Subject: [PATCH] lib/storage: reduce too big maxMetrics in getTagFilterWithMinMetricIDsCountAdaptive This should improve performance on inverted index search for big amount of unique time series when big -search.maxUniqueTimeseries is set. --- lib/storage/index_db.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index f4d6f9d29..415ca76fd 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -1206,6 +1206,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs } func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { + maxMetrics = is.adjustMaxMetricsAdaptive(maxMetrics) kb := &is.kb kb.B = tfs.marshal(kb.B[:0]) kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics)) @@ -1249,6 +1250,23 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters var errTooManyMetrics = errors.New("all the tag filters match too many metrics") +func (is *indexSearch) adjustMaxMetricsAdaptive(maxMetrics int) int { + if is.db.prevHourMetricIDs == nil { + return maxMetrics + } + hmPrev := is.db.prevHourMetricIDs.Load().(*hourMetricIDs) + if hmPrev == nil || !hmPrev.isFull { + return maxMetrics + } + hourMetrics := len(hmPrev.m) + if hourMetrics >= 256 && maxMetrics > hourMetrics/4 { + // It is cheaper to filter on the hour or day metrics if the minimum + // number of matching metrics across tfs exceeds hourMetrics / 4. + return hourMetrics / 4 + } + return maxMetrics +} + func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { var minMetricIDs map[uint64]struct{} var minTf *tagFilter @@ -1417,7 +1435,7 @@ func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs map[uint64]struct{ maxTimeRangeMetrics := 20 * maxMetrics metricIDsForTimeRange, err := is.getMetricIDsForTimeRange(tr, maxTimeRangeMetrics+1) if err == errMissingMetricIDsForDate { - return fmt.Errorf("cannot find tag filter matching less up to %d time series; either increase -search.maxUniqueTimeseries or use more specific tag filters", + return fmt.Errorf("cannot find tag filter matching less than %d time series; either increase -search.maxUniqueTimeseries or use more specific tag filters", maxMetrics) } if err != nil {