From 8b0a63722f5ee1f6bf95be17605a8cc276fbf1be Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 25 Jun 2019 19:55:21 +0300 Subject: [PATCH] lib/storage: reduce too big maxMetrics in getTagFilterWithMinMetricIDsCountAdaptive This should improve performance on inverted index search for big amount of unique time series when big -search.maxUniqueTimeseries is set. --- lib/storage/index_db.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 32975b9a9..00bd1049d 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -1256,6 +1256,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(metricIDs, srcMetricIDs } func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { + maxMetrics = is.adjustMaxMetricsAdaptive(maxMetrics) kb := &is.kb kb.B = tfs.marshal(kb.B[:0]) kb.B = encoding.MarshalUint64(kb.B, uint64(maxMetrics)) @@ -1299,6 +1300,23 @@ func (is *indexSearch) getTagFilterWithMinMetricIDsCountAdaptive(tfs *TagFilters var errTooManyMetrics = errors.New("all the tag filters match too many metrics") +func (is *indexSearch) adjustMaxMetricsAdaptive(maxMetrics int) int { + if is.db.prevHourMetricIDs == nil { + return maxMetrics + } + hmPrev := is.db.prevHourMetricIDs.Load().(*hourMetricIDs) + if hmPrev == nil || !hmPrev.isFull { + return maxMetrics + } + hourMetrics := len(hmPrev.m) + if hourMetrics >= 256 && maxMetrics > hourMetrics/4 { + // It is cheaper to filter on the hour or day metrics if the minimum + // number of matching metrics across tfs exceeds hourMetrics / 4. + return hourMetrics / 4 + } + return maxMetrics +} + func (is *indexSearch) getTagFilterWithMinMetricIDsCount(tfs *TagFilters, maxMetrics int) (*tagFilter, map[uint64]struct{}, error) { var minMetricIDs map[uint64]struct{} var minTf *tagFilter @@ -1466,7 +1484,7 @@ func (is *indexSearch) updateMetricIDsForTagFilters(metricIDs map[uint64]struct{ maxTimeRangeMetrics := 20 * maxMetrics metricIDsForTimeRange, err := is.getMetricIDsForTimeRange(tr, maxTimeRangeMetrics+1, tfs.accountID, tfs.projectID) if err == errMissingMetricIDsForDate { - return fmt.Errorf("cannot find tag filter matching less up to %d time series; either increase -search.maxUniqueTimeseries or use more specific tag filters", + return fmt.Errorf("cannot find tag filter matching less than %d time series; either increase -search.maxUniqueTimeseries or use more specific tag filters", maxMetrics) } if err != nil {