lib/storage: postpone label filters matching too many time series instead of giving up with error

This should reduce the frequency of the following errors:

    cannot find tag filter matching less than N time series; either increase -search.maxUniqueTimeseries or use more specific tag filters

    more than N time series found on the time range [...]; either increase -search.maxUniqueTimeseries or shrink the time range

lib/storage/index_db.go

@@ -2398,19 +2398,36 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
 	})
 
 	// Populate metricIDs with the first non-negative filter.
-	var tfFirst *tagFilter
+	var tfsPostponed []*tagFilter
+	var metricIDs *uint64set.Set
+	maxDateMetrics := maxMetrics * 50
+	tfsRemainingWithCount := tfsWithCount[:0]
 	for i := range tfsWithCount {
 		tf := tfsWithCount[i].tf
 		if tf.isNegative {
+			tfsRemainingWithCount = append(tfsRemainingWithCount, tfsWithCount[i])
 			continue
 		}
-		tfFirst = tf
+		m, err := is.getMetricIDsForDateTagFilter(tf, date, tfs.commonPrefix, maxDateMetrics)
+		if err != nil {
+			return nil, err
+		}
+		if m.Len() >= maxDateMetrics {
+			// Too many time series found by a single tag filter. Postpone applying this filter via metricName match.
+			tfsPostponed = append(tfsPostponed, tf)
+			continue
+		}
+		metricIDs = m
+		i++
+		for i < len(tfsWithCount) {
+			tfsRemainingWithCount = append(tfsRemainingWithCount, tfsWithCount[i])
+			i++
+		}
 		break
 	}
-	var metricIDs *uint64set.Set
+	if metricIDs == nil {
-	maxDateMetrics := maxMetrics * 50
+		// All the filters in tfs are negative or match too many time series.
-	if tfFirst == nil {
+		// Populate all the metricIDs for the given (date),
-		// All the filters in tfs are negative. Populate all the metricIDs for the given (date),
 		// so later they can be filtered out with negative filters.
 		m, err := is.getMetricIDsForDate(date, maxDateMetrics)
 		if err != nil {
@@ -2422,52 +2439,34 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
 			}
 			return nil, fmt.Errorf("cannot obtain all the metricIDs: %s", err)
 		}
-		metricIDs = m
+		if m.Len() >= maxDateMetrics {
-	} else {
+			// Too many time series found for the given (date). Fall back to global search.
-		// Populate metricIDs for the given tfFirst on the given (date)
-		m, err := is.getMetricIDsForDateTagFilter(tfFirst, date, tfs.commonPrefix, maxDateMetrics)
-		if err != nil {
-			return nil, err
-		}
-		metricIDs = m
-	}
-	if metricIDs.Len() >= maxDateMetrics {
-		// Too many time series found by a single tag filter. Fall back to global search.
 			return nil, errFallbackToMetricNameMatch
 		}
+		metricIDs = m
+	}
 
 	// Intersect metricIDs with the rest of filters.
-	for i := range tfsWithCount {
+	for i := range tfsRemainingWithCount {
-		tfWithCount := &tfsWithCount[i]
+		tfWithCount := tfsRemainingWithCount[i]
-		tf := tfWithCount.tf
+		if n := uint64(metricIDs.Len()); n < 1000 || (n < tfWithCount.count/maxIndexScanLoopsPerMetric && n < uint64(maxMetrics)/10) {
-		if tf == tfFirst {
-			continue
-		}
-		if n := uint64(metricIDs.Len()); n < 1000 || n < tfWithCount.count/maxIndexScanLoopsPerMetric {
 			// It should be faster performing metricName match on the remaining filters
 			// instead of scanning big number of entries in the inverted index for these filters.
-			tfsRemaining := tfsWithCount[i:]
+			for i < len(tfsRemainingWithCount) {
-			tfsPostponed := make([]*tagFilter, 0, len(tfsRemaining))
+				tfsPostponed = append(tfsPostponed, tfsRemainingWithCount[i].tf)
-			for j := range tfsRemaining {
+				i++
-				tf := tfsRemaining[j].tf
-				if tf == tfFirst {
-					continue
 			}
-				tfsPostponed = append(tfsPostponed, tf)
+			break
-			}
-			var m uint64set.Set
-			if err := is.updateMetricIDsByMetricNameMatch(&m, metricIDs, tfsPostponed); err != nil {
-				return nil, err
-			}
-			return &m, nil
 		}
+		tf := tfWithCount.tf
 		m, err := is.getMetricIDsForDateTagFilter(tf, date, tfs.commonPrefix, maxDateMetrics)
 		if err != nil {
 			return nil, err
 		}
 		if m.Len() >= maxDateMetrics {
-			// Too many time series found by a single tag filter. Fall back to global search.
+			// Too many time series found by a single tag filter. Postpone applying this filter via metricName match.
-			return nil, errFallbackToMetricNameMatch
+			tfsPostponed = append(tfsPostponed, tf)
+			continue
 		}
 		if tf.isNegative {
 			metricIDs.Subtract(m)
@@ -2479,6 +2478,19 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
 			return nil, nil
 		}
 	}
+	if len(tfsPostponed) > 0 {
+		if n := metricIDs.Len(); n > 50000 && n > maxMetrics/10 {
+			// It will be slow to perform metricName match on this number of time series.
+			// Fall back to global search.
+			return nil, errFallbackToMetricNameMatch
+		}
+		// Apply the postponed filters via metricName match.
+		var m uint64set.Set
+		if err := is.updateMetricIDsByMetricNameMatch(&m, metricIDs, tfsPostponed); err != nil {
+			return nil, err
+		}
+		return &m, nil
+	}
 	return metricIDs, nil
 }