lib/storage: properly handle {label=~"foo|"} filters as Prometheus does

Such filters must match all the time series with `label="foo"` plus all the time series without `label`

Previously only time series with `label="foo"` were matched.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/395

app/vmselect/netstorage/netstorage.go

@@ -576,6 +576,7 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 			}
 		}
 		tfss = append(tfss, tfs)
+		tfss = append(tfss, tfs.Finalize()...)
 	}
 	return tfss, nil
 }

lib/storage/tag_filters.go

@@ -33,6 +33,8 @@ func NewTagFilters() *TagFilters {
 // Add adds the given tag filter to tfs.
 //
 // MetricGroup must be encoded with nil key.
+//
+// Finalize must be called after tfs is constructed.
 func (tfs *TagFilters) Add(key, value []byte, isNegative, isRegexp bool) error {
 	// Verify whether tag filter is empty.
 	if len(value) == 0 {
@@ -66,6 +68,34 @@ func (tfs *TagFilters) Add(key, value []byte, isNegative, isRegexp bool) error {
 	return nil
 }
 
+// Finalize finalizes tfs and may return complementary TagFilters,
+// which must be added to the resulting set of tag filters.
+func (tfs *TagFilters) Finalize() []*TagFilters {
+	var tfssNew []*TagFilters
+	for i := range tfs.tfs {
+		tf := &tfs.tfs[i]
+		if tf.matchesEmptyValue {
+			// tf matches empty value, so it must be accompanied with `key!~".+"` tag filter
+			// in order to match time series without the given label.
+			tfssNew = append(tfssNew, tfs.cloneWithNegativeFilter(tf))
+		}
+	}
+	return tfssNew
+}
+
+func (tfs *TagFilters) cloneWithNegativeFilter(tfNegative *tagFilter) *TagFilters {
+	tfsNew := NewTagFilters()
+	for i := range tfs.tfs {
+		tf := &tfs.tfs[i]
+		if tf == tfNegative {
+			tfsNew.Add(tf.key, []byte(".+"), true, true)
+		} else {
+			tfsNew.Add(tf.key, tf.value, tf.isNegative, tf.isRegexp)
+		}
+	}
+	return tfsNew
+}
+
 // String returns human-readable value for tfs.
 func (tfs *TagFilters) String() string {
 	if len(tfs.tfs) == 0 {
@@ -102,7 +132,7 @@ type tagFilter struct {
 
 	// Prefix always contains {nsPrefixTagToMetricIDs, key}.
 	// Additionally it contains:
-	//  - value ending with tagSeparatorChar if !isRegexp.
+	//  - value if !isRegexp.
 	//  - non-regexp prefix if isRegexp.
 	prefix []byte
 
@@ -111,6 +141,11 @@ type tagFilter struct {
 
 	// Matches regexp suffix.
 	reSuffixMatch func(b []byte) bool
+
+	// Set to true for filter that matches empty value, i.e. "", "|foo" or ".*"
+	//
+	// Such a filter must be applied directly to metricNames.
+	matchesEmptyValue bool
 }
 
 // String returns human-readable tf value.
@@ -196,6 +231,9 @@ func (tf *tagFilter) Init(commonPrefix, key, value []byte, isNegative, isRegexp
 	}
 	tf.orSuffixes = append(tf.orSuffixes[:0], rcv.orValues...)
 	tf.reSuffixMatch = rcv.reMatch
+	if len(prefix) == 0 && !tf.isNegative && tf.reSuffixMatch(nil) {
+		tf.matchesEmptyValue = true
+	}
 	return nil
 }