From ee066aa0d5bf6d7bdefc3ae2c0cfdcdfbdcb8c2f Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Wed, 16 Feb 2022 18:14:55 +0200
Subject: [PATCH] lib/storage: use binary search instead of full scan for
 skipping artificial tags when searching for tag names or tag values

This should improve performance for /api/v1/labels and /api/v1/label/<label_name>/values

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2200
---
 docs/CHANGELOG.md       |  1 +
 lib/storage/index_db.go | 56 ++++++++++++++++++++---------------------
 2 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 3bf974a9d..2e60f09bb 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -15,6 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 ## tip
 
 
+* BUGFIX: fix a bug, which could significantly slow down requests to `/api/v1/labels` and `/api/v1/label/<label_name>/values`. These APIs are used by Grafana for auto-completion of label names and label values. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2200).
 * BUGFIX: vmalert: add support for `$externalLabels` and `$externalURL` template vars in the same way as Prometheus does. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2193).
 * BUGFIX: update default value for `-promscrape.fileSDCheckInterval`, so it matches default duration used by Prometheus for checking for updates in `file_sd_configs`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2187). Thanks to @corporate-gadfly for the fix.
 
diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go
index 2bff65894..277b36947 100644
--- a/lib/storage/index_db.go
+++ b/lib/storage/index_db.go
@@ -801,12 +801,9 @@ func (is *indexSearch) searchTagKeysOnDate(tks map[string]struct{}, date uint64,
 			continue
 		}
 		key := mp.Tag.Key
-		if isArtificialTagKey(key) {
-			// Skip artificially created tag key.
-			continue
+		if !isArtificialTagKey(key) {
+			tks[string(key)] = struct{}{}
 		}
-		// Store tag key.
-		tks[string(key)] = struct{}{}
 
 		// Search for the next tag key.
 		// The last char in kb.B must be tagSeparatorChar.
@@ -880,12 +877,9 @@ func (is *indexSearch) searchTagKeys(tks map[string]struct{}, maxTagKeys int) er
 			continue
 		}
 		key := mp.Tag.Key
-		if isArtificialTagKey(key) {
-			// Skip artificailly created tag keys.
-			continue
+		if !isArtificialTagKey(key) {
+			tks[string(key)] = struct{}{}
 		}
-		// Store tag key.
-		tks[string(key)] = struct{}{}
 
 		// Search for the next tag key.
 		// The last char in kb.B must be tagSeparatorChar.
@@ -1000,14 +994,15 @@ func (is *indexSearch) searchTagValuesOnDate(tvs map[string]struct{}, tagKey []b
 			continue
 		}
 
-		// Store tag value
-		tvs[string(mp.Tag.Value)] = struct{}{}
-
-		if mp.MetricIDsLen() < maxMetricIDsPerRow/2 {
-			// There is no need in searching for the next tag value,
-			// since it is likely it is located in the next row,
-			// because the current row contains incomplete metricIDs set.
-			continue
+		skipTag := isArtificialTagKey(mp.Tag.Key)
+		if !skipTag {
+			tvs[string(mp.Tag.Value)] = struct{}{}
+			if mp.MetricIDsLen() < maxMetricIDsPerRow/2 {
+				// There is no need in searching for the next tag value,
+				// since it is likely it is located in the next row,
+				// because the current row contains incomplete metricIDs set.
+				continue
+			}
 		}
 		// Search for the next tag value.
 		// The last char in kb.B must be tagSeparatorChar.
@@ -1015,7 +1010,9 @@ func (is *indexSearch) searchTagValuesOnDate(tvs map[string]struct{}, tagKey []b
 		kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixDateTagToMetricIDs)
 		kb.B = encoding.MarshalUint64(kb.B, date)
 		kb.B = marshalTagValue(kb.B, mp.Tag.Key)
-		kb.B = marshalTagValue(kb.B, mp.Tag.Value)
+		if !skipTag {
+			kb.B = marshalTagValue(kb.B, mp.Tag.Value)
+		}
 		kb.B[len(kb.B)-1]++
 		ts.Seek(kb.B)
 	}
@@ -1085,21 +1082,24 @@ func (is *indexSearch) searchTagValues(tvs map[string]struct{}, tagKey []byte, m
 			continue
 		}
 
-		// Store tag value
-		tvs[string(mp.Tag.Value)] = struct{}{}
-
-		if mp.MetricIDsLen() < maxMetricIDsPerRow/2 {
-			// There is no need in searching for the next tag value,
-			// since it is likely it is located in the next row,
-			// because the current row contains incomplete metricIDs set.
-			continue
+		skipTag := isArtificialTagKey(mp.Tag.Key)
+		if !skipTag {
+			tvs[string(mp.Tag.Value)] = struct{}{}
+			if mp.MetricIDsLen() < maxMetricIDsPerRow/2 {
+				// There is no need in searching for the next tag value,
+				// since it is likely it is located in the next row,
+				// because the current row contains incomplete metricIDs set.
+				continue
+			}
 		}
 		// Search for the next tag value.
 		// The last char in kb.B must be tagSeparatorChar.
 		// Just increment it in order to jump to the next tag value.
 		kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
 		kb.B = marshalTagValue(kb.B, mp.Tag.Key)
-		kb.B = marshalTagValue(kb.B, mp.Tag.Value)
+		if !skipTag {
+			kb.B = marshalTagValue(kb.B, mp.Tag.Value)
+		}
 		kb.B[len(kb.B)-1]++
 		ts.Seek(kb.B)
 	}