lib/storage: filter deleted label names and values from `/api/v1/labe… (#6342)

…ls` and `/api/v1/label/.../values`

Check for deleted metrics when `match[]` filter matches small number of
time series (optimized path).

The issue was introduced
[v1.81.0](https://docs.victoriametrics.com/changelog_2022/#v1810).

Related issue
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6300 Updates
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978

Signed-off-by: hagen1778 <roman@victoriametrics.com>

docs/CHANGELOG.md

@@ -70,6 +70,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): reduce the default value for `-maxLabelValueLen` command-line flag from `16KiB` to `1KiB`. This should prevent from issues like [this one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6176) when time series with too long labels are ingested into VictoriaMetrics.
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly release memory used for metrics during config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6247).
 * BUGFIX: [dashboards](https://grafana.com/orgs/victoriametrics): fix `AnnotationQueryRunner` error in Grafana when executing annotations query against Prometheus backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6309) for details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): filter deleted label names and values from [`/api/v1/labels`](https://docs.victoriametrics.com/url-examples/#apiv1labels) and [`/api/v1/label/.../values`](https://docs.victoriametrics.com/url-examples/#apiv1labelvalues) responses when `match[]` filter matches small number of time series. The issue was introduced [v1.81.0](https://docs.victoriametrics.com/changelog_2022/#v1810).
 
 * DEPRECATION: [vmagent](https://docs.victoriametrics.com/vmagent/): removed deprecated `-remoteWrite.multitenantURL` flag from vmagent. This flag was deprecated since [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0). Use `-enableMultitenantHandlers` instead, as it is easier to use and combine with [multitenant URL at vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels). See these [docs for details](https://docs.victoriametrics.com/vmagent.html#multitenancy).
 

lib/storage/index_db.go

@@ -747,10 +747,18 @@ func (is *indexSearch) getLabelNamesForMetricIDs(qt *querytracer.Tracer, metricI
 	if len(metricIDs) > 0 {
 		lns["__name__"] = struct{}{}
 	}
+
+	dmis := is.db.s.getDeletedMetricIDs()
+	checkDeleted := dmis.Len() > 0
+
 	var mn MetricName
 	foundLabelNames := 0
 	var buf []byte
 	for _, metricID := range metricIDs {
+		if checkDeleted && dmis.Has(metricID) {
+			// skip deleted IDs from result
+			continue
+		}
 		var ok bool
 		buf, ok = is.searchMetricNameWithCache(buf[:0], metricID)
 		if !ok {
@@ -946,10 +954,18 @@ func (is *indexSearch) getLabelValuesForMetricIDs(qt *querytracer.Tracer, lvs ma
 	if labelName == "" {
 		labelName = "__name__"
 	}
+
+	dmis := is.db.s.getDeletedMetricIDs()
+	checkDeleted := dmis.Len() > 0
+
 	var mn MetricName
 	foundLabelValues := 0
 	var buf []byte
 	for _, metricID := range metricIDs {
+		if checkDeleted && dmis.Has(metricID) {
+			// skip deleted IDs from result
+			continue
+		}
 		var ok bool
 		buf, ok = is.searchMetricNameWithCache(buf[:0], metricID)
 		if !ok {

lib/storage/index_db_test.go

@@ -1562,30 +1562,34 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
 		"testMetric",
 	}
 	sort.Strings(labelNames)
+
+	newMN := func(name string, day, metric int) MetricName {
+		var mn MetricName
+		mn.MetricGroup = []byte(name)
+		mn.AddTag(
+			"constant",
+			"const",
+		)
+		mn.AddTag(
+			"day",
+			fmt.Sprintf("%v", day),
+		)
+		mn.AddTag(
+			"UniqueId",
+			fmt.Sprintf("%v", metric),
+		)
+		mn.AddTag(
+			"some_unique_id",
+			fmt.Sprintf("%v", day),
+		)
+		mn.sortTags()
+		return mn
+	}
 	for day := 0; day < days; day++ {
 		date := baseDate - uint64(day)
 		var metricIDs uint64set.Set
 		for metric := 0; metric < metricsPerDay; metric++ {
-			var mn MetricName
-			mn.MetricGroup = []byte("testMetric")
-			mn.AddTag(
-				"constant",
-				"const",
-			)
-			mn.AddTag(
-				"day",
-				fmt.Sprintf("%v", day),
-			)
-			mn.AddTag(
-				"UniqueId",
-				fmt.Sprintf("%v", metric),
-			)
-			mn.AddTag(
-				"some_unique_id",
-				fmt.Sprintf("%v", day),
-			)
-			mn.sortTags()
-
+			mn := newMN("testMetric", day, metric)
 			metricNameBuf = mn.Marshal(metricNameBuf[:0])
 			var genTSID generationTSID
 			if !is.getTSIDByMetricName(&genTSID, metricNameBuf, date) {
@@ -1626,6 +1630,29 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
 	}
 	db.putIndexSearch(is2)
 
+	// add a metric that will be deleted shortly
+	is3 := db.getIndexSearch(noDeadline)
+	day := days
+	date := baseDate - uint64(day)
+	mn := newMN("deletedMetric", day, 999)
+	mn.AddTag(
+		"labelToDelete",
+		fmt.Sprintf("%v", day),
+	)
+	mn.sortTags()
+	metricNameBuf = mn.Marshal(metricNameBuf[:0])
+	var genTSID generationTSID
+	if !is3.getTSIDByMetricName(&genTSID, metricNameBuf, date) {
+		generateTSID(&genTSID.TSID, &mn)
+		createAllIndexesForMetricName(is3, &mn, &genTSID.TSID, date)
+	}
+	// delete the added metric. It is expected it won't be returned during searches
+	deletedSet := &uint64set.Set{}
+	deletedSet.Add(genTSID.TSID.MetricID)
+	s.setDeletedMetricIDs(deletedSet)
+	db.putIndexSearch(is3)
+	s.DebugFlush()
+
 	// Check SearchLabelNamesWithFiltersOnTimeRange with the specified time range.
 	tr := TimeRange{
 		MinTimestamp: int64(now) - msecPerDay,