app/vmselect/promql: return NaN values if invalid bucket counts are passed to histogram_quantile

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/136

app/vmselect/promql/exec_test.go

@@ -2158,6 +2158,26 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_quantile(negative-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_quantile(0.6,
+			label_set(90, "foo", "bar", "le", "10")
+			or label_set(-100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_quantile(nan-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_quantile(0.6,
+			label_set(90, "foo", "bar", "le", "10")
+			or label_set(NaN, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`median_over_time()`, func(t *testing.T) {
 		t.Parallel()
 		q := `median_over_time({})`

app/vmselect/promql/transform.go

@@ -331,13 +331,17 @@ func transformHistogramQuantile(tfa *transformFuncArg) ([]*timeseries, error) {
 			return inf
 		}
 		vReq := xss[len(xss)-1].ts.Values[i] * phi
+		if math.IsNaN(vReq) || vReq < 0 {
+			// Broken bucket.
+			return nan
+		}
 		for _, xs := range xss {
 			v := xs.ts.Values[i]
-			le := xs.le
+			if math.IsNaN(v) || v < 0 {
-			if v <= vPrev {
+				// Broken bucket.
-				v = vPrev
+				return nan
-				le = lePrev
 			}
+			le := xs.le
 			if v < vReq {
 				vPrev = v
 				lePrev = le