From b75f1854c55740ca13fc07aca5e93ac8d4c8ca04 Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Wed, 14 Sep 2022 17:41:09 +0300 Subject: [PATCH] vmselect/promql: add alphanumeric sort by label (sort_by_label_numeric) (#2982) * vmselect/promql: add alphanumeric sort by label (sort_by_label_numeric) * vmselect/promql: fix tests, add documentation * vmselect/promql: update test * vmselect/promql: update for alphanumeric sorting, fix tests * vmselect/promql: remove comments * vmselect/promql: cleanup * vmselect/promql: avoid memory allocations, update functions descriptions * vmselect/promql: make linter happy (remove ineffectual assigment) * vmselect/promql: add test case, fix behavior when strings are equal * vendor: update github.com/VictoriaMetrics/metricsql from v0.44.1 to v0.45.0 this adds support for sort_by_label_numeric and sort_by_label_numeric_desc functions * wip * lib/promscrape: read response body into memory in stream parsing mode before parsing it This reduces scrape duration for targets returning big responses. The response body was already read into memory in stream parsing mode before this change, so this commit shouldn't increase memory usage. * wip Co-authored-by: Aliaksandr Valialkin --- app/vmselect/promql/exec.go | 3 +- app/vmselect/promql/exec_test.go | 175 ++++++++++ app/vmselect/promql/transform.go | 313 ++++++++++++------ app/vmselect/promql/transform_test.go | 108 ++++++ docs/CHANGELOG.md | 3 +- docs/MetricsQL.md | 10 + go.mod | 2 +- go.sum | 4 +- .../VictoriaMetrics/metricsql/doc.go | 13 +- .../VictoriaMetrics/metricsql/optimizer.go | 6 +- .../VictoriaMetrics/metricsql/transform.go | 188 +++++------ vendor/modules.txt | 2 +- 12 files changed, 625 insertions(+), 202 deletions(-) diff --git a/app/vmselect/promql/exec.go b/app/vmselect/promql/exec.go index a605157ef..ab2753fc8 100644 --- a/app/vmselect/promql/exec.go +++ b/app/vmselect/promql/exec.go @@ -99,7 +99,8 @@ func maySortResults(e metricsql.Expr, tss []*timeseries) bool { case *metricsql.FuncExpr: switch strings.ToLower(v.Name) { case "sort", "sort_desc", - "sort_by_label", "sort_by_label_desc": + "sort_by_label", "sort_by_label_desc", + "sort_by_label_numeric", "sort_by_label_numeric_desc": return false } case *metricsql.AggrFuncExpr: diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 171aae0dc..af9895826 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -7738,6 +7738,178 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r1, r2, r3, r4} f(q, resultExpected) }) + t.Run(`sort_by_label_numeric(multiple_labels_only_string)`, func(t *testing.T) { + t.Parallel() + q := `sort_by_label_numeric(( + label_set(1, "x", "b", "y", "aa"), + label_set(2, "x", "a", "y", "aa"), + ), "y", "x")` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{2, 2, 2, 2, 2, 2}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{ + { + Key: []byte("x"), + Value: []byte("a"), + }, + { + Key: []byte("y"), + Value: []byte("aa"), + }, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 1, 1, 1, 1, 1}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{ + { + Key: []byte("x"), + Value: []byte("b"), + }, + { + Key: []byte("y"), + Value: []byte("aa"), + }, + } + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) + t.Run(`sort_by_label_numeric(multiple_labels_numbers_special_chars)`, func(t *testing.T) { + t.Parallel() + q := `sort_by_label_numeric(( + label_set(1, "x", "1:0:2", "y", "1:0:1"), + label_set(2, "x", "1:0:15", "y", "1:0:1"), + ), "x", "y")` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 1, 1, 1, 1, 1}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{ + { + Key: []byte("x"), + Value: []byte("1:0:2"), + }, + { + Key: []byte("y"), + Value: []byte("1:0:1"), + }, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{2, 2, 2, 2, 2, 2}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{ + { + Key: []byte("x"), + Value: []byte("1:0:15"), + }, + { + Key: []byte("y"), + Value: []byte("1:0:1"), + }, + } + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) + t.Run(`sort_by_label_numeric_desc(multiple_labels_numbers_special_chars)`, func(t *testing.T) { + t.Parallel() + q := `sort_by_label_numeric_desc(( + label_set(1, "x", "1:0:2", "y", "1:0:1"), + label_set(2, "x", "1:0:15", "y", "1:0:1"), + ), "x", "y")` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{2, 2, 2, 2, 2, 2}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{ + { + Key: []byte("x"), + Value: []byte("1:0:15"), + }, + { + Key: []byte("y"), + Value: []byte("1:0:1"), + }, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 1, 1, 1, 1, 1}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{ + { + Key: []byte("x"), + Value: []byte("1:0:2"), + }, + { + Key: []byte("y"), + Value: []byte("1:0:1"), + }, + } + resultExpected := []netstorage.Result{r1, r2} + f(q, resultExpected) + }) + t.Run(`sort_by_label_numeric(alias_numbers_with_special_chars)`, func(t *testing.T) { + t.Parallel() + q := `sort_by_label_numeric(( + label_set(4, "a", "DS50:1/0/15"), + label_set(1, "a", "DS50:1/0/0"), + label_set(2, "a", "DS50:1/0/1"), + label_set(3, "a", "DS50:1/0/2"), + ), "a")` + r1 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1, 1, 1, 1, 1, 1}, + Timestamps: timestampsExpected, + } + r1.MetricName.Tags = []storage.Tag{ + { + Key: []byte("a"), + Value: []byte("DS50:1/0/0"), + }, + } + r2 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{2, 2, 2, 2, 2, 2}, + Timestamps: timestampsExpected, + } + r2.MetricName.Tags = []storage.Tag{ + { + Key: []byte("a"), + Value: []byte("DS50:1/0/1"), + }, + } + r3 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{3, 3, 3, 3, 3, 3}, + Timestamps: timestampsExpected, + } + r3.MetricName.Tags = []storage.Tag{ + { + Key: []byte("a"), + Value: []byte("DS50:1/0/2"), + }, + } + r4 := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{4, 4, 4, 4, 4, 4}, + Timestamps: timestampsExpected, + } + r4.MetricName.Tags = []storage.Tag{ + { + Key: []byte("a"), + Value: []byte("DS50:1/0/15"), + }, + } + resultExpected := []netstorage.Result{r1, r2, r3, r4} + f(q, resultExpected) + }) } func TestExecError(t *testing.T) { @@ -7811,6 +7983,8 @@ func TestExecError(t *testing.T) { f(`sort_desc()`) f(`sort_by_label()`) f(`sort_by_label_desc()`) + f(`sort_by_label_numeric()`) + f(`sort_by_label_numeric_desc()`) f(`timestamp()`) f(`timestamp_with_name()`) f(`vector()`) @@ -7933,6 +8107,7 @@ func TestExecError(t *testing.T) { f(`round(1, 1 or label_set(2, "xx", "foo"))`) f(`histogram_quantile(1 or label_set(2, "xx", "foo"), 1)`) f(`histogram_quantiles("foo", 1 or label_set(2, "xxx", "foo"), 2)`) + f(`sort_by_label_numeric(1, 2)`) f(`label_set(1, 2, 3)`) f(`label_set(1, "foo", (label_set(1, "foo", bar") or label_set(2, "xxx", "yy")))`) f(`label_set(1, "foo", 3)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index b2696f775..739d6e7c4 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -14,104 +14,107 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metricsql" ) var transformFuncs = map[string]transformFunc{ - "": transformUnion, // empty func is a synonym to union - "abs": newTransformFuncOneArg(transformAbs), - "absent": transformAbsent, - "acos": newTransformFuncOneArg(transformAcos), - "acosh": newTransformFuncOneArg(transformAcosh), - "asin": newTransformFuncOneArg(transformAsin), - "asinh": newTransformFuncOneArg(transformAsinh), - "atan": newTransformFuncOneArg(transformAtan), - "atanh": newTransformFuncOneArg(transformAtanh), - "bitmap_and": newTransformBitmap(bitmapAnd), - "bitmap_or": newTransformBitmap(bitmapOr), - "bitmap_xor": newTransformBitmap(bitmapXor), - "buckets_limit": transformBucketsLimit, - "ceil": newTransformFuncOneArg(transformCeil), - "clamp": transformClamp, - "clamp_max": transformClampMax, - "clamp_min": transformClampMin, - "cos": newTransformFuncOneArg(transformCos), - "cosh": newTransformFuncOneArg(transformCosh), - "day_of_month": newTransformFuncDateTime(transformDayOfMonth), - "day_of_week": newTransformFuncDateTime(transformDayOfWeek), - "days_in_month": newTransformFuncDateTime(transformDaysInMonth), - "deg": newTransformFuncOneArg(transformDeg), - "drop_common_labels": transformDropCommonLabels, - "end": newTransformFuncZeroArgs(transformEnd), - "exp": newTransformFuncOneArg(transformExp), - "floor": newTransformFuncOneArg(transformFloor), - "histogram_avg": transformHistogramAvg, - "histogram_quantile": transformHistogramQuantile, - "histogram_quantiles": transformHistogramQuantiles, - "histogram_share": transformHistogramShare, - "histogram_stddev": transformHistogramStddev, - "histogram_stdvar": transformHistogramStdvar, - "hour": newTransformFuncDateTime(transformHour), - "interpolate": transformInterpolate, - "keep_last_value": transformKeepLastValue, - "keep_next_value": transformKeepNextValue, - "label_copy": transformLabelCopy, - "label_del": transformLabelDel, - "label_graphite_group": transformLabelGraphiteGroup, - "label_join": transformLabelJoin, - "label_keep": transformLabelKeep, - "label_lowercase": transformLabelLowercase, - "label_map": transformLabelMap, - "label_match": transformLabelMatch, - "label_mismatch": transformLabelMismatch, - "label_move": transformLabelMove, - "label_replace": transformLabelReplace, - "label_set": transformLabelSet, - "label_transform": transformLabelTransform, - "label_uppercase": transformLabelUppercase, - "label_value": transformLabelValue, - "limit_offset": transformLimitOffset, - "ln": newTransformFuncOneArg(transformLn), - "log2": newTransformFuncOneArg(transformLog2), - "log10": newTransformFuncOneArg(transformLog10), - "minute": newTransformFuncDateTime(transformMinute), - "month": newTransformFuncDateTime(transformMonth), - "now": transformNow, - "pi": transformPi, - "prometheus_buckets": transformPrometheusBuckets, - "rad": newTransformFuncOneArg(transformRad), - "rand": newTransformRand(newRandFloat64), - "rand_exponential": newTransformRand(newRandExpFloat64), - "rand_normal": newTransformRand(newRandNormFloat64), - "range_avg": newTransformFuncRange(runningAvg), - "range_first": transformRangeFirst, - "range_last": transformRangeLast, - "range_max": newTransformFuncRange(runningMax), - "range_min": newTransformFuncRange(runningMin), - "range_quantile": transformRangeQuantile, - "range_sum": newTransformFuncRange(runningSum), - "remove_resets": transformRemoveResets, - "round": transformRound, - "running_avg": newTransformFuncRunning(runningAvg), - "running_max": newTransformFuncRunning(runningMax), - "running_min": newTransformFuncRunning(runningMin), - "running_sum": newTransformFuncRunning(runningSum), - "scalar": transformScalar, - "sgn": transformSgn, - "sin": newTransformFuncOneArg(transformSin), - "sinh": newTransformFuncOneArg(transformSinh), - "smooth_exponential": transformSmoothExponential, - "sort": newTransformFuncSort(false), - "sort_by_label": newTransformFuncSortByLabel(false), - "sort_by_label_desc": newTransformFuncSortByLabel(true), - "sort_desc": newTransformFuncSort(true), - "sqrt": newTransformFuncOneArg(transformSqrt), - "start": newTransformFuncZeroArgs(transformStart), - "step": newTransformFuncZeroArgs(transformStep), - "tan": newTransformFuncOneArg(transformTan), - "tanh": newTransformFuncOneArg(transformTanh), - "time": transformTime, + "": transformUnion, // empty func is a synonym to union + "abs": newTransformFuncOneArg(transformAbs), + "absent": transformAbsent, + "acos": newTransformFuncOneArg(transformAcos), + "acosh": newTransformFuncOneArg(transformAcosh), + "asin": newTransformFuncOneArg(transformAsin), + "asinh": newTransformFuncOneArg(transformAsinh), + "atan": newTransformFuncOneArg(transformAtan), + "atanh": newTransformFuncOneArg(transformAtanh), + "bitmap_and": newTransformBitmap(bitmapAnd), + "bitmap_or": newTransformBitmap(bitmapOr), + "bitmap_xor": newTransformBitmap(bitmapXor), + "buckets_limit": transformBucketsLimit, + "ceil": newTransformFuncOneArg(transformCeil), + "clamp": transformClamp, + "clamp_max": transformClampMax, + "clamp_min": transformClampMin, + "cos": newTransformFuncOneArg(transformCos), + "cosh": newTransformFuncOneArg(transformCosh), + "day_of_month": newTransformFuncDateTime(transformDayOfMonth), + "day_of_week": newTransformFuncDateTime(transformDayOfWeek), + "days_in_month": newTransformFuncDateTime(transformDaysInMonth), + "deg": newTransformFuncOneArg(transformDeg), + "drop_common_labels": transformDropCommonLabels, + "end": newTransformFuncZeroArgs(transformEnd), + "exp": newTransformFuncOneArg(transformExp), + "floor": newTransformFuncOneArg(transformFloor), + "histogram_avg": transformHistogramAvg, + "histogram_quantile": transformHistogramQuantile, + "histogram_quantiles": transformHistogramQuantiles, + "histogram_share": transformHistogramShare, + "histogram_stddev": transformHistogramStddev, + "histogram_stdvar": transformHistogramStdvar, + "hour": newTransformFuncDateTime(transformHour), + "interpolate": transformInterpolate, + "keep_last_value": transformKeepLastValue, + "keep_next_value": transformKeepNextValue, + "label_copy": transformLabelCopy, + "label_del": transformLabelDel, + "label_graphite_group": transformLabelGraphiteGroup, + "label_join": transformLabelJoin, + "label_keep": transformLabelKeep, + "label_lowercase": transformLabelLowercase, + "label_map": transformLabelMap, + "label_match": transformLabelMatch, + "label_mismatch": transformLabelMismatch, + "label_move": transformLabelMove, + "label_replace": transformLabelReplace, + "label_set": transformLabelSet, + "label_transform": transformLabelTransform, + "label_uppercase": transformLabelUppercase, + "label_value": transformLabelValue, + "limit_offset": transformLimitOffset, + "ln": newTransformFuncOneArg(transformLn), + "log2": newTransformFuncOneArg(transformLog2), + "log10": newTransformFuncOneArg(transformLog10), + "minute": newTransformFuncDateTime(transformMinute), + "month": newTransformFuncDateTime(transformMonth), + "now": transformNow, + "pi": transformPi, + "prometheus_buckets": transformPrometheusBuckets, + "rad": newTransformFuncOneArg(transformRad), + "rand": newTransformRand(newRandFloat64), + "rand_exponential": newTransformRand(newRandExpFloat64), + "rand_normal": newTransformRand(newRandNormFloat64), + "range_avg": newTransformFuncRange(runningAvg), + "range_first": transformRangeFirst, + "range_last": transformRangeLast, + "range_max": newTransformFuncRange(runningMax), + "range_min": newTransformFuncRange(runningMin), + "range_quantile": transformRangeQuantile, + "range_sum": newTransformFuncRange(runningSum), + "remove_resets": transformRemoveResets, + "round": transformRound, + "running_avg": newTransformFuncRunning(runningAvg), + "running_max": newTransformFuncRunning(runningMax), + "running_min": newTransformFuncRunning(runningMin), + "running_sum": newTransformFuncRunning(runningSum), + "scalar": transformScalar, + "sgn": transformSgn, + "sin": newTransformFuncOneArg(transformSin), + "sinh": newTransformFuncOneArg(transformSinh), + "smooth_exponential": transformSmoothExponential, + "sort": newTransformFuncSort(false), + "sort_by_label": newTransformFuncSortByLabel(false), + "sort_by_label_desc": newTransformFuncSortByLabel(true), + "sort_by_label_numeric": newTransformFuncNumericSort(false), + "sort_by_label_numeric_desc": newTransformFuncNumericSort(true), + "sort_desc": newTransformFuncSort(true), + "sqrt": newTransformFuncOneArg(transformSqrt), + "start": newTransformFuncZeroArgs(transformStart), + "step": newTransformFuncZeroArgs(transformStep), + "tan": newTransformFuncOneArg(transformTan), + "tanh": newTransformFuncOneArg(transformTanh), + "time": transformTime, // "timestamp" has been moved to rollup funcs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 "timezone_offset": transformTimezoneOffset, "union": transformUnion, @@ -1992,6 +1995,130 @@ func newTransformFuncSortByLabel(isDesc bool) transformFunc { } } +func newTransformFuncNumericSort(isDesc bool) transformFunc { + return func(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if len(args) < 2 { + return nil, fmt.Errorf("expecting at least 2 args; got %d args", len(args)) + } + var labels []string + for i, arg := range args[1:] { + label, err := getString(arg, i+1) + if err != nil { + return nil, fmt.Errorf("cannot parse label #%d for sorting: %w", i+1, err) + } + labels = append(labels, label) + } + rvs := args[0] + sort.SliceStable(rvs, func(i, j int) bool { + for _, label := range labels { + a := rvs[i].MetricName.GetTagValue(label) + b := rvs[j].MetricName.GetTagValue(label) + if string(a) == string(b) { + continue + } + aStr := bytesutil.ToUnsafeString(a) + bStr := bytesutil.ToUnsafeString(b) + if isDesc { + return numericLess(bStr, aStr) + } + return numericLess(aStr, bStr) + } + return false + }) + return rvs, nil + } +} + +func numericLess(a, b string) bool { + for { + if len(b) == 0 { + return false + } + if len(a) == 0 { + return true + } + aPrefix := getNumPrefix(a) + bPrefix := getNumPrefix(b) + a = a[len(aPrefix):] + b = b[len(bPrefix):] + if len(aPrefix) > 0 || len(bPrefix) > 0 { + if len(aPrefix) == 0 { + return false + } + if len(bPrefix) == 0 { + return true + } + aNum := mustParseNum(aPrefix) + bNum := mustParseNum(bPrefix) + if aNum != bNum { + return aNum < bNum + } + } + aPrefix = getNonNumPrefix(a) + bPrefix = getNonNumPrefix(b) + a = a[len(aPrefix):] + b = b[len(bPrefix):] + if aPrefix != bPrefix { + return aPrefix < bPrefix + } + } +} + +func getNumPrefix(s string) string { + i := 0 + if len(s) > 0 { + switch s[0] { + case '-', '+': + i++ + } + } + hasNum := false + hasDot := false + for i < len(s) { + if !isDecimalChar(s[i]) { + if !hasDot && s[i] == '.' { + hasDot = true + i++ + continue + } + if !hasNum { + return "" + } + return s[:i] + } + hasNum = true + i++ + } + if !hasNum { + return "" + } + return s +} + +func getNonNumPrefix(s string) string { + i := 0 + for i < len(s) { + if isDecimalChar(s[i]) { + return s[:i] + } + i++ + } + return s +} + +func isDecimalChar(ch byte) bool { + return ch >= '0' && ch <= '9' +} + +func mustParseNum(s string) float64 { + f, err := strconv.ParseFloat(s, 64) + if err != nil { + logger.Panicf("BUG: unexpected error when parsing the number %q: %s", s, err) + } + return f +} + func newTransformFuncSort(isDesc bool) transformFunc { return func(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args diff --git a/app/vmselect/promql/transform_test.go b/app/vmselect/promql/transform_test.go index 5998c4d78..5c5d7e414 100644 --- a/app/vmselect/promql/transform_test.go +++ b/app/vmselect/promql/transform_test.go @@ -3,6 +3,7 @@ package promql import ( "fmt" "reflect" + "strconv" "strings" "testing" @@ -220,3 +221,110 @@ func timeseriesToPromMetrics(tss []*timeseries) string { } return strings.Join(a, "\n") } + +func TestGetNumPrefix(t *testing.T) { + f := func(s, prefixExpected string) { + t.Helper() + prefix := getNumPrefix(s) + if prefix != prefixExpected { + t.Fatalf("unexpected getNumPrefix(%q): got %q; want %q", s, prefix, prefixExpected) + } + if len(prefix) > 0 { + if _, err := strconv.ParseFloat(prefix, 64); err != nil { + t.Fatalf("cannot parse num %q: %s", prefix, err) + } + } + } + + f("", "") + f("foo", "") + f("-", "") + f(".", "") + f("-.", "") + f("+..", "") + f("1", "1") + f("12", "12") + f("1foo", "1") + f("-123", "-123") + f("-123bar", "-123") + f("+123", "+123") + f("+123.", "+123.") + f("+123..", "+123.") + f("+123.-", "+123.") + f("12.34..", "12.34") + f("-12.34..", "-12.34") + f("-12.-34..", "-12.") +} + +func TestNumericLess(t *testing.T) { + f := func(a, b string, want bool) { + t.Helper() + if got := numericLess(a, b); got != want { + t.Fatalf("unexpected numericLess(%q, %q): got %v; want %v", a, b, got, want) + } + } + // empty strings + f("", "", false) + f("", "321", true) + f("321", "", false) + f("", "abc", true) + f("abc", "", false) + f("foo", "123", false) + f("123", "foo", true) + // same length numbers + f("123", "321", true) + f("321", "123", false) + f("123", "123", false) + // same length strings + f("a", "b", true) + f("b", "a", false) + f("a", "a", false) + // identical string prefix + f("foo123", "foo", false) + f("foo", "foo123", true) + f("foo", "foo", false) + // identical num prefix + f("123foo", "123bar", false) + f("123bar", "123foo", true) + f("123bar", "123bar", false) + // numbers with special chars + f("1:0:0", "1:0:2", true) + // numbers with special chars and different number rank + f("1:0:15", "1:0:2", false) + // multiple zeroes" + f("0", "00", false) + // only chars + f("aa", "ab", true) + // strings with different lengths + f("ab", "abc", true) + // multiple zeroes after equal char + f("a0001", "a0000001", false) + // short first string with numbers and highest rank + f("a10", "abcdefgh2", true) + // less as second string + f("a1b", "a01b", false) + // equal strings by length with different number rank + f("a001b01", "a01b001", false) + // different numbers rank + f("a01b001", "a001b01", false) + // different numbers rank + f("a01b001", "a001b01", false) + // highest char and number + f("a1", "a1x", true) + // highest number reverse chars + f("1b", "1ax", false) + // numbers with leading zero + f("082", "83", true) + // numbers with leading zero and chars + f("083a", "9a", false) + f("083a", "94a", true) + // negative number + f("-123", "123", true) + f("-123", "+123", true) + f("-123", "-123", false) + f("123", "-123", false) + // fractional number + f("12.9", "12.56", false) + f("12.56", "12.9", true) + f("12.9", "12.9", false) +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 66d515108..61e248680 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -23,8 +23,9 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `debug` mode to the alerting rule settings for printing additional information into logs during evaluation. See `debug` param in [alerting rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules). -* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): minimize the time needed for reading large responses from scrape targets in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). This should reduce scrape durations for such targets as [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) running in a big Kubernetes cluster. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add experimental feature for displaying last 10 states of the rule (recording or alerting) evaluation. The state is available on the Rule page, which can be opened by clicking on `Details` link next to Rule's name on the `/groups` page. +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): minimize the time needed for reading large responses from scrape targets in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). This should reduce scrape durations for such targets as [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) running in a big Kubernetes cluster. +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [sort_by_label_numeric](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric) and [sort_by_label_numeric_desc](https://docs.victoriametrics.com/MetricsQL.html#sort_by_label_numeric_desc) functions for [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html) of input time series by the specified labels. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2938). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly calculate `rate_over_sum(m[d])` as `sum_over_time(m[d])/d`. Previously the `sum_over_time(m[d])` could be improperly divided by smaller than `d` time range. See [rate_over_sum() docs](https://docs.victoriametrics.com/MetricsQL.html#rate_over_sum) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3045). * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly calculate query results at `vmselect`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067). The issue has been introduced in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810). diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 5906007cc..07bfac00f 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -673,6 +673,16 @@ The list of supported transform functions: `sort_by_label_desc(q, label1, ... labelN)` sorts series in descending order by the given set of labels. For example, `sort_by_label(foo, "bar")` would sort `foo` series by values of the label `bar` in these series. See also [sort_by_label](#sort_by_label). +#### sort_by_label_numeric + +`sort_by_label_numeric(q, label1, ... labelN)` sorts series in ascending order by the given set of labels using [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html). For example, if `foo` series have `bar` label with values `1`, `101`, `15` and `2`, then `sort_by_label_numeric(foo, "bar")` would return series in the following order of `bar` label values: `1`, `2`, `15` and `101`. +See also [sort_by_label_numeric_desc](#sort_by_label_numeric_desc) and [sort_by_label](#sort_by_label). + +#### sort_by_label_numeric_desc + +`sort_by_label_numeric_desc(q, label1, ... labelN)` sorts series in descending order by the given set of labels using [numeric sort](https://www.gnu.org/software/coreutils/manual/html_node/Version-sort-is-not-the-same-as-numeric-sort.html). For example, if `foo` series have `bar` label with values `1`, `101`, `15` and `2`, then `sort_by_label_numeric(foo, "bar")` would return series in the following order of `bar` label values: `101`, `15`, `2` and `1`. +See also [sort_by_label_numeric](#sort_by_label_numeric) and [sort_by_label_desc](#sort_by_label_desc). + #### sort_desc `sort_desc(q)` sorts series in descending order by the last point in every time series returned by `q`. This function is supported by PromQL. See also [sort](#sort). diff --git a/go.mod b/go.mod index f8ec5aa1d..6b758edd9 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 github.com/VictoriaMetrics/metrics v1.22.2 - github.com/VictoriaMetrics/metricsql v0.44.1 + github.com/VictoriaMetrics/metricsql v0.45.0 github.com/aws/aws-sdk-go v1.44.96 github.com/cespare/xxhash/v2 v2.1.2 diff --git a/go.sum b/go.sum index 421e1c47a..656f8e70a 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metrics v1.22.2 h1:A6LsNidYwkAHetxsvNFaUWjtzu5ltdgNEoS6i7Bn+6I= github.com/VictoriaMetrics/metrics v1.22.2/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= -github.com/VictoriaMetrics/metricsql v0.44.1 h1:qGoRt0g84uMUscVjS7P3uDZKmjJubWKaIx9v0iHKgck= -github.com/VictoriaMetrics/metricsql v0.44.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= +github.com/VictoriaMetrics/metricsql v0.45.0 h1:kVQHnkDJm4qyJ8f5msTclmwqAtlUdPbbEJ7zoa/FTNs= +github.com/VictoriaMetrics/metricsql v0.45.0/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/doc.go b/vendor/github.com/VictoriaMetrics/metricsql/doc.go index 55f156849..ad99176ec 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/doc.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/doc.go @@ -5,11 +5,10 @@ // // Usage: // -// expr, err := metricsql.Parse(`sum(rate(foo{bar="baz"}[5m])) by (job)`) -// if err != nil { -// // parse error -// } -// // Now expr contains parsed MetricsQL as `*Expr` structs. -// // See Parse examples for more details. -// +// expr, err := metricsql.Parse(`sum(rate(foo{bar="baz"}[5m])) by (job)`) +// if err != nil { +// // parse error +// } +// // Now expr contains parsed MetricsQL as `*Expr` structs. +// // See Parse examples for more details. package metricsql diff --git a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go index 03410f053..e56659b86 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/optimizer.go @@ -10,9 +10,9 @@ import ( // // It performs the following optimizations: // -// - Adds missing filters to `foo{filters1} op bar{filters2}` -// according to https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization -// I.e. such query is converted to `foo{filters1, filters2} op bar{filters1, filters2}` +// - Adds missing filters to `foo{filters1} op bar{filters2}` +// according to https://utcc.utoronto.ca/~cks/space/blog/sysadmin/PrometheusLabelNonOptimization +// I.e. such query is converted to `foo{filters1, filters2} op bar{filters1, filters2}` func Optimize(e Expr) Expr { if !canOptimize(e) { return e diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index 865c68e87..44084bc3e 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -5,99 +5,101 @@ import ( ) var transformFuncs = map[string]bool{ - "": true, // empty func is a synonym to union - "abs": true, - "absent": true, - "acos": true, - "acosh": true, - "asin": true, - "asinh": true, - "atan": true, - "atanh": true, - "bitmap_and": true, - "bitmap_or": true, - "bitmap_xor": true, - "buckets_limit": true, - "ceil": true, - "clamp": true, - "clamp_max": true, - "clamp_min": true, - "cos": true, - "cosh": true, - "day_of_month": true, - "day_of_week": true, - "days_in_month": true, - "deg": true, - "drop_common_labels": true, - "end": true, - "exp": true, - "floor": true, - "histogram_avg": true, - "histogram_quantile": true, - "histogram_quantiles": true, - "histogram_share": true, - "histogram_stddev": true, - "histogram_stdvar": true, - "hour": true, - "interpolate": true, - "keep_last_value": true, - "keep_next_value": true, - "label_copy": true, - "label_del": true, - "label_graphite_group": true, - "label_join": true, - "label_keep": true, - "label_lowercase": true, - "label_map": true, - "label_match": true, - "label_mismatch": true, - "label_move": true, - "label_replace": true, - "label_set": true, - "label_transform": true, - "label_uppercase": true, - "label_value": true, - "limit_offset": true, - "ln": true, - "log2": true, - "log10": true, - "minute": true, - "month": true, - "now": true, - "pi": true, - "prometheus_buckets": true, - "rad": true, - "rand": true, - "rand_exponential": true, - "rand_normal": true, - "range_avg": true, - "range_first": true, - "range_last": true, - "range_max": true, - "range_min": true, - "range_quantile": true, - "range_sum": true, - "remove_resets": true, - "round": true, - "running_avg": true, - "running_max": true, - "running_min": true, - "running_sum": true, - "scalar": true, - "sgn": true, - "sin": true, - "sinh": true, - "smooth_exponential": true, - "sort": true, - "sort_by_label": true, - "sort_by_label_desc": true, - "sort_desc": true, - "sqrt": true, - "start": true, - "step": true, - "tan": true, - "tanh": true, - "time": true, + "": true, // empty func is a synonym to union + "abs": true, + "absent": true, + "acos": true, + "acosh": true, + "asin": true, + "asinh": true, + "atan": true, + "atanh": true, + "bitmap_and": true, + "bitmap_or": true, + "bitmap_xor": true, + "buckets_limit": true, + "ceil": true, + "clamp": true, + "clamp_max": true, + "clamp_min": true, + "cos": true, + "cosh": true, + "day_of_month": true, + "day_of_week": true, + "days_in_month": true, + "deg": true, + "drop_common_labels": true, + "end": true, + "exp": true, + "floor": true, + "histogram_avg": true, + "histogram_quantile": true, + "histogram_quantiles": true, + "histogram_share": true, + "histogram_stddev": true, + "histogram_stdvar": true, + "hour": true, + "interpolate": true, + "keep_last_value": true, + "keep_next_value": true, + "label_copy": true, + "label_del": true, + "label_graphite_group": true, + "label_join": true, + "label_keep": true, + "label_lowercase": true, + "label_map": true, + "label_match": true, + "label_mismatch": true, + "label_move": true, + "label_replace": true, + "label_set": true, + "label_transform": true, + "label_uppercase": true, + "label_value": true, + "limit_offset": true, + "ln": true, + "log2": true, + "log10": true, + "minute": true, + "month": true, + "now": true, + "pi": true, + "prometheus_buckets": true, + "rad": true, + "rand": true, + "rand_exponential": true, + "rand_normal": true, + "range_avg": true, + "range_first": true, + "range_last": true, + "range_max": true, + "range_min": true, + "range_quantile": true, + "range_sum": true, + "remove_resets": true, + "round": true, + "running_avg": true, + "running_max": true, + "running_min": true, + "running_sum": true, + "scalar": true, + "sgn": true, + "sin": true, + "sinh": true, + "smooth_exponential": true, + "sort": true, + "sort_by_label": true, + "sort_by_label_desc": true, + "sort_by_label_numeric": true, + "sort_by_label_numeric_desc": true, + "sort_desc": true, + "sqrt": true, + "start": true, + "step": true, + "tan": true, + "tanh": true, + "time": true, // "timestamp" has been moved to rollup funcs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 "timezone_offset": true, "union": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 25b0e3a57..17b196718 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -27,7 +27,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.22.2 ## explicit; go 1.15 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.44.1 +# github.com/VictoriaMetrics/metricsql v0.45.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop