From 9052c6b73459c65c5dd8529bbf6362ef47cb3ed9 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 28 May 2024 17:19:06 +0200 Subject: [PATCH] wip --- docs/VictoriaLogs/LogsQL.md | 47 ++++++++++++++++++++++++++++++++ lib/logstorage/pipe_math.go | 13 +++++++-- lib/logstorage/pipe_math_test.go | 6 ++-- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 7dfe0d727..00100b945 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -1158,6 +1158,7 @@ LogsQL supports the following pipes: - [`filter`](#filter-pipe) applies additional [filters](#filters) to results. - [`format`](#format-pipe) formats ouptut field from input [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). - [`limit`](#limit-pipe) limits the number selected logs. +- [`math`](#math-pipe) performs mathematical calculations over numeric values stored in [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). - [`offset`](#offset-pipe) skips the given number of selected logs. - [`pack_json`](#pack_json-pipe) packs [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) into JSON object. - [`rename`](#rename-pipe) renames [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). @@ -1269,6 +1270,7 @@ See also: - [Conditional extract](#conditional-extract) - [`unpack_json` pipe](#unpack_json-pipe) - [`unpack_logfmt` pipe](#unpack_logfmt-pipe) +- [`math` pipe](#math-pipe) #### Format for extract pipe pattern @@ -1526,6 +1528,50 @@ See also: - [`sort` pipe](#sort-pipe) - [`offset` pipe](#offset-pipe) +### math pipe + +`| math ...` [pipe](#pipes) performs mathematical calculations over numeric values stored in [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). +For example, the following query divides `duration_msecs` field value by 1000, then rounds them to integer and stores the result in the `duration_secs` field: + +```logsql +_time:5m | math round(duration_msecs / 1000) as duration_secs +``` + +The following mathematical operations are supported by `math` pipe: + +- `arg1 + arg2` - returns the sum of `arg1` and `arg2` +- `arg1 - arg2` - returns the difference between `arg1` and `arg2` +- `arg1 * arg2` - multiplies `arg1` by `arg2` +- `arg1 / arg2` - divides `arg1` by `arg2` +- `arg1 % arg2` - returns the remainder of the division of `arg1` by `arg2` +- `arg1 ^ arg2` - returns the power of `arg1` by `arg2` +- `abs(arg)` - returns an absolute values for the given `arg` +- `max(arg1, ..., argN)` - returns the maximum value among the given `arg1`, ..., `argN` +- `min(arg1, ..., argN)` - returns the minimum value among the given `arg1`, ..., `argN` +- `round(arg)` - returns rounded to integer value for the given `arg`. The `round()` accepts optional `nearest` arg, which allows rounding the number to the given `nearest` multiple. + For example, `round(temperature, 0.1)` rounds `temperature` field to one decimal digit after the point. + +Every `argX` argument in every mathematical operation can contain one of the following values: + +- The name of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). For example, `errors_total / requests_total`. +- Any [supported numeric value](#numeric-values). For example, `response_size_bytes / 1MiB`. +- Another mathematical expression. Optionally, it may be put inside `(...)`. For example, `(a + b) * c`. + +Multiple distinct results can be calculated in a single `math ...` pipe - just separate them with `,`. For example, the following query calculates the error rate +and the number of successful requests from `errors`, `warnings` and `requests` [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model): + +```logsql +_time:5m | math + (errors / requests) as error_rate, + (requests - (errors + warnings)) as success_requests +``` + +See also: + +- [`stats` pipe](#stats-pipe) +- [`extract` pipe](#extract-pipe) + + ### offset pipe If some selected logs must be skipped after [`sort`](#sort-pipe), then `| offset N` [pipe](#pipes) can be used, where `N` can contain any [supported integer numeric value](#numeric-values). @@ -1808,6 +1854,7 @@ See also: - [stats by IPv4 buckets](#stats-by-ipv4-buckets) - [stats with additional filters](#stats-with-additional-filters) - [stats pipe functions](#stats-pipe-functions) +- [`math` pipe](#math-pipe) - [`sort` pipe](#sort-pipe) diff --git a/lib/logstorage/pipe_math.go b/lib/logstorage/pipe_math.go index 0239db694..5792a4050 100644 --- a/lib/logstorage/pipe_math.go +++ b/lib/logstorage/pipe_math.go @@ -535,8 +535,8 @@ func parseMathExprRound(lex *lexer) (*mathExpr, error) { if err != nil { return nil, err } - if len(me.args) != 2 { - return nil, fmt.Errorf("'round' function needs 2 args; got %d args: [%s]", len(me.args), me) + if len(me.args) != 1 && len(me.args) != 2 { + return nil, fmt.Errorf("'round' function needs 1 or 2 args; got %d args: [%s]", len(me.args), me) } return me, nil } @@ -749,6 +749,15 @@ func mathFuncMin(result []float64, args [][]float64) { func mathFuncRound(result []float64, args [][]float64) { arg := args[0] + if len(args) == 1 { + // Round to integer + for i := range result { + result[i] = math.Round(arg[i]) + } + return + } + + // Round to nearest nearest := args[1] var f float64 for i := range result { diff --git a/lib/logstorage/pipe_math_test.go b/lib/logstorage/pipe_math_test.go index 4edc5fdc1..1500e33e6 100644 --- a/lib/logstorage/pipe_math_test.go +++ b/lib/logstorage/pipe_math_test.go @@ -20,6 +20,7 @@ func TestParsePipeMathSuccess(t *testing.T) { f(`math (foo / ((bar + baz) * abc) ^ -2) as a`) f(`math (foo + bar / baz - abc) as a`) f(`math min(3, foo, (1 + bar) / baz) as a, max(a, b) as b, (abs(c) + 5) as d`) + f(`math round(foo) as x`) f(`math round(foo, 0.1) as y`) } @@ -39,7 +40,6 @@ func TestParsePipeMathFailure(t *testing.T) { f(`math max() as x`) f(`math max(a) as x`) f(`math round() as x`) - f(`math round(a) as x`) f(`math round(a, b, c) as x`) } @@ -134,7 +134,7 @@ func TestPipeMath(t *testing.T) { }, }) - f("math abs(-min(a,b)) as min, max(b,c) as max", [][]Field{ + f("math abs(-min(a,b)) as min, round(max(40*b/30,c)) as max", [][]Field{ { {"a", "v1"}, {"b", "2"}, @@ -150,7 +150,7 @@ func TestPipeMath(t *testing.T) { }, }) - f("math round((2*c + (b%c))/(c-b)^(b-1), 0.001) as a", [][]Field{ + f("math round((2*c + (b%c))/(c-b)^(b-1), -0.001) as a", [][]Field{ { {"a", "v"}, {"b", "2"},