From 80a7c65ab72b1440a7c13b2e2c90f19f3f42f630 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 5 Jun 2024 10:07:49 +0200 Subject: [PATCH 1/3] lib/logstorage: allow using `eval` keyword instead of `math` keyword in `math` pipe --- docs/VictoriaLogs/CHANGELOG.md | 2 ++ docs/VictoriaLogs/LogsQL.md | 15 +++++++++++++++ lib/logstorage/pipe.go | 2 +- lib/logstorage/pipe_math.go | 4 ++-- lib/logstorage/pipe_math_test.go | 2 +- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index a94a2ec18..101d2ab63 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +* FEATURE: allow using `eval` instead of `math` keyword in [`math` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#math-pipe). + ## [v0.17.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.17.0-victorialogs) Released at 2024-06-05 diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 52d85e976..94e0d74e5 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -1681,10 +1681,25 @@ Every `argX` argument in every mathematical operation can contain one of the fol - Any [supported numeric value](#numeric-values), [rfc3339 time](https://www.rfc-editor.org/rfc/rfc3339) or IPv4 address. For example, `1MiB`, `"2024-05-15T10:20:30.934324Z"` or `"12.34.56.78"`. - Another mathematical expression, which can be put inside `(...)`. For example, `(a + b) * c`. +The parsed time, duration and IPv4 address can be converted back to string representation after math transformations with the help of [`format` pipe](#format-pipe). For example, +the following query rounds the `request_duration` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) to seconds before converting it back to string representation: + +```logsql +_time:5m | math round(request_duration, 1e9) as request_duration_nsecs | format '' as request_duration +``` + +The `eval` keyword can be used instead of `math` for convenince. For example, the following query calculates `duration_msecs` field +by multiplying `duration_secs` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) to `1000`: + +```logsql +_time:5m | eval (duration_secs * 1000) as duration_msecs +``` + See also: - [`stats` pipe](#stats-pipe) - [`extract` pipe](#extract-pipe) +- [`format` pipe](#format-pipe) ### offset pipe diff --git a/lib/logstorage/pipe.go b/lib/logstorage/pipe.go index 2c9063f4b..27aa21d1d 100644 --- a/lib/logstorage/pipe.go +++ b/lib/logstorage/pipe.go @@ -160,7 +160,7 @@ func parsePipe(lex *lexer) (pipe, error) { return nil, fmt.Errorf("cannot parse 'limit' pipe: %w", err) } return pl, nil - case lex.isKeyword("math"): + case lex.isKeyword("math", "eval"): pm, err := parsePipeMath(lex) if err != nil { return nil, fmt.Errorf("cannot parse 'math' pipe: %w", err) diff --git a/lib/logstorage/pipe_math.go b/lib/logstorage/pipe_math.go index 1138a38ea..b0f165b22 100644 --- a/lib/logstorage/pipe_math.go +++ b/lib/logstorage/pipe_math.go @@ -356,8 +356,8 @@ func (pmp *pipeMathProcessor) flush() error { } func parsePipeMath(lex *lexer) (*pipeMath, error) { - if !lex.isKeyword("math") { - return nil, fmt.Errorf("unexpected token: %q; want %q", lex.token, "math") + if !lex.isKeyword("math", "eval") { + return nil, fmt.Errorf("unexpected token: %q; want 'math' or 'eval'", lex.token) } lex.nextToken() diff --git a/lib/logstorage/pipe_math_test.go b/lib/logstorage/pipe_math_test.go index 67d446cc0..40b7dee50 100644 --- a/lib/logstorage/pipe_math_test.go +++ b/lib/logstorage/pipe_math_test.go @@ -77,7 +77,7 @@ func TestPipeMath(t *testing.T) { }, }) - f("math b+1 as a, a*2 as b, b-10.5+c as c", [][]Field{ + f("eval b+1 as a, a*2 as b, b-10.5+c as c", [][]Field{ { {"a", "v1"}, {"b", "2"}, From c2cd47da0744115d5b44fffcfd77b7d7a9860ac4 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 5 Jun 2024 10:12:51 +0200 Subject: [PATCH 2/3] docs/VictoriaLogs/LogsQL.md: substitute TAG with APP-NAME[PROCID] in rfc3164 syslog message format at `unpack_syslog` pipe docs --- docs/VictoriaLogs/LogsQL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 94e0d74e5..6c6871006 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -2375,7 +2375,7 @@ _time:5m | unpack_logfmt if (ip:"") from foo `| unpack_syslog from field_name` [pipe](#pipes) unpacks [syslog](https://en.wikipedia.org/wiki/Syslog) message from the given [`field_name`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). It understands the following Syslog formats: -- [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) aka `MMM DD hh:mm:ss HOSTNAME TAG: MESSAGE` +- [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) aka `MMM DD hh:mm:ss HOSTNAME APP-NAME[PROCID]: MESSAGE` - [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424) aka `1 TIMESTAMP HOSTNAME APP-NAME PROCID MSGID [STRUCTURED-DATA] MESSAGE` The following fields are unpacked: From 1d86ea49a0c267a223e2e0502137024ba1dabf2d Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 5 Jun 2024 10:15:46 +0200 Subject: [PATCH 3/3] docs/VictoriaLogs/LogsQL.md: typo fix: replace `from _json` with `from _msg` where appropriate --- docs/VictoriaLogs/LogsQL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 6c6871006..4ad0612c5 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -2214,7 +2214,7 @@ For example, the following query unpacks JSON fields from the [`_msg` field](htt _time:5m | unpack_json from _msg ``` -The `from _json` part can be omitted when JSON fields are unpacked from the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field). +The `from _msg` part can be omitted when JSON fields are unpacked from the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field). The following query is equivalent to the previous one: ```logsql @@ -2299,7 +2299,7 @@ across logs for the last 5 minutes: _time:5m | unpack_logfmt from _msg ``` -The `from _json` part can be omitted when [logfmt](https://brandur.org/logfmt) fields are unpacked from the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field). +The `from _msg` part can be omitted when [logfmt](https://brandur.org/logfmt) fields are unpacked from the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field). The following query is equivalent to the previous one: ```logsql @@ -2404,7 +2404,7 @@ across logs for the last 5 minutes: _time:5m | unpack_syslog from _msg ``` -The `from _json` part can be omitted when [syslog](https://en.wikipedia.org/wiki/Syslog) message is unpacked +The `from _msg` part can be omitted when [syslog](https://en.wikipedia.org/wiki/Syslog) message is unpacked from the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field). The following query is equivalent to the previous one: