From 7ac529c23516efee522046daf2a650fd057a7348 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 25 May 2024 22:59:13 +0200 Subject: [PATCH 1/4] lib/logstorage: work-in-progress --- docs/VictoriaLogs/LogsQL.md | 29 ++++++++++++++++++++++++++++- lib/logstorage/pipe_unroll.go | 7 ++++++- lib/regexutil/regex_test.go | 2 ++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index 74cec09c1..ec36f58ab 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -254,6 +254,7 @@ The list of LogsQL filters: - [Word filter](#word-filter) - matches logs with the given [word](#word) - [Phrase filter](#phrase-filter) - matches logs with the given phrase - [Prefix filter](#prefix-filter) - matches logs with the given word prefix or phrase prefix +- [Substring filter](#substring-filter) - matches logs with the given substring - [Empty value filter](#empty-value-filter) - matches logs without the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) - [Any value filter](#any-value-filter) - matches logs with the given non-empty [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) - [Exact filter](#exact-filter) - matches logs with the exact value @@ -490,7 +491,7 @@ This query matches the following [log messages](https://docs.victoriametrics.com This query doesn't match the following log messages: - `Error: foobar`, since the `Error` [word](#word) starts with capital letter. Use `i(err*)` for this case. See [these docs](#case-insensitive-filter) for details. -- `fooerror`, since the `fooerror` [word](#word) doesn't start with `err`. Use `~"err"` for this case. See [these docs](#regexp-filter) for details. +- `fooerror`, since the `fooerror` [word](#word) doesn't start with `err`. Use `~"err"` for this case. See [these docs](#substring-filter) for details. Prefix filter can be applied to [phrases](#phrase-filter). For example, the following query matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) containing phrases with `unexpected fail` prefix: @@ -549,6 +550,32 @@ See also: - [Logical filter](#logical-filter) +### Substring filter + +If it is needed to find logs with some substring, then `~"substring"` filter can be used. For example, the following query matches log entries, +which contain `ampl` text in the [`_msg` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field): + +```logsql +~"ampl" +``` + +It matches the following messages: + +- `Example message` +- `This is a sample` + +It doesn't match `EXAMPLE message`, since `AMPL` substring here is in uppercase. Use `~"(?i)ampl"` filter instead. Note that case-insensitive filter +may be much slower than case-sensitive one. + +Performance tip: prefer using [word filter](#word-filter) and [phrase filter](#phrase-filter), since substring filter may be quite slow. + +See also: + +- [Word filter](#word-filter) +- [Phrase filter](#phrase-filter) +- [Regexp filter](#regexp-filter) + + ### Empty value filter Sometimes it is needed to find log entries without the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). diff --git a/lib/logstorage/pipe_unroll.go b/lib/logstorage/pipe_unroll.go index e1e4e3be9..180b5687b 100644 --- a/lib/logstorage/pipe_unroll.go +++ b/lib/logstorage/pipe_unroll.go @@ -74,9 +74,10 @@ func (pu *pipeUnroll) updateNeededFields(neededFields, unneededFields fieldsSet) } } -func (pu *pipeUnroll) newPipeProcessor(workersCount int, _ <-chan struct{}, _ func(), ppNext pipeProcessor) pipeProcessor { +func (pu *pipeUnroll) newPipeProcessor(workersCount int, stopCh <-chan struct{}, _ func(), ppNext pipeProcessor) pipeProcessor { return &pipeUnrollProcessor{ pu: pu, + stopCh: stopCh, ppNext: ppNext, shards: make([]pipeUnrollProcessorShard, workersCount), @@ -85,6 +86,7 @@ func (pu *pipeUnroll) newPipeProcessor(workersCount int, _ <-chan struct{}, _ fu type pipeUnrollProcessor struct { pu *pipeUnroll + stopCh <-chan struct{} ppNext pipeProcessor shards []pipeUnrollProcessorShard @@ -139,6 +141,9 @@ func (pup *pipeUnrollProcessor) writeBlock(workerID uint, br *blockResult) { fields := shard.fields for rowIdx := range br.timestamps { if bm.isSetBit(rowIdx) { + if needStop(pup.stopCh) { + return + } shard.writeUnrolledFields(br, pu.fields, columnValues, rowIdx) } else { fields = fields[:0] diff --git a/lib/regexutil/regex_test.go b/lib/regexutil/regex_test.go index 2fedc2d81..a7208f962 100644 --- a/lib/regexutil/regex_test.go +++ b/lib/regexutil/regex_test.go @@ -165,6 +165,8 @@ func TestGetLiterals(t *testing.T) { f("foo.*bar(a|b)baz.+", []string{"foo", "bar", "baz"}) f("(foo[ab](?:bar))", []string{"foo", "bar"}) f("foo|bar", nil) + f("(?i)foo", nil) + f("foo((?i)bar)baz", []string{"foo", "baz"}) f("((foo|bar)baz xxx(?:yzabc))", []string{"baz xxxyzabc"}) f("((foo|bar)baz xxx(?:yzabc)*)", []string{"baz xxx"}) f("((foo|bar)baz? xxx(?:yzabc)*)", []string{"ba", " xxx"}) From 1e203f35f73cbdba10a1cfe0387050aee6554df6 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 26 May 2024 01:55:21 +0200 Subject: [PATCH 2/4] lib/logstorage: work-in-progress --- docs/VictoriaLogs/CHANGELOG.md | 10 +- docs/VictoriaLogs/LogsQL.md | 12 ++ lib/logstorage/parser.go | 14 +++ lib/logstorage/parser_test.go | 6 + lib/logstorage/pipe_filter.go | 2 +- lib/logstorage/pipe_stats.go | 10 +- lib/logstorage/pipe_utils_test.go | 14 ++- lib/logstorage/storage_search_test.go | 161 +++++++++++++++++++++++++- 8 files changed, 218 insertions(+), 11 deletions(-) diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 08ce42dee..b569f6195 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -19,6 +19,14 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +## [v0.12.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.12.1-victorialogs) + +Released at 2024-05-26 + +* FEATURE: add support for comments in multi-line LogsQL queries. See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#comments). + +* BUGFIX: properly apply [`in(...)` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter) inside `if (...)` conditions at various [pipes](https://docs.victoriametrics.com/victorialogs/logsql/#pipes). This bug has been introduced in [v0.12.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.12.0-victorialogs). + ## [v0.12.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.12.0-victorialogs) Released at 2024-05-26 @@ -31,7 +39,7 @@ Released at 2024-05-26 * BUGFIX: prevent from panic in [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) when VictoriaLogs runs on a system with one CPU core. * BUGFIX: do not return referenced fields if they weren't present in the original logs. For example, `_time:5m | format if (non_existing_field:"") "abc"` could return empty `non_exiting_field`, while it shuldn't be returned because it is missing in the original logs. -* BUGFIX: properly initialize values for [`in(...)` filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter) inside [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe) if the `in(...)` contains other [filters](https://docs.victoriametrics.com/victorialogs/logsql/#filters). For example, `_time:5m | filter ip:in(user_type:admin | fields ip)` now works correctly. +* BUGFIX: properly initialize values for [`in(...)` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter) inside [`filter` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#filter-pipe) if the `in(...)` contains other [filters](https://docs.victoriametrics.com/victorialogs/logsql/#filters). For example, `_time:5m | filter ip:in(user_type:admin | fields ip)` now works correctly. ## [v0.11.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.11.0-victorialogs) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index ec36f58ab..8b6e17467 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -2501,6 +2501,18 @@ LogsQL provides the following [pipes](#pipes) for limiting the number of returne Specific log fields can be queried via [`fields` pipe](#fields-pipe). +## Comments + +LogsQL query may contain comments at any place. The comment starts with `#` and continues until the end of the current line. +Example query with comments: + +```logsql +error # find logs with `error` word + | stats by (_stream) logs # then count the number of logs per `_stream` label + | sort by (logs) desc # then sort by the found logs in descending order + | limit 5 # and show top 5 streams with the biggest number of logs +``` + ## Numeric values LogsQL accepts numeric values in the following formats: diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index f613ea8c0..0220d9655 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -123,9 +123,12 @@ func (lex *lexer) nextToken() { lex.token = "" lex.rawToken = "" lex.isSkippedSpace = false + if len(s) == 0 { return } + +again: r, size := utf8.DecodeRuneInString(s) if r == utf8.RuneError { lex.nextCharToken(s, size) @@ -139,6 +142,17 @@ func (lex *lexer) nextToken() { r, size = utf8.DecodeRuneInString(s) } + if r == '#' { + // skip comment till \n + n := strings.IndexByte(s, '\n') + if n < 0 { + s = "" + } else { + s = s[n+1:] + } + goto again + } + // Try decoding simple token tokenLen := 0 for isTokenRune(r) || r == '.' { diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index 6bbd7e08d..4b80b66b8 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -1059,6 +1059,12 @@ func TestParseQuerySuccess(t *testing.T) { // multiple different pipes f(`* | fields foo, bar | limit 100 | stats by(foo,bar) count(baz) as qwert`, `* | fields foo, bar | limit 100 | stats by (foo, bar) count(baz) as qwert`) f(`* | skip 100 | head 20 | skip 10`, `* | offset 100 | limit 20 | offset 10`) + + // comments + f(`* # some comment | foo bar`, `*`) + f(`foo | # some comment | foo bar + fields x # another comment + |filter "foo#this#isn't a comment"#this is comment`, `foo | fields x | filter "foo#this#isn't a comment"`) } func TestParseQueryFailure(t *testing.T) { diff --git a/lib/logstorage/pipe_filter.go b/lib/logstorage/pipe_filter.go index ffc0b3f6c..daba8f5bc 100644 --- a/lib/logstorage/pipe_filter.go +++ b/lib/logstorage/pipe_filter.go @@ -43,7 +43,7 @@ func (pf *pipeFilter) initFilterInValues(cache map[string][]string, getFieldValu return nil, err } pfNew := *pf - pf.f = fNew + pfNew.f = fNew return &pfNew, nil } diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go index 77504dec2..90fdb0c73 100644 --- a/lib/logstorage/pipe_stats.go +++ b/lib/logstorage/pipe_stats.go @@ -133,16 +133,18 @@ func (ps *pipeStats) hasFilterInWithQuery() bool { func (ps *pipeStats) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { funcsNew := make([]pipeStatsFunc, len(ps.funcs)) - for i, f := range ps.funcs { + for i := range ps.funcs { + f := &ps.funcs[i] iffNew, err := f.iff.initFilterInValues(cache, getFieldValuesFunc) if err != nil { return nil, err } - f.iff = iffNew - funcsNew[i] = f + fNew := *f + fNew.iff = iffNew + funcsNew[i] = fNew } psNew := *ps - ps.funcs = funcsNew + psNew.funcs = funcsNew return &psNew, nil } diff --git a/lib/logstorage/pipe_utils_test.go b/lib/logstorage/pipe_utils_test.go index 7a6c6f7d2..af6c3b9fd 100644 --- a/lib/logstorage/pipe_utils_test.go +++ b/lib/logstorage/pipe_utils_test.go @@ -128,15 +128,21 @@ func (pp *testPipeProcessor) flush() error { func (pp *testPipeProcessor) expectRows(t *testing.T, expectedRows [][]Field) { t.Helper() - if len(pp.resultRows) != len(expectedRows) { + assertRowsEqual(t, pp.resultRows, expectedRows) +} + +func assertRowsEqual(t *testing.T, resultRows, expectedRows [][]Field) { + t.Helper() + + if len(resultRows) != len(expectedRows) { t.Fatalf("unexpected number of rows; got %d; want %d\nrows got\n%s\nrows expected\n%s", - len(pp.resultRows), len(expectedRows), rowsToString(pp.resultRows), rowsToString(expectedRows)) + len(resultRows), len(expectedRows), rowsToString(resultRows), rowsToString(expectedRows)) } - sortTestRows(pp.resultRows) + sortTestRows(resultRows) sortTestRows(expectedRows) - for i, resultRow := range pp.resultRows { + for i, resultRow := range resultRows { expectedRow := expectedRows[i] if len(resultRow) != len(expectedRow) { t.Fatalf("unexpected number of fields at row #%d; got %d; want %d\nrow got\n%s\nrow expected\n%s", diff --git a/lib/logstorage/storage_search_test.go b/lib/logstorage/storage_search_test.go index 9f9a43c49..f664a6f63 100644 --- a/lib/logstorage/storage_search_test.go +++ b/lib/logstorage/storage_search_test.go @@ -3,6 +3,8 @@ package logstorage import ( "context" "fmt" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -310,6 +312,163 @@ func TestStorageRunQuery(t *testing.T) { mustRunQuery(tenantIDs, q, writeBlock) }) + // Run more complex tests + f := func(t *testing.T, query string, rowsExpected [][]Field) { + t.Helper() + + q := mustParseQuery(query) + var resultRowsLock sync.Mutex + var resultRows [][]Field + writeBlock := func(_ uint, _ []int64, bcs []BlockColumn) { + if len(bcs) == 0 { + return + } + + for i := 0; i < len(bcs[0].Values); i++ { + row := make([]Field, len(bcs)) + for j, bc := range bcs { + row[j] = Field{ + Name: strings.Clone(bc.Name), + Value: strings.Clone(bc.Values[i]), + } + } + resultRowsLock.Lock() + resultRows = append(resultRows, row) + resultRowsLock.Unlock() + } + } + mustRunQuery(allTenantIDs, q, writeBlock) + + assertRowsEqual(t, resultRows, rowsExpected) + } + + t.Run("stats-count-total", func(t *testing.T) { + f(t, `* | stats count() rows`, [][]Field{ + { + {"rows", "1155"}, + }, + }) + }) + t.Run("in-filter-with-subquery-match", func(t *testing.T) { + f(t, `tenant.id:in(tenant.id:2 | fields tenant.id) | stats count() rows`, [][]Field{ + { + {"rows", "105"}, + }, + }) + }) + t.Run("in-filter-with-subquery-mismatch", func(t *testing.T) { + f(t, `tenant.id:in(tenant.id:23243 | fields tenant.id) | stats count() rows`, [][]Field{ + { + {"rows", "0"}, + }, + }) + }) + t.Run("conditional-stats", func(t *testing.T) { + f(t, `* | stats + count() rows_total, + count() if (stream-id:0) stream_0_rows, + count() if (stream-id:1123) stream_x_rows + `, [][]Field{ + { + {"rows_total", "1155"}, + {"stream_0_rows", "385"}, + {"stream_x_rows", "0"}, + }, + }) + }) + t.Run("in-filter-with-subquery-in-conditional-stats-mismatch", func(t *testing.T) { + f(t, `* | stats + count() rows_total, + count() if (tenant.id:in(tenant.id:3 | fields tenant.id)) rows_nonzero, + count() if (tenant.id:in(tenant.id:23243 | fields tenant.id)) rows_zero + `, [][]Field{ + { + {"rows_total", "1155"}, + {"rows_nonzero", "105"}, + {"rows_zero", "0"}, + }, + }) + }) + t.Run("pipe-extract", func(*testing.T) { + f(t, `* | extract "host-:" from instance | uniq (host) with hits | sort by (host)`, [][]Field{ + { + {"host", "0"}, + {"hits", "385"}, + }, + { + {"host", "1"}, + {"hits", "385"}, + }, + { + {"host", "2"}, + {"hits", "385"}, + }, + }) + }) + t.Run("pipe-extract-if-filter-with-subquery", func(*testing.T) { + f(t, `* | extract + if (tenant.id:in(tenant.id:(3 or 4) | fields tenant.id)) + "host-:" from instance + | filter host:~"1|2" + | uniq (tenant.id, host) with hits + | sort by (tenant.id, host)`, [][]Field{ + { + {"tenant.id", "{accountID=3,projectID=31}"}, + {"host", "1"}, + {"hits", "35"}, + }, + { + {"tenant.id", "{accountID=3,projectID=31}"}, + {"host", "2"}, + {"hits", "35"}, + }, + { + {"tenant.id", "{accountID=4,projectID=41}"}, + {"host", "1"}, + {"hits", "35"}, + }, + { + {"tenant.id", "{accountID=4,projectID=41}"}, + {"host", "2"}, + {"hits", "35"}, + }, + }) + }) + t.Run("pipe-extract-if-filter-with-subquery-non-empty-host", func(*testing.T) { + f(t, `* | extract + if (tenant.id:in(tenant.id:3 | fields tenant.id)) + "host-:" from instance + | filter host:* + | uniq (host) with hits + | sort by (host)`, [][]Field{ + { + {"host", "0"}, + {"hits", "35"}, + }, + { + {"host", "1"}, + {"hits", "35"}, + }, + { + {"host", "2"}, + {"hits", "35"}, + }, + }) + }) + t.Run("pipe-extract-if-filter-with-subquery-empty-host", func(*testing.T) { + f(t, `* | extract + if (tenant.id:in(tenant.id:3 | fields tenant.id)) + "host-:" from instance + | filter host:"" + | uniq (host) with hits + | sort by (host)`, [][]Field{ + { + {"host", ""}, + {"hits", "1050"}, + }, + }) + }) + // Close the storage and delete its data s.MustClose() fs.MustRemoveAll(path) @@ -318,7 +477,7 @@ func TestStorageRunQuery(t *testing.T) { func mustParseQuery(query string) *Query { q, err := ParseQuery(query) if err != nil { - panic(fmt.Errorf("BUG: cannot parse %s: %w", query, err)) + panic(fmt.Errorf("BUG: cannot parse [%s]: %w", query, err)) } return q } From 99138e15c094713cd696d7ab838cd0175998b1bb Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 26 May 2024 02:01:32 +0200 Subject: [PATCH 3/4] lib/logstorage: fix golangci-lint warnings --- lib/logstorage/pipe_copy.go | 2 +- lib/logstorage/pipe_delete.go | 2 +- lib/logstorage/pipe_field_names.go | 2 +- lib/logstorage/pipe_fields.go | 2 +- lib/logstorage/pipe_limit.go | 2 +- lib/logstorage/pipe_offset.go | 2 +- lib/logstorage/pipe_pack_json.go | 2 +- lib/logstorage/pipe_rename.go | 2 +- lib/logstorage/pipe_sort.go | 2 +- lib/logstorage/pipe_uniq.go | 2 +- lib/logstorage/pipe_unroll.go | 4 ++-- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/logstorage/pipe_copy.go b/lib/logstorage/pipe_copy.go index 340b1d97f..00d91edca 100644 --- a/lib/logstorage/pipe_copy.go +++ b/lib/logstorage/pipe_copy.go @@ -58,7 +58,7 @@ func (pc *pipeCopy) hasFilterInWithQuery() bool { return false } -func (pc *pipeCopy) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pc *pipeCopy) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pc, nil } diff --git a/lib/logstorage/pipe_delete.go b/lib/logstorage/pipe_delete.go index 0bf861076..9f2ca5a68 100644 --- a/lib/logstorage/pipe_delete.go +++ b/lib/logstorage/pipe_delete.go @@ -40,7 +40,7 @@ func (pd *pipeDelete) hasFilterInWithQuery() bool { return false } -func (pd *pipeDelete) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pd *pipeDelete) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pd, nil } diff --git a/lib/logstorage/pipe_field_names.go b/lib/logstorage/pipe_field_names.go index 3d4faf31a..1185eb75e 100644 --- a/lib/logstorage/pipe_field_names.go +++ b/lib/logstorage/pipe_field_names.go @@ -45,7 +45,7 @@ func (pf *pipeFieldNames) hasFilterInWithQuery() bool { return false } -func (pf *pipeFieldNames) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pf *pipeFieldNames) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pf, nil } diff --git a/lib/logstorage/pipe_fields.go b/lib/logstorage/pipe_fields.go index f0fb5873a..1724897df 100644 --- a/lib/logstorage/pipe_fields.go +++ b/lib/logstorage/pipe_fields.go @@ -57,7 +57,7 @@ func (pf *pipeFields) hasFilterInWithQuery() bool { return false } -func (pf *pipeFields) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pf *pipeFields) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pf, nil } diff --git a/lib/logstorage/pipe_limit.go b/lib/logstorage/pipe_limit.go index 2f4054393..f480cb46c 100644 --- a/lib/logstorage/pipe_limit.go +++ b/lib/logstorage/pipe_limit.go @@ -28,7 +28,7 @@ func (pl *pipeLimit) hasFilterInWithQuery() bool { return false } -func (pl *pipeLimit) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pl *pipeLimit) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pl, nil } diff --git a/lib/logstorage/pipe_offset.go b/lib/logstorage/pipe_offset.go index fc0363d6a..f5d07554c 100644 --- a/lib/logstorage/pipe_offset.go +++ b/lib/logstorage/pipe_offset.go @@ -28,7 +28,7 @@ func (po *pipeOffset) hasFilterInWithQuery() bool { return false } -func (po *pipeOffset) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (po *pipeOffset) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return po, nil } diff --git a/lib/logstorage/pipe_pack_json.go b/lib/logstorage/pipe_pack_json.go index ef647f6ec..d320fb6fb 100644 --- a/lib/logstorage/pipe_pack_json.go +++ b/lib/logstorage/pipe_pack_json.go @@ -42,7 +42,7 @@ func (pp *pipePackJSON) hasFilterInWithQuery() bool { return false } -func (pp *pipePackJSON) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pp *pipePackJSON) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pp, nil } diff --git a/lib/logstorage/pipe_rename.go b/lib/logstorage/pipe_rename.go index 44ded34ac..6911413fc 100644 --- a/lib/logstorage/pipe_rename.go +++ b/lib/logstorage/pipe_rename.go @@ -62,7 +62,7 @@ func (pr *pipeRename) hasFilterInWithQuery() bool { return false } -func (pr *pipeRename) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pr *pipeRename) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pr, nil } diff --git a/lib/logstorage/pipe_sort.go b/lib/logstorage/pipe_sort.go index da87c7a83..0b6d3cbbd 100644 --- a/lib/logstorage/pipe_sort.go +++ b/lib/logstorage/pipe_sort.go @@ -75,7 +75,7 @@ func (ps *pipeSort) hasFilterInWithQuery() bool { return false } -func (ps *pipeSort) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (ps *pipeSort) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return ps, nil } diff --git a/lib/logstorage/pipe_uniq.go b/lib/logstorage/pipe_uniq.go index c261aea05..c80c5c657 100644 --- a/lib/logstorage/pipe_uniq.go +++ b/lib/logstorage/pipe_uniq.go @@ -59,7 +59,7 @@ func (pu *pipeUniq) hasFilterInWithQuery() bool { return false } -func (pu *pipeUniq) initFilterInValues(cache map[string][]string, getFieldValuesFunc getFieldValuesFunc) (pipe, error) { +func (pu *pipeUniq) initFilterInValues(_ map[string][]string, _ getFieldValuesFunc) (pipe, error) { return pu, nil } diff --git a/lib/logstorage/pipe_unroll.go b/lib/logstorage/pipe_unroll.go index 180b5687b..2a2978f0b 100644 --- a/lib/logstorage/pipe_unroll.go +++ b/lib/logstorage/pipe_unroll.go @@ -144,7 +144,7 @@ func (pup *pipeUnrollProcessor) writeBlock(workerID uint, br *blockResult) { if needStop(pup.stopCh) { return } - shard.writeUnrolledFields(br, pu.fields, columnValues, rowIdx) + shard.writeUnrolledFields(pu.fields, columnValues, rowIdx) } else { fields = fields[:0] for i, f := range pu.fields { @@ -163,7 +163,7 @@ func (pup *pipeUnrollProcessor) writeBlock(workerID uint, br *blockResult) { shard.a.reset() } -func (shard *pipeUnrollProcessorShard) writeUnrolledFields(br *blockResult, fieldNames []string, columnValues [][]string, rowIdx int) { +func (shard *pipeUnrollProcessorShard) writeUnrolledFields(fieldNames []string, columnValues [][]string, rowIdx int) { // unroll values at rowIdx row shard.unrolledValues = slicesutil.SetLength(shard.unrolledValues, len(columnValues)) From 31e23c6f6fe026b2cdc38c6cb36a06b43ba60693 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 26 May 2024 02:07:28 +0200 Subject: [PATCH 4/4] deployment: update VictoriaLogs image from v0.12.0-victorialogs to v0.12.1-victorialogs See https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.12.1-victorialogs --- deployment/docker/docker-compose-victorialogs.yml | 2 +- .../docker/victorialogs/filebeat-docker/docker-compose.yml | 2 +- .../docker/victorialogs/filebeat-syslog/docker-compose.yml | 2 +- .../docker/victorialogs/fluentbit-docker/docker-compose.yml | 2 +- deployment/docker/victorialogs/logstash/docker-compose.yml | 2 +- deployment/docker/victorialogs/promtail/docker-compose.yml | 2 +- .../docker/victorialogs/vector-docker/docker-compose.yml | 2 +- deployment/logs-benchmark/docker-compose.yml | 2 +- docs/VictoriaLogs/QuickStart.md | 6 +++--- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/deployment/docker/docker-compose-victorialogs.yml b/deployment/docker/docker-compose-victorialogs.yml index a2fd3f17d..f3840bc0a 100644 --- a/deployment/docker/docker-compose-victorialogs.yml +++ b/deployment/docker/docker-compose-victorialogs.yml @@ -43,7 +43,7 @@ services: # storing logs and serving read queries. victorialogs: container_name: victorialogs - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs command: - "--storageDataPath=/vlogs" - "--httpListenAddr=:9428" diff --git a/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml b/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml index c6d7b6731..2e6d85dd4 100644 --- a/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat-docker/docker-compose.yml @@ -22,7 +22,7 @@ services: - -beat.uri=http://filebeat-victorialogs:5066 victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - victorialogs-filebeat-docker-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml b/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml index 6b09623fc..ad5e66002 100644 --- a/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml +++ b/deployment/docker/victorialogs/filebeat-syslog/docker-compose.yml @@ -13,7 +13,7 @@ services: - "5140:5140" victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - victorialogs-filebeat-syslog-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml index 8217d641e..27e622504 100644 --- a/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/fluentbit-docker/docker-compose.yml @@ -11,7 +11,7 @@ services: - "5140:5140" victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - victorialogs-fluentbit-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/logstash/docker-compose.yml b/deployment/docker/victorialogs/logstash/docker-compose.yml index 3849f8b72..2a4af75f0 100644 --- a/deployment/docker/victorialogs/logstash/docker-compose.yml +++ b/deployment/docker/victorialogs/logstash/docker-compose.yml @@ -14,7 +14,7 @@ services: - "5140:5140" victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - victorialogs-logstash-vl:/vlogs ports: diff --git a/deployment/docker/victorialogs/promtail/docker-compose.yml b/deployment/docker/victorialogs/promtail/docker-compose.yml index 821438218..39fdcec65 100644 --- a/deployment/docker/victorialogs/promtail/docker-compose.yml +++ b/deployment/docker/victorialogs/promtail/docker-compose.yml @@ -12,7 +12,7 @@ services: - "5140:5140" vlogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - victorialogs-promtail-docker:/vlogs ports: diff --git a/deployment/docker/victorialogs/vector-docker/docker-compose.yml b/deployment/docker/victorialogs/vector-docker/docker-compose.yml index 2c8f1f69b..9f1080a01 100644 --- a/deployment/docker/victorialogs/vector-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/vector-docker/docker-compose.yml @@ -22,7 +22,7 @@ services: condition: service_healthy victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - victorialogs-vector-docker-vl:/vlogs ports: diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml index 4020265f7..382e87e1f 100644 --- a/deployment/logs-benchmark/docker-compose.yml +++ b/deployment/logs-benchmark/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: # Run `make package-victoria-logs` to build victoria-logs image vlogs: - image: docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + image: docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs volumes: - vlogs:/vlogs ports: diff --git a/docs/VictoriaLogs/QuickStart.md b/docs/VictoriaLogs/QuickStart.md index 31e11131b..7dd2ecd15 100644 --- a/docs/VictoriaLogs/QuickStart.md +++ b/docs/VictoriaLogs/QuickStart.md @@ -34,8 +34,8 @@ Just download archive for the needed Operating system and architecture, unpack i For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it: ```sh -curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.12.0-victorialogs/victoria-logs-linux-amd64-v0.12.0-victorialogs.tar.gz -tar xzf victoria-logs-linux-amd64-v0.12.0-victorialogs.tar.gz +curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v0.12.1-victorialogs/victoria-logs-linux-amd64-v0.12.1-victorialogs.tar.gz +tar xzf victoria-logs-linux-amd64-v0.12.1-victorialogs.tar.gz ./victoria-logs-prod ``` @@ -59,7 +59,7 @@ Here is the command to run VictoriaLogs in a Docker container: ```sh docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \ - docker.io/victoriametrics/victoria-logs:v0.12.0-victorialogs + docker.io/victoriametrics/victoria-logs:v0.12.1-victorialogs ``` See also: