From 9c4423c1dbe10d6b005322bfdff10d2120b82d49 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 7 May 2024 23:38:09 +0200 Subject: [PATCH] wip --- docs/VictoriaLogs/LogsQL.md | 10 +++--- lib/logstorage/parser_test.go | 18 +++++----- lib/logstorage/pipe_stats.go | 6 ++-- ...ats_uniq_array.go => stats_uniq_values.go} | 36 +++++++++---------- 4 files changed, 35 insertions(+), 35 deletions(-) rename lib/logstorage/{stats_uniq_array.go => stats_uniq_values.go} (79%) diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index bfd4b0c59..af800a98b 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -1327,7 +1327,7 @@ LogsQL supports the following functions for [`stats` pipe](#stats-pipe): - [`min`](#min-stats) calculates the minumum value over the given numeric [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). - [`sum`](#sum-stats) calculates the sum for the given numeric [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). - [`uniq`](#uniq-stats) calculates the number of unique non-empty values for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). -- [`uniq_array`](#uniq_array-stats) returns unique non-empty values for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). +- [`uniq_values`](#uniq_values-stats) returns unique non-empty values for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). ### avg stats @@ -1475,12 +1475,12 @@ _time:5m | stats uniq(host, path) unique_host_path_pairs See also: -- [`uniq_array`](#uniq_array-stats) +- [`uniq_values`](#uniq_values-stats) - [`count`](#count-stats) -### uniq_array stats +### uniq_values stats -`uniq_array(field1, ..., fieldN)` [stats pipe](#stats-pipe) returns the unique non-empty values across +`uniq_values(field1, ..., fieldN)` [stats pipe](#stats-pipe) returns the unique non-empty values across the mentioned [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). The returned values are sorted and encoded in JSON array. @@ -1488,7 +1488,7 @@ For example, the following query returns unique non-empty values for the `ip` [f over logs for the last 5 minutes: ```logsql -_time:5m | stats uniq_array(ip) unique_ips +_time:5m | stats uniq_values(ip) unique_ips ``` See also: diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index e5eb544cd..06658344a 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -911,12 +911,12 @@ func TestParseQuerySuccess(t *testing.T) { f(`* | stats by(x) uniq() z`, `* | stats by (x) uniq(*) as z`) f(`* | stats by(x) uniq(a,*,b) z`, `* | stats by (x) uniq(*) as z`) - // stats pipe uniq_array - f(`* | stats uniq_array(foo) bar`, `* | stats uniq_array(foo) as bar`) - f(`* | stats by(x, y) uniq_array(foo, bar) as baz`, `* | stats by (x, y) uniq_array(foo, bar) as baz`) - f(`* | stats by(x) uniq_array(*) y`, `* | stats by (x) uniq_array(*) as y`) - f(`* | stats by(x) uniq_array() y`, `* | stats by (x) uniq_array(*) as y`) - f(`* | stats by(x) uniq_array(a,*,b) y`, `* | stats by (x) uniq_array(*) as y`) + // stats pipe uniq_values + f(`* | stats uniq_values(foo) bar`, `* | stats uniq_values(foo) as bar`) + f(`* | stats by(x, y) uniq_values(foo, bar) as baz`, `* | stats by (x, y) uniq_values(foo, bar) as baz`) + f(`* | stats by(x) uniq_values(*) y`, `* | stats by (x) uniq_values(*) as y`) + f(`* | stats by(x) uniq_values() y`, `* | stats by (x) uniq_values(*) as y`) + f(`* | stats by(x) uniq_values(a,*,b) y`, `* | stats by (x) uniq_values(*) as y`) // stats pipe multiple funcs f(`* | stats count() "foo.bar:baz", uniq(a) bar`, `* | stats count(*) as "foo.bar:baz", uniq(a) as bar`) @@ -1228,9 +1228,9 @@ func TestParseQueryFailure(t *testing.T) { f(`foo | stats uniq`) f(`foo | stats uniq()`) - // invalid stats uniq_array - f(`foo | stats uniq_array`) - f(`foo | stats uniq_array()`) + // invalid stats uniq_values + f(`foo | stats uniq_values`) + f(`foo | stats uniq_values()`) // invalid stats grouping fields f(`foo | stats by(foo:bar) count() baz`) diff --git a/lib/logstorage/pipe_stats.go b/lib/logstorage/pipe_stats.go index 28d1c4044..0288ac2ac 100644 --- a/lib/logstorage/pipe_stats.go +++ b/lib/logstorage/pipe_stats.go @@ -506,10 +506,10 @@ func parseStatsFunc(lex *lexer) (statsFunc, string, error) { return nil, "", fmt.Errorf("cannot parse 'avg' func: %w", err) } sf = sas - case lex.isKeyword("uniq_array"): - sus, err := parseStatsUniqArray(lex) + case lex.isKeyword("uniq_values"): + sus, err := parseStatsUniqValues(lex) if err != nil { - return nil, "", fmt.Errorf("cannot parse 'uniq_array' func: %w", err) + return nil, "", fmt.Errorf("cannot parse 'uniq_values' func: %w", err) } sf = sus default: diff --git a/lib/logstorage/stats_uniq_array.go b/lib/logstorage/stats_uniq_values.go similarity index 79% rename from lib/logstorage/stats_uniq_array.go rename to lib/logstorage/stats_uniq_values.go index 7b4423572..6542bb143 100644 --- a/lib/logstorage/stats_uniq_array.go +++ b/lib/logstorage/stats_uniq_values.go @@ -10,21 +10,21 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" ) -type statsUniqArray struct { +type statsUniqValues struct { fields []string containsStar bool } -func (su *statsUniqArray) String() string { - return "uniq_array(" + fieldNamesString(su.fields) + ")" +func (su *statsUniqValues) String() string { + return "uniq_values(" + fieldNamesString(su.fields) + ")" } -func (su *statsUniqArray) neededFields() []string { +func (su *statsUniqValues) neededFields() []string { return su.fields } -func (su *statsUniqArray) newStatsProcessor() (statsProcessor, int) { - sup := &statsUniqArrayProcessor{ +func (su *statsUniqValues) newStatsProcessor() (statsProcessor, int) { + sup := &statsUniqValuesProcessor{ su: su, m: make(map[string]struct{}), @@ -32,13 +32,13 @@ func (su *statsUniqArray) newStatsProcessor() (statsProcessor, int) { return sup, int(unsafe.Sizeof(*sup)) } -type statsUniqArrayProcessor struct { - su *statsUniqArray +type statsUniqValuesProcessor struct { + su *statsUniqValues m map[string]struct{} } -func (sup *statsUniqArrayProcessor) updateStatsForAllRows(br *blockResult) int { +func (sup *statsUniqValuesProcessor) updateStatsForAllRows(br *blockResult) int { stateSizeIncrease := 0 if sup.su.containsStar { columns := br.getColumns() @@ -54,7 +54,7 @@ func (sup *statsUniqArrayProcessor) updateStatsForAllRows(br *blockResult) int { return stateSizeIncrease } -func (sup *statsUniqArrayProcessor) updateStatsForAllRowsColumn(c *blockResultColumn, br *blockResult) int { +func (sup *statsUniqValuesProcessor) updateStatsForAllRowsColumn(c *blockResultColumn, br *blockResult) int { m := sup.m stateSizeIncrease := 0 if c.isConst { @@ -107,7 +107,7 @@ func (sup *statsUniqArrayProcessor) updateStatsForAllRowsColumn(c *blockResultCo return stateSizeIncrease } -func (sup *statsUniqArrayProcessor) updateStatsForRow(br *blockResult, rowIdx int) int { +func (sup *statsUniqValuesProcessor) updateStatsForRow(br *blockResult, rowIdx int) int { stateSizeIncrease := 0 if sup.su.containsStar { columns := br.getColumns() @@ -123,7 +123,7 @@ func (sup *statsUniqArrayProcessor) updateStatsForRow(br *blockResult, rowIdx in return stateSizeIncrease } -func (sup *statsUniqArrayProcessor) updateStatsForRowColumn(c *blockResultColumn, br *blockResult, rowIdx int) int { +func (sup *statsUniqValuesProcessor) updateStatsForRowColumn(c *blockResultColumn, br *blockResult, rowIdx int) int { m := sup.m stateSizeIncrease := 0 if c.isConst { @@ -170,8 +170,8 @@ func (sup *statsUniqArrayProcessor) updateStatsForRowColumn(c *blockResultColumn return stateSizeIncrease } -func (sup *statsUniqArrayProcessor) mergeState(sfp statsProcessor) { - src := sfp.(*statsUniqArrayProcessor) +func (sup *statsUniqValuesProcessor) mergeState(sfp statsProcessor) { + src := sfp.(*statsUniqValuesProcessor) m := sup.m for k := range src.m { if _, ok := m[k]; !ok { @@ -180,7 +180,7 @@ func (sup *statsUniqArrayProcessor) mergeState(sfp statsProcessor) { } } -func (sup *statsUniqArrayProcessor) finalizeStats() string { +func (sup *statsUniqValuesProcessor) finalizeStats() string { if len(sup.m) == 0 { return "[]" } @@ -214,12 +214,12 @@ func (sup *statsUniqArrayProcessor) finalizeStats() string { return bytesutil.ToUnsafeString(b) } -func parseStatsUniqArray(lex *lexer) (*statsUniqArray, error) { - fields, err := parseFieldNamesForStatsFunc(lex, "uniq_array") +func parseStatsUniqValues(lex *lexer) (*statsUniqValues, error) { + fields, err := parseFieldNamesForStatsFunc(lex, "uniq_values") if err != nil { return nil, err } - su := &statsUniqArray{ + su := &statsUniqValues{ fields: fields, containsStar: slices.Contains(fields, "*"), }