From 3da4b970d744782d1a865d25e9d631d982ffb9eb Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 17 May 2024 11:41:29 +0200 Subject: [PATCH] wip --- lib/logstorage/indexdb_test.go | 4 +- lib/logstorage/parser.go | 9 ---- lib/logstorage/parser_test.go | 45 ------------------ lib/logstorage/storage_search_test.go | 20 +++----- lib/logstorage/stream_filter_test.go | 68 +++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 70 deletions(-) create mode 100644 lib/logstorage/stream_filter_test.go diff --git a/lib/logstorage/indexdb_test.go b/lib/logstorage/indexdb_test.go index 2b3dbbfe8..513452f1a 100644 --- a/lib/logstorage/indexdb_test.go +++ b/lib/logstorage/indexdb_test.go @@ -50,7 +50,7 @@ func TestStorageSearchStreamIDs(t *testing.T) { f := func(filterStream string, expectedStreamIDs []streamID) { t.Helper() - sf := mustNewStreamFilter(filterStream) + sf := mustNewTestStreamFilter(filterStream) if expectedStreamIDs == nil { expectedStreamIDs = []streamID{} } @@ -68,7 +68,7 @@ func TestStorageSearchStreamIDs(t *testing.T) { AccountID: 1, ProjectID: 2, } - sf := mustNewStreamFilter(`{job="job-0",instance="instance-0"}`) + sf := mustNewTestStreamFilter(`{job="job-0",instance="instance-0"}`) for i := 0; i < 3; i++ { streamIDs := idb.searchStreamIDs([]TenantID{tenantID}, sf) if len(streamIDs) > 0 { diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 41a81fd3b..ddf3cb771 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -1111,15 +1111,6 @@ func parseFilterStream(lex *lexer) (*filterStream, error) { } } -func newStreamFilter(s string) (*StreamFilter, error) { - lex := newLexer(s) - fs, err := parseFilterStream(lex) - if err != nil { - return nil, err - } - return fs.f, nil -} - func parseAndStreamFilter(lex *lexer) (*andStreamFilter, error) { var filters []*streamTagFilter for { diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index f6c6f23a4..b7f52551a 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -34,51 +34,6 @@ func TestLexer(t *testing.T) { []string{"_stream", ":", "{", "foo", "=", "bar", ",", "a", "=~", "baz", ",", "b", "!=", "cd", ",", "d,}a", "!~", "abc", "}"}) } -func TestNewStreamFilterSuccess(t *testing.T) { - f := func(s, resultExpected string) { - t.Helper() - sf, err := newStreamFilter(s) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - result := sf.String() - if result != resultExpected { - t.Fatalf("unexpected StreamFilter; got %s; want %s", result, resultExpected) - } - } - - f("{}", "{}") - f(`{foo="bar"}`, `{foo="bar"}`) - f(`{ "foo" =~ "bar.+" , baz!="a" or x="y"}`, `{foo=~"bar.+",baz!="a" or x="y"}`) - f(`{"a b"='c}"d' OR de="aaa"}`, `{"a b"="c}\"d" or de="aaa"}`) - f(`{a="b", c="d" or x="y"}`, `{a="b",c="d" or x="y"}`) -} - -func TestNewStreamFilterFailure(t *testing.T) { - f := func(s string) { - t.Helper() - sf, err := newStreamFilter(s) - if err == nil { - t.Fatalf("expecting non-nil error") - } - if sf != nil { - t.Fatalf("expecting nil sf; got %v", sf) - } - } - - f("") - f("}") - f("{") - f("{foo") - f("{foo}") - f("{'foo") - f("{foo=") - f("{foo or bar}") - f("{foo=bar") - f("{foo=bar baz}") - f("{foo='bar' baz='x'}") -} - func TestParseTimeDuration(t *testing.T) { f := func(s string, durationExpected time.Duration) { t.Helper() diff --git a/lib/logstorage/storage_search_test.go b/lib/logstorage/storage_search_test.go index e83aad7d8..d2da8dea1 100644 --- a/lib/logstorage/storage_search_test.go +++ b/lib/logstorage/storage_search_test.go @@ -501,7 +501,7 @@ func TestStorageSearch(t *testing.T) { } }) t.Run("stream-filter-mismatch", func(_ *testing.T) { - sf := mustNewStreamFilter(`{job="foobar",instance=~"host-.+:2345"}`) + sf := mustNewTestStreamFilter(`{job="foobar",instance=~"host-.+:2345"}`) minTimestamp := baseTimestamp maxTimestamp := baseTimestamp + rowsPerBlock*1e9 + blocksPerStream f := getBaseFilter(minTimestamp, maxTimestamp, sf) @@ -517,7 +517,7 @@ func TestStorageSearch(t *testing.T) { }) t.Run("matching-stream-id", func(t *testing.T) { for i := 0; i < streamsPerTenant; i++ { - sf := mustNewStreamFilter(fmt.Sprintf(`{job="foobar",instance="host-%d:234"}`, i)) + sf := mustNewTestStreamFilter(fmt.Sprintf(`{job="foobar",instance="host-%d:234"}`, i)) tenantID := TenantID{ AccountID: 1, ProjectID: 11, @@ -543,7 +543,7 @@ func TestStorageSearch(t *testing.T) { } }) t.Run("matching-multiple-stream-ids", func(t *testing.T) { - sf := mustNewStreamFilter(`{job="foobar",instance=~"host-[^:]+:234"}`) + sf := mustNewTestStreamFilter(`{job="foobar",instance=~"host-[^:]+:234"}`) tenantID := TenantID{ AccountID: 1, ProjectID: 11, @@ -568,7 +568,7 @@ func TestStorageSearch(t *testing.T) { } }) t.Run("matching-multiple-stream-ids-with-re-filter", func(t *testing.T) { - sf := mustNewStreamFilter(`{job="foobar",instance=~"host-[^:]+:234"}`) + sf := mustNewTestStreamFilter(`{job="foobar",instance=~"host-[^:]+:234"}`) tenantID := TenantID{ AccountID: 1, ProjectID: 11, @@ -602,7 +602,7 @@ func TestStorageSearch(t *testing.T) { } }) t.Run("matching-stream-id-smaller-time-range", func(t *testing.T) { - sf := mustNewStreamFilter(`{job="foobar",instance="host-1:234"}`) + sf := mustNewTestStreamFilter(`{job="foobar",instance="host-1:234"}`) tenantID := TenantID{ AccountID: 1, ProjectID: 11, @@ -627,7 +627,7 @@ func TestStorageSearch(t *testing.T) { } }) t.Run("matching-stream-id-missing-time-range", func(_ *testing.T) { - sf := mustNewStreamFilter(`{job="foobar",instance="host-1:234"}`) + sf := mustNewTestStreamFilter(`{job="foobar",instance="host-1:234"}`) tenantID := TenantID{ AccountID: 1, ProjectID: 11, @@ -649,11 +649,3 @@ func TestStorageSearch(t *testing.T) { s.MustClose() fs.MustRemoveAll(path) } - -func mustNewStreamFilter(s string) *StreamFilter { - sf, err := newStreamFilter(s) - if err != nil { - panic(fmt.Errorf("unexpected error in newStreamFilter(%q): %w", s, err)) - } - return sf -} diff --git a/lib/logstorage/stream_filter_test.go b/lib/logstorage/stream_filter_test.go new file mode 100644 index 000000000..ed88bd1c6 --- /dev/null +++ b/lib/logstorage/stream_filter_test.go @@ -0,0 +1,68 @@ +package logstorage + +import ( + "fmt" + "testing" +) + +func TestNewTestStreamFilterSuccess(t *testing.T) { + f := func(s, resultExpected string) { + t.Helper() + sf, err := newTestStreamFilter(s) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + result := sf.String() + if result != resultExpected { + t.Fatalf("unexpected StreamFilter; got %s; want %s", result, resultExpected) + } + } + + f("{}", "{}") + f(`{foo="bar"}`, `{foo="bar"}`) + f(`{ "foo" =~ "bar.+" , baz!="a" or x="y"}`, `{foo=~"bar.+",baz!="a" or x="y"}`) + f(`{"a b"='c}"d' OR de="aaa"}`, `{"a b"="c}\"d" or de="aaa"}`) + f(`{a="b", c="d" or x="y"}`, `{a="b",c="d" or x="y"}`) +} + +func TestNewTestStreamFilterFailure(t *testing.T) { + f := func(s string) { + t.Helper() + sf, err := newTestStreamFilter(s) + if err == nil { + t.Fatalf("expecting non-nil error") + } + if sf != nil { + t.Fatalf("expecting nil sf; got %v", sf) + } + } + + f("") + f("}") + f("{") + f("{foo") + f("{foo}") + f("{'foo") + f("{foo=") + f("{foo or bar}") + f("{foo=bar") + f("{foo=bar baz}") + f("{foo='bar' baz='x'}") +} + +func mustNewTestStreamFilter(s string) *StreamFilter { + sf, err := newTestStreamFilter(s) + if err != nil { + panic(fmt.Errorf("unexpected error in newTestStreamFilter(%q): %w", s, err)) + } + return sf +} + +func newTestStreamFilter(s string) (*StreamFilter, error) { + lex := newLexer(s) + fs, err := parseFilterStream(lex) + if err != nil { + return nil, err + } + return fs.f, nil +}