github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

lib/storage: optimize convert multiple values regexp filter to composite tag filter (#1610)

Browse source 
* lib/storage: optimize convert multiple values regexp filter to composite tag filter

* Apply suggestions from code review

Co-authored-by: Aliaksandr Valialkin <valyala@gmail.com>
This commit is contained in:
faceair 2021-09-14 17:47:07 +08:00 committed by Aliaksandr Valialkin
parent 5e5ce27df7
commit 61a51f7c15
2 changed files with 357 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
lib/storage/tag_filters.go
View file

@ -19,58 +19,86 @@ import (
// //
// This converts `foo{bar="baz",x=~"a.+"}` to `{foo=bar="baz",foo=x=~"a.+"} filter. // This converts `foo{bar="baz",x=~"a.+"}` to `{foo=bar="baz",foo=x=~"a.+"} filter.
func convertToCompositeTagFilterss(tfss []*TagFilters) []*TagFilters { func convertToCompositeTagFilterss(tfss []*TagFilters) []*TagFilters {
tfssNew := make([]*TagFilters, len(tfss)) tfssNew := make([]*TagFilters, 0, len(tfss))
for i, tfs := range tfss { for _, tfs := range tfss {
tfssNew[i] = convertToCompositeTagFilters(tfs) tfssNew = append(tfss, convertToCompositeTagFilters(tfs)...)
} }
return tfssNew return tfssNew
} }
func convertToCompositeTagFilters(tfs *TagFilters) *TagFilters { func convertToCompositeTagFilters(tfs *TagFilters) []*TagFilters {
// Search for metric name filter, which must be used for creating composite filters. tfssCompiled := make([]*TagFilters, 0)
var name []byte
// Search for filters on metric name, which will be used for creating composite filters.
var names [][]byte
hasPositiveFilter := false hasPositiveFilter := false
for _, tf := range tfs.tfs { for _, tf := range tfs.tfs {
if len(tf.key) == 0 && !tf.isNegative && !tf.isRegexp { if len(tf.key) == 0 && !tf.isNegative && !tf.isRegexp {
name = tf.value names = [][]byte{tf.value}
} else if len(tf.key) == 0 && !tf.isNegative && tf.isRegexp && len(tf.orSuffixes) > 0 {
// Split the filter {__name__=~"name1|...|nameN", other_filters}
// into `name1{other_filters}`, ..., `nameN{other_filters}`
// and generate composite filters for each of them
names = names[:0]
for _, orSuffix := range tf.orSuffixes {
names = append(names, []byte(orSuffix))
}
} else if !tf.isNegative && !tf.isEmptyMatch { } else if !tf.isNegative && !tf.isEmptyMatch {
hasPositiveFilter = true hasPositiveFilter = true
} }
} }
if len(name) == 0 { if len(names) == 0 {
tfssCompiled = append(tfssCompiled, tfs)
atomic.AddUint64(&compositeFilterMissingConversions, 1) atomic.AddUint64(&compositeFilterMissingConversions, 1)
return tfs return tfssCompiled
} }
tfsNew := make([]tagFilter, 0, len(tfs.tfs))
var compositeKey []byte var compositeKey []byte
compositeFilters := 0 compositeFilters := 0
for _, tf := range tfs.tfs { for _, name := range names {
if len(tf.key) == 0 { tfsNew := make([]tagFilter, 0, len(tfs.tfs))
if !hasPositiveFilter || tf.isNegative || tf.isRegexp || string(tf.value) != string(name) { for _, tf := range tfs.tfs {
tfsNew = append(tfsNew, tf) if len(tf.key) == 0 {
sameOrSuffixes := true
if len(names) != len(tf.orSuffixes) {
sameOrSuffixes = false
} else {
for i, orSuffix := range tf.orSuffixes {
if string(names[i]) != orSuffix {
sameOrSuffixes = false
break
}
}
}
if !hasPositiveFilter || tf.isNegative || tf.isRegexp && !sameOrSuffixes || !tf.isRegexp && string(tf.value) != string(name) {
tfsNew = append(tfsNew, tf)
}
continue
} }
continue if string(tf.key) == "__graphite__" || bytes.Equal(tf.key, graphiteReverseTagKey) {
tfsNew = append(tfsNew, tf)
continue
}
compositeKey = marshalCompositeTagKey(compositeKey[:0], name, tf.key)
var tfNew tagFilter
if err := tfNew.Init(tfs.commonPrefix, compositeKey, tf.value, tf.isNegative, tf.isRegexp); err != nil {
logger.Panicf("BUG: unexpected error when creating composite tag filter for name=%q and key=%q: %s", name, tf.key, err)
}
tfsNew = append(tfsNew, tfNew)
compositeFilters++
} }
if string(tf.key) == "__graphite__" || bytes.Equal(tf.key, graphiteReverseTagKey) { tfsCompiled := NewTagFilters(tfs.accountID, tfs.projectID)
tfsNew = append(tfsNew, tf) tfsCompiled.tfs = tfsNew
continue tfssCompiled = append(tfssCompiled, tfsCompiled)
}
compositeKey = marshalCompositeTagKey(compositeKey[:0], name, tf.key)
var tfNew tagFilter
if err := tfNew.Init(tfs.commonPrefix, compositeKey, tf.value, tf.isNegative, tf.isRegexp); err != nil {
logger.Panicf("BUG: unexpected error when creating composite tag filter for name=%q and key=%q: %s", name, tf.key, err)
}
tfsNew = append(tfsNew, tfNew)
compositeFilters++
} }
if compositeFilters == 0 { if compositeFilters == 0 {
tfssCompiled = append(tfssCompiled[:0], tfs)
atomic.AddUint64(&compositeFilterMissingConversions, 1) atomic.AddUint64(&compositeFilterMissingConversions, 1)
return tfs return tfssCompiled
} }
tfsCompiled := NewTagFilters(tfs.accountID, tfs.projectID)
tfsCompiled.tfs = tfsNew
atomic.AddUint64(&compositeFilterSuccessConversions, 1) atomic.AddUint64(&compositeFilterSuccessConversions, 1)
return tfsCompiled return tfssCompiled
} }
var ( var (

435
lib/storage/tag_filters_test.go
View file

@ -7,7 +7,7 @@ import (
) )
func TestConvertToCompositeTagFilters(t *testing.T) { func TestConvertToCompositeTagFilters(t *testing.T) {
f := func(tfs, resultExpected []TagFilter) { f := func(tfs []TagFilter, resultExpected [][]TagFilter) {
t.Helper() t.Helper()
accountID := uint32(123) accountID := uint32(123)
projectID := uint32(456) projectID := uint32(456)
@ -17,21 +17,25 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
t.Fatalf("cannot add tf=%s: %s", tf.String(), err) t.Fatalf("cannot add tf=%s: %s", tf.String(), err)
} }
} }
resultCompiled := convertToCompositeTagFilters(tfsCompiled) resultCompileds := convertToCompositeTagFilters(tfsCompiled)
if resultCompiled.accountID != accountID { result := make([][]TagFilter, len(resultCompileds))
t.Fatalf("unexpected accountID; got %d; want %d", resultCompiled.accountID, accountID) for i, resultCompiled := range resultCompileds {
} if resultCompiled.accountID != accountID {
if resultCompiled.projectID != projectID { t.Fatalf("unexpected accountID; got %d; want %d", resultCompiled.accountID, accountID)
t.Fatalf("unexpected projectID; got %d; want %d", resultCompiled.projectID, projectID)
}
result := make([]TagFilter, len(resultCompiled.tfs))
for i, tf := range resultCompiled.tfs {
result[i] = TagFilter{
Key: tf.key,
Value: tf.value,
IsNegative: tf.isNegative,
IsRegexp: tf.isRegexp,
} }
if resultCompiled.projectID != projectID {
t.Fatalf("unexpected projectID; got %d; want %d", resultCompiled.projectID, projectID)
}
tfs := make([]TagFilter, len(resultCompiled.tfs))
for i, tf := range resultCompiled.tfs {
tfs[i] = TagFilter{
Key: tf.key,
Value: tf.value,
IsNegative: tf.isNegative,
IsRegexp: tf.isRegexp,
}
}
result[i] = tfs
} }
if !reflect.DeepEqual(result, resultExpected) { if !reflect.DeepEqual(result, resultExpected) {
t.Fatalf("unexpected result;\ngot\n%+v\nwant\n%+v", result, resultExpected) t.Fatalf("unexpected result;\ngot\n%+v\nwant\n%+v", result, resultExpected)
@ -39,7 +43,7 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
} }
// Empty filters // Empty filters
f(nil, []TagFilter{}) f(nil, [][]TagFilter{{}})
// A single non-name filter // A single non-name filter
f([]TagFilter{ f([]TagFilter{
@ -49,12 +53,14 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("foo"), {
Value: []byte("bar"), Key: []byte("foo"),
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -72,18 +78,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: true, IsNegative: true,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("foo"), {
Value: []byte("bar"), Key: []byte("foo"),
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
}, IsRegexp: false,
{ },
Key: []byte("x"), {
Value: []byte("yy"), Key: []byte("x"),
IsNegative: true, Value: []byte("yy"),
IsRegexp: false, IsNegative: true,
IsRegexp: false,
},
}, },
}) })
@ -95,12 +103,14 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar"), Key: nil,
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -118,18 +128,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar"), Key: nil,
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
}, IsRegexp: false,
{ },
Key: nil, {
Value: []byte("baz"), Key: nil,
IsNegative: false, Value: []byte("baz"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -147,12 +159,14 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("\xfe\x03barfoo"), {
Value: []byte("abc"), Key: []byte("\xfe\x03barfoo"),
IsNegative: false, Value: []byte("abc"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -170,18 +184,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: true, IsNegative: true,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar"), Key: nil,
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
}, IsRegexp: false,
{ },
Key: []byte("\xfe\x03barfoo"), {
Value: []byte("abc"), Key: []byte("\xfe\x03barfoo"),
IsNegative: true, Value: []byte("abc"),
IsRegexp: false, IsNegative: true,
IsRegexp: false,
},
}, },
}) })
@ -205,18 +221,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: true, IsRegexp: true,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("\xfe\x03barfoo"), {
Value: []byte("abc"), Key: []byte("\xfe\x03barfoo"),
IsNegative: true, Value: []byte("abc"),
IsRegexp: false, IsNegative: true,
}, IsRegexp: false,
{ },
Key: []byte("\xfe\x03bara"), {
Value: []byte("b.+"), Key: []byte("\xfe\x03bara"),
IsNegative: false, Value: []byte("b.+"),
IsRegexp: true, IsNegative: false,
IsRegexp: true,
},
}, },
}) })
@ -240,18 +258,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar"), Key: nil,
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
}, IsRegexp: false,
{ },
Key: []byte("\xfe\x03bazfoo"), {
Value: []byte("abc"), Key: []byte("\xfe\x03bazfoo"),
IsNegative: false, Value: []byte("abc"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -269,12 +289,14 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: true, IsRegexp: true,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("\xfe\x03barfoo"), {
Value: []byte("abc"), Key: []byte("\xfe\x03barfoo"),
IsNegative: false, Value: []byte("abc"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -292,12 +314,14 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: true, IsRegexp: true,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("\xfe\x03barfoo"), {
Value: []byte("abc.+"), Key: []byte("\xfe\x03barfoo"),
IsNegative: false, Value: []byte("abc.+"),
IsRegexp: true, IsNegative: false,
IsRegexp: true,
},
}, },
}) })
@ -315,18 +339,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar"), Key: nil,
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
}, IsRegexp: false,
{ },
Key: []byte("__graphite__"), {
Value: []byte("foo.*.bar"), Key: []byte("__graphite__"),
IsNegative: false, Value: []byte("foo.*.bar"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -350,18 +376,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("\xfe\x03barfoo"), {
Value: []byte("abc"), Key: []byte("\xfe\x03barfoo"),
IsNegative: false, Value: []byte("abc"),
IsRegexp: false, IsNegative: false,
}, IsRegexp: false,
{ },
Key: []byte("__graphite__"), {
Value: []byte("foo.*.bar"), Key: []byte("__graphite__"),
IsNegative: false, Value: []byte("foo.*.bar"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
@ -379,13 +407,144 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: []byte("\xfe\x03barfoo"), {
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc"),
IsNegative: false,
IsRegexp: false,
},
},
})
// Multiple values regexp filter, which can be converted to non-regexp, with non-name filter.
f([]TagFilter{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("foo"),
Value: []byte("abc"), Value: []byte("abc"),
IsNegative: false, IsNegative: false,
IsRegexp: false, IsRegexp: false,
}, },
}, [][]TagFilter{
{
{
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc"),
IsNegative: false,
IsRegexp: false,
},
},
{
{
Key: []byte("\xfe\x03foofoo"),
Value: []byte("abc"),
IsNegative: false,
IsRegexp: false,
},
},
})
// Two multiple values regexp filter, which can be converted to non-regexp, with non-name filter.
f([]TagFilter{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: nil,
Value: []byte("abc|def"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("face"),
Value: []byte("air"),
IsNegative: false,
IsRegexp: false,
},
}, [][]TagFilter{
{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("\xfe\x03abcface"),
Value: []byte("air"),
IsNegative: false,
IsRegexp: false,
},
},
{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("\xfe\x03defface"),
Value: []byte("air"),
IsNegative: false,
IsRegexp: false,
},
},
})
// Multiple values regexp filter with a single negative filter
f([]TagFilter{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("foo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: false,
},
}, [][]TagFilter{
{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("\xfe\x03barfoo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: false,
},
},
{
{
Key: nil,
Value: []byte("bar|foo"),
IsNegative: false,
IsRegexp: true,
},
{
Key: []byte("\xfe\x03foofoo"),
Value: []byte("abc"),
IsNegative: true,
IsRegexp: false,
},
},
}) })
// Regexp name filter with non-name filter. // Regexp name filter with non-name filter.
@ -402,18 +561,20 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: true, IsNegative: true,
IsRegexp: false, IsRegexp: false,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar.+"), Key: nil,
IsNegative: false, Value: []byte("bar.+"),
IsRegexp: true, IsNegative: false,
}, IsRegexp: true,
{ },
Key: []byte("foo"), {
Value: []byte("abc"), Key: []byte("foo"),
IsNegative: true, Value: []byte("abc"),
IsRegexp: false, IsNegative: true,
IsRegexp: false,
},
}, },
}) })
@ -431,12 +592,14 @@ func TestConvertToCompositeTagFilters(t *testing.T) {
IsNegative: false, IsNegative: false,
IsRegexp: true, IsRegexp: true,
}, },
}, []TagFilter{ }, [][]TagFilter{
{ {
Key: nil, {
Value: []byte("bar"), Key: nil,
IsNegative: false, Value: []byte("bar"),
IsRegexp: false, IsNegative: false,
IsRegexp: false,
},
}, },
}) })
} }