mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-31 15:06:26 +00:00
wip
This commit is contained in:
parent
a8dde0aeac
commit
ae0a11d7c1
6 changed files with 88 additions and 13 deletions
|
@ -320,10 +320,10 @@ func (q *Query) Optimize() {
|
|||
switch t := p.(type) {
|
||||
case *pipeStats:
|
||||
for _, f := range t.funcs {
|
||||
if f.iff != nil {
|
||||
optimizeFilterIn(f.iff)
|
||||
}
|
||||
optimizeFilterIn(f.iff)
|
||||
}
|
||||
case *pipeExtract:
|
||||
optimizeFilterIn(t.iff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -345,6 +345,10 @@ func removeStarFilters(f filter) filter {
|
|||
}
|
||||
|
||||
func optimizeFilterIn(f filter) {
|
||||
if f == nil {
|
||||
return
|
||||
}
|
||||
|
||||
visitFunc := func(f filter) bool {
|
||||
fi, ok := f.(*filterIn)
|
||||
if ok && fi.q != nil {
|
||||
|
|
|
@ -65,7 +65,10 @@ func parsePipes(lex *lexer) ([]pipe, error) {
|
|||
var pipes []pipe
|
||||
for !lex.isKeyword(")", "") {
|
||||
if !lex.isKeyword("|") {
|
||||
return nil, fmt.Errorf("expecting '|'; got %q", lex.token)
|
||||
if len(pipes) == 0 {
|
||||
return nil, fmt.Errorf("expecting '|' after the query filters; got %q", lex.token)
|
||||
}
|
||||
return nil, fmt.Errorf("expecting '|' after [%s] pipe; got %q", pipes[len(pipes)-1], lex.token)
|
||||
}
|
||||
lex.nextToken()
|
||||
p, err := parsePipe(lex)
|
||||
|
|
|
@ -205,7 +205,7 @@ func parsePipeExtract(lex *lexer) (*pipeExtract, error) {
|
|||
if lex.isKeyword("if") {
|
||||
iff, err := parseIfFilter(lex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse 'if' filter for %s: %w", pe, err)
|
||||
return nil, err
|
||||
}
|
||||
pe.iff = iff
|
||||
|
||||
|
|
|
@ -4,6 +4,35 @@ import (
|
|||
"testing"
|
||||
)
|
||||
|
||||
func TestParsePipeExtractSuccess(t *testing.T) {
|
||||
f := func(pipeStr string) {
|
||||
t.Helper()
|
||||
expectParsePipeSuccess(t, pipeStr)
|
||||
}
|
||||
|
||||
f(`extract "foo<bar>"`)
|
||||
f(`extract from x "foo<bar>"`)
|
||||
f(`extract from x "foo<bar>" if (y:in(a:foo bar | uniq by (qwe) limit 10))`)
|
||||
}
|
||||
|
||||
func TestParsePipeExtractFailure(t *testing.T) {
|
||||
f := func(pipeStr string) {
|
||||
t.Helper()
|
||||
expectParsePipeFailure(t, pipeStr)
|
||||
}
|
||||
|
||||
f(`extract`)
|
||||
f(`extract from`)
|
||||
f(`extract if (x:y)`)
|
||||
f(`extract if (x:y) "a<b>"`)
|
||||
f(`extract "a<b>" if`)
|
||||
f(`extract "a<b>" if (foo`)
|
||||
f(`extract "a<b>" if "foo"`)
|
||||
f(`extract "a"`)
|
||||
f(`extract "<a><b>"`)
|
||||
f(`extract "<*>foo<_>bar"`)
|
||||
}
|
||||
|
||||
func TestPipeExtract(t *testing.T) {
|
||||
f := func(pipeStr string, rows, rowsExpected [][]Field) {
|
||||
t.Helper()
|
||||
|
@ -211,3 +240,28 @@ func TestPipeExtractUpdateNeededFields(t *testing.T) {
|
|||
f("extract from x '<foo>x<bar>'", "f2,foo,x,y", "", "f2,x,y", "")
|
||||
f("extract from x '<foo>x<bar>' if (a:b foo:q)", "f2,foo,x,y", "", "a,f2,foo,x,y", "")
|
||||
}
|
||||
|
||||
func expectParsePipeFailure(t *testing.T, pipeStr string) {
|
||||
t.Helper()
|
||||
|
||||
lex := newLexer(pipeStr)
|
||||
p, err := parsePipe(lex)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting error when parsing [%s]; parsed result: [%s]", pipeStr, p)
|
||||
}
|
||||
}
|
||||
|
||||
func expectParsePipeSuccess(t *testing.T, pipeStr string) {
|
||||
t.Helper()
|
||||
|
||||
lex := newLexer(pipeStr)
|
||||
p, err := parsePipe(lex)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot parse [%s]: %s", pipeStr, err)
|
||||
}
|
||||
|
||||
pipeStrResult := p.String()
|
||||
if pipeStrResult != pipeStr {
|
||||
t.Fatalf("unexpected string representation of pipe; got\n%s\nwant\n%s", pipeStrResult, pipeStr)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -526,7 +526,7 @@ func parsePipeStats(lex *lexer) (*pipeStats, error) {
|
|||
if lex.isKeyword("if") {
|
||||
iff, err := parseIfFilter(lex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse 'if' filter for %s: %w", sf, err)
|
||||
return nil, err
|
||||
}
|
||||
f.iff = iff
|
||||
|
||||
|
@ -537,7 +537,7 @@ func parsePipeStats(lex *lexer) (*pipeStats, error) {
|
|||
|
||||
resultName, err := parseResultName(lex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse result name for %s: %w", sf, err)
|
||||
return nil, fmt.Errorf("cannot parse result name for [%s]: %w", sf, err)
|
||||
}
|
||||
f.resultName = resultName
|
||||
|
||||
|
|
|
@ -260,6 +260,9 @@ func (s *Storage) initFilterInValues(ctx context.Context, tenantIDs []TenantID,
|
|||
}
|
||||
|
||||
func hasFilterInWithQueryForFilter(f filter) bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
visitFunc := func(f filter) bool {
|
||||
fi, ok := f.(*filterIn)
|
||||
return ok && fi.needExecuteQuery
|
||||
|
@ -269,12 +272,15 @@ func hasFilterInWithQueryForFilter(f filter) bool {
|
|||
|
||||
func hasFilterInWithQueryForPipes(pipes []pipe) bool {
|
||||
for _, p := range pipes {
|
||||
ps, ok := p.(*pipeStats)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, f := range ps.funcs {
|
||||
if f.iff != nil && hasFilterInWithQueryForFilter(f.iff) {
|
||||
switch t := p.(type) {
|
||||
case *pipeStats:
|
||||
for _, f := range t.funcs {
|
||||
if hasFilterInWithQueryForFilter(f.iff) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case *pipeExtract:
|
||||
if hasFilterInWithQueryForFilter(t.iff) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
@ -333,6 +339,14 @@ func initFilterInValuesForPipes(cache map[string][]string, pipes []pipe, getFiel
|
|||
byFields: t.byFields,
|
||||
funcs: funcsNew,
|
||||
}
|
||||
case *pipeExtract:
|
||||
fNew, err := initFilterInValuesForFilter(cache, t.iff, getFieldValuesFunc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pe := *t
|
||||
pe.iff = fNew
|
||||
pipesNew[i] = &pe
|
||||
default:
|
||||
pipesNew[i] = p
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue