wip

lib/logstorage/parser.go

@@ -320,10 +320,10 @@ func (q *Query) Optimize() {
 		switch t := p.(type) {
 		case *pipeStats:
 			for _, f := range t.funcs {
-				if f.iff != nil {
+				optimizeFilterIn(f.iff)
-					optimizeFilterIn(f.iff)
-				}
 			}
+		case *pipeExtract:
+			optimizeFilterIn(t.iff)
 		}
 	}
 }
@@ -345,6 +345,10 @@ func removeStarFilters(f filter) filter {
 }
 
 func optimizeFilterIn(f filter) {
+	if f == nil {
+		return
+	}
+
 	visitFunc := func(f filter) bool {
 		fi, ok := f.(*filterIn)
 		if ok && fi.q != nil {

lib/logstorage/pipe.go

@@ -65,7 +65,10 @@ func parsePipes(lex *lexer) ([]pipe, error) {
 	var pipes []pipe
 	for !lex.isKeyword(")", "") {
 		if !lex.isKeyword("|") {
-			return nil, fmt.Errorf("expecting '|'; got %q", lex.token)
+			if len(pipes) == 0 {
+				return nil, fmt.Errorf("expecting '|' after the query filters; got %q", lex.token)
+			}
+			return nil, fmt.Errorf("expecting '|' after [%s] pipe; got %q", pipes[len(pipes)-1], lex.token)
 		}
 		lex.nextToken()
 		p, err := parsePipe(lex)

lib/logstorage/pipe_extract.go

@@ -205,7 +205,7 @@ func parsePipeExtract(lex *lexer) (*pipeExtract, error) {
 	if lex.isKeyword("if") {
 		iff, err := parseIfFilter(lex)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse 'if' filter for %s: %w", pe, err)
+			return nil, err
 		}
 		pe.iff = iff
 

lib/logstorage/pipe_extract_test.go

@@ -4,6 +4,35 @@ import (
 	"testing"
 )
 
+func TestParsePipeExtractSuccess(t *testing.T) {
+	f := func(pipeStr string) {
+		t.Helper()
+		expectParsePipeSuccess(t, pipeStr)
+	}
+
+	f(`extract "foo<bar>"`)
+	f(`extract from x "foo<bar>"`)
+	f(`extract from x "foo<bar>" if (y:in(a:foo bar | uniq by (qwe) limit 10))`)
+}
+
+func TestParsePipeExtractFailure(t *testing.T) {
+	f := func(pipeStr string) {
+		t.Helper()
+		expectParsePipeFailure(t, pipeStr)
+	}
+
+	f(`extract`)
+	f(`extract from`)
+	f(`extract if (x:y)`)
+	f(`extract if (x:y) "a<b>"`)
+	f(`extract "a<b>" if`)
+	f(`extract "a<b>" if (foo`)
+	f(`extract "a<b>" if "foo"`)
+	f(`extract "a"`)
+	f(`extract "<a><b>"`)
+	f(`extract "<*>foo<_>bar"`)
+}
+
 func TestPipeExtract(t *testing.T) {
 	f := func(pipeStr string, rows, rowsExpected [][]Field) {
 		t.Helper()
@@ -211,3 +240,28 @@ func TestPipeExtractUpdateNeededFields(t *testing.T) {
 	f("extract from x '<foo>x<bar>'", "f2,foo,x,y", "", "f2,x,y", "")
 	f("extract from x '<foo>x<bar>' if (a:b foo:q)", "f2,foo,x,y", "", "a,f2,foo,x,y", "")
 }
+
+func expectParsePipeFailure(t *testing.T, pipeStr string) {
+	t.Helper()
+
+	lex := newLexer(pipeStr)
+	p, err := parsePipe(lex)
+	if err == nil {
+		t.Fatalf("expecting error when parsing [%s]; parsed result: [%s]", pipeStr, p)
+	}
+}
+
+func expectParsePipeSuccess(t *testing.T, pipeStr string) {
+	t.Helper()
+
+	lex := newLexer(pipeStr)
+	p, err := parsePipe(lex)
+	if err != nil {
+		t.Fatalf("cannot parse [%s]: %s", pipeStr, err)
+	}
+
+	pipeStrResult := p.String()
+	if pipeStrResult != pipeStr {
+		t.Fatalf("unexpected string representation of pipe; got\n%s\nwant\n%s", pipeStrResult, pipeStr)
+	}
+}

lib/logstorage/pipe_stats.go

@@ -526,7 +526,7 @@ func parsePipeStats(lex *lexer) (*pipeStats, error) {
 		if lex.isKeyword("if") {
 			iff, err := parseIfFilter(lex)
 			if err != nil {
-				return nil, fmt.Errorf("cannot parse 'if' filter for %s: %w", sf, err)
+				return nil, err
 			}
 			f.iff = iff
 
@@ -537,7 +537,7 @@ func parsePipeStats(lex *lexer) (*pipeStats, error) {
 
 		resultName, err := parseResultName(lex)
 		if err != nil {
-			return nil, fmt.Errorf("cannot parse result name for %s: %w", sf, err)
+			return nil, fmt.Errorf("cannot parse result name for [%s]: %w", sf, err)
 		}
 		f.resultName = resultName
 

lib/logstorage/storage_search.go

@@ -260,6 +260,9 @@ func (s *Storage) initFilterInValues(ctx context.Context, tenantIDs []TenantID,
 }
 
 func hasFilterInWithQueryForFilter(f filter) bool {
+	if f == nil {
+		return false
+	}
 	visitFunc := func(f filter) bool {
 		fi, ok := f.(*filterIn)
 		return ok && fi.needExecuteQuery
@@ -269,12 +272,15 @@ func hasFilterInWithQueryForFilter(f filter) bool {
 
 func hasFilterInWithQueryForPipes(pipes []pipe) bool {
 	for _, p := range pipes {
-		ps, ok := p.(*pipeStats)
+		switch t := p.(type) {
-		if !ok {
+		case *pipeStats:
-			continue
+			for _, f := range t.funcs {
-		}
+				if hasFilterInWithQueryForFilter(f.iff) {
-		for _, f := range ps.funcs {
+					return true
-			if f.iff != nil && hasFilterInWithQueryForFilter(f.iff) {
+				}
+			}
+		case *pipeExtract:
+			if hasFilterInWithQueryForFilter(t.iff) {
 				return true
 			}
 		}
@@ -333,6 +339,14 @@ func initFilterInValuesForPipes(cache map[string][]string, pipes []pipe, getFiel
 				byFields: t.byFields,
 				funcs:    funcsNew,
 			}
+		case *pipeExtract:
+			fNew, err := initFilterInValuesForFilter(cache, t.iff, getFieldValuesFunc)
+			if err != nil {
+				return nil, err
+			}
+			pe := *t
+			pe.iff = fNew
+			pipesNew[i] = &pe
 		default:
 			pipesNew[i] = p
 		}