wip

lib/logstorage/parser.go

@@ -39,12 +39,17 @@ type lexer struct {
 	currentTimestamp int64
 }
 
+// newLexer returns new lexer for the given s.
+//
+// The lex.token points to the first token in s.
 func newLexer(s string) *lexer {
-	return &lexer{
+	lex := &lexer{
 		s:                s,
 		sOrig:            s,
 		currentTimestamp: time.Now().UnixNano(),
 	}
+	lex.nextToken()
+	return lex
 }
 
 func (lex *lexer) isEnd() bool {
@@ -240,7 +245,7 @@ func ParseQuery(s string) (*Query, error) {
 }
 
 func parseFilter(lex *lexer) (filter, error) {
-	if !lex.mustNextToken() || lex.isKeyword("|") {
+	if lex.isKeyword("|", "") {
 		return nil, fmt.Errorf("missing query")
 	}
 	fo, err := parseFilterOr(lex, "")
@@ -1026,9 +1031,6 @@ func parseFilterStream(lex *lexer) (*filterStream, error) {
 
 func newStreamFilter(s string) (*StreamFilter, error) {
 	lex := newLexer(s)
-	if !lex.mustNextToken() {
-		return nil, fmt.Errorf("missing '{' in _stream filter")
-	}
 	fs, err := parseFilterStream(lex)
 	if err != nil {
 		return nil, err

lib/logstorage/parser_test.go

@@ -12,12 +12,11 @@ func TestLexer(t *testing.T) {
 		t.Helper()
 		lex := newLexer(s)
 		for _, tokenExpected := range tokensExpected {
-			lex.nextToken()
 			if lex.token != tokenExpected {
 				t.Fatalf("unexpected token; got %q; want %q", lex.token, tokenExpected)
 			}
+			lex.nextToken()
 		}
-		lex.nextToken()
 		if lex.token != "" {
 			t.Fatalf("unexpected tail token: %q", lex.token)
 		}

lib/logstorage/pipe_copy_test.go

@@ -13,7 +13,6 @@ func TestPipeCopyUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeCopy(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)

lib/logstorage/pipe_delete_test.go

@@ -12,7 +12,6 @@ func TestPipeDeleteUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeDelete(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)

lib/logstorage/pipe_fields_test.go

@@ -12,7 +12,6 @@ func TestPipeFieldsUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeFields(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)

lib/logstorage/pipe_rename_test.go

@@ -12,7 +12,6 @@ func TestPipeRenameUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeRename(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)

lib/logstorage/pipe_sort_test.go

@@ -12,7 +12,6 @@ func TestPipeSortUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeSort(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)

lib/logstorage/pipe_stats_test.go

@@ -12,7 +12,6 @@ func TestPipeStatsUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeStats(lex)
 		if err != nil {
 			t.Fatalf("unexpected error when parsing %s: %s", s, err)