From 790a9491b23d0001021886bc255d5027bb9f0bb2 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Thu, 9 May 2024 14:02:53 +0200
Subject: [PATCH] wip

---
 lib/logstorage/parser.go           | 12 +++++++-----
 lib/logstorage/parser_test.go      |  3 +--
 lib/logstorage/pipe_copy_test.go   |  1 -
 lib/logstorage/pipe_delete_test.go |  1 -
 lib/logstorage/pipe_fields_test.go |  1 -
 lib/logstorage/pipe_rename_test.go |  1 -
 lib/logstorage/pipe_sort_test.go   |  1 -
 lib/logstorage/pipe_stats_test.go  |  1 -
 8 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go
index 04c910e71..d842e59f2 100644
--- a/lib/logstorage/parser.go
+++ b/lib/logstorage/parser.go
@@ -39,12 +39,17 @@ type lexer struct {
 	currentTimestamp int64
 }
 
+// newLexer returns new lexer for the given s.
+//
+// The lex.token points to the first token in s.
 func newLexer(s string) *lexer {
-	return &lexer{
+	lex := &lexer{
 		s:                s,
 		sOrig:            s,
 		currentTimestamp: time.Now().UnixNano(),
 	}
+	lex.nextToken()
+	return lex
 }
 
 func (lex *lexer) isEnd() bool {
@@ -240,7 +245,7 @@ func ParseQuery(s string) (*Query, error) {
 }
 
 func parseFilter(lex *lexer) (filter, error) {
-	if !lex.mustNextToken() || lex.isKeyword("|") {
+	if lex.isKeyword("|", "") {
 		return nil, fmt.Errorf("missing query")
 	}
 	fo, err := parseFilterOr(lex, "")
@@ -1026,9 +1031,6 @@ func parseFilterStream(lex *lexer) (*filterStream, error) {
 
 func newStreamFilter(s string) (*StreamFilter, error) {
 	lex := newLexer(s)
-	if !lex.mustNextToken() {
-		return nil, fmt.Errorf("missing '{' in _stream filter")
-	}
 	fs, err := parseFilterStream(lex)
 	if err != nil {
 		return nil, err
diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go
index a78fdf46d..97bd39af5 100644
--- a/lib/logstorage/parser_test.go
+++ b/lib/logstorage/parser_test.go
@@ -12,12 +12,11 @@ func TestLexer(t *testing.T) {
 		t.Helper()
 		lex := newLexer(s)
 		for _, tokenExpected := range tokensExpected {
-			lex.nextToken()
 			if lex.token != tokenExpected {
 				t.Fatalf("unexpected token; got %q; want %q", lex.token, tokenExpected)
 			}
+			lex.nextToken()
 		}
-		lex.nextToken()
 		if lex.token != "" {
 			t.Fatalf("unexpected tail token: %q", lex.token)
 		}
diff --git a/lib/logstorage/pipe_copy_test.go b/lib/logstorage/pipe_copy_test.go
index e29aece0b..32b7e2d63 100644
--- a/lib/logstorage/pipe_copy_test.go
+++ b/lib/logstorage/pipe_copy_test.go
@@ -13,7 +13,6 @@ func TestPipeCopyUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeCopy(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)
diff --git a/lib/logstorage/pipe_delete_test.go b/lib/logstorage/pipe_delete_test.go
index 5124a5fcc..f3db305f5 100644
--- a/lib/logstorage/pipe_delete_test.go
+++ b/lib/logstorage/pipe_delete_test.go
@@ -12,7 +12,6 @@ func TestPipeDeleteUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeDelete(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)
diff --git a/lib/logstorage/pipe_fields_test.go b/lib/logstorage/pipe_fields_test.go
index ff020f4a2..0dcc7bf39 100644
--- a/lib/logstorage/pipe_fields_test.go
+++ b/lib/logstorage/pipe_fields_test.go
@@ -12,7 +12,6 @@ func TestPipeFieldsUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeFields(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)
diff --git a/lib/logstorage/pipe_rename_test.go b/lib/logstorage/pipe_rename_test.go
index 0e92f3605..eda949214 100644
--- a/lib/logstorage/pipe_rename_test.go
+++ b/lib/logstorage/pipe_rename_test.go
@@ -12,7 +12,6 @@ func TestPipeRenameUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeRename(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)
diff --git a/lib/logstorage/pipe_sort_test.go b/lib/logstorage/pipe_sort_test.go
index dd4bcebae..2111409f0 100644
--- a/lib/logstorage/pipe_sort_test.go
+++ b/lib/logstorage/pipe_sort_test.go
@@ -12,7 +12,6 @@ func TestPipeSortUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeSort(lex)
 		if err != nil {
 			t.Fatalf("cannot parse %s: %s", s, err)
diff --git a/lib/logstorage/pipe_stats_test.go b/lib/logstorage/pipe_stats_test.go
index 7bf66d083..ec2a08e22 100644
--- a/lib/logstorage/pipe_stats_test.go
+++ b/lib/logstorage/pipe_stats_test.go
@@ -12,7 +12,6 @@ func TestPipeStatsUpdateNeededFields(t *testing.T) {
 		unfs := newTestFieldsSet(unneededFields)
 
 		lex := newLexer(s)
-		lex.nextToken()
 		p, err := parsePipeStats(lex)
 		if err != nil {
 			t.Fatalf("unexpected error when parsing %s: %s", s, err)