diff --git a/lib/logstorage/pattern.go b/lib/logstorage/pattern.go index 27b597eee..0ae0ef286 100644 --- a/lib/logstorage/pattern.go +++ b/lib/logstorage/pattern.go @@ -31,11 +31,47 @@ type patternStep struct { field string } -func newPattern(steps []patternStep) *pattern { - if len(steps) == 0 { - logger.Panicf("BUG: steps cannot be empty") +func (ptn *pattern) clone() *pattern { + steps := ptn.steps + fields, matches := newFieldsAndMatchesFromPatternSteps(steps) + if len(fields) == 0 { + logger.Panicf("BUG: fields cannot be empty for steps=%v", steps) + } + return &pattern{ + steps: steps, + matches: matches, + fields: fields, + } +} + +func parsePattern(s string) (*pattern, error) { + steps, err := parsePatternSteps(s) + if err != nil { + return nil, err } + // Verify that prefixes are non-empty between fields. The first prefix may be empty. + for i := 1; i < len(steps); i++ { + if steps[i].prefix == "" { + return nil, fmt.Errorf("missing delimiter between <%s> and <%s>", steps[i-1].field, steps[i].field) + } + } + + // Build pattern struct + fields, matches := newFieldsAndMatchesFromPatternSteps(steps) + if len(fields) == 0 { + return nil, fmt.Errorf("pattern %q must contain at least a single named field in the form ", s) + } + + ptn := &pattern{ + steps: steps, + matches: matches, + fields: fields, + } + return ptn, nil +} + +func newFieldsAndMatchesFromPatternSteps(steps []patternStep) ([]patternField, []string) { matches := make([]string, len(steps)) var fields []patternField @@ -47,22 +83,14 @@ func newPattern(steps []patternStep) *pattern { }) } } - if len(fields) == 0 { - logger.Panicf("BUG: fields cannot be empty") - } - ef := &pattern{ - steps: steps, - matches: matches, - fields: fields, - } - return ef + return fields, matches } -func (ef *pattern) apply(s string) { - clear(ef.matches) +func (ptn *pattern) apply(s string) { + clear(ptn.matches) - steps := ef.steps + steps := ptn.steps if prefix := steps[0].prefix; prefix != "" { n := strings.Index(s, prefix) @@ -73,7 +101,7 @@ func (ef *pattern) apply(s string) { s = s[n+len(prefix):] } - matches := ef.matches + matches := ptn.matches for i := range steps { nextPrefix := "" if i+1 < len(steps) { @@ -126,13 +154,18 @@ func tryUnquoteString(s string) (string, int) { } func parsePatternSteps(s string) ([]patternStep, error) { - var steps []patternStep + if len(s) == 0 { + return nil, nil + } - hasNamedField := false + var steps []patternStep n := strings.IndexByte(s, '<') if n < 0 { - return nil, fmt.Errorf("missing <...> fields") + steps = append(steps, patternStep{ + prefix: s, + }) + return steps, nil } prefix := s[:n] s = s[n+1:] @@ -151,9 +184,6 @@ func parsePatternSteps(s string) ([]patternStep, error) { prefix: prefix, field: field, }) - if !hasNamedField && field != "" { - hasNamedField = true - } if len(s) == 0 { break } @@ -165,17 +195,10 @@ func parsePatternSteps(s string) ([]patternStep, error) { }) break } - if n == 0 { - return nil, fmt.Errorf("missing delimiter after <%s>", field) - } prefix = s[:n] s = s[n+1:] } - if !hasNamedField { - return nil, fmt.Errorf("missing named fields like ") - } - for i := range steps { step := &steps[i] step.prefix = html.UnescapeString(step.prefix) diff --git a/lib/logstorage/pattern_test.go b/lib/logstorage/pattern_test.go index 2bfa2d1d6..d7fe960b1 100644 --- a/lib/logstorage/pattern_test.go +++ b/lib/logstorage/pattern_test.go @@ -6,24 +6,32 @@ import ( ) func TestPatternApply(t *testing.T) { - f := func(pattern, s string, resultsExpected []string) { + f := func(patternStr, s string, resultsExpected []string) { t.Helper() - steps, err := parsePatternSteps(pattern) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - ef := newPattern(steps) - ef.apply(s) - - if len(ef.fields) != len(resultsExpected) { - t.Fatalf("unexpected number of results; got %d; want %d", len(ef.fields), len(resultsExpected)) - } - for i, f := range ef.fields { - if v := *f.value; v != resultsExpected[i] { - t.Fatalf("unexpected value for field %q; got %q; want %q", f.name, v, resultsExpected[i]) + checkFields := func(ptn *pattern) { + t.Helper() + if len(ptn.fields) != len(resultsExpected) { + t.Fatalf("unexpected number of results; got %d; want %d", len(ptn.fields), len(resultsExpected)) + } + for i, f := range ptn.fields { + if v := *f.value; v != resultsExpected[i] { + t.Fatalf("unexpected value for field %q; got %q; want %q", f.name, v, resultsExpected[i]) + } } } + + ptn, err := parsePattern(patternStr) + if err != nil { + t.Fatalf("cannot parse %q: %s", patternStr, err) + } + ptn.apply(s) + checkFields(ptn) + + // clone pattern and check fields again + ptnCopy := ptn.clone() + ptnCopy.apply(s) + checkFields(ptn) } f("", "", []string{""}) @@ -57,6 +65,30 @@ func TestPatternApply(t *testing.T) { f(`,"bar`, `"foo,\"bar"`, []string{`foo,"bar`}) } +func TestParsePatternFailure(t *testing.T) { + f := func(patternStr string) { + t.Helper() + + ptn, err := parsePattern(patternStr) + if err == nil { + t.Fatalf("expecting error when parsing %q; got %v", patternStr, ptn) + } + } + + // Missing named fields + f("") + f("foobar") + f("<>") + f("<>foo<>bar") + + // Missing delimiter between fields + f("") + f("abcdef") + f("abc") + f("abc<_>") + f("abc<_><_>") +} + func TestParsePatternStepsSuccess(t *testing.T) { f := func(s string, stepsExpected []patternStep) { t.Helper() @@ -70,6 +102,33 @@ func TestParsePatternStepsSuccess(t *testing.T) { } } + f("", nil) + + f("foobar", []patternStep{ + { + prefix: "foobar", + }, + }) + + f("<>", []patternStep{ + {}, + }) + + f("foo<>", []patternStep{ + { + prefix: "foo", + }, + }) + + f("", []patternStep{ + { + field: "foo", + }, + { + field: "bar", + }, + }) + f("", []patternStep{ { field: "foo", @@ -141,38 +200,19 @@ func TestParsePatternStepsSuccess(t *testing.T) { prefix: ">", }, }) + } func TestParsePatternStepsFailure(t *testing.T) { f := func(s string) { t.Helper() - _, err := parsePatternSteps(s) + steps, err := parsePatternSteps(s) if err == nil { - t.Fatalf("expecting non-nil error when parsing %q", s) + t.Fatalf("expecting non-nil error when parsing %q; got steps: %v", s, steps) } } - // empty string - f("") - - // zero fields - f("foobar") - - // Zero named fields - f("<>") - f("foo<>") - f("<>foo") - f("foo<_>bar<*>baz<>xxx") - - // missing delimiter between fields - f("") - f("<>") - f("<>") - f("bb<>aa") - f("aa") - f("aabb") - // missing > f("