diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go index d7e23b234..9eff009bc 100644 --- a/app/vmalert/alerting.go +++ b/app/vmalert/alerting.go @@ -19,6 +19,7 @@ import ( // AlertingRule is basic alert entity type AlertingRule struct { + Type datasource.Type RuleID uint64 Name string Expr string @@ -50,6 +51,7 @@ type alertingRuleMetrics struct { func newAlertingRule(group *Group, cfg config.Rule) *AlertingRule { ar := &AlertingRule{ + Type: cfg.Type, RuleID: cfg.ID, Name: cfg.Alert, Expr: cfg.Expr, @@ -120,7 +122,7 @@ func (ar *AlertingRule) ID() uint64 { // Exec executes AlertingRule expression via the given Querier. // Based on the Querier results AlertingRule maintains notifier.Alerts func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series bool) ([]prompbmarshal.TimeSeries, error) { - qMetrics, err := q.Query(ctx, ar.Expr) + qMetrics, err := q.Query(ctx, ar.Expr, ar.Type) ar.mu.Lock() defer ar.mu.Unlock() @@ -137,7 +139,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, q datasource.Querier, series b } } - qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query) } + qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query, ar.Type) } updated := make(map[uint64]struct{}) // update list of active alerts for _, m := range qMetrics { @@ -310,6 +312,7 @@ func (ar *AlertingRule) RuleAPI() APIAlertingRule { // encode as strings to avoid rounding ID: fmt.Sprintf("%d", ar.ID()), GroupID: fmt.Sprintf("%d", ar.GroupID), + Type: ar.Type.String(), Name: ar.Name, Expression: ar.Expr, For: ar.For.String(), @@ -404,7 +407,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb return fmt.Errorf("querier is nil") } - qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query) } + qFn := func(query string) ([]datasource.Metric, error) { return q.Query(ctx, query, ar.Type) } // account for external labels in filter var labelsFilter string @@ -417,7 +420,7 @@ func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookb // remote write protocol which is used for state persistence in vmalert. expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])", alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds())) - qMetrics, err := q.Query(ctx, expr) + qMetrics, err := q.Query(ctx, expr, ar.Type) if err != nil { return err } diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index da998ca5c..f639febed 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" @@ -21,6 +23,7 @@ import ( // Group contains list of Rules grouped into // entity with one name and evaluation interval type Group struct { + Type datasource.Type `yaml:"type,omitempty"` File string Name string `yaml:"name"` Interval time.Duration `yaml:"interval,omitempty"` @@ -44,6 +47,19 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error { if err != nil { return fmt.Errorf("failed to marshal group configuration for checksum: %w", err) } + // change default value to prometheus datasource. + if g.Type.Get() == "" { + g.Type.Set(datasource.NewPrometheusType()) + } + // update rules with empty type. + for i, r := range g.Rules { + if r.Type.Get() == "" { + r.Type.Set(g.Type) + r.ID = HashRule(r) + g.Rules[i] = r + } + } + h := md5.New() h.Write(b) g.Checksum = fmt.Sprintf("%x", h.Sum(nil)) @@ -58,6 +74,7 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error { if len(g.Rules) == 0 { return fmt.Errorf("group %q can't contain no rules", g.Name) } + uniqueRules := map[uint64]struct{}{} for _, r := range g.Rules { ruleName := r.Record @@ -72,7 +89,13 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error { return fmt.Errorf("invalid rule %q.%q: %w", g.Name, ruleName, err) } if validateExpressions { - if _, err := metricsql.Parse(r.Expr); err != nil { + // its needed only for tests. + // because correct types must be inherited after unmarshalling. + exprValidator := g.Type.ValidateExpr + if r.Type.Get() != "" { + exprValidator = r.Type.ValidateExpr + } + if err := exprValidator(r.Expr); err != nil { return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err) } } @@ -92,6 +115,7 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error { // recording rule or alerting rule. type Rule struct { ID uint64 + Type datasource.Type `yaml:"type,omitempty"` Record string `yaml:"record,omitempty"` Alert string `yaml:"alert,omitempty"` Expr string `yaml:"expr"` @@ -169,6 +193,7 @@ func HashRule(r Rule) uint64 { h.Write([]byte("alerting")) h.Write([]byte(r.Alert)) } + h.Write([]byte(r.Type.Get())) kv := sortMap(r.Labels) for _, i := range kv { h.Write([]byte(i.key)) diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 13dd21731..1cdaa3b9a 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" + "gopkg.in/yaml.v2" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" @@ -53,6 +55,10 @@ func TestParseBad(t *testing.T) { []string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set", }, + { + []string{"testdata/rules1-bad.rules"}, + "bad graphite expr", + }, } for _, tc := range testCases { _, err := Parse(tc.path, true, true) @@ -215,6 +221,75 @@ func TestGroup_Validate(t *testing.T) { }, expErr: "", }, + { + group: &Group{Name: "test thanos", + Type: datasource.NewRawType("thanos"), + Rules: []Rule{ + {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ + "description": "{{ value|query }}", + }}, + }, + }, + validateExpressions: true, + expErr: "unknown datasource type", + }, + { + group: &Group{Name: "test graphite", + Type: datasource.NewGraphiteType(), + Rules: []Rule{ + {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ + "description": "some-description", + }}, + }, + }, + validateExpressions: true, + expErr: "", + }, + { + group: &Group{Name: "test prometheus", + Type: datasource.NewPrometheusType(), + Rules: []Rule{ + {Alert: "alert", Expr: "up == 1", Labels: map[string]string{ + "description": "{{ value|query }}", + }}, + }, + }, + validateExpressions: true, + expErr: "", + }, + { + group: &Group{ + Name: "test graphite inherit", + Type: datasource.NewGraphiteType(), + Rules: []Rule{ + { + Expr: "sumSeries(time('foo.bar',10))", + For: PromDuration{milliseconds: 10}, + }, + { + Expr: "sum(up == 0 ) by (host)", + Type: datasource.NewPrometheusType(), + }, + }, + }, + }, + { + group: &Group{ + Name: "test graphite prometheus bad expr", + Type: datasource.NewGraphiteType(), + Rules: []Rule{ + { + Expr: "sum(up == 0 ) by (host)", + For: PromDuration{milliseconds: 10}, + }, + { + Expr: "sumSeries(time('foo.bar',10))", + Type: datasource.NewPrometheusType(), + }, + }, + }, + expErr: "invalid rule", + }, } for _, tc := range testCases { err := tc.group.Validate(tc.validateAnnotations, tc.validateExpressions) diff --git a/app/vmalert/config/testdata/dir/rules-update0-good.rules b/app/vmalert/config/testdata/dir/rules-update0-good.rules new file mode 100644 index 000000000..ffbfdf30f --- /dev/null +++ b/app/vmalert/config/testdata/dir/rules-update0-good.rules @@ -0,0 +1,13 @@ +groups: + - name: TestUpdateGroup + interval: 2s + concurrency: 2 + type: prometheus + rules: + - alert: up + expr: up == 0 + for: 30s + - alert: up graphite + expr: filterSeries(time('host.1',20),'>','0') + for: 30s + type: graphite diff --git a/app/vmalert/config/testdata/dir/rules-update1-good.rules b/app/vmalert/config/testdata/dir/rules-update1-good.rules new file mode 100644 index 000000000..0fcad7687 --- /dev/null +++ b/app/vmalert/config/testdata/dir/rules-update1-good.rules @@ -0,0 +1,12 @@ +groups: + - name: TestUpdateGroup + interval: 30s + type: graphite + rules: + - alert: up + expr: filterSeries(time('host.2',20),'>','0') + for: 30s + - alert: up graphite + expr: filterSeries(time('host.1',20),'>','0') + for: 30s + type: graphite diff --git a/app/vmalert/config/testdata/rules1-bad.rules b/app/vmalert/config/testdata/rules1-bad.rules new file mode 100644 index 000000000..37913318d --- /dev/null +++ b/app/vmalert/config/testdata/rules1-bad.rules @@ -0,0 +1,12 @@ +groups: + - name: TestGraphiteBadGroup + interval: 2s + concurrency: 2 + type: graphite + rules: + - alert: Conns + expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500) by instance + for: 3m + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" \ No newline at end of file diff --git a/app/vmalert/config/testdata/rules3-good.rules b/app/vmalert/config/testdata/rules3-good.rules new file mode 100644 index 000000000..3e260b94d --- /dev/null +++ b/app/vmalert/config/testdata/rules3-good.rules @@ -0,0 +1,30 @@ +groups: + - name: TestGroup + interval: 2s + concurrency: 2 + type: graphite + rules: + - alert: Conns + expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500) + for: 3m + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" + - name: TestGroupPromMixed + interval: 2s + concurrency: 2 + type: prometheus + rules: + - alert: Conns + expr: sum(vm_tcplistener_conns) by (instance) > 1 + for: 3m + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" + - alert: HostDown + type: graphite + expr: filterSeries(sumSeries(host.receiver.interface.up),'last','=', 0) + for: 3m + annotations: + summary: Too high connection number for {{$labels.instance}} + description: "It is {{ $value }} connections for {{$labels.instance}}" diff --git a/app/vmalert/datasource/datasource.go b/app/vmalert/datasource/datasource.go index 5e6aa32d3..4625d09ee 100644 --- a/app/vmalert/datasource/datasource.go +++ b/app/vmalert/datasource/datasource.go @@ -1,12 +1,14 @@ package datasource -import "context" +import ( + "context" +) // Querier interface wraps Query method which // executes given query and returns list of Metrics // as result type Querier interface { - Query(ctx context.Context, query string) ([]Metric, error) + Query(ctx context.Context, query string, engine Type) ([]Metric, error) } // Metric is the basic entity which should be return by datasource diff --git a/app/vmalert/datasource/type.go b/app/vmalert/datasource/type.go new file mode 100644 index 000000000..10dfc0ca4 --- /dev/null +++ b/app/vmalert/datasource/type.go @@ -0,0 +1,89 @@ +package datasource + +import ( + "fmt" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql" + "github.com/VictoriaMetrics/metricsql" +) + +const graphiteType = "graphite" +const prometheusType = "prometheus" + +// Type represents data source type +type Type struct { + name string +} + +// NewPrometheusType returns prometheus datasource type +func NewPrometheusType() Type { + return Type{name: prometheusType} +} + +// NewGraphiteType returns graphite datasource type +func NewGraphiteType() Type { + return Type{name: graphiteType} +} + +// NewRawType returns datasource type from raw string +// without validation. +func NewRawType(d string) Type { + return Type{name: d} +} + +// Get returns datasource type +func (t *Type) Get() string { + return t.name +} + +// Set changes datasource type +func (t *Type) Set(d Type) { + t.name = d.name +} + +// String implements String interface with default value. +func (t Type) String() string { + if t.name == "" { + return prometheusType + } + return t.name +} + +// ValidateExpr validates query expression with datasource ql. +func (t *Type) ValidateExpr(expr string) error { + switch t.name { + case graphiteType: + if _, err := graphiteql.Parse(expr); err != nil { + return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err) + } + case "", prometheusType: + if _, err := metricsql.Parse(expr); err != nil { + return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err) + } + default: + return fmt.Errorf("unknown datasource type=%q", t.name) + } + return nil +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + switch s { + case "": + s = prometheusType + case graphiteType, prometheusType: + default: + return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, prometheusType, graphiteType) + } + t.name = s + return nil +} + +// MarshalYAML implements the yaml.Unmarshaler interface. +func (t Type) MarshalYAML() (interface{}, error) { + return t.name, nil +} diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go index ae99df121..0c0626fbc 100644 --- a/app/vmalert/datasource/vm.go +++ b/app/vmalert/datasource/vm.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "net/http" - "net/url" "strconv" "strings" "time" @@ -46,17 +45,45 @@ func (r response) metrics() ([]Metric, error) { return ms, nil } +type graphiteResponse []graphiteResponseTarget + +type graphiteResponseTarget struct { + Target string `json:"target"` + Tags map[string]string `json:"tags"` + DataPoints [][2]float64 `json:"datapoints"` +} + +func (r graphiteResponse) metrics() []Metric { + var ms []Metric + for _, res := range r { + if len(res.DataPoints) < 1 { + continue + } + var m Metric + // add only last value to the result. + last := res.DataPoints[len(res.DataPoints)-1] + m.Value = last[0] + m.Timestamp = int64(last[1]) + for k, v := range res.Tags { + m.AddLabel(k, v) + } + ms = append(ms, m) + } + return ms +} + // VMStorage represents vmstorage entity with ability to read and write metrics type VMStorage struct { c *http.Client - queryURL string + datasourceURL string basicAuthUser string basicAuthPass string lookBack time.Duration queryStep time.Duration } -const queryPath = "/api/v1/query?query=" +const queryPath = "/api/v1/query" +const graphitePath = "/render" // NewVMStorage is a constructor for VMStorage func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Duration, queryStep time.Duration, c *http.Client) *VMStorage { @@ -64,26 +91,31 @@ func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, lookBack time.Du c: c, basicAuthUser: basicAuthUser, basicAuthPass: basicAuthPass, - queryURL: strings.TrimSuffix(baseURL, "/") + queryPath, + datasourceURL: strings.TrimSuffix(baseURL, "/"), lookBack: lookBack, queryStep: queryStep, } } -// Query reads metrics from datasource by given query -func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) { - const ( - statusSuccess, statusError, rtVector = "success", "error", "vector" - ) - q := s.queryURL + url.QueryEscape(query) - if s.lookBack > 0 { - lookBack := time.Now().Add(-s.lookBack) - q += fmt.Sprintf("&time=%d", lookBack.Unix()) +// Query reads metrics from datasource by given query and type +func (s *VMStorage) Query(ctx context.Context, query string, dataSourceType Type) ([]Metric, error) { + switch dataSourceType.name { + case "", prometheusType: + return s.queryDataSource(ctx, query, s.setPrometheusReqParams, parsePrometheusResponse) + case graphiteType: + return s.queryDataSource(ctx, query, s.setGraphiteReqParams, parseGraphiteResponse) + default: + return nil, fmt.Errorf("engine not found: %q", dataSourceType) } - if s.queryStep > 0 { - q += fmt.Sprintf("&step=%s", s.queryStep.String()) - } - req, err := http.NewRequest("POST", q, nil) +} + +func (s *VMStorage) queryDataSource( + ctx context.Context, + query string, + setReqParams func(r *http.Request, query string), + processResponse func(r *http.Request, resp *http.Response, + ) ([]Metric, error)) ([]Metric, error) { + req, err := http.NewRequest("POST", s.datasourceURL, nil) if err != nil { return nil, err } @@ -91,6 +123,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) { if s.basicAuthPass != "" { req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass) } + setReqParams(req, query) resp, err := s.c.Do(req.WithContext(ctx)) if err != nil { return nil, fmt.Errorf("error getting response from %s: %w", req.URL, err) @@ -100,9 +133,46 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) { body, _ := ioutil.ReadAll(resp.Body) return nil, fmt.Errorf("datasource returns unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body) } + return processResponse(req, resp) +} + +func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) { + r.URL.Path += queryPath + q := r.URL.Query() + q.Set("query", query) + if s.lookBack > 0 { + lookBack := time.Now().Add(-s.lookBack) + q.Set("time", fmt.Sprintf("%d", lookBack.Unix())) + } + if s.queryStep > 0 { + q.Set("step", s.queryStep.String()) + } + r.URL.RawQuery = q.Encode() +} + +func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string) { + r.URL.Path += graphitePath + q := r.URL.Query() + q.Set("format", "json") + q.Set("target", query) + from := "-5min" + if s.lookBack > 0 { + lookBack := time.Now().Add(-s.lookBack) + from = strconv.FormatInt(lookBack.Unix(), 10) + } + q.Set("from", from) + q.Set("until", "now") + r.URL.RawQuery = q.Encode() +} + +const ( + statusSuccess, statusError, rtVector = "success", "error", "vector" +) + +func parsePrometheusResponse(req *http.Request, resp *http.Response) ([]Metric, error) { r := &response{} if err := json.NewDecoder(resp.Body).Decode(r); err != nil { - return nil, fmt.Errorf("error parsing metrics for %s: %w", req.URL, err) + return nil, fmt.Errorf("error parsing prometheus metrics for %s: %w", req.URL, err) } if r.Status == statusError { return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error) @@ -115,3 +185,11 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) { } return r.metrics() } + +func parseGraphiteResponse(req *http.Request, resp *http.Response) ([]Metric, error) { + r := &graphiteResponse{} + if err := json.NewDecoder(resp.Body).Decode(r); err != nil { + return nil, fmt.Errorf("error parsing graphite metrics for %s: %w", req.URL, err) + } + return r.metrics(), nil +} diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go index af99c8421..1afe0d8e3 100644 --- a/app/vmalert/datasource/vm_test.go +++ b/app/vmalert/datasource/vm_test.go @@ -14,6 +14,7 @@ var ( basicAuthName = "foo" basicAuthPass = "bar" query = "vm_rows" + queryRender = "constantLine(10)" ) func TestVMSelectQuery(t *testing.T) { @@ -22,6 +23,13 @@ func TestVMSelectQuery(t *testing.T) { t.Errorf("should not be called") }) c := -1 + mux.HandleFunc("/render", func(w http.ResponseWriter, request *http.Request) { + c++ + switch c { + case 7: + w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`)) + } + }) mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) { c++ if r.Method != http.MethodPost { @@ -62,25 +70,25 @@ func TestVMSelectQuery(t *testing.T) { srv := httptest.NewServer(mux) defer srv.Close() am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, time.Minute, 0, srv.Client()) - if _, err := am.Query(ctx, query); err == nil { + if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil { t.Fatalf("expected connection error got nil") } - if _, err := am.Query(ctx, query); err == nil { + if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil { t.Fatalf("expected invalid response status error got nil") } - if _, err := am.Query(ctx, query); err == nil { + if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil { t.Fatalf("expected response body error got nil") } - if _, err := am.Query(ctx, query); err == nil { + if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil { t.Fatalf("expected error status got nil") } - if _, err := am.Query(ctx, query); err == nil { + if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil { t.Fatalf("expected unknown status got nil") } - if _, err := am.Query(ctx, query); err == nil { + if _, err := am.Query(ctx, query, NewPrometheusType()); err == nil { t.Fatalf("expected non-vector resultType error got nil") } - m, err := am.Query(ctx, query) + m, err := am.Query(ctx, query, NewPrometheusType()) if err != nil { t.Fatalf("unexpected %s", err) } @@ -98,4 +106,22 @@ func TestVMSelectQuery(t *testing.T) { m[0].Labels[0].Name != expected.Labels[0].Name { t.Fatalf("unexpected metric %+v want %+v", m[0], expected) } + m, err = am.Query(ctx, queryRender, NewGraphiteType()) + if err != nil { + t.Fatalf("unexpected %s", err) + } + if len(m) != 1 { + t.Fatalf("expected 1 metric got %d in %+v", len(m), m) + } + expected = Metric{ + Labels: []Label{{Value: "constantLine(10)", Name: "name"}}, + Timestamp: 1611758403, + Value: 10, + } + if m[0].Timestamp != expected.Timestamp && + m[0].Value != expected.Value && + m[0].Labels[0].Value != expected.Labels[0].Value && + m[0].Labels[0].Name != expected.Labels[0].Name { + t.Fatalf("unexpected metric %+v want %+v", m[0], expected) + } } diff --git a/app/vmalert/group.go b/app/vmalert/group.go index f2a1b5bd7..e8a73b25f 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -22,6 +22,7 @@ type Group struct { Name string File string Rules []Rule + Type datasource.Type Interval time.Duration Concurrency int Checksum string @@ -50,6 +51,7 @@ func newGroupMetrics(name, file string) *groupMetrics { func newGroup(cfg config.Group, defaultInterval time.Duration, labels map[string]string) *Group { g := &Group{ + Type: cfg.Type, Name: cfg.Name, File: cfg.File, Interval: cfg.Interval, @@ -99,6 +101,7 @@ func (g *Group) ID() uint64 { hash.Write([]byte(g.File)) hash.Write([]byte("\xff")) hash.Write([]byte(g.Name)) + hash.Write([]byte(g.Type.Get())) return hash.Sum64() } @@ -157,6 +160,7 @@ func (g *Group) updateWith(newGroup *Group) error { for _, nr := range rulesRegistry { newRules = append(newRules, nr) } + g.Type = newGroup.Type g.Concurrency = newGroup.Concurrency g.Checksum = newGroup.Checksum g.Rules = newRules diff --git a/app/vmalert/helpers_test.go b/app/vmalert/helpers_test.go index e4fcc1d59..3f879d466 100644 --- a/app/vmalert/helpers_test.go +++ b/app/vmalert/helpers_test.go @@ -38,7 +38,7 @@ func (fq *fakeQuerier) add(metrics ...datasource.Metric) { fq.Unlock() } -func (fq *fakeQuerier) Query(_ context.Context, _ string) ([]datasource.Metric, error) { +func (fq *fakeQuerier) Query(_ context.Context, _ string, _ datasource.Type) ([]datasource.Metric, error) { fq.Lock() defer fq.Unlock() if fq.err != nil { diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go index 3abdd1340..baaf0effb 100644 --- a/app/vmalert/manager.go +++ b/app/vmalert/manager.go @@ -142,6 +142,7 @@ func (g *Group) toAPI() APIGroup { // encode as string to avoid rounding ID: fmt.Sprintf("%d", g.ID()), Name: g.Name, + Type: g.Type.String(), File: g.File, Interval: g.Interval.String(), Concurrency: g.Concurrency, diff --git a/app/vmalert/manager_test.go b/app/vmalert/manager_test.go index a3cd3d31f..8762ef6cb 100644 --- a/app/vmalert/manager_test.go +++ b/app/vmalert/manager_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" ) @@ -106,6 +108,18 @@ func TestManagerUpdate(t *testing.T) { Name: "ExampleAlertAlwaysFiring", Expr: "sum by(job) (up == 1)", } + ExampleAlertGraphite = &AlertingRule{ + Name: "up graphite", + Expr: "filterSeries(time('host.1',20),'>','0')", + Type: datasource.NewGraphiteType(), + For: defaultEvalInterval, + } + ExampleAlertGraphite2 = &AlertingRule{ + Name: "up", + Expr: "filterSeries(time('host.2',20),'>','0')", + Type: datasource.NewGraphiteType(), + For: defaultEvalInterval, + } ) testCases := []struct { @@ -122,6 +136,7 @@ func TestManagerUpdate(t *testing.T) { { File: "config/testdata/dir/rules1-good.rules", Name: "duplicatedGroupDiffFiles", + Type: datasource.NewPrometheusType(), Interval: defaultEvalInterval, Rules: []Rule{ &AlertingRule{ @@ -146,12 +161,14 @@ func TestManagerUpdate(t *testing.T) { { File: "config/testdata/rules0-good.rules", Name: "groupGorSingleAlert", + Type: datasource.NewPrometheusType(), Rules: []Rule{VMRows}, Interval: defaultEvalInterval, }, { File: "config/testdata/rules0-good.rules", Interval: defaultEvalInterval, + Type: datasource.NewPrometheusType(), Name: "TestGroup", Rules: []Rule{ Conns, ExampleAlertAlwaysFiring, @@ -166,13 +183,16 @@ func TestManagerUpdate(t *testing.T) { { File: "config/testdata/rules0-good.rules", Name: "groupGorSingleAlert", + Type: datasource.NewPrometheusType(), Interval: defaultEvalInterval, Rules: []Rule{VMRows}, }, { File: "config/testdata/rules0-good.rules", Interval: defaultEvalInterval, - Name: "TestGroup", Rules: []Rule{ + Name: "TestGroup", + Type: datasource.NewPrometheusType(), + Rules: []Rule{ Conns, ExampleAlertAlwaysFiring, }}, @@ -186,12 +206,14 @@ func TestManagerUpdate(t *testing.T) { { File: "config/testdata/rules0-good.rules", Name: "groupGorSingleAlert", + Type: datasource.NewPrometheusType(), Interval: defaultEvalInterval, Rules: []Rule{VMRows}, }, { File: "config/testdata/rules0-good.rules", Interval: defaultEvalInterval, + Type: datasource.NewPrometheusType(), Name: "TestGroup", Rules: []Rule{ Conns, ExampleAlertAlwaysFiring, @@ -199,6 +221,23 @@ func TestManagerUpdate(t *testing.T) { }, }, }, + { + name: "update prometheus to graphite type", + initPath: "config/testdata/dir/rules-update0-good.rules", + updatePath: "config/testdata/dir/rules-update1-good.rules", + want: []*Group{ + { + File: "config/testdata/dir/rules-update1-good.rules", + Interval: defaultEvalInterval, + Type: datasource.NewGraphiteType(), + Name: "TestUpdateGroup", + Rules: []Rule{ + ExampleAlertGraphite2, + ExampleAlertGraphite, + }, + }, + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/app/vmalert/recording.go b/app/vmalert/recording.go index 89f0c57e1..4f075cdf1 100644 --- a/app/vmalert/recording.go +++ b/app/vmalert/recording.go @@ -18,6 +18,7 @@ import ( // to evaluate configured Expression and // return TimeSeries as result. type RecordingRule struct { + Type datasource.Type RuleID uint64 Name string Expr string @@ -53,6 +54,7 @@ func (rr *RecordingRule) ID() uint64 { func newRecordingRule(group *Group, cfg config.Rule) *RecordingRule { rr := &RecordingRule{ + Type: cfg.Type, RuleID: cfg.ID, Name: cfg.Record, Expr: cfg.Expr, @@ -60,6 +62,7 @@ func newRecordingRule(group *Group, cfg config.Rule) *RecordingRule { GroupID: group.ID(), metrics: &recordingRuleMetrics{}, } + labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID()) rr.metrics.errors = getOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels), func() float64 { @@ -84,8 +87,7 @@ func (rr *RecordingRule) Exec(ctx context.Context, q datasource.Querier, series return nil, nil } - qMetrics, err := q.Query(ctx, rr.Expr) - + qMetrics, err := q.Query(ctx, rr.Expr, rr.Type) rr.mu.Lock() defer rr.mu.Unlock() @@ -162,6 +164,7 @@ func (rr *RecordingRule) RuleAPI() APIRecordingRule { ID: fmt.Sprintf("%d", rr.ID()), GroupID: fmt.Sprintf("%d", rr.GroupID), Name: rr.Name, + Type: rr.Type.String(), Expression: rr.Expr, LastError: lastErr, LastExec: rr.lastExecTime, diff --git a/app/vmalert/web_types.go b/app/vmalert/web_types.go index b26a20d2c..ca3cbdd05 100644 --- a/app/vmalert/web_types.go +++ b/app/vmalert/web_types.go @@ -21,6 +21,7 @@ type APIAlert struct { // APIGroup represents Group for WEB view type APIGroup struct { Name string `json:"name"` + Type string `json:"type"` ID string `json:"id"` File string `json:"file"` Interval string `json:"interval"` @@ -33,6 +34,7 @@ type APIGroup struct { type APIAlertingRule struct { ID string `json:"id"` Name string `json:"name"` + Type string `json:"type"` GroupID string `json:"group_id"` Expression string `json:"expression"` For string `json:"for"` @@ -46,6 +48,7 @@ type APIAlertingRule struct { type APIRecordingRule struct { ID string `json:"id"` Name string `json:"name"` + Type string `json:"type"` GroupID string `json:"group_id"` Expression string `json:"expression"` LastError string `json:"last_error"` diff --git a/app/vmselect/graphiteql/lexer.go b/app/vmselect/graphiteql/lexer.go new file mode 100644 index 000000000..3a519706c --- /dev/null +++ b/app/vmselect/graphiteql/lexer.go @@ -0,0 +1,417 @@ +package graphiteql + +import ( + "fmt" + "strings" +) + +type lexer struct { + // Token contains the currently parsed token. + // An empty token means EOF. + Token string + + sOrig string + sTail string + + err error +} + +func (lex *lexer) Context() string { + return fmt.Sprintf("%s%s", lex.Token, lex.sTail) +} + +func (lex *lexer) Init(s string) { + lex.Token = "" + + lex.sOrig = s + lex.sTail = s + + lex.err = nil +} + +func (lex *lexer) Next() error { + if lex.err != nil { + return lex.err + } + token, err := lex.next() + if err != nil { + lex.err = err + return err + } + lex.Token = token + return nil +} + +func (lex *lexer) next() (string, error) { + // Skip whitespace + s := lex.sTail + i := 0 + for i < len(s) && isSpaceChar(s[i]) { + i++ + } + s = s[i:] + lex.sTail = s + + if len(s) == 0 { + return "", nil + } + + var token string + var err error + switch s[0] { + case '(', ')', ',', '|', '=', '+', '-': + token = s[:1] + goto tokenFoundLabel + } + if isStringPrefix(s) { + token, err = scanString(s) + if err != nil { + return "", err + } + goto tokenFoundLabel + } + if isPositiveNumberPrefix(s) { + token, err = scanPositiveNumber(s) + if err != nil { + return "", err + } + goto tokenFoundLabel + } + token, err = scanIdent(s) + if err != nil { + return "", err + } + +tokenFoundLabel: + lex.sTail = s[len(token):] + return token, nil +} + +func scanString(s string) (string, error) { + if len(s) < 2 { + return "", fmt.Errorf("cannot find end of string in %q", s) + } + + quote := s[0] + i := 1 + for { + n := strings.IndexByte(s[i:], quote) + if n < 0 { + return "", fmt.Errorf("cannot find closing quote %c for the string %q", quote, s) + } + i += n + bs := 0 + for bs < i && s[i-bs-1] == '\\' { + bs++ + } + if bs%2 == 0 { + token := s[:i+1] + return token, nil + } + i++ + } +} + +func scanPositiveNumber(s string) (string, error) { + // Scan integer part. It may be empty if fractional part exists. + i := 0 + skipChars, isHex := scanSpecialIntegerPrefix(s) + i += skipChars + if isHex { + // Scan integer hex number + for i < len(s) && isHexChar(s[i]) { + i++ + } + if i == skipChars { + return "", fmt.Errorf("number cannot be empty") + } + return s[:i], nil + } + for i < len(s) && isDecimalChar(s[i]) { + i++ + } + + if i == len(s) { + if i == skipChars { + return "", fmt.Errorf("number cannot be empty") + } + return s, nil + } + if s[i] != '.' && s[i] != 'e' && s[i] != 'E' { + return s[:i], nil + } + + if s[i] == '.' { + // Scan fractional part. It cannot be empty. + i++ + j := i + for j < len(s) && isDecimalChar(s[j]) { + j++ + } + if j == i { + return "", fmt.Errorf("missing fractional part in %q", s) + } + i = j + if i == len(s) { + return s, nil + } + } + + if s[i] != 'e' && s[i] != 'E' { + return s[:i], nil + } + i++ + + // Scan exponent part. + if i == len(s) { + return "", fmt.Errorf("missing exponent part in %q", s) + } + if s[i] == '-' || s[i] == '+' { + i++ + } + j := i + for j < len(s) && isDecimalChar(s[j]) { + j++ + } + if j == i { + return "", fmt.Errorf("missing exponent part in %q", s) + } + return s[:j], nil +} + +func scanIdent(s string) (string, error) { + i := 0 + for i < len(s) { + switch s[i] { + case '\\': + // Skip the next char, since it is escaped + i += 2 + if i > len(s) { + return "", fmt.Errorf("missing escaped char in the end of %q", s) + } + case '[': + n := strings.IndexByte(s[i+1:], ']') + if n < 0 { + return "", fmt.Errorf("missing `]` char in %q", s) + } + i += n + 2 + case '{': + n := strings.IndexByte(s[i+1:], '}') + if n < 0 { + return "", fmt.Errorf("missing '}' char in %q", s) + } + i += n + 2 + case '*', '.': + i++ + default: + if !isIdentChar(s[i]) { + goto end + } + i++ + } + } +end: + if i == 0 { + return "", fmt.Errorf("cannot find a single ident char in %q", s) + } + return s[:i], nil +} + +func unescapeIdent(s string) string { + n := strings.IndexByte(s, '\\') + if n < 0 { + return s + } + dst := make([]byte, 0, len(s)) + for { + dst = append(dst, s[:n]...) + s = s[n+1:] + if len(s) == 0 { + return string(dst) + } + if s[0] == 'x' && len(s) >= 3 { + h1 := fromHex(s[1]) + h2 := fromHex(s[2]) + if h1 >= 0 && h2 >= 0 { + dst = append(dst, byte((h1<<4)|h2)) + s = s[3:] + } else { + dst = append(dst, s[0]) + s = s[1:] + } + } else { + dst = append(dst, s[0]) + s = s[1:] + } + n = strings.IndexByte(s, '\\') + if n < 0 { + dst = append(dst, s...) + return string(dst) + } + } +} + +func fromHex(ch byte) int { + if ch >= '0' && ch <= '9' { + return int(ch - '0') + } + if ch >= 'a' && ch <= 'f' { + return int((ch - 'a') + 10) + } + if ch >= 'A' && ch <= 'F' { + return int((ch - 'A') + 10) + } + return -1 +} + +func toHex(n byte) byte { + if n < 10 { + return '0' + n + } + return 'a' + (n - 10) +} + +func isMetricExprChar(ch byte) bool { + switch ch { + case '.', '*', '[', ']', '{', '}', ',': + return true + } + return false +} + +func appendEscapedIdent(dst []byte, s string) []byte { + for i := 0; i < len(s); i++ { + ch := s[i] + if isIdentChar(ch) || isMetricExprChar(ch) { + if i == 0 && !isFirstIdentChar(ch) { + // hex-encode the first char + dst = append(dst, '\\', 'x', toHex(ch>>4), toHex(ch&0xf)) + } else { + dst = append(dst, ch) + } + } else if ch >= 0x20 && ch < 0x7f { + // Leave ASCII printable chars as is + dst = append(dst, '\\', ch) + } else { + // hex-encode non-printable chars + dst = append(dst, '\\', 'x', toHex(ch>>4), toHex(ch&0xf)) + } + } + return dst +} + +func isEOF(s string) bool { + return len(s) == 0 +} + +func isBool(s string) bool { + s = strings.ToLower(s) + return s == "true" || s == "false" +} + +func isStringPrefix(s string) bool { + if len(s) == 0 { + return false + } + switch s[0] { + case '"', '\'': + return true + default: + return false + } +} + +func isPositiveNumberPrefix(s string) bool { + if len(s) == 0 { + return false + } + if isDecimalChar(s[0]) { + return true + } + + // Check for .234 numbers + if s[0] != '.' || len(s) < 2 { + return false + } + return isDecimalChar(s[1]) +} + +func isSpecialIntegerPrefix(s string) bool { + skipChars, _ := scanSpecialIntegerPrefix(s) + return skipChars > 0 +} + +func scanSpecialIntegerPrefix(s string) (skipChars int, isHex bool) { + if len(s) < 1 || s[0] != '0' { + return 0, false + } + s = strings.ToLower(s[1:]) + if len(s) == 0 { + return 0, false + } + if isDecimalChar(s[0]) { + // octal number: 0123 + return 1, false + } + if s[0] == 'x' { + // 0x + return 2, true + } + if s[0] == 'o' || s[0] == 'b' { + // 0x, 0o or 0b prefix + return 2, false + } + return 0, false +} + +func isDecimalChar(ch byte) bool { + return ch >= '0' && ch <= '9' +} + +func isHexChar(ch byte) bool { + return isDecimalChar(ch) || ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F' +} + +func isIdentPrefix(s string) bool { + if len(s) == 0 { + return false + } + if s[0] == '\\' { + // Assume this is an escape char for the next char. + return true + } + return isFirstIdentChar(s[0]) +} + +func isFirstIdentChar(ch byte) bool { + if !isIdentChar(ch) { + return false + } + if isDecimalChar(ch) { + return false + } + return true +} + +func isIdentChar(ch byte) bool { + if ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' { + return true + } + if isDecimalChar(ch) { + return true + } + switch ch { + case '-', '_', '$', ':', '*', '{', '[': + return true + } + return false +} + +func isSpaceChar(ch byte) bool { + switch ch { + case ' ', '\t', '\n', '\v', '\f', '\r': + return true + default: + return false + } +} diff --git a/app/vmselect/graphiteql/lexer_test.go b/app/vmselect/graphiteql/lexer_test.go new file mode 100644 index 000000000..bb070bf3a --- /dev/null +++ b/app/vmselect/graphiteql/lexer_test.go @@ -0,0 +1,151 @@ +package graphiteql + +import ( + "reflect" + "strings" + "testing" +) + +func TestScanStringSuccess(t *testing.T) { + f := func(s, sExpected string) { + t.Helper() + result, err := scanString(s) + if err != nil { + t.Fatalf("unexpected error in scanString(%s): %s", s, err) + } + if result != sExpected { + t.Fatalf("unexpected string scanned from %s; got %s; want %s", s, result, sExpected) + } + if !strings.HasPrefix(s, result) { + t.Fatalf("invalid prefix for scanne string %s: %s", s, result) + } + } + f(`""`, `""`) + f(`''`, `''`) + f(`""tail`, `""`) + f(`''tail`, `''`) + f(`"foo", bar`, `"foo"`) + f(`'foo', bar`, `'foo'`) + f(`"foo\.bar"`, `"foo\.bar"`) + f(`"foo\"bar\1"\"`, `"foo\"bar\1"`) + f(`"foo\\"bar\1"\"`, `"foo\\"`) + f(`'foo\\'bar\1"\"`, `'foo\\'`) +} + +func TestScanStringFailure(t *testing.T) { + f := func(s string) { + t.Helper() + result, err := scanString(s) + if err == nil { + t.Fatalf("expecting non-nil error for scanString(%s)", s) + } + if result != "" { + t.Fatalf("expecting empty result for scanString(%s); got %s", s, result) + } + } + f(``) + f(`"foo`) + f(`'bar`) +} + +func TestAppendEscapedIdent(t *testing.T) { + f := func(s, resultExpected string) { + t.Helper() + result := appendEscapedIdent(nil, s) + if string(result) != resultExpected { + t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected) + } + } + f("", "") + f("a", "a") + f("fo_o.$b-ar.b[a]*z{aa,bb}", "fo_o.$b-ar.b[a]*z{aa,bb}") + f("a(b =C)", `a\(b\ \=C\)`) +} + +func TestLexerSuccess(t *testing.T) { + f := func(s string, tokensExpected []string) { + t.Helper() + var lex lexer + var tokens []string + lex.Init(s) + for { + if err := lex.Next(); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if isEOF(lex.Token) { + break + } + tokens = append(tokens, lex.Token) + } + if !reflect.DeepEqual(tokens, tokensExpected) { + t.Fatalf("unexpected tokens; got\n%q\nwant\n%q", tokens, tokensExpected) + } + } + f("", nil) + f("a", []string{"a"}) + f("*", []string{"*"}) + f("*.a", []string{"*.a"}) + f("[a-z]zx", []string{"[a-z]zx"}) + f("fo_o.$ba-r", []string{"fo_o.$ba-r"}) + f("{foo,bar}", []string{"{foo,bar}"}) + f("[a-z]", []string{"[a-z]"}) + f("fo*.bar10[s-z]as{aaa,bb}.ss{aa*ss,DS[c-D],ss}ss", []string{"fo*.bar10[s-z]as{aaa,bb}.ss{aa*ss,DS[c-D],ss}ss"}) + f("FOO.bar:avg", []string{"FOO.bar:avg"}) + f("FOO.bar\\:avg", []string{"FOO.bar\\:avg"}) + f(`foo.Bar|aaa(b,cd,e=aa)`, []string{"foo.Bar", "|", "aaa", "(", "b", ",", "cd", ",", "e", "=", "aa", ")"}) + f(`foo.Bar\|aaa\(b\,cd\,e\=aa\)`, []string{`foo.Bar\|aaa\(b\,cd\,e\=aa\)`}) + f(`123`, []string{`123`}) + f(`12.34`, []string{`12.34`}) + f(`12.34e4`, []string{`12.34e4`}) + f(`12.34e-4`, []string{`12.34e-4`}) + f(`12E+45`, []string{`12E+45`}) + f(`+12.34`, []string{`+`, `12.34`}) + f("0xABcd", []string{`0xABcd`}) + f("f(0o765,0b1101,0734,12.34)", []string{"f", "(", "0o765", ",", "0b1101", ",", "0734", ",", "12.34", ")"}) + f(`f ( foo, -.54e6,bar)`, []string{"f", "(", "foo", ",", "-", ".54e6", ",", "bar", ")"}) + f(`"foo(b'ar:baz)"`, []string{`"foo(b'ar:baz)"`}) + f(`'a"bc'`, []string{`'a"bc'`}) + f(`"f\"oo\\b"`, []string{`"f\"oo\\b"`}) + f(`a("b,c", 'de')`, []string{`a`, `(`, `"b,c"`, `,`, `'de'`, `)`}) +} + +func TestLexerError(t *testing.T) { + f := func(s string) { + t.Helper() + var lex lexer + lex.Init(s) + for { + if err := lex.Next(); err != nil { + // Make sure lex.Next() consistently returns the error. + if err1 := lex.Next(); err1 != err { + t.Fatalf("unexpected error returned; got %v; want %v", err1, err) + } + return + } + if isEOF(lex.Token) { + t.Fatalf("expecting non-nil error when parsing %q", s) + } + } + } + + // Invalid identifier + f("foo\\") + f(`foo[bar`) + f(`foo{bar`) + f("~") + f(",~") + + // Invalid string + f(`"`) + f(`"foo`) + f(`'aa`) + + // Invalid number + f(`0x`) + f(`0o`) + f(`-0b`) + f(`13.`) + f(`1e`) + f(`1E+`) + f(`1.3e`) +} diff --git a/app/vmselect/graphiteql/parser.go b/app/vmselect/graphiteql/parser.go new file mode 100644 index 000000000..f24a2e8bf --- /dev/null +++ b/app/vmselect/graphiteql/parser.go @@ -0,0 +1,407 @@ +package graphiteql + +import ( + "fmt" + "strconv" + "strings" +) + +type parser struct { + lex lexer +} + +// Expr is Graphite expression for render API. +type Expr interface { + // AppendString appends Expr contents to dst and returns the result. + AppendString(dst []byte) []byte +} + +// Parse parses Graphite render API target expression. +// +// See https://graphite.readthedocs.io/en/stable/render_api.html +func Parse(s string) (Expr, error) { + var p parser + p.lex.Init(s) + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot parse target expression: %w; context: %q", err, p.lex.Context()) + } + expr, err := p.parseExpr() + if err != nil { + return nil, fmt.Errorf("cannot parse target expression: %w; context: %q", err, p.lex.Context()) + } + if !isEOF(p.lex.Token) { + return nil, fmt.Errorf("unexpected tail left after parsing %q; context: %q", expr.AppendString(nil), p.lex.Context()) + } + return expr, nil +} + +func (p *parser) parseExpr() (Expr, error) { + var expr Expr + var err error + token := p.lex.Token + switch { + case isPositiveNumberPrefix(token) || token == "+" || token == "-": + expr, err = p.parseNumber() + if err != nil { + return nil, err + } + case isStringPrefix(token): + expr, err = p.parseString() + if err != nil { + return nil, err + } + case isIdentPrefix(token): + expr, err = p.parseMetricExprOrFuncCall() + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unexpected token when parsing expression: %q", token) + } + + for { + switch p.lex.Token { + case "|": + // Chained function call. For example, `metric|func` + firstArg := &ArgExpr{ + Expr: expr, + } + expr, err = p.parseChainedFunc(firstArg) + if err != nil { + return nil, err + } + continue + default: + return expr, nil + } + } +} + +func (p *parser) parseNumber() (*NumberExpr, error) { + token := p.lex.Token + isMinus := false + if token == "-" || token == "+" { + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find number after %q token: %w", token, err) + } + isMinus = token == "-" + token = p.lex.Token + } + var n float64 + if isSpecialIntegerPrefix(token) { + d, err := strconv.ParseInt(token, 0, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse integer %q: %w", token, err) + } + n = float64(d) + } else { + f, err := strconv.ParseFloat(token, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse floating-point number %q: %w", token, err) + } + n = f + } + if isMinus { + n = -n + } + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find next token after %q: %w", token, err) + } + ne := &NumberExpr{ + N: n, + } + return ne, nil +} + +// NoneExpr contains None value +type NoneExpr struct{} + +// AppendString appends string representation of nne to dst and returns the result. +func (nne *NoneExpr) AppendString(dst []byte) []byte { + return append(dst, "None"...) +} + +// BoolExpr contains bool value (True or False). +type BoolExpr struct { + // B is bool value + B bool +} + +// AppendString appends string representation of be to dst and returns the result. +func (be *BoolExpr) AppendString(dst []byte) []byte { + if be.B { + return append(dst, "True"...) + } + return append(dst, "False"...) +} + +// NumberExpr contains float64 constant. +type NumberExpr struct { + // N is float64 constant + N float64 +} + +// AppendString appends string representation of ne to dst and returns the result. +func (ne *NumberExpr) AppendString(dst []byte) []byte { + return strconv.AppendFloat(dst, ne.N, 'g', -1, 64) +} + +func (p *parser) parseString() (*StringExpr, error) { + token := p.lex.Token + if len(token) < 2 || token[0] != token[len(token)-1] { + return nil, fmt.Errorf(`string literal contains unexpected trailing char; got %q`, token) + } + quote := string(append([]byte{}, token[0])) + s := token[1 : len(token)-1] + s = strings.ReplaceAll(s, `\`+quote, quote) + s = strings.ReplaceAll(s, `\\`, `\`) + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find next token after %s: %w", token, err) + } + se := &StringExpr{ + S: s, + } + return se, nil +} + +// StringExpr represents string contant. +type StringExpr struct { + // S contains unquoted string contents. + S string +} + +// AppendString appends se to dst and returns the result. +func (se *StringExpr) AppendString(dst []byte) []byte { + dst = append(dst, '\'') + s := strings.ReplaceAll(se.S, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `\'`) + dst = append(dst, s...) + dst = append(dst, '\'') + return dst +} + +// QuoteString quotes s, so it could be used in Graphite queries. +func QuoteString(s string) string { + se := &StringExpr{ + S: s, + } + return string(se.AppendString(nil)) +} + +func (p *parser) parseMetricExprOrFuncCall() (Expr, error) { + token := p.lex.Token + ident := unescapeIdent(token) + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find next token after %q: %w", token, err) + } + token = p.lex.Token + switch token { + case "(": + // Function call. For example, `func(foo,bar)` + funcName := ident + args, err := p.parseArgs() + if err != nil { + return nil, fmt.Errorf("cannot parse args for function %q: %w", funcName, err) + } + fe := &FuncExpr{ + FuncName: funcName, + Args: args, + printState: printStateNormal, + } + return fe, nil + default: + // Metric epxression or bool expression or None. + if isBool(ident) { + be := &BoolExpr{ + B: strings.ToLower(ident) == "true", + } + return be, nil + } + if strings.ToLower(ident) == "none" { + nne := &NoneExpr{} + return nne, nil + } + me := &MetricExpr{ + Query: ident, + } + return me, nil + } +} + +func (p *parser) parseChainedFunc(firstArg *ArgExpr) (*FuncExpr, error) { + for { + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find function name after %q|: %w", firstArg.AppendString(nil), err) + } + if !isIdentPrefix(p.lex.Token) { + return nil, fmt.Errorf("expecting function name after %q|, got %q", firstArg.AppendString(nil), p.lex.Token) + } + funcName := unescapeIdent(p.lex.Token) + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find next token after %q|%q: %w", firstArg.AppendString(nil), funcName, err) + } + fe := &FuncExpr{ + FuncName: funcName, + printState: printStateChained, + } + if p.lex.Token != "(" { + fe.Args = []*ArgExpr{firstArg} + } else { + args, err := p.parseArgs() + if err != nil { + return nil, fmt.Errorf("cannot parse args for %q|%q: %w", firstArg.AppendString(nil), funcName, err) + } + fe.Args = append([]*ArgExpr{firstArg}, args...) + } + if p.lex.Token != "|" { + return fe, nil + } + firstArg = &ArgExpr{ + Expr: fe, + } + } +} + +func (p *parser) parseArgs() ([]*ArgExpr, error) { + var args []*ArgExpr + for { + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find arg #%d: %w", len(args), err) + } + if p.lex.Token == ")" { + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find next token after function args: %w", err) + } + return args, nil + } + expr, err := p.parseExpr() + if err != nil { + return nil, fmt.Errorf("cannot parse arg #%d: %w", len(args), err) + } + if p.lex.Token == "=" { + // Named expression + me, ok := expr.(*MetricExpr) + if !ok { + return nil, fmt.Errorf("expecting a name for named expression; got %q", expr.AppendString(nil)) + } + argName := me.Query + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find named value for %q: %w", argName, err) + } + argValue, err := p.parseExpr() + if err != nil { + return nil, fmt.Errorf("cannot parse named value for %q: %w", argName, err) + } + args = append(args, &ArgExpr{ + Name: argName, + Expr: argValue, + }) + } else { + args = append(args, &ArgExpr{ + Expr: expr, + }) + } + switch p.lex.Token { + case ",": + // Continue parsing args + case ")": + // End of args + if err := p.lex.Next(); err != nil { + return nil, fmt.Errorf("cannot find next token after func args: %w", err) + } + return args, nil + default: + return nil, fmt.Errorf("unexpected token detected in func args: %q", p.lex.Token) + } + } +} + +// ArgExpr represents function arg (which may be named). +type ArgExpr struct { + // Name is named arg name. It is empty for positional arg. + Name string + + // Expr arg expression. + Expr Expr +} + +// AppendString appends string representation of ae to dst and returns the result. +func (ae *ArgExpr) AppendString(dst []byte) []byte { + if ae.Name != "" { + dst = appendEscapedIdent(dst, ae.Name) + dst = append(dst, '=') + } + dst = ae.Expr.AppendString(dst) + return dst +} + +// FuncExpr represents function call. +type FuncExpr struct { + // FuncName is the function name + FuncName string + + // Args is function args. + Args []*ArgExpr + + printState funcPrintState +} + +type funcPrintState int + +const ( + // Normal func call: `func(arg1, ..., argN)` + printStateNormal = funcPrintState(0) + + // Chained func call: `arg1|func(arg2, ..., argN)` + printStateChained = funcPrintState(1) +) + +// AppendString appends string representation of fe to dst and returns the result. +func (fe *FuncExpr) AppendString(dst []byte) []byte { + switch fe.printState { + case printStateNormal: + dst = appendEscapedIdent(dst, fe.FuncName) + dst = appendArgsString(dst, fe.Args) + case printStateChained: + if len(fe.Args) == 0 { + panic("BUG: chained func call must have at least a single arg") + } + firstArg := fe.Args[0] + tailArgs := fe.Args[1:] + if firstArg.Name != "" { + panic("BUG: the first chained arg must have no name") + } + dst = firstArg.AppendString(dst) + dst = append(dst, '|') + dst = appendEscapedIdent(dst, fe.FuncName) + if len(tailArgs) > 0 { + dst = appendArgsString(dst, tailArgs) + } + default: + panic(fmt.Sprintf("BUG: unexpected printState=%d", fe.printState)) + } + return dst +} + +// MetricExpr represents metric expression. +type MetricExpr struct { + // Query is the query for fetching metrics. + Query string +} + +// AppendString append string representation of me to dst and returns the result. +func (me *MetricExpr) AppendString(dst []byte) []byte { + return appendEscapedIdent(dst, me.Query) +} + +func appendArgsString(dst []byte, args []*ArgExpr) []byte { + dst = append(dst, '(') + for i, arg := range args { + dst = arg.AppendString(dst) + if i+1 < len(args) { + dst = append(dst, ',') + } + } + dst = append(dst, ')') + return dst +} diff --git a/app/vmselect/graphiteql/parser_test.go b/app/vmselect/graphiteql/parser_test.go new file mode 100644 index 000000000..287c4eac2 --- /dev/null +++ b/app/vmselect/graphiteql/parser_test.go @@ -0,0 +1,121 @@ +package graphiteql + +import ( + "testing" +) + +func TestQuoteString(t *testing.T) { + f := func(s, qExpected string) { + t.Helper() + q := QuoteString(s) + if q != qExpected { + t.Fatalf("unexpected result from QuoteString(%q); got %s; want %s", s, q, qExpected) + } + } + f(``, `''`) + f(`foo`, `'foo'`) + f(`f'o\ba"r`, `'f\'o\\ba"r'`) +} + +func TestParseSuccess(t *testing.T) { + another := func(s, resultExpected string) { + t.Helper() + expr, err := Parse(s) + if err != nil { + t.Fatalf("unexpected error when parsing %s: %s", s, err) + } + result := expr.AppendString(nil) + if string(result) != resultExpected { + t.Fatalf("unexpected result when marshaling %s;\ngot\n%s\nwant\n%s", s, result, resultExpected) + } + } + same := func(s string) { + t.Helper() + another(s, s) + } + // Metric expressions + same("a") + same("foo.bar.baz") + same("foo.bar.baz:aa:bb") + same("fOO.*.b[a-z]R{aa*,bb}s_s.$aaa") + same("*") + same("*.foo") + same("{x,y}.z") + same("[x-zaBc]DeF") + another(`\f\ oo`, `f\ oo`) + another(`f\x1B\x3a`, `f\x1b:`) + + // booleans + same("True") + same("False") + another("true", "True") + another("faLSe", "False") + + // Numbers + same("123") + same("-123") + another("+123", "123") + same("12.3") + same("-1.23") + another("+1.23", "1.23") + another("123e5", "1.23e+07") + another("-123e5", "-1.23e+07") + another("+123e5", "1.23e+07") + another("1.23E5", "123000") + another("-1.23e5", "-123000") + another("+1.23e5", "123000") + another("0xab", "171") + another("0b1011101", "93") + another("0O12345", "5349") + + // strings + another(`"foo'"`, `'foo\''`) + same(`'fo\'o"b\\ar'`) + another(`"f\\oo\.bar\1"`, `'f\\oo\\.bar\\1'`) + + // function calls + same("foo()") + another("foo(bar,)", "foo(bar)") + same("foo(bar,123,'baz')") + another("foo(foo(bar), BAZ = xx ( 123, x))", `foo(foo(bar),BAZ=xx(123,x))`) + + // chained functions + another("foo | bar", "foo|bar") + same("foo|bar|baz") + same("foo(sss)|bar(aa)|xxx.ss") + another(`foo|bar(1,"sdf")`, `foo|bar(1,'sdf')`) + + // mix + same(`f(a,xx=b|c|aa(124,'456'),aa=bb)`) +} + +func TestParseFailure(t *testing.T) { + f := func(s string) { + t.Helper() + expr, err := Parse(s) + if err == nil { + t.Fatalf("expecting error when parsing %s", s) + } + if expr != nil { + t.Fatalf("expecting nil expr when parsing %s; got %s", s, expr.AppendString(nil)) + } + } + f("") + f("'asdf") + f("foo bar") + f("f(a") + f("f(1.2.3") + f("foo|bar(") + f("+foo") + f("-bar") + f("123 '") + f("f '") + f("f|") + f("f|'") + f("f|123") + f("f('") + f("f(f()=123)") + f("f(a=')") + f("f(a=foo(") + f("f()'") +}