From 855c25b6c4d3f24378b84abc6ba32116e13c8e2f Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Wed, 25 Oct 2023 20:48:11 +0800 Subject: [PATCH] remove vmalert-tool code from branch cluster (#5229) Follow up https://github.com/VictoriaMetrics/VictoriaMetrics/commit/130e0ea5f02ba6d6238b530fdee90fb0c36462a5. vmalert-tool can't be easily adapted for vmcluster now, cause it needs to set up the whole vmcluster[vminsert+vmstorage+vmselect] first. You can use vmalert-tool to run unit tests for alerting and recording rules. It will perform the following actions: - sets up an isolated VictoriaMetrics instance; - simulates the periodic ingestion of time series; - queries the ingested data for recording and alerting rules evaluation like vmalert; But component packages have functions that not exported and variables with same name, so to implement this for cluster will need amount of code refactor and doesn't look like a good thing to themselves. So I want to remove it from the cluster branch. --- app/vmalert-tool/Makefile | 103 ---- app/vmalert-tool/main.go | 54 --- app/vmalert-tool/unittest/alerting.go | 19 - app/vmalert-tool/unittest/input.go | 182 ------- app/vmalert-tool/unittest/input_test.go | 93 ---- app/vmalert-tool/unittest/recording.go | 100 ---- .../testdata/disable-group-label.yaml | 43 -- .../unittest/testdata/failed-test.yaml | 49 -- .../unittest/testdata/long-period.yaml | 30 -- app/vmalert-tool/unittest/testdata/rules.yaml | 39 -- app/vmalert-tool/unittest/testdata/test1.yaml | 99 ---- app/vmalert-tool/unittest/testdata/test2.yaml | 46 -- app/vmalert-tool/unittest/type.go | 83 ---- app/vmalert-tool/unittest/unittest.go | 443 ------------------ app/vmalert-tool/unittest/unittest_test.go | 47 -- 15 files changed, 1430 deletions(-) delete mode 100644 app/vmalert-tool/Makefile delete mode 100644 app/vmalert-tool/main.go delete mode 100644 app/vmalert-tool/unittest/alerting.go delete mode 100644 app/vmalert-tool/unittest/input.go delete mode 100644 app/vmalert-tool/unittest/input_test.go delete mode 100644 app/vmalert-tool/unittest/recording.go delete mode 100644 app/vmalert-tool/unittest/testdata/disable-group-label.yaml delete mode 100644 app/vmalert-tool/unittest/testdata/failed-test.yaml delete mode 100644 app/vmalert-tool/unittest/testdata/long-period.yaml delete mode 100644 app/vmalert-tool/unittest/testdata/rules.yaml delete mode 100644 app/vmalert-tool/unittest/testdata/test1.yaml delete mode 100644 app/vmalert-tool/unittest/testdata/test2.yaml delete mode 100644 app/vmalert-tool/unittest/type.go delete mode 100644 app/vmalert-tool/unittest/unittest.go delete mode 100644 app/vmalert-tool/unittest/unittest_test.go diff --git a/app/vmalert-tool/Makefile b/app/vmalert-tool/Makefile deleted file mode 100644 index dbb6b373a..000000000 --- a/app/vmalert-tool/Makefile +++ /dev/null @@ -1,103 +0,0 @@ -# All these commands must run from repository root. - -vmalert-tool: - APP_NAME=vmalert-tool $(MAKE) app-local - -vmalert-tool-race: - APP_NAME=vmalert-tool RACE=-race $(MAKE) app-local - -vmalert-tool-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker - -vmalert-tool-pure-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-pure - -vmalert-tool-linux-amd64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-amd64 - -vmalert-tool-linux-arm-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm - -vmalert-tool-linux-arm64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm64 - -vmalert-tool-linux-ppc64le-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-ppc64le - -vmalert-tool-linux-386-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386 - -vmalert-tool-darwin-amd64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64 - -vmalert-tool-darwin-arm64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-arm64 - -vmalert-tool-freebsd-amd64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-freebsd-amd64 - -vmalert-tool-openbsd-amd64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-openbsd-amd64 - -vmalert-tool-windows-amd64-prod: - APP_NAME=vmalert-tool $(MAKE) app-via-docker-windows-amd64 - -package-vmalert-tool: - APP_NAME=vmalert-tool $(MAKE) package-via-docker - -package-vmalert-tool-pure: - APP_NAME=vmalert-tool $(MAKE) package-via-docker-pure - -package-vmalert-tool-amd64: - APP_NAME=vmalert-tool $(MAKE) package-via-docker-amd64 - -package-vmalert-tool-arm: - APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm - -package-vmalert-tool-arm64: - APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm64 - -package-vmalert-tool-ppc64le: - APP_NAME=vmalert-tool $(MAKE) package-via-docker-ppc64le - -package-vmalert-tool-386: - APP_NAME=vmalert-tool $(MAKE) package-via-docker-386 - -publish-vmalert-tool: - APP_NAME=vmalert-tool $(MAKE) publish-via-docker - -vmalert-tool-linux-amd64: - APP_NAME=vmalert-tool CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch - -vmalert-tool-linux-arm: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch - -vmalert-tool-linux-arm64: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch - -vmalert-tool-linux-ppc64le: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch - -vmalert-tool-linux-s390x: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch - -vmalert-tool-linux-386: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch - -vmalert-tool-darwin-amd64: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch - -vmalert-tool-darwin-arm64: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch - -vmalert-tool-freebsd-amd64: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch - -vmalert-tool-openbsd-amd64: - APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch - -vmalert-tool-windows-amd64: - GOARCH=amd64 APP_NAME=vmalert-tool $(MAKE) app-local-windows-goarch - -vmalert-tool-pure: - APP_NAME=vmalert-tool $(MAKE) app-local-pure diff --git a/app/vmalert-tool/main.go b/app/vmalert-tool/main.go deleted file mode 100644 index 790cebd8b..000000000 --- a/app/vmalert-tool/main.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "time" - - "github.com/urfave/cli/v2" - - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert-tool/unittest" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo" -) - -func main() { - start := time.Now() - app := &cli.App{ - Name: "vmalert-tool", - Usage: "VMAlert command-line tool", - UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html", - Version: buildinfo.Version, - Commands: []*cli.Command{ - { - Name: "unittest", - Usage: "Run unittest for alerting and recording rules.", - UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules", - Flags: []cli.Flag{ - &cli.StringSliceFlag{ - Name: "files", - Usage: "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.", - Required: true, - }, - &cli.BoolFlag{ - Name: "disableAlertgroupLabel", - Usage: "disable adding group's Name as label to generated alerts and time series.", - Required: false, - }, - }, - Action: func(c *cli.Context) error { - if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed { - return fmt.Errorf("unittest failed") - } - return nil - }, - }, - }, - } - - err := app.Run(os.Args) - if err != nil { - log.Fatalln(err) - } - log.Printf("Total time: %v", time.Since(start)) -} diff --git a/app/vmalert-tool/unittest/alerting.go b/app/vmalert-tool/unittest/alerting.go deleted file mode 100644 index a9ffe8a79..000000000 --- a/app/vmalert-tool/unittest/alerting.go +++ /dev/null @@ -1,19 +0,0 @@ -package unittest - -import ( - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" -) - -// alertTestCase holds alert_rule_test cases defined in test file -type alertTestCase struct { - EvalTime *promutils.Duration `yaml:"eval_time"` - GroupName string `yaml:"groupname"` - Alertname string `yaml:"alertname"` - ExpAlerts []expAlert `yaml:"exp_alerts"` -} - -// expAlert holds exp_alerts defined in test file -type expAlert struct { - ExpLabels map[string]string `yaml:"exp_labels"` - ExpAnnotations map[string]string `yaml:"exp_annotations"` -} diff --git a/app/vmalert-tool/unittest/input.go b/app/vmalert-tool/unittest/input.go deleted file mode 100644 index af4254a80..000000000 --- a/app/vmalert-tool/unittest/input.go +++ /dev/null @@ -1,182 +0,0 @@ -package unittest - -import ( - "bytes" - "fmt" - "io" - "net/http" - "regexp" - "strconv" - "strings" - "time" - - testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" - "github.com/VictoriaMetrics/metricsql" -) - -// series holds input_series defined in the test file -type series struct { - Series string `yaml:"series"` - Values string `yaml:"values"` -} - -// sequenceValue is an omittable value in a sequence of time series values. -type sequenceValue struct { - Value float64 - Omitted bool -} - -func httpWrite(address string, r io.Reader) { - resp, err := http.Post(address, "", r) - if err != nil { - logger.Fatalf("failed to send to storage: %v", err) - } - resp.Body.Close() -} - -// writeInputSeries send input series to vmstorage and flush them -func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error { - r := testutil.WriteRequest{} - for _, data := range input { - expr, err := metricsql.Parse(data.Series) - if err != nil { - return fmt.Errorf("failed to parse series %s: %v", data.Series, err) - } - promvals, err := parseInputValue(data.Values, true) - if err != nil { - return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err) - } - metricExpr, ok := expr.(*metricsql.MetricExpr) - if !ok { - return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err) - } - samples := make([]testutil.Sample, 0, len(promvals)) - ts := startStamp - for _, v := range promvals { - if !v.Omitted { - samples = append(samples, testutil.Sample{ - Timestamp: ts.UnixMilli(), - Value: v.Value, - }) - } - ts = ts.Add(interval.Duration()) - } - var ls []testutil.Label - for _, filter := range metricExpr.LabelFilterss[0] { - ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value}) - } - r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples}) - } - - data, err := testutil.Compress(r) - if err != nil { - return fmt.Errorf("failed to compress data: %v", err) - } - // write input series to vm - httpWrite(dst, bytes.NewBuffer(data)) - vmstorage.Storage.DebugFlush() - return nil -} - -// parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test. -func parseInputValue(input string, origin bool) ([]sequenceValue, error) { - var res []sequenceValue - items := strings.Split(input, " ") - reg := regexp.MustCompile(`\D?\d*\D?`) - for _, item := range items { - if item == "stale" { - res = append(res, sequenceValue{Value: decimal.StaleNaN}) - continue - } - vals := reg.FindAllString(item, -1) - switch len(vals) { - case 1: - if vals[0] == "_" { - res = append(res, sequenceValue{Omitted: true}) - continue - } - v, err := strconv.ParseFloat(vals[0], 64) - if err != nil { - return nil, err - } - res = append(res, sequenceValue{Value: v}) - continue - case 2: - p1 := vals[0][:len(vals[0])-1] - v2, err := strconv.ParseInt(vals[1], 10, 64) - if err != nil { - return nil, err - } - option := vals[0][len(vals[0])-1] - switch option { - case '+': - v1, err := strconv.ParseFloat(p1, 64) - if err != nil { - return nil, err - } - res = append(res, sequenceValue{Value: v1 + float64(v2)}) - case 'x': - for i := int64(0); i <= v2; i++ { - if p1 == "_" { - if i == 0 { - i = 1 - } - res = append(res, sequenceValue{Omitted: true}) - continue - } - v1, err := strconv.ParseFloat(p1, 64) - if err != nil { - return nil, err - } - if !origin || v1 == 0 { - res = append(res, sequenceValue{Value: v1 * float64(i)}) - continue - } - newVal := fmt.Sprintf("%s+0x%s", p1, vals[1]) - newRes, err := parseInputValue(newVal, false) - if err != nil { - return nil, err - } - res = append(res, newRes...) - break - } - - default: - return nil, fmt.Errorf("got invalid operation %b", option) - } - case 3: - r1, err := parseInputValue(fmt.Sprintf("%s%s", vals[1], vals[2]), false) - if err != nil { - return nil, err - } - p1 := vals[0][:len(vals[0])-1] - v1, err := strconv.ParseFloat(p1, 64) - if err != nil { - return nil, err - } - option := vals[0][len(vals[0])-1] - var isAdd bool - if option == '+' { - isAdd = true - } - for _, r := range r1 { - if isAdd { - res = append(res, sequenceValue{ - Value: r.Value + v1, - }) - } else { - res = append(res, sequenceValue{ - Value: v1 - r.Value, - }) - } - } - default: - return nil, fmt.Errorf("unsupported input %s", input) - } - } - return res, nil -} diff --git a/app/vmalert-tool/unittest/input_test.go b/app/vmalert-tool/unittest/input_test.go deleted file mode 100644 index 5f74de98b..000000000 --- a/app/vmalert-tool/unittest/input_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package unittest - -import ( - "testing" - - "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" -) - -func TestParseInputValue(t *testing.T) { - testCases := []struct { - input string - exp []sequenceValue - failed bool - }{ - { - "", - nil, - true, - }, - { - "testfailed", - nil, - true, - }, - // stale doesn't support operations - { - "stalex3", - nil, - true, - }, - { - "-4", - []sequenceValue{{Value: -4}}, - false, - }, - { - "_", - []sequenceValue{{Omitted: true}}, - false, - }, - { - "stale", - []sequenceValue{{Value: decimal.StaleNaN}}, - false, - }, - { - "-4x1", - []sequenceValue{{Value: -4}, {Value: -4}}, - false, - }, - { - "_x1", - []sequenceValue{{Omitted: true}}, - false, - }, - { - "1+1x4", - []sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 4}, {Value: 5}}, - false, - }, - { - "2-1x4", - []sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}}, - false, - }, - { - "1+1x1 _ -4 stale 3+20x1", - []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}}, - false, - }, - } - - for _, tc := range testCases { - output, err := parseInputValue(tc.input, true) - if err != nil != tc.failed { - t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil) - } - if len(tc.exp) != len(output) { - t.Fatalf("expect %v, got %v", tc.exp, output) - } - for i := 0; i < len(tc.exp); i++ { - if tc.exp[i].Omitted != output[i].Omitted { - t.Fatalf("expect %v, got %v", tc.exp, output) - } - if tc.exp[i].Value != output[i].Value { - if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) { - continue - } - t.Fatalf("expect %v, got %v", tc.exp, output) - } - } - } -} diff --git a/app/vmalert-tool/unittest/recording.go b/app/vmalert-tool/unittest/recording.go deleted file mode 100644 index a47d104d6..000000000 --- a/app/vmalert-tool/unittest/recording.go +++ /dev/null @@ -1,100 +0,0 @@ -package unittest - -import ( - "context" - "fmt" - "net/url" - "reflect" - "sort" - "time" - - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" - "github.com/VictoriaMetrics/metricsql" -) - -// metricsqlTestCase holds metricsql_expr_test cases defined in test file -type metricsqlTestCase struct { - Expr string `yaml:"expr"` - EvalTime *promutils.Duration `yaml:"eval_time"` - ExpSamples []expSample `yaml:"exp_samples"` -} - -type expSample struct { - Labels string `yaml:"labels"` - Value float64 `yaml:"value"` -} - -// checkMetricsqlCase will check metricsql_expr_test cases -func checkMetricsqlCase(cases []metricsqlTestCase, q datasource.QuerierBuilder) (checkErrs []error) { - queries := q.BuildWithParams(datasource.QuerierParams{QueryParams: url.Values{"nocache": {"1"}, "latency_offset": {"1ms"}}, DataSourceType: "prometheus"}) -Outer: - for _, mt := range cases { - result, _, err := queries.Query(context.Background(), mt.Expr, durationToTime(mt.EvalTime)) - if err != nil { - checkErrs = append(checkErrs, fmt.Errorf(" expr: %q, time: %s, err: %w", mt.Expr, - mt.EvalTime.Duration().String(), err)) - continue - } - var gotSamples []parsedSample - for _, s := range result.Data { - sort.Slice(s.Labels, func(i, j int) bool { - return s.Labels[i].Name < s.Labels[j].Name - }) - gotSamples = append(gotSamples, parsedSample{ - Labels: s.Labels, - Value: s.Values[0], - }) - } - var expSamples []parsedSample - for _, s := range mt.ExpSamples { - expLb := datasource.Labels{} - if s.Labels != "" { - metricsqlExpr, err := metricsql.Parse(s.Labels) - if err != nil { - checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s, err: %v", mt.Expr, - mt.EvalTime.Duration().String(), fmt.Errorf("failed to parse labels %q: %w", s.Labels, err))) - continue Outer - } - metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr) - if !ok { - checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s, err: %v", mt.Expr, - mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type"))) - continue Outer - } - for _, l := range metricsqlMetricExpr.LabelFilterss[0] { - expLb = append(expLb, datasource.Label{ - Name: l.Label, - Value: l.Value, - }) - } - } - sort.Slice(expLb, func(i, j int) bool { - return expLb[i].Name < expLb[j].Name - }) - expSamples = append(expSamples, parsedSample{ - Labels: expLb, - Value: s.Value, - }) - } - sort.Slice(expSamples, func(i, j int) bool { - return datasource.LabelCompare(expSamples[i].Labels, expSamples[j].Labels) <= 0 - }) - sort.Slice(gotSamples, func(i, j int) bool { - return datasource.LabelCompare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0 - }) - if !reflect.DeepEqual(expSamples, gotSamples) { - checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s,\n exp: %v\n got: %v", mt.Expr, - mt.EvalTime.Duration().String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples))) - } - - } - return -} - -func durationToTime(pd *promutils.Duration) time.Time { - if pd == nil { - return time.Time{} - } - return time.UnixMilli(pd.Duration().Milliseconds()) -} diff --git a/app/vmalert-tool/unittest/testdata/disable-group-label.yaml b/app/vmalert-tool/unittest/testdata/disable-group-label.yaml deleted file mode 100644 index 6060710b6..000000000 --- a/app/vmalert-tool/unittest/testdata/disable-group-label.yaml +++ /dev/null @@ -1,43 +0,0 @@ -rule_files: - - rules.yaml - -evaluation_interval: 1m - -tests: - - interval: 1m - input_series: - - series: 'up{job="vmagent2", instance="localhost:9090"}' - values: "0+0x1440" - - metricsql_expr_test: - - expr: suquery_interval_test - eval_time: 4m - exp_samples: - - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}' - value: 1 - - alert_rule_test: - - eval_time: 2h - alertname: InstanceDown - exp_alerts: - - exp_labels: - job: vmagent2 - severity: page - instance: localhost:9090 - datacenter: dc-123 - exp_annotations: - summary: "Instance localhost:9090 down" - description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes." - - - eval_time: 0 - alertname: AlwaysFiring - exp_alerts: - - exp_labels: - datacenter: dc-123 - - - eval_time: 0 - alertname: InstanceDown - exp_alerts: [] - - external_labels: - datacenter: dc-123 diff --git a/app/vmalert-tool/unittest/testdata/failed-test.yaml b/app/vmalert-tool/unittest/testdata/failed-test.yaml deleted file mode 100644 index 0a1de6365..000000000 --- a/app/vmalert-tool/unittest/testdata/failed-test.yaml +++ /dev/null @@ -1,49 +0,0 @@ -rule_files: - - rules.yaml - -tests: - - interval: 1m - name: "Failing test" - input_series: - - series: test - values: "0" - - metricsql_expr_test: - - expr: test - eval_time: 0m - exp_samples: - - value: 0 - labels: test - - # will failed cause there is no "Test" group and rule defined - alert_rule_test: - - eval_time: 0m - groupname: Test - alertname: Test - exp_alerts: - - exp_labels: {} - - - interval: 1m - name: Failing alert test - input_series: - - series: 'up{job="test"}' - values: 0x10 - - alert_rule_test: - # will failed cause rule is firing - - eval_time: 5m - groupname: group1 - alertname: InstanceDown - exp_alerts: [] - - - interval: 1m - name: Failing alert test with missing groupname - input_series: - - series: 'up{job="test"}' - values: 0x10 - - alert_rule_test: - # will failed cause missing groupname - - eval_time: 5m - alertname: AlwaysFiring - exp_alerts: [] diff --git a/app/vmalert-tool/unittest/testdata/long-period.yaml b/app/vmalert-tool/unittest/testdata/long-period.yaml deleted file mode 100644 index 052470b69..000000000 --- a/app/vmalert-tool/unittest/testdata/long-period.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# can be executed successfully but will take more than 1 minute -# not included in unit test now -evaluation_interval: 100d - -rule_files: - - rules.yaml - -tests: - - interval: 1d - input_series: - - series: test - # Max time in time.Duration is 106751d from 1970 (2^63/10^9), i.e. 2262. - # But VictoriaMetrics supports maxTimestamp value +2 days from now. see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/827. - # We input series to 2024-01-01T00:00:00 here. - values: "0+1x19723" - - metricsql_expr_test: - - expr: timestamp(test) - eval_time: 0m - exp_samples: - - value: 0 - - expr: test - eval_time: 100d - exp_samples: - - labels: test - value: 100 - - expr: timestamp(test) - eval_time: 19000d - exp_samples: - - value: 1641600000 # 19000d -> seconds. diff --git a/app/vmalert-tool/unittest/testdata/rules.yaml b/app/vmalert-tool/unittest/testdata/rules.yaml deleted file mode 100644 index f7455c6d8..000000000 --- a/app/vmalert-tool/unittest/testdata/rules.yaml +++ /dev/null @@ -1,39 +0,0 @@ -groups: - - name: group1 - rules: - - alert: InstanceDown - expr: up == 0 - for: 5m - labels: - severity: page - annotations: - summary: "Instance {{ $labels.instance }} down" - description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." - - alert: AlwaysFiring - expr: 1 - - alert: SameAlertNameWithDifferentGroup - expr: absent(test) - for: 1m - - - name: group2 - rules: - - record: t1 - expr: test - - record: job:test:count_over_time1m - expr: sum without(instance) (count_over_time(test[1m])) - - record: suquery_interval_test - expr: count_over_time(up[5m:]) - - - alert: SameAlertNameWithDifferentGroup - expr: absent(test) - for: 5m - - - name: group3 - rules: - - record: t2 - expr: t1 - - - name: group4 - rules: - - record: t3 - expr: t1 diff --git a/app/vmalert-tool/unittest/testdata/test1.yaml b/app/vmalert-tool/unittest/testdata/test1.yaml deleted file mode 100644 index d60e8587f..000000000 --- a/app/vmalert-tool/unittest/testdata/test1.yaml +++ /dev/null @@ -1,99 +0,0 @@ -rule_files: - - rules.yaml - -evaluation_interval: 1m -group_eval_order: ["group4", "group2", "group3"] - -tests: - - interval: 1m - name: "basic test" - input_series: - - series: "test" - values: "_x5 1x5 _ stale" - - alert_rule_test: - - eval_time: 1m - groupname: group1 - alertname: SameAlertNameWithDifferentGroup - exp_alerts: - - {} - - eval_time: 1m - groupname: group2 - alertname: SameAlertNameWithDifferentGroup - exp_alerts: [] - - eval_time: 6m - groupname: group1 - alertname: SameAlertNameWithDifferentGroup - exp_alerts: [] - - metricsql_expr_test: - - expr: test - eval_time: 11m - exp_samples: - - labels: '{__name__="test"}' - value: 1 - - expr: test - eval_time: 12m - exp_samples: [] - - - interval: 1m - name: "basic test2" - input_series: - - series: 'up{job="vmagent1", instance="localhost:9090"}' - values: "0+0x1440" - - series: "test" - values: "0+1x1440" - - metricsql_expr_test: - - expr: count(ALERTS) by (alertgroup, alertname, alertstate) - eval_time: 4m - exp_samples: - - labels: '{alertgroup="group1", alertname="AlwaysFiring", alertstate="firing"}' - value: 1 - - labels: '{alertgroup="group1", alertname="InstanceDown", alertstate="pending"}' - value: 1 - - expr: t1 - eval_time: 4m - exp_samples: - - value: 4 - labels: '{__name__="t1", datacenter="dc-123"}' - - expr: t2 - eval_time: 4m - exp_samples: - - value: 4 - labels: '{__name__="t2", datacenter="dc-123"}' - - expr: t3 - eval_time: 4m - exp_samples: - # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1 - - value: 3 - labels: '{__name__="t3", datacenter="dc-123"}' - - alert_rule_test: - - eval_time: 10m - groupname: group1 - alertname: InstanceDown - exp_alerts: - - exp_labels: - job: vmagent1 - severity: page - instance: localhost:9090 - datacenter: dc-123 - exp_annotations: - summary: "Instance localhost:9090 down" - description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes." - - - eval_time: 0 - groupname: group1 - alertname: AlwaysFiring - exp_alerts: - - exp_labels: - datacenter: dc-123 - - - eval_time: 0 - groupname: alerts - alertname: InstanceDown - exp_alerts: [] - - external_labels: - datacenter: dc-123 diff --git a/app/vmalert-tool/unittest/testdata/test2.yaml b/app/vmalert-tool/unittest/testdata/test2.yaml deleted file mode 100644 index 2d7825742..000000000 --- a/app/vmalert-tool/unittest/testdata/test2.yaml +++ /dev/null @@ -1,46 +0,0 @@ -rule_files: - - rules.yaml - -evaluation_interval: 1m - -tests: - - interval: 1m - input_series: - - series: 'up{job="vmagent2", instance="localhost:9090"}' - values: "0+0x1440" - - metricsql_expr_test: - - expr: suquery_interval_test - eval_time: 4m - exp_samples: - - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}' - value: 1 - - alert_rule_test: - - eval_time: 2h - groupname: group1 - alertname: InstanceDown - exp_alerts: - - exp_labels: - job: vmagent2 - severity: page - instance: localhost:9090 - datacenter: dc-123 - exp_annotations: - summary: "Instance localhost:9090 down" - description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes." - - - eval_time: 0 - groupname: group1 - alertname: AlwaysFiring - exp_alerts: - - exp_labels: - datacenter: dc-123 - - - eval_time: 0 - groupname: group1 - alertname: InstanceDown - exp_alerts: [] - - external_labels: - datacenter: dc-123 diff --git a/app/vmalert-tool/unittest/type.go b/app/vmalert-tool/unittest/type.go deleted file mode 100644 index 8915a8f5c..000000000 --- a/app/vmalert-tool/unittest/type.go +++ /dev/null @@ -1,83 +0,0 @@ -package unittest - -import ( - "fmt" - "strconv" - "strings" - - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" -) - -// parsedSample is a sample with parsed Labels -type parsedSample struct { - Labels datasource.Labels - Value float64 -} - -func (ps *parsedSample) String() string { - return ps.Labels.String() + " " + strconv.FormatFloat(ps.Value, 'E', -1, 64) -} - -func parsedSamplesString(pss []parsedSample) string { - if len(pss) == 0 { - return "nil" - } - s := pss[0].String() - for _, ps := range pss[1:] { - s += ", " + ps.String() - } - return s -} - -// labelAndAnnotation holds labels and annotations -type labelAndAnnotation struct { - Labels datasource.Labels - Annotations datasource.Labels -} - -func (la *labelAndAnnotation) String() string { - return "Labels:" + la.Labels.String() + "\nAnnotations:" + la.Annotations.String() -} - -// labelsAndAnnotations is collection of LabelAndAnnotation -type labelsAndAnnotations []labelAndAnnotation - -func (la labelsAndAnnotations) Len() int { return len(la) } - -func (la labelsAndAnnotations) Swap(i, j int) { la[i], la[j] = la[j], la[i] } -func (la labelsAndAnnotations) Less(i, j int) bool { - diff := datasource.LabelCompare(la[i].Labels, la[j].Labels) - if diff != 0 { - return diff < 0 - } - return datasource.LabelCompare(la[i].Annotations, la[j].Annotations) < 0 -} - -func (la labelsAndAnnotations) String() string { - if len(la) == 0 { - return "[]" - } - s := "[\n0:" + indentLines("\n"+la[0].String(), " ") - for i, l := range la[1:] { - s += ",\n" + fmt.Sprintf("%d", i+1) + ":" + indentLines("\n"+l.String(), " ") - } - s += "\n]" - - return s -} - -// indentLines prefixes each line in the supplied string with the given "indent" string. -func indentLines(lines, indent string) string { - sb := strings.Builder{} - n := strings.Split(lines, "\n") - for i, l := range n { - if i > 0 { - sb.WriteString(indent) - } - sb.WriteString(l) - if i != len(n)-1 { - sb.WriteRune('\n') - } - } - return sb.String() -} diff --git a/app/vmalert-tool/unittest/unittest.go b/app/vmalert-tool/unittest/unittest.go deleted file mode 100644 index 424a6132e..000000000 --- a/app/vmalert-tool/unittest/unittest.go +++ /dev/null @@ -1,443 +0,0 @@ -package unittest - -import ( - "context" - "flag" - "fmt" - "net/http" - "os" - "path/filepath" - "reflect" - "sort" - "time" - - "gopkg.in/yaml.v2" - - vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" - "github.com/VictoriaMetrics/metrics" -) - -var ( - storagePath string - httpListenAddr = ":8880" - // insert series from 1970-01-01T00:00:00 - testStartTime = time.Unix(0, 0).UTC() - - testPromWriteHTTPPath = "http://127.0.0.1" + httpListenAddr + "/api/v1/write" - testDataSourcePath = "http://127.0.0.1" + httpListenAddr + "/prometheus" - testRemoteWritePath = "http://127.0.0.1" + httpListenAddr - testHealthHTTPPath = "http://127.0.0.1" + httpListenAddr + "/health" - - disableAlertgroupLabel bool -) - -const ( - testStoragePath = "vmalert-unittest" - testLogLevel = "ERROR" -) - -// UnitTest runs unittest for files -func UnitTest(files []string, disableGroupLabel bool) bool { - if err := templates.Load([]string{}, true); err != nil { - logger.Fatalf("failed to load template: %v", err) - } - storagePath = filepath.Join(os.TempDir(), testStoragePath) - processFlags() - vminsert.Init() - vmselect.Init() - // storagePath will be created again when closing vmselect, so remove it again. - defer fs.MustRemoveAll(storagePath) - defer vminsert.Stop() - defer vmselect.Stop() - disableAlertgroupLabel = disableGroupLabel - return rulesUnitTest(files) -} - -func rulesUnitTest(files []string) bool { - var failed bool - for _, f := range files { - if err := ruleUnitTest(f); err != nil { - fmt.Println(" FAILED") - fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err) - failed = true - } else { - fmt.Println(" SUCCESS") - } - } - return failed -} - -func ruleUnitTest(filename string) []error { - fmt.Println("\nUnit Testing: ", filename) - b, err := os.ReadFile(filename) - if err != nil { - return []error{fmt.Errorf("failed to read file: %w", err)} - } - - var unitTestInp unitTestFile - if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil { - return []error{fmt.Errorf("failed to unmarshal file: %w", err)} - } - if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil { - return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)} - } - - if unitTestInp.EvaluationInterval.Duration() == 0 { - fmt.Println("evaluation_interval set to 1m by default") - unitTestInp.EvaluationInterval = &promutils.Duration{D: 1 * time.Minute} - } - - groupOrderMap := make(map[string]int) - for i, gn := range unitTestInp.GroupEvalOrder { - if _, ok := groupOrderMap[gn]; ok { - return []error{fmt.Errorf("group name repeated in `group_eval_order`: %s", gn)} - } - groupOrderMap[gn] = i - } - - testGroups, err := vmalertconfig.Parse(unitTestInp.RuleFiles, nil, true) - if err != nil { - return []error{fmt.Errorf("failed to parse `rule_files`: %w", err)} - } - - var errs []error - for _, t := range unitTestInp.Tests { - if err := verifyTestGroup(t); err != nil { - errs = append(errs, err) - continue - } - testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups) - errs = append(errs, testErrs...) - } - - if len(errs) > 0 { - return errs - } - return nil -} - -func verifyTestGroup(group testGroup) error { - var testGroupName string - if group.TestGroupName != "" { - testGroupName = fmt.Sprintf("testGroupName: %s\n", group.TestGroupName) - } - for _, at := range group.AlertRuleTests { - if at.Alertname == "" { - return fmt.Errorf("\n%s missing required filed \"alertname\"", testGroupName) - } - if !disableAlertgroupLabel && at.GroupName == "" { - return fmt.Errorf("\n%s missing required filed \"groupname\" when flag \"disableAlertGroupLabel\" is false", testGroupName) - } - if disableAlertgroupLabel && at.GroupName != "" { - return fmt.Errorf("\n%s shouldn't set filed \"groupname\" when flag \"disableAlertGroupLabel\" is true", testGroupName) - } - if at.EvalTime == nil { - return fmt.Errorf("\n%s missing required filed \"eval_time\"", testGroupName) - } - } - for _, et := range group.MetricsqlExprTests { - if et.Expr == "" { - return fmt.Errorf("\n%s missing required filed \"expr\"", testGroupName) - } - if et.EvalTime == nil { - return fmt.Errorf("\n%s missing required filed \"eval_time\"", testGroupName) - } - } - return nil -} - -func processFlags() { - flag.Parse() - for _, fv := range []struct { - flag string - value string - }{ - {flag: "storageDataPath", value: storagePath}, - {flag: "loggerLevel", value: testLogLevel}, - {flag: "search.disableCache", value: "true"}, - // set storage retention time to 100 years, allow to store series from 1970-01-01T00:00:00. - {flag: "retentionPeriod", value: "100y"}, - {flag: "datasource.url", value: testDataSourcePath}, - {flag: "remoteWrite.url", value: testRemoteWritePath}, - } { - // panics if flag doesn't exist - if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil { - logger.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err) - } - } -} - -func setUp() { - vmstorage.Init(promql.ResetRollupResultCacheIfNeeded) - go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool { - switch r.URL.Path { - case "/prometheus/api/v1/query": - if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil { - httpserver.Errorf(w, r, "%s", err) - } - return true - case "/prometheus/api/v1/write", "/api/v1/write": - if err := promremotewrite.InsertHandler(r); err != nil { - httpserver.Errorf(w, r, "%s", err) - } - return true - default: - } - return false - }) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - readyCheckFunc := func() bool { - resp, err := http.Get(testHealthHTTPPath) - if err != nil { - return false - } - _ = resp.Body.Close() - return resp.StatusCode == 200 - } -checkCheck: - for { - select { - case <-ctx.Done(): - logger.Fatalf("http server can't be ready in 30s") - default: - if readyCheckFunc() { - break checkCheck - } - time.Sleep(3 * time.Second) - } - } -} - -func tearDown() { - if err := httpserver.Stop(httpListenAddr); err != nil { - logger.Errorf("cannot stop the webservice: %s", err) - } - vmstorage.Stop() - metrics.UnregisterAllMetrics() - fs.MustRemoveAll(storagePath) -} - -// resolveAndGlobFilepaths joins all relative paths in a configuration -// with a given base directory and replaces all globs with matching files. -func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error { - for i, rf := range utf.RuleFiles { - if rf != "" && !filepath.IsAbs(rf) { - utf.RuleFiles[i] = filepath.Join(baseDir, rf) - } - } - - var globbedFiles []string - for _, rf := range utf.RuleFiles { - m, err := filepath.Glob(rf) - if err != nil { - return err - } - if len(m) == 0 { - fmt.Fprintln(os.Stderr, " WARNING: no file match pattern", rf) - } - globbedFiles = append(globbedFiles, m...) - } - utf.RuleFiles = globbedFiles - return nil -} - -func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) { - // set up vmstorage and http server for ingest and read queries - setUp() - // tear down vmstorage and clean the data dir - defer tearDown() - - err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, testPromWriteHTTPPath) - if err != nil { - return []error{err} - } - - q, err := datasource.Init(nil) - if err != nil { - return []error{fmt.Errorf("failed to init datasource: %v", err)} - } - rw, err := remotewrite.NewDebugClient() - if err != nil { - return []error{fmt.Errorf("failed to init wr: %v", err)} - } - - alertEvalTimesMap := map[time.Duration]struct{}{} - alertExpResultMap := map[time.Duration]map[string]map[string][]expAlert{} - for _, at := range tg.AlertRuleTests { - et := at.EvalTime.Duration() - alertEvalTimesMap[et] = struct{}{} - if _, ok := alertExpResultMap[et]; !ok { - alertExpResultMap[et] = make(map[string]map[string][]expAlert) - } - if _, ok := alertExpResultMap[et][at.GroupName]; !ok { - alertExpResultMap[et][at.GroupName] = make(map[string][]expAlert) - } - alertExpResultMap[et][at.GroupName][at.Alertname] = at.ExpAlerts - } - alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap)) - for k := range alertEvalTimesMap { - alertEvalTimes = append(alertEvalTimes, k) - } - sort.Slice(alertEvalTimes, func(i, j int) bool { - return alertEvalTimes[i] < alertEvalTimes[j] - }) - - // sort group eval order according to the given "group_eval_order". - sort.Slice(testGroups, func(i, j int) bool { - return groupOrderMap[testGroups[i].Name] < groupOrderMap[testGroups[j].Name] - }) - - // create groups with given rule - var groups []*rule.Group - for _, group := range testGroups { - ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels) - groups = append(groups, ng) - } - - evalIndex := 0 - maxEvalTime := testStartTime.Add(tg.maxEvalTime()) - for ts := testStartTime; ts.Before(maxEvalTime) || ts.Equal(maxEvalTime); ts = ts.Add(evalInterval) { - for _, g := range groups { - errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts) - for err := range errs { - if err != nil { - checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name, - ts, err)) - } - } - // flush series after each group evaluation - vmstorage.Storage.DebugFlush() - } - - // check alert_rule_test case at every eval time - for evalIndex < len(alertEvalTimes) { - if ts.Sub(testStartTime) > alertEvalTimes[evalIndex] || - alertEvalTimes[evalIndex] >= ts.Add(evalInterval).Sub(testStartTime) { - break - } - gotAlertsMap := map[string]map[string]labelsAndAnnotations{} - for _, g := range groups { - if disableAlertgroupLabel { - g.Name = "" - } - if _, ok := alertExpResultMap[time.Duration(ts.UnixNano())][g.Name]; !ok { - continue - } - if _, ok := gotAlertsMap[g.Name]; !ok { - gotAlertsMap[g.Name] = make(map[string]labelsAndAnnotations) - } - for _, r := range g.Rules { - ar, isAlertRule := r.(*rule.AlertingRule) - if !isAlertRule { - continue - } - if _, ok := alertExpResultMap[time.Duration(ts.UnixNano())][g.Name][ar.Name]; ok { - for _, got := range ar.GetAlerts() { - if got.State != notifier.StateFiring { - continue - } - if disableAlertgroupLabel { - delete(got.Labels, "alertgroup") - } - laa := labelAndAnnotation{ - Labels: datasource.ConvertToLabels(got.Labels), - Annotations: datasource.ConvertToLabels(got.Annotations), - } - gotAlertsMap[g.Name][ar.Name] = append(gotAlertsMap[g.Name][ar.Name], laa) - } - } - - } - } - for groupname, gres := range alertExpResultMap[alertEvalTimes[evalIndex]] { - for alertname, res := range gres { - var expAlerts labelsAndAnnotations - for _, expAlert := range res { - if expAlert.ExpLabels == nil { - expAlert.ExpLabels = make(map[string]string) - } - // alertGroupNameLabel is added as additional labels when `disableAlertGroupLabel` is false - if !disableAlertgroupLabel { - expAlert.ExpLabels["alertgroup"] = groupname - } - // alertNameLabel is added as additional labels in vmalert. - expAlert.ExpLabels["alertname"] = alertname - expAlerts = append(expAlerts, labelAndAnnotation{ - Labels: datasource.ConvertToLabels(expAlert.ExpLabels), - Annotations: datasource.ConvertToLabels(expAlert.ExpAnnotations), - }) - } - sort.Sort(expAlerts) - - gotAlerts := gotAlertsMap[groupname][alertname] - sort.Sort(gotAlerts) - if !reflect.DeepEqual(expAlerts, gotAlerts) { - var testGroupName string - if tg.TestGroupName != "" { - testGroupName = fmt.Sprintf("testGroupName: %s,\n", tg.TestGroupName) - } - expString := indentLines(expAlerts.String(), " ") - gotString := indentLines(gotAlerts.String(), " ") - checkErrs = append(checkErrs, fmt.Errorf("\n%s groupname: %s, alertname: %s, time: %s, \n exp:%v, \n got:%v ", - testGroupName, groupname, alertname, alertEvalTimes[evalIndex].String(), expString, gotString)) - } - } - } - evalIndex++ - } - - } - - checkErrs = append(checkErrs, checkMetricsqlCase(tg.MetricsqlExprTests, q)...) - return checkErrs -} - -// unitTestFile holds the contents of a single unit test file -type unitTestFile struct { - RuleFiles []string `yaml:"rule_files"` - EvaluationInterval *promutils.Duration `yaml:"evaluation_interval"` - GroupEvalOrder []string `yaml:"group_eval_order"` - Tests []testGroup `yaml:"tests"` -} - -// testGroup is a group of input series and test cases associated with it -type testGroup struct { - Interval *promutils.Duration `yaml:"interval"` - InputSeries []series `yaml:"input_series"` - AlertRuleTests []alertTestCase `yaml:"alert_rule_test"` - MetricsqlExprTests []metricsqlTestCase `yaml:"metricsql_expr_test"` - ExternalLabels map[string]string `yaml:"external_labels"` - TestGroupName string `yaml:"name"` -} - -// maxEvalTime returns the max eval time among all alert_rule_test and metricsql_expr_test -func (tg *testGroup) maxEvalTime() time.Duration { - var maxd time.Duration - for _, alert := range tg.AlertRuleTests { - if alert.EvalTime.Duration() > maxd { - maxd = alert.EvalTime.Duration() - } - } - for _, met := range tg.MetricsqlExprTests { - if met.EvalTime.Duration() > maxd { - maxd = met.EvalTime.Duration() - } - } - return maxd -} diff --git a/app/vmalert-tool/unittest/unittest_test.go b/app/vmalert-tool/unittest/unittest_test.go deleted file mode 100644 index c2c014d6e..000000000 --- a/app/vmalert-tool/unittest/unittest_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package unittest - -import ( - "os" - "testing" - - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates" -) - -func TestMain(m *testing.M) { - if err := templates.Load([]string{}, true); err != nil { - os.Exit(1) - } - os.Exit(m.Run()) -} - -func TestUnitRule(t *testing.T) { - testCases := []struct { - name string - disableGroupLabel bool - files []string - failed bool - }{ - { - name: "run multi files", - files: []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, - failed: false, - }, - { - name: "disable group label", - disableGroupLabel: true, - files: []string{"./testdata/disable-group-label.yaml"}, - failed: false, - }, - { - name: "failing test", - files: []string{"./testdata/failed-test.yaml"}, - failed: true, - }, - } - for _, tc := range testCases { - fail := UnitTest(tc.files, tc.disableGroupLabel) - if fail != tc.failed { - t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail) - } - } -}