mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2025-01-10 15:14:09 +00:00
remove vmalert-tool code from branch cluster (#5229)
Follow up 130e0ea5f0.
vmalert-tool can't be easily adapted for vmcluster now, cause it needs to set up the whole vmcluster[vminsert+vmstorage+vmselect] first.
You can use vmalert-tool to run unit tests for alerting and recording rules.
It will perform the following actions:
- sets up an isolated VictoriaMetrics instance;
- simulates the periodic ingestion of time series;
- queries the ingested data for recording and alerting rules evaluation like vmalert;
But component packages have functions that not exported and variables with same name, so to implement this for cluster will need amount of code refactor and doesn't look like a good thing to themselves.
So I want to remove it from the cluster branch.
This commit is contained in:
Hui Wang
GitHub
parent
6c63ca18f5
commit
855c25b6c4
15 changed files with 0 additions and 1430 deletions
|
|
@ -1,103 +0,0 @@
|
||||||
# All these commands must run from repository root.
|
|
||||||
|
|
||||||
vmalert-tool:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-local
|
|
||||||
|
|
||||||
vmalert-tool-race:
|
|
||||||
APP_NAME=vmalert-tool RACE=-race $(MAKE) app-local
|
|
||||||
|
|
||||||
vmalert-tool-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker
|
|
||||||
|
|
||||||
vmalert-tool-pure-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-pure
|
|
||||||
|
|
||||||
vmalert-tool-linux-amd64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-amd64
|
|
||||||
|
|
||||||
vmalert-tool-linux-arm-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm
|
|
||||||
|
|
||||||
vmalert-tool-linux-arm64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm64
|
|
||||||
|
|
||||||
vmalert-tool-linux-ppc64le-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-ppc64le
|
|
||||||
|
|
||||||
vmalert-tool-linux-386-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386
|
|
||||||
|
|
||||||
vmalert-tool-darwin-amd64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64
|
|
||||||
|
|
||||||
vmalert-tool-darwin-arm64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-arm64
|
|
||||||
|
|
||||||
vmalert-tool-freebsd-amd64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-freebsd-amd64
|
|
||||||
|
|
||||||
vmalert-tool-openbsd-amd64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-openbsd-amd64
|
|
||||||
|
|
||||||
vmalert-tool-windows-amd64-prod:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-via-docker-windows-amd64
|
|
||||||
|
|
||||||
package-vmalert-tool:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker
|
|
||||||
|
|
||||||
package-vmalert-tool-pure:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker-pure
|
|
||||||
|
|
||||||
package-vmalert-tool-amd64:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker-amd64
|
|
||||||
|
|
||||||
package-vmalert-tool-arm:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm
|
|
||||||
|
|
||||||
package-vmalert-tool-arm64:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm64
|
|
||||||
|
|
||||||
package-vmalert-tool-ppc64le:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker-ppc64le
|
|
||||||
|
|
||||||
package-vmalert-tool-386:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) package-via-docker-386
|
|
||||||
|
|
||||||
publish-vmalert-tool:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) publish-via-docker
|
|
||||||
|
|
||||||
vmalert-tool-linux-amd64:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-linux-arm:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-linux-arm64:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-linux-ppc64le:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-linux-s390x:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-linux-386:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-darwin-amd64:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-darwin-arm64:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-freebsd-amd64:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-openbsd-amd64:
|
|
||||||
APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
|
||||||
|
|
||||||
vmalert-tool-windows-amd64:
|
|
||||||
GOARCH=amd64 APP_NAME=vmalert-tool $(MAKE) app-local-windows-goarch
|
|
||||||
|
|
||||||
vmalert-tool-pure:
|
|
||||||
APP_NAME=vmalert-tool $(MAKE) app-local-pure
|
|
||||||
|
|
@ -1,54 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert-tool/unittest"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
start := time.Now()
|
|
||||||
app := &cli.App{
|
|
||||||
Name: "vmalert-tool",
|
|
||||||
Usage: "VMAlert command-line tool",
|
|
||||||
UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html",
|
|
||||||
Version: buildinfo.Version,
|
|
||||||
Commands: []*cli.Command{
|
|
||||||
{
|
|
||||||
Name: "unittest",
|
|
||||||
Usage: "Run unittest for alerting and recording rules.",
|
|
||||||
UsageText: "More info in https://docs.victoriametrics.com/vmalert-tool.html#Unit-testing-for-rules",
|
|
||||||
Flags: []cli.Flag{
|
|
||||||
&cli.StringSliceFlag{
|
|
||||||
Name: "files",
|
|
||||||
Usage: "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.",
|
|
||||||
Required: true,
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "disableAlertgroupLabel",
|
|
||||||
Usage: "disable adding group's Name as label to generated alerts and time series.",
|
|
||||||
Required: false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
|
|
||||||
return fmt.Errorf("unittest failed")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
err := app.Run(os.Args)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln(err)
|
|
||||||
}
|
|
||||||
log.Printf("Total time: %v", time.Since(start))
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
|
||||||
)
|
|
||||||
|
|
||||||
// alertTestCase holds alert_rule_test cases defined in test file
|
|
||||||
type alertTestCase struct {
|
|
||||||
EvalTime *promutils.Duration `yaml:"eval_time"`
|
|
||||||
GroupName string `yaml:"groupname"`
|
|
||||||
Alertname string `yaml:"alertname"`
|
|
||||||
ExpAlerts []expAlert `yaml:"exp_alerts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// expAlert holds exp_alerts defined in test file
|
|
||||||
type expAlert struct {
|
|
||||||
ExpLabels map[string]string `yaml:"exp_labels"`
|
|
||||||
ExpAnnotations map[string]string `yaml:"exp_annotations"`
|
|
||||||
}
|
|
||||||
|
|
@ -1,182 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
|
||||||
"github.com/VictoriaMetrics/metricsql"
|
|
||||||
)
|
|
||||||
|
|
||||||
// series holds input_series defined in the test file
|
|
||||||
type series struct {
|
|
||||||
Series string `yaml:"series"`
|
|
||||||
Values string `yaml:"values"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// sequenceValue is an omittable value in a sequence of time series values.
|
|
||||||
type sequenceValue struct {
|
|
||||||
Value float64
|
|
||||||
Omitted bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func httpWrite(address string, r io.Reader) {
|
|
||||||
resp, err := http.Post(address, "", r)
|
|
||||||
if err != nil {
|
|
||||||
logger.Fatalf("failed to send to storage: %v", err)
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeInputSeries send input series to vmstorage and flush them
|
|
||||||
func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
|
|
||||||
r := testutil.WriteRequest{}
|
|
||||||
for _, data := range input {
|
|
||||||
expr, err := metricsql.Parse(data.Series)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to parse series %s: %v", data.Series, err)
|
|
||||||
}
|
|
||||||
promvals, err := parseInputValue(data.Values, true)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
|
|
||||||
}
|
|
||||||
metricExpr, ok := expr.(*metricsql.MetricExpr)
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err)
|
|
||||||
}
|
|
||||||
samples := make([]testutil.Sample, 0, len(promvals))
|
|
||||||
ts := startStamp
|
|
||||||
for _, v := range promvals {
|
|
||||||
if !v.Omitted {
|
|
||||||
samples = append(samples, testutil.Sample{
|
|
||||||
Timestamp: ts.UnixMilli(),
|
|
||||||
Value: v.Value,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
ts = ts.Add(interval.Duration())
|
|
||||||
}
|
|
||||||
var ls []testutil.Label
|
|
||||||
for _, filter := range metricExpr.LabelFilterss[0] {
|
|
||||||
ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value})
|
|
||||||
}
|
|
||||||
r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := testutil.Compress(r)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to compress data: %v", err)
|
|
||||||
}
|
|
||||||
// write input series to vm
|
|
||||||
httpWrite(dst, bytes.NewBuffer(data))
|
|
||||||
vmstorage.Storage.DebugFlush()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test.
|
|
||||||
func parseInputValue(input string, origin bool) ([]sequenceValue, error) {
|
|
||||||
var res []sequenceValue
|
|
||||||
items := strings.Split(input, " ")
|
|
||||||
reg := regexp.MustCompile(`\D?\d*\D?`)
|
|
||||||
for _, item := range items {
|
|
||||||
if item == "stale" {
|
|
||||||
res = append(res, sequenceValue{Value: decimal.StaleNaN})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
vals := reg.FindAllString(item, -1)
|
|
||||||
switch len(vals) {
|
|
||||||
case 1:
|
|
||||||
if vals[0] == "_" {
|
|
||||||
res = append(res, sequenceValue{Omitted: true})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
v, err := strconv.ParseFloat(vals[0], 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res = append(res, sequenceValue{Value: v})
|
|
||||||
continue
|
|
||||||
case 2:
|
|
||||||
p1 := vals[0][:len(vals[0])-1]
|
|
||||||
v2, err := strconv.ParseInt(vals[1], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
option := vals[0][len(vals[0])-1]
|
|
||||||
switch option {
|
|
||||||
case '+':
|
|
||||||
v1, err := strconv.ParseFloat(p1, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res = append(res, sequenceValue{Value: v1 + float64(v2)})
|
|
||||||
case 'x':
|
|
||||||
for i := int64(0); i <= v2; i++ {
|
|
||||||
if p1 == "_" {
|
|
||||||
if i == 0 {
|
|
||||||
i = 1
|
|
||||||
}
|
|
||||||
res = append(res, sequenceValue{Omitted: true})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
v1, err := strconv.ParseFloat(p1, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if !origin || v1 == 0 {
|
|
||||||
res = append(res, sequenceValue{Value: v1 * float64(i)})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
newVal := fmt.Sprintf("%s+0x%s", p1, vals[1])
|
|
||||||
newRes, err := parseInputValue(newVal, false)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res = append(res, newRes...)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("got invalid operation %b", option)
|
|
||||||
}
|
|
||||||
case 3:
|
|
||||||
r1, err := parseInputValue(fmt.Sprintf("%s%s", vals[1], vals[2]), false)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
p1 := vals[0][:len(vals[0])-1]
|
|
||||||
v1, err := strconv.ParseFloat(p1, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
option := vals[0][len(vals[0])-1]
|
|
||||||
var isAdd bool
|
|
||||||
if option == '+' {
|
|
||||||
isAdd = true
|
|
||||||
}
|
|
||||||
for _, r := range r1 {
|
|
||||||
if isAdd {
|
|
||||||
res = append(res, sequenceValue{
|
|
||||||
Value: r.Value + v1,
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
res = append(res, sequenceValue{
|
|
||||||
Value: v1 - r.Value,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported input %s", input)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,93 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseInputValue(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
input string
|
|
||||||
exp []sequenceValue
|
|
||||||
failed bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"",
|
|
||||||
nil,
|
|
||||||
true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"testfailed",
|
|
||||||
nil,
|
|
||||||
true,
|
|
||||||
},
|
|
||||||
// stale doesn't support operations
|
|
||||||
{
|
|
||||||
"stalex3",
|
|
||||||
nil,
|
|
||||||
true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"-4",
|
|
||||||
[]sequenceValue{{Value: -4}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"_",
|
|
||||||
[]sequenceValue{{Omitted: true}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"stale",
|
|
||||||
[]sequenceValue{{Value: decimal.StaleNaN}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"-4x1",
|
|
||||||
[]sequenceValue{{Value: -4}, {Value: -4}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"_x1",
|
|
||||||
[]sequenceValue{{Omitted: true}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"1+1x4",
|
|
||||||
[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 4}, {Value: 5}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"2-1x4",
|
|
||||||
[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"1+1x1 _ -4 stale 3+20x1",
|
|
||||||
[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
output, err := parseInputValue(tc.input, true)
|
|
||||||
if err != nil != tc.failed {
|
|
||||||
t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
|
|
||||||
}
|
|
||||||
if len(tc.exp) != len(output) {
|
|
||||||
t.Fatalf("expect %v, got %v", tc.exp, output)
|
|
||||||
}
|
|
||||||
for i := 0; i < len(tc.exp); i++ {
|
|
||||||
if tc.exp[i].Omitted != output[i].Omitted {
|
|
||||||
t.Fatalf("expect %v, got %v", tc.exp, output)
|
|
||||||
}
|
|
||||||
if tc.exp[i].Value != output[i].Value {
|
|
||||||
if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
t.Fatalf("expect %v, got %v", tc.exp, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,100 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"reflect"
|
|
||||||
"sort"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
|
||||||
"github.com/VictoriaMetrics/metricsql"
|
|
||||||
)
|
|
||||||
|
|
||||||
// metricsqlTestCase holds metricsql_expr_test cases defined in test file
|
|
||||||
type metricsqlTestCase struct {
|
|
||||||
Expr string `yaml:"expr"`
|
|
||||||
EvalTime *promutils.Duration `yaml:"eval_time"`
|
|
||||||
ExpSamples []expSample `yaml:"exp_samples"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type expSample struct {
|
|
||||||
Labels string `yaml:"labels"`
|
|
||||||
Value float64 `yaml:"value"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkMetricsqlCase will check metricsql_expr_test cases
|
|
||||||
func checkMetricsqlCase(cases []metricsqlTestCase, q datasource.QuerierBuilder) (checkErrs []error) {
|
|
||||||
queries := q.BuildWithParams(datasource.QuerierParams{QueryParams: url.Values{"nocache": {"1"}, "latency_offset": {"1ms"}}, DataSourceType: "prometheus"})
|
|
||||||
Outer:
|
|
||||||
for _, mt := range cases {
|
|
||||||
result, _, err := queries.Query(context.Background(), mt.Expr, durationToTime(mt.EvalTime))
|
|
||||||
if err != nil {
|
|
||||||
checkErrs = append(checkErrs, fmt.Errorf(" expr: %q, time: %s, err: %w", mt.Expr,
|
|
||||||
mt.EvalTime.Duration().String(), err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var gotSamples []parsedSample
|
|
||||||
for _, s := range result.Data {
|
|
||||||
sort.Slice(s.Labels, func(i, j int) bool {
|
|
||||||
return s.Labels[i].Name < s.Labels[j].Name
|
|
||||||
})
|
|
||||||
gotSamples = append(gotSamples, parsedSample{
|
|
||||||
Labels: s.Labels,
|
|
||||||
Value: s.Values[0],
|
|
||||||
})
|
|
||||||
}
|
|
||||||
var expSamples []parsedSample
|
|
||||||
for _, s := range mt.ExpSamples {
|
|
||||||
expLb := datasource.Labels{}
|
|
||||||
if s.Labels != "" {
|
|
||||||
metricsqlExpr, err := metricsql.Parse(s.Labels)
|
|
||||||
if err != nil {
|
|
||||||
checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s, err: %v", mt.Expr,
|
|
||||||
mt.EvalTime.Duration().String(), fmt.Errorf("failed to parse labels %q: %w", s.Labels, err)))
|
|
||||||
continue Outer
|
|
||||||
}
|
|
||||||
metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr)
|
|
||||||
if !ok {
|
|
||||||
checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s, err: %v", mt.Expr,
|
|
||||||
mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type")))
|
|
||||||
continue Outer
|
|
||||||
}
|
|
||||||
for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
|
|
||||||
expLb = append(expLb, datasource.Label{
|
|
||||||
Name: l.Label,
|
|
||||||
Value: l.Value,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Slice(expLb, func(i, j int) bool {
|
|
||||||
return expLb[i].Name < expLb[j].Name
|
|
||||||
})
|
|
||||||
expSamples = append(expSamples, parsedSample{
|
|
||||||
Labels: expLb,
|
|
||||||
Value: s.Value,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
sort.Slice(expSamples, func(i, j int) bool {
|
|
||||||
return datasource.LabelCompare(expSamples[i].Labels, expSamples[j].Labels) <= 0
|
|
||||||
})
|
|
||||||
sort.Slice(gotSamples, func(i, j int) bool {
|
|
||||||
return datasource.LabelCompare(gotSamples[i].Labels, gotSamples[j].Labels) <= 0
|
|
||||||
})
|
|
||||||
if !reflect.DeepEqual(expSamples, gotSamples) {
|
|
||||||
checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s,\n exp: %v\n got: %v", mt.Expr,
|
|
||||||
mt.EvalTime.Duration().String(), parsedSamplesString(expSamples), parsedSamplesString(gotSamples)))
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func durationToTime(pd *promutils.Duration) time.Time {
|
|
||||||
if pd == nil {
|
|
||||||
return time.Time{}
|
|
||||||
}
|
|
||||||
return time.UnixMilli(pd.Duration().Milliseconds())
|
|
||||||
}
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
rule_files:
|
|
||||||
- rules.yaml
|
|
||||||
|
|
||||||
evaluation_interval: 1m
|
|
||||||
|
|
||||||
tests:
|
|
||||||
- interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: 'up{job="vmagent2", instance="localhost:9090"}'
|
|
||||||
values: "0+0x1440"
|
|
||||||
|
|
||||||
metricsql_expr_test:
|
|
||||||
- expr: suquery_interval_test
|
|
||||||
eval_time: 4m
|
|
||||||
exp_samples:
|
|
||||||
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
|
|
||||||
value: 1
|
|
||||||
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 2h
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels:
|
|
||||||
job: vmagent2
|
|
||||||
severity: page
|
|
||||||
instance: localhost:9090
|
|
||||||
datacenter: dc-123
|
|
||||||
exp_annotations:
|
|
||||||
summary: "Instance localhost:9090 down"
|
|
||||||
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
|
|
||||||
|
|
||||||
- eval_time: 0
|
|
||||||
alertname: AlwaysFiring
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels:
|
|
||||||
datacenter: dc-123
|
|
||||||
|
|
||||||
- eval_time: 0
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts: []
|
|
||||||
|
|
||||||
external_labels:
|
|
||||||
datacenter: dc-123
|
|
||||||
|
|
@ -1,49 +0,0 @@
|
||||||
rule_files:
|
|
||||||
- rules.yaml
|
|
||||||
|
|
||||||
tests:
|
|
||||||
- interval: 1m
|
|
||||||
name: "Failing test"
|
|
||||||
input_series:
|
|
||||||
- series: test
|
|
||||||
values: "0"
|
|
||||||
|
|
||||||
metricsql_expr_test:
|
|
||||||
- expr: test
|
|
||||||
eval_time: 0m
|
|
||||||
exp_samples:
|
|
||||||
- value: 0
|
|
||||||
labels: test
|
|
||||||
|
|
||||||
# will failed cause there is no "Test" group and rule defined
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 0m
|
|
||||||
groupname: Test
|
|
||||||
alertname: Test
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {}
|
|
||||||
|
|
||||||
- interval: 1m
|
|
||||||
name: Failing alert test
|
|
||||||
input_series:
|
|
||||||
- series: 'up{job="test"}'
|
|
||||||
values: 0x10
|
|
||||||
|
|
||||||
alert_rule_test:
|
|
||||||
# will failed cause rule is firing
|
|
||||||
- eval_time: 5m
|
|
||||||
groupname: group1
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts: []
|
|
||||||
|
|
||||||
- interval: 1m
|
|
||||||
name: Failing alert test with missing groupname
|
|
||||||
input_series:
|
|
||||||
- series: 'up{job="test"}'
|
|
||||||
values: 0x10
|
|
||||||
|
|
||||||
alert_rule_test:
|
|
||||||
# will failed cause missing groupname
|
|
||||||
- eval_time: 5m
|
|
||||||
alertname: AlwaysFiring
|
|
||||||
exp_alerts: []
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
# can be executed successfully but will take more than 1 minute
|
|
||||||
# not included in unit test now
|
|
||||||
evaluation_interval: 100d
|
|
||||||
|
|
||||||
rule_files:
|
|
||||||
- rules.yaml
|
|
||||||
|
|
||||||
tests:
|
|
||||||
- interval: 1d
|
|
||||||
input_series:
|
|
||||||
- series: test
|
|
||||||
# Max time in time.Duration is 106751d from 1970 (2^63/10^9), i.e. 2262.
|
|
||||||
# But VictoriaMetrics supports maxTimestamp value +2 days from now. see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/827.
|
|
||||||
# We input series to 2024-01-01T00:00:00 here.
|
|
||||||
values: "0+1x19723"
|
|
||||||
|
|
||||||
metricsql_expr_test:
|
|
||||||
- expr: timestamp(test)
|
|
||||||
eval_time: 0m
|
|
||||||
exp_samples:
|
|
||||||
- value: 0
|
|
||||||
- expr: test
|
|
||||||
eval_time: 100d
|
|
||||||
exp_samples:
|
|
||||||
- labels: test
|
|
||||||
value: 100
|
|
||||||
- expr: timestamp(test)
|
|
||||||
eval_time: 19000d
|
|
||||||
exp_samples:
|
|
||||||
- value: 1641600000 # 19000d -> seconds.
|
|
||||||
39
app/vmalert-tool/unittest/testdata/rules.yaml
vendored
39
app/vmalert-tool/unittest/testdata/rules.yaml
vendored
|
|
@ -1,39 +0,0 @@
|
||||||
groups:
|
|
||||||
- name: group1
|
|
||||||
rules:
|
|
||||||
- alert: InstanceDown
|
|
||||||
expr: up == 0
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: page
|
|
||||||
annotations:
|
|
||||||
summary: "Instance {{ $labels.instance }} down"
|
|
||||||
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
|
|
||||||
- alert: AlwaysFiring
|
|
||||||
expr: 1
|
|
||||||
- alert: SameAlertNameWithDifferentGroup
|
|
||||||
expr: absent(test)
|
|
||||||
for: 1m
|
|
||||||
|
|
||||||
- name: group2
|
|
||||||
rules:
|
|
||||||
- record: t1
|
|
||||||
expr: test
|
|
||||||
- record: job:test:count_over_time1m
|
|
||||||
expr: sum without(instance) (count_over_time(test[1m]))
|
|
||||||
- record: suquery_interval_test
|
|
||||||
expr: count_over_time(up[5m:])
|
|
||||||
|
|
||||||
- alert: SameAlertNameWithDifferentGroup
|
|
||||||
expr: absent(test)
|
|
||||||
for: 5m
|
|
||||||
|
|
||||||
- name: group3
|
|
||||||
rules:
|
|
||||||
- record: t2
|
|
||||||
expr: t1
|
|
||||||
|
|
||||||
- name: group4
|
|
||||||
rules:
|
|
||||||
- record: t3
|
|
||||||
expr: t1
|
|
||||||
99
app/vmalert-tool/unittest/testdata/test1.yaml
vendored
99
app/vmalert-tool/unittest/testdata/test1.yaml
vendored
|
|
@ -1,99 +0,0 @@
|
||||||
rule_files:
|
|
||||||
- rules.yaml
|
|
||||||
|
|
||||||
evaluation_interval: 1m
|
|
||||||
group_eval_order: ["group4", "group2", "group3"]
|
|
||||||
|
|
||||||
tests:
|
|
||||||
- interval: 1m
|
|
||||||
name: "basic test"
|
|
||||||
input_series:
|
|
||||||
- series: "test"
|
|
||||||
values: "_x5 1x5 _ stale"
|
|
||||||
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 1m
|
|
||||||
groupname: group1
|
|
||||||
alertname: SameAlertNameWithDifferentGroup
|
|
||||||
exp_alerts:
|
|
||||||
- {}
|
|
||||||
- eval_time: 1m
|
|
||||||
groupname: group2
|
|
||||||
alertname: SameAlertNameWithDifferentGroup
|
|
||||||
exp_alerts: []
|
|
||||||
- eval_time: 6m
|
|
||||||
groupname: group1
|
|
||||||
alertname: SameAlertNameWithDifferentGroup
|
|
||||||
exp_alerts: []
|
|
||||||
|
|
||||||
metricsql_expr_test:
|
|
||||||
- expr: test
|
|
||||||
eval_time: 11m
|
|
||||||
exp_samples:
|
|
||||||
- labels: '{__name__="test"}'
|
|
||||||
value: 1
|
|
||||||
- expr: test
|
|
||||||
eval_time: 12m
|
|
||||||
exp_samples: []
|
|
||||||
|
|
||||||
- interval: 1m
|
|
||||||
name: "basic test2"
|
|
||||||
input_series:
|
|
||||||
- series: 'up{job="vmagent1", instance="localhost:9090"}'
|
|
||||||
values: "0+0x1440"
|
|
||||||
- series: "test"
|
|
||||||
values: "0+1x1440"
|
|
||||||
|
|
||||||
metricsql_expr_test:
|
|
||||||
- expr: count(ALERTS) by (alertgroup, alertname, alertstate)
|
|
||||||
eval_time: 4m
|
|
||||||
exp_samples:
|
|
||||||
- labels: '{alertgroup="group1", alertname="AlwaysFiring", alertstate="firing"}'
|
|
||||||
value: 1
|
|
||||||
- labels: '{alertgroup="group1", alertname="InstanceDown", alertstate="pending"}'
|
|
||||||
value: 1
|
|
||||||
- expr: t1
|
|
||||||
eval_time: 4m
|
|
||||||
exp_samples:
|
|
||||||
- value: 4
|
|
||||||
labels: '{__name__="t1", datacenter="dc-123"}'
|
|
||||||
- expr: t2
|
|
||||||
eval_time: 4m
|
|
||||||
exp_samples:
|
|
||||||
- value: 4
|
|
||||||
labels: '{__name__="t2", datacenter="dc-123"}'
|
|
||||||
- expr: t3
|
|
||||||
eval_time: 4m
|
|
||||||
exp_samples:
|
|
||||||
# t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
|
|
||||||
- value: 3
|
|
||||||
labels: '{__name__="t3", datacenter="dc-123"}'
|
|
||||||
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 10m
|
|
||||||
groupname: group1
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels:
|
|
||||||
job: vmagent1
|
|
||||||
severity: page
|
|
||||||
instance: localhost:9090
|
|
||||||
datacenter: dc-123
|
|
||||||
exp_annotations:
|
|
||||||
summary: "Instance localhost:9090 down"
|
|
||||||
description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
|
|
||||||
|
|
||||||
- eval_time: 0
|
|
||||||
groupname: group1
|
|
||||||
alertname: AlwaysFiring
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels:
|
|
||||||
datacenter: dc-123
|
|
||||||
|
|
||||||
- eval_time: 0
|
|
||||||
groupname: alerts
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts: []
|
|
||||||
|
|
||||||
external_labels:
|
|
||||||
datacenter: dc-123
|
|
||||||
46
app/vmalert-tool/unittest/testdata/test2.yaml
vendored
46
app/vmalert-tool/unittest/testdata/test2.yaml
vendored
|
|
@ -1,46 +0,0 @@
|
||||||
rule_files:
|
|
||||||
- rules.yaml
|
|
||||||
|
|
||||||
evaluation_interval: 1m
|
|
||||||
|
|
||||||
tests:
|
|
||||||
- interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: 'up{job="vmagent2", instance="localhost:9090"}'
|
|
||||||
values: "0+0x1440"
|
|
||||||
|
|
||||||
metricsql_expr_test:
|
|
||||||
- expr: suquery_interval_test
|
|
||||||
eval_time: 4m
|
|
||||||
exp_samples:
|
|
||||||
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
|
|
||||||
value: 1
|
|
||||||
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 2h
|
|
||||||
groupname: group1
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels:
|
|
||||||
job: vmagent2
|
|
||||||
severity: page
|
|
||||||
instance: localhost:9090
|
|
||||||
datacenter: dc-123
|
|
||||||
exp_annotations:
|
|
||||||
summary: "Instance localhost:9090 down"
|
|
||||||
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
|
|
||||||
|
|
||||||
- eval_time: 0
|
|
||||||
groupname: group1
|
|
||||||
alertname: AlwaysFiring
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels:
|
|
||||||
datacenter: dc-123
|
|
||||||
|
|
||||||
- eval_time: 0
|
|
||||||
groupname: group1
|
|
||||||
alertname: InstanceDown
|
|
||||||
exp_alerts: []
|
|
||||||
|
|
||||||
external_labels:
|
|
||||||
datacenter: dc-123
|
|
||||||
|
|
@ -1,83 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
|
||||||
)
|
|
||||||
|
|
||||||
// parsedSample is a sample with parsed Labels
|
|
||||||
type parsedSample struct {
|
|
||||||
Labels datasource.Labels
|
|
||||||
Value float64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ps *parsedSample) String() string {
|
|
||||||
return ps.Labels.String() + " " + strconv.FormatFloat(ps.Value, 'E', -1, 64)
|
|
||||||
}
|
|
||||||
|
|
||||||
func parsedSamplesString(pss []parsedSample) string {
|
|
||||||
if len(pss) == 0 {
|
|
||||||
return "nil"
|
|
||||||
}
|
|
||||||
s := pss[0].String()
|
|
||||||
for _, ps := range pss[1:] {
|
|
||||||
s += ", " + ps.String()
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// labelAndAnnotation holds labels and annotations
|
|
||||||
type labelAndAnnotation struct {
|
|
||||||
Labels datasource.Labels
|
|
||||||
Annotations datasource.Labels
|
|
||||||
}
|
|
||||||
|
|
||||||
func (la *labelAndAnnotation) String() string {
|
|
||||||
return "Labels:" + la.Labels.String() + "\nAnnotations:" + la.Annotations.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// labelsAndAnnotations is collection of LabelAndAnnotation
|
|
||||||
type labelsAndAnnotations []labelAndAnnotation
|
|
||||||
|
|
||||||
func (la labelsAndAnnotations) Len() int { return len(la) }
|
|
||||||
|
|
||||||
func (la labelsAndAnnotations) Swap(i, j int) { la[i], la[j] = la[j], la[i] }
|
|
||||||
func (la labelsAndAnnotations) Less(i, j int) bool {
|
|
||||||
diff := datasource.LabelCompare(la[i].Labels, la[j].Labels)
|
|
||||||
if diff != 0 {
|
|
||||||
return diff < 0
|
|
||||||
}
|
|
||||||
return datasource.LabelCompare(la[i].Annotations, la[j].Annotations) < 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (la labelsAndAnnotations) String() string {
|
|
||||||
if len(la) == 0 {
|
|
||||||
return "[]"
|
|
||||||
}
|
|
||||||
s := "[\n0:" + indentLines("\n"+la[0].String(), " ")
|
|
||||||
for i, l := range la[1:] {
|
|
||||||
s += ",\n" + fmt.Sprintf("%d", i+1) + ":" + indentLines("\n"+l.String(), " ")
|
|
||||||
}
|
|
||||||
s += "\n]"
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// indentLines prefixes each line in the supplied string with the given "indent" string.
|
|
||||||
func indentLines(lines, indent string) string {
|
|
||||||
sb := strings.Builder{}
|
|
||||||
n := strings.Split(lines, "\n")
|
|
||||||
for i, l := range n {
|
|
||||||
if i > 0 {
|
|
||||||
sb.WriteString(indent)
|
|
||||||
}
|
|
||||||
sb.WriteString(l)
|
|
||||||
if i != len(n)-1 {
|
|
||||||
sb.WriteRune('\n')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sb.String()
|
|
||||||
}
|
|
||||||
|
|
@ -1,443 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"sort"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
|
|
||||||
vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
|
||||||
"github.com/VictoriaMetrics/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
storagePath string
|
|
||||||
httpListenAddr = ":8880"
|
|
||||||
// insert series from 1970-01-01T00:00:00
|
|
||||||
testStartTime = time.Unix(0, 0).UTC()
|
|
||||||
|
|
||||||
testPromWriteHTTPPath = "http://127.0.0.1" + httpListenAddr + "/api/v1/write"
|
|
||||||
testDataSourcePath = "http://127.0.0.1" + httpListenAddr + "/prometheus"
|
|
||||||
testRemoteWritePath = "http://127.0.0.1" + httpListenAddr
|
|
||||||
testHealthHTTPPath = "http://127.0.0.1" + httpListenAddr + "/health"
|
|
||||||
|
|
||||||
disableAlertgroupLabel bool
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
testStoragePath = "vmalert-unittest"
|
|
||||||
testLogLevel = "ERROR"
|
|
||||||
)
|
|
||||||
|
|
||||||
// UnitTest runs unittest for files
|
|
||||||
func UnitTest(files []string, disableGroupLabel bool) bool {
|
|
||||||
if err := templates.Load([]string{}, true); err != nil {
|
|
||||||
logger.Fatalf("failed to load template: %v", err)
|
|
||||||
}
|
|
||||||
storagePath = filepath.Join(os.TempDir(), testStoragePath)
|
|
||||||
processFlags()
|
|
||||||
vminsert.Init()
|
|
||||||
vmselect.Init()
|
|
||||||
// storagePath will be created again when closing vmselect, so remove it again.
|
|
||||||
defer fs.MustRemoveAll(storagePath)
|
|
||||||
defer vminsert.Stop()
|
|
||||||
defer vmselect.Stop()
|
|
||||||
disableAlertgroupLabel = disableGroupLabel
|
|
||||||
return rulesUnitTest(files)
|
|
||||||
}
|
|
||||||
|
|
||||||
func rulesUnitTest(files []string) bool {
|
|
||||||
var failed bool
|
|
||||||
for _, f := range files {
|
|
||||||
if err := ruleUnitTest(f); err != nil {
|
|
||||||
fmt.Println(" FAILED")
|
|
||||||
fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err)
|
|
||||||
failed = true
|
|
||||||
} else {
|
|
||||||
fmt.Println(" SUCCESS")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return failed
|
|
||||||
}
|
|
||||||
|
|
||||||
func ruleUnitTest(filename string) []error {
|
|
||||||
fmt.Println("\nUnit Testing: ", filename)
|
|
||||||
b, err := os.ReadFile(filename)
|
|
||||||
if err != nil {
|
|
||||||
return []error{fmt.Errorf("failed to read file: %w", err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
var unitTestInp unitTestFile
|
|
||||||
if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
|
|
||||||
return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
|
|
||||||
}
|
|
||||||
if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
|
|
||||||
return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
if unitTestInp.EvaluationInterval.Duration() == 0 {
|
|
||||||
fmt.Println("evaluation_interval set to 1m by default")
|
|
||||||
unitTestInp.EvaluationInterval = &promutils.Duration{D: 1 * time.Minute}
|
|
||||||
}
|
|
||||||
|
|
||||||
groupOrderMap := make(map[string]int)
|
|
||||||
for i, gn := range unitTestInp.GroupEvalOrder {
|
|
||||||
if _, ok := groupOrderMap[gn]; ok {
|
|
||||||
return []error{fmt.Errorf("group name repeated in `group_eval_order`: %s", gn)}
|
|
||||||
}
|
|
||||||
groupOrderMap[gn] = i
|
|
||||||
}
|
|
||||||
|
|
||||||
testGroups, err := vmalertconfig.Parse(unitTestInp.RuleFiles, nil, true)
|
|
||||||
if err != nil {
|
|
||||||
return []error{fmt.Errorf("failed to parse `rule_files`: %w", err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
var errs []error
|
|
||||||
for _, t := range unitTestInp.Tests {
|
|
||||||
if err := verifyTestGroup(t); err != nil {
|
|
||||||
errs = append(errs, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
|
|
||||||
errs = append(errs, testErrs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(errs) > 0 {
|
|
||||||
return errs
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func verifyTestGroup(group testGroup) error {
|
|
||||||
var testGroupName string
|
|
||||||
if group.TestGroupName != "" {
|
|
||||||
testGroupName = fmt.Sprintf("testGroupName: %s\n", group.TestGroupName)
|
|
||||||
}
|
|
||||||
for _, at := range group.AlertRuleTests {
|
|
||||||
if at.Alertname == "" {
|
|
||||||
return fmt.Errorf("\n%s missing required filed \"alertname\"", testGroupName)
|
|
||||||
}
|
|
||||||
if !disableAlertgroupLabel && at.GroupName == "" {
|
|
||||||
return fmt.Errorf("\n%s missing required filed \"groupname\" when flag \"disableAlertGroupLabel\" is false", testGroupName)
|
|
||||||
}
|
|
||||||
if disableAlertgroupLabel && at.GroupName != "" {
|
|
||||||
return fmt.Errorf("\n%s shouldn't set filed \"groupname\" when flag \"disableAlertGroupLabel\" is true", testGroupName)
|
|
||||||
}
|
|
||||||
if at.EvalTime == nil {
|
|
||||||
return fmt.Errorf("\n%s missing required filed \"eval_time\"", testGroupName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, et := range group.MetricsqlExprTests {
|
|
||||||
if et.Expr == "" {
|
|
||||||
return fmt.Errorf("\n%s missing required filed \"expr\"", testGroupName)
|
|
||||||
}
|
|
||||||
if et.EvalTime == nil {
|
|
||||||
return fmt.Errorf("\n%s missing required filed \"eval_time\"", testGroupName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func processFlags() {
|
|
||||||
flag.Parse()
|
|
||||||
for _, fv := range []struct {
|
|
||||||
flag string
|
|
||||||
value string
|
|
||||||
}{
|
|
||||||
{flag: "storageDataPath", value: storagePath},
|
|
||||||
{flag: "loggerLevel", value: testLogLevel},
|
|
||||||
{flag: "search.disableCache", value: "true"},
|
|
||||||
// set storage retention time to 100 years, allow to store series from 1970-01-01T00:00:00.
|
|
||||||
{flag: "retentionPeriod", value: "100y"},
|
|
||||||
{flag: "datasource.url", value: testDataSourcePath},
|
|
||||||
{flag: "remoteWrite.url", value: testRemoteWritePath},
|
|
||||||
} {
|
|
||||||
// panics if flag doesn't exist
|
|
||||||
if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
|
|
||||||
logger.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func setUp() {
|
|
||||||
vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
|
|
||||||
go httpserver.Serve(httpListenAddr, false, func(w http.ResponseWriter, r *http.Request) bool {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/prometheus/api/v1/query":
|
|
||||||
if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
|
|
||||||
httpserver.Errorf(w, r, "%s", err)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
case "/prometheus/api/v1/write", "/api/v1/write":
|
|
||||||
if err := promremotewrite.InsertHandler(r); err != nil {
|
|
||||||
httpserver.Errorf(w, r, "%s", err)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
})
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
readyCheckFunc := func() bool {
|
|
||||||
resp, err := http.Get(testHealthHTTPPath)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
return resp.StatusCode == 200
|
|
||||||
}
|
|
||||||
checkCheck:
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
logger.Fatalf("http server can't be ready in 30s")
|
|
||||||
default:
|
|
||||||
if readyCheckFunc() {
|
|
||||||
break checkCheck
|
|
||||||
}
|
|
||||||
time.Sleep(3 * time.Second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func tearDown() {
|
|
||||||
if err := httpserver.Stop(httpListenAddr); err != nil {
|
|
||||||
logger.Errorf("cannot stop the webservice: %s", err)
|
|
||||||
}
|
|
||||||
vmstorage.Stop()
|
|
||||||
metrics.UnregisterAllMetrics()
|
|
||||||
fs.MustRemoveAll(storagePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveAndGlobFilepaths joins all relative paths in a configuration
|
|
||||||
// with a given base directory and replaces all globs with matching files.
|
|
||||||
func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
|
|
||||||
for i, rf := range utf.RuleFiles {
|
|
||||||
if rf != "" && !filepath.IsAbs(rf) {
|
|
||||||
utf.RuleFiles[i] = filepath.Join(baseDir, rf)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var globbedFiles []string
|
|
||||||
for _, rf := range utf.RuleFiles {
|
|
||||||
m, err := filepath.Glob(rf)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(m) == 0 {
|
|
||||||
fmt.Fprintln(os.Stderr, " WARNING: no file match pattern", rf)
|
|
||||||
}
|
|
||||||
globbedFiles = append(globbedFiles, m...)
|
|
||||||
}
|
|
||||||
utf.RuleFiles = globbedFiles
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
|
|
||||||
// set up vmstorage and http server for ingest and read queries
|
|
||||||
setUp()
|
|
||||||
// tear down vmstorage and clean the data dir
|
|
||||||
defer tearDown()
|
|
||||||
|
|
||||||
err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, testPromWriteHTTPPath)
|
|
||||||
if err != nil {
|
|
||||||
return []error{err}
|
|
||||||
}
|
|
||||||
|
|
||||||
q, err := datasource.Init(nil)
|
|
||||||
if err != nil {
|
|
||||||
return []error{fmt.Errorf("failed to init datasource: %v", err)}
|
|
||||||
}
|
|
||||||
rw, err := remotewrite.NewDebugClient()
|
|
||||||
if err != nil {
|
|
||||||
return []error{fmt.Errorf("failed to init wr: %v", err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
alertEvalTimesMap := map[time.Duration]struct{}{}
|
|
||||||
alertExpResultMap := map[time.Duration]map[string]map[string][]expAlert{}
|
|
||||||
for _, at := range tg.AlertRuleTests {
|
|
||||||
et := at.EvalTime.Duration()
|
|
||||||
alertEvalTimesMap[et] = struct{}{}
|
|
||||||
if _, ok := alertExpResultMap[et]; !ok {
|
|
||||||
alertExpResultMap[et] = make(map[string]map[string][]expAlert)
|
|
||||||
}
|
|
||||||
if _, ok := alertExpResultMap[et][at.GroupName]; !ok {
|
|
||||||
alertExpResultMap[et][at.GroupName] = make(map[string][]expAlert)
|
|
||||||
}
|
|
||||||
alertExpResultMap[et][at.GroupName][at.Alertname] = at.ExpAlerts
|
|
||||||
}
|
|
||||||
alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
|
|
||||||
for k := range alertEvalTimesMap {
|
|
||||||
alertEvalTimes = append(alertEvalTimes, k)
|
|
||||||
}
|
|
||||||
sort.Slice(alertEvalTimes, func(i, j int) bool {
|
|
||||||
return alertEvalTimes[i] < alertEvalTimes[j]
|
|
||||||
})
|
|
||||||
|
|
||||||
// sort group eval order according to the given "group_eval_order".
|
|
||||||
sort.Slice(testGroups, func(i, j int) bool {
|
|
||||||
return groupOrderMap[testGroups[i].Name] < groupOrderMap[testGroups[j].Name]
|
|
||||||
})
|
|
||||||
|
|
||||||
// create groups with given rule
|
|
||||||
var groups []*rule.Group
|
|
||||||
for _, group := range testGroups {
|
|
||||||
ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
|
|
||||||
groups = append(groups, ng)
|
|
||||||
}
|
|
||||||
|
|
||||||
evalIndex := 0
|
|
||||||
maxEvalTime := testStartTime.Add(tg.maxEvalTime())
|
|
||||||
for ts := testStartTime; ts.Before(maxEvalTime) || ts.Equal(maxEvalTime); ts = ts.Add(evalInterval) {
|
|
||||||
for _, g := range groups {
|
|
||||||
errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
|
|
||||||
for err := range errs {
|
|
||||||
if err != nil {
|
|
||||||
checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,
|
|
||||||
ts, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// flush series after each group evaluation
|
|
||||||
vmstorage.Storage.DebugFlush()
|
|
||||||
}
|
|
||||||
|
|
||||||
// check alert_rule_test case at every eval time
|
|
||||||
for evalIndex < len(alertEvalTimes) {
|
|
||||||
if ts.Sub(testStartTime) > alertEvalTimes[evalIndex] ||
|
|
||||||
alertEvalTimes[evalIndex] >= ts.Add(evalInterval).Sub(testStartTime) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
gotAlertsMap := map[string]map[string]labelsAndAnnotations{}
|
|
||||||
for _, g := range groups {
|
|
||||||
if disableAlertgroupLabel {
|
|
||||||
g.Name = ""
|
|
||||||
}
|
|
||||||
if _, ok := alertExpResultMap[time.Duration(ts.UnixNano())][g.Name]; !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, ok := gotAlertsMap[g.Name]; !ok {
|
|
||||||
gotAlertsMap[g.Name] = make(map[string]labelsAndAnnotations)
|
|
||||||
}
|
|
||||||
for _, r := range g.Rules {
|
|
||||||
ar, isAlertRule := r.(*rule.AlertingRule)
|
|
||||||
if !isAlertRule {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, ok := alertExpResultMap[time.Duration(ts.UnixNano())][g.Name][ar.Name]; ok {
|
|
||||||
for _, got := range ar.GetAlerts() {
|
|
||||||
if got.State != notifier.StateFiring {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if disableAlertgroupLabel {
|
|
||||||
delete(got.Labels, "alertgroup")
|
|
||||||
}
|
|
||||||
laa := labelAndAnnotation{
|
|
||||||
Labels: datasource.ConvertToLabels(got.Labels),
|
|
||||||
Annotations: datasource.ConvertToLabels(got.Annotations),
|
|
||||||
}
|
|
||||||
gotAlertsMap[g.Name][ar.Name] = append(gotAlertsMap[g.Name][ar.Name], laa)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for groupname, gres := range alertExpResultMap[alertEvalTimes[evalIndex]] {
|
|
||||||
for alertname, res := range gres {
|
|
||||||
var expAlerts labelsAndAnnotations
|
|
||||||
for _, expAlert := range res {
|
|
||||||
if expAlert.ExpLabels == nil {
|
|
||||||
expAlert.ExpLabels = make(map[string]string)
|
|
||||||
}
|
|
||||||
// alertGroupNameLabel is added as additional labels when `disableAlertGroupLabel` is false
|
|
||||||
if !disableAlertgroupLabel {
|
|
||||||
expAlert.ExpLabels["alertgroup"] = groupname
|
|
||||||
}
|
|
||||||
// alertNameLabel is added as additional labels in vmalert.
|
|
||||||
expAlert.ExpLabels["alertname"] = alertname
|
|
||||||
expAlerts = append(expAlerts, labelAndAnnotation{
|
|
||||||
Labels: datasource.ConvertToLabels(expAlert.ExpLabels),
|
|
||||||
Annotations: datasource.ConvertToLabels(expAlert.ExpAnnotations),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
sort.Sort(expAlerts)
|
|
||||||
|
|
||||||
gotAlerts := gotAlertsMap[groupname][alertname]
|
|
||||||
sort.Sort(gotAlerts)
|
|
||||||
if !reflect.DeepEqual(expAlerts, gotAlerts) {
|
|
||||||
var testGroupName string
|
|
||||||
if tg.TestGroupName != "" {
|
|
||||||
testGroupName = fmt.Sprintf("testGroupName: %s,\n", tg.TestGroupName)
|
|
||||||
}
|
|
||||||
expString := indentLines(expAlerts.String(), " ")
|
|
||||||
gotString := indentLines(gotAlerts.String(), " ")
|
|
||||||
checkErrs = append(checkErrs, fmt.Errorf("\n%s groupname: %s, alertname: %s, time: %s, \n exp:%v, \n got:%v ",
|
|
||||||
testGroupName, groupname, alertname, alertEvalTimes[evalIndex].String(), expString, gotString))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
evalIndex++
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
checkErrs = append(checkErrs, checkMetricsqlCase(tg.MetricsqlExprTests, q)...)
|
|
||||||
return checkErrs
|
|
||||||
}
|
|
||||||
|
|
||||||
// unitTestFile holds the contents of a single unit test file
|
|
||||||
type unitTestFile struct {
|
|
||||||
RuleFiles []string `yaml:"rule_files"`
|
|
||||||
EvaluationInterval *promutils.Duration `yaml:"evaluation_interval"`
|
|
||||||
GroupEvalOrder []string `yaml:"group_eval_order"`
|
|
||||||
Tests []testGroup `yaml:"tests"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// testGroup is a group of input series and test cases associated with it
|
|
||||||
type testGroup struct {
|
|
||||||
Interval *promutils.Duration `yaml:"interval"`
|
|
||||||
InputSeries []series `yaml:"input_series"`
|
|
||||||
AlertRuleTests []alertTestCase `yaml:"alert_rule_test"`
|
|
||||||
MetricsqlExprTests []metricsqlTestCase `yaml:"metricsql_expr_test"`
|
|
||||||
ExternalLabels map[string]string `yaml:"external_labels"`
|
|
||||||
TestGroupName string `yaml:"name"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// maxEvalTime returns the max eval time among all alert_rule_test and metricsql_expr_test
|
|
||||||
func (tg *testGroup) maxEvalTime() time.Duration {
|
|
||||||
var maxd time.Duration
|
|
||||||
for _, alert := range tg.AlertRuleTests {
|
|
||||||
if alert.EvalTime.Duration() > maxd {
|
|
||||||
maxd = alert.EvalTime.Duration()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, met := range tg.MetricsqlExprTests {
|
|
||||||
if met.EvalTime.Duration() > maxd {
|
|
||||||
maxd = met.EvalTime.Duration()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return maxd
|
|
||||||
}
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
package unittest
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
|
||||||
if err := templates.Load([]string{}, true); err != nil {
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
os.Exit(m.Run())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnitRule(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
name string
|
|
||||||
disableGroupLabel bool
|
|
||||||
files []string
|
|
||||||
failed bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "run multi files",
|
|
||||||
files: []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
|
|
||||||
failed: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "disable group label",
|
|
||||||
disableGroupLabel: true,
|
|
||||||
files: []string{"./testdata/disable-group-label.yaml"},
|
|
||||||
failed: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "failing test",
|
|
||||||
files: []string{"./testdata/failed-test.yaml"},
|
|
||||||
failed: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range testCases {
|
|
||||||
fail := UnitTest(tc.files, tc.disableGroupLabel)
|
|
||||||
if fail != tc.failed {
|
|
||||||
t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in a new issue