diff --git a/Makefile b/Makefile index 8b2dd3344..8909af47e 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,8 @@ all: \ vmauth-prod \ vmbackup-prod \ vmrestore-prod \ - vmctl-prod + vmctl-prod \ + vmalert-tool-prod clean: rm -rf bin/* @@ -40,7 +41,8 @@ publish: package-base \ publish-vmauth \ publish-vmbackup \ publish-vmrestore \ - publish-vmctl + publish-vmctl \ + publish-vmalert-tool package: \ package-victoria-metrics \ @@ -50,7 +52,8 @@ package: \ package-vmauth \ package-vmbackup \ package-vmrestore \ - package-vmctl + package-vmctl \ + package-vmalert-tool vmutils: \ vmagent \ @@ -58,7 +61,8 @@ vmutils: \ vmauth \ vmbackup \ vmrestore \ - vmctl + vmctl \ + vmalert-tool vmutils-pure: \ vmagent-pure \ @@ -66,7 +70,8 @@ vmutils-pure: \ vmauth-pure \ vmbackup-pure \ vmrestore-pure \ - vmctl-pure + vmctl-pure \ + vmalert-tool-pure vmutils-linux-amd64: \ vmagent-linux-amd64 \ @@ -74,7 +79,8 @@ vmutils-linux-amd64: \ vmauth-linux-amd64 \ vmbackup-linux-amd64 \ vmrestore-linux-amd64 \ - vmctl-linux-amd64 + vmctl-linux-amd64 \ + vmalert-tool-linux-amd64 vmutils-linux-arm64: \ vmagent-linux-arm64 \ @@ -82,7 +88,8 @@ vmutils-linux-arm64: \ vmauth-linux-arm64 \ vmbackup-linux-arm64 \ vmrestore-linux-arm64 \ - vmctl-linux-arm64 + vmctl-linux-arm64 \ + vmalert-tool-linux-arm64 vmutils-linux-arm: \ vmagent-linux-arm \ @@ -90,7 +97,8 @@ vmutils-linux-arm: \ vmauth-linux-arm \ vmbackup-linux-arm \ vmrestore-linux-arm \ - vmctl-linux-arm + vmctl-linux-arm \ + vmalert-tool-linux-arm vmutils-linux-386: \ vmagent-linux-386 \ @@ -98,7 +106,8 @@ vmutils-linux-386: \ vmauth-linux-386 \ vmbackup-linux-386 \ vmrestore-linux-386 \ - vmctl-linux-386 + vmctl-linux-386 \ + vmalert-tool-linux-386 vmutils-linux-ppc64le: \ vmagent-linux-ppc64le \ @@ -106,7 +115,8 @@ vmutils-linux-ppc64le: \ vmauth-linux-ppc64le \ vmbackup-linux-ppc64le \ vmrestore-linux-ppc64le \ - vmctl-linux-ppc64le + vmctl-linux-ppc64le \ + vmalert-tool-linux-ppc64le vmutils-darwin-amd64: \ vmagent-darwin-amd64 \ @@ -114,7 +124,8 @@ vmutils-darwin-amd64: \ vmauth-darwin-amd64 \ vmbackup-darwin-amd64 \ vmrestore-darwin-amd64 \ - vmctl-darwin-amd64 + vmctl-darwin-amd64 \ + vmalert-tool-darwin-amd64 vmutils-darwin-arm64: \ vmagent-darwin-arm64 \ @@ -122,7 +133,8 @@ vmutils-darwin-arm64: \ vmauth-darwin-arm64 \ vmbackup-darwin-arm64 \ vmrestore-darwin-arm64 \ - vmctl-darwin-arm64 + vmctl-darwin-arm64 \ + vmalert-tool-darwin-arm64 vmutils-freebsd-amd64: \ vmagent-freebsd-amd64 \ @@ -130,7 +142,8 @@ vmutils-freebsd-amd64: \ vmauth-freebsd-amd64 \ vmbackup-freebsd-amd64 \ vmrestore-freebsd-amd64 \ - vmctl-freebsd-amd64 + vmctl-freebsd-amd64 \ + vmalert-tool-freebsd-amd64 vmutils-openbsd-amd64: \ vmagent-openbsd-amd64 \ @@ -138,7 +151,8 @@ vmutils-openbsd-amd64: \ vmauth-openbsd-amd64 \ vmbackup-openbsd-amd64 \ vmrestore-openbsd-amd64 \ - vmctl-openbsd-amd64 + vmctl-openbsd-amd64 \ + vmalert-tool-openbsd-amd64 vmutils-windows-amd64: \ vmagent-windows-amd64 \ @@ -146,7 +160,8 @@ vmutils-windows-amd64: \ vmauth-windows-amd64 \ vmbackup-windows-amd64 \ vmrestore-windows-amd64 \ - vmctl-windows-amd64 + vmctl-windows-amd64 \ + vmalert-tool-windows-amd64 victoria-metrics-crossbuild: \ victoria-metrics-linux-386 \ @@ -342,7 +357,8 @@ release-vmutils-goos-goarch: \ vmauth-$(GOOS)-$(GOARCH)-prod \ vmbackup-$(GOOS)-$(GOARCH)-prod \ vmrestore-$(GOOS)-$(GOARCH)-prod \ - vmctl-$(GOOS)-$(GOARCH)-prod + vmctl-$(GOOS)-$(GOARCH)-prod \ + vmalert-tool-$(GOOS)-$(GOARCH)-prod cd bin && \ tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \ vmagent-$(GOOS)-$(GOARCH)-prod \ @@ -351,6 +367,7 @@ release-vmutils-goos-goarch: \ vmbackup-$(GOOS)-$(GOARCH)-prod \ vmrestore-$(GOOS)-$(GOARCH)-prod \ vmctl-$(GOOS)-$(GOARCH)-prod \ + vmalert-tool-$(GOOS)-$(GOARCH)-prod && sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \ vmagent-$(GOOS)-$(GOARCH)-prod \ vmalert-$(GOOS)-$(GOARCH)-prod \ @@ -358,6 +375,7 @@ release-vmutils-goos-goarch: \ vmbackup-$(GOOS)-$(GOARCH)-prod \ vmrestore-$(GOOS)-$(GOARCH)-prod \ vmctl-$(GOOS)-$(GOARCH)-prod \ + vmalert-tool-$(GOOS)-$(GOARCH)-prod \ | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt cd bin && rm -rf \ vmagent-$(GOOS)-$(GOARCH)-prod \ @@ -365,7 +383,8 @@ release-vmutils-goos-goarch: \ vmauth-$(GOOS)-$(GOARCH)-prod \ vmbackup-$(GOOS)-$(GOARCH)-prod \ vmrestore-$(GOOS)-$(GOARCH)-prod \ - vmctl-$(GOOS)-$(GOARCH)-prod + vmctl-$(GOOS)-$(GOARCH)-prod \ + vmalert-tool-$(GOOS)-$(GOARCH)-prod release-vmutils-windows-goarch: \ vmagent-windows-$(GOARCH)-prod \ @@ -373,7 +392,8 @@ release-vmutils-windows-goarch: \ vmauth-windows-$(GOARCH)-prod \ vmbackup-windows-$(GOARCH)-prod \ vmrestore-windows-$(GOARCH)-prod \ - vmctl-windows-$(GOARCH)-prod + vmctl-windows-$(GOARCH)-prod \ + vmalert-tool-windows-$(GOARCH)-prod cd bin && \ zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \ vmagent-windows-$(GOARCH)-prod.exe \ @@ -382,6 +402,7 @@ release-vmutils-windows-goarch: \ vmbackup-windows-$(GOARCH)-prod.exe \ vmrestore-windows-$(GOARCH)-prod.exe \ vmctl-windows-$(GOARCH)-prod.exe \ + vmalert-tool-windows-$(GOARCH)-prod.exe \ && sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \ vmagent-windows-$(GOARCH)-prod.exe \ vmalert-windows-$(GOARCH)-prod.exe \ @@ -389,6 +410,7 @@ release-vmutils-windows-goarch: \ vmbackup-windows-$(GOARCH)-prod.exe \ vmrestore-windows-$(GOARCH)-prod.exe \ vmctl-windows-$(GOARCH)-prod.exe \ + vmalert-tool-windows-$(GOARCH)-prod.exe \ > vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt cd bin && rm -rf \ vmagent-windows-$(GOARCH)-prod.exe \ @@ -396,7 +418,8 @@ release-vmutils-windows-goarch: \ vmauth-windows-$(GOARCH)-prod.exe \ vmbackup-windows-$(GOARCH)-prod.exe \ vmrestore-windows-$(GOARCH)-prod.exe \ - vmctl-windows-$(GOARCH)-prod.exe + vmctl-windows-$(GOARCH)-prod.exe \ + vmalert-tool-windows-$(GOARCH)-prod.exe pprof-cpu: go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE) @@ -514,3 +537,4 @@ docs-sync: SRC=app/vmctl/README.md DST=docs/vmctl.md OLD_URL='/vmctl.html' ORDER=8 TITLE=vmctl $(MAKE) copy-docs SRC=app/vmgateway/README.md DST=docs/vmgateway.md OLD_URL='/vmgateway.html' ORDER=9 TITLE=vmgateway $(MAKE) copy-docs SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md OLD_URL='/vmbackupmanager.html' ORDER=10 TITLE=vmbackupmanager $(MAKE) copy-docs + SRC=app/vmalert-tool/README.md DST=docs/vmalert-tool.md OLD_URL='' ORDER=12 TITLE=vmalert-tool $(MAKE) copy-docs diff --git a/app/vmalert-tool/Makefile b/app/vmalert-tool/Makefile new file mode 100644 index 000000000..dbb6b373a --- /dev/null +++ b/app/vmalert-tool/Makefile @@ -0,0 +1,103 @@ +# All these commands must run from repository root. + +vmalert-tool: + APP_NAME=vmalert-tool $(MAKE) app-local + +vmalert-tool-race: + APP_NAME=vmalert-tool RACE=-race $(MAKE) app-local + +vmalert-tool-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker + +vmalert-tool-pure-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-pure + +vmalert-tool-linux-amd64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-amd64 + +vmalert-tool-linux-arm-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm + +vmalert-tool-linux-arm64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-arm64 + +vmalert-tool-linux-ppc64le-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-ppc64le + +vmalert-tool-linux-386-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386 + +vmalert-tool-darwin-amd64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64 + +vmalert-tool-darwin-arm64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-arm64 + +vmalert-tool-freebsd-amd64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-freebsd-amd64 + +vmalert-tool-openbsd-amd64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-openbsd-amd64 + +vmalert-tool-windows-amd64-prod: + APP_NAME=vmalert-tool $(MAKE) app-via-docker-windows-amd64 + +package-vmalert-tool: + APP_NAME=vmalert-tool $(MAKE) package-via-docker + +package-vmalert-tool-pure: + APP_NAME=vmalert-tool $(MAKE) package-via-docker-pure + +package-vmalert-tool-amd64: + APP_NAME=vmalert-tool $(MAKE) package-via-docker-amd64 + +package-vmalert-tool-arm: + APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm + +package-vmalert-tool-arm64: + APP_NAME=vmalert-tool $(MAKE) package-via-docker-arm64 + +package-vmalert-tool-ppc64le: + APP_NAME=vmalert-tool $(MAKE) package-via-docker-ppc64le + +package-vmalert-tool-386: + APP_NAME=vmalert-tool $(MAKE) package-via-docker-386 + +publish-vmalert-tool: + APP_NAME=vmalert-tool $(MAKE) publish-via-docker + +vmalert-tool-linux-amd64: + APP_NAME=vmalert-tool CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch + +vmalert-tool-linux-arm: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch + +vmalert-tool-linux-arm64: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch + +vmalert-tool-linux-ppc64le: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch + +vmalert-tool-linux-s390x: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch + +vmalert-tool-linux-386: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch + +vmalert-tool-darwin-amd64: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch + +vmalert-tool-darwin-arm64: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch + +vmalert-tool-freebsd-amd64: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch + +vmalert-tool-openbsd-amd64: + APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch + +vmalert-tool-windows-amd64: + GOARCH=amd64 APP_NAME=vmalert-tool $(MAKE) app-local-windows-goarch + +vmalert-tool-pure: + APP_NAME=vmalert-tool $(MAKE) app-local-pure diff --git a/app/vmalert-tool/README.md b/app/vmalert-tool/README.md new file mode 100644 index 000000000..e25465275 --- /dev/null +++ b/app/vmalert-tool/README.md @@ -0,0 +1,244 @@ + +# vmalert-tool + +VMAlert command-line tool + +## Unit testing for rules + +You can use `vmalert-tool` to run unit tests for alerting and recording rules. +It will perform the following actions: +* sets up an isolated VictoriaMetrics instance; +* simulates the periodic ingestion of time series; +* queries the ingested data for recording and alerting rules evaluation like [vmalert](https://docs.victoriametrics.com/vmalert.html); +* checks whether the firing alerts or resulting recording rules match the expected results. + +See how to run vmalert-tool for unit test below: +``` +# Run vmalert-tool with one or multiple test files via --files cmd-line flag +./vmalert-tool unittest --files test1.yaml --files test2.yaml +``` + +vmalert-tool unittest is compatible with [Prometheus config format for tests](https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/#test-file-format) +except `promql_expr_test` field. Use `metricsql_expr_test` field name instead. The name is different because vmalert-tool +validates and executes [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expressions, +which aren't always backward compatible with [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/). + +### Test file format + +The configuration format for files specified in `--files` cmd-line flag is the following: +``` +# Path to the files or http url containing [rule groups](https://docs.victoriametrics.com/vmalert.html#groups) configuration. +# Enterprise version of vmalert-tool supports S3 and GCS paths to rules. +rule_files: + [ - ] + +# The evaluation interval for rules specified in `rule_files` +[ evaluation_interval: | default = 1m ] + +# Groups listed below will be evaluated by order. +# Not All the groups need not be mentioned, if not, they will be evaluated by define order in rule_files. +group_eval_order: + [ - ] + +# The list of unit test files to be checked during evaluation. +tests: + [ - ] +``` + +#### `` + +``` +# Interval between samples for input series +interval: +# Time series to persist into the database according to configured before running tests. +input_series: + [ - ] + +# Name of the test group, optional +[ name: ] + +# Unit tests for alerting rules +alert_rule_test: + [ - ] + +# Unit tests for Metricsql expressions. +metricsql_expr_test: + [ - ] + +# External labels accessible for templating. +external_labels: + [ : ... ] + +``` + +#### `` + +``` +# series in the following format '{