VictoriaMetrics/Makefile

280 lines
7.7 KiB
Makefile
Raw Normal View History

2019-05-22 21:16:55 +00:00
PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
2019-05-22 21:16:55 +00:00
BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
LATEST_TAG ?= cluster-latest
2019-05-22 21:16:55 +00:00
PKG_TAG ?= $(shell git tag -l --points-at HEAD)
ifeq ($(PKG_TAG),)
PKG_TAG := $(BUILDINFO_TAG)
endif
GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
TAR_OWNERSHIP ?= --owner=1000 --group=1000
2019-05-22 21:16:55 +00:00
.PHONY: $(MAKECMDGOALS)
include app/*/Makefile
include cspell/Makefile
include docs/Makefile
include deployment/*/Makefile
include dashboards/Makefile
include package/release/Makefile
2019-05-22 21:16:55 +00:00
all: \
2019-05-22 21:23:23 +00:00
vminsert \
vmselect \
vmstorage
2019-05-22 21:16:55 +00:00
2019-07-23 16:54:50 +00:00
all-pure: \
vminsert-pure \
vmselect-pure \
vmstorage-pure
2019-07-23 16:54:50 +00:00
2019-05-22 21:16:55 +00:00
clean:
rm -rf bin/*
vmcluster-linux-amd64: \
vminsert-linux-amd64 \
vmselect-linux-amd64 \
vmstorage-linux-amd64
vmcluster-linux-arm64: \
vminsert-linux-arm64 \
vmselect-linux-arm64 \
vmstorage-linux-arm64
vmcluster-linux-arm: \
vminsert-linux-arm \
vmselect-linux-arm \
vmstorage-linux-arm
vmcluster-linux-ppc64le: \
vminsert-linux-ppc64le \
vmselect-linux-ppc64le \
vmstorage-linux-ppc64le
vmcluster-linux-386: \
vminsert-linux-386 \
vmselect-linux-386 \
vmstorage-linux-386
vmcluster-freebsd-amd64: \
vminsert-freebsd-amd64 \
vmselect-freebsd-amd64 \
vmstorage-freebsd-amd64
vmcluster-openbsd-amd64: \
vminsert-openbsd-amd64 \
vmselect-openbsd-amd64 \
vmstorage-openbsd-amd64
vmcluster-windows-amd64: \
vminsert-windows-amd64 \
vmselect-windows-amd64 \
vmstorage-windows-amd64
vmcluster-darwin-amd64: \
vminsert-darwin-amd64 \
vmselect-darwin-amd64 \
vmstorage-darwin-amd64
vmcluster-darwin-arm64: \
vminsert-darwin-arm64 \
vmselect-darwin-arm64 \
vmstorage-darwin-arm64
crossbuild: vmcluster-crossbuild
vmcluster-crossbuild:
$(MAKE_PARALLEL) vmcluster-linux-amd64 \
vmcluster-linux-arm64 \
vmcluster-linux-arm \
vmcluster-linux-ppc64le \
vmcluster-linux-386 \
vmcluster-freebsd-amd64 \
vmcluster-openbsd-amd64
publish: \
publish-vminsert \
publish-vmselect \
publish-vmstorage
2019-05-22 21:23:23 +00:00
package: \
package-vminsert \
package-vmselect \
package-vmstorage
2019-05-22 21:16:55 +00:00
publish-release:
rm -rf bin/*
git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
release:
$(MAKE_PARALLEL) release-vmcluster
release-vmcluster: \
release-vmcluster-linux-amd64 \
release-vmcluster-linux-arm64 \
release-vmcluster-freebsd-amd64 \
all: add Windows build for VictoriaMetrics This commit changes background merge algorithm, so it becomes compatible with Windows file semantics. The previous algorithm for background merge: 1. Merge source parts into a destination part inside tmp directory. 2. Create a file in txn directory with instructions on how to atomically swap source parts with the destination part. 3. Perform instructions from the file. 4. Delete the file with instructions. This algorithm guarantees that either source parts or destination part is visible in the partition after unclean shutdown at any step above, since the remaining files with instructions is replayed on the next restart, after that the remaining contents of the tmp directory is deleted. Unfortunately this algorithm doesn't work under Windows because it disallows removing and moving files, which are in use. So the new algorithm for background merge has been implemented: 1. Merge source parts into a destination part inside the partition directory itself. E.g. now the partition directory may contain both complete and incomplete parts. 2. Atomically update the parts.json file with the new list of parts after the merge, e.g. remove the source parts from the list and add the destination part to the list before storing it to parts.json file. 3. Remove the source parts from disk when they are no longer used. This algorithm guarantees that either source parts or destination part is visible in the partition after unclean shutdown at any step above, since incomplete partitions from step 1 or old source parts from step 3 are removed on the next startup by inspecting parts.json file. This algorithm should work under Windows, since it doesn't remove or move files in use. This algorithm has also the following benefits: - It should work better for NFS. - It fits object storage semantics. The new algorithm changes data storage format, so it is impossible to downgrade to the previous versions of VictoriaMetrics after upgrading to this algorithm. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3236 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3821 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
2023-03-19 08:36:05 +00:00
release-vmcluster-openbsd-amd64 \
release-vmcluster-windows-amd64 \
release-vmcluster-darwin-amd64 \
release-vmcluster-darwin-arm64
release-vmcluster-linux-amd64:
GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
release-vmcluster-linux-arm64:
GOOS=linux GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
release-vmcluster-freebsd-amd64:
GOOS=freebsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
release-vmcluster-openbsd-amd64:
GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
all: add Windows build for VictoriaMetrics This commit changes background merge algorithm, so it becomes compatible with Windows file semantics. The previous algorithm for background merge: 1. Merge source parts into a destination part inside tmp directory. 2. Create a file in txn directory with instructions on how to atomically swap source parts with the destination part. 3. Perform instructions from the file. 4. Delete the file with instructions. This algorithm guarantees that either source parts or destination part is visible in the partition after unclean shutdown at any step above, since the remaining files with instructions is replayed on the next restart, after that the remaining contents of the tmp directory is deleted. Unfortunately this algorithm doesn't work under Windows because it disallows removing and moving files, which are in use. So the new algorithm for background merge has been implemented: 1. Merge source parts into a destination part inside the partition directory itself. E.g. now the partition directory may contain both complete and incomplete parts. 2. Atomically update the parts.json file with the new list of parts after the merge, e.g. remove the source parts from the list and add the destination part to the list before storing it to parts.json file. 3. Remove the source parts from disk when they are no longer used. This algorithm guarantees that either source parts or destination part is visible in the partition after unclean shutdown at any step above, since incomplete partitions from step 1 or old source parts from step 3 are removed on the next startup by inspecting parts.json file. This algorithm should work under Windows, since it doesn't remove or move files in use. This algorithm has also the following benefits: - It should work better for NFS. - It fits object storage semantics. The new algorithm changes data storage format, so it is impossible to downgrade to the previous versions of VictoriaMetrics after upgrading to this algorithm. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3236 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3821 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
2023-03-19 08:36:05 +00:00
release-vmcluster-windows-amd64:
GOARCH=amd64 $(MAKE) release-vmcluster-windows-goarch
release-vmcluster-darwin-amd64:
GOOS=darwin GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
release-vmcluster-darwin-arm64:
GOOS=darwin GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
release-vmcluster-goos-goarch: \
vminsert-$(GOOS)-$(GOARCH)-prod \
vmselect-$(GOOS)-$(GOARCH)-prod \
vmstorage-$(GOOS)-$(GOARCH)-prod
cd bin && \
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
vminsert-$(GOOS)-$(GOARCH)-prod \
vmselect-$(GOOS)-$(GOARCH)-prod \
vmstorage-$(GOOS)-$(GOARCH)-prod \
&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
vminsert-$(GOOS)-$(GOARCH)-prod \
vmselect-$(GOOS)-$(GOARCH)-prod \
vmstorage-$(GOOS)-$(GOARCH)-prod \
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
vminsert-$(GOOS)-$(GOARCH)-prod \
vmselect-$(GOOS)-$(GOARCH)-prod \
vmstorage-$(GOOS)-$(GOARCH)-prod
all: add Windows build for VictoriaMetrics This commit changes background merge algorithm, so it becomes compatible with Windows file semantics. The previous algorithm for background merge: 1. Merge source parts into a destination part inside tmp directory. 2. Create a file in txn directory with instructions on how to atomically swap source parts with the destination part. 3. Perform instructions from the file. 4. Delete the file with instructions. This algorithm guarantees that either source parts or destination part is visible in the partition after unclean shutdown at any step above, since the remaining files with instructions is replayed on the next restart, after that the remaining contents of the tmp directory is deleted. Unfortunately this algorithm doesn't work under Windows because it disallows removing and moving files, which are in use. So the new algorithm for background merge has been implemented: 1. Merge source parts into a destination part inside the partition directory itself. E.g. now the partition directory may contain both complete and incomplete parts. 2. Atomically update the parts.json file with the new list of parts after the merge, e.g. remove the source parts from the list and add the destination part to the list before storing it to parts.json file. 3. Remove the source parts from disk when they are no longer used. This algorithm guarantees that either source parts or destination part is visible in the partition after unclean shutdown at any step above, since incomplete partitions from step 1 or old source parts from step 3 are removed on the next startup by inspecting parts.json file. This algorithm should work under Windows, since it doesn't remove or move files in use. This algorithm has also the following benefits: - It should work better for NFS. - It fits object storage semantics. The new algorithm changes data storage format, so it is impossible to downgrade to the previous versions of VictoriaMetrics after upgrading to this algorithm. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3236 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3821 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70
2023-03-19 08:36:05 +00:00
release-vmcluster-windows-goarch: \
vminsert-windows-$(GOARCH)-prod \
vmselect-windows-$(GOARCH)-prod \
vmstorage-windows-$(GOARCH)-prod
cd bin && \
zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
vminsert-windows-$(GOARCH)-prod.exe \
vmselect-windows-$(GOARCH)-prod.exe \
vmstorage-windows-$(GOARCH)-prod.exe \
&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
vminsert-windows-$(GOARCH)-prod.exe \
vmselect-windows-$(GOARCH)-prod.exe \
vmstorage-windows-$(GOARCH)-prod.exe \
> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
vminsert-windows-$(GOARCH)-prod.exe \
vmselect-windows-$(GOARCH)-prod.exe \
vmstorage-windows-$(GOARCH)-prod.exe
pprof-cpu:
go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
2019-05-22 21:16:55 +00:00
fmt:
gofmt -l -w -s ./lib
gofmt -l -w -s ./app
tests: Initial version of integration tests (#7253) ### Describe Your Changes Related issue: #7199 This is the initial version of the integration tests for cluster. See `README.md` for details. Currently cluster only, but it can also be used for vm-single if needed. The code has been added to the apptest package that resides in the root directory of the VM codebase. This is done to exclude the integration tests from regular testing build targets because: - Most of the test variants do not apply to integration testing (such as pure or race). - The integtation tests may also be slow because each test must wait for 2 seconds so vmstorage flushes pending content). It may be okay when there are a few tests but when there is a 100 of them running tests will require much more time which will affect the developer wait time and CI workflows. - Finally, the integration tests may be flaky especially short term. An alternative approach would be placing apptest under app package and exclude apptest from packages under test, but that is not trivial. The integration tests rely on retrieving some application runtime info from the application logs, namely the application's host:port. Therefore some changes to lib/httpserver/httpserver.go were necessary, such as reporting the effective host:port instead the one from the flag. ### Checklist The following checks are **mandatory**: - [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2024-10-30 14:22:06 +00:00
gofmt -l -w -s ./apptest
2019-05-22 21:16:55 +00:00
vet:
go vet ./lib/...
go vet ./app/...
tests: Initial version of integration tests (#7253) ### Describe Your Changes Related issue: #7199 This is the initial version of the integration tests for cluster. See `README.md` for details. Currently cluster only, but it can also be used for vm-single if needed. The code has been added to the apptest package that resides in the root directory of the VM codebase. This is done to exclude the integration tests from regular testing build targets because: - Most of the test variants do not apply to integration testing (such as pure or race). - The integtation tests may also be slow because each test must wait for 2 seconds so vmstorage flushes pending content). It may be okay when there are a few tests but when there is a 100 of them running tests will require much more time which will affect the developer wait time and CI workflows. - Finally, the integration tests may be flaky especially short term. An alternative approach would be placing apptest under app package and exclude apptest from packages under test, but that is not trivial. The integration tests rely on retrieving some application runtime info from the application logs, namely the application's host:port. Therefore some changes to lib/httpserver/httpserver.go were necessary, such as reporting the effective host:port instead the one from the flag. ### Checklist The following checks are **mandatory**: - [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2024-10-30 14:22:06 +00:00
go vet ./apptest/...
2019-05-22 21:16:55 +00:00
check-all: fmt vet golangci-lint govulncheck
clean-checkers: remove-golangci-lint remove-govulncheck
2019-05-22 21:16:55 +00:00
test:
testing: allow disabling fsync to make tests run faster (#6871) ### Describe Your Changes fsync() ensures that the data is written to disk. In production this is needed for data durability. However, during the development, when the unit tests are run, this level of durability is not needed. Therefore fsync() can be disabled which will makes test runs two times faster. The disabling is done by setting the `DISABLE_FSYNC_FOR_TESTING` environment variable. The valid values for this variable are the same as the values of the arg of `go doc strconv.ParseBool`: ``` 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. ``` Any other value means `false`. The variable is set for all test build targets. Compare running times: Build Target | DISABLE_FSYNC_FOR_TESTING=0 | DISABLE_FSYNC_FOR_TESTING=1 ----------------- | ------------------------------------------------ | ------------------------------------------------- make test | 1m5s | 0m22s make test-race | 3m1s | 1m42s make test-pure | 1m7s | 0m20s make test-full | 1m21s | 0m32s make test-full-386 | 1m42s | 0m36s When running tests for a given package, fsync can be disabled as follows: ```shell DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/storage ``` Disabling fsync() is intended for testing purposes only and the name of the variables reflects that. What could also have been done but haven't: - lib/filestream/filestream.go: `Writer.MustFlush()` also uses f.Sync() but nothing has been done to it, because the Writer.MustFlush() is not used anywhere in the VM codebase. A side question: what is the general policy for the unused code? - lib/filestream/filestream.go: Writer.Write() calls `adviceDontNeed()` which calls unix.Fdatasync(). Disabling it could potentially improve running time, but running tests with this code disabled has shown otherwise. ### Checklist The following checks are **mandatory**: - [ x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <wwctrsrx@gmail.com> (cherry picked from commit 334cd92a6c3cdac1aa2102653ffc5e7d9382bfbc) Signed-off-by: hagen1778 <roman@victoriametrics.com>
2024-08-30 08:54:46 +00:00
DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
test-race:
testing: allow disabling fsync to make tests run faster (#6871) ### Describe Your Changes fsync() ensures that the data is written to disk. In production this is needed for data durability. However, during the development, when the unit tests are run, this level of durability is not needed. Therefore fsync() can be disabled which will makes test runs two times faster. The disabling is done by setting the `DISABLE_FSYNC_FOR_TESTING` environment variable. The valid values for this variable are the same as the values of the arg of `go doc strconv.ParseBool`: ``` 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. ``` Any other value means `false`. The variable is set for all test build targets. Compare running times: Build Target | DISABLE_FSYNC_FOR_TESTING=0 | DISABLE_FSYNC_FOR_TESTING=1 ----------------- | ------------------------------------------------ | ------------------------------------------------- make test | 1m5s | 0m22s make test-race | 3m1s | 1m42s make test-pure | 1m7s | 0m20s make test-full | 1m21s | 0m32s make test-full-386 | 1m42s | 0m36s When running tests for a given package, fsync can be disabled as follows: ```shell DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/storage ``` Disabling fsync() is intended for testing purposes only and the name of the variables reflects that. What could also have been done but haven't: - lib/filestream/filestream.go: `Writer.MustFlush()` also uses f.Sync() but nothing has been done to it, because the Writer.MustFlush() is not used anywhere in the VM codebase. A side question: what is the general policy for the unused code? - lib/filestream/filestream.go: Writer.Write() calls `adviceDontNeed()` which calls unix.Fdatasync(). Disabling it could potentially improve running time, but running tests with this code disabled has shown otherwise. ### Checklist The following checks are **mandatory**: - [ x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <wwctrsrx@gmail.com> (cherry picked from commit 334cd92a6c3cdac1aa2102653ffc5e7d9382bfbc) Signed-off-by: hagen1778 <roman@victoriametrics.com>
2024-08-30 08:54:46 +00:00
DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
test-pure:
testing: allow disabling fsync to make tests run faster (#6871) ### Describe Your Changes fsync() ensures that the data is written to disk. In production this is needed for data durability. However, during the development, when the unit tests are run, this level of durability is not needed. Therefore fsync() can be disabled which will makes test runs two times faster. The disabling is done by setting the `DISABLE_FSYNC_FOR_TESTING` environment variable. The valid values for this variable are the same as the values of the arg of `go doc strconv.ParseBool`: ``` 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. ``` Any other value means `false`. The variable is set for all test build targets. Compare running times: Build Target | DISABLE_FSYNC_FOR_TESTING=0 | DISABLE_FSYNC_FOR_TESTING=1 ----------------- | ------------------------------------------------ | ------------------------------------------------- make test | 1m5s | 0m22s make test-race | 3m1s | 1m42s make test-pure | 1m7s | 0m20s make test-full | 1m21s | 0m32s make test-full-386 | 1m42s | 0m36s When running tests for a given package, fsync can be disabled as follows: ```shell DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/storage ``` Disabling fsync() is intended for testing purposes only and the name of the variables reflects that. What could also have been done but haven't: - lib/filestream/filestream.go: `Writer.MustFlush()` also uses f.Sync() but nothing has been done to it, because the Writer.MustFlush() is not used anywhere in the VM codebase. A side question: what is the general policy for the unused code? - lib/filestream/filestream.go: Writer.Write() calls `adviceDontNeed()` which calls unix.Fdatasync(). Disabling it could potentially improve running time, but running tests with this code disabled has shown otherwise. ### Checklist The following checks are **mandatory**: - [ x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <wwctrsrx@gmail.com> (cherry picked from commit 334cd92a6c3cdac1aa2102653ffc5e7d9382bfbc) Signed-off-by: hagen1778 <roman@victoriametrics.com>
2024-08-30 08:54:46 +00:00
DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
2019-07-23 16:54:50 +00:00
test-full:
testing: allow disabling fsync to make tests run faster (#6871) ### Describe Your Changes fsync() ensures that the data is written to disk. In production this is needed for data durability. However, during the development, when the unit tests are run, this level of durability is not needed. Therefore fsync() can be disabled which will makes test runs two times faster. The disabling is done by setting the `DISABLE_FSYNC_FOR_TESTING` environment variable. The valid values for this variable are the same as the values of the arg of `go doc strconv.ParseBool`: ``` 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. ``` Any other value means `false`. The variable is set for all test build targets. Compare running times: Build Target | DISABLE_FSYNC_FOR_TESTING=0 | DISABLE_FSYNC_FOR_TESTING=1 ----------------- | ------------------------------------------------ | ------------------------------------------------- make test | 1m5s | 0m22s make test-race | 3m1s | 1m42s make test-pure | 1m7s | 0m20s make test-full | 1m21s | 0m32s make test-full-386 | 1m42s | 0m36s When running tests for a given package, fsync can be disabled as follows: ```shell DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/storage ``` Disabling fsync() is intended for testing purposes only and the name of the variables reflects that. What could also have been done but haven't: - lib/filestream/filestream.go: `Writer.MustFlush()` also uses f.Sync() but nothing has been done to it, because the Writer.MustFlush() is not used anywhere in the VM codebase. A side question: what is the general policy for the unused code? - lib/filestream/filestream.go: Writer.Write() calls `adviceDontNeed()` which calls unix.Fdatasync(). Disabling it could potentially improve running time, but running tests with this code disabled has shown otherwise. ### Checklist The following checks are **mandatory**: - [ x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <wwctrsrx@gmail.com> (cherry picked from commit 334cd92a6c3cdac1aa2102653ffc5e7d9382bfbc) Signed-off-by: hagen1778 <roman@victoriametrics.com>
2024-08-30 08:54:46 +00:00
DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
test-full-386:
testing: allow disabling fsync to make tests run faster (#6871) ### Describe Your Changes fsync() ensures that the data is written to disk. In production this is needed for data durability. However, during the development, when the unit tests are run, this level of durability is not needed. Therefore fsync() can be disabled which will makes test runs two times faster. The disabling is done by setting the `DISABLE_FSYNC_FOR_TESTING` environment variable. The valid values for this variable are the same as the values of the arg of `go doc strconv.ParseBool`: ``` 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. ``` Any other value means `false`. The variable is set for all test build targets. Compare running times: Build Target | DISABLE_FSYNC_FOR_TESTING=0 | DISABLE_FSYNC_FOR_TESTING=1 ----------------- | ------------------------------------------------ | ------------------------------------------------- make test | 1m5s | 0m22s make test-race | 3m1s | 1m42s make test-pure | 1m7s | 0m20s make test-full | 1m21s | 0m32s make test-full-386 | 1m42s | 0m36s When running tests for a given package, fsync can be disabled as follows: ```shell DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/storage ``` Disabling fsync() is intended for testing purposes only and the name of the variables reflects that. What could also have been done but haven't: - lib/filestream/filestream.go: `Writer.MustFlush()` also uses f.Sync() but nothing has been done to it, because the Writer.MustFlush() is not used anywhere in the VM codebase. A side question: what is the general policy for the unused code? - lib/filestream/filestream.go: Writer.Write() calls `adviceDontNeed()` which calls unix.Fdatasync(). Disabling it could potentially improve running time, but running tests with this code disabled has shown otherwise. ### Checklist The following checks are **mandatory**: - [ x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <wwctrsrx@gmail.com> (cherry picked from commit 334cd92a6c3cdac1aa2102653ffc5e7d9382bfbc) Signed-off-by: hagen1778 <roman@victoriametrics.com>
2024-08-30 08:54:46 +00:00
DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
tests: Initial version of integration tests (#7253) ### Describe Your Changes Related issue: #7199 This is the initial version of the integration tests for cluster. See `README.md` for details. Currently cluster only, but it can also be used for vm-single if needed. The code has been added to the apptest package that resides in the root directory of the VM codebase. This is done to exclude the integration tests from regular testing build targets because: - Most of the test variants do not apply to integration testing (such as pure or race). - The integtation tests may also be slow because each test must wait for 2 seconds so vmstorage flushes pending content). It may be okay when there are a few tests but when there is a 100 of them running tests will require much more time which will affect the developer wait time and CI workflows. - Finally, the integration tests may be flaky especially short term. An alternative approach would be placing apptest under app package and exclude apptest from packages under test, but that is not trivial. The integration tests rely on retrieving some application runtime info from the application logs, namely the application's host:port. Therefore some changes to lib/httpserver/httpserver.go were necessary, such as reporting the effective host:port instead the one from the flag. ### Checklist The following checks are **mandatory**: - [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2024-10-30 14:22:06 +00:00
integration-test: all
go test ./apptest/... -skip="^TestSingle.*" -v
tests: Initial version of integration tests (#7253) ### Describe Your Changes Related issue: #7199 This is the initial version of the integration tests for cluster. See `README.md` for details. Currently cluster only, but it can also be used for vm-single if needed. The code has been added to the apptest package that resides in the root directory of the VM codebase. This is done to exclude the integration tests from regular testing build targets because: - Most of the test variants do not apply to integration testing (such as pure or race). - The integtation tests may also be slow because each test must wait for 2 seconds so vmstorage flushes pending content). It may be okay when there are a few tests but when there is a 100 of them running tests will require much more time which will affect the developer wait time and CI workflows. - Finally, the integration tests may be flaky especially short term. An alternative approach would be placing apptest under app package and exclude apptest from packages under test, but that is not trivial. The integration tests rely on retrieving some application runtime info from the application logs, namely the application's host:port. Therefore some changes to lib/httpserver/httpserver.go were necessary, such as reporting the effective host:port instead the one from the flag. ### Checklist The following checks are **mandatory**: - [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2024-10-30 14:22:06 +00:00
2019-05-22 21:16:55 +00:00
benchmark:
go test -bench=. ./lib/...
go test -bench=. ./app/...
2019-05-22 21:16:55 +00:00
benchmark-pure:
CGO_ENABLED=0 go test -bench=. ./lib/...
CGO_ENABLED=0 go test -bench=. ./app/...
2019-05-22 21:16:55 +00:00
vendor-update:
go get -u ./lib/...
go get -u ./app/...
go mod tidy -compat=1.23
go mod vendor
2019-05-22 21:16:55 +00:00
2019-05-22 21:23:23 +00:00
app-local:
CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
2019-07-23 16:54:50 +00:00
app-local-pure:
CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
2019-05-22 21:23:23 +00:00
app-local-goos-goarch:
CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
app-local-windows-goarch:
CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
2019-05-22 21:16:55 +00:00
quicktemplate-gen: install-qtc
qtc
install-qtc:
which qtc || go install github.com/valyala/quicktemplate/qtc@latest
golangci-lint: install-golangci-lint
golangci-lint run
install-golangci-lint:
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
remove-golangci-lint:
rm -rf `which golangci-lint`
govulncheck: install-govulncheck
govulncheck ./...
install-govulncheck:
which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
remove-govulncheck:
rm -rf `which govulncheck`
install-wwhrd:
which wwhrd || go install github.com/frapposelli/wwhrd@latest
check-licenses: install-wwhrd
wwhrd check -f .wwhrd.yml