Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

This commit is contained in:
Aliaksandr Valialkin 2022-08-31 02:32:32 +03:00
commit b9668d5294
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
9 changed files with 181 additions and 14 deletions

View file

@ -16,6 +16,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
include app/*/Makefile
include deployment/*/Makefile
include snap/local/Makefile
include package/release/Makefile
all: \
victoria-metrics-prod \
@ -168,7 +169,9 @@ publish-release:
git checkout $(TAG) && $(MAKE) release publish && \
git checkout $(TAG)-cluster && $(MAKE) release publish && \
git checkout $(TAG)-enterprise && $(MAKE) release publish && \
git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish && \
$(MAKE) github-create-release && \
$(MAKE) github-upload-assets
release: \
release-victoria-metrics \

View file

@ -62,6 +62,7 @@ VictoriaMetrics has the following prominent features:
* [JSON line format](#how-to-import-data-in-json-line-format).
* [Arbitrary CSV data](#how-to-import-csv-data).
* [Native binary format](#how-to-import-data-in-native-format).
* [DataDog agent or DogStatsD](#how-to-send-data-from-datadog-agent).
* It supports metrics [relabeling](#relabeling).
* It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/).
@ -1834,8 +1835,10 @@ It is safe sharing the collected profiles from security point of view, since the
See [these docs](https://github.com/netdata/netdata#integrations).
* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
See [this example](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml).
* [Ansible role for installing single-node VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
* [Ansible role for installing cluster VictoriaMetrics](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
* [vmalert-cli](https://github.com/aorfanos/vmalert-cli) - a CLI application for managing [vmalert](https://docs.victoriametrics.com/vmalert.html).

View file

@ -74,6 +74,20 @@ Released at 08-08-2022
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly show date picker at `Table` tab. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2874).
* BUGFIX: properly generate http redirects if `-http.pathPrefix` command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2918).
## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3)
Released at 30-08-2022
**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
* SECURITY: [vmalert](https://docs.victoriametrics.com/vmalert.html): do not expose `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flag values in logs and at `http://vmalert:8880/flags` page by default, since they may contain sensitive data such as auth keys. This aligns `vmalert` behaviour with [vmagent](https://docs.victoriametrics.com/vmagent.html), which doesn't expose `-remoteWrite.url` command-line flag value in logs and at `http://vmagent:8429/flags` page by default. Specify `-remoteWrite.showURL`, `-remoteRead.showURL` and `-datasource.showURL` command-line flags for showing values for the corresponding `-*.url` flags in logs. Thanks to @mble for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2965).
* SECURITY: upgrade base docker image (alpine) from 3.16.1 to 3.16.2. See [alpine 3.16.2 release notes](https://alpinelinux.org/posts/Alpine-3.13.12-3.14.8-3.15.6-3.16.2-released.html).
* BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode).
* BUGFIX: improve performance for requests to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues) when the filter in the `match[]` query arg matches small number of time series. The performance for this case has been reduced in [v1.78.0](https://docs.victoriametrics.com/CHANGELOG.html#v1780). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1533) issues.
* BUGFIX: increase the default limit on the number of concurrent merges for small parts from 8 to 16. This should help resolving potential issues with heavy data ingestion. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2673#issuecomment-1218185978) from @lukepalmer .
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): fix panic when incorrect arg is passed as `phi` into [histogram_quantiles](https://docs.victoriametrics.com/MetricsQL.html#histogram_quantiles) function. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3026).
## [v1.79.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.2)

View file

@ -13,6 +13,7 @@ for ingestion rates lower than a million data points per second.
The single-node version [scales perfectly](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
with the number of CPU cores, RAM and available storage space.
The single-node version is easier to configure and operate compared to the cluster version, so think twice before choosing the cluster version.
See [this question](https://docs.victoriametrics.com/FAQ.html#which-victoriametrics-type-is-recommended-for-use-in-production---single-node-or-cluster) for more details.
Join [our Slack](https://slack.victoriametrics.com/) or [contact us](mailto:info@victoriametrics.com) with consulting and support questions.

View file

@ -301,6 +301,41 @@ VictoriaMetrics maintains in-memory cache for mapping of [active time series](#w
See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986).
## Which VictoriaMetrics type is recommended for use in production - single-node or cluster?
Both [single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) and
[VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) are production-ready.
Single-node VictoriaMetrics is able to handle quite big workloads in production
with tens of millions of [active time series](https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series)
at the ingestion rate of million of samples per second. See [this case study](https://docs.victoriametrics.com/CaseStudies.html#wixcom).
Single-node VictoriaMetrics requires lower amounts of CPU and RAM for handling the same workload comparing
to cluster version of VictoriaMetrics, since it doesn't need to pass the encoded data over the network
between [cluster components](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#architecture-overview).
The performance of a single-node VictoriaMetrics scales almost perfectly with the available CPU, RAM and disk IO resources on the host where it runs -
see [this article](https://valyala.medium.com/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
Given the facts above **it is recommended to use single-node VictoriaMetrics in the majority of cases**.
Cluster version of VictoriaMetrics may be preferred over single-node VictoriaMetrics in the following relatively rare cases:
- If [multitenancy support](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) is needed,
since single-node VictoriaMetrics doesn't support multitenancy. Though it is possible to run multiple single-node VictoriaMetrics
instances - one per each tenant - and route incoming requests from particular tenant to the needed VictoriaMetrics instance
via [vmauth](https://docs.victoriametrics.com/vmauth.html).
- If the current workload cannot be handled by a single-node VictoriaMetrics. For example, if you are going to ingest hundreds of millions of active time series
at ingestion rates exceeding a million samples per second, then it is better to use cluster version of VictoriaMetrics,
since its capacity can [scale horizontally with the number of nodes in the cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-resizing-and-scalability).
## How to migrate data from single-node VictoriaMetrics to cluster version?
Single-node VictoriaMetrics stores data on disk in slightly different format comparing to cluster version of VictoriaMetrics.
So it is impossible to just copy the on-disk data from `-storageDataPath` directory from single-node VictoriaMetrics to a `vmstorage` node in VictoriaMetrics cluster.
If you need migrating data from single-node VictoriaMetrics to cluster version, then [follow these instructions](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics).
## Why isn't MetricsQL 100% compatible with PromQL?
[MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) provides better user experience than PromQL. It fixes a few annoying issues in PromQL. This prevents MetricsQL to be 100% compatible with PromQL. See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.

View file

@ -62,6 +62,7 @@ VictoriaMetrics has the following prominent features:
* [JSON line format](#how-to-import-data-in-json-line-format).
* [Arbitrary CSV data](#how-to-import-csv-data).
* [Native binary format](#how-to-import-data-in-native-format).
* [DataDog agent or DogStatsD](#how-to-send-data-from-datadog-agent).
* It supports metrics [relabeling](#relabeling).
* It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/).
@ -1834,8 +1835,10 @@ It is safe sharing the collected profiles from security point of view, since the
See [these docs](https://github.com/netdata/netdata#integrations).
* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
See [this example](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml).
* [Ansible role for installing single-node VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
* [Ansible role for installing cluster VictoriaMetrics](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
* [vmalert-cli](https://github.com/aorfanos/vmalert-cli) - a CLI application for managing [vmalert](https://docs.victoriametrics.com/vmalert.html).

View file

@ -4,22 +4,56 @@ sort: 18
# Release process guidance
## Prereqs
1. Make sure you have enterprise remote configured
```
git remote add enterprise <url>
```
2. Make sure you have singing key configured
3. Make sure you have github token with at least `read:org, repo, write:packages` permissions exported under `GITHUB_TOKEN` env variable.
You can create token [here](https://github.com/settings/tokens)
## Release version and Docker images
0. Make sure that the release commits have no security issues.
1a. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
1b. Add `(available in v1.xx.y)` line to feature docs introduced in the upcoming release.
1b. Add `(available starting from v1.xx.y)` line to feature docs introduced in the upcoming release.
2. Create the following release tags:
* `git tag -s v1.xx.y` in `master` branch
* `git tag -s v1.xx.y-cluster` in `cluster` branch
* `git tag -s v1.xx.y-enterprise` in `enterprise` branch
* `git tag -s v1.xx.y-enterprise-cluster` in `enterprise-cluster` branch
3. Run `TAG=v1.xx.y make publish-release`. It will create `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory and publish Docker images for the given `TAG`, `TAG-cluster`, `TAG-enterprise` and `TAG-enterprise-cluster`.
4. Push release tags to <https://github.com/VictoriaMetrics/VictoriaMetrics> : `git push origin v1.xx.y` and `git push origin v1.xx.y-cluster`. Do not push `-enterprise` tags to public repository.
5. Go to <https://github.com/VictoriaMetrics/VictoriaMetrics/releases> , create new release from the pushed tag on step 4 and upload `*.tar.gz` archive with the corresponding `_checksums.txt` from step 3.
6. Copy the [CHANGELOG](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) for this release to [releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page.
7. Bump version of the VictoriaMetrics cluster setup in for [sandbox environment](https://github.com/VictoriaMetrics/ops/blob/main/sandbox/manifests/benchmark-vm/vmcluster.yaml)
by [opening and merging PR](https://github.com/VictoriaMetrics/ops/pull/58).
3. Run `TAG=v1.xx.y make publish-release`. This command performs the following tasks:
a) Build and package binaries in `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory.
This step can be run manually with the command `make release` from the needed git tag.
b) Build and publish [multi-platform Docker images](https://docs.docker.com/build/buildx/multiplatform-images/)
for the given `TAG`, `TAG-cluster`, `TAG-enterprise` and `TAG-enterprise-cluster`.
The multi-platform Docker image is built for the following platforms:
* linux/amd64
* linux/arm64
* linux/arm
* linux/ppc64le
* linux/386
This step can be run manually with the command `make publish` from the needed git tag.
c) Create draft GitHub release with the name `TAG`. This step can be run manually
with the command `TAG=v1.xx.y make github-create-release`.
The release id is stored at `/tmp/vm-github-release` file.
d) Upload all the binaries and checksums created at step `a` to that release.
This step can be run manually with the command `make github-upload-assets`.
It is expected that the needed release id is stored at `/tmp/vm-github-release` file,
which must be created at the step `c`.
If the upload process is interrupted by any reason, then the following recovery steps must be performed:
- To delete the created draft release by running the command `make github-delete-release`.
This command expects that the id of the release to delete is located at `/tmp/vm-github-release`
file created at the step `c`.
- To run the command `TAG=v1.xx.y make github-create-release github-upload-assets`, so new release is created
and all the needed assets are re-uploaded to it.
5. Go to <https://github.com/VictoriaMetrics/VictoriaMetrics/releases> and verify that draft release with the name `TAG` has been created
and this release contains all the needed binaries and checksums.
6. Update the release description with the [CHANGELOG](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) for this release.
7. Remove the `draft` checkbox for the `TAG` release and manually publish it.
8. Bump version of the VictoriaMetrics cluster in the [sandbox environment](https://github.com/VictoriaMetrics/ops/blob/main/sandbox/manifests/benchmark-vm/vmcluster.yaml)
by [opening and merging PR](https://github.com/VictoriaMetrics/ops/pull/58).
## Building snap package

View file

@ -66,6 +66,7 @@ VictoriaMetrics has the following prominent features:
* [JSON line format](#how-to-import-data-in-json-line-format).
* [Arbitrary CSV data](#how-to-import-csv-data).
* [Native binary format](#how-to-import-data-in-native-format).
* [DataDog agent or DogStatsD](#how-to-send-data-from-datadog-agent).
* It supports metrics [relabeling](#relabeling).
* It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://victoriametrics.com/products/enterprise/).
@ -1838,8 +1839,10 @@ It is safe sharing the collected profiles from security point of view, since the
See [these docs](https://github.com/netdata/netdata#integrations).
* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
See [this example](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml).
* [Ansible role for installing single-node VictoriaMetrics](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
* [Ansible role for installing cluster VictoriaMetrics](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
* [Ansible role for installing cluster VictoriaMetrics (by VictoriaMetrics)](https://github.com/VictoriaMetrics/ansible-playbooks).
* [Ansible role for installing cluster VictoriaMetrics (by community)](https://github.com/Slapper/ansible-victoriametrics-cluster-role).
* [Ansible role for installing single-node VictoriaMetrics (by community)](https://github.com/dreamteam-gg/ansible-victoriametrics-role).
* [Snap package for VictoriaMetrics](https://snapcraft.io/victoriametrics).
* [vmalert-cli](https://github.com/aorfanos/vmalert-cli) - a CLI application for managing [vmalert](https://docs.victoriametrics.com/vmalert.html).

71
package/release/Makefile Normal file
View file

@ -0,0 +1,71 @@
GITHUB_RELEASE_SPEC_FILE="/tmp/vm-github-release"
GITHUB_DEBUG_FILE="/tmp/vm-github-debug"
github-token-check:
ifndef GITHUB_TOKEN
$(error missing GITHUB_TOKEN env var. It must contain github token for VictoriaMetrics project obtained from https://github.com/settings/tokens)
endif
github-tag-check:
ifndef TAG
$(error missing TAG env var. It must contain github release tag to create)
endif
github-create-release: github-token-check github-tag-check
@result=$$(curl -o $(GITHUB_RELEASE_SPEC_FILE) -s -w "%{http_code}" \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: token $(GITHUB_TOKEN)" \
https://api.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases \
-d '{"tag_name":"$(TAG)","name":"$(TAG)","body":"TODO: put here the changelog for $(TAG) release from docs/CHANGELOG.md","draft":true,"prerelease":false,"generate_release_notes":false}'); \
if [ $${result} = 201 ]; then \
release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
printf "Created release $(TAG) with id=$${release_id}\n"; \
else \
printf "Failed to create release $(TAG)\n"; \
cat $(GITHUB_RELEASE_SPEC_FILE); \
exit 1; \
fi
github-upload-assets:
@release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
$(foreach file, $(wildcard bin/*.zip), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="application/zip" $(MAKE) github-upload-asset || exit 1;) \
$(foreach file, $(wildcard bin/*.tar.gz), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="application/x-gzip" $(MAKE) github-upload-asset || exit 1;) \
$(foreach file, $(wildcard bin/*_checksums.txt), FILE=$(file) RELEASE_ID=$${release_id} CONTENT_TYPE="text/plain" $(MAKE) github-upload-asset || exit 1;)
github-upload-asset: github-token-check
ifndef FILE
$(error missing FILE env var. It must contain path to file to upload to github release)
endif
@printf "Uploading $(FILE)\n"
@result=$$(curl -o $(GITHUB_DEBUG_FILE) -w "%{http_code}" \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: token $(GITHUB_TOKEN)" \
-H "Content-Type: $(CONTENT_TYPE)" \
--data-binary "@$(FILE)" \
https://uploads.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases/$(RELEASE_ID)/assets?name=$(notdir $(FILE))); \
if [ $${result} = 201 ]; then \
printf "Upload OK: $${result}\n"; \
elif [ $${result} = 422 ]; then \
printf "Asset already uploaded, you need to delete it from UI if you want to re-upload it\n"; \
else \
printf "Upload failed: $${result}\n"; \
cat $(GITHUB_DEBUG_FILE); \
exit 1; \
fi
github-delete-release: github-token-check
@release_id=$$(cat $(GITHUB_RELEASE_SPEC_FILE) | grep '"id"' -m 1 | sed -E 's/.* ([[:digit:]]+)\,/\1/'); \
result=$$(curl -o $(GITHUB_DEBUG_FILE) -s -w "%{http_code}" \
-X DELETE \
-H "Accept: application/vnd.github+json" \
-H "Authorization: token $(GITHUB_TOKEN)" \
https://api.github.com/repos/VictoriaMetrics/VictoriaMetrics/releases/$${release_id}); \
if [ $${result} = 204 ]; then \
printf "Deleted release with id=$${release_id}\n"; \
else \
printf "Failed to delete release with id=$${release_id}\n"; \
cat $(GITHUB_DEBUG_FILE); \
exit 1; \
fi