mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-01 14:47:38 +00:00
Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files
This commit is contained in:
commit
bd79706eb3
41 changed files with 805 additions and 47 deletions
|
@ -2342,6 +2342,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-usePromCompatibleNaming
|
||||
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
|
||||
-version
|
||||
|
|
14
SECURITY.md
Normal file
14
SECURITY.md
Normal file
|
@ -0,0 +1,14 @@
|
|||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Version | Supported |
|
||||
|---------|--------------------|
|
||||
| 1.81.x | :white_check_mark: |
|
||||
| 1.80.x | :x: |
|
||||
| 1.79.x | :white_check_mark: |
|
||||
| < 1.78 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report any security issues to security@victoriametrics.com
|
|
@ -1269,6 +1269,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-usePromCompatibleNaming
|
||||
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
|
||||
-version
|
||||
|
|
|
@ -1069,6 +1069,8 @@ The shortlist of configuration flags is the following:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -308,6 +308,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -266,6 +266,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -72,14 +72,14 @@ Backup manager launched with the following configuration:
|
|||
```console
|
||||
export NODE_IP=192.168.0.10
|
||||
export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
|
||||
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
|
||||
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
|
||||
```
|
||||
|
||||
Expected logs in vmbackupmanager:
|
||||
|
||||
```console
|
||||
info lib/backup/actions/backup.go:131 server-side copied 81 out of 81 parts from GCS{bucket: "vmstorage-data", dir: "192.168.0.10//latest/"} to GCS{bucket: "vmstorage-data", dir: "192.168.0.10//weekly/2020-34/"} in 2.549833008s
|
||||
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
|
||||
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
|
||||
```
|
||||
|
||||
Expected logs in vmstorage:
|
||||
|
@ -93,7 +93,7 @@ info VictoriaMetrics/lib/storage/storage.go:319 deleted snapshot "/vmstora
|
|||
The result on the GCS bucket
|
||||
|
||||
* The root folder
|
||||
|
||||
|
||||
![root](vmbackupmanager_root_folder.png)
|
||||
|
||||
* The latest folder
|
||||
|
@ -265,6 +265,8 @@ vmbackupmanager performs regular backups according to the provided configs.
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -149,7 +149,7 @@ cat << EOF > limit.yaml
|
|||
limits:
|
||||
- type: queries
|
||||
value: 100
|
||||
- type: rows_inserted
|
||||
- type: rows_inserted
|
||||
value: 100000
|
||||
- type: new_series
|
||||
value: 1000
|
||||
|
@ -168,7 +168,7 @@ curl 'http://localhost:8431/api/v1/import/prometheus' -X POST -d 'foo{bar="baz1
|
|||
# read metric from tenant 1:5
|
||||
curl 'http://localhost:8431/api/v1/labels' -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MjAxNjIwMDAwMDAsInZtX2FjY2VzcyI6eyJ0ZW5hbnRfaWQiOnsiYWNjb3VudF9pZCI6MTV9fX0.PB1_KXDKPUp-40pxOGk6lt_jt9Yq80PIMpWVJqSForQ'
|
||||
|
||||
# check rate limit
|
||||
# check rate limit
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
@ -199,11 +199,11 @@ The shortlist of configuration flags include the following:
|
|||
-datasource.maxIdleConnections int
|
||||
Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
|
||||
-datasource.oauth2.clientID string
|
||||
Optional OAuth2 clientID to use for -datasource.url.
|
||||
Optional OAuth2 clientID to use for -datasource.url.
|
||||
-datasource.oauth2.clientSecret string
|
||||
Optional OAuth2 clientSecret to use for -datasource.url.
|
||||
-datasource.oauth2.clientSecretFile string
|
||||
Optional OAuth2 clientSecretFile to use for -datasource.url.
|
||||
Optional OAuth2 clientSecretFile to use for -datasource.url.
|
||||
-datasource.oauth2.scopes string
|
||||
Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
|
||||
-datasource.oauth2.tokenUrl string
|
||||
|
@ -315,6 +315,8 @@ The shortlist of configuration flags include the following:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
-write.url string
|
||||
|
|
|
@ -166,6 +166,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
12
deployment/marketplace/vultr/Makefile
Normal file
12
deployment/marketplace/vultr/Makefile
Normal file
|
@ -0,0 +1,12 @@
|
|||
RELEASE_NAME := vm-vultr-server
|
||||
VM_VERSION ?= $(shell git describe --abbrev=0 --tags)
|
||||
PACKER_LOG := 1
|
||||
PACKER_LOG_PATH := packer.log
|
||||
|
||||
.PHONY: $(MAKECMDGOALS)
|
||||
|
||||
release-victoria-metrics-vultr-server:
|
||||
cp ./victoriametrics-single/etc/update-motd.d/99-one-click.tpl ./victoriametrics-single/etc/update-motd.d/99-one-click
|
||||
sed -i -e "s/VM_VERSION/${VM_VERSION}/g" ./victoriametrics-single/etc/update-motd.d/99-one-click
|
||||
packer init victoriametrics-single/victoriametrics-single.pkr.hcl
|
||||
packer build victoriametrics-single/victoriametrics-single.pkr.hcl
|
55
deployment/marketplace/vultr/README.md
Normal file
55
deployment/marketplace/vultr/README.md
Normal file
|
@ -0,0 +1,55 @@
|
|||
## Application summary
|
||||
|
||||
VictoriaMetrics is a fast and scalable open source time series database and monitoring solution.
|
||||
|
||||
## Description
|
||||
|
||||
VictoriaMetrics is a free [open source time series database](https://en.wikipedia.org/wiki/Time_series_database) (TSDB) and monitoring solution, designed to collect, store and process real-time metrics.
|
||||
|
||||
It supports the [Prometheus](https://en.wikipedia.org/wiki/Prometheus_(software)) pull model and various push protocols ([Graphite](https://en.wikipedia.org/wiki/Graphite_(software)), [InfluxDB](https://en.wikipedia.org/wiki/InfluxDB), OpenTSDB) for data ingestion. It is optimized for storage with high-latency IO, low IOPS and time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
|
||||
|
||||
For reading the data and evaluating alerting rules, VictoriaMetrics supports the PromQL, [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) and Graphite query languages. VictoriaMetrics Single is fully autonomous and can be used as a long-term storage for time series.
|
||||
|
||||
[VictoriaMetrics Single](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) = Hassle-free monitoring solution. Easily handles 10M+ of active time series on a single instance. Perfect for small and medium environments.
|
||||
|
||||
## Getting started after deploying VictoriaMetrics Single
|
||||
|
||||
### Config
|
||||
|
||||
VictoriaMetrics configuration is located at `/etc/victoriametrics/single/scrape.yml` on the server.
|
||||
This One Click app uses 8428, 2003, 4242 and 8089 ports to accept metrics from different protocols. It's recommended to disable ports for protocols which are not needed. [Ubuntu firewall](https://help.ubuntu.com/community/UFW) can be used to easily disable access for specific ports.
|
||||
|
||||
### Scraping metrics
|
||||
|
||||
VictoriaMetrics supports metrics scraping in the same way as Prometheus does. Check the configuration file to edit scraping targets. See more details about scraping at [How to scrape Prometheus exporters](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-scrape-prometheus-exporters-such-as-node-exporter).
|
||||
|
||||
### Sending metrics
|
||||
|
||||
Besides scraping, VictoriaMetrics accepts write requests for various ingestion protocols. This One Click app supports the following protocols:
|
||||
|
||||
- [Datadog](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent), [Influx (telegraph)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf), [JSON](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format), [CSV](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data), [Prometheus](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) on port :8428
|
||||
- [Graphite (statsd)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) on port :2003 tcp/udp
|
||||
- [OpenTSDB](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents) on port :4242
|
||||
- Influx (telegraph) on port :8089 tcp/udp
|
||||
|
||||
See more details and examples in [official documentation](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html).
|
||||
|
||||
### UI
|
||||
|
||||
VictoriaMetrics provides a [User Interface (UI)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmui) for query troubleshooting and exploration. The UI is available at `http://your_server_public_ipv4:8428/vmui`. It lets users explore query results via graphs and tables.
|
||||
|
||||
To check it, open the following in your browser `http://your_server_public_ipv4:8428/vmui` and then enter `vm_app_uptime_seconds` to the Query Field to Execute the Query.
|
||||
|
||||
Run the following command to query and retrieve a result from VictoriaMetrics Single with `curl`:
|
||||
|
||||
```console
|
||||
curl -sg http://your_server_public_ipv4:8428/api/v1/query_range?query=vm_app_uptime_seconds | jq
|
||||
```
|
||||
|
||||
### Accessing
|
||||
|
||||
Once the **server** is created, you can use Vultr's web console to start a session or SSH directly to the server as root:
|
||||
|
||||
```console
|
||||
ssh root@your_server_public_ipv4
|
||||
```
|
12
deployment/marketplace/vultr/RELEASE_GUIDE.md
Normal file
12
deployment/marketplace/vultr/RELEASE_GUIDE.md
Normal file
|
@ -0,0 +1,12 @@
|
|||
## Release guide for Vultr Marketplace
|
||||
|
||||
### Build image
|
||||
|
||||
1. To build the snapshot in Vultr account you will need `VULTR_API_KEY` and [packer](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli).
|
||||
2. `VULTR_API_KEY` can be generated on [https://my.vultr.com/settings/#settingsapi](https://my.vultr.com/settings/#settingsapi) or use already generated from OnePassword.
|
||||
3. Choose prefered version of VictoriaMetrics on [Github releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page.
|
||||
4. Set variables `VULTR_API_KEY` with `VM_VERSION` for `packer` environment and run make from example below:
|
||||
|
||||
```console
|
||||
make release-victoria-metrics-vultr-server VULTR_API_KEY="your_token_here" VM_VERSION="prefered_release_version"
|
||||
```
|
317
deployment/marketplace/vultr/helper-scripts/vultr-helper.sh
Executable file
317
deployment/marketplace/vultr/helper-scripts/vultr-helper.sh
Executable file
|
@ -0,0 +1,317 @@
|
|||
#!/bin/bash
|
||||
|
||||
# shopt -s inherit_errexit
|
||||
set -o errexit
|
||||
|
||||
###################################################################
|
||||
## Vultr Marketplace Helper Functions
|
||||
|
||||
function error_detect_on()
|
||||
{
|
||||
set -euo pipefail
|
||||
}
|
||||
|
||||
function error_detect_off()
|
||||
{
|
||||
set +euo pipefail
|
||||
}
|
||||
|
||||
function enable_verbose_commands()
|
||||
{
|
||||
set -x
|
||||
}
|
||||
|
||||
function disable_verbose_commands()
|
||||
{
|
||||
set +x
|
||||
}
|
||||
|
||||
function get_metadata_item()
|
||||
{
|
||||
local item_path="${1:-}"
|
||||
local item_value
|
||||
|
||||
item_value="$(curl --fail --silent --header "Metadata-Token: vultr" "http://169.254.169.254/${item_path}")"
|
||||
|
||||
echo "${item_value}"
|
||||
}
|
||||
|
||||
function get_hostname()
|
||||
{
|
||||
get_metadata_item "latest/meta-data/hostname"
|
||||
}
|
||||
|
||||
function get_userdata()
|
||||
{
|
||||
get_metadata_item "latest/user-data"
|
||||
}
|
||||
|
||||
function get_sshkeys()
|
||||
{
|
||||
get_metadata_item "current/ssh-keys"
|
||||
}
|
||||
|
||||
function get_var()
|
||||
{
|
||||
local var_name="${1:-}"
|
||||
local var_val
|
||||
var_val="$(get_metadata_item "v1/internal/app-${var_name}" 2>/dev/null)"
|
||||
|
||||
eval "${var_name}='${var_val}'"
|
||||
}
|
||||
|
||||
function get_ip()
|
||||
{
|
||||
local ip_var="${1:-}"
|
||||
local ip_val
|
||||
ip_val="$(get_metadata_item "latest/meta-data/public-ipv4" 2>/dev/null)"
|
||||
|
||||
eval "${ip_var}='${ip_val}'"
|
||||
}
|
||||
|
||||
function wait_on_apt_lock()
|
||||
{
|
||||
until ! lsof -t /var/cache/apt/archives/lock /var/lib/apt/lists/lock /var/lib/dpkg/lock >/dev/null 2>&1
|
||||
do
|
||||
echo "Waiting 3 for apt lock currently held by another process."
|
||||
sleep 3
|
||||
done
|
||||
}
|
||||
|
||||
function apt_safe()
|
||||
{
|
||||
wait_on_apt_lock
|
||||
apt install -y "$@"
|
||||
}
|
||||
|
||||
function apt_update_safe()
|
||||
{
|
||||
wait_on_apt_lock
|
||||
apt update -y
|
||||
}
|
||||
|
||||
function apt_upgrade_safe()
|
||||
{
|
||||
wait_on_apt_lock
|
||||
DEBIAN_FRONTEND=noninteractive apt upgrade -y
|
||||
}
|
||||
|
||||
function apt_remove_safe()
|
||||
{
|
||||
wait_on_apt_lock
|
||||
apt remove -y --auto-remove "$@"
|
||||
}
|
||||
|
||||
function apt_clean_safe()
|
||||
{
|
||||
wait_on_apt_lock
|
||||
apt autoremove -y
|
||||
|
||||
wait_on_apt_lock
|
||||
apt autoclean -y
|
||||
}
|
||||
|
||||
function update_and_clean_packages()
|
||||
{
|
||||
# RHEL/CentOS
|
||||
if [[ -f /etc/redhat-release ]]; then
|
||||
yum update -y
|
||||
yum clean all
|
||||
# Ubuntu / Debian
|
||||
elif grep -qs "debian" /etc/os-release 2>/dev/null; then
|
||||
apt_update_safe
|
||||
apt_upgrade_safe
|
||||
apt_clean_safe
|
||||
fi
|
||||
}
|
||||
|
||||
function set_vultr_kernel_option()
|
||||
{
|
||||
# RHEL/CentOS
|
||||
if [[ -f /etc/redhat-release ]]; then
|
||||
/sbin/grubby --update-kernel=ALL --args vultr
|
||||
# Ubuntu / Debian
|
||||
elif grep -qs "debian" /etc/os-release 2>/dev/null; then
|
||||
sed -i -e "/^GRUB_CMDLINE_LINUX_DEFAULT=/ s/\"$/ vultr\"/" /etc/default/grub
|
||||
update-grub
|
||||
fi
|
||||
}
|
||||
|
||||
function install_cloud_init()
|
||||
{
|
||||
local cloud_init_exe
|
||||
cloud_init_exe="$(command -v cloud-init >/dev/null 2>&1)"
|
||||
if [[ -x "${cloud_init_exe}" ]]; then
|
||||
echo "cloud-init is already installed."
|
||||
return
|
||||
fi
|
||||
|
||||
local release_version="${1:-"latest"}"
|
||||
if [[ "${release_version}" != "latest" && "${release_version}" != "nightly" ]]; then
|
||||
echo "${release_version} is an invalid release option. Allowed: latest, nightly"
|
||||
exit 255
|
||||
fi
|
||||
|
||||
# Lets remove all traces of previously installed cloud-init
|
||||
# Ubuntu installs have proven problematic with their left over
|
||||
# configs for the installer in recent versions
|
||||
cleanup_cloudinit
|
||||
|
||||
update_and_clean_packages
|
||||
|
||||
local build_type
|
||||
local package_ext
|
||||
|
||||
[[ -e /etc/os-release ]] && . /etc/os-release
|
||||
case "${ID:-}" in
|
||||
debian)
|
||||
build_type="debian"
|
||||
package_ext="deb"
|
||||
;;
|
||||
fedora)
|
||||
build_type="rhel"
|
||||
package_ext="rpm"
|
||||
;;
|
||||
ubuntu)
|
||||
build_type="universal"
|
||||
package_ext="deb"
|
||||
;;
|
||||
*)
|
||||
case "${ID_LIKE:-}" in
|
||||
*rhel*)
|
||||
build_type="rhel"
|
||||
package_ext="rpm"
|
||||
;;
|
||||
*)
|
||||
echo "Unable to determine OS. Please install from source!"
|
||||
exit 255
|
||||
esac
|
||||
esac
|
||||
|
||||
local cloud_init_package="cloud-init_${build_type}_${release_version}.${package_ext}"
|
||||
wget -O "/tmp/${cloud_init_package}" "https://ewr1.vultrobjects.com/cloud_init_beta/${cloud_init_package}"
|
||||
|
||||
case "${package_ext}" in
|
||||
rpm)
|
||||
yum install -y "/tmp/${cloud_init_package}"
|
||||
;;
|
||||
deb)
|
||||
apt_safe "/tmp/${cloud_init_package}"
|
||||
;;
|
||||
*)
|
||||
echo "Unable to determine package installation method."
|
||||
exit 255
|
||||
esac
|
||||
|
||||
rm -f "/tmp/${cloud_init_package}"
|
||||
}
|
||||
|
||||
function cleanup_cloudinit()
|
||||
{
|
||||
rm -rf \
|
||||
/etc/cloud \
|
||||
/etc/systemd/system/cloud-init.target.wants/* \
|
||||
/lib/systemd/system/cloud* \
|
||||
/run/cloud-init \
|
||||
/usr/bin/cloud* \
|
||||
/usr/lib/cloud* \
|
||||
/usr/local/bin/cloud* \
|
||||
/usr/src/cloud* \
|
||||
/var/log/cloud*
|
||||
}
|
||||
|
||||
function clean_tmp()
|
||||
{
|
||||
mkdir -p /tmp
|
||||
chmod 1777 /tmp
|
||||
rm -rf /tmp/* /var/tmp/*
|
||||
}
|
||||
|
||||
function clean_keys()
|
||||
{
|
||||
rm -f /root/.ssh/authorized_keys /etc/ssh/*key*
|
||||
touch /etc/ssh/revoked_keys
|
||||
chmod 600 /etc/ssh/revoked_keys
|
||||
}
|
||||
|
||||
function clean_logs()
|
||||
{
|
||||
find /var/log -mtime -1 -type f -exec truncate -s 0 {} \;
|
||||
rm -rf \
|
||||
/var/log/*.[0-9] \
|
||||
/var/log/*.gz \
|
||||
/var/log/*.log \
|
||||
/var/log/lastlog \
|
||||
/var/log/wtmp
|
||||
|
||||
: > /var/log/auth.log
|
||||
}
|
||||
|
||||
function clean_history()
|
||||
{
|
||||
history -c
|
||||
: > /root/.bash_history
|
||||
unset HISTFILE
|
||||
}
|
||||
|
||||
function clean_mloc()
|
||||
{
|
||||
/usr/bin/updatedb || true
|
||||
}
|
||||
|
||||
function clean_random()
|
||||
{
|
||||
rm -f /var/lib/systemd/random-seed
|
||||
}
|
||||
|
||||
function clean_machine_id()
|
||||
{
|
||||
[[ -e /etc/machine-id ]] && : > /etc/machine-id
|
||||
[[ -e /var/lib/dbus/machine-id ]] && : > /var/lib/dbus/machine-id
|
||||
}
|
||||
|
||||
function clean_free_space()
|
||||
{
|
||||
dd if=/dev/zero of=/zerofile || true
|
||||
sync
|
||||
rm -f /zerofile
|
||||
sync
|
||||
}
|
||||
|
||||
function trim_ssd()
|
||||
{
|
||||
fstrim / || true
|
||||
}
|
||||
|
||||
function cleanup_marketplace_scripts()
|
||||
{
|
||||
rm -f /root/*.sh
|
||||
}
|
||||
|
||||
function disable_network_manager()
|
||||
{
|
||||
## Disable NetworkManager, replace with network-scripts
|
||||
systemctl disable --now NetworkManager
|
||||
sed -i \
|
||||
-e 's/^ONBOOT.*/ONBOOT=yes/g' \
|
||||
-e 's/^NM_CONTROLLED.*/NM_CONTROLLED=no/g' /etc/sysconfig/network-scripts/ifcfg-*
|
||||
yum install -y network-scripts
|
||||
}
|
||||
|
||||
function clean_system()
|
||||
{
|
||||
|
||||
update_and_clean_packages
|
||||
set_vultr_kernel_option
|
||||
clean_tmp
|
||||
clean_keys
|
||||
clean_logs
|
||||
clean_history
|
||||
clean_random
|
||||
clean_machine_id
|
||||
clean_mloc
|
||||
clean_free_space
|
||||
trim_ssd
|
||||
|
||||
cleanup_marketplace_scripts
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
[Unit]
|
||||
Description=VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
|
||||
# https://docs.victoriametrics.com
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=victoriametrics
|
||||
Group=victoriametrics
|
||||
WorkingDirectory=/var/lib/victoria-metrics-data
|
||||
StartLimitBurst=5
|
||||
StartLimitInterval=0
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
EnvironmentFile=-/etc/victoriametrics/single/victoriametrics.conf
|
||||
ExecStart=/usr/bin/victoria-metrics-prod $ARGS
|
||||
ExecStop=/bin/kill -s SIGTERM $MAINPID
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
# See docs https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#tuning
|
||||
ProtectSystem=full
|
||||
LimitNOFILE=1048576
|
||||
LimitNPROC=1048576
|
||||
LimitCORE=infinity
|
||||
StandardOutput=syslog
|
||||
StandardError=syslog
|
||||
SyslogIdentifier=vmsingle
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# Configured as part of the DigitalOcean 1-Click Image build process
|
||||
|
||||
myip=$(hostname -I | awk '{print$1}')
|
||||
cat <<EOF
|
||||
********************************************************************************
|
||||
|
||||
Welcome to VictoriaMetrics server!
|
||||
To keep this server secure, the UFW firewall is enabled.
|
||||
All ports are BLOCKED except 22 (SSH), 80 (HTTP), and 443 (HTTPS), 8428 (VictoriaMetrics HTTP), 8089 (VictoriaMetrics Influx),
|
||||
4242 (VictoriaMetrics OpenTSDB), 2003 (VictoriaMetrics Graphite)
|
||||
|
||||
In a web browser, you can view:
|
||||
* The VictoriaMetrics Quickstart guide: https://kutt.it/1click-quickstart
|
||||
|
||||
On the server:
|
||||
* The default VictoriaMetrics root is located at /var/lib/victoria-metrics-data
|
||||
* VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
|
||||
|
||||
********************************************************************************
|
||||
# This image includes version VM_VERSION of VictoriaMetrics.
|
||||
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/VM_VERSION
|
||||
|
||||
# Website: https://victoriametrics.com
|
||||
# Documentation: https://docs.victoriametrics.com
|
||||
# VictoriaMetrics Github : https://github.com/VictoriaMetrics/VictoriaMetrics
|
||||
# VictoriaMetrics Slack Community: https://slack.victoriametrics.com
|
||||
# VictoriaMetrics Telegram Community: https://t.me/VictoriaMetrics_en
|
||||
# VictoriaMetrics in Twitter: https://twitter.com/VictoriaMetrics
|
||||
|
||||
# VictoriaMetrics config: /etc/victoriametrics/single/victoriametrics.conf
|
||||
|
||||
********************************************************************************
|
||||
EOF
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# Configured as part of the DigitalOcean 1-Click Image build process
|
||||
|
||||
myip=$(hostname -I | awk '{print$1}')
|
||||
cat <<EOF
|
||||
********************************************************************************
|
||||
|
||||
Welcome to VictoriaMetrics server!
|
||||
To keep this server secure, the UFW firewall is enabled.
|
||||
All ports are BLOCKED except 22 (SSH), 80 (HTTP), and 443 (HTTPS), 8428 (VictoriaMetrics HTTP), 8089 (VictoriaMetrics Influx),
|
||||
4242 (VictoriaMetrics OpenTSDB), 2003 (VictoriaMetrics Graphite)
|
||||
|
||||
In a web browser, you can view:
|
||||
* The VictoriaMetrics Quickstart guide: https://kutt.it/1click-quickstart
|
||||
|
||||
On the server:
|
||||
* The default VictoriaMetrics root is located at /var/lib/victoria-metrics-data
|
||||
* VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
|
||||
|
||||
********************************************************************************
|
||||
# This image includes version VM_VERSION of VictoriaMetrics.
|
||||
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/VM_VERSION
|
||||
|
||||
# Website: https://victoriametrics.com
|
||||
# Documentation: https://docs.victoriametrics.com
|
||||
# VictoriaMetrics Github : https://github.com/VictoriaMetrics/VictoriaMetrics
|
||||
# VictoriaMetrics Slack Community: https://slack.victoriametrics.com
|
||||
# VictoriaMetrics Telegram Community: https://t.me/VictoriaMetrics_en
|
||||
# VictoriaMetrics in Twitter: https://twitter.com/VictoriaMetrics
|
||||
|
||||
# VictoriaMetrics config: /etc/victoriametrics/single/victoriametrics.conf
|
||||
|
||||
********************************************************************************
|
||||
EOF
|
|
@ -0,0 +1,7 @@
|
|||
# Scrape config example
|
||||
#
|
||||
scrape_configs:
|
||||
- job_name: self_scrape
|
||||
scrape_interval: 10s
|
||||
static_configs:
|
||||
- targets: ['127.0.0.1:8428']
|
|
@ -0,0 +1 @@
|
|||
ARGS="-promscrape.config=/etc/victoriametrics/single/scrape.yml -storageDataPath=/var/lib/victoria-metrics-data -retentionPeriod=12 -httpListenAddr=:8428 -graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -influxListenAddr=:8089 -enableTCP6"
|
4
deployment/marketplace/vultr/victoriametrics-single/setup-per-boot.sh
Executable file
4
deployment/marketplace/vultr/victoriametrics-single/setup-per-boot.sh
Executable file
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
|
||||
## Run on every boot.
|
||||
echo $(date -u) ": System booted." >> /var/log/per-boot.log
|
29
deployment/marketplace/vultr/victoriametrics-single/setup-per-instance.sh
Executable file
29
deployment/marketplace/vultr/victoriametrics-single/setup-per-instance.sh
Executable file
|
@ -0,0 +1,29 @@
|
|||
#!/bin/bash
|
||||
## Runs once-and-only-once at first boot per instance.
|
||||
|
||||
## Report the start time to a logfile.
|
||||
echo $(date -u) ": System provisioning started." >> /var/log/per-instance.log
|
||||
|
||||
apt update
|
||||
DEBIAN_FRONTEND=noninteractive apt -qq full-upgrade -y
|
||||
DEBIAN_FRONTEND=noninteractive apt -qq install -y ufw wget software-properties-common ssh
|
||||
|
||||
# Configure UFW
|
||||
|
||||
sed -e 's|DEFAULT_FORWARD_POLICY=.*|DEFAULT_FORWARD_POLICY="ACCEPT"|g' \
|
||||
-i /etc/default/ufw
|
||||
|
||||
ufw allow ssh comment "SSH port"
|
||||
ufw allow http comment "HTTP port"
|
||||
ufw allow https comment "HTTPS port"
|
||||
ufw allow 8428 comment "VictoriaMetrics Single HTTP port"
|
||||
ufw allow 8089/tcp comment "TCP Influx Listen port for VictoriaMetrics"
|
||||
ufw allow 8089/udp comment "UDP Influx Listen port for VictoriaMetrics"
|
||||
ufw allow 2003/tcp comment "TCP Graphite Listen port for VictoriaMetrics"
|
||||
ufw allow 2003/udp comment "UDP Graphite Listen port for VictoriaMetrics"
|
||||
ufw allow 4242 comment "OpenTSDB Listen port for VictoriaMetrics"
|
||||
|
||||
ufw --force enable
|
||||
|
||||
## Report the end time to a logfile.
|
||||
echo $(date -u) ": System provisioning script is complete." >> /var/log/per-instance.log
|
|
@ -0,0 +1,65 @@
|
|||
variable "vultr_api_key" {
|
||||
type = string
|
||||
default = "${env("VULTR_API_KEY")}"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "victoriametrics_version" {
|
||||
type = string
|
||||
default = "${env("VM_VERSION")}"
|
||||
description = "Version number of the desired VictoriaMetrics binary."
|
||||
}
|
||||
|
||||
packer {
|
||||
required_plugins {
|
||||
vultr = {
|
||||
version = ">=v2.3.2"
|
||||
source = "github.com/vultr/vultr"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
source "vultr" "victoriametrics-single" {
|
||||
api_key = "${var.vultr_api_key}"
|
||||
os_id = "387"
|
||||
plan_id = "vc2-1c-1gb"
|
||||
region_id = "ewr"
|
||||
snapshot_description = "victoriametrics-snapshot-${formatdate("YYYY-MM-DD hh:mm", timestamp())}"
|
||||
ssh_username = "root"
|
||||
state_timeout = "10m"
|
||||
}
|
||||
|
||||
build {
|
||||
sources = ["source.vultr.victoriametrics-single"]
|
||||
|
||||
provisioner "file" {
|
||||
source = "helper-scripts/vultr-helper.sh"
|
||||
destination = "/root/vultr-helper.sh"
|
||||
}
|
||||
|
||||
provisioner "file" {
|
||||
source = "victoriametrics-single/setup-per-boot.sh"
|
||||
destination = "/root/setup-per-boot.sh"
|
||||
}
|
||||
|
||||
# Copy configuration files
|
||||
provisioner "file" {
|
||||
destination = "/etc/"
|
||||
source = "victoriametrics-single/etc/"
|
||||
}
|
||||
|
||||
provisioner "file" {
|
||||
source = "victoriametrics-single/setup-per-instance.sh"
|
||||
destination = "/root/setup-per-instance.sh"
|
||||
}
|
||||
|
||||
provisioner "shell" {
|
||||
environment_vars = [
|
||||
"VM_VERSION=${var.victoriametrics_version}",
|
||||
"DEBIAN_FRONTEND=noninteractive"
|
||||
]
|
||||
script = "victoriametrics-single/victoriametrics-single.sh"
|
||||
remote_folder = "/root"
|
||||
remote_file = "victoriametrics-single.sh"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,41 @@
|
|||
#!/bin/bash
|
||||
################################################
|
||||
## Prerequisites
|
||||
chmod +x /root/vultr-helper.sh
|
||||
. /root/vultr-helper.sh
|
||||
error_detect_on
|
||||
install_cloud_init latest
|
||||
|
||||
################################################
|
||||
## Create victoriametrics user
|
||||
groupadd -r victoriametrics
|
||||
useradd -g victoriametrics -d /var/lib/victoria-metrics-data -s /sbin/nologin --system victoriametrics
|
||||
|
||||
mkdir -p /var/lib/victoria-metrics-data
|
||||
chown -R victoriametrics:victoriametrics /var/lib/victoria-metrics-data
|
||||
|
||||
################################################
|
||||
## Download VictoriaMetrics
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/victoria-metrics-linux-amd64-${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz
|
||||
tar xvf /tmp/victoria-metrics.tar.gz -C /usr/bin
|
||||
chmod +x /usr/bin/victoria-metrics-prod
|
||||
chown root:root /usr/bin/victoria-metrics-prod
|
||||
|
||||
################################################
|
||||
## Install provisioning scripts
|
||||
mkdir -p /var/lib/cloud/scripts/per-boot/
|
||||
mkdir -p /var/lib/cloud/scripts/per-instance/
|
||||
|
||||
mv /root/setup-per-boot.sh /var/lib/cloud/scripts/per-boot/setup-per-boot.sh
|
||||
mv /root/setup-per-instance.sh /var/lib/cloud/scripts/per-instance/setup-per-instance.sh
|
||||
|
||||
chmod +x /var/lib/cloud/scripts/per-boot/setup-per-boot.sh
|
||||
chmod +x /var/lib/cloud/scripts/per-instance/setup-per-instance.sh
|
||||
|
||||
# Enable VictoriaMetrics on boot
|
||||
systemctl enable vmsingle.service
|
||||
|
||||
################################################
|
||||
## Prepare server for Marketplace snapshot
|
||||
|
||||
clean_system
|
|
@ -25,6 +25,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
* FEATURE: check the correctess of raw sample timestamps stored on disk when reading them. This reduces the probability of possible silent corruption of the data stored on disk. This should help [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2998) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3011).
|
||||
* FEATURE: atomically delete directories with snapshots, parts and partitions at [storage level](https://docs.victoriametrics.com/#storage). Previously such directories can be left in partially deleted state when the deletion operation was interrupted by unclean shutdown. This may result in `cannot open file ...: no such file or directory` error on the next start. The probability of this error was quite high when NFS or EFS was used as persistent storage for VictoriaMetrics data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038).
|
||||
* FEATURE: set the `start` arg to `end - 5 minutes` if isn't passed explicitly to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3052).
|
||||
* FEATURE: allow to define the minimum TLS version to use when accepting https requests to VictoriaMetrics components if `-tls` command-line flag is set. The minimum TLS version can be set via `-tlsMinVersion` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3090).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `debug` mode to the alerting rule settings for printing additional information into logs during evaluation. See `debug` param in [alerting rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules).
|
||||
|
@ -38,7 +39,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
|||
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly calculate query results at `vmselect`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067). The issue has been introduced in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810).
|
||||
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): log clear error when multiple identical `-storageNode` command-line flags are passed to `vmselect` or to `vminsert`. Previously these components were crashed with cryptic panic `metric ... is already registered` in this case. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3076).
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `RangeError: Maximum call stack size exceeded` error when the query returns too many data points at `Table` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3092/files).
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119).
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119).
|
||||
* BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810) wasn't complete.
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval.
|
||||
|
||||
## [v1.81.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.2)
|
||||
|
|
|
@ -777,6 +777,8 @@ Below is the output for `/path/to/vminsert -help`:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-usePromCompatibleNaming
|
||||
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
|
||||
-version
|
||||
|
@ -981,6 +983,8 @@ Below is the output for `/path/to/vmselect -help`:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
-vmalert.proxyURL string
|
||||
|
@ -1141,6 +1145,8 @@ Below is the output for `/path/to/vmstorage -help`:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
-vminsertAddr string
|
||||
|
|
|
@ -806,7 +806,9 @@ sum by (__name__) (
|
|||
|
||||
#### label_value
|
||||
|
||||
`label_value(q, "label")` returns number values for the given `label` for every time series returned by `q`. For example, if `label_value(foo, "bar")` is applied to `foo{bar="1.234"}`, then it will return a time series `foo{bar="1.234"}` with `1.234` value.
|
||||
`label_value(q, "label")` returns numeric values for the given `label` for every time series returned by `q`.
|
||||
For example, if `label_value(foo, "bar")` is applied to `foo{bar="1.234"}`, then it will return a time series
|
||||
`foo{bar="1.234"}` with `1.234` value. Function will return no data for non-numeric label values.
|
||||
|
||||
### Aggregate functions
|
||||
|
||||
|
|
|
@ -2342,6 +2342,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-usePromCompatibleNaming
|
||||
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
|
||||
-version
|
||||
|
|
|
@ -16,7 +16,7 @@ git remote add enterprise <url>
|
|||
## Release version and Docker images
|
||||
|
||||
0. Make sure that the release commits have no security issues.
|
||||
1a. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
|
||||
1a. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) and update version if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/SECURITY.md)
|
||||
1b. Add `(available starting from v1.xx.y)` line to feature docs introduced in the upcoming release.
|
||||
2. Create the following release tags:
|
||||
* `git tag -s v1.xx.y` in `master` branch
|
||||
|
|
|
@ -2346,6 +2346,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-usePromCompatibleNaming
|
||||
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
|
||||
-version
|
||||
|
|
|
@ -94,7 +94,7 @@ Supported token types are `Read-Only`, `Write-Only` and `Read-Write`. Click on t
|
|||
to see usage examples:
|
||||
|
||||
<p>
|
||||
<img src="quickstart-tokens-usage.png" width="800">
|
||||
<img src="quickstart-token-usage.png" width="800">
|
||||
</p>
|
||||
|
||||
Follow usage example in order to configure access to VictoriaMetrics for your Prometheus,
|
||||
|
|
|
@ -1273,6 +1273,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-usePromCompatibleNaming
|
||||
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
|
||||
-version
|
||||
|
|
|
@ -1073,6 +1073,8 @@ The shortlist of configuration flags is the following:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -312,6 +312,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -270,6 +270,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -76,14 +76,14 @@ Backup manager launched with the following configuration:
|
|||
```console
|
||||
export NODE_IP=192.168.0.10
|
||||
export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
|
||||
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
|
||||
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
|
||||
```
|
||||
|
||||
Expected logs in vmbackupmanager:
|
||||
|
||||
```console
|
||||
info lib/backup/actions/backup.go:131 server-side copied 81 out of 81 parts from GCS{bucket: "vmstorage-data", dir: "192.168.0.10//latest/"} to GCS{bucket: "vmstorage-data", dir: "192.168.0.10//weekly/2020-34/"} in 2.549833008s
|
||||
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
|
||||
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
|
||||
```
|
||||
|
||||
Expected logs in vmstorage:
|
||||
|
@ -97,7 +97,7 @@ info VictoriaMetrics/lib/storage/storage.go:319 deleted snapshot "/vmstora
|
|||
The result on the GCS bucket
|
||||
|
||||
* The root folder
|
||||
|
||||
|
||||
![root](vmbackupmanager_root_folder.png)
|
||||
|
||||
* The latest folder
|
||||
|
@ -269,6 +269,8 @@ vmbackupmanager performs regular backups according to the provided configs.
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -153,7 +153,7 @@ cat << EOF > limit.yaml
|
|||
limits:
|
||||
- type: queries
|
||||
value: 100
|
||||
- type: rows_inserted
|
||||
- type: rows_inserted
|
||||
value: 100000
|
||||
- type: new_series
|
||||
value: 1000
|
||||
|
@ -172,7 +172,7 @@ curl 'http://localhost:8431/api/v1/import/prometheus' -X POST -d 'foo{bar="baz1
|
|||
# read metric from tenant 1:5
|
||||
curl 'http://localhost:8431/api/v1/labels' -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MjAxNjIwMDAwMDAsInZtX2FjY2VzcyI6eyJ0ZW5hbnRfaWQiOnsiYWNjb3VudF9pZCI6MTV9fX0.PB1_KXDKPUp-40pxOGk6lt_jt9Yq80PIMpWVJqSForQ'
|
||||
|
||||
# check rate limit
|
||||
# check rate limit
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
@ -203,11 +203,11 @@ The shortlist of configuration flags include the following:
|
|||
-datasource.maxIdleConnections int
|
||||
Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
|
||||
-datasource.oauth2.clientID string
|
||||
Optional OAuth2 clientID to use for -datasource.url.
|
||||
Optional OAuth2 clientID to use for -datasource.url.
|
||||
-datasource.oauth2.clientSecret string
|
||||
Optional OAuth2 clientSecret to use for -datasource.url.
|
||||
-datasource.oauth2.clientSecretFile string
|
||||
Optional OAuth2 clientSecretFile to use for -datasource.url.
|
||||
Optional OAuth2 clientSecretFile to use for -datasource.url.
|
||||
-datasource.oauth2.scopes string
|
||||
Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
|
||||
-datasource.oauth2.tokenUrl string
|
||||
|
@ -319,6 +319,8 @@ The shortlist of configuration flags include the following:
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
-write.url string
|
||||
|
|
|
@ -170,6 +170,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
|
|||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
-tlsKeyFile string
|
||||
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
|
||||
-tlsMinVersion string
|
||||
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
|
||||
-version
|
||||
Show VictoriaMetrics version
|
||||
```
|
||||
|
|
|
@ -35,6 +35,8 @@ var (
|
|||
tlsCertFile = flag.String("tlsCertFile", "", "Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated")
|
||||
tlsKeyFile = flag.String("tlsKeyFile", "", "Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated")
|
||||
tlsCipherSuites = flagutil.NewArray("tlsCipherSuites", "Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants")
|
||||
tlsMinVersion = flag.String("tlsMinVersion", "", "Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. "+
|
||||
"Supported values: TLS10, TLS11, TLS12, TLS13")
|
||||
|
||||
pathPrefix = flag.String("http.pathPrefix", "", "An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, "+
|
||||
"then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. "+
|
||||
|
@ -95,9 +97,9 @@ func Serve(addr string, rh RequestHandler) {
|
|||
logger.Infof("pprof handlers are exposed at %s://%s/debug/pprof/", scheme, hostAddr)
|
||||
var tlsConfig *tls.Config
|
||||
if *tlsEnable {
|
||||
tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsCipherSuites)
|
||||
tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot load TLS cert from -tlsCertFile=%q, -tlsKeyFile=%q: %s", *tlsCertFile, *tlsKeyFile, err)
|
||||
logger.Fatalf("cannot load TLS cert from -tlsCertFile=%q, -tlsKeyFile=%q, -tlsMinVersion=%q: %s", *tlsCertFile, *tlsKeyFile, *tlsMinVersion, err)
|
||||
}
|
||||
tlsConfig = tc
|
||||
}
|
||||
|
|
|
@ -709,7 +709,7 @@ func (tb *Table) mergeRawItemsBlocks(ibs []*inmemoryBlock, isFinal bool) {
|
|||
atomic.AddUint64(&tb.assistedMerges, 1)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
|
||||
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
|
||||
return
|
||||
}
|
||||
logger.Panicf("FATAL: cannot merge small parts: %s", err)
|
||||
|
@ -788,12 +788,14 @@ func (tb *Table) canBackgroundMerge() bool {
|
|||
return atomic.LoadUint32(tb.isReadOnly) == 0
|
||||
}
|
||||
|
||||
var errReadOnlyMode = fmt.Errorf("storage is in readonly mode")
|
||||
|
||||
func (tb *Table) mergeExistingParts(isFinal bool) error {
|
||||
if !tb.canBackgroundMerge() {
|
||||
// Do not perform background merge in read-only mode
|
||||
// in order to prevent from disk space shortage.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
|
||||
return nil
|
||||
return errReadOnlyMode
|
||||
}
|
||||
n := fs.MustGetFreeSpace(tb.path)
|
||||
// Divide free space by the max number of concurrent merges.
|
||||
|
@ -832,7 +834,7 @@ func (tb *Table) partMerger() error {
|
|||
// The merger has been stopped.
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, errNothingToMerge) {
|
||||
if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) {
|
||||
return err
|
||||
}
|
||||
if fasttime.UnixTimestamp()-lastMergeTime > 30 {
|
||||
|
|
|
@ -10,7 +10,7 @@ import (
|
|||
)
|
||||
|
||||
// GetServerTLSConfig returns TLS config for the server.
|
||||
func GetServerTLSConfig(tlsCertFile, tlsKeyFile string, tlsCipherSuites []string) (*tls.Config, error) {
|
||||
func GetServerTLSConfig(tlsCertFile, tlsKeyFile, tlsMinVersion string, tlsCipherSuites []string) (*tls.Config, error) {
|
||||
var certLock sync.Mutex
|
||||
var certDeadline uint64
|
||||
var cert *tls.Certificate
|
||||
|
@ -22,10 +22,15 @@ func GetServerTLSConfig(tlsCertFile, tlsKeyFile string, tlsCipherSuites []string
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot use TLS cipher suites from tlsCipherSuites=%q: %w", tlsCipherSuites, err)
|
||||
}
|
||||
minVersion, err := ParseTLSVersion(tlsMinVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannnot use TLS min version from tlsMinVersion=%q. Supported TLS versions (TLS10, TLS11, TLS12, TLS13): %w", tlsMinVersion, err)
|
||||
}
|
||||
cert = &c
|
||||
cfg := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
PreferServerCipherSuites: true,
|
||||
MinVersion: minVersion,
|
||||
// Do not set MaxVersion, since this has no sense from security PoV.
|
||||
// This can only result in lower security level if improperly set.
|
||||
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
certLock.Lock()
|
||||
defer certLock.Unlock()
|
||||
|
@ -63,3 +68,22 @@ func cipherSuitesFromNames(cipherSuiteNames []string) ([]uint16, error) {
|
|||
}
|
||||
return cipherSuites, nil
|
||||
}
|
||||
|
||||
// ParseTLSVersion returns tls version from the given string s.
|
||||
func ParseTLSVersion(s string) (uint16, error) {
|
||||
switch strings.ToUpper(s) {
|
||||
case "":
|
||||
// Special case - use default TLS version provided by tls package.
|
||||
return 0, nil
|
||||
case "TLS13":
|
||||
return tls.VersionTLS13, nil
|
||||
case "TLS12":
|
||||
return tls.VersionTLS12, nil
|
||||
case "TLS11":
|
||||
return tls.VersionTLS11, nil
|
||||
case "TLS10":
|
||||
return tls.VersionTLS10, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported TLS version %q", s)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package netutil
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
@ -76,3 +77,44 @@ func TestCipherSuitesFromNames(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTLSVersionSuccess(t *testing.T) {
|
||||
f := func(s string, want uint16) {
|
||||
t.Helper()
|
||||
got, err := ParseTLSVersion(s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for ParseTLSVersion(%q): %s", s, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("unexpected value got from ParseTLSVersion(%q); got %d; want %d", s, got, want)
|
||||
}
|
||||
}
|
||||
// lowercase tlsName
|
||||
f("tls10", tls.VersionTLS10)
|
||||
f("tls11", tls.VersionTLS11)
|
||||
f("tls12", tls.VersionTLS12)
|
||||
f("tls13", tls.VersionTLS13)
|
||||
// uppercase tlsName
|
||||
f("TLS10", tls.VersionTLS10)
|
||||
f("TLS11", tls.VersionTLS11)
|
||||
f("TLS12", tls.VersionTLS12)
|
||||
f("TLS13", tls.VersionTLS13)
|
||||
// empty tlsName
|
||||
f("", 0)
|
||||
}
|
||||
|
||||
func TestParseTLSVersionFailure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
_, err := ParseTLSVersion(s)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error for ParseTLSVersion(%q)", s)
|
||||
}
|
||||
}
|
||||
// incorrect tlsName
|
||||
f("123")
|
||||
// incorrect tlsName with correct prefix
|
||||
f("TLS1")
|
||||
// incorrect tls version in tlsName
|
||||
f("TLS14")
|
||||
}
|
||||
|
|
|
@ -15,6 +15,7 @@ import (
|
|||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/fasthttp"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"golang.org/x/oauth2"
|
||||
|
@ -79,6 +80,8 @@ type TLSConfig struct {
|
|||
ServerName string `yaml:"server_name,omitempty"`
|
||||
InsecureSkipVerify bool `yaml:"insecure_skip_verify,omitempty"`
|
||||
MinVersion string `yaml:"min_version,omitempty"`
|
||||
// Do not define MaxVersion field (max_version), since this has no sense from security PoV.
|
||||
// This can only result in lower security level if improperly set.
|
||||
}
|
||||
|
||||
// String returns human-readable representation of tc
|
||||
|
@ -399,6 +402,8 @@ func (ac *Config) NewTLSConfig() *tls.Config {
|
|||
tlsCfg.ServerName = ac.TLSServerName
|
||||
tlsCfg.InsecureSkipVerify = ac.TLSInsecureSkipVerify
|
||||
tlsCfg.MinVersion = ac.TLSMinVersion
|
||||
// Do not set tlsCfg.MaxVersion, since this has no sense from security PoV.
|
||||
// This can only result in lower security level if improperly set.
|
||||
return tlsCfg
|
||||
}
|
||||
|
||||
|
@ -713,27 +718,10 @@ func (tctx *tlsContext) initFromTLSConfig(baseDir string, tc *TLSConfig) error {
|
|||
return fmt.Errorf("cannot parse data from `ca_file` %q", tc.CAFile)
|
||||
}
|
||||
}
|
||||
if tc.MinVersion != "" {
|
||||
v, err := parseTLSVersion(tc.MinVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse `min_version`: %w", err)
|
||||
}
|
||||
tctx.minVersion = v
|
||||
v, err := netutil.ParseTLSVersion(tc.MinVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse `min_version`: %w", err)
|
||||
}
|
||||
tctx.minVersion = v
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseTLSVersion(s string) (uint16, error) {
|
||||
switch strings.ToUpper(s) {
|
||||
case "TLS13":
|
||||
return tls.VersionTLS13, nil
|
||||
case "TLS12":
|
||||
return tls.VersionTLS12, nil
|
||||
case "TLS11":
|
||||
return tls.VersionTLS11, nil
|
||||
case "TLS10":
|
||||
return tls.VersionTLS10, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported TLS version %q", s)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue