Merge branch 'public-single-node' into pmm-6401-read-prometheus-data-files

This commit is contained in:
Aliaksandr Valialkin 2022-09-26 18:03:30 +03:00
commit bd79706eb3
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
41 changed files with 805 additions and 47 deletions

View file

@ -2342,6 +2342,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-usePromCompatibleNaming
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-version

14
SECURITY.md Normal file
View file

@ -0,0 +1,14 @@
# Security Policy
## Supported Versions
| Version | Supported |
|---------|--------------------|
| 1.81.x | :white_check_mark: |
| 1.80.x | :x: |
| 1.79.x | :white_check_mark: |
| < 1.78 | :x: |
## Reporting a Vulnerability
Please report any security issues to security@victoriametrics.com

View file

@ -1269,6 +1269,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-usePromCompatibleNaming
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-version

View file

@ -1069,6 +1069,8 @@ The shortlist of configuration flags is the following:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -308,6 +308,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -266,6 +266,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -72,14 +72,14 @@ Backup manager launched with the following configuration:
```console
export NODE_IP=192.168.0.10
export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
```
Expected logs in vmbackupmanager:
```console
info lib/backup/actions/backup.go:131 server-side copied 81 out of 81 parts from GCS{bucket: "vmstorage-data", dir: "192.168.0.10//latest/"} to GCS{bucket: "vmstorage-data", dir: "192.168.0.10//weekly/2020-34/"} in 2.549833008s
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
```
Expected logs in vmstorage:
@ -93,7 +93,7 @@ info VictoriaMetrics/lib/storage/storage.go:319 deleted snapshot "/vmstora
The result on the GCS bucket
* The root folder
![root](vmbackupmanager_root_folder.png)
* The latest folder
@ -265,6 +265,8 @@ vmbackupmanager performs regular backups according to the provided configs.
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -149,7 +149,7 @@ cat << EOF > limit.yaml
limits:
- type: queries
value: 100
- type: rows_inserted
- type: rows_inserted
value: 100000
- type: new_series
value: 1000
@ -168,7 +168,7 @@ curl 'http://localhost:8431/api/v1/import/prometheus' -X POST -d 'foo{bar="baz1
# read metric from tenant 1:5
curl 'http://localhost:8431/api/v1/labels' -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MjAxNjIwMDAwMDAsInZtX2FjY2VzcyI6eyJ0ZW5hbnRfaWQiOnsiYWNjb3VudF9pZCI6MTV9fX0.PB1_KXDKPUp-40pxOGk6lt_jt9Yq80PIMpWVJqSForQ'
# check rate limit
# check rate limit
```
## Configuration
@ -199,11 +199,11 @@ The shortlist of configuration flags include the following:
-datasource.maxIdleConnections int
Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
-datasource.oauth2.clientID string
Optional OAuth2 clientID to use for -datasource.url.
Optional OAuth2 clientID to use for -datasource.url.
-datasource.oauth2.clientSecret string
Optional OAuth2 clientSecret to use for -datasource.url.
-datasource.oauth2.clientSecretFile string
Optional OAuth2 clientSecretFile to use for -datasource.url.
Optional OAuth2 clientSecretFile to use for -datasource.url.
-datasource.oauth2.scopes string
Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
-datasource.oauth2.tokenUrl string
@ -315,6 +315,8 @@ The shortlist of configuration flags include the following:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
-write.url string

View file

@ -166,6 +166,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -0,0 +1,12 @@
RELEASE_NAME := vm-vultr-server
VM_VERSION ?= $(shell git describe --abbrev=0 --tags)
PACKER_LOG := 1
PACKER_LOG_PATH := packer.log
.PHONY: $(MAKECMDGOALS)
release-victoria-metrics-vultr-server:
cp ./victoriametrics-single/etc/update-motd.d/99-one-click.tpl ./victoriametrics-single/etc/update-motd.d/99-one-click
sed -i -e "s/VM_VERSION/${VM_VERSION}/g" ./victoriametrics-single/etc/update-motd.d/99-one-click
packer init victoriametrics-single/victoriametrics-single.pkr.hcl
packer build victoriametrics-single/victoriametrics-single.pkr.hcl

View file

@ -0,0 +1,55 @@
## Application summary
VictoriaMetrics is a fast and scalable open source time series database and monitoring solution.
## Description
VictoriaMetrics is a free [open source time series database](https://en.wikipedia.org/wiki/Time_series_database) (TSDB) and monitoring solution, designed to collect, store and process real-time metrics.
It supports the [Prometheus](https://en.wikipedia.org/wiki/Prometheus_(software)) pull model and various push protocols ([Graphite](https://en.wikipedia.org/wiki/Graphite_(software)), [InfluxDB](https://en.wikipedia.org/wiki/InfluxDB), OpenTSDB) for data ingestion. It is optimized for storage with high-latency IO, low IOPS and time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
For reading the data and evaluating alerting rules, VictoriaMetrics supports the PromQL, [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) and Graphite query languages. VictoriaMetrics Single is fully autonomous and can be used as a long-term storage for time series.
[VictoriaMetrics Single](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) = Hassle-free monitoring solution. Easily handles 10M+ of active time series on a single instance. Perfect for small and medium environments.
## Getting started after deploying VictoriaMetrics Single
### Config
VictoriaMetrics configuration is located at `/etc/victoriametrics/single/scrape.yml` on the server.
This One Click app uses 8428, 2003, 4242 and 8089 ports to accept metrics from different protocols. It's recommended to disable ports for protocols which are not needed. [Ubuntu firewall](https://help.ubuntu.com/community/UFW) can be used to easily disable access for specific ports.
### Scraping metrics
VictoriaMetrics supports metrics scraping in the same way as Prometheus does. Check the configuration file to edit scraping targets. See more details about scraping at [How to scrape Prometheus exporters](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-scrape-prometheus-exporters-such-as-node-exporter).
### Sending metrics
Besides scraping, VictoriaMetrics accepts write requests for various ingestion protocols. This One Click app supports the following protocols:
- [Datadog](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent), [Influx (telegraph)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf), [JSON](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format), [CSV](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data), [Prometheus](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) on port :8428
- [Graphite (statsd)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) on port :2003 tcp/udp
- [OpenTSDB](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents) on port :4242
- Influx (telegraph) on port :8089 tcp/udp
See more details and examples in [official documentation](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html).
### UI
VictoriaMetrics provides a [User Interface (UI)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmui) for query troubleshooting and exploration. The UI is available at `http://your_server_public_ipv4:8428/vmui`. It lets users explore query results via graphs and tables.
To check it, open the following in your browser `http://your_server_public_ipv4:8428/vmui` and then enter `vm_app_uptime_seconds` to the Query Field to Execute the Query.
Run the following command to query and retrieve a result from VictoriaMetrics Single with `curl`:
```console
curl -sg http://your_server_public_ipv4:8428/api/v1/query_range?query=vm_app_uptime_seconds | jq
```
### Accessing
Once the **server** is created, you can use Vultr's web console to start a session or SSH directly to the server as root:
```console
ssh root@your_server_public_ipv4
```

View file

@ -0,0 +1,12 @@
## Release guide for Vultr Marketplace
### Build image
1. To build the snapshot in Vultr account you will need `VULTR_API_KEY` and [packer](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli).
2. `VULTR_API_KEY` can be generated on [https://my.vultr.com/settings/#settingsapi](https://my.vultr.com/settings/#settingsapi) or use already generated from OnePassword.
3. Choose prefered version of VictoriaMetrics on [Github releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page.
4. Set variables `VULTR_API_KEY` with `VM_VERSION` for `packer` environment and run make from example below:
```console
make release-victoria-metrics-vultr-server VULTR_API_KEY="your_token_here" VM_VERSION="prefered_release_version"
```

View file

@ -0,0 +1,317 @@
#!/bin/bash
# shopt -s inherit_errexit
set -o errexit
###################################################################
## Vultr Marketplace Helper Functions
function error_detect_on()
{
set -euo pipefail
}
function error_detect_off()
{
set +euo pipefail
}
function enable_verbose_commands()
{
set -x
}
function disable_verbose_commands()
{
set +x
}
function get_metadata_item()
{
local item_path="${1:-}"
local item_value
item_value="$(curl --fail --silent --header "Metadata-Token: vultr" "http://169.254.169.254/${item_path}")"
echo "${item_value}"
}
function get_hostname()
{
get_metadata_item "latest/meta-data/hostname"
}
function get_userdata()
{
get_metadata_item "latest/user-data"
}
function get_sshkeys()
{
get_metadata_item "current/ssh-keys"
}
function get_var()
{
local var_name="${1:-}"
local var_val
var_val="$(get_metadata_item "v1/internal/app-${var_name}" 2>/dev/null)"
eval "${var_name}='${var_val}'"
}
function get_ip()
{
local ip_var="${1:-}"
local ip_val
ip_val="$(get_metadata_item "latest/meta-data/public-ipv4" 2>/dev/null)"
eval "${ip_var}='${ip_val}'"
}
function wait_on_apt_lock()
{
until ! lsof -t /var/cache/apt/archives/lock /var/lib/apt/lists/lock /var/lib/dpkg/lock >/dev/null 2>&1
do
echo "Waiting 3 for apt lock currently held by another process."
sleep 3
done
}
function apt_safe()
{
wait_on_apt_lock
apt install -y "$@"
}
function apt_update_safe()
{
wait_on_apt_lock
apt update -y
}
function apt_upgrade_safe()
{
wait_on_apt_lock
DEBIAN_FRONTEND=noninteractive apt upgrade -y
}
function apt_remove_safe()
{
wait_on_apt_lock
apt remove -y --auto-remove "$@"
}
function apt_clean_safe()
{
wait_on_apt_lock
apt autoremove -y
wait_on_apt_lock
apt autoclean -y
}
function update_and_clean_packages()
{
# RHEL/CentOS
if [[ -f /etc/redhat-release ]]; then
yum update -y
yum clean all
# Ubuntu / Debian
elif grep -qs "debian" /etc/os-release 2>/dev/null; then
apt_update_safe
apt_upgrade_safe
apt_clean_safe
fi
}
function set_vultr_kernel_option()
{
# RHEL/CentOS
if [[ -f /etc/redhat-release ]]; then
/sbin/grubby --update-kernel=ALL --args vultr
# Ubuntu / Debian
elif grep -qs "debian" /etc/os-release 2>/dev/null; then
sed -i -e "/^GRUB_CMDLINE_LINUX_DEFAULT=/ s/\"$/ vultr\"/" /etc/default/grub
update-grub
fi
}
function install_cloud_init()
{
local cloud_init_exe
cloud_init_exe="$(command -v cloud-init >/dev/null 2>&1)"
if [[ -x "${cloud_init_exe}" ]]; then
echo "cloud-init is already installed."
return
fi
local release_version="${1:-"latest"}"
if [[ "${release_version}" != "latest" && "${release_version}" != "nightly" ]]; then
echo "${release_version} is an invalid release option. Allowed: latest, nightly"
exit 255
fi
# Lets remove all traces of previously installed cloud-init
# Ubuntu installs have proven problematic with their left over
# configs for the installer in recent versions
cleanup_cloudinit
update_and_clean_packages
local build_type
local package_ext
[[ -e /etc/os-release ]] && . /etc/os-release
case "${ID:-}" in
debian)
build_type="debian"
package_ext="deb"
;;
fedora)
build_type="rhel"
package_ext="rpm"
;;
ubuntu)
build_type="universal"
package_ext="deb"
;;
*)
case "${ID_LIKE:-}" in
*rhel*)
build_type="rhel"
package_ext="rpm"
;;
*)
echo "Unable to determine OS. Please install from source!"
exit 255
esac
esac
local cloud_init_package="cloud-init_${build_type}_${release_version}.${package_ext}"
wget -O "/tmp/${cloud_init_package}" "https://ewr1.vultrobjects.com/cloud_init_beta/${cloud_init_package}"
case "${package_ext}" in
rpm)
yum install -y "/tmp/${cloud_init_package}"
;;
deb)
apt_safe "/tmp/${cloud_init_package}"
;;
*)
echo "Unable to determine package installation method."
exit 255
esac
rm -f "/tmp/${cloud_init_package}"
}
function cleanup_cloudinit()
{
rm -rf \
/etc/cloud \
/etc/systemd/system/cloud-init.target.wants/* \
/lib/systemd/system/cloud* \
/run/cloud-init \
/usr/bin/cloud* \
/usr/lib/cloud* \
/usr/local/bin/cloud* \
/usr/src/cloud* \
/var/log/cloud*
}
function clean_tmp()
{
mkdir -p /tmp
chmod 1777 /tmp
rm -rf /tmp/* /var/tmp/*
}
function clean_keys()
{
rm -f /root/.ssh/authorized_keys /etc/ssh/*key*
touch /etc/ssh/revoked_keys
chmod 600 /etc/ssh/revoked_keys
}
function clean_logs()
{
find /var/log -mtime -1 -type f -exec truncate -s 0 {} \;
rm -rf \
/var/log/*.[0-9] \
/var/log/*.gz \
/var/log/*.log \
/var/log/lastlog \
/var/log/wtmp
: > /var/log/auth.log
}
function clean_history()
{
history -c
: > /root/.bash_history
unset HISTFILE
}
function clean_mloc()
{
/usr/bin/updatedb || true
}
function clean_random()
{
rm -f /var/lib/systemd/random-seed
}
function clean_machine_id()
{
[[ -e /etc/machine-id ]] && : > /etc/machine-id
[[ -e /var/lib/dbus/machine-id ]] && : > /var/lib/dbus/machine-id
}
function clean_free_space()
{
dd if=/dev/zero of=/zerofile || true
sync
rm -f /zerofile
sync
}
function trim_ssd()
{
fstrim / || true
}
function cleanup_marketplace_scripts()
{
rm -f /root/*.sh
}
function disable_network_manager()
{
## Disable NetworkManager, replace with network-scripts
systemctl disable --now NetworkManager
sed -i \
-e 's/^ONBOOT.*/ONBOOT=yes/g' \
-e 's/^NM_CONTROLLED.*/NM_CONTROLLED=no/g' /etc/sysconfig/network-scripts/ifcfg-*
yum install -y network-scripts
}
function clean_system()
{
update_and_clean_packages
set_vultr_kernel_option
clean_tmp
clean_keys
clean_logs
clean_history
clean_random
clean_machine_id
clean_mloc
clean_free_space
trim_ssd
cleanup_marketplace_scripts
}

View file

@ -0,0 +1,29 @@
[Unit]
Description=VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
# https://docs.victoriametrics.com
After=network.target
[Service]
Type=simple
User=victoriametrics
Group=victoriametrics
WorkingDirectory=/var/lib/victoria-metrics-data
StartLimitBurst=5
StartLimitInterval=0
Restart=on-failure
RestartSec=5
EnvironmentFile=-/etc/victoriametrics/single/victoriametrics.conf
ExecStart=/usr/bin/victoria-metrics-prod $ARGS
ExecStop=/bin/kill -s SIGTERM $MAINPID
ExecReload=/bin/kill -HUP $MAINPID
# See docs https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#tuning
ProtectSystem=full
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=vmsingle
[Install]
WantedBy=multi-user.target

View file

@ -0,0 +1,35 @@
#!/bin/sh
#
# Configured as part of the DigitalOcean 1-Click Image build process
myip=$(hostname -I | awk '{print$1}')
cat <<EOF
********************************************************************************
Welcome to VictoriaMetrics server!
To keep this server secure, the UFW firewall is enabled.
All ports are BLOCKED except 22 (SSH), 80 (HTTP), and 443 (HTTPS), 8428 (VictoriaMetrics HTTP), 8089 (VictoriaMetrics Influx),
4242 (VictoriaMetrics OpenTSDB), 2003 (VictoriaMetrics Graphite)
In a web browser, you can view:
* The VictoriaMetrics Quickstart guide: https://kutt.it/1click-quickstart
On the server:
* The default VictoriaMetrics root is located at /var/lib/victoria-metrics-data
* VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
********************************************************************************
# This image includes version VM_VERSION of VictoriaMetrics.
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/VM_VERSION
# Website: https://victoriametrics.com
# Documentation: https://docs.victoriametrics.com
# VictoriaMetrics Github : https://github.com/VictoriaMetrics/VictoriaMetrics
# VictoriaMetrics Slack Community: https://slack.victoriametrics.com
# VictoriaMetrics Telegram Community: https://t.me/VictoriaMetrics_en
# VictoriaMetrics in Twitter: https://twitter.com/VictoriaMetrics
# VictoriaMetrics config: /etc/victoriametrics/single/victoriametrics.conf
********************************************************************************
EOF

View file

@ -0,0 +1,35 @@
#!/bin/sh
#
# Configured as part of the DigitalOcean 1-Click Image build process
myip=$(hostname -I | awk '{print$1}')
cat <<EOF
********************************************************************************
Welcome to VictoriaMetrics server!
To keep this server secure, the UFW firewall is enabled.
All ports are BLOCKED except 22 (SSH), 80 (HTTP), and 443 (HTTPS), 8428 (VictoriaMetrics HTTP), 8089 (VictoriaMetrics Influx),
4242 (VictoriaMetrics OpenTSDB), 2003 (VictoriaMetrics Graphite)
In a web browser, you can view:
* The VictoriaMetrics Quickstart guide: https://kutt.it/1click-quickstart
On the server:
* The default VictoriaMetrics root is located at /var/lib/victoria-metrics-data
* VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
********************************************************************************
# This image includes version VM_VERSION of VictoriaMetrics.
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/VM_VERSION
# Website: https://victoriametrics.com
# Documentation: https://docs.victoriametrics.com
# VictoriaMetrics Github : https://github.com/VictoriaMetrics/VictoriaMetrics
# VictoriaMetrics Slack Community: https://slack.victoriametrics.com
# VictoriaMetrics Telegram Community: https://t.me/VictoriaMetrics_en
# VictoriaMetrics in Twitter: https://twitter.com/VictoriaMetrics
# VictoriaMetrics config: /etc/victoriametrics/single/victoriametrics.conf
********************************************************************************
EOF

View file

@ -0,0 +1,7 @@
# Scrape config example
#
scrape_configs:
- job_name: self_scrape
scrape_interval: 10s
static_configs:
- targets: ['127.0.0.1:8428']

View file

@ -0,0 +1 @@
ARGS="-promscrape.config=/etc/victoriametrics/single/scrape.yml -storageDataPath=/var/lib/victoria-metrics-data -retentionPeriod=12 -httpListenAddr=:8428 -graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -influxListenAddr=:8089 -enableTCP6"

View file

@ -0,0 +1,4 @@
#!/bin/bash
## Run on every boot.
echo $(date -u) ": System booted." >> /var/log/per-boot.log

View file

@ -0,0 +1,29 @@
#!/bin/bash
## Runs once-and-only-once at first boot per instance.
## Report the start time to a logfile.
echo $(date -u) ": System provisioning started." >> /var/log/per-instance.log
apt update
DEBIAN_FRONTEND=noninteractive apt -qq full-upgrade -y
DEBIAN_FRONTEND=noninteractive apt -qq install -y ufw wget software-properties-common ssh
# Configure UFW
sed -e 's|DEFAULT_FORWARD_POLICY=.*|DEFAULT_FORWARD_POLICY="ACCEPT"|g' \
-i /etc/default/ufw
ufw allow ssh comment "SSH port"
ufw allow http comment "HTTP port"
ufw allow https comment "HTTPS port"
ufw allow 8428 comment "VictoriaMetrics Single HTTP port"
ufw allow 8089/tcp comment "TCP Influx Listen port for VictoriaMetrics"
ufw allow 8089/udp comment "UDP Influx Listen port for VictoriaMetrics"
ufw allow 2003/tcp comment "TCP Graphite Listen port for VictoriaMetrics"
ufw allow 2003/udp comment "UDP Graphite Listen port for VictoriaMetrics"
ufw allow 4242 comment "OpenTSDB Listen port for VictoriaMetrics"
ufw --force enable
## Report the end time to a logfile.
echo $(date -u) ": System provisioning script is complete." >> /var/log/per-instance.log

View file

@ -0,0 +1,65 @@
variable "vultr_api_key" {
type = string
default = "${env("VULTR_API_KEY")}"
sensitive = true
}
variable "victoriametrics_version" {
type = string
default = "${env("VM_VERSION")}"
description = "Version number of the desired VictoriaMetrics binary."
}
packer {
required_plugins {
vultr = {
version = ">=v2.3.2"
source = "github.com/vultr/vultr"
}
}
}
source "vultr" "victoriametrics-single" {
api_key = "${var.vultr_api_key}"
os_id = "387"
plan_id = "vc2-1c-1gb"
region_id = "ewr"
snapshot_description = "victoriametrics-snapshot-${formatdate("YYYY-MM-DD hh:mm", timestamp())}"
ssh_username = "root"
state_timeout = "10m"
}
build {
sources = ["source.vultr.victoriametrics-single"]
provisioner "file" {
source = "helper-scripts/vultr-helper.sh"
destination = "/root/vultr-helper.sh"
}
provisioner "file" {
source = "victoriametrics-single/setup-per-boot.sh"
destination = "/root/setup-per-boot.sh"
}
# Copy configuration files
provisioner "file" {
destination = "/etc/"
source = "victoriametrics-single/etc/"
}
provisioner "file" {
source = "victoriametrics-single/setup-per-instance.sh"
destination = "/root/setup-per-instance.sh"
}
provisioner "shell" {
environment_vars = [
"VM_VERSION=${var.victoriametrics_version}",
"DEBIAN_FRONTEND=noninteractive"
]
script = "victoriametrics-single/victoriametrics-single.sh"
remote_folder = "/root"
remote_file = "victoriametrics-single.sh"
}
}

View file

@ -0,0 +1,41 @@
#!/bin/bash
################################################
## Prerequisites
chmod +x /root/vultr-helper.sh
. /root/vultr-helper.sh
error_detect_on
install_cloud_init latest
################################################
## Create victoriametrics user
groupadd -r victoriametrics
useradd -g victoriametrics -d /var/lib/victoria-metrics-data -s /sbin/nologin --system victoriametrics
mkdir -p /var/lib/victoria-metrics-data
chown -R victoriametrics:victoriametrics /var/lib/victoria-metrics-data
################################################
## Download VictoriaMetrics
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/victoria-metrics-linux-amd64-${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz
tar xvf /tmp/victoria-metrics.tar.gz -C /usr/bin
chmod +x /usr/bin/victoria-metrics-prod
chown root:root /usr/bin/victoria-metrics-prod
################################################
## Install provisioning scripts
mkdir -p /var/lib/cloud/scripts/per-boot/
mkdir -p /var/lib/cloud/scripts/per-instance/
mv /root/setup-per-boot.sh /var/lib/cloud/scripts/per-boot/setup-per-boot.sh
mv /root/setup-per-instance.sh /var/lib/cloud/scripts/per-instance/setup-per-instance.sh
chmod +x /var/lib/cloud/scripts/per-boot/setup-per-boot.sh
chmod +x /var/lib/cloud/scripts/per-instance/setup-per-instance.sh
# Enable VictoriaMetrics on boot
systemctl enable vmsingle.service
################################################
## Prepare server for Marketplace snapshot
clean_system

View file

@ -25,6 +25,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: check the correctess of raw sample timestamps stored on disk when reading them. This reduces the probability of possible silent corruption of the data stored on disk. This should help [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2998) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3011).
* FEATURE: atomically delete directories with snapshots, parts and partitions at [storage level](https://docs.victoriametrics.com/#storage). Previously such directories can be left in partially deleted state when the deletion operation was interrupted by unclean shutdown. This may result in `cannot open file ...: no such file or directory` error on the next start. The probability of this error was quite high when NFS or EFS was used as persistent storage for VictoriaMetrics data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3038).
* FEATURE: set the `start` arg to `end - 5 minutes` if isn't passed explicitly to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3052).
* FEATURE: allow to define the minimum TLS version to use when accepting https requests to VictoriaMetrics components if `-tls` command-line flag is set. The minimum TLS version can be set via `-tlsMinVersion` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3090).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `vm-native-step-interval` command line flag for `vm-native` mode. New option allows splitting the import process into chunks by time interval. This helps migrating data sets with high churn rate and provides better control over the process. See [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2733).
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add `top queries` tab, which shows various stats for recently executed queries. See [these docs](https://docs.victoriametrics.com/#top-queries) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2707).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add `debug` mode to the alerting rule settings for printing additional information into logs during evaluation. See `debug` param in [alerting rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules).
@ -38,7 +39,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly calculate query results at `vmselect`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067). The issue has been introduced in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810).
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): log clear error when multiple identical `-storageNode` command-line flags are passed to `vmselect` or to `vminsert`. Previously these components were crashed with cryptic panic `metric ... is already registered` in this case. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3076).
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `RangeError: Maximum call stack size exceeded` error when the query returns too many data points at `Table` view. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3092/files).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119).
* BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810) wasn't complete.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval.
## [v1.81.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.2)

View file

@ -777,6 +777,8 @@ Below is the output for `/path/to/vminsert -help`:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-usePromCompatibleNaming
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-version
@ -981,6 +983,8 @@ Below is the output for `/path/to/vmselect -help`:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
-vmalert.proxyURL string
@ -1141,6 +1145,8 @@ Below is the output for `/path/to/vmstorage -help`:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
-vminsertAddr string

View file

@ -806,7 +806,9 @@ sum by (__name__) (
#### label_value
`label_value(q, "label")` returns number values for the given `label` for every time series returned by `q`. For example, if `label_value(foo, "bar")` is applied to `foo{bar="1.234"}`, then it will return a time series `foo{bar="1.234"}` with `1.234` value.
`label_value(q, "label")` returns numeric values for the given `label` for every time series returned by `q`.
For example, if `label_value(foo, "bar")` is applied to `foo{bar="1.234"}`, then it will return a time series
`foo{bar="1.234"}` with `1.234` value. Function will return no data for non-numeric label values.
### Aggregate functions

View file

@ -2342,6 +2342,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-usePromCompatibleNaming
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-version

View file

@ -16,7 +16,7 @@ git remote add enterprise <url>
## Release version and Docker images
0. Make sure that the release commits have no security issues.
1a. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md).
1a. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) and update version if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/SECURITY.md)
1b. Add `(available starting from v1.xx.y)` line to feature docs introduced in the upcoming release.
2. Create the following release tags:
* `git tag -s v1.xx.y` in `master` branch

View file

@ -2346,6 +2346,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-usePromCompatibleNaming
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-version

View file

@ -94,7 +94,7 @@ Supported token types are `Read-Only`, `Write-Only` and `Read-Write`. Click on t
to see usage examples:
<p>
<img src="quickstart-tokens-usage.png" width="800">
<img src="quickstart-token-usage.png" width="800">
</p>
Follow usage example in order to configure access to VictoriaMetrics for your Prometheus,

View file

@ -1273,6 +1273,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-usePromCompatibleNaming
Whether to replace characters unsupported by Prometheus with underscores in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
-version

View file

@ -1073,6 +1073,8 @@ The shortlist of configuration flags is the following:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -312,6 +312,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -270,6 +270,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -76,14 +76,14 @@ Backup manager launched with the following configuration:
```console
export NODE_IP=192.168.0.10
export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula
```
Expected logs in vmbackupmanager:
```console
info lib/backup/actions/backup.go:131 server-side copied 81 out of 81 parts from GCS{bucket: "vmstorage-data", dir: "192.168.0.10//latest/"} to GCS{bucket: "vmstorage-data", dir: "192.168.0.10//weekly/2020-34/"} in 2.549833008s
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
info lib/backup/actions/backup.go:169 backed up 853315 bytes in 2.882 seconds; deleted 0 bytes; server-side copied 853315 bytes; uploaded 0 bytes
```
Expected logs in vmstorage:
@ -97,7 +97,7 @@ info VictoriaMetrics/lib/storage/storage.go:319 deleted snapshot "/vmstora
The result on the GCS bucket
* The root folder
![root](vmbackupmanager_root_folder.png)
* The latest folder
@ -269,6 +269,8 @@ vmbackupmanager performs regular backups according to the provided configs.
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -153,7 +153,7 @@ cat << EOF > limit.yaml
limits:
- type: queries
value: 100
- type: rows_inserted
- type: rows_inserted
value: 100000
- type: new_series
value: 1000
@ -172,7 +172,7 @@ curl 'http://localhost:8431/api/v1/import/prometheus' -X POST -d 'foo{bar="baz1
# read metric from tenant 1:5
curl 'http://localhost:8431/api/v1/labels' -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2MjAxNjIwMDAwMDAsInZtX2FjY2VzcyI6eyJ0ZW5hbnRfaWQiOnsiYWNjb3VudF9pZCI6MTV9fX0.PB1_KXDKPUp-40pxOGk6lt_jt9Yq80PIMpWVJqSForQ'
# check rate limit
# check rate limit
```
## Configuration
@ -203,11 +203,11 @@ The shortlist of configuration flags include the following:
-datasource.maxIdleConnections int
Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
-datasource.oauth2.clientID string
Optional OAuth2 clientID to use for -datasource.url.
Optional OAuth2 clientID to use for -datasource.url.
-datasource.oauth2.clientSecret string
Optional OAuth2 clientSecret to use for -datasource.url.
-datasource.oauth2.clientSecretFile string
Optional OAuth2 clientSecretFile to use for -datasource.url.
Optional OAuth2 clientSecretFile to use for -datasource.url.
-datasource.oauth2.scopes string
Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'
-datasource.oauth2.tokenUrl string
@ -319,6 +319,8 @@ The shortlist of configuration flags include the following:
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
-write.url string

View file

@ -170,6 +170,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
Supports an array of values separated by comma or specified via multiple flags.
-tlsKeyFile string
Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated
-tlsMinVersion string
Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13
-version
Show VictoriaMetrics version
```

View file

@ -35,6 +35,8 @@ var (
tlsCertFile = flag.String("tlsCertFile", "", "Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated")
tlsKeyFile = flag.String("tlsKeyFile", "", "Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated")
tlsCipherSuites = flagutil.NewArray("tlsCipherSuites", "Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants")
tlsMinVersion = flag.String("tlsMinVersion", "", "Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. "+
"Supported values: TLS10, TLS11, TLS12, TLS13")
pathPrefix = flag.String("http.pathPrefix", "", "An optional prefix to add to all the paths handled by http server. For example, if '-http.pathPrefix=/foo/bar' is set, "+
"then all the http requests will be handled on '/foo/bar/*' paths. This may be useful for proxied requests. "+
@ -95,9 +97,9 @@ func Serve(addr string, rh RequestHandler) {
logger.Infof("pprof handlers are exposed at %s://%s/debug/pprof/", scheme, hostAddr)
var tlsConfig *tls.Config
if *tlsEnable {
tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsCipherSuites)
tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites)
if err != nil {
logger.Fatalf("cannot load TLS cert from -tlsCertFile=%q, -tlsKeyFile=%q: %s", *tlsCertFile, *tlsKeyFile, err)
logger.Fatalf("cannot load TLS cert from -tlsCertFile=%q, -tlsKeyFile=%q, -tlsMinVersion=%q: %s", *tlsCertFile, *tlsKeyFile, *tlsMinVersion, err)
}
tlsConfig = tc
}

View file

@ -709,7 +709,7 @@ func (tb *Table) mergeRawItemsBlocks(ibs []*inmemoryBlock, isFinal bool) {
atomic.AddUint64(&tb.assistedMerges, 1)
continue
}
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return
}
logger.Panicf("FATAL: cannot merge small parts: %s", err)
@ -788,12 +788,14 @@ func (tb *Table) canBackgroundMerge() bool {
return atomic.LoadUint32(tb.isReadOnly) == 0
}
var errReadOnlyMode = fmt.Errorf("storage is in readonly mode")
func (tb *Table) mergeExistingParts(isFinal bool) error {
if !tb.canBackgroundMerge() {
// Do not perform background merge in read-only mode
// in order to prevent from disk space shortage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
return nil
return errReadOnlyMode
}
n := fs.MustGetFreeSpace(tb.path)
// Divide free space by the max number of concurrent merges.
@ -832,7 +834,7 @@ func (tb *Table) partMerger() error {
// The merger has been stopped.
return nil
}
if !errors.Is(err, errNothingToMerge) {
if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) {
return err
}
if fasttime.UnixTimestamp()-lastMergeTime > 30 {

View file

@ -10,7 +10,7 @@ import (
)
// GetServerTLSConfig returns TLS config for the server.
func GetServerTLSConfig(tlsCertFile, tlsKeyFile string, tlsCipherSuites []string) (*tls.Config, error) {
func GetServerTLSConfig(tlsCertFile, tlsKeyFile, tlsMinVersion string, tlsCipherSuites []string) (*tls.Config, error) {
var certLock sync.Mutex
var certDeadline uint64
var cert *tls.Certificate
@ -22,10 +22,15 @@ func GetServerTLSConfig(tlsCertFile, tlsKeyFile string, tlsCipherSuites []string
if err != nil {
return nil, fmt.Errorf("cannot use TLS cipher suites from tlsCipherSuites=%q: %w", tlsCipherSuites, err)
}
minVersion, err := ParseTLSVersion(tlsMinVersion)
if err != nil {
return nil, fmt.Errorf("cannnot use TLS min version from tlsMinVersion=%q. Supported TLS versions (TLS10, TLS11, TLS12, TLS13): %w", tlsMinVersion, err)
}
cert = &c
cfg := &tls.Config{
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
MinVersion: minVersion,
// Do not set MaxVersion, since this has no sense from security PoV.
// This can only result in lower security level if improperly set.
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
certLock.Lock()
defer certLock.Unlock()
@ -63,3 +68,22 @@ func cipherSuitesFromNames(cipherSuiteNames []string) ([]uint16, error) {
}
return cipherSuites, nil
}
// ParseTLSVersion returns tls version from the given string s.
func ParseTLSVersion(s string) (uint16, error) {
switch strings.ToUpper(s) {
case "":
// Special case - use default TLS version provided by tls package.
return 0, nil
case "TLS13":
return tls.VersionTLS13, nil
case "TLS12":
return tls.VersionTLS12, nil
case "TLS11":
return tls.VersionTLS11, nil
case "TLS10":
return tls.VersionTLS10, nil
default:
return 0, fmt.Errorf("unsupported TLS version %q", s)
}
}

View file

@ -1,6 +1,7 @@
package netutil
import (
"crypto/tls"
"reflect"
"testing"
)
@ -76,3 +77,44 @@ func TestCipherSuitesFromNames(t *testing.T) {
})
}
}
func TestParseTLSVersionSuccess(t *testing.T) {
f := func(s string, want uint16) {
t.Helper()
got, err := ParseTLSVersion(s)
if err != nil {
t.Fatalf("unexpected error for ParseTLSVersion(%q): %s", s, err)
}
if got != want {
t.Fatalf("unexpected value got from ParseTLSVersion(%q); got %d; want %d", s, got, want)
}
}
// lowercase tlsName
f("tls10", tls.VersionTLS10)
f("tls11", tls.VersionTLS11)
f("tls12", tls.VersionTLS12)
f("tls13", tls.VersionTLS13)
// uppercase tlsName
f("TLS10", tls.VersionTLS10)
f("TLS11", tls.VersionTLS11)
f("TLS12", tls.VersionTLS12)
f("TLS13", tls.VersionTLS13)
// empty tlsName
f("", 0)
}
func TestParseTLSVersionFailure(t *testing.T) {
f := func(s string) {
t.Helper()
_, err := ParseTLSVersion(s)
if err == nil {
t.Fatalf("expecting non-nil error for ParseTLSVersion(%q)", s)
}
}
// incorrect tlsName
f("123")
// incorrect tlsName with correct prefix
f("TLS1")
// incorrect tls version in tlsName
f("TLS14")
}

View file

@ -15,6 +15,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
"github.com/VictoriaMetrics/fasthttp"
"github.com/cespare/xxhash/v2"
"golang.org/x/oauth2"
@ -79,6 +80,8 @@ type TLSConfig struct {
ServerName string `yaml:"server_name,omitempty"`
InsecureSkipVerify bool `yaml:"insecure_skip_verify,omitempty"`
MinVersion string `yaml:"min_version,omitempty"`
// Do not define MaxVersion field (max_version), since this has no sense from security PoV.
// This can only result in lower security level if improperly set.
}
// String returns human-readable representation of tc
@ -399,6 +402,8 @@ func (ac *Config) NewTLSConfig() *tls.Config {
tlsCfg.ServerName = ac.TLSServerName
tlsCfg.InsecureSkipVerify = ac.TLSInsecureSkipVerify
tlsCfg.MinVersion = ac.TLSMinVersion
// Do not set tlsCfg.MaxVersion, since this has no sense from security PoV.
// This can only result in lower security level if improperly set.
return tlsCfg
}
@ -713,27 +718,10 @@ func (tctx *tlsContext) initFromTLSConfig(baseDir string, tc *TLSConfig) error {
return fmt.Errorf("cannot parse data from `ca_file` %q", tc.CAFile)
}
}
if tc.MinVersion != "" {
v, err := parseTLSVersion(tc.MinVersion)
if err != nil {
return fmt.Errorf("cannot parse `min_version`: %w", err)
}
tctx.minVersion = v
v, err := netutil.ParseTLSVersion(tc.MinVersion)
if err != nil {
return fmt.Errorf("cannot parse `min_version`: %w", err)
}
tctx.minVersion = v
return nil
}
func parseTLSVersion(s string) (uint16, error) {
switch strings.ToUpper(s) {
case "TLS13":
return tls.VersionTLS13, nil
case "TLS12":
return tls.VersionTLS12, nil
case "TLS11":
return tls.VersionTLS11, nil
case "TLS10":
return tls.VersionTLS10, nil
default:
return 0, fmt.Errorf("unsupported TLS version %q", s)
}
}