From 32be84fc759e7b2ca705d0456c5bd21e76309e17 Mon Sep 17 00:00:00 2001 From: Denys Holius <5650611+denisgolius@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:44:36 +0300 Subject: [PATCH] Adds packer build for server with VM Single node in vultr.com marketplace (#3142) * adds packer build for server with VM Single node in vultr.com marketplace * fix missed varibale --- deployment/marketplace/vultr/Makefile | 12 + deployment/marketplace/vultr/README.md | 55 +++ deployment/marketplace/vultr/RELEASE_GUIDE.md | 12 + .../vultr/helper-scripts/vultr-helper.sh | 317 ++++++++++++++++++ .../etc/systemd/system/vmsingle.service | 29 ++ .../etc/update-motd.d/99-one-click | 35 ++ .../etc/update-motd.d/99-one-click.tpl | 35 ++ .../etc/victoriametrics/single/scrape.yml | 7 + .../single/victoriametrics.conf | 1 + .../victoriametrics-single/setup-per-boot.sh | 4 + .../setup-per-instance.sh | 29 ++ .../victoriametrics-single.pkr.hcl | 65 ++++ .../victoriametrics-single.sh | 41 +++ 13 files changed, 642 insertions(+) create mode 100644 deployment/marketplace/vultr/Makefile create mode 100644 deployment/marketplace/vultr/README.md create mode 100644 deployment/marketplace/vultr/RELEASE_GUIDE.md create mode 100755 deployment/marketplace/vultr/helper-scripts/vultr-helper.sh create mode 100644 deployment/marketplace/vultr/victoriametrics-single/etc/systemd/system/vmsingle.service create mode 100644 deployment/marketplace/vultr/victoriametrics-single/etc/update-motd.d/99-one-click create mode 100644 deployment/marketplace/vultr/victoriametrics-single/etc/update-motd.d/99-one-click.tpl create mode 100644 deployment/marketplace/vultr/victoriametrics-single/etc/victoriametrics/single/scrape.yml create mode 100644 deployment/marketplace/vultr/victoriametrics-single/etc/victoriametrics/single/victoriametrics.conf create mode 100755 deployment/marketplace/vultr/victoriametrics-single/setup-per-boot.sh create mode 100755 deployment/marketplace/vultr/victoriametrics-single/setup-per-instance.sh create mode 100644 deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.pkr.hcl create mode 100644 deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh diff --git a/deployment/marketplace/vultr/Makefile b/deployment/marketplace/vultr/Makefile new file mode 100644 index 000000000..9849619d1 --- /dev/null +++ b/deployment/marketplace/vultr/Makefile @@ -0,0 +1,12 @@ +RELEASE_NAME := vm-vultr-server +VM_VERSION ?= $(shell git describe --abbrev=0 --tags) +PACKER_LOG := 1 +PACKER_LOG_PATH := packer.log + +.PHONY: $(MAKECMDGOALS) + +release-victoria-metrics-vultr-server: + cp ./victoriametrics-single/etc/update-motd.d/99-one-click.tpl ./victoriametrics-single/etc/update-motd.d/99-one-click + sed -i -e "s/VM_VERSION/${VM_VERSION}/g" ./victoriametrics-single/etc/update-motd.d/99-one-click + packer init victoriametrics-single/victoriametrics-single.pkr.hcl + packer build victoriametrics-single/victoriametrics-single.pkr.hcl \ No newline at end of file diff --git a/deployment/marketplace/vultr/README.md b/deployment/marketplace/vultr/README.md new file mode 100644 index 000000000..db2f39fb0 --- /dev/null +++ b/deployment/marketplace/vultr/README.md @@ -0,0 +1,55 @@ +## Application summary + +VictoriaMetrics is a fast and scalable open source time series database and monitoring solution. + +## Description + +VictoriaMetrics is a free [open source time series database](https://en.wikipedia.org/wiki/Time_series_database) (TSDB) and monitoring solution, designed to collect, store and process real-time metrics. + +It supports the [Prometheus](https://en.wikipedia.org/wiki/Prometheus_(software)) pull model and various push protocols ([Graphite](https://en.wikipedia.org/wiki/Graphite_(software)), [InfluxDB](https://en.wikipedia.org/wiki/InfluxDB), OpenTSDB) for data ingestion. It is optimized for storage with high-latency IO, low IOPS and time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). + +For reading the data and evaluating alerting rules, VictoriaMetrics supports the PromQL, [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) and Graphite query languages. VictoriaMetrics Single is fully autonomous and can be used as a long-term storage for time series. + +[VictoriaMetrics Single](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) = Hassle-free monitoring solution. Easily handles 10M+ of active time series on a single instance. Perfect for small and medium environments. + +## Getting started after deploying VictoriaMetrics Single + +### Config + +VictoriaMetrics configuration is located at `/etc/victoriametrics/single/scrape.yml` on the server. +This One Click app uses 8428, 2003, 4242 and 8089 ports to accept metrics from different protocols. It's recommended to disable ports for protocols which are not needed. [Ubuntu firewall](https://help.ubuntu.com/community/UFW) can be used to easily disable access for specific ports. + +### Scraping metrics + +VictoriaMetrics supports metrics scraping in the same way as Prometheus does. Check the configuration file to edit scraping targets. See more details about scraping at [How to scrape Prometheus exporters](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-scrape-prometheus-exporters-such-as-node-exporter). + +### Sending metrics + +Besides scraping, VictoriaMetrics accepts write requests for various ingestion protocols. This One Click app supports the following protocols: + +- [Datadog](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent), [Influx (telegraph)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf), [JSON](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format), [CSV](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data), [Prometheus](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) on port :8428 +- [Graphite (statsd)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) on port :2003 tcp/udp +- [OpenTSDB](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents) on port :4242 +- Influx (telegraph) on port :8089 tcp/udp + +See more details and examples in [official documentation](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html). + +### UI + +VictoriaMetrics provides a [User Interface (UI)](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmui) for query troubleshooting and exploration. The UI is available at `http://your_server_public_ipv4:8428/vmui`. It lets users explore query results via graphs and tables. + +To check it, open the following in your browser `http://your_server_public_ipv4:8428/vmui` and then enter `vm_app_uptime_seconds` to the Query Field to Execute the Query. + +Run the following command to query and retrieve a result from VictoriaMetrics Single with `curl`: + +```console +curl -sg http://your_server_public_ipv4:8428/api/v1/query_range?query=vm_app_uptime_seconds | jq +``` + +### Accessing + +Once the **server** is created, you can use Vultr's web console to start a session or SSH directly to the server as root: + +```console +ssh root@your_server_public_ipv4 +``` diff --git a/deployment/marketplace/vultr/RELEASE_GUIDE.md b/deployment/marketplace/vultr/RELEASE_GUIDE.md new file mode 100644 index 000000000..f68c882b5 --- /dev/null +++ b/deployment/marketplace/vultr/RELEASE_GUIDE.md @@ -0,0 +1,12 @@ +## Release guide for Vultr Marketplace + +### Build image + +1. To build the snapshot in Vultr account you will need `VULTR_API_KEY` and [packer](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli). +2. `VULTR_API_KEY` can be generated on [https://my.vultr.com/settings/#settingsapi](https://my.vultr.com/settings/#settingsapi) or use already generated from OnePassword. +3. Choose prefered version of VictoriaMetrics on [Github releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) page. +4. Set variables `VULTR_API_KEY` with `VM_VERSION` for `packer` environment and run make from example below: + +```console +make release-victoria-metrics-vultr-server VULTR_API_KEY="your_token_here" VM_VERSION="prefered_release_version" +``` diff --git a/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh b/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh new file mode 100755 index 000000000..97607b141 --- /dev/null +++ b/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh @@ -0,0 +1,317 @@ +#!/bin/bash + +# shopt -s inherit_errexit +set -o errexit + +################################################################### +## Vultr Marketplace Helper Functions + +function error_detect_on() +{ + set -euo pipefail +} + +function error_detect_off() +{ + set +euo pipefail +} + +function enable_verbose_commands() +{ + set -x +} + +function disable_verbose_commands() +{ + set +x +} + +function get_metadata_item() +{ + local item_path="${1:-}" + local item_value + + item_value="$(curl --fail --silent --header "Metadata-Token: vultr" "http://169.254.169.254/${item_path}")" + + echo "${item_value}" +} + +function get_hostname() +{ + get_metadata_item "latest/meta-data/hostname" +} + +function get_userdata() +{ + get_metadata_item "latest/user-data" +} + +function get_sshkeys() +{ + get_metadata_item "current/ssh-keys" +} + +function get_var() +{ + local var_name="${1:-}" + local var_val + var_val="$(get_metadata_item "v1/internal/app-${var_name}" 2>/dev/null)" + + eval "${var_name}='${var_val}'" +} + +function get_ip() +{ + local ip_var="${1:-}" + local ip_val + ip_val="$(get_metadata_item "latest/meta-data/public-ipv4" 2>/dev/null)" + + eval "${ip_var}='${ip_val}'" +} + +function wait_on_apt_lock() +{ + until ! lsof -t /var/cache/apt/archives/lock /var/lib/apt/lists/lock /var/lib/dpkg/lock >/dev/null 2>&1 + do + echo "Waiting 3 for apt lock currently held by another process." + sleep 3 + done +} + +function apt_safe() +{ + wait_on_apt_lock + apt install -y "$@" +} + +function apt_update_safe() +{ + wait_on_apt_lock + apt update -y +} + +function apt_upgrade_safe() +{ + wait_on_apt_lock + DEBIAN_FRONTEND=noninteractive apt upgrade -y +} + +function apt_remove_safe() +{ + wait_on_apt_lock + apt remove -y --auto-remove "$@" +} + +function apt_clean_safe() +{ + wait_on_apt_lock + apt autoremove -y + + wait_on_apt_lock + apt autoclean -y +} + +function update_and_clean_packages() +{ + # RHEL/CentOS + if [[ -f /etc/redhat-release ]]; then + yum update -y + yum clean all + # Ubuntu / Debian + elif grep -qs "debian" /etc/os-release 2>/dev/null; then + apt_update_safe + apt_upgrade_safe + apt_clean_safe + fi +} + +function set_vultr_kernel_option() +{ + # RHEL/CentOS + if [[ -f /etc/redhat-release ]]; then + /sbin/grubby --update-kernel=ALL --args vultr + # Ubuntu / Debian + elif grep -qs "debian" /etc/os-release 2>/dev/null; then + sed -i -e "/^GRUB_CMDLINE_LINUX_DEFAULT=/ s/\"$/ vultr\"/" /etc/default/grub + update-grub + fi +} + +function install_cloud_init() +{ + local cloud_init_exe + cloud_init_exe="$(command -v cloud-init >/dev/null 2>&1)" + if [[ -x "${cloud_init_exe}" ]]; then + echo "cloud-init is already installed." + return + fi + + local release_version="${1:-"latest"}" + if [[ "${release_version}" != "latest" && "${release_version}" != "nightly" ]]; then + echo "${release_version} is an invalid release option. Allowed: latest, nightly" + exit 255 + fi + + # Lets remove all traces of previously installed cloud-init + # Ubuntu installs have proven problematic with their left over + # configs for the installer in recent versions + cleanup_cloudinit + + update_and_clean_packages + + local build_type + local package_ext + + [[ -e /etc/os-release ]] && . /etc/os-release + case "${ID:-}" in + debian) + build_type="debian" + package_ext="deb" + ;; + fedora) + build_type="rhel" + package_ext="rpm" + ;; + ubuntu) + build_type="universal" + package_ext="deb" + ;; + *) + case "${ID_LIKE:-}" in + *rhel*) + build_type="rhel" + package_ext="rpm" + ;; + *) + echo "Unable to determine OS. Please install from source!" + exit 255 + esac + esac + + local cloud_init_package="cloud-init_${build_type}_${release_version}.${package_ext}" + wget -O "/tmp/${cloud_init_package}" "https://ewr1.vultrobjects.com/cloud_init_beta/${cloud_init_package}" + + case "${package_ext}" in + rpm) + yum install -y "/tmp/${cloud_init_package}" + ;; + deb) + apt_safe "/tmp/${cloud_init_package}" + ;; + *) + echo "Unable to determine package installation method." + exit 255 + esac + + rm -f "/tmp/${cloud_init_package}" +} + +function cleanup_cloudinit() +{ + rm -rf \ + /etc/cloud \ + /etc/systemd/system/cloud-init.target.wants/* \ + /lib/systemd/system/cloud* \ + /run/cloud-init \ + /usr/bin/cloud* \ + /usr/lib/cloud* \ + /usr/local/bin/cloud* \ + /usr/src/cloud* \ + /var/log/cloud* +} + +function clean_tmp() +{ + mkdir -p /tmp + chmod 1777 /tmp + rm -rf /tmp/* /var/tmp/* +} + +function clean_keys() +{ + rm -f /root/.ssh/authorized_keys /etc/ssh/*key* + touch /etc/ssh/revoked_keys + chmod 600 /etc/ssh/revoked_keys +} + +function clean_logs() +{ + find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; + rm -rf \ + /var/log/*.[0-9] \ + /var/log/*.gz \ + /var/log/*.log \ + /var/log/lastlog \ + /var/log/wtmp + + : > /var/log/auth.log +} + +function clean_history() +{ + history -c + : > /root/.bash_history + unset HISTFILE +} + +function clean_mloc() +{ + /usr/bin/updatedb || true +} + +function clean_random() +{ + rm -f /var/lib/systemd/random-seed +} + +function clean_machine_id() +{ + [[ -e /etc/machine-id ]] && : > /etc/machine-id + [[ -e /var/lib/dbus/machine-id ]] && : > /var/lib/dbus/machine-id +} + +function clean_free_space() +{ + dd if=/dev/zero of=/zerofile || true + sync + rm -f /zerofile + sync +} + +function trim_ssd() +{ + fstrim / || true +} + +function cleanup_marketplace_scripts() +{ + rm -f /root/*.sh +} + +function disable_network_manager() +{ + ## Disable NetworkManager, replace with network-scripts + systemctl disable --now NetworkManager + sed -i \ + -e 's/^ONBOOT.*/ONBOOT=yes/g' \ + -e 's/^NM_CONTROLLED.*/NM_CONTROLLED=no/g' /etc/sysconfig/network-scripts/ifcfg-* + yum install -y network-scripts +} + +function clean_system() +{ + + update_and_clean_packages + set_vultr_kernel_option + clean_tmp + clean_keys + clean_logs + clean_history + clean_random + clean_machine_id + clean_mloc + clean_free_space + trim_ssd + + cleanup_marketplace_scripts +} diff --git a/deployment/marketplace/vultr/victoriametrics-single/etc/systemd/system/vmsingle.service b/deployment/marketplace/vultr/victoriametrics-single/etc/systemd/system/vmsingle.service new file mode 100644 index 000000000..601e72386 --- /dev/null +++ b/deployment/marketplace/vultr/victoriametrics-single/etc/systemd/system/vmsingle.service @@ -0,0 +1,29 @@ +[Unit] +Description=VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database. +# https://docs.victoriametrics.com +After=network.target + +[Service] +Type=simple +User=victoriametrics +Group=victoriametrics +WorkingDirectory=/var/lib/victoria-metrics-data +StartLimitBurst=5 +StartLimitInterval=0 +Restart=on-failure +RestartSec=5 +EnvironmentFile=-/etc/victoriametrics/single/victoriametrics.conf +ExecStart=/usr/bin/victoria-metrics-prod $ARGS +ExecStop=/bin/kill -s SIGTERM $MAINPID +ExecReload=/bin/kill -HUP $MAINPID +# See docs https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#tuning +ProtectSystem=full +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=vmsingle + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deployment/marketplace/vultr/victoriametrics-single/etc/update-motd.d/99-one-click b/deployment/marketplace/vultr/victoriametrics-single/etc/update-motd.d/99-one-click new file mode 100644 index 000000000..cc9609012 --- /dev/null +++ b/deployment/marketplace/vultr/victoriametrics-single/etc/update-motd.d/99-one-click @@ -0,0 +1,35 @@ +#!/bin/sh +# +# Configured as part of the DigitalOcean 1-Click Image build process + +myip=$(hostname -I | awk '{print$1}') +cat <> /var/log/per-boot.log \ No newline at end of file diff --git a/deployment/marketplace/vultr/victoriametrics-single/setup-per-instance.sh b/deployment/marketplace/vultr/victoriametrics-single/setup-per-instance.sh new file mode 100755 index 000000000..36bd29b6f --- /dev/null +++ b/deployment/marketplace/vultr/victoriametrics-single/setup-per-instance.sh @@ -0,0 +1,29 @@ +#!/bin/bash +## Runs once-and-only-once at first boot per instance. + +## Report the start time to a logfile. +echo $(date -u) ": System provisioning started." >> /var/log/per-instance.log + +apt update +DEBIAN_FRONTEND=noninteractive apt -qq full-upgrade -y +DEBIAN_FRONTEND=noninteractive apt -qq install -y ufw wget software-properties-common ssh + +# Configure UFW + +sed -e 's|DEFAULT_FORWARD_POLICY=.*|DEFAULT_FORWARD_POLICY="ACCEPT"|g' \ + -i /etc/default/ufw + +ufw allow ssh comment "SSH port" +ufw allow http comment "HTTP port" +ufw allow https comment "HTTPS port" +ufw allow 8428 comment "VictoriaMetrics Single HTTP port" +ufw allow 8089/tcp comment "TCP Influx Listen port for VictoriaMetrics" +ufw allow 8089/udp comment "UDP Influx Listen port for VictoriaMetrics" +ufw allow 2003/tcp comment "TCP Graphite Listen port for VictoriaMetrics" +ufw allow 2003/udp comment "UDP Graphite Listen port for VictoriaMetrics" +ufw allow 4242 comment "OpenTSDB Listen port for VictoriaMetrics" + +ufw --force enable + +## Report the end time to a logfile. +echo $(date -u) ": System provisioning script is complete." >> /var/log/per-instance.log \ No newline at end of file diff --git a/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.pkr.hcl b/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.pkr.hcl new file mode 100644 index 000000000..84f527c79 --- /dev/null +++ b/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.pkr.hcl @@ -0,0 +1,65 @@ +variable "vultr_api_key" { + type = string + default = "${env("VULTR_API_KEY")}" + sensitive = true +} + +variable "victoriametrics_version" { + type = string + default = "${env("VM_VERSION")}" + description = "Version number of the desired VictoriaMetrics binary." +} + +packer { + required_plugins { + vultr = { + version = ">=v2.3.2" + source = "github.com/vultr/vultr" + } + } +} + +source "vultr" "victoriametrics-single" { + api_key = "${var.vultr_api_key}" + os_id = "387" + plan_id = "vc2-1c-1gb" + region_id = "ewr" + snapshot_description = "victoriametrics-snapshot-${formatdate("YYYY-MM-DD hh:mm", timestamp())}" + ssh_username = "root" + state_timeout = "10m" +} + +build { + sources = ["source.vultr.victoriametrics-single"] + + provisioner "file" { + source = "helper-scripts/vultr-helper.sh" + destination = "/root/vultr-helper.sh" + } + + provisioner "file" { + source = "victoriametrics-single/setup-per-boot.sh" + destination = "/root/setup-per-boot.sh" + } + + # Copy configuration files + provisioner "file" { + destination = "/etc/" + source = "victoriametrics-single/etc/" + } + + provisioner "file" { + source = "victoriametrics-single/setup-per-instance.sh" + destination = "/root/setup-per-instance.sh" + } + + provisioner "shell" { + environment_vars = [ + "VM_VERSION=${var.victoriametrics_version}", + "DEBIAN_FRONTEND=noninteractive" + ] + script = "victoriametrics-single/victoriametrics-single.sh" + remote_folder = "/root" + remote_file = "victoriametrics-single.sh" + } +} diff --git a/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh b/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh new file mode 100644 index 000000000..55d3a146a --- /dev/null +++ b/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh @@ -0,0 +1,41 @@ +#!/bin/bash +################################################ +## Prerequisites +chmod +x /root/vultr-helper.sh +. /root/vultr-helper.sh +error_detect_on +install_cloud_init latest + +################################################ +## Create victoriametrics user +groupadd -r victoriametrics +useradd -g victoriametrics -d /var/lib/victoria-metrics-data -s /sbin/nologin --system victoriametrics + +mkdir -p /var/lib/victoria-metrics-data +chown -R victoriametrics:victoriametrics /var/lib/victoria-metrics-data + +################################################ +## Download VictoriaMetrics +wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/victoria-metrics-linux-amd64-${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz +tar xvf /tmp/victoria-metrics.tar.gz -C /usr/bin +chmod +x /usr/bin/victoria-metrics-prod +chown root:root /usr/bin/victoria-metrics-prod + +################################################ +## Install provisioning scripts +mkdir -p /var/lib/cloud/scripts/per-boot/ +mkdir -p /var/lib/cloud/scripts/per-instance/ + +mv /root/setup-per-boot.sh /var/lib/cloud/scripts/per-boot/setup-per-boot.sh +mv /root/setup-per-instance.sh /var/lib/cloud/scripts/per-instance/setup-per-instance.sh + +chmod +x /var/lib/cloud/scripts/per-boot/setup-per-boot.sh +chmod +x /var/lib/cloud/scripts/per-instance/setup-per-instance.sh + +# Enable VictoriaMetrics on boot +systemctl enable vmsingle.service + +################################################ +## Prepare server for Marketplace snapshot + +clean_system \ No newline at end of file