From 24915fd4bc7e7180013c4ab6c36278e8b1f166f0 Mon Sep 17 00:00:00 2001 From: Denys Holius <5650611+denisgolius@users.noreply.github.com> Date: Thu, 23 Feb 2023 10:42:55 +0200 Subject: [PATCH] Fix some typos and adds improvements for packer builds (#3825) * update helper scripts to latest versions * added missed command for initialisation variables from Makefile * deployment/marketplace/vultr/helper-scripts/vultr-helper.sh: update helper script to latest version * fixed typo for using VM_VERSION variable * added an example of specifying the VM_VERSION and tokens for API's * set packer logging to STDOUT by default --- .../digitialocean/one-click-droplet/Makefile | 1 + .../one-click-droplet/RELEASE_GUIDE.md | 3 +- .../files/etc/update-motd.d/99-one-click | 4 +- .../files/etc/update-motd.d/99-one-click.tpl | 4 +- .../scripts/04-install-victoriametrics.sh | 2 +- .../one-click-droplet/scripts/90-cleanup.sh | 4 + .../one-click-droplet/scripts/99-img-check.sh | 215 ++++----- deployment/marketplace/vultr/Makefile | 2 - deployment/marketplace/vultr/RELEASE_GUIDE.md | 2 +- .../vultr/helper-scripts/vultr-helper.sh | 413 +++++++++--------- .../victoriametrics-single.sh | 2 +- 11 files changed, 307 insertions(+), 345 deletions(-) diff --git a/deployment/marketplace/digitialocean/one-click-droplet/Makefile b/deployment/marketplace/digitialocean/one-click-droplet/Makefile index 8d67f78f0..c607b4b9c 100644 --- a/deployment/marketplace/digitialocean/one-click-droplet/Makefile +++ b/deployment/marketplace/digitialocean/one-click-droplet/Makefile @@ -6,4 +6,5 @@ VM_VERSION ?= $(shell git describe --abbrev=0 --tags) release-victoria-metrics-digitalocean-oneclick-droplet: cp ./files/etc/update-motd.d/99-one-click.tpl ./files/etc/update-motd.d/99-one-click sed -i -e "s/VM_VERSION/${VM_VERSION}/g" ./files/etc/update-motd.d/99-one-click + packer init template.pkr.hcl packer build template.pkr.hcl \ No newline at end of file diff --git a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md index 40fdcd22d..8e6521ed9 100644 --- a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md +++ b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md @@ -8,9 +8,10 @@ 4. Set variables `DIGITALOCEAN_API_TOKEN` with `VM_VERSION` for `packer` environment and run make from example below: ```console -make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="your_token_here" VM_VERSION="prefered_release_version" +make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.87.1" ``` + ## Release guide for DigitalOcean Kubernetes 1-Click App ### Submit a pull request diff --git a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click index 1f747b6c2..e2a0e9480 100755 --- a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click +++ b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click @@ -19,8 +19,8 @@ On the server: * VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface. ******************************************************************************** - # This image includes version VM_VERSION of VictoriaMetrics. - # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/VM_VERSION + # This image includes 1.87.1 version of VictoriaMetrics. + # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.87.1 # Welcome to VictoriaMetrics droplet! diff --git a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click.tpl b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click.tpl index 1f747b6c2..1b7d1abbf 100644 --- a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click.tpl +++ b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click.tpl @@ -19,8 +19,8 @@ On the server: * VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface. ******************************************************************************** - # This image includes version VM_VERSION of VictoriaMetrics. - # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/VM_VERSION + # This image includes VM_VERSION version of VictoriaMetrics. + # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/vVM_VERSION # Welcome to VictoriaMetrics droplet! diff --git a/deployment/marketplace/digitialocean/one-click-droplet/scripts/04-install-victoriametrics.sh b/deployment/marketplace/digitialocean/one-click-droplet/scripts/04-install-victoriametrics.sh index be606c912..29b6e8149 100755 --- a/deployment/marketplace/digitialocean/one-click-droplet/scripts/04-install-victoriametrics.sh +++ b/deployment/marketplace/digitialocean/one-click-droplet/scripts/04-install-victoriametrics.sh @@ -3,7 +3,7 @@ # Wait for cloud-init cloud-init status --wait -wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/victoria-metrics-linux-amd64-${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz +wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v${VM_VERSION}/victoria-metrics-linux-amd64-v${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz tar xvf /tmp/victoria-metrics.tar.gz -C /usr/bin chmod +x /usr/bin/victoria-metrics-prod chown root:root /usr/bin/victoria-metrics-prod diff --git a/deployment/marketplace/digitialocean/one-click-droplet/scripts/90-cleanup.sh b/deployment/marketplace/digitialocean/one-click-droplet/scripts/90-cleanup.sh index f6b0f526d..ba673980c 100755 --- a/deployment/marketplace/digitialocean/one-click-droplet/scripts/90-cleanup.sh +++ b/deployment/marketplace/digitialocean/one-click-droplet/scripts/90-cleanup.sh @@ -22,6 +22,10 @@ rm -rf /var/lib/cloud/instances/* rm -f /root/.ssh/authorized_keys /etc/ssh/*key* touch /etc/ssh/revoked_keys chmod 600 /etc/ssh/revoked_keys +sudo apt-get purge droplet-agent -y +history -c +cat /dev/null > /root/.bash_history +unset HISTFILE # Securely erase the unused portion of the filesystem GREEN='\033[0;32m' diff --git a/deployment/marketplace/digitialocean/one-click-droplet/scripts/99-img-check.sh b/deployment/marketplace/digitialocean/one-click-droplet/scripts/99-img-check.sh index 00b547641..4c3c63aca 100755 --- a/deployment/marketplace/digitialocean/one-click-droplet/scripts/99-img-check.sh +++ b/deployment/marketplace/digitialocean/one-click-droplet/scripts/99-img-check.sh @@ -1,10 +1,10 @@ #!/bin/bash # DigitalOcean Marketplace Image Validation Tool -# © 2021 DigitalOcean LLC. +# © 2021-2022 DigitalOcean LLC. # This code is licensed under Apache 2.0 license (see LICENSE.md for details) -VERSION="v. 1.6" +VERSION="v. 1.8.1" RUNDATE=$( date ) # Script should be run with SUDO @@ -31,6 +31,7 @@ cmdExists() { function getDistro { if [ -f /etc/os-release ]; then # freedesktop.org and systemd + # shellcheck disable=SC1091 . /etc/os-release OS=$NAME VER=$VERSION_ID @@ -40,6 +41,7 @@ elif type lsb_release >/dev/null 2>&1; then VER=$(lsb_release -sr) elif [ -f /etc/lsb-release ]; then # For some versions of Debian/Ubuntu without lsb_release command + # shellcheck disable=SC1091 . /etc/lsb-release OS=$DISTRIB_ID VER=$DISTRIB_RELEASE @@ -52,8 +54,8 @@ elif [ -f /etc/SuSe-release ]; then : elif [ -f /etc/redhat-release ]; then # Older Red Hat, CentOS, etc. - VER=$( cat /etc/redhat-release | cut -d" " -f3 | cut -d "." -f1) - d=$( cat /etc/redhat-release | cut -d" " -f1 | cut -d "." -f1) + VER=$(cut -d" " -f3 < /etc/redhat-release | cut -d "." -f1) + d=$(cut -d" " -f1 < /etc/redhat-release | cut -d "." -f1) if [[ $d == "CentOS" ]]; then OS="CentOS Linux" fi @@ -68,15 +70,16 @@ SHADOW=$(cat /etc/shadow) } function checkAgent { - # Check for the presence of the do-agent in the filesystem - if [ -d /var/opt/digitalocean/do-agent ];then - echo -en "\e[41m[FAIL]\e[0m DigitalOcean Monitoring Agent detected.\n" + # Check for the presence of the DO directory in the filesystem + if [ -d /opt/digitalocean ];then + echo -en "\e[41m[FAIL]\e[0m DigitalOcean directory detected.\n" ((FAIL++)) STATUS=2 - if [[ $OS == "CentOS Linux" ]]; then - echo "The agent can be removed with 'sudo yum remove do-agent' " - elif [[ $OS == "Ubuntu" ]]; then - echo "The agent can be removed with 'sudo apt-get purge do-agent' " + if [[ $OS == "CentOS Linux" ]] || [[ $OS == "CentOS Stream" ]] || [[ $OS == "Rocky Linux" ]] || [[ $OS == "AlmaLinux" ]]; then + echo "To uninstall the agent: 'sudo yum remove droplet-agent'" + echo "To remove the DO directory: 'find /opt/digitalocean/ -type d -empty -delete'" + elif [[ $OS == "Ubuntu" ]] || [[ $OS == "Debian" ]]; then + echo "To uninstall the agent and remove the DO directory: 'sudo apt-get purge droplet-agent'" fi else echo -en "\e[32m[PASS]\e[0m DigitalOcean Monitoring agent was not found\n" @@ -90,8 +93,9 @@ function checkLogs { # Check if there are log archives or log files that have not been recently cleared. for f in /var/log/*-????????; do [[ -e $f ]] || break - if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m Log archive ${f} found\n" + if [ "${f}" != "${cp_ignore}" ]; then + echo -en "\e[93m[WARN]\e[0m Log archive ${f} found; Contents:\n" + cat "${f}" ((WARN++)) if [[ $STATUS != 2 ]]; then STATUS=1 @@ -100,7 +104,8 @@ function checkLogs { done for f in /var/log/*.[0-9];do [[ -e $f ]] || break - echo -en "\e[93m[WARN]\e[0m Log archive ${f} found\n" + echo -en "\e[93m[WARN]\e[0m Log archive ${f} found; Contents:\n" + cat "${f}" ((WARN++)) if [[ $STATUS != 2 ]]; then STATUS=1 @@ -108,17 +113,19 @@ function checkLogs { done for f in /var/log/*.log; do [[ -e $f ]] || break - if [[ "${f}" = '/var/log/lfd.log' && "$( cat "${f}" | egrep -v '/var/log/messages has been reset| Watching /var/log/messages' | wc -c)" -gt 50 ]]; then - if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" + if [[ "${f}" = '/var/log/lfd.log' && "$(grep -E -v '/var/log/messages has been reset| Watching /var/log/messages' "${f}" | wc -c)" -gt 50 ]]; then + if [ "${f}" != "${cp_ignore}" ]; then + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found; Contents:\n" + cat "${f}" ((WARN++)) if [[ $STATUS != 2 ]]; then STATUS=1 fi fi - elif [[ "${f}" != '/var/log/lfd.log' && "$( cat "${f}" | wc -c)" -gt 50 ]]; then - if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" + elif [[ "${f}" != '/var/log/lfd.log' && "$(wc -c < "${f}")" -gt 50 ]]; then + if [ "${f}" != "${cp_ignore}" ]; then + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found; Contents:\n" + cat "${f}" ((WARN++)) if [[ $STATUS != 2 ]]; then STATUS=1 @@ -151,25 +158,25 @@ function checkRoot { if [ -d ${uhome}/ ]; then if [ -d ${uhome}/.ssh/ ]; then if ls ${uhome}/.ssh/*> /dev/null 2>&1; then - for key in ${uhome}/.ssh/* + for key in "${uhome}"/.ssh/* do if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then - if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + if [ "$(wc -c < "${key}")" -gt 50 ]; then echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) + akey=$(cat "${key}") echo "File Contents:" - echo $akey + echo "$akey" echo "--------------" ((FAIL++)) STATUS=2 fi elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then - if [ "$( cat "${key}" | wc -c)" -gt 0 ]; then + if [ "$(wc -c < "${key}")" -gt 0 ]; then echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) + akey=$(cat "${key}") echo "File Contents:" - echo $akey + echo "$akey" echo "--------------" ((FAIL++)) STATUS=2 @@ -187,7 +194,7 @@ function checkRoot { STATUS=1 fi else - if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + if [ "$(wc -c < "${key}")" -gt 50 ]; then echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a populated known_hosts file in \e[93m${key}\e[0m\n" ((WARN++)) if [[ $STATUS != 2 ]]; then @@ -204,7 +211,7 @@ function checkRoot { fi if [ -f /root/.bash_history ];then - BH_S=$( cat /root/.bash_history | wc -c) + BH_S=$(wc -c < /root/.bash_history) if [[ $BH_S -lt 200 ]]; then echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" @@ -229,7 +236,7 @@ function checkRoot { function checkUsers { # Check each user-created account - for user in $(awk -F: '$3 >= 1000 && $1 != "nobody" {print $1}' /etc/passwd;) + awk -F: '$3 >= 1000 && $1 != "nobody" {print $1}' < /etc/passwd | while IFS= read -r user; do # Skip some other non-user system accounts if [[ $user == "centos" ]]; then @@ -244,9 +251,11 @@ function checkUsers { if [[ "${u[0]}" == "${user}" ]]; then if [[ ${u[1]} == "!" ]] || [[ ${u[1]} == "!!" ]] || [[ ${u[1]} == "*" ]]; then echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" + # shellcheck disable=SC2030 ((PASS++)) else echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account. Only system users are allowed on the image.\n" + # shellcheck disable=SC2030 ((FAIL++)) STATUS=2 fi @@ -257,29 +266,30 @@ function checkUsers { if [ -d "${uhome}/" ]; then if [ -d "${uhome}/.ssh/" ]; then if ls "${uhome}/.ssh/*"> /dev/null 2>&1; then - for key in ${uhome}/.ssh/* + for key in "${uhome}"/.ssh/* do if [ "${key}" == "${uhome}/.ssh/authorized_keys" ]; then - if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + if [ "$(wc -c < "${key}")" -gt 50 ]; then echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a populated authorized_keys file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) + akey=$(cat "${key}") echo "File Contents:" - echo $akey + echo "$akey" echo "--------------" ((FAIL++)) STATUS=2 fi elif [ "${key}" == "${uhome}/.ssh/id_rsa" ]; then - if [ "$( cat "${key}" | wc -c)" -gt 0 ]; then + if [ "$(wc -c < "${key}")" -gt 0 ]; then echo -en "\e[41m[FAIL]\e[0m User \e[1m${user}\e[0m has a private key file in \e[93m${key}\e[0m\n" - akey=$(cat ${key}) + akey=$(cat "${key}") echo "File Contents:" - echo $akey + echo "$akey" echo "--------------" ((FAIL++)) STATUS=2 else echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has empty private key file in \e[93m${key}\e[0m\n" + # shellcheck disable=SC2030 ((WARN++)) if [[ $STATUS != 2 ]]; then STATUS=1 @@ -294,7 +304,7 @@ function checkUsers { fi else - if [ "$( cat "${key}" | wc -c)" -gt 50 ]; then + if [ "$(wc -c < "${key}")" -gt 50 ]; then echo -en "\e[93m[WARN]\e[0m User \e[1m${user}\e[0m has a known_hosts file in \e[93m${key}\e[0m\n" ((WARN++)) if [[ $STATUS != 2 ]]; then @@ -317,7 +327,7 @@ function checkUsers { # Check for an uncleared .bash_history for this user if [ -f "${uhome}/.bash_history" ]; then - BH_S=$( cat "${uhome}/.bash_history" | wc -c ) + BH_S=$(wc -c < "${uhome}/.bash_history") if [[ $BH_S -lt 200 ]]; then echo -en "\e[32m[PASS]\e[0m ${user}'s Bash History appears to have been cleared\n" @@ -340,12 +350,14 @@ function checkFirewall { ufwa=$(ufw status |head -1| sed -e "s/^Status:\ //") if [[ $ufwa == "active" ]]; then FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" + # shellcheck disable=SC2031 ((PASS++)) else FW_VER="\e[93m[WARN]\e[0m No firewall is configured. Ensure ${fw} is installed and configured\n" + # shellcheck disable=SC2031 ((WARN++)) fi - elif [[ $OS == "CentOS Linux" ]]; then + elif [[ $OS == "CentOS Linux" ]] || [[ $OS == "CentOS Stream" ]] || [[ $OS == "Rocky Linux" ]] || [[ $OS == "AlmaLinux" ]]; then if [ -f /usr/lib/systemd/system/csf.service ]; then fw="csf" if [[ $(systemctl status $fw >/dev/null 2>&1) ]]; then @@ -399,7 +411,7 @@ function checkFirewall { else # user could be using vanilla iptables, check if kernel module is loaded fw="iptables" - if [[ $(lsmod | grep -q '^ip_tables' 2>/dev/null) ]]; then + if lsmod | grep -q '^ip_tables' 2>/dev/null; then FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" ((PASS++)) else @@ -423,9 +435,9 @@ function checkUpdates { echo -en "\nUpdating apt package database to check for security updates, this may take a minute...\n\n" apt-get -y update > /dev/null - uc=$(apt-get --just-print upgrade | grep -i "security" | wc -l) + uc=$(apt-get --just-print upgrade | grep -i "security" -c) if [[ $uc -gt 0 ]]; then - update_count=$(( ${uc} / 2 )) + update_count=$(( uc / 2 )) else update_count=0 fi @@ -437,12 +449,14 @@ function checkUpdates { sleep 2 apt-get --just-print upgrade | grep -i security | awk '{print $2}' | awk '!seen[$0]++' echo -en + # shellcheck disable=SC2031 ((FAIL++)) STATUS=2 else echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n\n" + ((PASS++)) fi - elif [[ $OS == "CentOS Linux" ]]; then + elif [[ $OS == "CentOS Linux" ]] || [[ $OS == "CentOS Stream" ]] || [[ $OS == "Rocky Linux" ]] || [[ $OS == "AlmaLinux" ]]; then echo -en "\nChecking for available security updates, this may take a minute...\n\n" update_count=$(yum check-update --security --quiet | wc -l) @@ -473,83 +487,6 @@ function checkCloudInit { fi return 1 } -function checkMongoDB { - # Check if MongoDB is installed - # If it is, verify the version is allowed (non-SSPL) - - if [[ $OS == "Ubuntu" ]] || [[ "$OS" =~ Debian.* ]]; then - - if [[ -f "/usr/bin/mongod" ]]; then - version=$(/usr/bin/mongod --version --quiet | grep "db version" | sed -e "s/^db\ version\ v//") - - if version_gt $version 4.0.0; then - if version_gt $version 4.0.3; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present, ${version}" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed, ${version} is not under the SSPL" - ((PASS++)) - fi - else - if version_gt $version 3.6.8; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present, ${version}" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed, ${version} is not under the SSPL" - ((PASS++)) - fi - fi - - - else - echo -en "\e[32m[PASS]\e[0m MongoDB is not installed" - ((PASS++)) - fi - - elif [[ $OS == "CentOS Linux" ]]; then - - if [[ -f "/usr/bin/mongod" ]]; then - version=$(/usr/bin/mongod --version --quiet | grep "db version" | sed -e "s/^db\ version\ v//") - - - if version_gt $version 4.0.0; then - if version_gt $version 4.0.3; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed is not under the SSPL" - ((PASS++)) - fi - else - if version_gt $version 3.6.8; then - echo -en "\e[41m[FAIL]\e[0m An SSPL version of MongoDB is present" - ((FAIL++)) - STATUS=2 - else - echo -en "\e[32m[PASS]\e[0m The version of MongoDB installed is not under the SSPL" - ((PASS++)) - fi - fi - - - - else - echo -en "\e[32m[PASS]\e[0m MongoDB is not installed" - ((PASS++)) - fi - - else - echo "ERROR: Unable to identify distribution" - ((FAIL++)) - STATUS 2 - return 1 - fi - - -} function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } @@ -569,14 +506,8 @@ osv=0 if [[ $OS == "Ubuntu" ]]; then ost=1 - if [[ $VER == "20.04" ]]; then + if [[ $VER == "22.10" ]] || [[ $VER == "22.04" ]] || [[ $VER == "20.04" ]] || [[ $VER == "18.04" ]] || [[ $VER == "16.04" ]]; then osv=1 - elif [[ $VER == "18.04" ]]; then - osv=1 - elif [[ $VER == "16.04" ]]; then - osv=1 - else - osv=0 fi elif [[ "$OS" =~ Debian.* ]]; then @@ -588,6 +519,9 @@ elif [[ "$OS" =~ Debian.* ]]; then 10) osv=1 ;; + 11) + osv=1 + ;; *) osv=2 ;; @@ -604,6 +538,27 @@ elif [[ $OS == "CentOS Linux" ]]; then else osv=2 fi +elif [[ $OS == "CentOS Stream" ]]; then + ost=1 + if [[ $VER == "8" ]]; then + osv=1 + else + osv=2 + fi +elif [[ $OS == "Rocky Linux" ]]; then + ost=1 + if [[ $VER =~ 8\. ]]; then + osv=1 + else + osv=2 + fi +elif [[ $OS == "AlmaLinux" ]]; then + ost=1 + if [[ "$VERSION" =~ 8.* ]] || [[ "$VERSION" =~ 9.* ]]; then + osv=1 + else + osv=2 + fi else ost=0 fi @@ -651,8 +606,6 @@ checkRoot checkAgent -checkMongoDB - # Summary echo -en "\n\n---------------------------------------------------------------------------------------------------\n" @@ -679,4 +632,4 @@ elif [[ $STATUS == 1 ]]; then else echo -en "Some critical tests failed. These items must be resolved and this scan re-run before you submit your image to the DigitalOcean Marketplace.\n\n" exit 1 -fi \ No newline at end of file +fi diff --git a/deployment/marketplace/vultr/Makefile b/deployment/marketplace/vultr/Makefile index 9849619d1..9e9550186 100644 --- a/deployment/marketplace/vultr/Makefile +++ b/deployment/marketplace/vultr/Makefile @@ -1,7 +1,5 @@ RELEASE_NAME := vm-vultr-server VM_VERSION ?= $(shell git describe --abbrev=0 --tags) -PACKER_LOG := 1 -PACKER_LOG_PATH := packer.log .PHONY: $(MAKECMDGOALS) diff --git a/deployment/marketplace/vultr/RELEASE_GUIDE.md b/deployment/marketplace/vultr/RELEASE_GUIDE.md index f68c882b5..45d0aad73 100644 --- a/deployment/marketplace/vultr/RELEASE_GUIDE.md +++ b/deployment/marketplace/vultr/RELEASE_GUIDE.md @@ -8,5 +8,5 @@ 4. Set variables `VULTR_API_KEY` with `VM_VERSION` for `packer` environment and run make from example below: ```console -make release-victoria-metrics-vultr-server VULTR_API_KEY="your_token_here" VM_VERSION="prefered_release_version" +make release-victoria-metrics-vultr-server VULTR_API_KEY="5FI5J9PZCCN1TAXPHI8UMDH5ZX8JIHJKTSLB" VM_VERSION="1.87.1" ``` diff --git a/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh b/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh index 97607b141..6b0fbe188 100755 --- a/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh +++ b/deployment/marketplace/vultr/helper-scripts/vultr-helper.sh @@ -1,317 +1,322 @@ #!/bin/bash -# shopt -s inherit_errexit -set -o errexit - ################################################################### ## Vultr Marketplace Helper Functions -function error_detect_on() +inherit_errexit() { - set -euo pipefail + if ! shopt -sq inherit_errexit > /dev/null 2>&1; then + echo "Unable to enable inherit_errexit" + fi } -function error_detect_off() +error_detect_on() { - set +euo pipefail + set -euo pipefail + inherit_errexit } -function enable_verbose_commands() +error_detect_off() { - set -x + set +euo pipefail + inherit_errexit } -function disable_verbose_commands() +enable_verbose_commands() { - set +x + set -x } -function get_metadata_item() +disable_verbose_commands() { - local item_path="${1:-}" - local item_value - - item_value="$(curl --fail --silent --header "Metadata-Token: vultr" "http://169.254.169.254/${item_path}")" - - echo "${item_value}" + set +x } -function get_hostname() +get_metadata_item() { - get_metadata_item "latest/meta-data/hostname" + local item_value + item_value="$(curl --fail --silent --header "Metadata-Token: vultr" "http://169.254.169.254/${1:-}")" + + echo "${item_value}" } -function get_userdata() +get_hostname() { - get_metadata_item "latest/user-data" + get_metadata_item "latest/meta-data/hostname" } -function get_sshkeys() +get_userdata() { - get_metadata_item "current/ssh-keys" + get_metadata_item "latest/user-data" } -function get_var() +get_sshkeys() { - local var_name="${1:-}" - local var_val - var_val="$(get_metadata_item "v1/internal/app-${var_name}" 2>/dev/null)" - - eval "${var_name}='${var_val}'" + get_metadata_item "current/ssh-keys" } -function get_ip() +# shellcheck disable=SC2034 +get_var() { - local ip_var="${1:-}" - local ip_val - ip_val="$(get_metadata_item "latest/meta-data/public-ipv4" 2>/dev/null)" + local var_name="${1:-}" var_path="${2:-}" var_val + var_val="$(get_metadata_item "${var_path:-"v1/internal/app-${var_name}"}" 2> /dev/null)" - eval "${ip_var}='${ip_val}'" + eval "${var_name}=\${var_val}" } -function wait_on_apt_lock() +# shellcheck disable=SC2034 +get_ip() { - until ! lsof -t /var/cache/apt/archives/lock /var/lib/apt/lists/lock /var/lib/dpkg/lock >/dev/null 2>&1 - do - echo "Waiting 3 for apt lock currently held by another process." - sleep 3 - done + local ip_var="${1:-}" ip_val + ip_val="$(get_var "${ip_var}" "latest/meta-data/public-ipv4")" + + eval "${ip_var}=\${ip_val}" } -function apt_safe() +wait_on_apt_lock() { - wait_on_apt_lock - apt install -y "$@" + until ! lsof -t /var/cache/apt/archives/lock /var/lib/apt/lists/lock /var/lib/dpkg/lock > /dev/null 2>&1; do + echo "Waiting 3 for apt lock currently held by another process." + sleep 3 + done } -function apt_update_safe() +apt_safe() { - wait_on_apt_lock - apt update -y + wait_on_apt_lock + apt install -y "$@" } -function apt_upgrade_safe() +apt_update_safe() { - wait_on_apt_lock - DEBIAN_FRONTEND=noninteractive apt upgrade -y + wait_on_apt_lock + apt update -y } -function apt_remove_safe() +apt_upgrade_safe() { - wait_on_apt_lock - apt remove -y --auto-remove "$@" + wait_on_apt_lock + DEBIAN_FRONTEND=noninteractive apt upgrade -y } -function apt_clean_safe() +apt_remove_safe() { - wait_on_apt_lock - apt autoremove -y - - wait_on_apt_lock - apt autoclean -y + wait_on_apt_lock + apt remove -y --auto-remove "$@" } -function update_and_clean_packages() +apt_clean_safe() { - # RHEL/CentOS - if [[ -f /etc/redhat-release ]]; then - yum update -y - yum clean all - # Ubuntu / Debian - elif grep -qs "debian" /etc/os-release 2>/dev/null; then - apt_update_safe - apt_upgrade_safe - apt_clean_safe - fi + wait_on_apt_lock + apt autoremove -y + + wait_on_apt_lock + apt autoclean -y } -function set_vultr_kernel_option() +update_and_clean_packages() { - # RHEL/CentOS - if [[ -f /etc/redhat-release ]]; then - /sbin/grubby --update-kernel=ALL --args vultr - # Ubuntu / Debian - elif grep -qs "debian" /etc/os-release 2>/dev/null; then - sed -i -e "/^GRUB_CMDLINE_LINUX_DEFAULT=/ s/\"$/ vultr\"/" /etc/default/grub - update-grub - fi + # RHEL/CentOS + if [[ -f /etc/redhat-release ]]; then + yum update -y + yum clean all + # Ubuntu / Debian + elif grep -qs "debian" /etc/os-release 2> /dev/null; then + apt_update_safe + apt_upgrade_safe + apt_clean_safe + fi } -function install_cloud_init() +set_vultr_kernel_option() { - local cloud_init_exe - cloud_init_exe="$(command -v cloud-init >/dev/null 2>&1)" - if [[ -x "${cloud_init_exe}" ]]; then - echo "cloud-init is already installed." - return - fi + # RHEL/CentOS + if [[ -f /etc/redhat-release ]]; then + /sbin/grubby --update-kernel=ALL --args vultr + # Ubuntu / Debian + elif grep -qs "debian" /etc/os-release 2> /dev/null; then + sed -i -e "/^GRUB_CMDLINE_LINUX_DEFAULT=/ s/\"$/ vultr\"/" /etc/default/grub + update-grub + fi +} + +install_cloud_init() +{ + local cloudinit_exe="" + if cloudinit_exe="$(command -v cloud-init 2> /dev/null)" && [[ -x "${cloudinit_exe}" ]]; then + echo "cloud-init is already installed." + return + fi local release_version="${1:-"latest"}" - if [[ "${release_version}" != "latest" && "${release_version}" != "nightly" ]]; then - echo "${release_version} is an invalid release option. Allowed: latest, nightly" - exit 255 - fi + if [[ "${release_version}" != "latest" && "${release_version}" != "nightly" ]]; then + echo "${release_version} is an invalid release option. Allowed: latest, nightly" + exit 255 + fi - # Lets remove all traces of previously installed cloud-init - # Ubuntu installs have proven problematic with their left over - # configs for the installer in recent versions - cleanup_cloudinit + # Lets remove all traces of previously installed cloud-init + # Ubuntu installs have proven problematic with their left over + # configs for the installer in recent versions + cleanup_cloudinit - update_and_clean_packages + update_and_clean_packages - local build_type - local package_ext + local build_type + local package_ext - [[ -e /etc/os-release ]] && . /etc/os-release - case "${ID:-}" in - debian) - build_type="debian" - package_ext="deb" - ;; - fedora) - build_type="rhel" - package_ext="rpm" - ;; - ubuntu) - build_type="universal" - package_ext="deb" - ;; - *) - case "${ID_LIKE:-}" in - *rhel*) - build_type="rhel" - package_ext="rpm" - ;; - *) - echo "Unable to determine OS. Please install from source!" - exit 255 - esac - esac + [[ -e /etc/os-release ]] && . /etc/os-release + case "${ID:-}" in + debian) + build_type="debian" + package_ext="deb" + ;; + fedora) + build_type="rhel" + package_ext="rpm" + ;; + ubuntu) + build_type="universal" + package_ext="deb" + ;; + *) + case "${ID_LIKE:-}" in + *rhel*) + build_type="rhel" + package_ext="rpm" + ;; + *) + echo "Unable to determine OS. Please install from source!" + exit 255 + ;; + esac + ;; + esac - local cloud_init_package="cloud-init_${build_type}_${release_version}.${package_ext}" - wget -O "/tmp/${cloud_init_package}" "https://ewr1.vultrobjects.com/cloud_init_beta/${cloud_init_package}" + local cloud_init_package="cloud-init_${build_type}_${release_version}.${package_ext}" + wget -O "/tmp/${cloud_init_package}" "https://ewr1.vultrobjects.com/cloud_init_beta/${cloud_init_package}" - case "${package_ext}" in - rpm) - yum install -y "/tmp/${cloud_init_package}" - ;; - deb) - apt_safe "/tmp/${cloud_init_package}" - ;; - *) - echo "Unable to determine package installation method." - exit 255 - esac + case "${package_ext}" in + rpm) + yum install -y "/tmp/${cloud_init_package}" + ;; + deb) + apt_safe "/tmp/${cloud_init_package}" + ;; + *) + echo "Unable to determine package installation method." + exit 255 + ;; + esac - rm -f "/tmp/${cloud_init_package}" + rm -f "/tmp/${cloud_init_package}" } -function cleanup_cloudinit() +cleanup_cloudinit() { - rm -rf \ - /etc/cloud \ - /etc/systemd/system/cloud-init.target.wants/* \ - /lib/systemd/system/cloud* \ - /run/cloud-init \ - /usr/bin/cloud* \ - /usr/lib/cloud* \ - /usr/local/bin/cloud* \ - /usr/src/cloud* \ - /var/log/cloud* + rm -rf \ + /etc/cloud \ + /etc/systemd/system/cloud-init.target.wants/* \ + /lib/systemd/system/cloud* \ + /run/cloud-init \ + /usr/bin/cloud* \ + /usr/lib/cloud* \ + /usr/local/bin/cloud* \ + /usr/src/cloud* \ + /var/log/cloud* } -function clean_tmp() +clean_tmp() { - mkdir -p /tmp - chmod 1777 /tmp - rm -rf /tmp/* /var/tmp/* + mkdir -p /tmp + chmod 1777 /tmp + rm -rf /tmp/* /var/tmp/* } -function clean_keys() +clean_keys() { - rm -f /root/.ssh/authorized_keys /etc/ssh/*key* - touch /etc/ssh/revoked_keys - chmod 600 /etc/ssh/revoked_keys + rm -f /root/.ssh/authorized_keys /etc/ssh/*key* + touch /etc/ssh/revoked_keys + chmod 600 /etc/ssh/revoked_keys } -function clean_logs() +clean_logs() { - find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; - rm -rf \ - /var/log/*.[0-9] \ - /var/log/*.gz \ - /var/log/*.log \ - /var/log/lastlog \ - /var/log/wtmp + find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; + rm -rf \ + /var/log/*.[0-9] \ + /var/log/*.gz \ + /var/log/*.log \ + /var/log/lastlog \ + /var/log/wtmp - : > /var/log/auth.log + : > /var/log/auth.log } -function clean_history() +clean_history() { - history -c - : > /root/.bash_history - unset HISTFILE + history -c + : > /root/.bash_history + unset HISTFILE } -function clean_mloc() +clean_mloc() { - /usr/bin/updatedb || true + /usr/bin/updatedb || true } -function clean_random() +clean_random() { - rm -f /var/lib/systemd/random-seed + rm -f /var/lib/systemd/random-seed } -function clean_machine_id() +clean_machine_id() { - [[ -e /etc/machine-id ]] && : > /etc/machine-id - [[ -e /var/lib/dbus/machine-id ]] && : > /var/lib/dbus/machine-id + [[ -e /etc/machine-id ]] && : > /etc/machine-id + [[ -e /var/lib/dbus/machine-id ]] && : > /var/lib/dbus/machine-id } -function clean_free_space() +clean_free_space() { - dd if=/dev/zero of=/zerofile || true - sync - rm -f /zerofile - sync + dd if=/dev/zero of=/zerofile || true + sync + rm -f /zerofile + sync } -function trim_ssd() +trim_ssd() { - fstrim / || true + fstrim / || true } -function cleanup_marketplace_scripts() +cleanup_marketplace_scripts() { - rm -f /root/*.sh + rm -f /root/*.sh } -function disable_network_manager() +disable_network_manager() { - ## Disable NetworkManager, replace with network-scripts - systemctl disable --now NetworkManager - sed -i \ - -e 's/^ONBOOT.*/ONBOOT=yes/g' \ - -e 's/^NM_CONTROLLED.*/NM_CONTROLLED=no/g' /etc/sysconfig/network-scripts/ifcfg-* - yum install -y network-scripts + ## Disable NetworkManager, replace with network-scripts + systemctl disable --now NetworkManager + sed -i \ + -e 's/^ONBOOT.*/ONBOOT=yes/g' \ + -e 's/^NM_CONTROLLED.*/NM_CONTROLLED=no/g' /etc/sysconfig/network-scripts/ifcfg-* + yum install -y network-scripts } -function clean_system() +clean_system() { - update_and_clean_packages - set_vultr_kernel_option - clean_tmp - clean_keys - clean_logs - clean_history - clean_random - clean_machine_id - clean_mloc - clean_free_space - trim_ssd + update_and_clean_packages + set_vultr_kernel_option + clean_tmp + clean_keys + clean_logs + clean_history + clean_random + clean_machine_id + clean_mloc + clean_free_space + trim_ssd - cleanup_marketplace_scripts + cleanup_marketplace_scripts } diff --git a/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh b/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh index 55d3a146a..8454808e2 100644 --- a/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh +++ b/deployment/marketplace/vultr/victoriametrics-single/victoriametrics-single.sh @@ -16,7 +16,7 @@ chown -R victoriametrics:victoriametrics /var/lib/victoria-metrics-data ################################################ ## Download VictoriaMetrics -wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/victoria-metrics-linux-amd64-${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz +wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v${VM_VERSION}/victoria-metrics-linux-amd64-v${VM_VERSION}.tar.gz -O /tmp/victoria-metrics.tar.gz tar xvf /tmp/victoria-metrics.tar.gz -C /usr/bin chmod +x /usr/bin/victoria-metrics-prod chown root:root /usr/bin/victoria-metrics-prod