lib/envtemplate: allow referring env vars from other env vars via %{ENV_VAR} syntax

This is a follow-up for 02096e06d0

README.md

@@ -180,10 +180,13 @@ It is possible manualy setting up a toy cluster on a single host. In this case e
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
-This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself.
+This expansion is performed by VictoriaMetrics itself.
+
+VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup.
+For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`.
 
 Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules:
 

app/vmalert/config/config.go

@@ -245,7 +245,7 @@ func parseFile(path string) ([]Group, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error reading alert rule file %q: %w", path, err)
 	}
-	data, err = envtemplate.Replace(data)
+	data, err = envtemplate.ReplaceBytes(data)
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err)
 	}

app/vmauth/auth_config.go

@@ -251,7 +251,7 @@ func readAuthConfig(path string) (map[string]*UserInfo, error) {
 
 func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 	var err error
-	data, err = envtemplate.Replace(data)
+	data, err = envtemplate.ReplaceBytes(data)
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}

docs/CHANGELOG.md

@@ -20,7 +20,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [VictoriaMetric enterprise](https://docs.victoriametrics.com/enterprise.html): allow configuring multiple retentions for distinct sets of time series. See [these docs](https://docs.victoriametrics.com/#retention-filters), [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/143) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/289) feature request.
 * FEATURE: [VictoriaMetric cluster enterprise](https://docs.victoriametrics.com/enterprise.html): add support for multiple retentions for distinct tenants - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#retention-filters) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/143) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/289) feature request.
 * FEATURE: allow limiting memory usage on a per-query basis with `-search.maxMemoryPerQuery` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3203).
-* FEATURE: allow using environment variables inside command-line flags passed to all VictoriaMetrics components. For example, if `AUTH_KEY=top-secret` environment variable is set, then `-metricsAuthKey=%{AUTH_KEY}` command-line flag is automatically expanded to `-storageDataPath=top-secret` at VictoriaMetrics startup. See [these docs](https://docs.victoriametrics.com/#environment-variables) for details.
+* FEATURE: allow referring environment variables inside command-line flags via `%{ENV_VAR}` syntax. For example, if `AUTH_KEY=top-secret` environment variable is set, then `-metricsAuthKey=%{AUTH_KEY}` command-line flag is automatically expanded to `-storageDataPath=top-secret` at VictoriaMetrics startup. See [these docs](https://docs.victoriametrics.com/#environment-variables) for details.
+* FEATURE: allow referring environment variables inside other environment variables via `%{ENV_VAR}` syntax. For example, if `A=a-%{B}`, `B=b-%{C}` and 'C=c` env vars are set, then VictoriaMetrics components automatically expand them to `A=a-b-c`, `B=b-c` and `C=c` on startup.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): drop all the labels with `__` prefix from discovered targets in the same way as Prometheus does according to [this article](https://www.robustperception.io/life-of-a-label/). Previously the following labels were available during [metric-level relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs): `__address__`, `__scheme__`, `__metrics_path__`, `__scrape_interval__`, `__scrape_timeout__`, `__param_*`. Now these labels are available only during [target-level relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config). This should reduce CPU usage and memory usage for `vmagent` setups, which scrape big number of targets.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): improve the performance for metric-level [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling), which can be applied via `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs), via `-remoteWrite.relabelConfig` or via `-remoteWrite.urlRelabelConfig` command-line options.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying full url in scrape target addresses (aka `__address__` label). This makes valid the following `-promscrape.config`:

docs/Cluster-VictoriaMetrics.md

@@ -184,10 +184,13 @@ It is possible manualy setting up a toy cluster on a single host. In this case e
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
-This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself.
+This expansion is performed by VictoriaMetrics itself.
+
+VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup.
+For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`.
 
 Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules:
 

docs/README.md

@@ -133,10 +133,13 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
-This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself.
+This expansion is performed by VictoriaMetrics itself.
+
+VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup.
+For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`.
 
 Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules:
 

docs/Single-server-VictoriaMetrics.md

@@ -136,10 +136,13 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
-This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself.
+This expansion is performed by VictoriaMetrics itself.
+
+VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup.
+For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`.
 
 Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules:
 

lib/backup/azremote/azblob.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 	"time"
 
@@ -19,6 +18,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -59,15 +59,15 @@ func (fs *FS) Init() error {
 
 	var sc *service.Client
 	var err error
-	if cs, ok := os.LookupEnv(envStorageAccCs); ok {
+	if cs, ok := envtemplate.LookupEnv(envStorageAccCs); ok {
 		sc, err = service.NewClientFromConnectionString(cs, nil)
 		if err != nil {
 			return fmt.Errorf("failed to create AZBlob service client from connection string: %w", err)
 		}
 	}
 
-	accountName, ok1 := os.LookupEnv(envStorageAcctName)
+	accountName, ok1 := envtemplate.LookupEnv(envStorageAcctName)
-	accountKey, ok2 := os.LookupEnv(envStorageAccKey)
+	accountKey, ok2 := envtemplate.LookupEnv(envStorageAccKey)
 	if ok1 && ok2 {
 		creds, err := azblob.NewSharedKeyCredential(accountName, accountKey)
 		if err != nil {

lib/envflag/envflag.go

@@ -26,12 +26,13 @@ func Parse() {
 	args := os.Args[1:]
 	dstArgs := args[:0]
 	for _, arg := range args {
-		b, err := envtemplate.Replace([]byte(arg))
+		s, err := envtemplate.ReplaceString(arg)
 		if err != nil {
+			// Do not use lib/logger here, since it is uninitialized yet.
 			log.Fatalf("cannot process arg %q: %s", arg, err)
 		}
-		if len(b) > 0 {
+		if len(s) > 0 {
-			dstArgs = append(dstArgs, string(b))
+			dstArgs = append(dstArgs, s)
 		}
 	}
 	os.Args = os.Args[:1+len(dstArgs)]
@@ -56,7 +57,7 @@ func Parse() {
 		}
 		// Get flag value from environment var.
 		fname := getEnvFlagName(f.Name)
-		if v, ok := os.LookupEnv(fname); ok {
+		if v, ok := envtemplate.LookupEnv(fname); ok {
 			if err := flag.Set(f.Name, v); err != nil {
 				// Do not use lib/logger here, since it is uninitialized yet.
 				log.Fatalf("cannot set flag %s to %q, which is read from environment variable %q: %s", f.Name, v, fname, err)

lib/envtemplate/envtemplate.go

@@ -1,31 +1,106 @@
 package envtemplate
 
 import (
-	"bytes"
 	"fmt"
 	"io"
+	"log"
 	"os"
+	"strings"
 
 	"github.com/valyala/fasttemplate"
 )
 
-// Replace replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values.
+// ReplaceBytes replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values.
 //
 // Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder.
-func Replace(b []byte) ([]byte, error) {
+func ReplaceBytes(b []byte) ([]byte, error) {
-	if !bytes.Contains(b, []byte("%{")) {
+	result, err := expand(envVars, string(b))
-		// Fast path - nothing to replace.
+	if err != nil {
-		return b, nil
+		return nil, err
 	}
-	s, err := fasttemplate.ExecuteFuncStringWithErr(string(b), "%{", "}", func(w io.Writer, tag string) (int, error) {
+	return []byte(result), nil
-		v, ok := os.LookupEnv(tag)
+}
+
+// ReplaceString replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values.
+//
+// Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder.
+func ReplaceString(s string) (string, error) {
+	result, err := expand(envVars, s)
+	if err != nil {
+		return "", err
+	}
+	return result, nil
+}
+
+// LookupEnv returns the expanded environment variable value for the given name.
+//
+// The expanded means that `%{ENV_VAR}` placeholders in env var value are replaced
+// with the corresponding ENV_VAR values (recursively).
+//
+// false is returned if environment variable isn't found.
+func LookupEnv(name string) (string, bool) {
+	value, ok := envVars[name]
+	return value, ok
+}
+
+var envVars = func() map[string]string {
+	envs := os.Environ()
+	m := parseEnvVars(envs)
+	return expandTemplates(m)
+}()
+
+func parseEnvVars(envs []string) map[string]string {
+	m := make(map[string]string, len(envs))
+	for _, env := range envs {
+		n := strings.IndexByte(env, '=')
+		if n < 0 {
+			m[env] = ""
+			continue
+		}
+		name := env[:n]
+		value := env[n+1:]
+		m[name] = value
+	}
+	return m
+}
+
+func expandTemplates(m map[string]string) map[string]string {
+	for i := 0; i < len(m); i++ {
+		mExpanded := make(map[string]string, len(m))
+		expands := 0
+		for name, value := range m {
+			valueExpanded, err := expand(m, value)
+			if err != nil {
+				// Do not use lib/logger here, since it is uninitialized yet.
+				log.Fatalf("cannot expand %q env var value %q: %s", name, value, err)
+			}
+			mExpanded[name] = valueExpanded
+			if valueExpanded != value {
+				expands++
+			}
+		}
+		if expands == 0 {
+			return mExpanded
+		}
+		m = mExpanded
+	}
+	return m
+}
+
+func expand(m map[string]string, s string) (string, error) {
+	if !strings.Contains(s, "%{") {
+		// Fast path - nothing to expand
+		return s, nil
+	}
+	result, err := fasttemplate.ExecuteFuncStringWithErr(s, "%{", "}", func(w io.Writer, tag string) (int, error) {
+		v, ok := m[tag]
 		if !ok {
-			return 0, fmt.Errorf("missing %q environment variable", tag)
+			return 0, fmt.Errorf("missing %q env var", tag)
 		}
 		return w.Write([]byte(v))
 	})
 	if err != nil {
-		return nil, err
+		return "", err
 	}
-	return []byte(s), nil
+	return result, nil
 }

lib/envtemplate/envtemplate_test.go

@@ -1,22 +1,70 @@
 package envtemplate
 
 import (
-	"os"
+	"reflect"
+	"sort"
 	"testing"
 )
 
+func TestExpandTemplates(t *testing.T) {
+	f := func(envs, resultExpected []string) {
+		t.Helper()
+		m := parseEnvVars(envs)
+		mExpanded := expandTemplates(m)
+		result := make([]string, 0, len(mExpanded))
+		for k, v := range mExpanded {
+			result = append(result, k+"="+v)
+		}
+		sort.Strings(result)
+		if !reflect.DeepEqual(result, resultExpected) {
+			t.Fatalf("unexpected result;\ngot\n%q\nwant\n%q", result, resultExpected)
+		}
+	}
+	f(nil, []string{})
+	f([]string{"foo=%{bar}", "bar=x"}, []string{"bar=x", "foo=x"})
+	f([]string{"a=x%{b}", "b=y%{c}z%{d}", "c=123", "d=qwe"}, []string{"a=xy123zqwe", "b=y123zqwe", "c=123", "d=qwe"})
+	f([]string{"a=x%{b}y", "b=z%{a}q", "c"}, []string{"a=xzxzxzxz%{a}qyqyqyqy", "b=zxzxzxzx%{b}yqyqyqyq", "c="})
+}
+
+func TestLookupEnv(t *testing.T) {
+	envVars = map[string]string{
+		"foo": "bar",
+	}
+	result, ok := LookupEnv("foo")
+	if result != "bar" {
+		t.Fatalf("unexpected result; got %q; want %q", result, "bar")
+	}
+	if !ok {
+		t.Fatalf("unexpected ok=false")
+	}
+	result, ok = LookupEnv("bar")
+	if result != "" {
+		t.Fatalf("unexpected non-empty result: %q", result)
+	}
+	if ok {
+		t.Fatalf("unexpected ok=true")
+	}
+}
+
 func TestReplaceSuccess(t *testing.T) {
-	if err := os.Setenv("foo", "bar"); err != nil {
+	envVars = map[string]string{
-		t.Fatalf("cannot set env var: %s", err)
+		"foo": "bar",
 	}
 	f := func(s, resultExpected string) {
 		t.Helper()
-		result, err := Replace([]byte(s))
+		result, err := ReplaceBytes([]byte(s))
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if string(result) != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%q\nwant\n%q", result, resultExpected)
+			t.Fatalf("unexpected result for ReplaceBytes(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected)
+		}
+		resultS, err := ReplaceString(s)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if resultS != resultExpected {
+			t.Fatalf("unexpected result for ReplaceString(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected)
 		}
 	}
 	f("", "")
@@ -27,9 +75,11 @@ func TestReplaceSuccess(t *testing.T) {
 func TestReplaceFailure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
-		_, err := Replace([]byte(s))
+		if _, err := ReplaceBytes([]byte(s)); err == nil {
-		if err == nil {
+			t.Fatalf("expecting non-nil error for ReplaceBytes(%q)", s)
-			t.Fatalf("expecting non-nil error")
+		}
+		if _, err := ReplaceString(s); err == nil {
+			t.Fatalf("expecting non-nil error for ReplaceString(%q)", s)
 		}
 	}
 	f("foo %{bar} %{baz}")

lib/promrelabel/config.go

@@ -152,7 +152,7 @@ func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err)
 	}
-	data, err = envtemplate.Replace(data)
+	data, err = envtemplate.ReplaceBytes(data)
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars at %q: %w", path, err)
 	}

lib/promscrape/config.go

@@ -93,7 +93,7 @@ type Config struct {
 
 func (cfg *Config) unmarshal(data []byte, isStrict bool) error {
 	var err error
-	data, err = envtemplate.Replace(data)
+	data, err = envtemplate.ReplaceBytes(data)
 	if err != nil {
 		return fmt.Errorf("cannot expand environment variables: %w", err)
 	}
@@ -375,7 +375,7 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot read `static_configs` from %q: %w", path, err)
 	}
-	data, err = envtemplate.Replace(data)
+	data, err = envtemplate.ReplaceBytes(data)
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err)
 	}
@@ -419,7 +419,7 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]*Scrap
 			if err != nil {
 				return nil, nil, fmt.Errorf("cannot load %q: %w", path, err)
 			}
-			data, err = envtemplate.Replace(data)
+			data, err = envtemplate.ReplaceBytes(data)
 			if err != nil {
 				return nil, nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err)
 			}