diff --git a/README.md b/README.md index b5ffac1c5..431f1f6af 100644 --- a/README.md +++ b/README.md @@ -132,10 +132,13 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c ### Environment variables -All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax. +All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax. For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. -This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself. +This expansion is performed by VictoriaMetrics itself. + +VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup. +For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`. Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index d0a2ef68a..e9f96a64f 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -245,7 +245,7 @@ func parseFile(path string) ([]Group, error) { if err != nil { return nil, fmt.Errorf("error reading alert rule file %q: %w", path, err) } - data, err = envtemplate.Replace(data) + data, err = envtemplate.ReplaceBytes(data) if err != nil { return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err) } diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 46eee7524..4c7fa1527 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -251,7 +251,7 @@ func readAuthConfig(path string) (map[string]*UserInfo, error) { func parseAuthConfig(data []byte) (map[string]*UserInfo, error) { var err error - data, err = envtemplate.Replace(data) + data, err = envtemplate.ReplaceBytes(data) if err != nil { return nil, fmt.Errorf("cannot expand environment vars: %w", err) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d9a116242..ae7c585cc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -20,7 +20,8 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [VictoriaMetric enterprise](https://docs.victoriametrics.com/enterprise.html): allow configuring multiple retentions for distinct sets of time series. See [these docs](https://docs.victoriametrics.com/#retention-filters), [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/143) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/289) feature request. * FEATURE: [VictoriaMetric cluster enterprise](https://docs.victoriametrics.com/enterprise.html): add support for multiple retentions for distinct tenants - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#retention-filters) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/143) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/289) feature request. * FEATURE: allow limiting memory usage on a per-query basis with `-search.maxMemoryPerQuery` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3203). -* FEATURE: allow using environment variables inside command-line flags passed to all VictoriaMetrics components. For example, if `AUTH_KEY=top-secret` environment variable is set, then `-metricsAuthKey=%{AUTH_KEY}` command-line flag is automatically expanded to `-storageDataPath=top-secret` at VictoriaMetrics startup. See [these docs](https://docs.victoriametrics.com/#environment-variables) for details. +* FEATURE: allow referring environment variables inside command-line flags via `%{ENV_VAR}` syntax. For example, if `AUTH_KEY=top-secret` environment variable is set, then `-metricsAuthKey=%{AUTH_KEY}` command-line flag is automatically expanded to `-storageDataPath=top-secret` at VictoriaMetrics startup. See [these docs](https://docs.victoriametrics.com/#environment-variables) for details. +* FEATURE: allow referring environment variables inside other environment variables via `%{ENV_VAR}` syntax. For example, if `A=a-%{B}`, `B=b-%{C}` and 'C=c` env vars are set, then VictoriaMetrics components automatically expand them to `A=a-b-c`, `B=b-c` and `C=c` on startup. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): drop all the labels with `__` prefix from discovered targets in the same way as Prometheus does according to [this article](https://www.robustperception.io/life-of-a-label/). Previously the following labels were available during [metric-level relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs): `__address__`, `__scheme__`, `__metrics_path__`, `__scrape_interval__`, `__scrape_timeout__`, `__param_*`. Now these labels are available only during [target-level relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config). This should reduce CPU usage and memory usage for `vmagent` setups, which scrape big number of targets. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): improve the performance for metric-level [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling), which can be applied via `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs), via `-remoteWrite.relabelConfig` or via `-remoteWrite.urlRelabelConfig` command-line options. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying full url in scrape target addresses (aka `__address__` label). This makes valid the following `-promscrape.config`: diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 2f316ab9e..310fe8a08 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -184,10 +184,13 @@ It is possible manualy setting up a toy cluster on a single host. In this case e ### Environment variables -All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax. +All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax. For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. -This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself. +This expansion is performed by VictoriaMetrics itself. + +VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup. +For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`. Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: diff --git a/docs/README.md b/docs/README.md index 489e73c3f..cc2ef665e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -133,10 +133,13 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c ### Environment variables -All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax. +All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax. For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. -This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself. +This expansion is performed by VictoriaMetrics itself. + +VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup. +For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`. Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index c801869b1..af101cd53 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -136,10 +136,13 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c ### Environment variables -All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax. +All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax. For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. -This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself. +This expansion is performed by VictoriaMetrics itself. + +VictoriaMetrics recursively expands `%{ENV_VAR}` references in environment variables on startup. +For example, `FOO=%{BAR}` environment variable is expanded to `FOO=abc` if `BAR=a%{BAZ}` and `BAZ=bc`. Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: diff --git a/lib/backup/azremote/azblob.go b/lib/backup/azremote/azblob.go index 11b02f911..b17f2acd8 100644 --- a/lib/backup/azremote/azblob.go +++ b/lib/backup/azremote/azblob.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "os" "strings" "time" @@ -19,6 +18,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common" "github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fscommon" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) @@ -59,15 +59,15 @@ func (fs *FS) Init() error { var sc *service.Client var err error - if cs, ok := os.LookupEnv(envStorageAccCs); ok { + if cs, ok := envtemplate.LookupEnv(envStorageAccCs); ok { sc, err = service.NewClientFromConnectionString(cs, nil) if err != nil { return fmt.Errorf("failed to create AZBlob service client from connection string: %w", err) } } - accountName, ok1 := os.LookupEnv(envStorageAcctName) - accountKey, ok2 := os.LookupEnv(envStorageAccKey) + accountName, ok1 := envtemplate.LookupEnv(envStorageAcctName) + accountKey, ok2 := envtemplate.LookupEnv(envStorageAccKey) if ok1 && ok2 { creds, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { diff --git a/lib/envflag/envflag.go b/lib/envflag/envflag.go index 6b333c8c7..1430ca8b4 100644 --- a/lib/envflag/envflag.go +++ b/lib/envflag/envflag.go @@ -26,12 +26,13 @@ func Parse() { args := os.Args[1:] dstArgs := args[:0] for _, arg := range args { - b, err := envtemplate.Replace([]byte(arg)) + s, err := envtemplate.ReplaceString(arg) if err != nil { + // Do not use lib/logger here, since it is uninitialized yet. log.Fatalf("cannot process arg %q: %s", arg, err) } - if len(b) > 0 { - dstArgs = append(dstArgs, string(b)) + if len(s) > 0 { + dstArgs = append(dstArgs, s) } } os.Args = os.Args[:1+len(dstArgs)] @@ -56,7 +57,7 @@ func Parse() { } // Get flag value from environment var. fname := getEnvFlagName(f.Name) - if v, ok := os.LookupEnv(fname); ok { + if v, ok := envtemplate.LookupEnv(fname); ok { if err := flag.Set(f.Name, v); err != nil { // Do not use lib/logger here, since it is uninitialized yet. log.Fatalf("cannot set flag %s to %q, which is read from environment variable %q: %s", f.Name, v, fname, err) diff --git a/lib/envtemplate/envtemplate.go b/lib/envtemplate/envtemplate.go index 14a84a90c..9d06435bf 100644 --- a/lib/envtemplate/envtemplate.go +++ b/lib/envtemplate/envtemplate.go @@ -1,31 +1,106 @@ package envtemplate import ( - "bytes" "fmt" "io" + "log" "os" + "strings" "github.com/valyala/fasttemplate" ) -// Replace replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values. +// ReplaceBytes replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values. // // Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder. -func Replace(b []byte) ([]byte, error) { - if !bytes.Contains(b, []byte("%{")) { - // Fast path - nothing to replace. - return b, nil +func ReplaceBytes(b []byte) ([]byte, error) { + result, err := expand(envVars, string(b)) + if err != nil { + return nil, err } - s, err := fasttemplate.ExecuteFuncStringWithErr(string(b), "%{", "}", func(w io.Writer, tag string) (int, error) { - v, ok := os.LookupEnv(tag) + return []byte(result), nil +} + +// ReplaceString replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values. +// +// Error is returned if ENV_VAR isn't set for some `%{ENV_VAR}` placeholder. +func ReplaceString(s string) (string, error) { + result, err := expand(envVars, s) + if err != nil { + return "", err + } + return result, nil +} + +// LookupEnv returns the expanded environment variable value for the given name. +// +// The expanded means that `%{ENV_VAR}` placeholders in env var value are replaced +// with the corresponding ENV_VAR values (recursively). +// +// false is returned if environment variable isn't found. +func LookupEnv(name string) (string, bool) { + value, ok := envVars[name] + return value, ok +} + +var envVars = func() map[string]string { + envs := os.Environ() + m := parseEnvVars(envs) + return expandTemplates(m) +}() + +func parseEnvVars(envs []string) map[string]string { + m := make(map[string]string, len(envs)) + for _, env := range envs { + n := strings.IndexByte(env, '=') + if n < 0 { + m[env] = "" + continue + } + name := env[:n] + value := env[n+1:] + m[name] = value + } + return m +} + +func expandTemplates(m map[string]string) map[string]string { + for i := 0; i < len(m); i++ { + mExpanded := make(map[string]string, len(m)) + expands := 0 + for name, value := range m { + valueExpanded, err := expand(m, value) + if err != nil { + // Do not use lib/logger here, since it is uninitialized yet. + log.Fatalf("cannot expand %q env var value %q: %s", name, value, err) + } + mExpanded[name] = valueExpanded + if valueExpanded != value { + expands++ + } + } + if expands == 0 { + return mExpanded + } + m = mExpanded + } + return m +} + +func expand(m map[string]string, s string) (string, error) { + if !strings.Contains(s, "%{") { + // Fast path - nothing to expand + return s, nil + } + result, err := fasttemplate.ExecuteFuncStringWithErr(s, "%{", "}", func(w io.Writer, tag string) (int, error) { + v, ok := m[tag] if !ok { - return 0, fmt.Errorf("missing %q environment variable", tag) + return 0, fmt.Errorf("missing %q env var", tag) } return w.Write([]byte(v)) }) if err != nil { - return nil, err + return "", err } - return []byte(s), nil + return result, nil } diff --git a/lib/envtemplate/envtemplate_test.go b/lib/envtemplate/envtemplate_test.go index db899c80d..375ece944 100644 --- a/lib/envtemplate/envtemplate_test.go +++ b/lib/envtemplate/envtemplate_test.go @@ -1,22 +1,70 @@ package envtemplate import ( - "os" + "reflect" + "sort" "testing" ) +func TestExpandTemplates(t *testing.T) { + f := func(envs, resultExpected []string) { + t.Helper() + m := parseEnvVars(envs) + mExpanded := expandTemplates(m) + result := make([]string, 0, len(mExpanded)) + for k, v := range mExpanded { + result = append(result, k+"="+v) + } + sort.Strings(result) + if !reflect.DeepEqual(result, resultExpected) { + t.Fatalf("unexpected result;\ngot\n%q\nwant\n%q", result, resultExpected) + } + } + f(nil, []string{}) + f([]string{"foo=%{bar}", "bar=x"}, []string{"bar=x", "foo=x"}) + f([]string{"a=x%{b}", "b=y%{c}z%{d}", "c=123", "d=qwe"}, []string{"a=xy123zqwe", "b=y123zqwe", "c=123", "d=qwe"}) + f([]string{"a=x%{b}y", "b=z%{a}q", "c"}, []string{"a=xzxzxzxz%{a}qyqyqyqy", "b=zxzxzxzx%{b}yqyqyqyq", "c="}) +} + +func TestLookupEnv(t *testing.T) { + envVars = map[string]string{ + "foo": "bar", + } + result, ok := LookupEnv("foo") + if result != "bar" { + t.Fatalf("unexpected result; got %q; want %q", result, "bar") + } + if !ok { + t.Fatalf("unexpected ok=false") + } + result, ok = LookupEnv("bar") + if result != "" { + t.Fatalf("unexpected non-empty result: %q", result) + } + if ok { + t.Fatalf("unexpected ok=true") + } +} + func TestReplaceSuccess(t *testing.T) { - if err := os.Setenv("foo", "bar"); err != nil { - t.Fatalf("cannot set env var: %s", err) + envVars = map[string]string{ + "foo": "bar", } f := func(s, resultExpected string) { t.Helper() - result, err := Replace([]byte(s)) + result, err := ReplaceBytes([]byte(s)) if err != nil { t.Fatalf("unexpected error: %s", err) } if string(result) != resultExpected { - t.Fatalf("unexpected result;\ngot\n%q\nwant\n%q", result, resultExpected) + t.Fatalf("unexpected result for ReplaceBytes(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected) + } + resultS, err := ReplaceString(s) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if resultS != resultExpected { + t.Fatalf("unexpected result for ReplaceString(%q);\ngot\n%q\nwant\n%q", s, result, resultExpected) } } f("", "") @@ -27,9 +75,11 @@ func TestReplaceSuccess(t *testing.T) { func TestReplaceFailure(t *testing.T) { f := func(s string) { t.Helper() - _, err := Replace([]byte(s)) - if err == nil { - t.Fatalf("expecting non-nil error") + if _, err := ReplaceBytes([]byte(s)); err == nil { + t.Fatalf("expecting non-nil error for ReplaceBytes(%q)", s) + } + if _, err := ReplaceString(s); err == nil { + t.Fatalf("expecting non-nil error for ReplaceString(%q)", s) } } f("foo %{bar} %{baz}") diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 16bf3e9e9..11553ed43 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -152,7 +152,7 @@ func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error) if err != nil { return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err) } - data, err = envtemplate.Replace(data) + data, err = envtemplate.ReplaceBytes(data) if err != nil { return nil, fmt.Errorf("cannot expand environment vars at %q: %w", path, err) } diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 4f7cc0351..0a5bf2c4a 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -93,7 +93,7 @@ type Config struct { func (cfg *Config) unmarshal(data []byte, isStrict bool) error { var err error - data, err = envtemplate.Replace(data) + data, err = envtemplate.ReplaceBytes(data) if err != nil { return fmt.Errorf("cannot expand environment variables: %w", err) } @@ -375,7 +375,7 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) { if err != nil { return nil, fmt.Errorf("cannot read `static_configs` from %q: %w", path, err) } - data, err = envtemplate.Replace(data) + data, err = envtemplate.ReplaceBytes(data) if err != nil { return nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err) } @@ -419,7 +419,7 @@ func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]*Scrap if err != nil { return nil, nil, fmt.Errorf("cannot load %q: %w", path, err) } - data, err = envtemplate.Replace(data) + data, err = envtemplate.ReplaceBytes(data) if err != nil { return nil, nil, fmt.Errorf("cannot expand environment vars in %q: %w", path, err) }