From c402265e883dac8045c76f8a6232540d47fa496a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 13 Aug 2020 16:43:55 +0300 Subject: [PATCH] all: support `%{ENV_VAR}` placeholders in yaml configs in all the vm* components Such placeholders are substituted by the corresponding environment variable values. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/583 --- README.md | 4 +- app/vmagent/README.md | 2 + app/vmalert/config/config.go | 2 + app/vmalert/main.go | 3 +- app/vmauth/README.md | 3 + app/vmauth/auth_config.go | 2 + docs/Single-server-VictoriaMetrics.md | 4 +- docs/vmagent.md | 2 + docs/vmauth.md | 3 + go.mod | 1 + go.sum | 2 + lib/envtemplate/envtemplate.go | 25 + lib/envtemplate/envtemplate_test.go | 19 + lib/promrelabel/config.go | 2 + lib/promscrape/config.go | 3 + lib/promscrape/scraper.go | 2 +- .../github.com/valyala/fasttemplate/LICENSE | 22 + .../github.com/valyala/fasttemplate/README.md | 85 ++++ vendor/github.com/valyala/fasttemplate/go.mod | 5 + vendor/github.com/valyala/fasttemplate/go.sum | 2 + .../valyala/fasttemplate/template.go | 437 ++++++++++++++++++ .../github.com/valyala/fasttemplate/unsafe.go | 21 + .../valyala/fasttemplate/unsafe_gae.go | 11 + vendor/modules.txt | 2 + 24 files changed, 660 insertions(+), 4 deletions(-) create mode 100644 lib/envtemplate/envtemplate.go create mode 100644 lib/envtemplate/envtemplate_test.go create mode 100644 vendor/github.com/valyala/fasttemplate/LICENSE create mode 100644 vendor/github.com/valyala/fasttemplate/README.md create mode 100644 vendor/github.com/valyala/fasttemplate/go.mod create mode 100644 vendor/github.com/valyala/fasttemplate/go.sum create mode 100644 vendor/github.com/valyala/fasttemplate/template.go create mode 100644 vendor/github.com/valyala/fasttemplate/unsafe.go create mode 100644 vendor/github.com/valyala/fasttemplate/unsafe_gae.go diff --git a/README.md b/README.md index 172a5229e..9c173e86f 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la In the future other `*_sd_config` types will be supported. +The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. + VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format). See also [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md), which can be used as drop-in replacement for Prometheus. @@ -768,7 +770,7 @@ and importing them concurrently. Note that the original file must be split on ne VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries. -Additionally VictoriaMetrics provides the following extra actions for relabeling rules: +VictoriaMetrics provides the following extra actions for relabeling rules: * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`. * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`. diff --git a/app/vmagent/README.md b/app/vmagent/README.md index d17ce5c54..f7c79b227 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -161,6 +161,8 @@ Note that `vmagent` doesn't support `refresh_interval` option these scrape confi command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs` entries to 60s. Run `vmagent -help` in order to see default values for `-promscrape.*CheckInterval` flags. +The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. + ### Adding labels to metrics diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 3d2ce6dfd..33864f64b 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -10,6 +10,7 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" "github.com/VictoriaMetrics/metricsql" "gopkg.in/yaml.v2" ) @@ -179,6 +180,7 @@ func parseFile(path string) ([]Group, error) { if err != nil { return nil, fmt.Errorf("error reading alert rule file: %w", err) } + data = envtemplate.Replace(data) g := struct { Groups []Group `yaml:"groups"` // Catches all undefined fields and must be empty after parsing. diff --git a/app/vmalert/main.go b/app/vmalert/main.go index a57361136..44edc2ef4 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -31,7 +31,8 @@ Supports patterns. Flag can be specified multiple times. Examples: -rule /path/to/file. Path to a single file with alerting rules -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, -absolute path to all .yaml files in root.`) +absolute path to all .yaml files in root. +Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.`) httpListenAddr = flag.String("httpListenAddr", ":8880", "Address to listen for http connections") evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules") diff --git a/app/vmauth/README.md b/app/vmauth/README.md index 8d8ec7f96..74b921692 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -64,6 +64,9 @@ users: url_prefix: "http://vminsert:8480/insert/42/prometheus" ``` +The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. +This may be useful for passing secrets to the config. + ### Security diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 02dc96b4b..85832719f 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -9,6 +9,7 @@ import ( "sync" "sync/atomic" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" "github.com/VictoriaMetrics/metrics" @@ -93,6 +94,7 @@ func readAuthConfig(path string) (map[string]*UserInfo, error) { } func parseAuthConfig(data []byte) (map[string]*UserInfo, error) { + data = envtemplate.Replace(data) var ac AuthConfig if err := yaml.UnmarshalStrict(data, &ac); err != nil { return nil, fmt.Errorf("cannot unmarshal AuthConfig data: %w", err) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 172a5229e..9c173e86f 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -288,6 +288,8 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la In the future other `*_sd_config` types will be supported. +The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. + VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format). See also [vmagent](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/README.md), which can be used as drop-in replacement for Prometheus. @@ -768,7 +770,7 @@ and importing them concurrently. Note that the original file must be split on ne VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points to a file containing a list of [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) entries. -Additionally VictoriaMetrics provides the following extra actions for relabeling rules: +VictoriaMetrics provides the following extra actions for relabeling rules: * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`. * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`. diff --git a/docs/vmagent.md b/docs/vmagent.md index d17ce5c54..f7c79b227 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -161,6 +161,8 @@ Note that `vmagent` doesn't support `refresh_interval` option these scrape confi command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs` entries to 60s. Run `vmagent -help` in order to see default values for `-promscrape.*CheckInterval` flags. +The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. + ### Adding labels to metrics diff --git a/docs/vmauth.md b/docs/vmauth.md index 8d8ec7f96..74b921692 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -64,6 +64,9 @@ users: url_prefix: "http://vminsert:8480/insert/42/prometheus" ``` +The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. +This may be useful for passing secrets to the config. + ### Security diff --git a/go.mod b/go.mod index 93c13fd9d..f68993f85 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/klauspost/compress v1.10.10 github.com/valyala/fastjson v1.5.4 github.com/valyala/fastrand v1.0.0 + github.com/valyala/fasttemplate v1.2.1 github.com/valyala/gozstd v1.7.0 github.com/valyala/histogram v1.1.2 github.com/valyala/quicktemplate v1.6.2 diff --git a/go.sum b/go.sum index 2b0b11fa8..c05b6749a 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,8 @@ github.com/valyala/fastjson v1.5.4 h1:r8gpiVwdzDU09NrlN38OyL5dUFpdwGQR5RQEBqY+hL github.com/valyala/fastjson v1.5.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ= github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1h8= diff --git a/lib/envtemplate/envtemplate.go b/lib/envtemplate/envtemplate.go new file mode 100644 index 000000000..c92eb827f --- /dev/null +++ b/lib/envtemplate/envtemplate.go @@ -0,0 +1,25 @@ +package envtemplate + +import ( + "bytes" + "io" + "os" + + "github.com/valyala/fasttemplate" +) + +// Replace replaces `%{ENV_VAR}` placeholders in b with the corresponding ENV_VAR values. +func Replace(b []byte) []byte { + if !bytes.Contains(b, []byte("%{")) { + // Fast path - nothing to replace. + return b + } + s := fasttemplate.ExecuteFuncString(string(b), "%{", "}", func(w io.Writer, tag string) (int, error) { + v := os.Getenv(tag) + if v == "" { + v = "%{" + tag + "}" + } + return w.Write([]byte(v)) + }) + return []byte(s) +} diff --git a/lib/envtemplate/envtemplate_test.go b/lib/envtemplate/envtemplate_test.go new file mode 100644 index 000000000..4ba4633cd --- /dev/null +++ b/lib/envtemplate/envtemplate_test.go @@ -0,0 +1,19 @@ +package envtemplate + +import ( + "testing" +) + +func TestReplace(t *testing.T) { + f := func(s, resultExpected string) { + t.Helper() + result := Replace([]byte(s)) + if string(result) != resultExpected { + t.Fatalf("unexpected result;\ngot\n%q\nwant\n%q", result, resultExpected) + } + } + f("", "") + f("foo", "foo") + f("%{foo}", "%{foo}") + f("foo %{bar} %{baz}", "foo %{bar} %{baz}") +} diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 8904ee033..911eea2a3 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" "gopkg.in/yaml.v2" ) @@ -28,6 +29,7 @@ func LoadRelabelConfigs(path string) ([]ParsedRelabelConfig, error) { if err != nil { return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err) } + data = envtemplate.Replace(data) var rcs []RelabelConfig if err := yaml.UnmarshalStrict(data, &rcs); err != nil { return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err) diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 7b72d09c6..1ce083c65 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -11,6 +11,7 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -105,6 +106,7 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) { if err != nil { return nil, fmt.Errorf("cannot read `static_configs` from %q: %w", path, err) } + data = envtemplate.Replace(data) var stcs []StaticConfig if err := yaml.UnmarshalStrict(data, &stcs); err != nil { return nil, fmt.Errorf("cannot unmarshal `static_configs` from %q: %w", path, err) @@ -153,6 +155,7 @@ func (cfg *Config) parse(data []byte, path string) error { } func unmarshalMaybeStrict(data []byte, dst interface{}) error { + data = envtemplate.Replace(data) var err error if *strictParse || *dryRun { err = yaml.UnmarshalStrict(data, dst) diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index 37c64b330..189f6f0f3 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -34,7 +34,7 @@ var ( "This works only if `gce_sd_configs` is configured in '-promscrape.config' file. "+ "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details") promscrapeConfigFile = flag.String("promscrape.config", "", "Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. "+ - "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config for details") + "See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details") ) // CheckConfig checks -promscrape.config for errors and unsupported options. diff --git a/vendor/github.com/valyala/fasttemplate/LICENSE b/vendor/github.com/valyala/fasttemplate/LICENSE new file mode 100644 index 000000000..7125a63c4 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Aliaksandr Valialkin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/vendor/github.com/valyala/fasttemplate/README.md b/vendor/github.com/valyala/fasttemplate/README.md new file mode 100644 index 000000000..2839ed0f7 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/README.md @@ -0,0 +1,85 @@ +fasttemplate +============ + +Simple and fast template engine for Go. + +Fasttemplate performs only a single task - it substitutes template placeholders +with user-defined values. At high speed :) + +Take a look at [quicktemplate](https://github.com/valyala/quicktemplate) if you need fast yet powerful html template engine. + +*Please note that fasttemplate doesn't do any escaping on template values +unlike [html/template](http://golang.org/pkg/html/template/) do. So values +must be properly escaped before passing them to fasttemplate.* + +Fasttemplate is faster than [text/template](http://golang.org/pkg/text/template/), +[strings.Replace](http://golang.org/pkg/strings/#Replace), +[strings.Replacer](http://golang.org/pkg/strings/#Replacer) +and [fmt.Fprintf](https://golang.org/pkg/fmt/#Fprintf) on placeholders' substitution. + +Below are benchmark results comparing fasttemplate performance to text/template, +strings.Replace, strings.Replacer and fmt.Fprintf: + +``` +$ go test -bench=. -benchmem +PASS +BenchmarkFmtFprintf-4 2000000 790 ns/op 0 B/op 0 allocs/op +BenchmarkStringsReplace-4 500000 3474 ns/op 2112 B/op 14 allocs/op +BenchmarkStringsReplacer-4 500000 2657 ns/op 2256 B/op 23 allocs/op +BenchmarkTextTemplate-4 500000 3333 ns/op 336 B/op 19 allocs/op +BenchmarkFastTemplateExecuteFunc-4 5000000 349 ns/op 0 B/op 0 allocs/op +BenchmarkFastTemplateExecute-4 3000000 383 ns/op 0 B/op 0 allocs/op +BenchmarkFastTemplateExecuteFuncString-4 3000000 549 ns/op 144 B/op 1 allocs/op +BenchmarkFastTemplateExecuteString-4 3000000 572 ns/op 144 B/op 1 allocs/op +BenchmarkFastTemplateExecuteTagFunc-4 2000000 743 ns/op 144 B/op 3 allocs/op +``` + + +Docs +==== + +See http://godoc.org/github.com/valyala/fasttemplate . + + +Usage +===== + +```go + template := "http://{{host}}/?q={{query}}&foo={{bar}}{{bar}}" + t := fasttemplate.New(template, "{{", "}}") + s := t.ExecuteString(map[string]interface{}{ + "host": "google.com", + "query": url.QueryEscape("hello=world"), + "bar": "foobar", + }) + fmt.Printf("%s", s) + + // Output: + // http://google.com/?q=hello%3Dworld&foo=foobarfoobar +``` + + +Advanced usage +============== + +```go + template := "Hello, [user]! You won [prize]!!! [foobar]" + t, err := fasttemplate.NewTemplate(template, "[", "]") + if err != nil { + log.Fatalf("unexpected error when parsing template: %s", err) + } + s := t.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + switch tag { + case "user": + return w.Write([]byte("John")) + case "prize": + return w.Write([]byte("$100500")) + default: + return w.Write([]byte(fmt.Sprintf("[unknown tag %q]", tag))) + } + }) + fmt.Printf("%s", s) + + // Output: + // Hello, John! You won $100500!!! [unknown tag "foobar"] +``` diff --git a/vendor/github.com/valyala/fasttemplate/go.mod b/vendor/github.com/valyala/fasttemplate/go.mod new file mode 100644 index 000000000..37a50e261 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/go.mod @@ -0,0 +1,5 @@ +module github.com/valyala/fasttemplate + +go 1.12 + +require github.com/valyala/bytebufferpool v1.0.0 diff --git a/vendor/github.com/valyala/fasttemplate/go.sum b/vendor/github.com/valyala/fasttemplate/go.sum new file mode 100644 index 000000000..c10c48c24 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/go.sum @@ -0,0 +1,2 @@ +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= diff --git a/vendor/github.com/valyala/fasttemplate/template.go b/vendor/github.com/valyala/fasttemplate/template.go new file mode 100644 index 000000000..186200134 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/template.go @@ -0,0 +1,437 @@ +// Package fasttemplate implements simple and fast template library. +// +// Fasttemplate is faster than text/template, strings.Replace +// and strings.Replacer. +// +// Fasttemplate ideally fits for fast and simple placeholders' substitutions. +package fasttemplate + +import ( + "bytes" + "fmt" + "io" + + "github.com/valyala/bytebufferpool" +) + +// ExecuteFunc calls f on each template tag (placeholder) occurrence. +// +// Returns the number of bytes written to w. +// +// This function is optimized for constantly changing templates. +// Use Template.ExecuteFunc for frozen templates. +func ExecuteFunc(template, startTag, endTag string, w io.Writer, f TagFunc) (int64, error) { + s := unsafeString2Bytes(template) + a := unsafeString2Bytes(startTag) + b := unsafeString2Bytes(endTag) + + var nn int64 + var ni int + var err error + for { + n := bytes.Index(s, a) + if n < 0 { + break + } + ni, err = w.Write(s[:n]) + nn += int64(ni) + if err != nil { + return nn, err + } + + s = s[n+len(a):] + n = bytes.Index(s, b) + if n < 0 { + // cannot find end tag - just write it to the output. + ni, _ = w.Write(a) + nn += int64(ni) + break + } + + ni, err = f(w, unsafeBytes2String(s[:n])) + nn += int64(ni) + if err != nil { + return nn, err + } + s = s[n+len(b):] + } + ni, err = w.Write(s) + nn += int64(ni) + + return nn, err +} + +// Execute substitutes template tags (placeholders) with the corresponding +// values from the map m and writes the result to the given writer w. +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// Returns the number of bytes written to w. +// +// This function is optimized for constantly changing templates. +// Use Template.Execute for frozen templates. +func Execute(template, startTag, endTag string, w io.Writer, m map[string]interface{}) (int64, error) { + return ExecuteFunc(template, startTag, endTag, w, func(w io.Writer, tag string) (int, error) { return stdTagFunc(w, tag, m) }) +} + +// ExecuteStd works the same way as Execute, but keeps the unknown placeholders. +// This can be used as a drop-in replacement for strings.Replacer +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// Returns the number of bytes written to w. +// +// This function is optimized for constantly changing templates. +// Use Template.ExecuteStd for frozen templates. +func ExecuteStd(template, startTag, endTag string, w io.Writer, m map[string]interface{}) (int64, error) { + return ExecuteFunc(template, startTag, endTag, w, func(w io.Writer, tag string) (int, error) { return keepUnknownTagFunc(w, startTag, endTag, tag, m) }) +} + +// ExecuteFuncString calls f on each template tag (placeholder) occurrence +// and substitutes it with the data written to TagFunc's w. +// +// Returns the resulting string. +// +// This function is optimized for constantly changing templates. +// Use Template.ExecuteFuncString for frozen templates. +func ExecuteFuncString(template, startTag, endTag string, f TagFunc) string { + s, err := ExecuteFuncStringWithErr(template, startTag, endTag, f) + if err != nil { + panic(fmt.Sprintf("unexpected error: %s", err)) + } + return s +} + +// ExecuteFuncStringWithErr is nearly the same as ExecuteFuncString +// but when f returns an error, ExecuteFuncStringWithErr won't panic like ExecuteFuncString +// it just returns an empty string and the error f returned +func ExecuteFuncStringWithErr(template, startTag, endTag string, f TagFunc) (string, error) { + tagsCount := bytes.Count(unsafeString2Bytes(template), unsafeString2Bytes(startTag)) + if tagsCount == 0 { + return template, nil + } + + bb := byteBufferPool.Get() + if _, err := ExecuteFunc(template, startTag, endTag, bb, f); err != nil { + bb.Reset() + byteBufferPool.Put(bb) + return "", err + } + s := string(bb.B) + bb.Reset() + byteBufferPool.Put(bb) + return s, nil +} + +var byteBufferPool bytebufferpool.Pool + +// ExecuteString substitutes template tags (placeholders) with the corresponding +// values from the map m and returns the result. +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// This function is optimized for constantly changing templates. +// Use Template.ExecuteString for frozen templates. +func ExecuteString(template, startTag, endTag string, m map[string]interface{}) string { + return ExecuteFuncString(template, startTag, endTag, func(w io.Writer, tag string) (int, error) { return stdTagFunc(w, tag, m) }) +} + +// ExecuteStringStd works the same way as ExecuteString, but keeps the unknown placeholders. +// This can be used as a drop-in replacement for strings.Replacer +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// This function is optimized for constantly changing templates. +// Use Template.ExecuteStringStd for frozen templates. +func ExecuteStringStd(template, startTag, endTag string, m map[string]interface{}) string { + return ExecuteFuncString(template, startTag, endTag, func(w io.Writer, tag string) (int, error) { return keepUnknownTagFunc(w, startTag, endTag, tag, m) }) +} + +// Template implements simple template engine, which can be used for fast +// tags' (aka placeholders) substitution. +type Template struct { + template string + startTag string + endTag string + + texts [][]byte + tags []string + byteBufferPool bytebufferpool.Pool +} + +// New parses the given template using the given startTag and endTag +// as tag start and tag end. +// +// The returned template can be executed by concurrently running goroutines +// using Execute* methods. +// +// New panics if the given template cannot be parsed. Use NewTemplate instead +// if template may contain errors. +func New(template, startTag, endTag string) *Template { + t, err := NewTemplate(template, startTag, endTag) + if err != nil { + panic(err) + } + return t +} + +// NewTemplate parses the given template using the given startTag and endTag +// as tag start and tag end. +// +// The returned template can be executed by concurrently running goroutines +// using Execute* methods. +func NewTemplate(template, startTag, endTag string) (*Template, error) { + var t Template + err := t.Reset(template, startTag, endTag) + if err != nil { + return nil, err + } + return &t, nil +} + +// TagFunc can be used as a substitution value in the map passed to Execute*. +// Execute* functions pass tag (placeholder) name in 'tag' argument. +// +// TagFunc must be safe to call from concurrently running goroutines. +// +// TagFunc must write contents to w and return the number of bytes written. +type TagFunc func(w io.Writer, tag string) (int, error) + +// Reset resets the template t to new one defined by +// template, startTag and endTag. +// +// Reset allows Template object re-use. +// +// Reset may be called only if no other goroutines call t methods at the moment. +func (t *Template) Reset(template, startTag, endTag string) error { + // Keep these vars in t, so GC won't collect them and won't break + // vars derived via unsafe* + t.template = template + t.startTag = startTag + t.endTag = endTag + t.texts = t.texts[:0] + t.tags = t.tags[:0] + + if len(startTag) == 0 { + panic("startTag cannot be empty") + } + if len(endTag) == 0 { + panic("endTag cannot be empty") + } + + s := unsafeString2Bytes(template) + a := unsafeString2Bytes(startTag) + b := unsafeString2Bytes(endTag) + + tagsCount := bytes.Count(s, a) + if tagsCount == 0 { + return nil + } + + if tagsCount+1 > cap(t.texts) { + t.texts = make([][]byte, 0, tagsCount+1) + } + if tagsCount > cap(t.tags) { + t.tags = make([]string, 0, tagsCount) + } + + for { + n := bytes.Index(s, a) + if n < 0 { + t.texts = append(t.texts, s) + break + } + t.texts = append(t.texts, s[:n]) + + s = s[n+len(a):] + n = bytes.Index(s, b) + if n < 0 { + return fmt.Errorf("Cannot find end tag=%q in the template=%q starting from %q", endTag, template, s) + } + + t.tags = append(t.tags, unsafeBytes2String(s[:n])) + s = s[n+len(b):] + } + + return nil +} + +// ExecuteFunc calls f on each template tag (placeholder) occurrence. +// +// Returns the number of bytes written to w. +// +// This function is optimized for frozen templates. +// Use ExecuteFunc for constantly changing templates. +func (t *Template) ExecuteFunc(w io.Writer, f TagFunc) (int64, error) { + var nn int64 + + n := len(t.texts) - 1 + if n == -1 { + ni, err := w.Write(unsafeString2Bytes(t.template)) + return int64(ni), err + } + + for i := 0; i < n; i++ { + ni, err := w.Write(t.texts[i]) + nn += int64(ni) + if err != nil { + return nn, err + } + + ni, err = f(w, t.tags[i]) + nn += int64(ni) + if err != nil { + return nn, err + } + } + ni, err := w.Write(t.texts[n]) + nn += int64(ni) + return nn, err +} + +// Execute substitutes template tags (placeholders) with the corresponding +// values from the map m and writes the result to the given writer w. +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// Returns the number of bytes written to w. +func (t *Template) Execute(w io.Writer, m map[string]interface{}) (int64, error) { + return t.ExecuteFunc(w, func(w io.Writer, tag string) (int, error) { return stdTagFunc(w, tag, m) }) +} + +// ExecuteStd works the same way as Execute, but keeps the unknown placeholders. +// This can be used as a drop-in replacement for strings.Replacer +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// Returns the number of bytes written to w. +func (t *Template) ExecuteStd(w io.Writer, m map[string]interface{}) (int64, error) { + return t.ExecuteFunc(w, func(w io.Writer, tag string) (int, error) { return keepUnknownTagFunc(w, t.startTag, t.endTag, tag, m) }) +} + +// ExecuteFuncString calls f on each template tag (placeholder) occurrence +// and substitutes it with the data written to TagFunc's w. +// +// Returns the resulting string. +// +// This function is optimized for frozen templates. +// Use ExecuteFuncString for constantly changing templates. +func (t *Template) ExecuteFuncString(f TagFunc) string { + s, err := t.ExecuteFuncStringWithErr(f) + if err != nil { + panic(fmt.Sprintf("unexpected error: %s", err)) + } + return s +} + +// ExecuteFuncStringWithErr calls f on each template tag (placeholder) occurrence +// and substitutes it with the data written to TagFunc's w. +// +// Returns the resulting string. +// +// This function is optimized for frozen templates. +// Use ExecuteFuncString for constantly changing templates. +func (t *Template) ExecuteFuncStringWithErr(f TagFunc) (string, error) { + bb := t.byteBufferPool.Get() + if _, err := t.ExecuteFunc(bb, f); err != nil { + bb.Reset() + t.byteBufferPool.Put(bb) + return "", err + } + s := string(bb.Bytes()) + bb.Reset() + t.byteBufferPool.Put(bb) + return s, nil +} + +// ExecuteString substitutes template tags (placeholders) with the corresponding +// values from the map m and returns the result. +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// This function is optimized for frozen templates. +// Use ExecuteString for constantly changing templates. +func (t *Template) ExecuteString(m map[string]interface{}) string { + return t.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { return stdTagFunc(w, tag, m) }) +} + +// ExecuteStringStd works the same way as ExecuteString, but keeps the unknown placeholders. +// This can be used as a drop-in replacement for strings.Replacer +// +// Substitution map m may contain values with the following types: +// * []byte - the fastest value type +// * string - convenient value type +// * TagFunc - flexible value type +// +// This function is optimized for frozen templates. +// Use ExecuteStringStd for constantly changing templates. +func (t *Template) ExecuteStringStd(m map[string]interface{}) string { + return t.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { return keepUnknownTagFunc(w, t.startTag, t.endTag, tag, m) }) +} + +func stdTagFunc(w io.Writer, tag string, m map[string]interface{}) (int, error) { + v := m[tag] + if v == nil { + return 0, nil + } + switch value := v.(type) { + case []byte: + return w.Write(value) + case string: + return w.Write([]byte(value)) + case TagFunc: + return value(w, tag) + default: + panic(fmt.Sprintf("tag=%q contains unexpected value type=%#v. Expected []byte, string or TagFunc", tag, v)) + } +} + +func keepUnknownTagFunc(w io.Writer, startTag, endTag, tag string, m map[string]interface{}) (int, error) { + v, ok := m[tag] + if !ok { + if _, err := w.Write(unsafeString2Bytes(startTag)); err != nil { + return 0, err + } + if _, err := w.Write(unsafeString2Bytes(tag)); err != nil { + return 0, err + } + if _, err := w.Write(unsafeString2Bytes(endTag)); err != nil { + return 0, err + } + return len(startTag) + len(tag) + len(endTag), nil + } + if v == nil { + return 0, nil + } + switch value := v.(type) { + case []byte: + return w.Write(value) + case string: + return w.Write([]byte(value)) + case TagFunc: + return value(w, tag) + default: + panic(fmt.Sprintf("tag=%q contains unexpected value type=%#v. Expected []byte, string or TagFunc", tag, v)) + } +} diff --git a/vendor/github.com/valyala/fasttemplate/unsafe.go b/vendor/github.com/valyala/fasttemplate/unsafe.go new file mode 100644 index 000000000..1020ca387 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/unsafe.go @@ -0,0 +1,21 @@ +// +build !appengine + +package fasttemplate + +import ( + "reflect" + "unsafe" +) + +func unsafeBytes2String(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +func unsafeString2Bytes(s string) (b []byte) { + sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh.Data = sh.Data + bh.Cap = sh.Len + bh.Len = sh.Len + return b +} diff --git a/vendor/github.com/valyala/fasttemplate/unsafe_gae.go b/vendor/github.com/valyala/fasttemplate/unsafe_gae.go new file mode 100644 index 000000000..cc4ce1516 --- /dev/null +++ b/vendor/github.com/valyala/fasttemplate/unsafe_gae.go @@ -0,0 +1,11 @@ +// +build appengine + +package fasttemplate + +func unsafeBytes2String(b []byte) string { + return string(b) +} + +func unsafeString2Bytes(s string) []byte { + return []byte(s) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 9548b12b8..51bfec8f6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -104,6 +104,8 @@ github.com/valyala/fastjson github.com/valyala/fastjson/fastfloat # github.com/valyala/fastrand v1.0.0 github.com/valyala/fastrand +# github.com/valyala/fasttemplate v1.2.1 +github.com/valyala/fasttemplate # github.com/valyala/gozstd v1.7.0 github.com/valyala/gozstd # github.com/valyala/histogram v1.1.2