VictoriaMetrics/app/vmalert/templates/template.go

// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package templates

import (
	"errors"
	"fmt"
	htmlTpl "html/template"
	"io/ioutil"
	"math"
	"net"
	"net/url"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	textTpl "text/template"

	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
)

// go template execution fails when it's tree is empty
const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`

var tplMu sync.RWMutex

type textTemplate struct {
	current     *textTpl.Template
	replacement *textTpl.Template
}

var masterTmpl textTemplate

func newTemplate() *textTpl.Template {
	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
	return textTpl.Must(tmpl.Parse(defaultTemplate))
}

// Load func loads templates from multiple globs specified in pathPatterns and either
// sets them directly to current template if it's undefined or with overwrite=true
// or sets replacement templates and adds templates with new names to a current
func Load(pathPatterns []string, overwrite bool) error {
	var err error
	tmpl := newTemplate()
	for _, tp := range pathPatterns {
		p, err := filepath.Glob(tp)
		if err != nil {
			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
		}
		if len(p) > 0 {
			tmpl, err = tmpl.ParseGlob(tp)
			if err != nil {
				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
			}
		}
	}
	if len(tmpl.Templates()) > 0 {
		err := tmpl.Execute(ioutil.Discard, nil)
		if err != nil {
			return fmt.Errorf("failed to execute template: %w", err)
		}
	}
	tplMu.Lock()
	defer tplMu.Unlock()
	if masterTmpl.current == nil || overwrite {
		masterTmpl.replacement = nil
		masterTmpl.current = newTemplate()
	} else {
		masterTmpl.replacement = newTemplate()
		if err = copyTemplates(tmpl, masterTmpl.replacement, overwrite); err != nil {
			return err
		}
	}
	return copyTemplates(tmpl, masterTmpl.current, overwrite)
}

func copyTemplates(from *textTpl.Template, to *textTpl.Template, overwrite bool) error {
	if from == nil {
		return nil
	}
	if to == nil {
		to = newTemplate()
	}
	tmpl, err := from.Clone()
	if err != nil {
		return err
	}
	for _, t := range tmpl.Templates() {
		if to.Lookup(t.Name()) == nil || overwrite {
			to, err = to.AddParseTree(t.Name(), t.Tree)
			if err != nil {
				return fmt.Errorf("failed to add template %q: %w", t.Name(), err)
			}
		}
	}
	return nil
}

// Reload func replaces current template with a replacement template
// which was set by Load with override=false
func Reload() {
	tplMu.Lock()
	defer tplMu.Unlock()
	if masterTmpl.replacement != nil {
		masterTmpl.current = masterTmpl.replacement
		masterTmpl.replacement = nil
	}
}

// metric is private copy of datasource.Metric,
// it is used for templating annotations,
// Labels as map simplifies templates evaluation.
type metric struct {
	Labels    map[string]string
	Timestamp int64
	Value     float64
}

// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
	mss := make([]metric, 0, len(ms))
	for _, m := range ms {
		labelsMap := make(map[string]string, len(m.Labels))
		for _, labelValue := range m.Labels {
			labelsMap[labelValue.Name] = labelValue.Value
		}
		mss = append(mss, metric{
			Labels:    labelsMap,
			Timestamp: m.Timestamps[0],
			Value:     m.Values[0]})
	}
	return mss
}

// QueryFn is used to wrap a call to datasource into simple-to-use function
// for templating functions.
type QueryFn func(query string) ([]datasource.Metric, error)

// UpdateWithFuncs updates existing or sets a new function map for a template
func UpdateWithFuncs(funcs textTpl.FuncMap) {
	tplMu.Lock()
	defer tplMu.Unlock()
	masterTmpl.current = masterTmpl.current.Funcs(funcs)
}

// GetWithFuncs returns a copy of current template with additional FuncMap
// provided with funcs argument
func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
	tplMu.RLock()
	defer tplMu.RUnlock()
	tmpl, err := masterTmpl.current.Clone()
	if err != nil {
		return nil, err
	}
	return tmpl.Funcs(funcs), nil
}

// Get returns a copy of a template
func Get() (*textTpl.Template, error) {
	tplMu.RLock()
	defer tplMu.RUnlock()
	return masterTmpl.current.Clone()
}

// FuncsWithQuery returns a function map that depends on metric data
func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
	return textTpl.FuncMap{
		"query": func(q string) ([]metric, error) {
			result, err := query(q)
			if err != nil {
				return nil, err
			}
			return datasourceMetricsToTemplateMetrics(result), nil
		},
	}
}

// FuncsWithExternalURL returns a function map that depends on externalURL value
func FuncsWithExternalURL(externalURL *url.URL) textTpl.FuncMap {
	return textTpl.FuncMap{
		"externalURL": func() string {
			return externalURL.String()
		},

		"pathPrefix": func() string {
			return externalURL.Path
		},
	}
}

// templateFuncs initiates template helper functions
func templateFuncs() textTpl.FuncMap {
	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
	return textTpl.FuncMap{
		/* Strings */

		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
		// so for instance $1 represents the text of the first submatch.
		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
		"reReplaceAll": func(pattern, repl, text string) string {
			re := regexp.MustCompile(pattern)
			return re.ReplaceAllString(text, repl)
		},

		// match reports whether the string s
		// contains any match of the regular expression pattern.
		// alias for https://golang.org/pkg/regexp/#MatchString
		"match": regexp.MatchString,

		// title returns a copy of the string s with all Unicode letters
		// that begin words mapped to their Unicode title case.
		// alias for https://golang.org/pkg/strings/#Title
		"title": strings.Title,

		// toUpper returns s with all Unicode letters mapped to their upper case.
		// alias for https://golang.org/pkg/strings/#ToUpper
		"toUpper": strings.ToUpper,

		// toLower returns s with all Unicode letters mapped to their lower case.
		// alias for https://golang.org/pkg/strings/#ToLower
		"toLower": strings.ToLower,

		// stripPort splits string into host and port, then returns only host.
		"stripPort": func(hostPort string) string {
			host, _, err := net.SplitHostPort(hostPort)
			if err != nil {
				return hostPort
			}
			return host
		},

		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
		"parseDuration": func(s string) (float64, error) {
			d, err := promutils.ParseDuration(s)
			if err != nil {
				return 0, err
			}
			return d.Seconds(), nil
		},

		/* Numbers */

		// humanize converts given number to a human readable format
		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
		"humanize": func(i interface{}) (string, error) {
			v, err := toFloat64(i)
			if err != nil {
				return "", err
			}
			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
				return fmt.Sprintf("%.4g", v), nil
			}
			if math.Abs(v) >= 1 {
				prefix := ""
				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
					if math.Abs(v) < 1000 {
						break
					}
					prefix = p
					v /= 1000
				}
				return fmt.Sprintf("%.4g%s", v, prefix), nil
			}
			prefix := ""
			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
				if math.Abs(v) >= 1 {
					break
				}
				prefix = p
				v *= 1000
			}
			return fmt.Sprintf("%.4g%s", v, prefix), nil
		},

		// humanize1024 converts given number to a human readable format with 1024 as base
		"humanize1024": func(i interface{}) (string, error) {
			v, err := toFloat64(i)
			if err != nil {
				return "", err
			}
			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
				return fmt.Sprintf("%.4g", v), nil
			}
			prefix := ""
			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
				if math.Abs(v) < 1024 {
					break
				}
				prefix = p
				v /= 1024
			}
			return fmt.Sprintf("%.4g%s", v, prefix), nil
		},

		// humanizeDuration converts given seconds to a human readable duration
		"humanizeDuration": func(i interface{}) (string, error) {
			v, err := toFloat64(i)
			if err != nil {
				return "", err
			}
			if math.IsNaN(v) || math.IsInf(v, 0) {
				return fmt.Sprintf("%.4g", v), nil
			}
			if v == 0 {
				return fmt.Sprintf("%.4gs", v), nil
			}
			if math.Abs(v) >= 1 {
				sign := ""
				if v < 0 {
					sign = "-"
					v = -v
				}
				seconds := int64(v) % 60
				minutes := (int64(v) / 60) % 60
				hours := (int64(v) / 60 / 60) % 24
				days := int64(v) / 60 / 60 / 24
				// For days to minutes, we display seconds as an integer.
				if days != 0 {
					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
				}
				if hours != 0 {
					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
				}
				if minutes != 0 {
					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
				}
				// For seconds, we display 4 significant digits.
				return fmt.Sprintf("%s%.4gs", sign, v), nil
			}
			prefix := ""
			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
				if math.Abs(v) >= 1 {
					break
				}
				prefix = p
				v *= 1000
			}
			return fmt.Sprintf("%.4g%ss", v, prefix), nil
		},

		// humanizePercentage converts given ratio value to a fraction of 100
		"humanizePercentage": func(i interface{}) (string, error) {
			v, err := toFloat64(i)
			if err != nil {
				return "", err
			}
			return fmt.Sprintf("%.4g%%", v*100), nil
		},

		// humanizeTimestamp converts given timestamp to a human readable time equivalent
		"humanizeTimestamp": func(i interface{}) (string, error) {
			v, err := toFloat64(i)
			if err != nil {
				return "", err
			}
			if math.IsNaN(v) || math.IsInf(v, 0) {
				return fmt.Sprintf("%.4g", v), nil
			}
			t := timeFromUnixTimestamp(v).Time().UTC()
			return fmt.Sprint(t), nil
		},

		// toTime converts given timestamp to a time.Time.
		"toTime": func(i interface{}) (time.Time, error) {
			v, err := toFloat64(i)
			if err != nil {
				return time.Time{}, err
			}
			if math.IsNaN(v) || math.IsInf(v, 0) {
				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
			}
			t := timeFromUnixTimestamp(v).Time().UTC()
			return t, nil
		},

		/* URLs */

		// externalURL returns value of `external.url` flag
		"externalURL": func() string {
			// externalURL function supposed to be substituted at FuncsWithExteralURL().
			// it is present here only for validation purposes, when there is no
			// provided datasource.
			//
			// return non-empty slice to pass validation with chained functions in template
			return ""
		},

		// pathPrefix returns a Path segment from the URL value in `external.url` flag
		"pathPrefix": func() string {
			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
			// it is present here only for validation purposes, when there is no
			// provided datasource.
			//
			// return non-empty slice to pass validation with chained functions in template
			return ""
		},

		// pathEscape escapes the string so it can be safely placed inside a URL path segment,
		// replacing special characters (including /) with %XX sequences as needed.
		// alias for https://golang.org/pkg/net/url/#PathEscape
		"pathEscape": func(u string) string {
			return url.PathEscape(u)
		},

		// queryEscape escapes the string so it can be safely placed
		// inside a URL query.
		// alias for https://golang.org/pkg/net/url/#QueryEscape
		"queryEscape": func(q string) string {
			return url.QueryEscape(q)
		},

		// crlfEscape replaces new line chars to skip URL encoding.
		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/890
		"crlfEscape": func(q string) string {
			q = strings.Replace(q, "\n", `\n`, -1)
			return strings.Replace(q, "\r", `\r`, -1)
		},

		// quotesEscape escapes quote char
		"quotesEscape": func(q string) string {
			return strings.Replace(q, `"`, `\"`, -1)
		},

		// query executes the MetricsQL/PromQL query against
		// configured `datasource.url` address.
		// For example, {{ query "foo" | first | value }} will
		// execute "/api/v1/query?query=foo" request and will return
		// the first value in response.
		"query": func(q string) ([]metric, error) {
			// query function supposed to be substituted at FuncsWithQuery().
			// it is present here only for validation purposes, when there is no
			// provided datasource.
			//
			// return non-empty slice to pass validation with chained functions in template
			// see issue #989 for details
			return []metric{{}}, nil
		},

		// first returns the first by order element from the given metrics list.
		// usually used alongside with `query` template function.
		"first": func(metrics []metric) (metric, error) {
			if len(metrics) > 0 {
				return metrics[0], nil
			}
			return metric{}, errors.New("first() called on vector with no elements")
		},

		// label returns the value of the given label name for the given metric.
		// usually used alongside with `query` template function.
		"label": func(label string, m metric) string {
			return m.Labels[label]
		},

		// sortByLabel sorts the given metrics by provided label key
		"sortByLabel": func(label string, metrics []metric) []metric {
			sort.SliceStable(metrics, func(i, j int) bool {
				return metrics[i].Labels[label] < metrics[j].Labels[label]
			})
			return metrics
		},

		// value returns the value of the given metric.
		// usually used alongside with `query` template function.
		"value": func(m metric) float64 {
			return m.Value
		},

		/* Helpers */

		// Converts a list of objects to a map with keys arg0, arg1 etc.
		// This is intended to allow multiple arguments to be passed to templates.
		"args": func(args ...interface{}) map[string]interface{} {
			result := make(map[string]interface{})
			for i, a := range args {
				result[fmt.Sprintf("arg%d", i)] = a
			}
			return result
		},

		// safeHtml marks string as HTML not requiring auto-escaping.
		"safeHtml": func(text string) htmlTpl.HTML {
			return htmlTpl.HTML(text)
		},
	}
}

// Time is the number of milliseconds since the epoch
// (1970-01-01 00:00 UTC) excluding leap seconds.
type Time int64

// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
func timeFromUnixTimestamp(t float64) Time {
	return Time(t * 1e3)
}

// The number of nanoseconds per minimum tick.
const nanosPerTick = int64(minimumTick / time.Nanosecond)

// MinimumTick is the minimum supported time resolution. This has to be
// at least time.Second in order for the code below to work.
const minimumTick = time.Millisecond

// second is the Time duration equivalent to one second.
const second = int64(time.Second / minimumTick)

// Time returns the time.Time representation of t.
func (t Time) Time() time.Time {
	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
}

func toFloat64(v interface{}) (float64, error) {
	switch i := v.(type) {
	case float64:
		return i, nil
	case float32:
		return float64(i), nil
	case int64:
		return float64(i), nil
	case int32:
		return float64(i), nil
	case int:
		return float64(i), nil
	case uint64:
		return float64(i), nil
	case uint32:
		return float64(i), nil
	case uint:
		return float64(i), nil
	case string:
		return strconv.ParseFloat(i, 64)
	default:
		return 0, fmt.Errorf("unexpected value type %v", i)
	}
}