diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 2e2612707..81e3f0819 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) // Group contains list of Rules grouped into @@ -25,7 +26,7 @@ type Group struct { Type datasource.Type `yaml:"type,omitempty"` File string Name string `yaml:"name"` - Interval utils.PromDuration `yaml:"interval"` + Interval promutils.Duration `yaml:"interval"` Rules []Rule `yaml:"rules"` Concurrency int `yaml:"concurrency"` // ExtraFilterLabels is a list label filters applied to every rule @@ -129,7 +130,7 @@ type Rule struct { Record string `yaml:"record,omitempty"` Alert string `yaml:"alert,omitempty"` Expr string `yaml:"expr"` - For utils.PromDuration `yaml:"for"` + For promutils.Duration `yaml:"for"` Labels map[string]string `yaml:"labels,omitempty"` Annotations map[string]string `yaml:"annotations,omitempty"` diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 0d685f5a1..2b84c3eff 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -11,7 +11,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) func TestMain(m *testing.M) { @@ -260,7 +260,7 @@ func TestGroup_Validate(t *testing.T) { Rules: []Rule{ { Expr: "sumSeries(time('foo.bar',10))", - For: utils.NewPromDuration(10 * time.Millisecond), + For: promutils.NewDuration(10 * time.Millisecond), }, { Expr: "sum(up == 0 ) by (host)", @@ -275,7 +275,7 @@ func TestGroup_Validate(t *testing.T) { Rules: []Rule{ { Expr: "sum(up == 0 ) by (host)", - For: utils.NewPromDuration(10 * time.Millisecond), + For: promutils.NewDuration(10 * time.Millisecond), }, { Expr: "sumSeries(time('foo.bar',10))", @@ -342,7 +342,7 @@ func TestHashRule(t *testing.T) { true, }, { - Rule{Alert: "alert", Expr: "up == 1", For: utils.NewPromDuration(time.Minute)}, + Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)}, Rule{Alert: "alert", Expr: "up == 1"}, true, }, diff --git a/app/vmalert/group_test.go b/app/vmalert/group_test.go index 5cbbab0ad..6aa15dd77 100644 --- a/app/vmalert/group_test.go +++ b/app/vmalert/group_test.go @@ -9,7 +9,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) func init() { @@ -34,7 +34,7 @@ func TestUpdateWith(t *testing.T) { []config.Rule{{ Alert: "foo", Expr: "up > 0", - For: utils.NewPromDuration(time.Second), + For: promutils.NewDuration(time.Second), Labels: map[string]string{ "bar": "baz", }, @@ -46,7 +46,7 @@ func TestUpdateWith(t *testing.T) { []config.Rule{{ Alert: "foo", Expr: "up > 10", - For: utils.NewPromDuration(time.Second), + For: promutils.NewDuration(time.Second), Labels: map[string]string{ "baz": "bar", }, diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go index 72e0663b2..07258c40e 100644 --- a/app/vmalert/notifier/config.go +++ b/app/vmalert/notifier/config.go @@ -11,11 +11,11 @@ import ( "strings" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) // Config contains list of supported configuration settings @@ -38,7 +38,7 @@ type Config struct { RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"` // The timeout used when sending alerts. - Timeout utils.PromDuration `yaml:"timeout,omitempty"` + Timeout promutils.Duration `yaml:"timeout,omitempty"` // Checksum stores the hash of yaml definition for the config. // May be used to detect any changes to the config file. @@ -71,7 +71,7 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { cfg.Scheme = "http" } if cfg.Timeout.Duration() == 0 { - cfg.Timeout = utils.NewPromDuration(time.Second * 10) + cfg.Timeout = promutils.NewDuration(time.Second * 10) } rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false) if err != nil { diff --git a/app/vmalert/notifier/template_func.go b/app/vmalert/notifier/template_func.go index 4a42c07e7..6cd3c0afe 100644 --- a/app/vmalert/notifier/template_func.go +++ b/app/vmalert/notifier/template_func.go @@ -27,7 +27,7 @@ import ( textTpl "text/template" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" - "github.com/VictoriaMetrics/metricsql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) // metric is private copy of datasource.Metric, @@ -104,12 +104,12 @@ func InitTemplateFunc(externalURL *url.URL) { }, // parseDuration parses a duration string such as "1h" into the number of seconds it represents - "parseDuration": func(d string) (float64, error) { - ms, err := metricsql.DurationValue(d, 0) + "parseDuration": func(s string) (float64, error) { + d, err := promutils.ParseDuration(s) if err != nil { return 0, err } - return float64(ms) / 1000, nil + return d.Seconds(), nil }, /* Numbers */ diff --git a/app/vmalert/replay_test.go b/app/vmalert/replay_test.go index 782377cb9..003e916a9 100644 --- a/app/vmalert/replay_test.go +++ b/app/vmalert/replay_test.go @@ -8,7 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) type fakeReplayQuerier struct { @@ -83,7 +83,7 @@ func TestReplay(t *testing.T) { to: "2021-01-01T15:02:30.000Z", maxDP: 60, cfg: []config.Group{ - {Interval: utils.NewPromDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}}, + {Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}}, }, qb: &fakeReplayQuerier{ registry: map[string]map[string]struct{}{ diff --git a/app/vmalert/utils/prom_duration.go b/app/vmalert/utils/prom_duration.go deleted file mode 100644 index bf9fa5dbd..000000000 --- a/app/vmalert/utils/prom_duration.go +++ /dev/null @@ -1,43 +0,0 @@ -package utils - -import ( - "time" - - "github.com/VictoriaMetrics/metricsql" -) - -// PromDuration is Prometheus duration. -type PromDuration struct { - milliseconds int64 -} - -// NewPromDuration returns PromDuration for given d. -func NewPromDuration(d time.Duration) PromDuration { - return PromDuration{ - milliseconds: d.Milliseconds(), - } -} - -// MarshalYAML implements yaml.Marshaler interface. -func (pd PromDuration) MarshalYAML() (interface{}, error) { - return pd.Duration().String(), nil -} - -// UnmarshalYAML implements yaml.Unmarshaler interface. -func (pd *PromDuration) UnmarshalYAML(unmarshal func(interface{}) error) error { - var s string - if err := unmarshal(&s); err != nil { - return err - } - ms, err := metricsql.DurationValue(s, 0) - if err != nil { - return err - } - pd.milliseconds = ms - return nil -} - -// Duration returns duration for pd. -func (pd *PromDuration) Duration() time.Duration { - return time.Duration(pd.milliseconds) * time.Millisecond -} diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index a1e5e0af5..42c169aaf 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -10,6 +10,7 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metricsql" ) @@ -50,14 +51,14 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { return maxTimeMsecs, nil } // Try parsing duration relative to the current time - d, err1 := metricsql.DurationValue(argValue, 0) + d, err1 := promutils.ParseDuration(argValue) if err1 != nil { return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) } if d > 0 { d = -d } - t = time.Now().Add(time.Duration(d) * time.Millisecond) + t = time.Now().Add(d) } secs = float64(t.UnixNano()) / 1e9 } @@ -93,11 +94,11 @@ func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, err secs, err := strconv.ParseFloat(argValue, 64) if err != nil { // Try parsing string format - d, err := metricsql.DurationValue(argValue, 0) + d, err := promutils.ParseDuration(argValue) if err != nil { return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) } - secs = float64(d) / 1000 + secs = d.Seconds() } msecs := int64(secs * 1e3) if msecs <= 0 || msecs > maxDurationMsecs { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 27ffd5720..56d6fc388 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -31,6 +31,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): show the total number of scrapes and the total number of scrape errors per target at `/targets` page. This information may be useful when debugging unreliable scrape targets. * FEATURE: vmagent and single-node VictoriaMetrics: disallow unknown fields at `-promscrape.config` file. Previously unknown fields were allowed. This could lead to long-living silent config errors. The previous behaviour can be returned by passing `-promscrape.config.strictParse=false` command-line flag. * FEATURE: add `__meta_kubernetes_endpointslice_label*` and `__meta_kubernetes_endpointslice_annotation*` labels for `role: endpointslice` targets in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) to be consistent with other `role` values. See [this issue](https://github.com/prometheus/prometheus/issues/10284). +* FEATURE: vmagent: support Prometheus-like durations in `-promscrape.config`. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/817#issuecomment-1033384766). * BUGFIX: return proper results from `highestMax()` function at [Graphite render API](https://docs.victoriametrics.com/#graphite-render-api-usage). Previously it was incorrectly returning timeseries with min peaks instead of max peaks. * BUGFIX: properly limit indexdb cache sizes. Previously they could exceed values set via `-memory.allowedPercent` and/or `-memory.allowedBytes` when `indexdb` contained many data parts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2007). diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 97f867775..e74cea346 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -30,6 +30,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" "github.com/VictoriaMetrics/metrics" xxhash "github.com/cespare/xxhash/v2" @@ -105,9 +106,9 @@ func (cfg *Config) getJobNames() []string { // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/ type GlobalConfig struct { - ScrapeInterval time.Duration `yaml:"scrape_interval,omitempty"` - ScrapeTimeout time.Duration `yaml:"scrape_timeout,omitempty"` - ExternalLabels map[string]string `yaml:"external_labels,omitempty"` + ScrapeInterval promutils.Duration `yaml:"scrape_interval,omitempty"` + ScrapeTimeout promutils.Duration `yaml:"scrape_timeout,omitempty"` + ExternalLabels map[string]string `yaml:"external_labels,omitempty"` } // ScrapeConfig represents essential parts for `scrape_config` section of Prometheus config. @@ -115,8 +116,8 @@ type GlobalConfig struct { // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config type ScrapeConfig struct { JobName string `yaml:"job_name"` - ScrapeInterval time.Duration `yaml:"scrape_interval,omitempty"` - ScrapeTimeout time.Duration `yaml:"scrape_timeout,omitempty"` + ScrapeInterval promutils.Duration `yaml:"scrape_interval,omitempty"` + ScrapeTimeout promutils.Duration `yaml:"scrape_timeout,omitempty"` MetricsPath string `yaml:"metrics_path,omitempty"` HonorLabels bool `yaml:"honor_labels,omitempty"` HonorTimestamps *bool `yaml:"honor_timestamps,omitempty"` @@ -149,8 +150,8 @@ type ScrapeConfig struct { DisableCompression bool `yaml:"disable_compression,omitempty"` DisableKeepAlive bool `yaml:"disable_keepalive,omitempty"` StreamParse bool `yaml:"stream_parse,omitempty"` - ScrapeAlignInterval time.Duration `yaml:"scrape_align_interval,omitempty"` - ScrapeOffset time.Duration `yaml:"scrape_offset,omitempty"` + ScrapeAlignInterval promutils.Duration `yaml:"scrape_align_interval,omitempty"` + ScrapeOffset promutils.Duration `yaml:"scrape_offset,omitempty"` SeriesLimit int `yaml:"series_limit,omitempty"` ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` @@ -705,16 +706,16 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf if jobName == "" { return nil, fmt.Errorf("missing `job_name` field in `scrape_config`") } - scrapeInterval := sc.ScrapeInterval + scrapeInterval := sc.ScrapeInterval.Duration() if scrapeInterval <= 0 { - scrapeInterval = globalCfg.ScrapeInterval + scrapeInterval = globalCfg.ScrapeInterval.Duration() if scrapeInterval <= 0 { scrapeInterval = defaultScrapeInterval } } - scrapeTimeout := sc.ScrapeTimeout + scrapeTimeout := sc.ScrapeTimeout.Duration() if scrapeTimeout <= 0 { - scrapeTimeout = globalCfg.ScrapeTimeout + scrapeTimeout = globalCfg.ScrapeTimeout.Duration() if scrapeTimeout <= 0 { scrapeTimeout = defaultScrapeTimeout } @@ -788,8 +789,8 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf disableCompression: sc.DisableCompression, disableKeepAlive: sc.DisableKeepAlive, streamParse: sc.StreamParse, - scrapeAlignInterval: sc.ScrapeAlignInterval, - scrapeOffset: sc.ScrapeOffset, + scrapeAlignInterval: sc.ScrapeAlignInterval.Duration(), + scrapeOffset: sc.ScrapeOffset.Duration(), seriesLimit: sc.SeriesLimit, } return swc, nil @@ -1057,7 +1058,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel // Read __scrape_interval__ and __scrape_timeout__ from labels. scrapeInterval := swc.scrapeInterval if s := promrelabel.GetLabelValueByName(labels, "__scrape_interval__"); len(s) > 0 { - d, err := time.ParseDuration(s) + d, err := promutils.ParseDuration(s) if err != nil { return nil, fmt.Errorf("cannot parse __scrape_interval__=%q: %w", s, err) } @@ -1065,7 +1066,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel } scrapeTimeout := swc.scrapeTimeout if s := promrelabel.GetLabelValueByName(labels, "__scrape_timeout__"); len(s) > 0 { - d, err := time.ParseDuration(s) + d, err := promutils.ParseDuration(s) if err != nil { return nil, fmt.Errorf("cannot parse __scrape_timeout__=%q: %w", s, err) } diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 47657b061..59a0b3826 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -1648,6 +1648,59 @@ scrape_configs: ProxyAuthConfig: &promauth.Config{}, }, }) + f(` +global: + scrape_timeout: 1d +scrape_configs: +- job_name: foo + scrape_interval: 1w + scrape_align_interval: 1d + scrape_offset: 2d + static_configs: + - targets: ["foo.bar:1234"] +`, []*ScrapeWork{ + { + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: time.Hour * 24 * 7, + ScrapeTimeout: time.Hour * 24, + ScrapeAlignInterval: time.Hour * 24, + ScrapeOffset: time.Hour * 24 * 2, + HonorTimestamps: true, + Labels: []prompbmarshal.Label{ + { + Name: "__address__", + Value: "foo.bar:1234", + }, + { + Name: "__metrics_path__", + Value: "/metrics", + }, + { + Name: "__scheme__", + Value: "http", + }, + { + Name: "__scrape_interval__", + Value: "168h0m0s", + }, + { + Name: "__scrape_timeout__", + Value: "24h0m0s", + }, + { + Name: "instance", + Value: "foo.bar:1234", + }, + { + Name: "job", + Value: "foo", + }, + }, + AuthConfig: &promauth.Config{}, + ProxyAuthConfig: &promauth.Config{}, + jobNameOriginal: "foo", + }, + }) } func equalStaticConfigForScrapeWorks(a, b []*ScrapeWork) bool { diff --git a/lib/promutils/duration.go b/lib/promutils/duration.go new file mode 100644 index 000000000..382b6f53c --- /dev/null +++ b/lib/promutils/duration.go @@ -0,0 +1,52 @@ +package promutils + +import ( + "time" + + "github.com/VictoriaMetrics/metricsql" +) + +// Duration is duration, which must be used in Prometheus-compatible yaml configs. +type Duration struct { + d time.Duration +} + +// NewDuration returns Duration for given d. +func NewDuration(d time.Duration) Duration { + return Duration{ + d: d, + } +} + +// MarshalYAML implements yaml.Marshaler interface. +func (pd Duration) MarshalYAML() (interface{}, error) { + return pd.d.String(), nil +} + +// UnmarshalYAML implements yaml.Unmarshaler interface. +func (pd *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + ms, err := metricsql.DurationValue(s, 0) + if err != nil { + return err + } + pd.d = time.Duration(ms) * time.Millisecond + return nil +} + +// Duration returns duration for pd. +func (pd Duration) Duration() time.Duration { + return pd.d +} + +// ParseDuration parses duration string in Prometheus format +func ParseDuration(s string) (time.Duration, error) { + ms, err := metricsql.DurationValue(s, 0) + if err != nil { + return 0, err + } + return time.Duration(ms) * time.Millisecond, nil +} diff --git a/lib/promutils/duration_test.go b/lib/promutils/duration_test.go new file mode 100644 index 000000000..2c09b68d8 --- /dev/null +++ b/lib/promutils/duration_test.go @@ -0,0 +1,42 @@ +package promutils + +import ( + "testing" + "time" +) + +func TestDuration(t *testing.T) { + if _, err := ParseDuration("foobar"); err == nil { + t.Fatalf("expecting error for invalid duration") + } + dNative, err := ParseDuration("1w") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if dNative != 7*24*time.Hour { + t.Fatalf("unexpected duration; got %s; want %s", dNative, 7*24*time.Hour) + } + d := NewDuration(dNative) + if d.Duration() != dNative { + t.Fatalf("unexpected duration; got %s; want %s", d.Duration(), dNative) + } + v, err := d.MarshalYAML() + if err != nil { + t.Fatalf("unexpected error in MarshalYAML(): %s", err) + } + sExpected := "168h0m0s" + if s := v.(string); s != sExpected { + t.Fatalf("unexpected value from MarshalYAML(); got %q; want %q", s, sExpected) + } + if err := d.UnmarshalYAML(func(v interface{}) error { + sp := v.(*string) + s := "1w3d5h" + *sp = s + return nil + }); err != nil { + t.Fatalf("unexpected error in UnmarshalYAML(): %s", err) + } + if dNative := d.Duration(); dNative != (10*24+5)*time.Hour { + t.Fatalf("unexpected value; got %s; want %s", dNative, (10*24+5)*time.Hour) + } +} diff --git a/lib/storage/part_header.go b/lib/storage/part_header.go index ec874a23d..af62a9438 100644 --- a/lib/storage/part_header.go +++ b/lib/storage/part_header.go @@ -11,7 +11,7 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" - "github.com/VictoriaMetrics/metricsql" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) // partHeader represents part header. @@ -140,11 +140,11 @@ func (ph *partHeader) readMinDedupInterval(partPath string) error { } return fmt.Errorf("cannot read %q: %w", filePath, err) } - dedupInterval, err := metricsql.DurationValue(string(data), 0) + dedupInterval, err := promutils.ParseDuration(string(data)) if err != nil { return fmt.Errorf("cannot parse minimum dedup interval %q at %q: %w", data, filePath, err) } - ph.MinDedupInterval = dedupInterval + ph.MinDedupInterval = dedupInterval.Milliseconds() return nil }