From ebaa1c7ad574526fb26cf868138b960ebc19998a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 16 Apr 2022 14:25:54 +0300 Subject: [PATCH] lib/promscrape: follow-up after baa1c24b368ccecf1ad243e571fe4ae5a2d15b5f --- app/vmalert/config/config.go | 20 ++++++------ app/vmalert/notifier/config.go | 2 +- docs/CHANGELOG.md | 1 + lib/promscrape/config.go | 42 ++++++++++++------------ lib/promscrape/config_test.go | 58 ++++++++++++++++++++++++++++++++++ lib/promutils/duration.go | 9 ++++-- 6 files changed, 97 insertions(+), 35 deletions(-) diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 81e3f0819..ec961b092 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -25,10 +25,10 @@ import ( type Group struct { Type datasource.Type `yaml:"type,omitempty"` File string - Name string `yaml:"name"` - Interval promutils.Duration `yaml:"interval"` - Rules []Rule `yaml:"rules"` - Concurrency int `yaml:"concurrency"` + Name string `yaml:"name"` + Interval *promutils.Duration `yaml:"interval,omitempty"` + Rules []Rule `yaml:"rules"` + Concurrency int `yaml:"concurrency"` // ExtraFilterLabels is a list label filters applied to every rule // request withing a group. Is compatible only with VM datasources. // See https://docs.victoriametrics.com#prometheus-querying-api-enhancements @@ -127,12 +127,12 @@ func (g *Group) Validate(validateAnnotations, validateExpressions bool) error { // recording rule or alerting rule. type Rule struct { ID uint64 - Record string `yaml:"record,omitempty"` - Alert string `yaml:"alert,omitempty"` - Expr string `yaml:"expr"` - For promutils.Duration `yaml:"for"` - Labels map[string]string `yaml:"labels,omitempty"` - Annotations map[string]string `yaml:"annotations,omitempty"` + Record string `yaml:"record,omitempty"` + Alert string `yaml:"alert,omitempty"` + Expr string `yaml:"expr"` + For *promutils.Duration `yaml:"for,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + Annotations map[string]string `yaml:"annotations,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go index 3eb776fbb..d43cced4d 100644 --- a/app/vmalert/notifier/config.go +++ b/app/vmalert/notifier/config.go @@ -44,7 +44,7 @@ type Config struct { // AlertRelabelConfigs contains list of relabeling rules alert labels AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"` // The timeout used when sending alerts. - Timeout promutils.Duration `yaml:"timeout,omitempty"` + Timeout *promutils.Duration `yaml:"timeout,omitempty"` // Checksum stores the hash of yaml definition for the config. // May be used to detect any changes to the config file. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e87e7eabf..afa5d3dec 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add support for DNS-based discovery for notifiers in the same way as Prometheus does. See [these docs](https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2460). * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): return non-zero exit code on error. This allows handling `vmctl` errors in shell scripts. Previously `vmctl` was returning 0 exit code on error. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2322). +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly show `scrape_timeout` and `scrape_interval` options at `http://vmagent:8429/config` page. Previously these options weren't displayed even if they were set in `-promscrape.config`. ## [v1.76.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.76.1) diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index bca411b2f..83ffc3c27 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -83,6 +83,19 @@ type Config struct { baseDir string } +func (cfg *Config) unmarshal(data []byte, isStrict bool) error { + data = envtemplate.Replace(data) + var err error + if isStrict { + if err = yaml.UnmarshalStrict(data, cfg); err != nil { + err = fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err) + } + } else { + err = yaml.Unmarshal(data, cfg) + } + return err +} + func (cfg *Config) marshal() []byte { data, err := yaml.Marshal(cfg) if err != nil { @@ -124,9 +137,9 @@ func (cfg *Config) getJobNames() []string { // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/ type GlobalConfig struct { - ScrapeInterval promutils.Duration `yaml:"scrape_interval,omitempty"` - ScrapeTimeout promutils.Duration `yaml:"scrape_timeout,omitempty"` - ExternalLabels map[string]string `yaml:"external_labels,omitempty"` + ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"` + ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` + ExternalLabels map[string]string `yaml:"external_labels,omitempty"` } // ScrapeConfig represents essential parts for `scrape_config` section of Prometheus config. @@ -134,8 +147,8 @@ type GlobalConfig struct { // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config type ScrapeConfig struct { JobName string `yaml:"job_name"` - ScrapeInterval promutils.Duration `yaml:"scrape_interval"` - ScrapeTimeout promutils.Duration `yaml:"scrape_timeout"` + ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"` + ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` MetricsPath string `yaml:"metrics_path,omitempty"` HonorLabels bool `yaml:"honor_labels,omitempty"` HonorTimestamps *bool `yaml:"honor_timestamps,omitempty"` @@ -168,8 +181,8 @@ type ScrapeConfig struct { DisableCompression bool `yaml:"disable_compression,omitempty"` DisableKeepAlive bool `yaml:"disable_keepalive,omitempty"` StreamParse bool `yaml:"stream_parse,omitempty"` - ScrapeAlignInterval promutils.Duration `yaml:"scrape_align_interval"` - ScrapeOffset promutils.Duration `yaml:"scrape_offset"` + ScrapeAlignInterval *promutils.Duration `yaml:"scrape_align_interval,omitempty"` + ScrapeOffset *promutils.Duration `yaml:"scrape_offset,omitempty"` SeriesLimit int `yaml:"series_limit,omitempty"` ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` @@ -309,7 +322,7 @@ func IsDryRun() bool { } func (cfg *Config) parseData(data []byte, path string) ([]byte, error) { - if err := unmarshalMaybeStrict(data, cfg); err != nil { + if err := cfg.unmarshal(data, *strictParse); err != nil { return nil, fmt.Errorf("cannot unmarshal data: %w", err) } absPath, err := filepath.Abs(path) @@ -349,19 +362,6 @@ func (cfg *Config) parseData(data []byte, path string) ([]byte, error) { return dataNew, nil } -func unmarshalMaybeStrict(data []byte, dst interface{}) error { - data = envtemplate.Replace(data) - var err error - if *strictParse { - if err = yaml.UnmarshalStrict(data, dst); err != nil { - err = fmt.Errorf("%w; pass -promscrape.config.strictParse=false command-line flag for ignoring unknown fields in yaml config", err) - } - } else { - err = yaml.Unmarshal(data, dst) - } - return err -} - func getSWSByJob(sws []*ScrapeWork) map[string][]*ScrapeWork { m := make(map[string][]*ScrapeWork) for _, sw := range sws { diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 59a0b3826..bc2147a4f 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "testing" "time" @@ -13,6 +14,63 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" ) +func TestScrapeConfigUnmarshalMarshal(t *testing.T) { + f := func(data string) { + t.Helper() + var cfg Config + data = strings.TrimSpace(data) + if err := cfg.unmarshal([]byte(data), true); err != nil { + t.Fatalf("parse error: %s\ndata:\n%s", err, data) + } + resultData := string(cfg.marshal()) + result := strings.TrimSpace(resultData) + if result != data { + t.Fatalf("unexpected marshaled config:\ngot\n%s\nwant\n%s", result, data) + } + } + f(` +global: + scrape_interval: 10s +`) + f(` +scrape_config_files: +- foo +- bar +`) + f(` +scrape_configs: +- job_name: foo + scrape_timeout: 1.5s + static_configs: + - targets: + - foo + - bar + labels: + foo: bar +`) + f(` +scrape_configs: +- job_name: foo + honor_labels: true + honor_timestamps: false + scheme: https + params: + foo: + - x + authorization: + type: foobar + relabel_configs: + - source_labels: [abc] + static_configs: + - targets: + - foo + relabel_debug: true + scrape_align_interval: 1h30m0s + proxy_bearer_token_file: file.txt +`) + +} + func TestNeedSkipScrapeWork(t *testing.T) { f := func(key string, membersCount, replicationFactor, memberNum int, needSkipExpected bool) { t.Helper() diff --git a/lib/promutils/duration.go b/lib/promutils/duration.go index 382b6f53c..edfe0dd1d 100644 --- a/lib/promutils/duration.go +++ b/lib/promutils/duration.go @@ -12,8 +12,8 @@ type Duration struct { } // NewDuration returns Duration for given d. -func NewDuration(d time.Duration) Duration { - return Duration{ +func NewDuration(d time.Duration) *Duration { + return &Duration{ d: d, } } @@ -38,7 +38,10 @@ func (pd *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { } // Duration returns duration for pd. -func (pd Duration) Duration() time.Duration { +func (pd *Duration) Duration() time.Duration { + if pd == nil { + return 0 + } return pd.d }