diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index c48143926..ae5234ff4 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -176,10 +176,10 @@ func Init() { } var ( - configReloads = metrics.NewCounter(`vm_relabel_config_reloads_total`) - configReloadErrors = metrics.NewCounter(`vm_relabel_config_reloads_errors_total`) - configSuccess = metrics.NewCounter(`vm_relabel_config_last_reload_successful`) - configTimestamp = metrics.NewCounter(`vm_relabel_config_last_reload_success_timestamp_seconds`) + configReloads = metrics.NewCounter(`vmagent_relabel_config_reloads_total`) + configReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`) + configSuccess = metrics.NewCounter(`vmagent_relabel_config_last_reload_successful`) + configTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`) ) func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx { diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go index 494280089..5c8db252e 100644 --- a/app/vminsert/relabel/relabel.go +++ b/app/vminsert/relabel/relabel.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" @@ -38,23 +39,38 @@ func Init() { logger.Fatalf("cannot load relabelConfig: %s", err) } pcsGlobal.Store(pcs) + configSuccess.Set(1) + configTimestamp.Set(fasttime.UnixTimestamp()) + if len(*relabelConfig) == 0 { return } go func() { for range sighupCh { + configReloads.Inc() logger.Infof("received SIGHUP; reloading -relabelConfig=%q...", *relabelConfig) pcs, err := loadRelabelConfig() if err != nil { + configReloadErrors.Inc() + configSuccess.Set(0) logger.Errorf("cannot load the updated relabelConfig: %s; preserving the previous config", err) continue } pcsGlobal.Store(pcs) + configSuccess.Set(1) + configTimestamp.Set(fasttime.UnixTimestamp()) logger.Infof("successfully reloaded -relabelConfig=%q", *relabelConfig) } }() } +var ( + configReloads = metrics.NewCounter(`vm_relabel_config_reloads_total`) + configReloadErrors = metrics.NewCounter(`vm_relabel_config_reloads_errors_total`) + configSuccess = metrics.NewCounter(`vm_relabel_config_last_reload_successful`) + configTimestamp = metrics.NewCounter(`vm_relabel_config_last_reload_success_timestamp_seconds`) +) + var pcsGlobal atomic.Value func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) { diff --git a/deployment/docker/alerts-vmagent.yml b/deployment/docker/alerts-vmagent.yml index 560e7be86..3c65b3009 100644 --- a/deployment/docker/alerts-vmagent.yml +++ b/deployment/docker/alerts-vmagent.yml @@ -125,10 +125,10 @@ groups: expr: | vm_promscrape_config_last_reload_successful != 1 or - vm_relabel_config_last_reload_successful != 1 + vmagent_relabel_config_last_reload_successful != 1 labels: severity: warning annotations: summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}" description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. - Check vmagent's logs for detailed error message." \ No newline at end of file + Check vmagent's logs for detailed error message." diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b9163388d..c16d9eec3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,7 +17,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html): add `-storageNode.filter` command-line flag for filtering the [discovered vmstorage nodes](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#automatic-vmstorage-discovery) with arbitrary regular expressions. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3353). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): allow using numeric values with `K`, `Ki`, `M`, `Mi`, `G`, `Gi`, `T` and `Ti` suffixes inside MetricsQL queries. For example `8Ki` equals to `8*1024`, while `8.2M` equals to `8.2*1000*1000`. -* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_normalize](https://docs.victoriametrics.com/MetricsQL.html#range_normalize) function for normalizing multiple time series into `[0...1]` value range. This function is useful for correlation analyzis of time series with distinct value ranges. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3167). +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_normalize](https://docs.victoriametrics.com/MetricsQL.html#range_normalize) function for normalizing multiple time series into `[0...1]` value range. This function is useful for correlation analysis of time series with distinct value ranges. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3167). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_linear_regression](https://docs.victoriametrics.com/MetricsQL.html#range_linear_regression) function for calculating [simple linear regression](https://en.wikipedia.org/wiki/Simple_linear_regression) over the input time series on the selected time range. This function is useful for predictions and capacity planning. For example, `range_linear_regression(process_resident_memory_bytes)` can predict future memory usage based on the past memory usage. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add [range_stddev](https://docs.victoriametrics.com/MetricsQL.html#range_stddev) and [range_stdvar](https://docs.victoriametrics.com/MetricsQL.html#range_stdvar) functions. * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): optimize `expr1 op expr2` query when `expr1` returns an empty result. In this case there is no sense in executing `expr2` for `op` not equal to `or`, since the end result will be empty according to [PromQL series matching rules](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349). Thanks to @jianglinjian for pointing to this case. @@ -27,7 +27,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add copy button to row on Table view. The button copies row in MetricQL format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2815). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the ability to "stick" a tooltip on the chart by clicking on a data point. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3321) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3376) * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): add default alert list for vmalert's metrics. See [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmalert.yml). -* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `vm_relabel_config_*` and `vm_promscrape_config_*` metrics for tracking relabel and scrape configuration hot-reloads. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3345). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `vmagent_relabel_config_*`, `vm_relabel_config_*` and `vm_promscrape_config_*` metrics for tracking relabel and scrape configuration hot-reloads. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3345). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly return an empty result from [limit_offset](https://docs.victoriametrics.com/MetricsQL.html#limit_offset) if the `offset` arg exceeds the number of inner time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3312). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly discover GCE zones when `filter` option is set at [gce_sd_configs](https://docs.victoriametrics.com/sd_configs.html#gce_sd_configs). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3202).