From c25f053945dbe55fb7dd42a9869177fd00b4900c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 28 Jul 2023 11:30:44 -0700 Subject: [PATCH 01/69] docs/VictoriaLogs/data-ingestion/README.md: add checkboxes for Loki format support across log shippers Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4728 --- docs/VictoriaLogs/data-ingestion/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/VictoriaLogs/data-ingestion/README.md b/docs/VictoriaLogs/data-ingestion/README.md index aa86c705a6..9876edacb8 100644 --- a/docs/VictoriaLogs/data-ingestion/README.md +++ b/docs/VictoriaLogs/data-ingestion/README.md @@ -255,10 +255,10 @@ VictoriaLogs exposes various [metrics](https://docs.victoriametrics.com/Victoria Here is the list of log collectors and their ingestion formats supported by VictoriaLogs: -| How to setup the collector | Format: Elasticsearch | Format: JSON Stream | -|------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|---------------------------------------------------------------| -| [Filebeat](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Filebeat.html) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | -| [Fluentbit](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Fluentbit.html) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | -| [Logstash](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Logstash.html) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | -| [Vector](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Vector.html) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | No | -| [Promtail](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Promtail.html) | No | No | +| How to setup the collector | Format: Elasticsearch | Format: JSON Stream | Format: Loki | +|------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|---------------------------------------------------------------|-------------------------------------------------------------------------------------| +| [Filebeat](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Filebeat.html) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | No | +| [Fluentbit](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Fluentbit.html) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) | +| [Logstash](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Logstash.html) | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | No | +| [Vector](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Vector.html) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | No | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) | +| [Promtail](https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/Promtail.html) | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) | From 9082a84566c288df98bf265b4e7564a83ae261fb Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 28 Jul 2023 19:47:02 -0700 Subject: [PATCH 02/69] lib/storage: update nextRotationTimestamp relative to the timestamp of the indexdb rotation Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4563 --- lib/storage/index_db_test.go | 2 +- lib/storage/storage.go | 9 +++++---- lib/storage/storage_test.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go index 446feb803a..074613f334 100644 --- a/lib/storage/index_db_test.go +++ b/lib/storage/index_db_test.go @@ -1479,7 +1479,7 @@ func TestIndexDBRepopulateAfterRotation(t *testing.T) { prevGeneration := db.generation // force index rotation - s.mustRotateIndexDB() + s.mustRotateIndexDB(time.Now()) // check tsidCache wasn't reset after the rotation var cs2 fastcache.Stats diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 9986191ee6..5ef0a82909 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -692,8 +692,8 @@ func (s *Storage) retentionWatcher() { select { case <-s.stop: return - case <-time.After(time.Second * time.Duration(d)): - s.mustRotateIndexDB() + case currentTime := <-time.After(time.Second * time.Duration(d)): + s.mustRotateIndexDB(currentTime) } } } @@ -750,14 +750,15 @@ func (s *Storage) nextDayMetricIDsUpdater() { } } -func (s *Storage) mustRotateIndexDB() { +func (s *Storage) mustRotateIndexDB(currentTime time.Time) { // Create new indexdb table, which will be used as idbNext newTableName := nextIndexDBTableName() idbNewPath := filepath.Join(s.path, indexdbDirname, newTableName) idbNew := mustOpenIndexDB(idbNewPath, s, &s.isReadOnly) // Update nextRotationTimestamp - atomic.AddInt64(&s.nextRotationTimestamp, s.retentionMsecs/1000) + nextRotationTimestamp := currentTime.UnixMilli() + s.retentionMsecs/1000 + atomic.StoreInt64(&s.nextRotationTimestamp, nextRotationTimestamp) // Set idbNext to idbNew idbNext := s.idbNext.Load() diff --git a/lib/storage/storage_test.go b/lib/storage/storage_test.go index 19d3713b34..a098fc72a5 100644 --- a/lib/storage/storage_test.go +++ b/lib/storage/storage_test.go @@ -1094,7 +1094,7 @@ func TestStorageRotateIndexDB(t *testing.T) { return default: time.Sleep(time.Millisecond) - s.mustRotateIndexDB() + s.mustRotateIndexDB(time.Now()) } } }() From e3ef3df938fb8f6afe8694ad9308e22e5f0b8ba8 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 28 Jul 2023 21:08:41 -0700 Subject: [PATCH 03/69] lib/promscrape: use local scrape timestamp for scraped metrics unless `honor_timestamps: true` is set explicitly This fixes the case with gaps for metrics collected from cadvisor, which exports invalid timestamps, which break staleness detection at VictoriaMetrics side. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697 , https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1656540535 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1773 --- docs/CHANGELOG.md | 1 + docs/sd_configs.md | 3 +- lib/promscrape/config.go | 7 +- lib/promscrape/config_test.go | 164 +++++++++++++++------------------- 4 files changed, 75 insertions(+), 100 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c86155e17d..157ed265f4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components Released at 2023-07-28 +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): revert unit test feature for alerting and recording rules introduced in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4596). See the following [change](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4734). ## [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) diff --git a/docs/sd_configs.md b/docs/sd_configs.md index 9c0c18a5e8..fc35c54095 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -1445,7 +1445,8 @@ scrape_configs: # If honor_timestamps is set to "false", the timestamps of the metrics exposed # by the target will be ignored. # - # By default, honor_timestamps is set to true. + # By default, honor_timestamps is set to false. + # See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1656540535 for details. # honor_timestamps: # scheme configures the protocol scheme used for requests. diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index cc1b9529af..0c26bdb0a2 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -244,7 +244,7 @@ type ScrapeConfig struct { ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` MetricsPath string `yaml:"metrics_path,omitempty"` HonorLabels bool `yaml:"honor_labels,omitempty"` - HonorTimestamps *bool `yaml:"honor_timestamps,omitempty"` + HonorTimestamps bool `yaml:"honor_timestamps,omitempty"` Scheme string `yaml:"scheme,omitempty"` Params map[string][]string `yaml:"params,omitempty"` HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` @@ -984,10 +984,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf scrapeTimeout = scrapeInterval } honorLabels := sc.HonorLabels - honorTimestamps := true - if sc.HonorTimestamps != nil { - honorTimestamps = *sc.HonorTimestamps - } + honorTimestamps := sc.HonorTimestamps denyRedirects := false if sc.HTTPClientConfig.FollowRedirects != nil { denyRedirects = !*sc.HTTPClientConfig.FollowRedirects diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 55c29f2089..7fe042b5f8 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -89,7 +89,7 @@ scrape_configs: scrape_configs: - job_name: foo honor_labels: true - honor_timestamps: false + honor_timestamps: true scheme: https params: foo: @@ -243,10 +243,9 @@ scrape_configs: resetNonEssentialFields(sws) swsExpected := []*ScrapeWork{ { - ScrapeURL: "http://host1:80/metric/path1?x=y", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://host1:80/metric/path1?x=y", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "host1:80", "job": "abc", @@ -256,10 +255,9 @@ scrape_configs: jobNameOriginal: "abc", }, { - ScrapeURL: "https://host2:443/metric/path2?x=y", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "https://host2:443/metric/path2?x=y", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "host2:443", "job": "abc", @@ -269,10 +267,9 @@ scrape_configs: jobNameOriginal: "abc", }, { - ScrapeURL: "http://host3:1234/metric/path3?arg1=value1&x=y", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://host3:1234/metric/path3?arg1=value1&x=y", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "host3:1234", "job": "abc", @@ -282,10 +279,9 @@ scrape_configs: jobNameOriginal: "abc", }, { - ScrapeURL: "https://host4:1234/foo/bar?x=y", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "https://host4:1234/foo/bar?x=y", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "host4:1234", "job": "abc", @@ -330,10 +326,9 @@ scrape_configs: sws := cfg.getStaticScrapeWork() resetNonEssentialFields(sws) swsExpected := []*ScrapeWork{{ - ScrapeURL: "http://black:9115/probe?module=dns_udp_example&target=8.8.8.8", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://black:9115/probe?module=dns_udp_example&target=8.8.8.8", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "8.8.8.8", "job": "blackbox", @@ -757,10 +752,9 @@ scrape_configs: - files: ["testdata/file_sd.json", "testdata/file_sd*.yml"] `, []*ScrapeWork{ { - ScrapeURL: "http://host1:80/abc/de", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://host1:80/abc/de", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "host1:80", "job": "foo", @@ -771,10 +765,9 @@ scrape_configs: jobNameOriginal: "foo", }, { - ScrapeURL: "http://host2:80/abc/de", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://host2:80/abc/de", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "host2:80", "job": "foo", @@ -785,10 +778,9 @@ scrape_configs: jobNameOriginal: "foo", }, { - ScrapeURL: "http://localhost:9090/abc/de", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://localhost:9090/abc/de", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "localhost:9090", "job": "foo", @@ -821,10 +813,9 @@ scrape_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "foo", @@ -845,10 +836,9 @@ scrape_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "foo", @@ -873,7 +863,7 @@ scrape_configs: metrics_path: /foo/bar scheme: https honor_labels: true - honor_timestamps: false + honor_timestamps: true follow_redirects: false params: p: ["x&y", "="] @@ -899,7 +889,7 @@ scrape_configs: ScrapeInterval: 54 * time.Second, ScrapeTimeout: 5 * time.Second, HonorLabels: true, - HonorTimestamps: false, + HonorTimestamps: true, DenyRedirects: true, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:443", @@ -916,7 +906,7 @@ scrape_configs: ScrapeInterval: 54 * time.Second, ScrapeTimeout: 5 * time.Second, HonorLabels: true, - HonorTimestamps: false, + HonorTimestamps: true, DenyRedirects: true, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "aaa:443", @@ -929,10 +919,9 @@ scrape_configs: jobNameOriginal: "foo", }, { - ScrapeURL: "http://1.2.3.4:80/metrics", - ScrapeInterval: 8 * time.Second, - ScrapeTimeout: 8 * time.Second, - HonorTimestamps: true, + ScrapeURL: "http://1.2.3.4:80/metrics", + ScrapeInterval: 8 * time.Second, + ScrapeTimeout: 8 * time.Second, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "1.2.3.4:80", "job": "qwer", @@ -945,10 +934,9 @@ scrape_configs: jobNameOriginal: "qwer", }, { - ScrapeURL: "http://foobar:80/metrics", - ScrapeInterval: 8 * time.Second, - ScrapeTimeout: 8 * time.Second, - HonorTimestamps: true, + ScrapeURL: "http://foobar:80/metrics", + ScrapeInterval: 8 * time.Second, + ScrapeTimeout: 8 * time.Second, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foobar:80", "job": "asdf", @@ -995,10 +983,9 @@ scrape_configs: - targets: ["foo.bar:1234", "drop-this-target"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics?x=keep_me", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics?x=keep_me", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "hash": "82", "instance": "foo.bar:1234", @@ -1039,10 +1026,9 @@ scrape_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ { - ScrapeURL: "mailto://foo.bar:1234/abc.de?a=b", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "mailto://foo.bar:1234/abc.de?a=b", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "fake.addr", "job": "https", @@ -1074,10 +1060,9 @@ scrape_configs: - targets: ["foo.bar:1234", "xyz"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "3", @@ -1098,10 +1083,9 @@ scrape_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "foo", @@ -1118,10 +1102,9 @@ scrape_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "foo", @@ -1138,10 +1121,9 @@ scrape_configs: - targets: ["foo.bar:1234"] `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "foo", @@ -1172,10 +1154,9 @@ scrape_configs: job: yyy `, []*ScrapeWork{ { - ScrapeURL: "http://pp:80/metrics?a=c&a=xy", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://pp:80/metrics?a=c&a=xy", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "foo": "bar", "instance": "pp:80", @@ -1239,10 +1220,9 @@ scrape_configs: replacement: true `, []*ScrapeWork{ { - ScrapeURL: "http://127.0.0.1:9116/snmp?module=if_mib&target=192.168.1.2", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://127.0.0.1:9116/snmp?module=if_mib&target=192.168.1.2", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "192.168.1.2", "job": "snmp", @@ -1269,10 +1249,9 @@ scrape_configs: target_label: __metrics_path__ `, []*ScrapeWork{ { - ScrapeURL: "http://foo.bar:1234/metricspath", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - HonorTimestamps: true, + ScrapeURL: "http://foo.bar:1234/metricspath", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", "job": "path wo slash", @@ -1300,7 +1279,6 @@ scrape_configs: ScrapeTimeout: time.Hour * 24, ScrapeAlignInterval: time.Hour * 24, ScrapeOffset: time.Hour * 24 * 2, - HonorTimestamps: true, NoStaleMarkers: true, Labels: promutils.NewLabelsFromMap(map[string]string{ "instance": "foo.bar:1234", @@ -1340,16 +1318,14 @@ func TestScrapeConfigClone(t *testing.T) { f(&ScrapeConfig{}) - bFalse := false var ie promrelabel.IfExpression if err := ie.Parse(`{foo=~"bar",baz!="z"}`); err != nil { t.Fatalf("unexpected error: %s", err) } f(&ScrapeConfig{ - JobName: "foo", - ScrapeInterval: promutils.NewDuration(time.Second * 47), - HonorLabels: true, - HonorTimestamps: &bFalse, + JobName: "foo", + ScrapeInterval: promutils.NewDuration(time.Second * 47), + HonorLabels: true, Params: map[string][]string{ "foo": {"bar", "baz"}, }, From d18ff993e67627e8c855c6862670c2aba4057277 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 28 Jul 2023 21:36:16 -0700 Subject: [PATCH 04/69] lib/promscrape: add a comment why `honor_timestamps` is set to false by default This should prevent from returning it back to true in the future Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697 --- docs/CHANGELOG.md | 2 +- lib/promscrape/config.go | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 157ed265f4..f284ace23f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,7 +28,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components Released at 2023-07-28 -* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): revert unit test feature for alerting and recording rules introduced in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4596). See the following [change](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4734). ## [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 0c26bdb0a2..5cf006c009 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -239,12 +239,17 @@ type GlobalConfig struct { // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config type ScrapeConfig struct { - JobName string `yaml:"job_name"` - ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"` - ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` - MetricsPath string `yaml:"metrics_path,omitempty"` - HonorLabels bool `yaml:"honor_labels,omitempty"` - HonorTimestamps bool `yaml:"honor_timestamps,omitempty"` + JobName string `yaml:"job_name"` + ScrapeInterval *promutils.Duration `yaml:"scrape_interval,omitempty"` + ScrapeTimeout *promutils.Duration `yaml:"scrape_timeout,omitempty"` + MetricsPath string `yaml:"metrics_path,omitempty"` + HonorLabels bool `yaml:"honor_labels,omitempty"` + + // HonorTimestamps is set to false by default contrary to Prometheus, which sets it to true by default, + // because of the issue with gaps on graphs when scraping cadvisor or similar targets, which export invalid timestamps. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799 for details. + HonorTimestamps bool `yaml:"honor_timestamps,omitempty"` + Scheme string `yaml:"scheme,omitempty"` Params map[string][]string `yaml:"params,omitempty"` HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` From 8e38efaa7b2a842072d3be80faa07f0ad305e51a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 28 Jul 2023 23:01:30 -0700 Subject: [PATCH 05/69] docs/CHANGELOG.md: move bugfix description to `tip` chapter, since it isnt released yet Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697 --- docs/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f284ace23f..f9fefc65aa 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,11 +24,13 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). + + ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) Released at 2023-07-28 -* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): revert unit test feature for alerting and recording rules introduced in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4596). See the following [change](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4734). ## [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) From 4283eb4626e14a39f856610d342342cd98c91cde Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Mon, 31 Jul 2023 09:21:05 +0200 Subject: [PATCH 06/69] docs: remove anchors from the 1.92 release Adding anchors to the 1.92 changelog breaks consistency of navigation section at https://docs.victoriametrics.com/CHANGELOG.html All other releases do not have subsections, so should 1.92. Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f9fefc65aa..1fe81b9b9d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -49,7 +49,7 @@ The previous behavior can be restored in the following ways: - by passing `-streamAggr.dropInput` command-line flag to single-node VictoriaMetrics; - by passing `-remoteWrite.streamAggr.dropInput` command-line flag per each configured `-remoteWrite.streamAggr.config` at `vmagent`. -### CHANGES +--- * SECURITY: upgrade base docker image (alpine) from 3.18.0 to 3.18.2. See [alpine 3.18.2 release notes](https://alpinelinux.org/posts/Alpine-3.15.9-3.16.6-3.17.4-3.18.2-released.html). * SECURITY: upgrade Go builder from Go1.20.5 to Go1.20.6. See [the list of issues addressed in Go1.20.6](https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved). From 9abf8535acb42f63abb1291c8113f708f810c7f0 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Mon, 31 Jul 2023 14:05:17 +0200 Subject: [PATCH 07/69] docs: add link to release-guide follow-up Signed-off-by: hagen1778 --- docs/Release-Guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Release-Guide.md b/docs/Release-Guide.md index 3f7e22ab43..ab195951c5 100644 --- a/docs/Release-Guide.md +++ b/docs/Release-Guide.md @@ -98,6 +98,7 @@ and make it merged. See example in this [commit](https://github.com/VictoriaMetr 1. Bump version of the VictoriaMetrics cluster in the [sandbox environment](https://github.com/VictoriaMetrics/ops/blob/main/gcp-test/sandbox/manifests/benchmark-vm/vmcluster.yaml) by [opening and merging PR](https://github.com/VictoriaMetrics/ops/pull/58). 1. Bump VictoriaMetrics version at `deployment/docker/docker-compose.yml` and at `deployment/docker/docker-compose-cluster.yml`. +1. Follow the instructions in [release follow-up](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/master/Release-Guide.md). ## Building snap package From 3f6efab6ae4b3d5f8d95b749858823dfc62fe4fc Mon Sep 17 00:00:00 2001 From: Damon07 <1490818749@qq.com> Date: Mon, 31 Jul 2023 21:23:59 +0800 Subject: [PATCH 08/69] {app/vmselect,docs}: support share_eq_over_time#4441 (#4725) https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441 Co-authored-by: wangm --- app/vmselect/promql/exec_test.go | 11 +++++++++++ app/vmselect/promql/rollup.go | 5 +++++ app/vmselect/promql/rollup_test.go | 19 +++++++++++++++++++ docs/MetricsQL.md | 8 ++++++++ 4 files changed, 43 insertions(+) diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index e298e729fc..b39887bea1 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -5826,6 +5826,17 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run(`share_eq_over_time`, func(t *testing.T) { + t.Parallel() + q := `share_eq_over_time(round(5*rand(0))[200s:10s], 1)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0.1, 0.2, 0.25, 0.1, 0.3, 0.3}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run(`count_gt_over_time`, func(t *testing.T) { t.Parallel() q := `count_gt_over_time(rand(0)[200s:10s], 0.7)` diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index f585440e5a..d7c742bec2 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -78,6 +78,7 @@ var rollupFuncs = map[string]newRollupFunc{ "scrape_interval": newRollupFuncOneArg(rollupScrapeInterval), "share_gt_over_time": newRollupShareGT, "share_le_over_time": newRollupShareLE, + "share_eq_over_time": newRollupShareEQ, "stale_samples_over_time": newRollupFuncOneArg(rollupStaleSamples), "stddev_over_time": newRollupFuncOneArg(rollupStddev), "stdvar_over_time": newRollupFuncOneArg(rollupStdvar), @@ -1106,6 +1107,10 @@ func countFilterGT(values []float64, gt float64) int { return n } +func newRollupShareEQ(args []interface{}) (rollupFunc, error) { + return newRollupShareFilter(args, countFilterEQ) +} + func countFilterEQ(values []float64, eq float64) int { n := 0 for _, v := range values { diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 25caa7c328..970b0b3baf 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -261,6 +261,25 @@ func TestRollupShareGTOverTime(t *testing.T) { f(1000, 0) } +func TestRollupShareEQOverTime(t *testing.T) { + f := func(eq, vExpected float64) { + t.Helper() + eqs := []*timeseries{{ + Values: []float64{eq}, + Timestamps: []int64{123}, + }} + var me metricsql.MetricExpr + args := []interface{}{&metricsql.RollupExpr{Expr: &me}, eqs} + testRollupFunc(t, "share_eq_over_time", args, &me, vExpected) + } + + f(-123, 0) + f(34, 0.3333333333333333) + f(44, 0.16666666666666666) + f(123, 0.08333333333333333) + f(1000, 0) +} + func TestRollupCountLEOverTime(t *testing.T) { f := func(le, vExpected float64) { t.Helper() diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index 2f59fda7e5..e5106fe6cf 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -744,6 +744,14 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k See also [share_gt_over_time](#share_gt_over_time). +#### share_eq_over_time + +`share_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which returns share (in the range `[0...1]`) of raw samples +on the given lookbehind window `d`, which are equal to `eq`. It is calculated independently per each time series returned +from the given [series_selector](https://docs.victoriametrics.com/keyConcepts.html#filtering). + +Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. + #### stale_samples_over_time `stale_samples_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number From 1a43ee11d1186f72d95c7f0fdfaf58cc6974a5bc Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Mon, 31 Jul 2023 15:27:26 +0200 Subject: [PATCH 09/69] docs: mention https://github.com/VictoriaMetrics/VictoriaMetrics/commit/3f6efab6ae4b3d5f8d95b749858823dfc62fe4fc in changelog Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1fe81b9b9d..8d00e396e2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,8 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip +* FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). + * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). From 4eecd4d0b362363bb32b0b9ffc74ab359c330325 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Mon, 31 Jul 2023 15:39:42 +0200 Subject: [PATCH 10/69] vendor: `make vendor-update` Follow-up after 3f6efab6ae4b3d5f8d95b749858823dfc62fe4fc Related to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441 Signed-off-by: hagen1778 --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/VictoriaMetrics/metricsql/rollup.go | 1 + vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6ab2a92bfe..2bab47497e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.2.0 github.com/VictoriaMetrics/metrics v1.24.0 - github.com/VictoriaMetrics/metricsql v0.61.1 + github.com/VictoriaMetrics/metricsql v0.62.0 github.com/aws/aws-sdk-go-v2 v1.19.0 github.com/aws/aws-sdk-go-v2/config v1.18.29 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.73 diff --git a/go.sum b/go.sum index 401f669256..b61a1689ef 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/VictoriaMetrics/fasthttp v1.2.0 h1:nd9Wng4DlNtaI27WlYh5mGXCJOmee/2c2b github.com/VictoriaMetrics/fasthttp v1.2.0/go.mod h1:zv5YSmasAoSyv8sBVexfArzFDIGGTN4TfCKAtAw7IfE= github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw= github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys= -github.com/VictoriaMetrics/metricsql v0.61.1 h1:vYkVDVa+ROvrDkhlrCzBLKB273tE6N681ftfZP+Lav8= -github.com/VictoriaMetrics/metricsql v0.61.1/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= +github.com/VictoriaMetrics/metricsql v0.62.0 h1:g8Iv8HPuAtgZAJhIIHTHct3eby/ZTpW1zMlJBE3O01c= +github.com/VictoriaMetrics/metricsql v0.62.0/go.mod h1:k4UaP/+CjuZslIjd+kCigNG9TQmUqh5v0TP/nMEy90I= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go index b049508955..da3204adc1 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/rollup.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/rollup.go @@ -64,6 +64,7 @@ var rollupFuncs = map[string]bool{ "scrape_interval": true, "share_gt_over_time": true, "share_le_over_time": true, + "share_eq_over_time": true, "stale_samples_over_time": true, "stddev_over_time": true, "stdvar_over_time": true, diff --git a/vendor/modules.txt b/vendor/modules.txt index 31f359cb40..bafba7cee8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.24.0 ## explicit; go 1.20 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.61.1 +# github.com/VictoriaMetrics/metricsql v0.62.0 ## explicit; go 1.13 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop From 9ede3e996b91fcec90389756d5b4b3a7d7ef6016 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 31 Jul 2023 16:39:57 +0200 Subject: [PATCH 11/69] vmalert: remove deprecated in v1.61.0 `-rule.configCheckInterval` (#4745) Use `-configCheckInterval` command-line flag instead. Signed-off-by: hagen1778 --- app/vmalert/README.md | 2 -- app/vmalert/main.go | 7 ------- app/vmalert/main_test.go | 10 +++++----- docs/CHANGELOG.md | 1 + docs/vmalert.md | 2 -- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 4015aaa2c4..4f06a64317 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -1243,8 +1243,6 @@ The shortlist of configuration flags is the following: See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage Supports an array of values separated by comma or specified via multiple flags. - -rule.configCheckInterval duration - Interval for checking for changes in '-rule' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead -rule.maxResolveDuration duration Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group. -rule.resendDelay duration diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 358dae7104..5a15eff229 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -55,9 +55,6 @@ absolute path to all .tpl files in root. -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively. `) - rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+ - "By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead") - configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+ "By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.") @@ -311,10 +308,6 @@ See the docs at https://docs.victoriametrics.com/vmalert.html . func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) { var configCheckCh <-chan time.Time checkInterval := *configCheckInterval - if checkInterval == 0 && *rulesCheckInterval > 0 { - logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead") - checkInterval = *rulesCheckInterval - } if checkInterval > 0 { ticker := time.NewTicker(checkInterval) configCheckCh = ticker.C diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go index ce495df4cd..ace1118514 100644 --- a/app/vmalert/main_test.go +++ b/app/vmalert/main_test.go @@ -92,7 +92,7 @@ groups: } writeToFile(t, f.Name(), rules1) - *rulesCheckInterval = 200 * time.Millisecond + *configCheckInterval = 200 * time.Millisecond *rulePath = []string{f.Name()} ctx, cancel := context.WithCancel(context.Background()) @@ -117,14 +117,14 @@ groups: return len(m.groups) } - time.Sleep(*rulesCheckInterval * 2) + time.Sleep(*configCheckInterval * 2) groupsLen := lenLocked(m) if groupsLen != 1 { t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen) } writeToFile(t, f.Name(), rules2) - time.Sleep(*rulesCheckInterval * 2) + time.Sleep(*configCheckInterval * 2) groupsLen = lenLocked(m) if groupsLen != 2 { fmt.Println(m.groups) @@ -133,7 +133,7 @@ groups: writeToFile(t, f.Name(), rules1) procutil.SelfSIGHUP() - time.Sleep(*rulesCheckInterval / 2) + time.Sleep(*configCheckInterval / 2) groupsLen = lenLocked(m) if groupsLen != 1 { t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen) @@ -141,7 +141,7 @@ groups: writeToFile(t, f.Name(), `corrupted`) procutil.SelfSIGHUP() - time.Sleep(*rulesCheckInterval / 2) + time.Sleep(*configCheckInterval / 2) groupsLen = lenLocked(m) if groupsLen != 1 { // should remain unchanged t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8d00e396e2..916f832508 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -25,6 +25,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). diff --git a/docs/vmalert.md b/docs/vmalert.md index b70bfd0e86..384fd8c832 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -1254,8 +1254,6 @@ The shortlist of configuration flags is the following: See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage Supports an array of values separated by comma or specified via multiple flags. - -rule.configCheckInterval duration - Interval for checking for changes in '-rule' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead -rule.maxResolveDuration duration Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group. -rule.resendDelay duration From 3b524f671b8a0cafdb07669a869bc85565978f87 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Mon, 31 Jul 2023 16:47:50 +0200 Subject: [PATCH 12/69] docs: rm typo of naming vmalert as a stateless service Signed-off-by: hagen1778 --- app/vmalert/README.md | 6 +++--- docs/vmalert.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 4f06a64317..72077ebef5 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -361,9 +361,9 @@ For recording rules to work `-remoteWrite.url` must be specified. ### Alerts state on restarts -`vmalert` is stateless, it holds alerts state in the process memory. Restarting of `vmalert` process -will reset alerts state in memory. To prevent `vmalert` from losing alerts state it should be configured -to persist the state to the remote destination via the following flags: +`vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts +in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state +to the remote database via the following flags: * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state to the configured address in the form of [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series) diff --git a/docs/vmalert.md b/docs/vmalert.md index 384fd8c832..d941caf0c4 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -372,9 +372,9 @@ For recording rules to work `-remoteWrite.url` must be specified. ### Alerts state on restarts -`vmalert` is stateless, it holds alerts state in the process memory. Restarting of `vmalert` process -will reset alerts state in memory. To prevent `vmalert` from losing alerts state it should be configured -to persist the state to the remote destination via the following flags: +`vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts +in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state +to the remote database via the following flags: * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state to the configured address in the form of [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series) From 216d4091f7a8c78066cecccb2ec5fb6c042f5bd5 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 31 Jul 2023 16:51:41 +0200 Subject: [PATCH 13/69] vmalert: remove deprecated in v1.79.0 web links with `*/status` suffix (#4747) Links of form `/api/v1///status` were deprecated in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links in v1.79.0. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825 This change removes code responsible for deprecated functionality. Signed-off-by: hagen1778 --- app/vmalert/web.go | 58 ++--------------------------------------- app/vmalert/web_test.go | 17 ------------ docs/CHANGELOG.md | 1 + 3 files changed, 3 insertions(+), 73 deletions(-) diff --git a/app/vmalert/web.go b/app/vmalert/web.go index ec24a3edb5..3d91fb6aa7 100644 --- a/app/vmalert/web.go +++ b/app/vmalert/web.go @@ -138,27 +138,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool { return true default: - // Support of deprecated links: - // * /api/v1///status - // * //status - // TODO: to remove in next versions - - if !strings.HasSuffix(r.URL.Path, "/status") { - httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path) - return false - } - alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/")) - if err != nil { - httpserver.Errorf(w, r, "%s", err) - return true - } - - redirectURL := alert.WebLink() - if strings.HasPrefix(r.URL.Path, "/api/v1/") { - redirectURL = alert.APILink() - } - httpserver.Redirect(w, "/"+redirectURL) - return true + httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path) + return false } } @@ -302,41 +283,6 @@ func (rh *requestHandler) listAlerts() ([]byte, error) { return b, nil } -func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) { - if strings.HasPrefix(path, "/vmalert") { - path = strings.TrimLeft(path, "/vmalert") - } - parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", -1) - if len(parts) != 3 { - return nil, &httpserver.ErrorWithStatusCode{ - Err: fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path), - StatusCode: http.StatusBadRequest, - } - } - groupID, err := uint64FromPath(parts[0]) - if err != nil { - return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err)) - } - alertID, err := uint64FromPath(parts[1]) - if err != nil { - return nil, badRequest(fmt.Errorf(`cannot parse alertID: %w`, err)) - } - resp, err := rh.m.AlertAPI(groupID, alertID) - if err != nil { - return nil, errResponse(err, http.StatusNotFound) - } - return resp, nil -} - -func uint64FromPath(path string) (uint64, error) { - s := strings.TrimRight(path, "/") - return strconv.ParseUint(s, 10, 0) -} - -func badRequest(err error) *httpserver.ErrorWithStatusCode { - return errResponse(err, http.StatusBadRequest) -} - func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode { return &httpserver.ErrorWithStatusCode{ Err: err, diff --git a/app/vmalert/web_test.go b/app/vmalert/web_test.go index a036a586fb..951e92e96e 100644 --- a/app/vmalert/web_test.go +++ b/app/vmalert/web_test.go @@ -145,23 +145,6 @@ func TestHandler(t *testing.T) { t.Errorf("expected 1 group got %d", length) } }) - - // check deprecated links support - // TODO: remove as soon as deprecated links removed - t.Run("/api/v1/0/0/status", func(t *testing.T) { - alert := &APIAlert{} - getResp(ts.URL+"/api/v1/0/0/status", alert, 200) - expAlert := ar.newAlertAPI(*ar.alerts[0]) - if !reflect.DeepEqual(alert, expAlert) { - t.Errorf("expected %v is equal to %v", alert, expAlert) - } - }) - t.Run("/api/v1/0/1/status", func(t *testing.T) { - getResp(ts.URL+"/api/v1/0/1/status", nil, 404) - }) - t.Run("/api/v1/1/0/status", func(t *testing.T) { - getResp(ts.URL+"/api/v1/1/0/status", nil, 404) - }) } func TestEmptyResponse(t *testing.T) { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 916f832508..626f61e06d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -26,6 +26,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead. +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). From 525c44e916b59942d3844e06e154cdb62cbede5c Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Mon, 31 Jul 2023 16:53:32 +0200 Subject: [PATCH 14/69] app/vmctl: remove ping from remote read protocol (#4749) Ping method was never used, so we remove it. --- app/vmctl/remoteread/remoteread.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/app/vmctl/remoteread/remoteread.go b/app/vmctl/remoteread/remoteread.go index c27ed799a6..fe458f0184 100644 --- a/app/vmctl/remoteread/remoteread.go +++ b/app/vmctl/remoteread/remoteread.go @@ -23,7 +23,6 @@ import ( const ( defaultReadTimeout = 5 * time.Minute remoteReadPath = "/api/v1/read" - healthPath = "/-/healthy" ) // StreamCallback is a callback function for processing time series @@ -154,23 +153,6 @@ func (c *Client) do(req *http.Request) (*http.Response, error) { return c.c.Do(req) } -// Ping checks the health of the read source -func (c *Client) Ping() error { - url := c.addr + healthPath - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return fmt.Errorf("cannot create request to %q: %s", url, err) - } - resp, err := c.do(req) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status code: %d", resp.StatusCode) - } - return nil -} - func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback) error { r := bytes.NewReader(data) url := c.addr + remoteReadPath From d322ee4b35ab1a2335618406fc2bccf789d1a22b Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Mon, 31 Jul 2023 16:55:59 +0200 Subject: [PATCH 15/69] app/vmctl: add support the `week` step for time-based chunks (#4743) https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738 --- app/vmctl/stepper/split.go | 6 +++ app/vmctl/stepper/split_test.go | 76 +++++++++++++++++++++++++++++++++ docs/CHANGELOG.md | 1 + 3 files changed, 83 insertions(+) diff --git a/app/vmctl/stepper/split.go b/app/vmctl/stepper/split.go index 11714cfb5b..8f890f6590 100644 --- a/app/vmctl/stepper/split.go +++ b/app/vmctl/stepper/split.go @@ -10,6 +10,8 @@ const ( StepMonth string = "month" // StepDay represents a one day interval StepDay string = "day" + // StepWeek represents a one week interval + StepWeek string = "week" // StepHour represents a one hour interval StepHour string = "hour" // StepMinute represents a one minute interval @@ -40,6 +42,10 @@ func SplitDateRange(start, end time.Time, step string) ([][]time.Time, error) { nextStep = func(t time.Time) (time.Time, time.Time) { return t, t.AddDate(0, 0, 1) } + case StepWeek: + nextStep = func(t time.Time) (time.Time, time.Time) { + return t, t.Add(7 * 24 * time.Hour) + } case StepHour: nextStep = func(t time.Time) (time.Time, time.Time) { return t, t.Add(time.Hour * 1) diff --git a/app/vmctl/stepper/split_test.go b/app/vmctl/stepper/split_test.go index 438485d4ae..062ffbffad 100644 --- a/app/vmctl/stepper/split_test.go +++ b/app/vmctl/stepper/split_test.go @@ -170,6 +170,82 @@ func Test_splitDateRange(t *testing.T) { }, wantErr: false, }, + { + name: "week chunking with not full week", + args: args{ + start: "2023-07-30T00:00:00Z", + end: "2023-08-05T23:59:59.999999999Z", + granularity: StepWeek, + }, + want: []testTimeRange{ + { + "2023-07-30T00:00:00Z", + "2023-08-05T23:59:59.999999999Z", + }, + }, + }, + { + name: "week chunking with start of the week and end of the week", + args: args{ + start: "2023-07-30T00:00:00Z", + end: "2023-08-06T00:00:00Z", + granularity: StepWeek, + }, + want: []testTimeRange{ + { + "2023-07-30T00:00:00Z", + "2023-08-06T00:00:00Z", + }, + }, + }, + { + name: "week chunking with next one day week", + args: args{ + start: "2023-07-30T00:00:00Z", + end: "2023-08-07T01:12:00Z", + granularity: StepWeek, + }, + want: []testTimeRange{ + { + "2023-07-30T00:00:00Z", + "2023-08-06T00:00:00Z", + }, + { + "2023-08-06T00:00:00Z", + "2023-08-07T01:12:00Z", + }, + }, + }, + { + name: "week chunking with month and not full week representation", + args: args{ + start: "2023-07-30T00:00:00Z", + end: "2023-09-01T01:12:00Z", + granularity: StepWeek, + }, + want: []testTimeRange{ + { + "2023-07-30T00:00:00Z", + "2023-08-06T00:00:00Z", + }, + { + "2023-08-06T00:00:00Z", + "2023-08-13T00:00:00Z", + }, + { + "2023-08-13T00:00:00Z", + "2023-08-20T00:00:00Z", + }, + { + "2023-08-20T00:00:00Z", + "2023-08-27T00:00:00Z", + }, + { + "2023-08-27T00:00:00Z", + "2023-09-01T01:12:00Z", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 626f61e06d..b1e851a076 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,6 +29,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From f35d27aa2beebde23e152dab1aeb40d708b8d496 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 31 Jul 2023 07:45:52 -0700 Subject: [PATCH 16/69] app/vlstorage: expose vl_data_size_bytes metric at /metrics page for tracking the on-disk data size (both indexdb and the data itself) --- app/vlstorage/main.go | 31 +++++++++++++++++++++++++------ dashboards/victorialogs.json | 4 ++-- docs/VictoriaLogs/CHANGELOG.md | 4 ++++ lib/logstorage/indexdb.go | 20 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/app/vlstorage/main.go b/app/vlstorage/main.go index 79bcdd93c4..efd8dfc219 100644 --- a/app/vlstorage/main.go +++ b/app/vlstorage/main.go @@ -121,24 +121,25 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set { return float64(m().FileMergesTotal) }) - ms.NewGauge(`vl_rows{type="inmemory"}`, func() float64 { + ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 { return float64(m().InmemoryRowsCount) }) - ms.NewGauge(`vl_rows{type="file"}`, func() float64 { + ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 { return float64(m().FileRowsCount) }) - ms.NewGauge(`vl_parts{type="inmemory"}`, func() float64 { + ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 { return float64(m().InmemoryParts) }) - ms.NewGauge(`vl_parts{type="file"}`, func() float64 { + ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 { return float64(m().FileParts) }) - ms.NewGauge(`vl_blocks{type="inmemory"}`, func() float64 { + ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 { return float64(m().InmemoryBlocks) }) - ms.NewGauge(`vl_blocks{type="file"}`, func() float64 { + ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 { return float64(m().FileBlocks) }) + ms.NewGauge(`vl_partitions`, func() float64 { return float64(m().PartitionsCount) }) @@ -146,6 +147,24 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set { return float64(m().StreamsCreatedTotal) }) + ms.NewGauge(`vl_indexdb_rows`, func() float64 { + return float64(m().IndexdbItemsCount) + }) + ms.NewGauge(`vl_indexdb_parts`, func() float64 { + return float64(m().IndexdbPartsCount) + }) + ms.NewGauge(`vl_indexdb_blocks`, func() float64 { + return float64(m().IndexdbBlocksCount) + }) + + ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 { + return float64(m().IndexdbSizeBytes) + }) + ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 { + dm := m() + return float64(dm.CompressedInmemorySize + dm.CompressedFileSize) + }) + ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 { return float64(m().CompressedInmemorySize) }) diff --git a/dashboards/victorialogs.json b/dashboards/victorialogs.json index 487216fea2..ff6fc7f3ff 100644 --- a/dashboards/victorialogs.json +++ b/dashboards/victorialogs.json @@ -98,7 +98,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(vl_rows)", + "expr": "sum(vl_storage_rows)", "format": "time_series", "instant": true, "interval": "", @@ -776,4 +776,4 @@ "uid": "OqPIZTX4z", "version": 4, "weekStart": "" -} \ No newline at end of file +} diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index c23bf0e36c..be91f975c6 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -5,6 +5,10 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta ## tip +* FEATURE: expose the following metrics at [/metrics](monitoring) page: + * `vl_data_size_bytes{type="storage"}` - on-disk size for data excluding [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes. + * `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes. + ## [v0.3.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.3.0-victorialogs) Released at 2023-07-20 diff --git a/lib/logstorage/indexdb.go b/lib/logstorage/indexdb.go index 4e69a9cdf3..04a1b82f59 100644 --- a/lib/logstorage/indexdb.go +++ b/lib/logstorage/indexdb.go @@ -33,6 +33,18 @@ const ( type IndexdbStats struct { // StreamsCreatedTotal is the number of log streams created since the indexdb initialization. StreamsCreatedTotal uint64 + + // IndexdbSizeBytes is the size of data in indexdb. + IndexdbSizeBytes uint64 + + // IndexdbItemsCount is the number of items in indexdb. + IndexdbItemsCount uint64 + + // IndexdbBlocksCount is the number of blocks in indexdb. + IndexdbBlocksCount uint64 + + // IndexdbPartsCount is the number of parts in indexdb. + IndexdbPartsCount uint64 } type indexdb struct { @@ -88,6 +100,14 @@ func (idb *indexdb) debugFlush() { func (idb *indexdb) updateStats(d *IndexdbStats) { d.StreamsCreatedTotal += atomic.LoadUint64(&idb.streamsCreatedTotal) + + var tm mergeset.TableMetrics + idb.tb.UpdateMetrics(&tm) + + d.IndexdbSizeBytes += tm.InmemorySizeBytes + tm.FileSizeBytes + d.IndexdbItemsCount += tm.InmemoryItemsCount + tm.FileItemsCount + d.IndexdbPartsCount += tm.InmemoryPartsCount + tm.FilePartsCount + d.IndexdbBlocksCount += tm.InmemoryBlocksCount + tm.FileBlocksCount } func (idb *indexdb) appendStreamTagsByStreamID(dst []byte, sid *streamID) []byte { From 837445b81bb6fe7f4e6da72559e297a3df23dc1a Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Tue, 1 Aug 2023 02:59:39 +0800 Subject: [PATCH 17/69] Automatic update operator docs from VictoriaMetrics/operator@dd980c8 (#4751) --- docs/operator/CHANGELOG.md | 1441 ++++++++++++++++++++++++++++++++++++ docs/operator/api.md | 99 ++- docs/operator/vars.md | 23 +- 3 files changed, 1526 insertions(+), 37 deletions(-) create mode 100644 docs/operator/CHANGELOG.md diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md new file mode 100644 index 0000000000..45dc37c40f --- /dev/null +++ b/docs/operator/CHANGELOG.md @@ -0,0 +1,1441 @@ +# CHANGELOG + +## tip + +### Breaking changes + +- **[vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): Field `OAuth2` in (HTTPAuth)[https://docs.victoriametrics.com/operator/api.html#httpauth] was renamed to `oauth2`. See [this issue](https://github.com/VictoriaMetrics/operator/issues/522) and [this PR](https://github.com/VictoriaMetrics/operator/pull/689) for details.** +- **[vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): Field `bearerTokenFilePath` in (BearerAuth)[https://docs.victoriametrics.com/operator/api.html#bearerauth] was renamed to `bearerTokenFile`. See [this issue](https://github.com/VictoriaMetrics/operator/issues/522) and [this PR](https://github.com/VictoriaMetrics/operator/pull/688/) for details.** +- **These changes affect following fields:** + - **`VMAlert.spec.datasource.OAuth2` -> `VMAlert.spec.datasource.oauth2`,** + - **`VMAlert.spec.notifier.OAuth2` -> `VMAlert.spec.notifier.oauth2`,** + - **`VMAlert.spec.notifiers[].OAuth2` -> `VMAlert.spec.notifiers[].oauth2`,** + - **`VMAlert.spec.remoteRead.OAuth2` -> `VMAlert.spec.remoteRead.oauth2`,** + - **`VMAlert.spec.remoteWrite.OAuth2` -> `VMAlert.spec.remoteWrite.oauth2`,** + - **`VMAlert.spec.datasource.bearerTokenFilePath` --> `VMAlert.spec.datasource.bearerTokenFile`,** + - **`VMAlert.spec.notifier.bearerTokenFilePath` --> `VMAlert.spec.notifier.bearerTokenFile`,** + - **`VMAlert.spec.notifiers[].bearerTokenFile` --> `VMAlert.spec.notifiers[].bearerTokenFile`,** + - **`VMAlert.spec.remoteRead.bearerTokenFilePath` --> `VMAlert.spec.remoteRead.bearerTokenFile`,** + - **`VMAlert.spec.remoteWrite.bearerTokenFilePath` --> `VMAlert.spec.remoteWrite.bearerTokenFile`.** + +### Fixes + +- operator set resource requests for config-reloader container by default. See [this PR](https://github.com/VictoriaMetrics/operator/pull/695/) for details. +- fix `attachMetadata` value miscovert for scrape objects. See [this issue](https://github.com/VictoriaMetrics/operator/issues/697) and [this PR](https://github.com/VictoriaMetrics/operator/pull/698) for details. +- [vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): fix `tlsCAFile` argument value generation when using secret or configMap. See [this issue](https://github.com/VictoriaMetrics/operator/issues/699) and [this PR](https://github.com/VictoriaMetrics/operator/issues/699) for details. +- [vmalertmanager](https://docs.victoriametrics.com/operator/api.html#vmalertmanager): fix default request memory and apply default resources if not set. See [this issue](https://github.com/VictoriaMetrics/operator/issues/706) and [this PR](https://github.com/VictoriaMetrics/operator/pull/710) for details. + +### Features + +- [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): add [example config](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmagent_stateful_with_sharding.yaml) for vmagent statefulmode. +- [vmcluster](https://docs.victoriametrics.com/operator/api.html#vmagent): add [example config](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmcluster_with_additional_claim.yaml) for cluster with custom storage claims. +- [vmrule](https://docs.victoriametrics.com/operator/api.html#vmrule): support `update_entries_limit` field in rules, refer to [alerting rules](https://docs.victoriametrics.com/vmalert.html#alerting-rules). See [this PR](https://github.com/VictoriaMetrics/operator/pull/691) for details. +- [vmrule](https://docs.victoriametrics.com/operator/api.html#vmrule): support `keep_firing_for` field in rules, refer to [alerting rules](https://docs.victoriametrics.com/vmalert.html#alerting-rules). See [this PR](https://github.com/VictoriaMetrics/operator/pull/711) for details. +- [vmoperator parameters](https://docs.victoriametrics.com/operator/vars.html): Add option `VM_ENABLESTRICTSECURITY` and enable strict security context by default. See [this issue](https://github.com/VictoriaMetrics/operator/issues/637) and [this PR](https://github.com/VictoriaMetrics/operator/pull/692/) for details. + + +## [v0.35.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.35.1) - 12 Jul 2023 + +### Fixes + +- [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): fixes regression with remoteWrite authorization (basicAuth/token). When `UseCustomConfigReloader` option was set, operator incorrectly rendered mounts for `vmagent` container. https://github.com/VictoriaMetrics/operator/commit/f2b8cf701a33f91cef19848c857fd6efb7db59dd + +[Changes][v0.35.1] + + + +## [v0.35.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.35.0) - 03 Jul 2023 + +### Fixes + +* [vmuser](https://docs.victoriametrics.com/operator/api.html#vmuser): fix vmselect url_map in vmuser. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/655). Thanks [@Haleygo](https://github.com/Haleygo) +* [vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): correctly set default port for vmauth components discovery. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/658). Thanks [@Haleygo](https://github.com/Haleygo) +* [vmuser](https://docs.victoriametrics.com/operator/api.html#vmuser): remove rate limit on delete. In https://github.com/VictoriaMetrics/operator/pull/672. Thanks [@Haleygo](https://github.com/Haleygo) +* [vmcluster](https://docs.victoriametrics.com/operator/api.html#vmuser): fix spec change check. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/677). Thanks [@Haleygo](https://github.com/Haleygo) +* Correctly publish multi-arch release at https://github.com/VictoriaMetrics/operator/pull/681. Thanks [@Haleygo](https://github.com/Haleygo) + +### Features + +* [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent):add validation when generate static scrape config. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/677). Thanks [@Haleygo](https://github.com/Haleygo) +* [vmalertmanagerconfig](https://docs.victoriametrics.com/operator/api.html#vmalertmanagerconfig) : add validation for slack receiver url. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/661). Thanks [@Haleygo](https://github.com/Haleygo) +* [vmauth/vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): implement configuration initiation for custom config reloader. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/619). Thanks [@Haleygo](https://github.com/Haleygo) +* add more generators Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/668 +* [vmsingle](https://docs.victoriametrics.com/operator/api.html#vmsingle): add status field. See [this issue for details](https://github.com/VictoriaMetrics/operator/issues/670). Thanks [@Haleygo](https://github.com/Haleygo) + +[Changes][v0.35.0] + + + +## [v0.34.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.34.1) - 29 May 2023 + +### Fixes + +- [VMCluster] - fail fast on misconfigured or missing kubernetes pods. It should prevent rare bug with cascade pod deletion. See this [issue](https://github.com/VictoriaMetrics/operator/issues/643) for details +- [VMAuth/VMAgent]- correctly renders initConfig image with global container registry domain. See this [issue](https://github.com/VictoriaMetrics/operator/issues/654) for details. +- [VMAgent] - correctly set RBAC permissions for single namespace mode and custom config reloader image. See this [issue](https://github.com/VictoriaMetrics/operator/issues/653) for details. + +[Changes][v0.34.1] + + + +## [v0.34.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.34.0) - 24 May 2023 + +### Breaking changes + +* **[Operator]: allows to properly run operator with single namespace. It changes default behavior with WATCH_NAMESPACE param is set. Operator will no longer make any calls for cluster wide resources and create only single namespace config for `VMAgent`. https://github.com/VictoriaMetrics/operator/issues/641** + +### Fixes + +* [VMNodeScrape]: fixed selectors for Exists and NotExists operators with empty label Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/646 +* [VMRule] Add config for vmrule in validating webhook Thanks in https://github.com/VictoriaMetrics/operator/pull/650 +* [VMAgent]: skips misconfigured objects with missed secret references: https://github.com/VictoriaMetrics/operator/issues/648 +* [VMAgent]: correctly renders initContainer for configuration download: https://github.com/VictoriaMetrics/operator/issues/649 + +### Features + +* [VMAlertmanager]: Bump alertmanager to v0.25.0 Thanks [@tamcore](https://github.com/tamcore) in https://github.com/VictoriaMetrics/operator/pull/636 +* [VMCluster] added `clusterNativePort` field to VMSelect/VMInsert for multi-level cluster setup ([#634](https://github.com/VictoriaMetrics/operator/issues/634)) Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/639 +* [VMRule]: add notifierHeader field in vmrule spec Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/622 +* [VMPodScrape]: adds FilterRunning option as prometheus does in https://github.com/VictoriaMetrics/operator/pull/640 +* [VMAuth]: adds latest features in https://github.com/VictoriaMetrics/operator/pull/642 + +[Changes][v0.34.0] + + + +## [v0.33.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.33.0) - 19 Apr 2023 + +### Fixes + +* [VMAlert]: skip bad rules and improve logging for rules exceed max configmap size https://github.com/VictoriaMetrics/operator/commit/bb754d5c20bb371a197cd6ff5afac1ba86a4d92b +* [VMAlertmanagerConfig]: fixed error with headers in VMAlertmanagerConfig.Receivers.EmailConfigs.Headers unmarshalling. Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/610 +* [VMAgent]: fixed keepInput setting for streaming aggregation. Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/618 +* [VMAlertmanagerConfig]: fix webhook config maxAlerts not work. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/625 +* [VMAgent]: Remove single quotes from remote write headers. Thanks [@axelsccp](https://github.com/axelsccp) in https://github.com/VictoriaMetrics/operator/pull/613 +* [VMAlertmanagerConfig]: fix parse route error and some comments. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/630 +* [VMUser]: properly removes finalizers for objects https://github.com/VictoriaMetrics/operator/commit/8f10113920a353f21fbcc8637076905f2e57bb34 + +### Features + +* [VMAlertmanager]: add option to disable route continue enforce. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/621 +* [VMAlertmanagerConfig]: support set require_tls to false. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/624 +* [VMAlertmanagerConfig]: add sanity check. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/627 +* Makefile: bump Alpine base image to latest v3.17.3. Thanks [@denisgolius](https://github.com/denisgolius) in https://github.com/VictoriaMetrics/operator/pull/628 +* [VMAlertmanagerConfig]: support sound field in pushover config. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/631 +* [VMAgent/VMAuth]: download initial config with initContainer https://github.com/VictoriaMetrics/operator/commit/612e7c8f40659731e7938ef9556eb088c67eb4b7 + +[Changes][v0.33.0] + + + +## [v0.32.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.32.1) - 16 Mar 2023 + +### Fixes + +* [config]: fixes typo at default vm apps version https://github.com/VictoriaMetrics/operator/issues/608 +* [vmsingle]: conditionally adds stream aggregation config https://github.com/VictoriaMetrics/operator/commit/4a0ca54113afcde439ca4c77e22d3ef1c0d36241 + +[Changes][v0.32.1] + + + +## [v0.32.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.32.0) - 15 Mar 2023 + +### Fixes + +* [security]: builds docker image with latest `alpine` base image and go `v1.20`. + +### Features + +* [vmauth]: automatically configures `proxy-protocol` client and `reloadAuthKey` for `config-reloader` container. https://github.com/VictoriaMetrics/operator/commit/611819233bf595a4dbd04b07d7be24b7e994379c +* [vmagent]: adds `scrapeTimeout` global configuration for `VMAgent` https://github.com/VictoriaMetrics/operator/commit/d1d5024c6befa0961f8d56c82a0554935a4b1878 +* [vmagent]: adds [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation.html) for `remoteWrite` targets https://github.com/VictoriaMetrics/operator/commit/b8baa6c2b72bdda64ebfcc9c3d86d846cd9b3c98 Thanks [@Amper](https://github.com/Amper) +* [vmsingle]: adds [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation.html) as global configuration for database https://github.com/VictoriaMetrics/operator/commit/b8baa6c2b72bdda64ebfcc9c3d86d846cd9b3c98 Thanks [@Amper](https://github.com/Amper) + +[Changes][v0.32.0] + + + +## [v0.31.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.31.0) - 02 Mar 2023 + +### Fixes + +* [hpa]: Fix hpa object since v2beta deprecated in 1.26+ Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/593 +* [api]: adds missing generated client CRD entities https://github.com/VictoriaMetrics/operator/issues/599 + +### Features + +* [vmalertmanager]: Add support of vmalertmanager.spec.templates and autoreload dirs for templates and configmaps thanks [@Amper](https://github.com/Amper) https://github.com/VictoriaMetrics/operator/issues/590 https://github.com/VictoriaMetrics/operator/issues/592 +* [vmalertmanager]: Add support "%SHARD_NUM%" placeholder for vmagent sts/deployment Thanks [@Amper](https://github.com/Amper) https://github.com/VictoriaMetrics/operator/issues/508 + +[Changes][v0.31.0] + + + +## [v0.30.4](https://github.com/VictoriaMetrics/operator/releases/tag/v0.30.4) - 27 Jan 2023 + +### Fixes + +- vmalertmanagerconfig: properly build `name` setting for `mute_time_intervals`. It must be uniq https://github.com/VictoriaMetrics/operator/commit/4db1c89abd5360a119e68874d51c27872265acb6 +- vmcluster: add `dedupMinScrape` only if replicationFactor > 1. It must improve overall cluster perfomance. Thanks [@hagen1778](https://github.com/hagen1778) https://github.com/VictoriaMetrics/operator/commit/837d6e71c6298e5a44c3f73f85235560aec4ee60 +- controllers/vmalert: do not delete annotations from created secret. Thanks [@zoetrope](https://github.com/zoetrope) https://github.com/VictoriaMetrics/operator/pull/588 + +### Features + +- vmalertmanagerconfig: adds location, active_time_intervals https://github.com/VictoriaMetrics/operator/commit/66ee8e544f480be386a4a126a6163599ed338705 + +[Changes][v0.30.4] + + + +## [v0.30.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.30.3) - 16 Jan 2023 + +### Fixes + +- controllers: pass correct selector labels for pvc resize function https://github.com/VictoriaMetrics/operator/commit/e7b57dd73b4fd8dc37b42b7ad7bf5a4d3483caae +- controllers: kubernetes 1.26+ deprecates v2 autoscaling, add api check for it https://github.com/VictoriaMetrics/operator/issues/583 + +[Changes][v0.30.3] + + + +## [v0.30.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.30.2) - 12 Jan 2023 + +### Upgrade notes + +* It's recommend to upgrade for this release when `vmagent.spec.statefulMode` is used. + +### Fixes + +- controllers/vmagent: fixes degradation for vmagent statefulMode https://github.com/VictoriaMetrics/operator/commit/6c26786db2ba0b2e85277418e588eac79e886b6e + +[Changes][v0.30.2] + + + +## [v0.30.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.30.1) - 09 Jan 2023 + +### Fixes + +- controllers/vmalert: correctly filter notifiers for namespace selector https://github.com/VictoriaMetrics/operator/commit/2290729fcc1b3775141b54ff71a295bd29457fbd +- dependency: upgrade deps for fs-notify https://github.com/VictoriaMetrics/operator/pull/576 Thanks [@yanggangtony](https://github.com/yanggangtony) +- controllers/options: fixes incorrectly used flags at options https://github.com/VictoriaMetrics/operator/commit/eac040c947ab4821bf6eb0eeae22b9b2d02b938c +- controllers/self-serviceScrape: prevents matching for auto-created serviceScrapes https://github.com/VictoriaMetrics/operator/issues/578 +- controllers/vmauth: fixes missing ows for serviceScrape https://github.com/VictoriaMetrics/operator/issues/579 + +### Features + +- adds `/ready` and `/health` api endpoints for probes https://github.com/VictoriaMetrics/operator/commit/b74d103998547fae5e69966bb68eddd08ae1ac00 +- controllers/concurrency: introduce new setting for reconcilation concurrency `controller.maxConcurrentReconciles` https://github.com/VictoriaMetrics/operator/commit/e8bbf9159cd61257d11e515fa77510ab2444a557 https://github.com/VictoriaMetrics/operator/issues/575 +- api/relabelConfig: adds missing `if`, `labels` and `match` actions https://github.com/VictoriaMetrics/operator/commit/93c9e780981ceb6869ee2953056a9bd3b6e6eae7 + +[Changes][v0.30.1] + + + +## [v0.30.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.30.0) - 29 Dec 2022 + +### Fixes + +- vmalertmanagerconfig: fixes duplicates at configuration https://github.com/VictoriaMetrics/operator/issues/554 +- controllers: correctly set current and update revisions for statefulset https://github.com/VictoriaMetrics/operator/issues/547 +- controller/factory: fix typo in urlRelabelingName Thanks [@dmitryk-dk](https://github.com/dmitryk-dk) in https://github.com/VictoriaMetrics/operator/pull/572 +- controllers/vmalert: fixes notifier selector incorrect matching https://github.com/VictoriaMetrics/operator/issues/569 +- controllers/cluster: fixes HPA labels for vminsert https://github.com/VictoriaMetrics/operator/issues/562 + +### Features + +- adds Scaling subresource for `VMAgent`. https://github.com/VictoriaMetrics/operator/issues/570 +- add optional namespace label matcher to inhibit rule thanks [@okzheng](https://github.com/okzheng) in https://github.com/VictoriaMetrics/operator/pull/559 +- provide crds yaml as release asset Thanks [@avthart](https://github.com/avthart) in https://github.com/VictoriaMetrics/operator/pull/566 +- child labels filtering https://github.com/VictoriaMetrics/operator/pull/571 +- controllers/vmalert: adds oauth2 and bearer auth for remote dbs in https://github.com/VictoriaMetrics/operator/pull/573 + +[Changes][v0.30.0] + + + +## [v0.29.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.29.2) - 17 Nov 2022 + +### Fixes + +- vmalertmanagerconfig: fixes duplicates at configuration https://github.com/VictoriaMetrics/operator/issues/554 +- controllers: correctly set current and update revisions for statefulset https://github.com/VictoriaMetrics/operator/issues/547 + +[Changes][v0.29.2] + + + +## [v0.29.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.29.1) - 14 Nov 2022 + +### Fixes + +- some typos https://github.com/VictoriaMetrics/operator/pull/548 Thanks [@fatsheep9146](https://github.com/fatsheep9146) +- update description for parameter to match behaviour https://github.com/VictoriaMetrics/operator/pull/549 thanks [@zekker6](https://github.com/zekker6) +- controllers/factory: fix resizing of PVC for vmsingle https://github.com/VictoriaMetrics/operator/pull/551 thanks [@zekker6](https://github.com/zekker6) + +### Features + +- Expose no_stale_markers through vm_scrape_params in https://github.com/VictoriaMetrics/operator/pull/546 Thanks [@tamcore](https://github.com/tamcore) +- {api/vmsingle,api/vmcluster}: add support of `vmbackupmanager` restore on pod start https://github.com/VictoriaMetrics/operator/pull/544 thanks [@zekker6](https://github.com/zekker6) +- api: changes errors handling for objects unmarshal https://github.com/VictoriaMetrics/operator/pull/550 + +[Changes][v0.29.1] + + + +## [v0.29.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.29.0) - 24 Oct 2022 + +### Fixes + +* vmcluster: reconcile VMStorage in VMCluster even if PodDisruptionBudget does not exist by [@miketth](https://github.com/miketth) in https://github.com/VictoriaMetrics/operator/pull/535 +* crash on Kubernetes 1.25 by [@miketth](https://github.com/miketth) in https://github.com/VictoriaMetrics/operator/pull/536 +* throttling for vmagent and vmalert https://github.com/VictoriaMetrics/operator/commit/63ca52bf140b033ecbc3c40f9efc8579b936ea29 +* vmalertmanagerconfig: parsing for nested routes https://github.com/VictoriaMetrics/operator/commit/f2bc0c09069c0cec9bec8757fc3bc339231ccfdd https://github.com/VictoriaMetrics/operator/commit/9472f1fe6e69fd4bfc63d5fb3da14c02b6fb4788 +* vmalertmanagerconfig: ownerreference set correctly https://github.com/VictoriaMetrics/operator/commit/2bb5d0234c7b32f27c3f82b007fea409887b54b9 +* vmagent: allows to set maxDiskUsage more then 1GB https://github.com/VictoriaMetrics/operator/commit/47f2b508ee503d03111ec03215466a123e2d3978 +* vmagent: properly merge ports for additional service https://github.com/VictoriaMetrics/operator/commit/05d332d704fd9cf9c490de22a554badc61e86f51 +* vmprobe: correctly set labels for ingress targets https://github.com/VictoriaMetrics/operator/commit/976315cd3dbf57d576414340b1d444d63f8d460d + +### Features + +* podDistruptionBudget: adds configurable selectors https://github.com/VictoriaMetrics/operator/commit/4f3f5eaf29ad85c6e9b142be5b05ef57b962fcb6 + +### New Contributors + +* [@miketth](https://github.com/miketth) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/535 + +[Changes][v0.29.0] + + + +## [v0.28.5](https://github.com/VictoriaMetrics/operator/releases/tag/v0.28.5) - 13 Sep 2022 + +### Fixes + +* authorization cache usage https://github.com/VictoriaMetrics/operator/commit/e43bdb6c975b712bf5f169b8fa74c8f7760c82f5 Thanks [@AndrewChubatiuk](https://github.com/AndrewChubatiuk) +* claimTemplates: fixes CRD for it https://github.com/VictoriaMetrics/operator/commit/a5d2f9f61ecfc37a776d8f8c1b0f1385536e773c +* vmrules: supress notFound errors https://github.com/VictoriaMetrics/operator/issues/524 +* vmagent: fixes regression at default values for tmpDataPath and maxDiskUsage flags https://github.com/VictoriaMetrics/operator/issues/523 + +### Features + +* vmalertmanager: ignore broken receivers https://github.com/VictoriaMetrics/operator/commit/68bbce1f7809d35b42a39925c09a4ddd61f64a9c +* service accounts: do not set labels and annotations for external service accounts https://github.com/VictoriaMetrics/operator/commit/2ea1e640c362271484d0627c4ca571fd0afd74b2 + +[Changes][v0.28.5] + + + +## [v0.28.4](https://github.com/VictoriaMetrics/operator/releases/tag/v0.28.4) - 12 Sep 2022 + +### Fixes + +* authorization cache usage https://github.com/VictoriaMetrics/operator/commit/e43bdb6c975b712bf5f169b8fa74c8f7760c82f5 Thanks [@AndrewChubatiuk](https://github.com/AndrewChubatiuk) +* claimTemplates: fixes CRD for it https://github.com/VictoriaMetrics/operator/commit/a5d2f9f61ecfc37a776d8f8c1b0f1385536e773c +* vmrules: supress notFound errors https://github.com/VictoriaMetrics/operator/issues/524 +* vmagent: fixes regression at default values for tmpDataPath and maxDiskUsage flags https://github.com/VictoriaMetrics/operator/issues/523 + +### Features + +* vmalertmanager: ignore broken receivers https://github.com/VictoriaMetrics/operator/commit/68bbce1f7809d35b42a39925c09a4ddd61f64a9c +* service accounts: do not set labels and annotations for external service accounts https://github.com/VictoriaMetrics/operator/commit/2ea1e640c362271484d0627c4ca571fd0afd74b2 + +[Changes][v0.28.4] + + + +## [v0.28.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.28.3) - 02 Sep 2022 + +### Fixes + +* vmalertmanagerConfig: regression at nested routes parsing https://github.com/VictoriaMetrics/operator/commit/07ce4ca80d3ba09506fc41baaecd7087f799a8aa +* vmagent: password_file option was ignored https://github.com/VictoriaMetrics/operator/commit/5ef9710976534be651687aaa71b2110b0a1a348f + +[Changes][v0.28.3] + + + +## [v0.28.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.28.2) - 01 Sep 2022 + +### Fixes + +* vmalert: regression at basicAuth https://github.com/VictoriaMetrics/operator/commit/f92463949c9fd8be961c52d98ac7f1f956f7eba3 +* converter/alertmanager: changes parsing for nested routes - added more context and validation webhook https://github.com/VictoriaMetrics/operator/commit/6af6071db733bbccfe066b45c73d0377a082b822 + +[Changes][v0.28.2] + + + +## [v0.28.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.28.1) - 31 Aug 2022 + +### Fixes + +* vmalert: fixes generated crd https://github.com/VictoriaMetrics/operator/commit/7b5b5b27c00e6ef42edb906ff00912157d21acea + + +[Changes][v0.28.1] + + + +## [v0.28.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.28.0) - 30 Aug 2022 + +### Fixes + +* security: changes base docker image https://github.com/VictoriaMetrics/operator/commit/cda21275517f84b66786e25c5f6b76977ee27a49 +* vmagent: fixes incorrect usage of remoteWriteSettings https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2946 +* vmalert: password_file usage https://github.com/VictoriaMetrics/operator/commit/45163164662934587eafd6afed7709efa31ddbe8 + +### Features + +* converter: adds support for prometheus `AlertmanagerConfig`. It converts into `VMAlertmanagerConfig`. https://github.com/VictoriaMetrics/operator/commit/0b99bc09b2bb1fede612bc509237f6ee6c7617a5 +* vmalert: tokenFilePath support for any remote endpoint https://github.com/VictoriaMetrics/operator/commit/5b010f4abcd778d35dca7c826bfb84af0e46e08d + +[Changes][v0.28.0] + + + +## [v0.27.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.27.2) - 22 Aug 2022 + +### Fixes + +* controllers: fixes `password_file` usage at basicAuth https://github.com/VictoriaMetrics/operator/commit/979f6375d43e33c35137c1006dc3b4be4dba8528 +* config-reloader: properly call gzip.Close method https://github.com/VictoriaMetrics/operator/commit/0d3aac72caf3710172c404fbf89f9a4b125dd97c thanks [@Cosrider](https://github.com/Cosrider) + +[Changes][v0.27.2] + + + +## [v0.27.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.27.1) - 17 Aug 2022 + +### Fixes + +* controllers: fixes policy/v1 api detection https://github.com/VictoriaMetrics/operator/pull/513 + +### Features + +* vmalert: added `headers` setting for `remoteRead`, `remoteWrite` and `dataSource` https://github.com/VictoriaMetrics/operator/issues/492 + +[Changes][v0.27.1] + + + +## [v0.27.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.27.0) - 16 Aug 2022 + +### Fixes + +* Adding support tls endpoint for vmauth config reloader by [@mayurvaid-redvest](https://github.com/mayurvaid-redvest) in https://github.com/VictoriaMetrics/operator/pull/511 +* Custom config-reloader incorrectly watch for directory at `VMAgent` https://github.com/VictoriaMetrics/operator/issues/510 +* Removes validation for `telegram_configs` `parse_mode` validation https://github.com/VictoriaMetrics/operator/issues/506 +* Deletion of `VMAgent` in `StatefulMode` https://github.com/VictoriaMetrics/operator/issues/505 + +### Features + +* Allows ignoring objects at argo-cd converted from prometheus CRD with env var: `VM_PROMETHEUSCONVERTERADDARGOCDIGNOREANNOTATIONS=true` https://github.com/VictoriaMetrics/operator/issues/509 +* `claimTemplates` now supported at `VMCluster`, `VMAlertmanager`, `VMAgent` https://github.com/VictoriaMetrics/operator/issues/507 +* `readinessGates` now supported by CRD objects https://github.com/VictoriaMetrics/operator/commit/29807e65ec817f8a4f095ba5804d0644a4855e46 +* HealthChecks now respects `tls` configured at CRD objects https://github.com/VictoriaMetrics/operator/commit/e43a4d5b22d9a507b2a65839a4ca2ce56f08dff8 + +### New Contributors + +* [@mayurvaid-redvest](https://github.com/mayurvaid-redvest) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/511 + +[Changes][v0.27.0] + + + +## [v0.26.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.26.3) - 26 Jul 2022 + +### Fixes + +* removes breaking changes introduced at v0.26.0. Operator added `docker.io` as container registry prefix and it may break applications, if private repository was configured at spec.repository.image. Now container registry is not set by default. +* alertmanager: removes breaking changes introduced at 0.26.0 release with extraArgs https://github.com/VictoriaMetrics/operator/commit/918595389e62e144c8f5ebae7472bcff62ccef44 + +[Changes][v0.26.3] + + + +## [v0.26.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.26.0) - 25 Jul 2022 + +### Breaking changes + +**This release contains breaking changes that was fixed at v0.26.2 release. It's recommended to use it instead of upgrading to v0.26.0** + +### Fixes + +* security: new alpine image with security fixes https://github.com/VictoriaMetrics/operator/commit/c991b5f315ebb3176b98f5cb00c64430efa0d9c1 +* alertmanager: metrics endpoint when routePrefix is configured https://github.com/VictoriaMetrics/operator/pull/488 Thanks [@blesswinsamuel](https://github.com/blesswinsamuel) +* alertmanager: Automaticly disable high availability mode for 1 replica in https://github.com/VictoriaMetrics/operator/pull/495. Thanks [@hadesy](https://github.com/hadesy) +* vmalertmanager: fix extraArgs, add two dashes https://github.com/VictoriaMetrics/operator/pull/503 Thanks [@flokli](https://github.com/flokli) +* vmcluster: disables selectNode arg passing to vmselect with enabled `HPA`. It should prevent vmselect cascade restarts https://github.com/VictoriaMetrics/operator/issues/499 +* controllers: changes default rate limiter max delay from 16minutes to 2 minutes. https://github.com/VictoriaMetrics/operator/issues/500 +* vmagent: now properly changes size for volumes at persistentMode https://github.com/VictoriaMetrics/operator/commit/81f09af5fd3b96c975cdd7b797d02e442e2d96d0 +* prometheus converter: adds some missing fields, bumps version dependecy https://github.com/VictoriaMetrics/operator/commit/35f1c26d98e10db06f561e51ee5ff02b9ad72f9d + +### Features + +* api/v1beta1/VMUser: adds tokenRef https://github.com/VictoriaMetrics/operator/pull/489 +* api/vmauth: adds host param for ingress https://github.com/VictoriaMetrics/operator/pull/490 +* api/vmcluster: reworks expanding for cluster https://github.com/VictoriaMetrics/operator/pull/494 +* global setting to override container registry by in https://github.com/VictoriaMetrics/operator/pull/501 Thanks [@tamcore](https://github.com/tamcore) +* api: new versioned kubernetes client https://github.com/VictoriaMetrics/operator/issues/481 +* api: adds `authorization` configuration for scrape targets +* api: adds `headers` fields for custom headers passing to targets https://github.com/VictoriaMetrics/operator/commit/0553b60090e51ec800bdbc3698b16752c6551944 +* vmagent: adds `headers` configuration per remote storage urls https://github.com/VictoriaMetrics/operator/commit/e0567210098ad53f9c17cc3e260eaab5f754b2f9 +* vmagent: allow configuring multitenant mode for remote storage urls https://github.com/VictoriaMetrics/operator/commit/e0567210098ad53f9c17cc3e260eaab5f754b2f9 + +### New Contributors + +* [@blesswinsamuel](https://github.com/blesswinsamuel) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/488 +* [@hadesy](https://github.com/hadesy) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/495 +* [@tamcore](https://github.com/tamcore) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/501 + +[Changes][v0.26.0] + + + +## [v0.25.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.25.1) - 20 May 2022 + +### Fixes + +- PersistentVolumeClaim creation for StatefulSet https://github.com/VictoriaMetrics/operator/pull/483 Thanks [@cnych](https://github.com/cnych) + +[Changes][v0.25.1] + + + +## [v0.25.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.25.0) - 19 May 2022 + +### Breaking changes +- **Changes `VMRule` API, now `expr` field can be only `string`, `integer` values are not supported anymore. https://github.com/VictoriaMetrics/operator/commit/f468ae02690e79ed72638f845535d19418b042af** + +### Fixes + +- PagerDuty config generation https://github.com/VictoriaMetrics/operator/commit/eef8e2eece269d1c64094b2f7cdf69beabaa3739 thanks [@okzheng](https://github.com/okzheng) +- missing `honorTimestamps` for `ServiceMonitor` to `VMServiceScrape` conversion https://github.com/VictoriaMetrics/operator/commit/6728391cc76576fd97571b2efc3bd24c94a4f083 thanks [@gotosre](https://github.com/gotosre) +- PVC volume automatic expansion for `VMCluster` and `VMAlertmanager` https://github.com/VictoriaMetrics/operator/commit/1eac5826b07e7255309b1b9971730e2b79610f85 + +### Features + +- Added `name` field for `VMUser` https://github.com/VictoriaMetrics/operator/issues/472 thanks [@pavan541cs](https://github.com/pavan541cs) +- Added `StatefulMode` for `VMAgent` it allows to use `Statefulset` instead of `Deployment` https://github.com/VictoriaMetrics/operator/issues/219 +- Added `Validation Webhook` for `VMRule`, it allows check errors at rules https://github.com/VictoriaMetrics/operator/issues/471 +- Added additional metrics for operator `operator_log_messages_total`, `operator_controller_objects_count`, `operator_reconcile_throttled_events_total`, `vm_app_version`, `vm_app_uptime_seconds`, `vm_app_start_timestamp` https://github.com/VictoriaMetrics/operator/commit/b941a42fb6fdfd8ea99ff190e822cb9314efb9d0 https://github.com/VictoriaMetrics/operator/commit/b3c7286e7dc737c46c4d33aa203c0b598a5ef187 +- Adds rate limiting for `VMAgent` and `VMAlert` reconcilation https://github.com/VictoriaMetrics/operator/commit/dfb6a14e1193089ba5ab112e0acf4e459aba68b4 + +### New Contributors +* [@pavan541cs](https://github.com/pavan541cs) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/473 +* [@gotosre](https://github.com/gotosre) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/475 + +[Changes][v0.25.0] + + + +## [v0.24.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.24.0) - 11 Apr 2022 + +### Fixes + +- Finalizers at UrlRelabelConfig and additionalScrapeConfigs https://github.com/VictoriaMetrics/operator/issues/442 +- vmagent config update after scrape objects secret data changes https://github.com/VictoriaMetrics/operator/issues/443 +- Log typos https://github.com/VictoriaMetrics/operator/issues/459 +- Correctly renders `opsgenia_config` for `VMAlertmanagerConfig` https://github.com/VictoriaMetrics/operator/commit/9128b7f24d5d6d98dcf7abc6f212d57cd39b0e7d thanks [@iyuroch](https://github.com/iyuroch) +- Updates basic image with CVE fix https://github.com/VictoriaMetrics/operator/commit/f4a9e530be6d5ebd6e450085ec807117b05e80a8 +- Adds missing finalizer for `VMSingle` deployment https://github.com/VictoriaMetrics/operator/commit/06dada488d629d4d321985e80d14ee04e099bdfd thanks [@lujiajing1126](https://github.com/lujiajing1126) +- `pager_duty` generation for `VMAlertmanagerConfig` https://github.com/VictoriaMetrics/operator/pull/439/files thanks [@okzheng](https://github.com/okzheng) +- `VMServiceScrape` generation for `vminsert`, previously opentsdb-http port could be included into it https://github.com/VictoriaMetrics/operator/issues/420 + +### Features + +- Allows filtering for Converted Prometheus CRD objects https://github.com/VictoriaMetrics/operator/issues/444 +- Allows overwriting for default arg params https://github.com/VictoriaMetrics/operator/issues/448 +- Allows customization for VMServiceScrape objects generated by operator for it's resources https://github.com/VictoriaMetrics/operator/issues/454 https://github.com/VictoriaMetrics/operator/commit/130e54781e1b193e9e65573df0b76440560db57e Thanks [@artifactori](https://github.com/artifactori) +- Allows configure `terminationGracePeriodSeconds` for CRD objects https://github.com/VictoriaMetrics/operator/issues/460 +- Allows configure `dnsConfig` for CRD objects https://github.com/VictoriaMetrics/operator/commit/dca0b48a175635cecdaf2fe04ea714eb74eecc79 thanks [@fatsheep9146](https://github.com/fatsheep9146) +- Adds `telegram_configs` for `VMAlertmanagerConfig` https://github.com/VictoriaMetrics/operator/commit/076b7d9665e6ac2979421bd8445083dc08cc32ee +- Allows set retentionPeriod less then 1 month https://github.com/VictoriaMetrics/operator/issues/430 + +### New Contributors + +* [@okzheng](https://github.com/okzheng) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/439 +* [@iyuroch](https://github.com/iyuroch) made their first contribution in https://github.com/VictoriaMetrics/operator/pull/464 + +[Changes][v0.24.0] + + + +## [v0.23.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.23.3) - 21 Feb 2022 + +### Fixes + +- fixes retention period for VMSingle and VMCluster, allows to set retentionPeriod lower than 1 month https://github.com/VictoriaMetrics/operator/issues/430 + +### Features + +- allows to control max and min scrape interval for `VMAgent`'s targets with `minScrapeInterval` and `maxScrapeInterval` https://github.com/VictoriaMetrics/operator/commit/3d8183205bef78e877b4f54d7892c4bad47b3971 + +[Changes][v0.23.3] + + + +## [v0.23.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.23.2) - 14 Feb 2022 + +### Fixes + +- fixed issue with parsing of kubernetes server version https://github.com/VictoriaMetrics/operator/issues/428 + +[Changes][v0.23.2] + + + +## [v0.23.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.23.1) - 10 Feb 2022 + +### Fixes + +- issue with incorrect vmservicescrape created for vminsert https://github.com/VictoriaMetrics/operator/issues/420 + +[Changes][v0.23.1] + + + +## [v0.23.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.23.0) - 09 Feb 2022 + +### Breaking changes + +- **job name label was changed, new prefix added with CRD type - probe, podScrape,serviceScrape, nodeScrape and staticScrape** + +### Fixes + +- fixes job name label with CRD type prefix, it must prevent possible job names collision https://github.com/VictoriaMetrics/operator/commit/3efe28b2de32485aa889118c63093adb291a82ff thanks [@tommy351](https://github.com/tommy351) +- fixes bearerToken usage for VMAgent remoteWriteSpec https://github.com/VictoriaMetrics/operator/issues/422 thanks [@artifactori](https://github.com/artifactori) + +### Features + +- check kubernetes api server version for deprecated objects and use proper API for it. First of all it's related with `PodSecurityPolicy` and `PodDisruptionBudget` https://github.com/VictoriaMetrics/operator/commit/5a64f6c01d535f5500a9d9a81ac851f9f12d547a + +[Changes][v0.23.0] + + + +## [v0.22.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.22.1) - 21 Jan 2022 + +### Fixes + +- fixes CSV configuration for operator-hub. It allows to launch operator in single-namespace mode https://github.com/VictoriaMetrics/operator/commit/94c7466224bff664552bae4424a54a036d72886b +- fixes annotations merge for deployments, it should fix endless reconcile loop https://github.com/VictoriaMetrics/operator/commit/7d26398ac3303f6684dd01ae12e376b05dd16ac8 + +### Features + +- bumps VictoriaMetrics appllications versions to the v1.72.0 https://github.com/VictoriaMetrics/operator/commit/de289af8af8472e5299fc6ff6e99749b58012edd + +[Changes][v0.22.1] + + + +## [v0.22.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.22.0) - 26 Dec 2021 + +### Fixes + +- fixes regression for VMAlert rules selector https://github.com/VictoriaMetrics/operator/issues/394 +- fixes build for go 1.17. Removed unneeded deps, upgraded lib versions https://github.com/VictoriaMetrics/operator/issues/392 +- fixes docs example https://github.com/VictoriaMetrics/operator/issues/391 + +### Features + +- moves operator API objects into separate go package. It allows to use operator API without import whole operator package. https://github.com/VictoriaMetrics/operator/commit/9fec1898617ba9f73c6c6c78cdebc1535514e263 +- allows to set `rollingUpdateStrategy` for statefullsets. With optional `rollingUpdateStrategy: rollingUpdate` operator uses kubernetes controller-manager updates for statefulsets, instead of own implementation. Allows kubectl rollout restart command for deployments and statefulsets https://github.com/VictoriaMetrics/operator/issues/389 +- allows to disable namespace label matcher for VMAlertmanager with global option `disableNamespaceMatcher` https://github.com/VictoriaMetrics/operator/issues/390 + +[Changes][v0.22.0] + + + +## [v0.21.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.21.0) - 30 Nov 2021 + +### Breaking changes + +- **Rollback changes for default behavior for CR selectors, such as serviceScrapeSelector at vmagent.spec. With new option `spec.selectAllByDefault: true` default behavior changes for select all on nil (as was at 0.20 version). https://github.com/VictoriaMetrics/operator/issues/383** +- **moves `ingress` api to `networking/v1` for `VMAuth`, minimal kubernetes supported version for `VMAuth` 1.19 https://github.com/VictoriaMetrics/operator/commit/2c6f81eb91452a7672907aa25acd392ef0777941** + +### Fixes + +- removes HPA from cache watch, it must remove errors at cluster without such api https://github.com/VictoriaMetrics/operator/commit/04bab9c486babed100522ec12fce3967e4dd5a13 +- labels and annotations update for auto-generated serviceScrape components. +- typos at quick-start https://github.com/VictoriaMetrics/operator/commit/e411cfe75b4ff3d57fd532e12c901eda5934645c thanks [@marcbachmann](https://github.com/marcbachmann) + +### Features + +- Adds alertmanager service scrape auto generation https://github.com/VictoriaMetrics/operator/issues/385 thanks [@FRosner](https://github.com/FRosner) +- Auto-add routing for vminsert and vmselect CRD components for `VMUser` https://github.com/VictoriaMetrics/operator/issues/379 +- Updates docs for `VMAuth`https://github.com/VictoriaMetrics/operator/blob/master/docs/auth.MD +- Allows changing default disk space usage for `VMAgent` https://github.com/VictoriaMetrics/operator/pull/381 thanks [@arctan90](https://github.com/arctan90) +- Adds Arch labels for clusterversion template https://github.com/VictoriaMetrics/operator/commit/9e89c3b2459fb85faa8e973fa1f1558d924000f3 thanks [@yselkowitz](https://github.com/yselkowitz) +- improves docs and fixes typos https://github.com/VictoriaMetrics/operator/commit/ae248dcb352a092d9f9caee87454b1ad25650a4c thanks [@flokli](https://github.com/flokli) + +[Changes][v0.21.0] + + + +## [v0.20.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.20.3) - 10 Nov 2021 + +#### Fixes + +- changes v1.SecretKeySelector value for pointer, it should help mitigate null error for v1.SecretKeySelector.Key https://github.com/VictoriaMetrics/operator/issues/365 +- Fixes `VMAlertmanagerConfig` - some configurations didn't add `send_resolved` option properly to the configration. https://github.com/VictoriaMetrics/operator/commit/6ee75053a4af2a163619908cd10ba4ec051755ab + +[Changes][v0.20.3] + + + +## [v0.20.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.20.2) - 07 Nov 2021 + +#### Fixes + +- regression at statefulset update process https://github.com/VictoriaMetrics/operator/issues/366 +- adds nullable option for v1.SecretKeySelector https://github.com/VictoriaMetrics/operator/issues/365 + +[Changes][v0.20.2] + + + +## [v0.20.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.20.1) - 28 Oct 2021 + +#### Fixes + +- regression at alertmanager config generation https://github.com/VictoriaMetrics/operator/commit/0f4368be57b2ccb2fbaebe9ce5fb4394299d89b3 + +[Changes][v0.20.1] + + + +## [v0.20.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.20.0) - 28 Oct 2021 + +### Breaking changes + +- **changes default behavior for CR selectors, such serviceScrapeSelector at vmagent.spec. Now it select all targets if is missing https://github.com/VictoriaMetrics/operator/commit/519e89b457576099288af2ea135878f6da25b567 See more at docs https://github.com/VictoriaMetrics/operator/blob/master/docs/quick-start.MD#object-selectors** +- **operator doesn't add cluster domain name for in-cluster communication, now its empty value. It should resolve issue with using operator at clusters with custom k8s domain https://github.com/VictoriaMetrics/operator/issues/354 thanks [@flokli](https://github.com/flokli)** + +### Features + +- adds ability to set custom headers to the `VMUser` target ref https://github.com/VictoriaMetrics/operator/issues/360 + +### Fixes + +- bearer token at staticScrape https://github.com/VictoriaMetrics/operator/issues/357 thanks [@addreas](https://github.com/addreas) +- path for the backups at vmcluster https://github.com/VictoriaMetrics/operator/issues/349 +- possible race condition for the cluster backups, now operator adds storage node name into backup path https://github.com/VictoriaMetrics/operator/issues/349 +- secret finalizer deletion for vmagent https://github.com/VictoriaMetrics/operator/issues/343 +- probes for vmagent https://github.com/VictoriaMetrics/operator/commit/f6de9c5774be0a5cd797c145553579e2e76a8df7 +- alertmanagerConfiguration build for slack https://github.com/VictoriaMetrics/operator/issues/339 + +[Changes][v0.20.0] + + + +## [v0.19.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.19.1) - 28 Sep 2021 + +### Fixes + +- Regression at `VMStaticScrape` - basic auth was incorrectly handled https://github.com/VictoriaMetrics/operator/issues/337 +- Convesion from `PodMonitor` to `VMPodScrape` https://github.com/VictoriaMetrics/operator/issues/335 + +[Changes][v0.19.1] + + + +## [v0.19.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.19.0) - 24 Sep 2021 + +### Features + +- Adds single-namespace mode for operator https://github.com/VictoriaMetrics/operator/issues/239 Thanks [@g7r](https://github.com/g7r) +- improves e2e tests thanks [@g7r](https://github.com/g7r) +- Adds `VMAlert` `Notifier` service discovery https://github.com/VictoriaMetrics/operator/pull/334 +- Updates `VMRule` - now it can use `vmalert` specific features https://github.com/VictoriaMetrics/operator/pull/331 +- Disables client caching for `Pod`, `Deployment` and `Statefulset`, it should reduce memory consumption https://github.com/VictoriaMetrics/operator/commit/9cfea5d091f072d1a0c6f8115a5e7652b94c6536 + +### Fixes + +- fixes psp rolebinding for operator https://github.com/VictoriaMetrics/operator/issues/323 +- fixes `VMAgent` reconciliation loop https://github.com/VictoriaMetrics/operator/issues/325 Thanks [@silverlyra](https://github.com/silverlyra) + +[Changes][v0.19.0] + + + +## [v0.18.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.18.2) - 03 Sep 2021 + +### Fixes + +- Fixes regression at CRD generation https://github.com/VictoriaMetrics/operator/issues/321 https://github.com/VictoriaMetrics/helm-charts/issues/199 + +[Changes][v0.18.2] + + + +## [v0.18.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.18.1) - 30 Aug 2021 + +### Fixes + +- Fixes regression at CRD generation https://github.com/VictoriaMetrics/operator/issues/316 Thanks [@Cosrider](https://github.com/Cosrider) + +[Changes][v0.18.1] + + + +## [v0.18.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.18.0) - 24 Aug 2021 + +### Deprecations + +- **Deprecates `apiextensions.k8s.io/v1beta1` API for CRD. Its still available at legacy mode.** + +### Features + +- Adds OAuth2 configuration for `VMagent`s remoteWrites and scrape endpoints +- Adds `TLSConfig` for `VMProbes` +- Major API update for `VMServiceScrape`, `VMPodScrape`, `VMProbe`, `VMStaticScrape` and `VMNodeScrape`: +- adds missing config params (sampleLimit and etc) +- Adds new config options `vm_scrape_params` https://github.com/VictoriaMetrics/operator/issues/303 +- Adds proxyAuth, that allows to authenticate proxy requests https://docs.victoriametrics.com/vmagent.html#scraping-targets-via-a-proxy +- Adds OAuth2 support. +- Adds `apiextensions.k8s.io/v1` `CRD` generation, `v1beta1` is now legacy https://github.com/VictoriaMetrics/operator/issues/291 +- Adds new `CRD` `VMAlertmanagerConfig`, it supports only v0.22 `alertmanager` version or above https://github.com/VictoriaMetrics/operator/issues/188 +- Makes `spec.selector` optional for `VMPodScrape` and `VMServiceScrape` https://github.com/VictoriaMetrics/operator/issues/307 +- Bumps alpine image for `3.14.1` - it should fixes security issues. +- Adds more unit tests and fixes some bugs + +### Fixes + +- Fixes bug for incorrect finalizer remove https://github.com/VictoriaMetrics/operator/issues/302 + +[Changes][v0.18.0] + + + +## [v0.17.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.17.2) - 31 Jul 2021 + +### Features + +- Updated docs. + +### Fixes + +- fixes vmauth default version +- fixes HPA deletion https://github.com/VictoriaMetrics/operator/issues/296 +- fixes VMAlert datasource TlsConfig https://github.com/VictoriaMetrics/operator/issues/298 +- fixes VMUser target_path_suffix typo at tags. + +[Changes][v0.17.2] + + + +## [v0.17.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.17.1) - 28 Jul 2021 + +### Features + +- Updated default versions for vm apps to v1.63.0 version +- Updated docs. + +[Changes][v0.17.1] + + + +## [v0.17.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.17.0) - 27 Jul 2021 + +### Features + +- Changes `VMAuth` config generation, now its possible to add `target_path_suffix` with optional query params https://github.com/VictoriaMetrics/operator/issues/245 +- Changes `VMAuth` config generation - in case of `/` it can generate simple config without url_map and regexp https://github.com/VictoriaMetrics/operator/commit/5dcd998b1814b26f75e3f6b5a38f8c3ee20552ec +- Reworks `annotations` merge https://github.com/VictoriaMetrics/operator/commit/90ae15e300bff68b9140e65819b2a5e1e972b9a0 + +### Fixes + +- Reduces memory usage - coz of improper label selectors and cache usage operator consumed a lot of memory https://github.com/VictoriaMetrics/operator/issues/285 +- Fixes VMAlert default image tag typo https://github.com/VictoriaMetrics/operator/issues/287 +- Fixes logging configuration https://github.com/VictoriaMetrics/operator/issues/281 +- Fixes new config reloader watch logic: https://github.com/VictoriaMetrics/operator/commit/35cadb04b828238ffdec67b3fd1ae7430543055d +- Fixes `VMServiceScrape` for `VMAgent` https://github.com/VictoriaMetrics/operator/commit/7bbbf2cd0557260b419e188b72a001572f848e35 + +[Changes][v0.17.0] + + + +## [v0.16.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.16.0) - 11 Jul 2021 + +### Breaking Changes + +- Changes `VMAgent` `RemoteWriteSpec` - some options were moved to `RemoteWriteSettings` https://github.com/VictoriaMetrics/operator/pull/273 + +### Features +- Adds experimental config-reloader implementation, it should help mitigate long configuration sync. It can be enabled with envvar `VM_USECUSTOMCONFIGRELOADER=true` https://github.com/VictoriaMetrics/operator/issues/124 +- Reduces load on kubernetes apiserver for `VMPodScrape` resources https://github.com/VictoriaMetrics/operator/pull/267 thanks [@fatsheep9146](https://github.com/fatsheep9146) +- Adds `/debug/pprof` handler at `0.0.0.0:8435` http server. + +### Fixes + +- Fixes Tls ingress for `VMAuth` https://github.com/VictoriaMetrics/operator/pull/270 +- Fixes endless loop for service account reconciliation https://github.com/VictoriaMetrics/operator/issues/277 +- Fixes `VMAlertmanager` update process https://github.com/VictoriaMetrics/operator/issues/271 +- Fixes ownership for `ArgoCD` based deployments - https://github.com/VictoriaMetrics/operator/issues/255 +- Fixes doc typos https://github.com/VictoriaMetrics/operator/pull/269 thanks [@zasdaym](https://github.com/zasdaym) + +[Changes][v0.16.0] + + + +## [v0.15.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.15.2) - 17 Jun 2021 + +### Features + +- reduced CRD size, it should fix operator-hub deployment +- updated lib versions. +- updated docs. + +[Changes][v0.15.2] + + + +## [v0.15.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.15.1) - 16 Jun 2021 + +### Fixes + +- Fixed panic at `VMCluster` https://github.com/VictoriaMetrics/operator/issues/264 + +[Changes][v0.15.1] + + + +## [v0.15.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.15.0) - 14 Jun 2021 + +### Features + +- Adds nodeSelector to all CRD Objects https://github.com/VictoriaMetrics/operator/issues/254 +- Adds HPA for `vminsert` and `vmselect` https://github.com/VictoriaMetrics/operator/issues/247 +- Adds new CRD resources - `VMAuth` and `VMUser` https://github.com/VictoriaMetrics/operator/issues/245 +- Adds hostPath support with ability to override `storageDataPath` setting https://github.com/VictoriaMetrics/operator/issues/240 + +### Fixes + +- Adds prometheus-config-reloader version check and updates its version https://github.com/VictoriaMetrics/operator/issues/259 +- Adds ownerReference to ServiceAccounts, it should mitigate ArgoCD issue https://github.com/VictoriaMetrics/operator/issues/255 +- Fixes cluster status update process https://github.com/VictoriaMetrics/operator/issues/253 +- Fixes `VMAlertmanager` config generation https://github.com/VictoriaMetrics/operator/issues/244 + +[Changes][v0.15.0] + + + +## [v0.14.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.14.2) - 26 Apr 2021 + +### Fixes + +- fixes insertPorts type for `VMCluster` + +[Changes][v0.14.2] + + + +## [v0.14.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.14.1) - 22 Apr 2021 + +### Fixes + +- fixes missing args for inline relabel configs. + +[Changes][v0.14.1] + + + +## [v0.14.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.14.0) - 22 Apr 2021 + +### Fixes + +- fixes incorrect tlsConfig handling for vmalert https://github.com/VictoriaMetrics/operator/issues/224 +- fixes config sync for relabeling https://github.com/VictoriaMetrics/operator/issues/222 + +### Features + +- improves statefulset rolling update https://github.com/VictoriaMetrics/operator/issues/217 +- adds ability to remove vmstorage from cluster routing https://github.com/VictoriaMetrics/operator/issues/218 +- adds `inlineRelabelConfig` and `inlineUrlRelabelConfig` for vmagent, it allows to define relabeling rules directly at vmagent CR https://github.com/VictoriaMetrics/operator/issues/154 +- adds `inlineScrapeConfig` https://github.com/VictoriaMetrics/operator/pull/230/files +- adds new RBAC permissions for `vmagent`, it should help to monitor `openshift` cluster correctly https://github.com/VictoriaMetrics/operator/issues/229 + +[Changes][v0.14.0] + + + +## [v0.13.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.13.1) - 13 Apr 2021 + +### Fixes + +- fixes operator role - added missing permission. +- fixes operator crash and improper tlsConfig build https://github.com/VictoriaMetrics/operator/issues/215 + +[Changes][v0.13.1] + + + +## [v0.13.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.13.0) - 09 Apr 2021 + +### Fixes + +- storage resize detection https://github.com/VictoriaMetrics/operator/pull/211 thanks [@lujiajing1126](https://github.com/lujiajing1126) +- vmagent rbac role https://github.com/VictoriaMetrics/operator/pull/213 thanks [@viperstars](https://github.com/viperstars) +- fixes CRD for kubernetes version less then 1.16 https://github.com/VictoriaMetrics/operator/pull/210 + +### Features + +- adds probes customization via CRD https://github.com/VictoriaMetrics/operator/pull/204 thanks [@preved911](https://github.com/preved911) + +[Changes][v0.13.0] + + + +## [v0.12.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.12.2) - 31 Mar 2021 + +### Fixes + +- fixes serviceAccount update https://github.com/VictoriaMetrics/operator/issues/207 + +[Changes][v0.12.2] + + + +## [v0.12.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.12.1) - 30 Mar 2021 + +### Fixes + +- removes liveness probe from vmstorage and `VMSingle` https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1158 +- fixes update process for `VMCluster` and `VMAlertmanager` + +[Changes][v0.12.1] + + + +## [v0.12.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.12.0) - 29 Mar 2021 + +### Breaking changes + +- operator automatically resizes `PVC` and recreates `StatefulSet` for `VMCluster` components if needed, be careful with upgrade, if you are manually edited `PVC` size. In common cases, it must be safe. + +### Features + +- Adds scraping sharding for `VMAgent` https://github.com/VictoriaMetrics/operator/issues/177 +- Adds pvc resizing for `VMCluster` and `VMAletermanager`, it also allows to change storage params https://github.com/VictoriaMetrics/operator/issues/161 +- Adds `PodDisruptionBudget` for `VMAgent`, `VMCluster`, `VMAlert` and `VMAlertmanager` https://github.com/VictoriaMetrics/operator/issues/191 Thanks [@umezawatakeshi](https://github.com/umezawatakeshi) +- Simplifies `topologySpreadConstraints` configuration https://github.com/VictoriaMetrics/operator/issues/191, thanks [@umezawatakeshi](https://github.com/umezawatakeshi) + +### Fixes + +- Fixes `VMAlert` `rule` arg - it was unproperly escaped https://github.com/VictoriaMetrics/operator/commit/870f258b324dbaec1e3d0d8739ff2feffc27bf0a +- Fixes `VMProbes`, now it supports relabeling for static targets https://github.com/VictoriaMetrics/operator/commit/b4db7d5128a22d4979d7284e15576322acbc9b4c +- Fixes `VMStaticScrape` - adds `honorLabels` and `honorTimestamps` setting to CRD + +[Changes][v0.12.0] + + + +## [v0.11.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.11.0) - 22 Mar 2021 + +### Breaking changes + +- Adds acceptEULA setting to `VMBackuper`, without it backuper cannot be used. https://github.com/VictoriaMetrics/operator/commit/dc7f9e0f830d1e5f1010e7e96ae99f1932fe549f + +### Features + +- Adds additional service for all components, its useful for service exposition https://github.com/VictoriaMetrics/operator/issues/163 thanks [@TinySong](https://github.com/TinySong) + +### Fixes + +- fixes bug with insert ports. +- minor fixes to examples. + +[Changes][v0.11.0] + + + +## [v0.10.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.10.0) - 14 Mar 2021 + +### Features + +- Added finalizers to objects created by operator. It must fix an issue with resource deletion by controller manager. Note, it requires additional rbac access. https://github.com/VictoriaMetrics/operator/issues/159 https://github.com/VictoriaMetrics/operator/pull/189 +- Added new resouce for static targets scrapping - `VMStaticScrape` https://github.com/VictoriaMetrics/operator/issues/155 +- Added `unlimited` param for default resources - https://github.com/VictoriaMetrics/operator/issues/181 +- Added clusterVersion spec to `VMCluster` it should simplify management https://github.com/VictoriaMetrics/operator/issues/176 + +### Fixes + +- fixes bug with incorrect object reconciliation - labelMatch heuristic was broken. +- fixes race condition on vmagent reconciliation. +- fixes `VMAlertmanager` version parse https://github.com/VictoriaMetrics/operator/pull/179 thanks [@morimoto-cybozu](https://github.com/morimoto-cybozu) +- other little improvements. + +[Changes][v0.10.0] + + + +## [v0.9.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.9.1) - 22 Feb 2021 + +### Features + +- adds externalLabels for vmalert https://github.com/VictoriaMetrics/operator/issues/160 + +### Fixes + +- rbac role namespace. + +[Changes][v0.9.1] + + + +## [v0.9.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.9.0) - 21 Feb 2021 + +### Features + +- adds finalizers to the CRDs, it must prevent deletion by controller manager and clean-up created resources properly. https://github.com/VictoriaMetrics/operator/issues/159 + +### Fixes + +- rbac role https://github.com/VictoriaMetrics/operator/issues/166 +- fixes incorrect converter start and race condition. + +[Changes][v0.9.0] + + + +## [v0.8.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.8.0) - 09 Feb 2021 + +### Features + +- adds VMPodScrape basic auth, token and tls connection support https://github.com/VictoriaMetrics/operator/issues/151 +- adds `insertPorts` for `VMSingle` and `VMCluster`, it allows to configure ingestion ports for OpenTSDB,Graphite and Influx servers https://github.com/VictoriaMetrics/operator/pull/157 + +### Fixes + +- fixes operator-hub docs broken links. +- fixes panic at vmcluster. + +[Changes][v0.8.0] + + + +## [v0.7.4](https://github.com/VictoriaMetrics/operator/releases/tag/v0.7.4) - 25 Jan 2021 + +### Fixes + +- fixed ExtraArgs typo https://github.com/VictoriaMetrics/operator/pull/150 thanks [@jansyk13](https://github.com/jansyk13) + +[Changes][v0.7.4] + + + +## [v0.7.3](https://github.com/VictoriaMetrics/operator/releases/tag/v0.7.3) - 20 Jan 2021 + +### Fixes + +- fixed panic at vmcluster https://github.com/VictoriaMetrics/operator/issues/147 thanks [@gideshrp1JL](https://github.com/gideshrp1JL) + +[Changes][v0.7.3] + + + +## [v0.7.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.7.2) - 17 Jan 2021 + +### Fixes + +- serverName for tlsConfig https://github.com/VictoriaMetrics/operator/issues/144 +- minScrapeInterval for vmstorage https://github.com/VictoriaMetrics/operator/pull/143 Thansk [@umezawatakeshi](https://github.com/umezawatakeshi) + +[Changes][v0.7.2] + + + +## [v0.7.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.7.1) - 01 Jan 2021 + +### Fixes + +- `VMAlert` deploy inconsistent update https://github.com/VictoriaMetrics/operator/issues/140 + +### Features + +- adds heuristic for selector match between `VMRule`, `VMNodeScrape`, `VMProbe`, `VMServiceScrape` and `VMPodScrape` and corresponding object - `VMAlert` or `VMAgent. It must speed up reconciliation in case of multi-tenancy. + +[Changes][v0.7.1] + + + +## [v0.7.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.7.0) - 30 Dec 2020 + +### Fixes + +- https://github.com/VictoriaMetrics/operator/pull/133 VMNodeScrape - fixes nodeScrapeNamespaceSelector. Thanks [@umezawatakeshi](https://github.com/umezawatakeshi) +- VMAlert notifiers support per notifier tlsInSecure. Note, you have to upgrade `vmalert` to v1.51 release. +- Removes null Status and creationTimestamp fields for CRDs. +- https://github.com/VictoriaMetrics/operator/issues/132 - fixes behavior if object was deleted. +- minor fixes to samples for operator-hub. + +### Features + +- https://github.com/VictoriaMetrics/operator/issues/131 adds support for classic relabelConfigs `target_label` and `source_labels`. +- https://github.com/VictoriaMetrics/operator/issues/127 adds `discoveryRole` with `endpoints`, `endpointslices` and `service` options. + +[Changes][v0.7.0] + + + +## [v0.6.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.6.1) - 16 Dec 2020 + +### Fixes + +- VMAlert TLSConfig build was fixed. +- Fixes docs for operator-hub. + +[Changes][v0.6.1] + + + +## [v0.6.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.6.0) - 15 Dec 2020 + +### Breaking changes: + +- `VMAgent` RemoteWriteSpec was changed, now it doesnt support `flushInterval,maxBlockSize,maxDiskUsagePerURL and queues`. Because its global flags at `vmagent`. Added `remoteWriteSettings` instead with corresponding settings. + +### Features + +- New CRD type `VMNodeScrape`, it's useful for kubernetes nodes exporters scraping. See details at https://github.com/VictoriaMetrics/operator/issues/125. +- `VMAlert` support multiple notifiers with `notifiers` spec. See details at https://github.com/VictoriaMetrics/operator/issues/117. +- `VMRule` support `concurrency` for group execution, see detail at vmalert docs https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmalert#groups. + +### Fixes + +- Updated docs, thanks [@umezawatakeshi](https://github.com/umezawatakeshi) +- Fixes `VMProbe` spec https://github.com/VictoriaMetrics/operator/issues/125 +- Fixes remoteWrite.labels + +[Changes][v0.6.0] + + + +## [v0.5.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.5.0) - 04 Dec 2020 + +### Breaking changes: + +- `VMCluster`'s `serviceAccountName` moved from `VMCluster.spec.vm....serviceAccountName` to the root of spec, and now its located at `VMCluster.spec.serviceAccountName`. +- Operator requires additional rbac permissions. + +### Features + +- PodSecurityPolicy automatically created for each object, with own ServiceAccount, ClusterRole and ClusterRoleBinding. Its possible to use custom PSP. https://github.com/VictoriaMetrics/operator/issues/109 +- Adds `VMAgent` rbac auto-creation. +- Adds ServiceAccount auto-creation. Its possible to use custome ServiceAccount instead of default. +- Adds `ownerReferences` for converted resources from `Prometheus-operator` CRDs, https://github.com/VictoriaMetrics/operator/pull/105 thanks [@teqwve](https://github.com/teqwve) . +- Adds `runtimeClassName`, `schedulerName` for all VictoriaMetrics applications. +- Adds `topologySpreadConstraints` for all VictoriaMetrics applications. https://github.com/VictoriaMetrics/operator/issues/107. +- Adds `hostAliases` for `VMAgent` and `VMSingle` applications. + +### Fixes + +- Fixes rbac for openshift deployment, adds emptyDir for `VMAgent`s persistent queue with 1gb size limit. https://github.com/VictoriaMetrics/operator/issues/106 +- Fixes `VMAlert` deployment serviceAccountName. +- Fixes logger levels for operator. +- Fixes labels, now is forbidden to change Selector labels for for all VictoriaMetrics applications. This changes will be ignored. +- Reduces size of CRDs. + +[Changes][v0.5.0] + + + +## [v0.4.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.4.0) - 15 Nov 2020 + +* Adds `VMRules` de-duplication with annotation https://github.com/VictoriaMetrics/operator/issues/99 +* Adds Operator-Hub integration https://github.com/VictoriaMetrics/operator/issues/33 +* Fixes deployment `Resource` definition (omit limits/requests if provided only one specification). +* Fixes Volumes mounts https://github.com/VictoriaMetrics/operator/issues/97 +* Fixes deployments update loop with extra-args https://github.com/VictoriaMetrics/operator/pull/100 . Thanks [@zhiyin009](https://github.com/zhiyin009) +* Fixes securityContext field https://github.com/VictoriaMetrics/operator/pull/101 . Thanks [@zhiyin009](https://github.com/zhiyin009) +* Fixes `VMAgent` start-up error https://github.com/VictoriaMetrics/VictoriaMetrics/issues/879 + +[Changes][v0.4.0] + + + +## [v0.3.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.3.0) - 29 Oct 2020 + +* adds fast config update for `VMAlert` https://github.com/VictoriaMetrics/operator/issues/86 +* adds docker multiarch support +* updates docs and examples https://github.com/VictoriaMetrics/operator/issues/85 thanks [@elmariofredo](https://github.com/elmariofredo) +* fixes env variables usage with applications https://github.com/VictoriaMetrics/operator/issues/89 +* fixes prometheus relabel config inconsistency https://github.com/VictoriaMetrics/operator/issues/92 +* fixes vmselect args https://github.com/VictoriaMetrics/operator/pull/95 thanks [@zhiyin009](https://github.com/zhiyin009) + +[Changes][v0.3.0] + + + +## [v0.2.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.2.1) - 28 Aug 2020 + +- [#78](https://github.com/VictoriaMetrics/operator/issues/78) fixed bug with rbac - without access to vmsingles api resource, operator wasn't able to start reconciliation loop. +- [#76](https://github.com/VictoriaMetrics/operator/issues/76) added path prefix support if extraArgs was specified. +- [#71](https://github.com/VictoriaMetrics/operator/issues/71) arm support with cross compilation. + +[Changes][v0.2.1] + + + +## [v0.2.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.2.0) - 23 Aug 2020 + +- Added VMProbe [#59](https://github.com/VictoriaMetrics/operator/issues/59) +- Fixed various bug with prometheus api objects conversion. +- added annotations for control conversion flow [#68](https://github.com/VictoriaMetrics/operator/issues/68) + +[Changes][v0.2.0] + + + +## [v0.1.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.1.2) - 21 Aug 2020 + +- [#66](https://github.com/VictoriaMetrics/operator/issues/66) added path replacement for `CAfile`, `Certfile`, `KeyFile`, `BearerTokenFile` at prometheus api converter. +- [#65](https://github.com/VictoriaMetrics/operator/issues/65) fixed tlsConfig logic, now configuration file renders correctly, if empty value for Cert, Ca or KeySecret defined at tlsConf +- minor documentation update + +[Changes][v0.1.2] + + + +## [v0.1.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.1.1) - 18 Aug 2020 + +- fixed issues with crd patching for 1.18 kubernetes version +- fixed issue with rbac roles +- upgraded go version to 1.15 +- upgraded operator-sdk version to 1.0.0 + +[Changes][v0.1.1] + + + +## [v0.1.0](https://github.com/VictoriaMetrics/operator/releases/tag/v0.1.0) - 12 Aug 2020 + +Starting point of operator releases +- Documentation update + +[Changes][v0.1.0] + + + +## [v0.0.6](https://github.com/VictoriaMetrics/operator/releases/tag/v0.0.6) - 26 Jul 2020 + +- breaking changes to api (changed group name to operator.victoriametrics.com) +- changed build and release process +- migrated to operator sdk 0.19 + +[Changes][v0.0.6] + + + +## [v0.0.2](https://github.com/VictoriaMetrics/operator/releases/tag/v0.0.2) - 12 Jun 2020 + +- fixed panic at vmSingle update +- added support for scraping tls targets with ServiceMonitor TLSConfig + +[Changes][v0.0.2] + + + +## [v0.0.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.0.1) - 06 Jun 2020 + +it contains basic api objects support: +1) vmAgent +2) vmAlert +3) vmSingle +4) vmAlertmanager + ++ prometheus-operator objects: +1) prometheusRule +2) serviceMonitor +3) podMonitor + +[Changes][v0.0.1] + + +[v0.35.1]: https://github.com/VictoriaMetrics/operator/compare/v0.35.0...v0.35.1 +[v0.35.0]: https://github.com/VictoriaMetrics/operator/compare/v0.34.1...v0.35.0 +[v0.34.1]: https://github.com/VictoriaMetrics/operator/compare/v0.34.0...v0.34.1 +[v0.34.0]: https://github.com/VictoriaMetrics/operator/compare/v0.33.0...v0.34.0 +[v0.33.0]: https://github.com/VictoriaMetrics/operator/compare/v0.32.1...v0.33.0 +[v0.32.1]: https://github.com/VictoriaMetrics/operator/compare/v0.32.0...v0.32.1 +[v0.32.0]: https://github.com/VictoriaMetrics/operator/compare/v0.31.0...v0.32.0 +[v0.31.0]: https://github.com/VictoriaMetrics/operator/compare/v0.30.4...v0.31.0 +[v0.30.4]: https://github.com/VictoriaMetrics/operator/compare/v0.30.3...v0.30.4 +[v0.30.3]: https://github.com/VictoriaMetrics/operator/compare/v0.30.2...v0.30.3 +[v0.30.2]: https://github.com/VictoriaMetrics/operator/compare/v0.30.1...v0.30.2 +[v0.30.1]: https://github.com/VictoriaMetrics/operator/compare/v0.30.0...v0.30.1 +[v0.30.0]: https://github.com/VictoriaMetrics/operator/compare/v0.29.2...v0.30.0 +[v0.29.2]: https://github.com/VictoriaMetrics/operator/compare/v0.29.1...v0.29.2 +[v0.29.1]: https://github.com/VictoriaMetrics/operator/compare/v0.29.0...v0.29.1 +[v0.29.0]: https://github.com/VictoriaMetrics/operator/compare/v0.28.5...v0.29.0 +[v0.28.5]: https://github.com/VictoriaMetrics/operator/compare/v0.28.4...v0.28.5 +[v0.28.4]: https://github.com/VictoriaMetrics/operator/compare/v0.28.3...v0.28.4 +[v0.28.3]: https://github.com/VictoriaMetrics/operator/compare/v0.28.2...v0.28.3 +[v0.28.2]: https://github.com/VictoriaMetrics/operator/compare/v0.28.1...v0.28.2 +[v0.28.1]: https://github.com/VictoriaMetrics/operator/compare/v0.28.0...v0.28.1 +[v0.28.0]: https://github.com/VictoriaMetrics/operator/compare/v0.27.2...v0.28.0 +[v0.27.2]: https://github.com/VictoriaMetrics/operator/compare/v0.27.1...v0.27.2 +[v0.27.1]: https://github.com/VictoriaMetrics/operator/compare/v0.27.0...v0.27.1 +[v0.27.0]: https://github.com/VictoriaMetrics/operator/compare/v0.26.3...v0.27.0 +[v0.26.3]: https://github.com/VictoriaMetrics/operator/compare/v0.26.0...v0.26.3 +[v0.26.0]: https://github.com/VictoriaMetrics/operator/compare/v0.25.1...v0.26.0 +[v0.25.1]: https://github.com/VictoriaMetrics/operator/compare/v0.25.0...v0.25.1 +[v0.25.0]: https://github.com/VictoriaMetrics/operator/compare/v0.24.0...v0.25.0 +[v0.24.0]: https://github.com/VictoriaMetrics/operator/compare/v0.23.3...v0.24.0 +[v0.23.3]: https://github.com/VictoriaMetrics/operator/compare/v0.23.2...v0.23.3 +[v0.23.2]: https://github.com/VictoriaMetrics/operator/compare/v0.23.1...v0.23.2 +[v0.23.1]: https://github.com/VictoriaMetrics/operator/compare/v0.23.0...v0.23.1 +[v0.23.0]: https://github.com/VictoriaMetrics/operator/compare/v0.22.1...v0.23.0 +[v0.22.1]: https://github.com/VictoriaMetrics/operator/compare/v0.22.0...v0.22.1 +[v0.22.0]: https://github.com/VictoriaMetrics/operator/compare/v0.21.0...v0.22.0 +[v0.21.0]: https://github.com/VictoriaMetrics/operator/compare/v0.20.3...v0.21.0 +[v0.20.3]: https://github.com/VictoriaMetrics/operator/compare/v0.20.2...v0.20.3 +[v0.20.2]: https://github.com/VictoriaMetrics/operator/compare/v0.20.1...v0.20.2 +[v0.20.1]: https://github.com/VictoriaMetrics/operator/compare/v0.20.0...v0.20.1 +[v0.20.0]: https://github.com/VictoriaMetrics/operator/compare/v0.19.1...v0.20.0 +[v0.19.1]: https://github.com/VictoriaMetrics/operator/compare/v0.19.0...v0.19.1 +[v0.19.0]: https://github.com/VictoriaMetrics/operator/compare/v0.18.2...v0.19.0 +[v0.18.2]: https://github.com/VictoriaMetrics/operator/compare/v0.18.1...v0.18.2 +[v0.18.1]: https://github.com/VictoriaMetrics/operator/compare/v0.18.0...v0.18.1 +[v0.18.0]: https://github.com/VictoriaMetrics/operator/compare/v0.17.2...v0.18.0 +[v0.17.2]: https://github.com/VictoriaMetrics/operator/compare/v0.17.1...v0.17.2 +[v0.17.1]: https://github.com/VictoriaMetrics/operator/compare/v0.17.0...v0.17.1 +[v0.17.0]: https://github.com/VictoriaMetrics/operator/compare/v0.16.0...v0.17.0 +[v0.16.0]: https://github.com/VictoriaMetrics/operator/compare/v0.15.2...v0.16.0 +[v0.15.2]: https://github.com/VictoriaMetrics/operator/compare/v0.15.1...v0.15.2 +[v0.15.1]: https://github.com/VictoriaMetrics/operator/compare/v0.15.0...v0.15.1 +[v0.15.0]: https://github.com/VictoriaMetrics/operator/compare/v0.14.2...v0.15.0 +[v0.14.2]: https://github.com/VictoriaMetrics/operator/compare/v0.14.1...v0.14.2 +[v0.14.1]: https://github.com/VictoriaMetrics/operator/compare/v0.14.0...v0.14.1 +[v0.14.0]: https://github.com/VictoriaMetrics/operator/compare/v0.13.1...v0.14.0 +[v0.13.1]: https://github.com/VictoriaMetrics/operator/compare/v0.13.0...v0.13.1 +[v0.13.0]: https://github.com/VictoriaMetrics/operator/compare/v0.12.2...v0.13.0 +[v0.12.2]: https://github.com/VictoriaMetrics/operator/compare/v0.12.1...v0.12.2 +[v0.12.1]: https://github.com/VictoriaMetrics/operator/compare/v0.12.0...v0.12.1 +[v0.12.0]: https://github.com/VictoriaMetrics/operator/compare/v0.11.0...v0.12.0 +[v0.11.0]: https://github.com/VictoriaMetrics/operator/compare/v0.10.0...v0.11.0 +[v0.10.0]: https://github.com/VictoriaMetrics/operator/compare/v0.9.1...v0.10.0 +[v0.9.1]: https://github.com/VictoriaMetrics/operator/compare/v0.9.0...v0.9.1 +[v0.9.0]: https://github.com/VictoriaMetrics/operator/compare/v0.8.0...v0.9.0 +[v0.8.0]: https://github.com/VictoriaMetrics/operator/compare/v0.7.4...v0.8.0 +[v0.7.4]: https://github.com/VictoriaMetrics/operator/compare/v0.7.3...v0.7.4 +[v0.7.3]: https://github.com/VictoriaMetrics/operator/compare/v0.7.2...v0.7.3 +[v0.7.2]: https://github.com/VictoriaMetrics/operator/compare/v0.7.1...v0.7.2 +[v0.7.1]: https://github.com/VictoriaMetrics/operator/compare/v0.7.0...v0.7.1 +[v0.7.0]: https://github.com/VictoriaMetrics/operator/compare/v0.6.1...v0.7.0 +[v0.6.1]: https://github.com/VictoriaMetrics/operator/compare/v0.6.0...v0.6.1 +[v0.6.0]: https://github.com/VictoriaMetrics/operator/compare/v0.5.0...v0.6.0 +[v0.5.0]: https://github.com/VictoriaMetrics/operator/compare/v0.4.0...v0.5.0 +[v0.4.0]: https://github.com/VictoriaMetrics/operator/compare/v0.3.0...v0.4.0 +[v0.3.0]: https://github.com/VictoriaMetrics/operator/compare/v0.2.1...v0.3.0 +[v0.2.1]: https://github.com/VictoriaMetrics/operator/compare/v0.2.0...v0.2.1 +[v0.2.0]: https://github.com/VictoriaMetrics/operator/compare/v0.1.2...v0.2.0 +[v0.1.2]: https://github.com/VictoriaMetrics/operator/compare/v0.1.1...v0.1.2 +[v0.1.1]: https://github.com/VictoriaMetrics/operator/compare/v0.1.0...v0.1.1 +[v0.1.0]: https://github.com/VictoriaMetrics/operator/compare/v0.0.6...v0.1.0 +[v0.0.6]: https://github.com/VictoriaMetrics/operator/compare/v0.0.2...v0.0.6 +[v0.0.2]: https://github.com/VictoriaMetrics/operator/compare/v0.0.1...v0.0.2 +[v0.0.1]: https://github.com/VictoriaMetrics/operator/tree/v0.0.1 diff --git a/docs/operator/api.md b/docs/operator/api.md index 6ca5a68f41..f592a2362a 100644 --- a/docs/operator/api.md +++ b/docs/operator/api.md @@ -63,6 +63,7 @@ This Document documents the types introduced by the VictoriaMetrics to be consum * [EmbeddedPodDisruptionBudgetSpec](#embeddedpoddisruptionbudgetspec) * [EmbeddedProbes](#embeddedprobes) * [HTTPAuth](#httpauth) +* [KeyValue](#keyvalue) * [ServiceSpec](#servicespec) * [StorageSpec](#storagespec) * [StreamAggrConfig](#streamaggrconfig) @@ -122,12 +123,14 @@ This Document documents the types introduced by the VictoriaMetrics to be consum * [StaticRef](#staticref) * [TargetRef](#targetref) * [VMUser](#vmuser) +* [VMUserIPFilters](#vmuseripfilters) * [VMUserList](#vmuserlist) * [VMUserSpec](#vmuserspec) * [EmbeddedIngress](#embeddedingress) * [VMAuth](#vmauth) * [VMAuthList](#vmauthlist) * [VMAuthSpec](#vmauthspec) +* [VMAuthUnauthorizedPath](#vmauthunauthorizedpath) * [TargetEndpoint](#targetendpoint) * [VMStaticScrape](#vmstaticscrape) * [VMStaticScrapeList](#vmstaticscrapelist) @@ -256,7 +259,7 @@ EmailConfig configures notifications via Email. | auth_password | AuthPassword defines secret name and key at CRD namespace. | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | auth_secret | AuthSecret defines secrent name and key at CRD namespace. It must contain the CRAM-MD5 secret. | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | auth_identity | The identity to use for authentication. | string | false | -| headers | Further headers email header key/value pairs. Overrides any headers previously set by the notification implementation. | map[string]string | false | +| headers | Further headers email header key/value pairs. Overrides any headers previously set by the notification implementation. | EmailConfigHeaders | false | | html | The HTML body of the email notification. | string | false | | text | The text body of the email notification. | string | false | | require_tls | The SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. | *bool | false | @@ -504,7 +507,7 @@ SlackConfirmationField protect users from destructive actions or particularly di ## SlackField -See https://api.slack.com/docs/message-attachments#fields for more information. +SlackField configures a single Slack field that is sent with each notification. See https://api.slack.com/docs/message-attachments#fields for more information. | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | @@ -807,7 +810,7 @@ VMAgentSpec defines the desired state of VMAgent ## VMAgentStatus -VmAgentStatus defines the observed state of VmAgent +VMAgentStatus defines the observed state of VmAgent | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | @@ -838,7 +841,7 @@ BearerAuth defines auth with bearer token | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| bearerTokenFilePath | | string | false | +| bearerTokenFile | Path to bearer token file | string | false | | bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | [Back to TOC](#table-of-contents) @@ -932,14 +935,25 @@ HTTPAuth generic auth used with http protocols | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | | basicAuth | | *[BasicAuth](#basicauth) | false | -| OAuth2 | | *[OAuth2](#oauth2) | false | +| oauth2 | | *[OAuth2](#oauth2) | false | | tlsConfig | | *[TLSConfig](#tlsconfig) | false | -| bearerTokenFilePath | | string | false | +| bearerTokenFile | Path to bearer token file | string | false | | bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false | [Back to TOC](#table-of-contents) +## KeyValue + +KeyValue defines a (key, value) tuple. + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| key | Key of the tuple. | string | true | +| value | Value of the tuple. | string | true | + +[Back to TOC](#table-of-contents) + ## ServiceSpec ServiceSpec defines additional service for CRD with user-defined params. by default, some of fields can be inherited from default service definition for the CRD: labels,selector, ports. if metadata.name is not defined, service will have format {{CRD_TYPE}}-{{CRD_NAME}}-additional-service. @@ -1005,15 +1019,15 @@ VMAlert executes a list of given alerting or recording rules against configured ## VMAlertDatasourceSpec -VMAgentRemoteReadSpec defines the remote storage configuration for VmAlert to read alerts from +VMAlertDatasourceSpec defines the remote storage configuration for VmAlert to read alerts from | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | | url | Victoria Metrics or VMSelect url. Required parameter. E.g. http://127.0.0.1:8428 | string | true | | basicAuth | | *[BasicAuth](#basicauth) | false | -| OAuth2 | | *[OAuth2](#oauth2) | false | +| oauth2 | | *[OAuth2](#oauth2) | false | | tlsConfig | | *[TLSConfig](#tlsconfig) | false | -| bearerTokenFilePath | | string | false | +| bearerTokenFile | Path to bearer token file | string | false | | bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false | @@ -1039,9 +1053,9 @@ VMAlertNotifierSpec defines the notifier url for sending information about alert | url | AlertManager url. E.g. http://127.0.0.1:9093 | string | false | | selector | Selector allows service discovery for alertmanager in this case all matched vmalertmanager replicas will be added into vmalert notifier.url as statefulset pod.fqdn | *[DiscoverySelector](#discoveryselector) | false | | basicAuth | | *[BasicAuth](#basicauth) | false | -| OAuth2 | | *[OAuth2](#oauth2) | false | +| oauth2 | | *[OAuth2](#oauth2) | false | | tlsConfig | | *[TLSConfig](#tlsconfig) | false | -| bearerTokenFilePath | | string | false | +| bearerTokenFile | Path to bearer token file | string | false | | bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false | @@ -1049,16 +1063,16 @@ VMAlertNotifierSpec defines the notifier url for sending information about alert ## VMAlertRemoteReadSpec -VMAgentRemoteReadSpec defines the remote storage configuration for VmAlert to read alerts from +VMAlertRemoteReadSpec defines the remote storage configuration for VmAlert to read alerts from | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | | url | URL of the endpoint to send samples to. | string | true | | lookback | Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s) Applied only to RemoteReadSpec | *string | false | | basicAuth | | *[BasicAuth](#basicauth) | false | -| OAuth2 | | *[OAuth2](#oauth2) | false | +| oauth2 | | *[OAuth2](#oauth2) | false | | tlsConfig | | *[TLSConfig](#tlsconfig) | false | -| bearerTokenFilePath | | string | false | +| bearerTokenFile | Path to bearer token file | string | false | | bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false | @@ -1066,7 +1080,7 @@ VMAgentRemoteReadSpec defines the remote storage configuration for VmAlert to re ## VMAlertRemoteWriteSpec -VMAgentRemoteWriteSpec defines the remote storage configuration for VmAlert +VMAlertRemoteWriteSpec defines the remote storage configuration for VmAlert | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | @@ -1076,9 +1090,9 @@ VMAgentRemoteWriteSpec defines the remote storage configuration for VmAlert | maxBatchSize | Defines defines max number of timeseries to be flushed at once (default 1000) | *int32 | false | | maxQueueSize | Defines the max number of pending datapoints to remote write endpoint (default 100000) | *int32 | false | | basicAuth | | *[BasicAuth](#basicauth) | false | -| OAuth2 | | *[OAuth2](#oauth2) | false | +| oauth2 | | *[OAuth2](#oauth2) | false | | tlsConfig | | *[TLSConfig](#tlsconfig) | false | -| bearerTokenFilePath | | string | false | +| bearerTokenFile | Path to bearer token file | string | false | | bearerTokenSecret | Optional bearer auth token to use for -remoteWrite.url | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#secretkeyselector-v1-core) | false | | headers | Headers allow configuring custom http headers Must be in form of semicolon separated header with value e.g. headerName:headerValue vmalert supports it since 1.79.0 version | []string | false | @@ -1114,7 +1128,7 @@ VMAlertSpec defines the desired state of VMAlert | hostNetwork | HostNetwork controls whether the pod may use the node network namespace | bool | false | | dnsPolicy | DNSPolicy sets DNS policy for the pod | [v1.DNSPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#pod-v1-core) | false | | topologySpreadConstraints | TopologySpreadConstraints embedded kubernetes pod configuration option, controls how pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ | [][v1.TopologySpreadConstraint](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) | false | -| evaluationInterval | EvaluationInterval how often evalute rules by default | string | false | +| evaluationInterval | EvaluationInterval defines how often to evaluate rules by default | string | false | | enforcedNamespaceLabel | EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. | string | false | | selectAllByDefault | SelectAllByDefault changes default behavior for empty CRD selectors, such RuleSelector. with selectAllByDefault: true and empty serviceScrapeSelector and RuleNamespaceSelector Operator selects all exist serviceScrapes with selectAllByDefault: false - selects nothing | bool | false | | ruleSelector | RuleSelector selector to select which VMRules to mount for loading alerting rules from. Works in combination with NamespaceSelector. If both nil - behaviour controlled by selectAllByDefault NamespaceSelector nil - only objects at VMAlert namespace. | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#labelselector-v1-meta) | false | @@ -1147,7 +1161,7 @@ VMAlertSpec defines the desired state of VMAlert ## VMAlertStatus -VmAlertStatus defines the observed state of VmAlert +VMAlertStatus defines the observed state of VmAlert | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | @@ -1241,10 +1255,12 @@ VMSingleStatus defines the observed state of VMSingle | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| replicas | ReplicaCount Total number of non-terminated pods targeted by this VMAlert cluster (their labels match the selector). | int32 | true | -| updatedReplicas | UpdatedReplicas Total number of non-terminated pods targeted by this VMAlert cluster that have the desired version spec. | int32 | true | -| availableReplicas | AvailableReplicas Total number of available pods (ready for at least minReadySeconds) targeted by this VMAlert cluster. | int32 | true | -| unavailableReplicas | UnavailableReplicas Total number of unavailable pods targeted by this VMAlert cluster. | int32 | true | +| replicas | ReplicaCount Total number of non-terminated pods targeted by this VMSingle. | int32 | true | +| updatedReplicas | UpdatedReplicas Total number of non-terminated pods targeted by this VMSingle. | int32 | true | +| availableReplicas | AvailableReplicas Total number of available pods (ready for at least minReadySeconds) targeted by this VMSingle. | int32 | true | +| unavailableReplicas | UnavailableReplicas Total number of unavailable pods targeted by this VMSingle. | int32 | true | +| singleStatus | | SingleStatus | true | +| reason | | string | false | [Back to TOC](#table-of-contents) @@ -1259,8 +1275,10 @@ Rule describes an alerting or recording rule. | expr | Expr is query, that will be evaluated at dataSource | string | true | | debug | Debug enables logging for rule it useful for tracking | *bool | false | | for | For evaluation interval in time.Duration format 30s, 1m, 1h or nanoseconds | string | false | +| keep_firing_for | KeepFiringFor will make alert continue firing for this long even when the alerting expression no longer has results. Use time.Duration format, 30s, 1m, 1h or nanoseconds | string | false | | labels | Labels will be added to rule configuration | map[string]string | false | | annotations | Annotations will be added to rule configuration | map[string]string | false | +| update_entries_limit | UpdateEntriesLimit defines max number of rule's state updates stored in memory. Overrides `-rule.updateEntriesLimit` in vmalert. | *int | false | [Back to TOC](#table-of-contents) @@ -1281,6 +1299,7 @@ RuleGroup is a list of sequentially evaluated recording and alerting rules. | params | Params optional HTTP URL parameters added to each rule request | url.Values | false | | type | Type defines datasource type for enterprise version of vmalert possible values - prometheus,graphite | string | false | | headers | Headers contains optional HTTP headers added to each rule request Must be in form `header-name: value` For example:\n headers:\n - \"CustomHeader: foo\"\n - \"CustomHeader2: bar\" | []string | false | +| notifier_headers | NotifierHeaders contains optional HTTP headers added to each alert request which will send to notifier Must be in form `header-name: value` For example:\n headers:\n - \"CustomHeader: foo\"\n - \"CustomHeader2: bar\" | []string | false | [Back to TOC](#table-of-contents) @@ -1572,6 +1591,7 @@ PodMetricsEndpoint defines a scrapeable endpoint of a Kubernetes Pod serving Pro | authorization | Authorization with http header Authorization | *[Authorization](#authorization) | false | | vm_scrape_params | VMScrapeParams defines VictoriaMetrics specific scrape parametrs | *[VMScrapeParams](#vmscrapeparams) | false | | attach_metadata | AttachMetadata configures metadata attaching from service discovery | [AttachMetadata](#attachmetadata) | false | +| filterRunning | FilterRunning applies filter with pod status == running it prevents from scrapping metrics at failed or succeed state pods. enabled by default | *bool | false | [Back to TOC](#table-of-contents) @@ -1969,7 +1989,8 @@ StaticRef - user-defined routing host address. | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| url | URL http url for given staticRef. | string | true | +| url | URL http url for given staticRef. | string | false | +| urls | URLs allows setting multiple urls for load-balancing at vmauth-side. | []string | false | [Back to TOC](#table-of-contents) @@ -1984,6 +2005,7 @@ TargetRef describes target for user traffic forwarding. one of target types can | paths | Paths - matched path to route. | []string | false | | target_path_suffix | QueryParams []string `json:\"queryParams,omitempty\"` TargetPathSuffix allows to add some suffix to the target path It allows to hide tenant configuration from user with crd as ref. it also may contain any url encoded params. | string | false | | headers | Headers represent additional http headers, that vmauth uses in form of [\"header_key: header_value\"] multiple values for header key: [\"header_key: value1,value2\"] it's available since 1.68.0 version of vmauth | []string | false | +| ip_filters | IPFilters defines per target src ip filters supported only with enterprise version of vmauth https://docs.victoriametrics.com/vmauth.html#ip-filters | [VMUserIPFilters](#vmuseripfilters) | false | [Back to TOC](#table-of-contents) @@ -1999,6 +2021,17 @@ VMUser is the Schema for the vmusers API [Back to TOC](#table-of-contents) +## VMUserIPFilters + +VMUserIPFilters defines filters for IP addresses supported only with enterprise version of vmauth https://docs.victoriametrics.com/vmauth.html#ip-filters + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| deny_list | | []string | false | +| allow_list | | []string | false | + +[Back to TOC](#table-of-contents) + ## VMUserList VMUserList contains a list of VMUser @@ -2024,6 +2057,7 @@ VMUserSpec defines the desired state of VMUser | generatePassword | GeneratePassword instructs operator to generate password for user if spec.password if empty. | bool | false | | bearerToken | BearerToken Authorization header value for accessing protected endpoint. | *string | false | | targetRefs | TargetRefs - reference to endpoints, which user may access. | [][TargetRef](#targetref) | true | +| default_url | DefaultURLs backend url for non-matching paths filter usually used for default backend with error message | []string | false | [Back to TOC](#table-of-contents) @@ -2106,7 +2140,7 @@ VMAuthSpec defines the desired state of VMAuth | userNamespaceSelector | UserNamespaceSelector Namespaces to be selected for VMAuth discovery. Works in combination with Selector. NamespaceSelector nil - only objects at VMAuth namespace. Selector nil - only objects at NamespaceSelector namespaces. If both nil - behaviour controlled by selectAllByDefault | *[metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#labelselector-v1-meta) | false | | extraArgs | ExtraArgs that will be passed to VMAuth pod for example remoteWrite.tmpDataPath: /tmp | map[string]string | false | | extraEnvs | ExtraEnvs that will be added to VMAuth pod | [][v1.EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#envvar-v1-core) | false | -| serviceSpec | ServiceSpec that will be added to vmauth service spec | *[ServiceSpec](#servicespec) | false | +| serviceSpec | ServiceSpec that will be added to vmsingle service spec | *[ServiceSpec](#servicespec) | false | | serviceScrapeSpec | ServiceScrapeSpec that will be added to vmauth VMServiceScrape spec | *[VMServiceScrapeSpec](#vmservicescrapespec) | false | | podDisruptionBudget | PodDisruptionBudget created by operator | *[EmbeddedPodDisruptionBudgetSpec](#embeddedpoddisruptionbudgetspec) | false | | ingress | Ingress enables ingress configuration for VMAuth. | *[EmbeddedIngress](#embeddedingress) | false | @@ -2116,6 +2150,19 @@ VMAuthSpec defines the desired state of VMAuth | nodeSelector | NodeSelector Define which Nodes the Pods are scheduled on. | map[string]string | false | | terminationGracePeriodSeconds | TerminationGracePeriodSeconds period for container graceful termination | *int64 | false | | readinessGates | ReadinessGates defines pod readiness gates | []v1.PodReadinessGate | false | +| unauthorizedAccessConfig | UnauthorizedAccessConfig configures access for un authorized users | [][VMAuthUnauthorizedPath](#vmauthunauthorizedpath) | false | + +[Back to TOC](#table-of-contents) + +## VMAuthUnauthorizedPath + +VMAuthUnauthorizedPath defines url_map for unauthorized access + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| src_paths | Paths src request paths | []string | false | +| url_prefix | URLs defines url_prefix for dst routing | []string | false | +| ip_filters | IPFilters defines filter for src ip address enterprise only | [VMUserIPFilters](#vmuseripfilters) | false | [Back to TOC](#table-of-contents) @@ -2200,7 +2247,7 @@ ProbeTargetIngress defines the set of Ingress objects considered for probing. ## VMProbe -\n VMProbe defines a probe for targets, that will be executed with prober,\n like blackbox exporter.\nIt helps to monitor reachability of target with various checks. +VMProbe defines a probe for targets, that will be executed with prober, like blackbox exporter. It helps to monitor reachability of target with various checks. | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | diff --git a/docs/operator/vars.md b/docs/operator/vars.md index 7e279d1602..ac3bc15288 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -9,8 +9,8 @@ menu: aliases: - /operator/vars.html --- -# Auto Generated vars for package config -updated at Mon May 8 06:43:29 UTC 2023 +# Auto Generated vars for package config + updated at Mon Jul 31 18:56:28 UTC 2023 | varible name | variable default value | variable required | variable description | @@ -20,7 +20,7 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_CUSTOMCONFIGRELOADERIMAGE | victoriametrics/operator:config-reloader-v0.32.0 | false | - | | VM_PSPAUTOCREATEENABLED | true | false | - | | VM_VMALERTDEFAULT_IMAGE | victoriametrics/vmalert | false | - | -| VM_VMALERTDEFAULT_VERSION | v1.89.1 | false | - | +| VM_VMALERTDEFAULT_VERSION | v1.91.3 | false | - | | VM_VMALERTDEFAULT_PORT | 8080 | false | - | | VM_VMALERTDEFAULT_USEDEFAULTRESOURCES | true | false | - | | VM_VMALERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - | @@ -31,7 +31,7 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_VMALERTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - | | VM_VMALERTDEFAULT_CONFIGRELOADIMAGE | jimmidyson/configmap-reload:v0.3.0 | false | - | | VM_VMAGENTDEFAULT_IMAGE | victoriametrics/vmagent | false | - | -| VM_VMAGENTDEFAULT_VERSION | v1.89.1 | false | - | +| VM_VMAGENTDEFAULT_VERSION | v1.91.3 | false | - | | VM_VMAGENTDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.58.0 | false | - | | VM_VMAGENTDEFAULT_PORT | 8429 | false | - | | VM_VMAGENTDEFAULT_USEDEFAULTRESOURCES | true | false | - | @@ -42,7 +42,7 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_VMAGENTDEFAULT_CONFIGRELOADERCPU | 100m | false | - | | VM_VMAGENTDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - | | VM_VMSINGLEDEFAULT_IMAGE | victoriametrics/victoria-metrics | false | - | -| VM_VMSINGLEDEFAULT_VERSION | v1.89.1 | false | - | +| VM_VMSINGLEDEFAULT_VERSION | v1.91.3 | false | - | | VM_VMSINGLEDEFAULT_PORT | 8429 | false | - | | VM_VMSINGLEDEFAULT_USEDEFAULTRESOURCES | true | false | - | | VM_VMSINGLEDEFAULT_RESOURCE_LIMIT_MEM | 1500Mi | false | - | @@ -53,14 +53,14 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_VMSINGLEDEFAULT_CONFIGRELOADERMEMORY | 25Mi | false | - | | VM_VMCLUSTERDEFAULT_USEDEFAULTRESOURCES | true | false | - | | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_IMAGE | victoriametrics/vmselect | false | - | -| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.89.1-cluster | false | - | +| VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_VERSION | v1.91.3-cluster | false | - | | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_PORT | 8481 | false | - | | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_MEM | 1000Mi | false | - | | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - | | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - | | VM_VMCLUSTERDEFAULT_VMSELECTDEFAULT_RESOURCE_REQUEST_CPU | 100m | false | - | | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_IMAGE | victoriametrics/vmstorage | false | - | -| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.89.1-cluster | false | - | +| VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VERSION | v1.91.3-cluster | false | - | | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMINSERTPORT | 8400 | false | - | | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_VMSELECTPORT | 8401 | false | - | | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_PORT | 8482 | false | - | @@ -69,7 +69,7 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_MEM | 500Mi | false | - | | VM_VMCLUSTERDEFAULT_VMSTORAGEDEFAULT_RESOURCE_REQUEST_CPU | 250m | false | - | | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_IMAGE | victoriametrics/vminsert | false | - | -| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.89.1-cluster | false | - | +| VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_VERSION | v1.91.3-cluster | false | - | | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_PORT | 8480 | false | - | | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_MEM | 500Mi | false | - | | VM_VMCLUSTERDEFAULT_VMINSERTDEFAULT_RESOURCE_LIMIT_CPU | 500m | false | - | @@ -88,7 +88,7 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_VMALERTMANAGER_RESOURCE_REQUEST_CPU | 30m | false | - | | VM_DISABLESELFSERVICESCRAPECREATION | false | false | - | | VM_VMBACKUP_IMAGE | victoriametrics/vmbackupmanager | false | - | -| VM_VMBACKUP_VERSION | v1.89.1-enterprise | false | - | +| VM_VMBACKUP_VERSION | v1.91.3-enterprise | false | - | | VM_VMBACKUP_PORT | 8300 | false | - | | VM_VMBACKUP_USEDEFAULTRESOURCES | true | false | - | | VM_VMBACKUP_RESOURCE_LIMIT_MEM | 500Mi | false | - | @@ -97,7 +97,7 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_VMBACKUP_RESOURCE_REQUEST_CPU | 150m | false | - | | VM_VMBACKUP_LOGLEVEL | INFO | false | - | | VM_VMAUTHDEFAULT_IMAGE | victoriametrics/vmauth | false | - | -| VM_VMAUTHDEFAULT_VERSION | v1.89.1 | false | - | +| VM_VMAUTHDEFAULT_VERSION | v1.91.3 | false | - | | VM_VMAUTHDEFAULT_CONFIGRELOADIMAGE | quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1 | false | - | | VM_VMAUTHDEFAULT_PORT | 8427 | false | - | | VM_VMAUTHDEFAULT_USEDEFAULTRESOURCES | true | false | - | @@ -126,4 +126,5 @@ updated at Mon May 8 06:43:29 UTC 2023 | VM_PODWAITREADYTIMEOUT | 80s | false | - | | VM_PODWAITREADYINTERVALCHECK | 5s | false | - | | VM_PODWAITREADYINITDELAY | 10s | false | - | -| VM_FORCERESYNCINTERVAL | 60s | false | configures force resync interval for VMAgent, VMAlert and VMAlertmanager | +| VM_FORCERESYNCINTERVAL | 60s | false | configures force resync interval for VMAgent, VMAlert, VMAlertmanager and VMAuth. | +| VM_ENABLESTRICTSECURITY | true | false | EnableStrictSecurity will add default `securityContext` to pods and containers created by operatorDefault PodSecurityContext include:1. RunAsNonRoot: true2. RunAsUser/RunAsGroup/FSGroup: 65534'65534' refers to 'nobody' in all the used default images like alpine, busybox.If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext.Default container SecurityContext include:1. AllowPrivilegeEscalation: false2. ReadOnlyRootFilesystem: true | From 8d0823792310d45b808729eab6dff4b89cea9bc7 Mon Sep 17 00:00:00 2001 From: zhaojinxin409 <5874804+zhaojinxin409@users.noreply.github.com> Date: Tue, 1 Aug 2023 15:09:01 +0800 Subject: [PATCH 18/69] Update stream-aggregation.md for `speed` misspelling (#4752) --- docs/stream-aggregation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stream-aggregation.md b/docs/stream-aggregation.md index acbb9ecb71..9be338bac3 100644 --- a/docs/stream-aggregation.md +++ b/docs/stream-aggregation.md @@ -83,7 +83,7 @@ Sometimes [alerting queries](https://docs.victoriametrics.com/vmalert.html#alert disk IO and network bandwidth at metrics storage side. For example, if `http_request_duration_seconds` histogram is generated by thousands of application instances, then the alerting query `histogram_quantile(0.99, sum(increase(http_request_duration_seconds_bucket[5m])) without (instance)) > 0.5` can become slow, since it needs to scan too big number of unique [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series) -with `http_request_duration_seconds_bucket` name. This alerting query can be sped up by pre-calculating +with `http_request_duration_seconds_bucket` name. This alerting query can be speed up by pre-calculating the `sum(increase(http_request_duration_seconds_bucket[5m])) without (instance)` via [recording rule](https://docs.victoriametrics.com/vmalert.html#recording-rules). But this recording rule may take too much time to execute too. In this case the slow recording rule can be substituted with the following [stream aggregation config](#stream-aggregation-config): From 6289a21d24a7c2918a86f9d1d5c4ffaf5c36cf32 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Tue, 1 Aug 2023 11:33:46 +0400 Subject: [PATCH 19/69] docs: add changelog entry for #4704 (#4753) Signed-off-by: Zakhar Bessarab --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b1e851a076..641e29d976 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). +* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From 833ab331b1d7953b9be320e3514254f89fa1b61a Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Tue, 1 Aug 2023 09:45:50 +0200 Subject: [PATCH 20/69] vmctl: allow disabling binary export protocol (#4716) Binary export API protocol can be disabled via `-vm-native-disable-binary-protocol` cmd-line flag when migrating data from VictoriaMetrics. Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need to be configured at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. Signed-off-by: hagen1778 --- app/vmctl/flags.go | 13 +++++++++++-- app/vmctl/main.go | 1 + app/vmctl/vm_native.go | 12 +++++++++--- app/vmctl/vm_native_test.go | 1 + docs/CHANGELOG.md | 7 ++++--- 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index c41c2cef39..4d72e18678 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -325,8 +325,9 @@ const ( vmNativeFilterTimeEnd = "vm-native-filter-time-end" vmNativeStepInterval = "vm-native-step-interval" - vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive" - vmNativeDisableRetries = "vm-native-disable-retries" + vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol" + vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive" + vmNativeDisableRetries = "vm-native-disable-retries" vmNativeSrcAddr = "vm-native-src-addr" vmNativeSrcUser = "vm-native-src-user" @@ -450,6 +451,14 @@ var ( Usage: "Defines whether to disable retries with backoff policy for migration process", Value: false, }, + &cli.BoolFlag{ + Name: vmNativeDisableBinaryProtocol, + Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format" + + "instead of https://docs.victoriametrics.com/#how-to-export-data-in-native-format API." + + "Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." + + "Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.", + Value: false, + }, } ) diff --git a/app/vmctl/main.go b/app/vmctl/main.go index e81746a762..ba6c570b66 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -254,6 +254,7 @@ func main() { cc: c.Int(vmConcurrency), disableRetries: c.Bool(vmNativeDisableRetries), isSilent: c.Bool(globalSilent), + isNative: !c.Bool(vmNativeDisableBinaryProtocol), } return p.run(ctx) }, diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go index 3c8392b9af..8d94d2bbe1 100644 --- a/app/vmctl/vm_native.go +++ b/app/vmctl/vm_native.go @@ -34,11 +34,12 @@ type vmNativeProcessor struct { cc int disableRetries bool isSilent bool + isNative bool } const ( - nativeExportAddr = "api/v1/export/native" - nativeImportAddr = "api/v1/import/native" + nativeExportAddr = "api/v1/export" + nativeImportAddr = "api/v1/import" nativeWithBackoffTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "█") "▒" "]" }} {{ percent . }}` nativeSingleProcessTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}` ) @@ -159,9 +160,14 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time, silent bool) error { exportAddr := nativeExportAddr + importAddr := nativeImportAddr + if p.isNative { + exportAddr += "/native" + importAddr += "/native" + } srcURL := fmt.Sprintf("%s/%s", p.src.Addr, exportAddr) - importAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.ExtraLabels) + importAddr, err := vm.AddExtraLabelsToImportPath(importAddr, p.dst.ExtraLabels) if err != nil { return fmt.Errorf("failed to add labels to import path: %s", err) } diff --git a/app/vmctl/vm_native_test.go b/app/vmctl/vm_native_test.go index 457437f75d..217df70aa0 100644 --- a/app/vmctl/vm_native_test.go +++ b/app/vmctl/vm_native_test.go @@ -228,6 +228,7 @@ func Test_vmNativeProcessor_run(t *testing.T) { interCluster: tt.fields.interCluster, cc: tt.fields.cc, isSilent: tt.args.silent, + isNative: true, } if err := p.run(tt.args.ctx); (err != nil) != tt.wantErr { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 641e29d976..ec1db9cca3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,10 +27,11 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. - -* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). -* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). + +* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From 5aed369132b3e38d85bb29709b23d861503d9ff0 Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Tue, 1 Aug 2023 16:43:00 +0200 Subject: [PATCH 21/69] app/vmctl: add flag where use can define path to the source remote read protocol (#4744) https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655 --- app/vmctl/flags.go | 6 ++++ app/vmctl/main.go | 1 + app/vmctl/remoteread/remoteread.go | 44 ++++++++++++++++++++---------- docs/CHANGELOG.md | 3 +- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index 4d72e18678..da88ba3f1c 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -477,6 +477,7 @@ const ( remoteReadHTTPTimeout = "remote-read-http-timeout" remoteReadHeaders = "remote-read-headers" remoteReadInsecureSkipVerify = "remote-read-insecure-skip-verify" + remoteReadDisablePathAppend = "remote-read-disable-path-append" ) var ( @@ -553,6 +554,11 @@ var ( Usage: "Whether to skip TLS certificate verification when connecting to the remote read address", Value: false, }, + &cli.BoolFlag{ + Name: remoteReadDisablePathAppend, + Usage: "Whether to disable automatic appending of the path to the remote storage.", + Value: true, + }, } ) diff --git a/app/vmctl/main.go b/app/vmctl/main.go index ba6c570b66..383869d002 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -133,6 +133,7 @@ func main() { LabelName: c.String(remoteReadFilterLabel), LabelValue: c.String(remoteReadFilterLabelValue), InsecureSkipVerify: c.Bool(remoteReadInsecureSkipVerify), + DisablePathAppend: c.Bool(remoteReadDisablePathAppend), }) if err != nil { return fmt.Errorf("error create remote read client: %s", err) diff --git a/app/vmctl/remoteread/remoteread.go b/app/vmctl/remoteread/remoteread.go index fe458f0184..5752ac08a4 100644 --- a/app/vmctl/remoteread/remoteread.go +++ b/app/vmctl/remoteread/remoteread.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "time" @@ -31,19 +32,22 @@ type StreamCallback func(series *vm.TimeSeries) error // Client is an HTTP client for reading // time series via remote read protocol. type Client struct { - addr string - c *http.Client - user string - password string - useStream bool - headers []keyValue - matchers []*prompb.LabelMatcher + addr string + disablePathAppend bool + c *http.Client + user string + password string + useStream bool + headers []keyValue + matchers []*prompb.LabelMatcher } // Config is config for remote read. type Config struct { // Addr of remote storage Addr string + // DisablePathAppend disable automatic appending of the remote read path + DisablePathAppend bool // Timeout defines timeout for HTTP requests // made by remote read client Timeout time.Duration @@ -104,13 +108,15 @@ func NewClient(cfg Config) (*Client, error) { Timeout: cfg.Timeout, Transport: utils.Transport(cfg.Addr, cfg.InsecureSkipVerify), }, - addr: strings.TrimSuffix(cfg.Addr, "/"), - user: cfg.Username, - password: cfg.Password, - useStream: cfg.UseStream, - headers: headers, - matchers: []*prompb.LabelMatcher{m}, + addr: strings.TrimSuffix(cfg.Addr, "/"), + disablePathAppend: cfg.DisablePathAppend, + user: cfg.Username, + password: cfg.Password, + useStream: cfg.UseStream, + headers: headers, + matchers: []*prompb.LabelMatcher{m}, } + return c, nil } @@ -155,8 +161,16 @@ func (c *Client) do(req *http.Request) (*http.Response, error) { func (c *Client) fetch(ctx context.Context, data []byte, streamCb StreamCallback) error { r := bytes.NewReader(data) - url := c.addr + remoteReadPath - req, err := http.NewRequest(http.MethodPost, url, r) + // by default, we are using a common remote read path + u, err := url.JoinPath(c.addr, remoteReadPath) + if err != nil { + return fmt.Errorf("error create url from addr %s and default remote read path %s", c.addr, remoteReadPath) + } + // we should use full address from the remote-read-src-addr flag + if c.disablePathAppend { + u = c.addr + } + req, err := http.NewRequest(http.MethodPost, u, r) if err != nil { return fmt.Errorf("failed to create new HTTP request: %w", err) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ec1db9cca3..71a6b4caa6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,9 +29,10 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). -* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). +* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From f1a4c0b614a87efd5617d895a87d76c218e70bf5 Mon Sep 17 00:00:00 2001 From: Anton Tykhyy Date: Wed, 2 Aug 2023 14:04:38 +0300 Subject: [PATCH 22/69] Remove some repetitions in docker Makefile (#4764) --- deployment/docker/Makefile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 32897e32de..6029bf6fcf 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -12,19 +12,21 @@ CERTS_IMAGE := alpine:3.17.3 GO_BUILDER_IMAGE := golang:1.20.6-alpine BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) +DOCKER_BUILD ?= docker build DOCKER_COMPOSE ?= docker compose +DOCKER_IMAGE_LS ?= docker image ls --format '{{.Repository}}:{{.Tag}}' package-base: - (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BASE_IMAGE)$$') \ - || docker build \ + ($(DOCKER_IMAGE_LS) | grep -q '$(BASE_IMAGE)$$') \ + || $(DOCKER_BUILD) \ --build-arg root_image=$(ROOT_IMAGE) \ --build-arg certs_image=$(CERTS_IMAGE) \ --tag $(BASE_IMAGE) \ deployment/docker/base package-builder: - (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(BUILDER_IMAGE)$$') \ - || docker build \ + ($(DOCKER_IMAGE_LS) | grep -q '$(BUILDER_IMAGE)$$') \ + || $(DOCKER_BUILD) \ --build-arg go_builder_image=$(GO_BUILDER_IMAGE) \ --tag $(BUILDER_IMAGE) \ deployment/docker/builder @@ -60,9 +62,9 @@ app-via-docker-windows: package-builder -o bin/$(APP_NAME)-windows$(APP_SUFFIX)-prod.exe $(PKG_PREFIX)/app/$(APP_NAME) package-via-docker: package-base - (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\ + ($(DOCKER_IMAGE_LS) | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE)$$') || (\ $(MAKE) app-via-docker && \ - docker build \ + $(DOCKER_BUILD) \ --build-arg src_binary=$(APP_NAME)$(APP_SUFFIX)-prod \ --build-arg base_image=$(BASE_IMAGE) \ --tag $(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(APP_SUFFIX)$(RACE) \ @@ -170,7 +172,7 @@ package-via-docker-386: GOARCH=386 $(MAKE) package-via-docker-goarch-nocgo remove-docker-images: - docker image ls --format '{{.Repository}}\t{{.ID}}' | awk '{print $$2}' | xargs docker image rm -f + docker image ls --format '{{.ID}}' | xargs docker image rm -f docker-single-up: $(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml up -d From 0c3d61b211b9016f4519362e0ba9c9ea8975ca9d Mon Sep 17 00:00:00 2001 From: Artem Navoiev Date: Wed, 2 Aug 2023 05:20:48 -0700 Subject: [PATCH 23/69] docs: add Naver Case Study (#4755) Signed-off-by: Artem Navoiev Co-authored-by: Roman Khavronenko --- README.md | 1 + docs/CaseStudies.md | 13 +++++++++++++ docs/README.md | 1 + docs/Single-server-VictoriaMetrics.md | 1 + 4 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 63eb1e0713..e59605d4c3 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Case studies: * [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x) * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode) * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind) +* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver) * [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay) * [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox) diff --git a/docs/CaseStudies.md b/docs/CaseStudies.md index 42634c7c4c..15a362f9c6 100644 --- a/docs/CaseStudies.md +++ b/docs/CaseStudies.md @@ -30,6 +30,7 @@ where you can chat with VictoriaMetrics users to get additional references, revi - [Groove X](#groove-x) - [Idealo.de](#idealode) - [MHI Vestas Offshore Wind](#mhi-vestas-offshore-wind) + - [Naver][#naver] - [Percona](#percona) - [Razorpay](#razorpay) - [Roblox](#roblox) @@ -420,6 +421,18 @@ Numbers with current, limited roll out: - Data size on disk: 800 GiB - Retention period: 3 years +## Naver + +See [our](https://www.navercorp.com/en/) video ["Time Series in the Multiverse of Madness" (in Korean)](https://www.youtube.com/watch?v=OUyXPgVcdw4) about the comparison of Time Series Database, why we have chosen VictoriaMetrics +We also covered the internals of the VictoriaMetrics data model and Cluster. +The key areas: + +* Explanation of the importance and role of Monitoring for NaverCorp +* History overview of Time Series Databases +* VictoriaMetrics Data model - read and write paths, index structure, compression and the crucial role of the churn rate +* Time series in the Multiverse Madness +* HA and Fault Tolerance - write without data loss, read for no downtime, management of Multiverse + ## Percona [Percona](https://www.percona.com/) is a leader in providing best-of-breed enterprise-class support, consulting, managed services, training and software for MySQL®, MariaDB®, MongoDB®, PostgreSQL® and other open source databases in on-premises and cloud environments. diff --git a/docs/README.md b/docs/README.md index 52d1de8510..6bb73fb490 100644 --- a/docs/README.md +++ b/docs/README.md @@ -120,6 +120,7 @@ Case studies: * [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x) * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode) * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind) +* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver) * [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay) * [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 2c3517e9b8..580587cb5f 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -128,6 +128,7 @@ Case studies: * [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x) * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode) * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind) +* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver) * [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay) * [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona) * [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox) From 8f4961fbbd72ddca57fcb0ffb48b54376def1179 Mon Sep 17 00:00:00 2001 From: Yury Molodov Date: Wed, 2 Aug 2023 14:21:52 +0200 Subject: [PATCH 24/69] vmui: display partial response warning (#4742) https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721 --- app/vmui/packages/vmui/src/api/types.ts | 1 + .../Configurators/QueryEditor/QueryEditor.tsx | 24 ++++++++++++------- .../Configurators/QueryEditor/style.scss | 8 +++++++ .../Configurators/QueryEditor/warningText.ts | 7 ++++++ .../packages/vmui/src/hooks/useFetchQuery.ts | 1 + docs/CHANGELOG.md | 1 + 6 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts diff --git a/app/vmui/packages/vmui/src/api/types.ts b/app/vmui/packages/vmui/src/api/types.ts index 19b3549fb0..17c46f821e 100644 --- a/app/vmui/packages/vmui/src/api/types.ts +++ b/app/vmui/packages/vmui/src/api/types.ts @@ -24,6 +24,7 @@ export interface TracingData { export interface QueryStats { seriesFetched?: string; resultLength?: number; + isPartial?: boolean; } export interface Logs { diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx index f0ef16c602..863918c2aa 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx +++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx @@ -7,6 +7,7 @@ import "./style.scss"; import { QueryStats } from "../../../api/types"; import Tooltip from "../../Main/Tooltip/Tooltip"; import { WarningIcon } from "../../Main/Icons"; +import { partialWarning, seriesFetchedWarning } from "./warningText"; export interface QueryEditorProps { onChange: (query: string) => void; @@ -39,7 +40,17 @@ const QueryEditor: FC = ({ const [openAutocomplete, setOpenAutocomplete] = useState(false); const autocompleteAnchorEl = useRef(null); - const showSeriesFetchedWarning = stats?.seriesFetched === "0" && !stats.resultLength; + + const warnings = [ + { + show: stats?.seriesFetched === "0" && !stats.resultLength, + text: seriesFetchedWarning + }, + { + show: stats?.isPartial, + text: partialWarning + } + ].filter((warning) => warning.show); const handleSelect = (val: string) => { onChange(val); @@ -108,17 +119,14 @@ const QueryEditor: FC = ({ onFoundOptions={handleChangeFoundOptions} /> )} - {showSeriesFetchedWarning && ( + {!!warnings.length && (
- {`No match! - This query hasn't selected any time series from database. - Either the requested metrics are missing in the database, - or there is a typo in series selector.`} - +
+ {warnings.map((warning, index) =>

{warning.text}

)} +
)} > diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss index 3381222c10..7c8354845b 100644 --- a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss +++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/style.scss @@ -22,6 +22,14 @@ &__tooltip { white-space: pre-line; + + p { + margin-bottom: $padding-small; + + &:last-child { + margin-bottom: 0; + } + } } } } diff --git a/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts new file mode 100644 index 0000000000..de9a9ebfc6 --- /dev/null +++ b/app/vmui/packages/vmui/src/components/Configurators/QueryEditor/warningText.ts @@ -0,0 +1,7 @@ +export const seriesFetchedWarning = `No match! +This query hasn't selected any time series from database. +Either the requested metrics are missing in the database, +or there is a typo in series selector.`; + +export const partialWarning = `The shown results are marked as PARTIAL. +The result is marked as partial if one or more vmstorage nodes failed to respond to the query.`; diff --git a/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts b/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts index 53a333f5a2..7576ecda26 100644 --- a/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts +++ b/app/vmui/packages/vmui/src/hooks/useFetchQuery.ts @@ -103,6 +103,7 @@ export const useFetchQuery = ({ if (response.ok) { setQueryStats(prev => [...prev, { ...resp?.stats, + isPartial: resp?.isPartial, resultLength: resp.data.result.length, }]); setQueryErrors(prev => [...prev, ""]); diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 71a6b4caa6..942142726f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -30,6 +30,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). +* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). From df37a47d4b50f2bfb50e67ca45d7b28a3dec1a7a Mon Sep 17 00:00:00 2001 From: SunKyu Lee Date: Wed, 2 Aug 2023 21:30:21 +0900 Subject: [PATCH 25/69] vmauth: add broken backend backoff flag (#4416) vmauth: allow configuring deadline for a backend to be excluded from the rotation The new flag `-failTimeout` allows overriding default time for a bad backend to be excluded from rotation. The override option could be useful for systems where it is expected for backends to be off for significant periods of time. Co-authored-by: Zakhar Bessarab --- app/vmauth/README.md | 2 ++ app/vmauth/auth_config.go | 2 +- app/vmauth/main.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/vmauth/README.md b/app/vmauth/README.md index 72daeffc24..cf6ea3a192 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -351,6 +351,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Prefix for environment variables if -envflag.enable is set -eula By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -failTimeout duration + Sets a delay period for load balancing to skip a malfunctioning backend. (defaults 3s) -flagsAuthKey string Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -fs.disableMmap diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 3c7ce00ed6..6fab91dc27 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -134,7 +134,7 @@ func (bu *backendURL) isBroken() bool { } func (bu *backendURL) setBroken() { - deadline := fasttime.UnixTimestamp() + 3 + deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds()) atomic.StoreUint64(&bu.brokenDeadline, deadline) } diff --git a/app/vmauth/main.go b/app/vmauth/main.go index c859e913cf..925bfb7c60 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -41,6 +41,7 @@ var ( reloadAuthKey = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+ `Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`) + failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend.") ) func main() { From 061f68fe5ef2caf6dd0669a1d8ddf074f46563b9 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Wed, 2 Aug 2023 14:35:37 +0200 Subject: [PATCH 26/69] docs: follow-up after df37a47d4b50f2bfb50e67ca45d7b28a3dec1a7a https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415 Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 942142726f..d76839d1eb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -25,6 +25,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow configuring deadline for a backend to be excluded from the rotation on errors via `-failTimeout` cmd-line flag. This feature could be useful when it is expected for backends to be not available for significant periods of time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415) for details. Thanks to @SunKyu for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4416). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. From 093d43de4573c1f78b737e08878896f7d8d76687 Mon Sep 17 00:00:00 2001 From: Artem Navoiev Date: Wed, 2 Aug 2023 23:00:57 +0200 Subject: [PATCH 27/69] add link to the slides for Naver case study Signed-off-by: Artem Navoiev --- docs/CaseStudies.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/CaseStudies.md b/docs/CaseStudies.md index 15a362f9c6..5d5eaffead 100644 --- a/docs/CaseStudies.md +++ b/docs/CaseStudies.md @@ -433,6 +433,8 @@ The key areas: * Time series in the Multiverse Madness * HA and Fault Tolerance - write without data loss, read for no downtime, management of Multiverse +[Slides](https://deview.kr/data/deview/session/attach/%5B2B4%5DVictoriaMetrics_%E1%84%89%E1%85%B5%E1%84%80%E1%85%A8%E1%84%8B%E1%85%A7%E1%86%AF_%E1%84%83%E1%85%A6%E1%84%8B%E1%85%B5%E1%84%90%E1%85%A5_%E1%84%83%E1%85%A2%E1%84%92%E1%85%A9%E1%86%AB%E1%84%83%E1%85%A9%E1%86%AB%E1%84%8B%E1%85%B4_%E1%84%86%E1%85%A5%E1%86%AF%E1%84%90%E1%85%B5%E1%84%87%E1%85%A5%E1%84%89%E1%85%B3_Kor+Eng.pdf) in English and Korean + ## Percona [Percona](https://www.percona.com/) is a leader in providing best-of-breed enterprise-class support, consulting, managed services, training and software for MySQL®, MariaDB®, MongoDB®, PostgreSQL® and other open source databases in on-premises and cloud environments. From 5d73a07cc33c68fb8e4f57a01644ba178d31387b Mon Sep 17 00:00:00 2001 From: Jan Kielmann Date: Thu, 3 Aug 2023 09:28:41 +0200 Subject: [PATCH 28/69] Fix spelling mistake in opentelemetry ingestion API path --- README.md | 2 +- docs/Cluster-VictoriaMetrics.md | 2 +- docs/README.md | 2 +- docs/Single-server-VictoriaMetrics.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e59605d4c3..38c53cdc34 100644 --- a/README.md +++ b/README.md @@ -1361,7 +1361,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc ## Sending data via OpenTelemetry -VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentemetry/api/v1/push` path. +VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path. VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`. Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`. diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index af7289dbbe..06aa25cee8 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -340,7 +340,7 @@ Check practical examples of VictoriaMetrics API [here](https://docs.victoriametr - `prometheus/api/v1/import/native` - for importing data obtained via `api/v1/export/native` on `vmselect` (see below). - `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-csv-data) for details. - `prometheus/api/v1/import/prometheus` - for importing data in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md). This endpoint also supports [Pushgateway protocol](https://github.com/prometheus/pushgateway#url). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-prometheus-exposition-format) for details. - - `opentemetry/api/v1/push` - for ingesting data via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-data-via-opentelemetry). + - `opentelemetry/api/v1/push` - for ingesting data via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-data-via-opentelemetry). - `datadog/api/v1/series` - for ingesting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent) for details. - `influx/write` and `influx/api/v2/write` - for ingesting data with [InfluxDB line protocol](https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/). See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details. - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html). This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-opentsdb-data-via-http-apiput-requests) for details. diff --git a/docs/README.md b/docs/README.md index 6bb73fb490..e7bb8d4884 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1364,7 +1364,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc ## Sending data via OpenTelemetry -VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentemetry/api/v1/push` path. +VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path. VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`. Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 580587cb5f..78df61bd24 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1372,7 +1372,7 @@ VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-sc ## Sending data via OpenTelemetry -VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentemetry/api/v1/push` path. +VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path. VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`. Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`. From 8895fb1d5f54f39c5826dd10d57fa1a25f8cbfa5 Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Thu, 3 Aug 2023 15:56:18 +0800 Subject: [PATCH 29/69] Automatic update operator docs from VictoriaMetrics/operator@29a7495 (#4770) --- docs/operator/CHANGELOG.md | 1 + docs/operator/vars.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index 45dc37c40f..db501ef387 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -24,6 +24,7 @@ - fix `attachMetadata` value miscovert for scrape objects. See [this issue](https://github.com/VictoriaMetrics/operator/issues/697) and [this PR](https://github.com/VictoriaMetrics/operator/pull/698) for details. - [vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): fix `tlsCAFile` argument value generation when using secret or configMap. See [this issue](https://github.com/VictoriaMetrics/operator/issues/699) and [this PR](https://github.com/VictoriaMetrics/operator/issues/699) for details. - [vmalertmanager](https://docs.victoriametrics.com/operator/api.html#vmalertmanager): fix default request memory and apply default resources if not set. See [this issue](https://github.com/VictoriaMetrics/operator/issues/706) and [this PR](https://github.com/VictoriaMetrics/operator/pull/710) for details. +- [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): fix missing additional VolumeClaimTemplates when using `ClaimTemplates` under StatefulMode. ### Features diff --git a/docs/operator/vars.md b/docs/operator/vars.md index ac3bc15288..be4340deec 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ aliases: - /operator/vars.html --- # Auto Generated vars for package config - updated at Mon Jul 31 18:56:28 UTC 2023 + updated at Thu Aug 3 03:30:07 UTC 2023 | varible name | variable default value | variable required | variable description | From e311a7bf802900c6d9c9a5c0ecdd2aeaa71b4983 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 3 Aug 2023 09:28:47 +0200 Subject: [PATCH 30/69] dashboards: add `Concurrent inserts` panel to vmagent's dasbhoard The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. The panel contains recommendation what to do if limit is reached. Signed-off-by: hagen1778 --- dashboards/vmagent.json | 209 +++++++++++++++++++++++++++++++--------- docs/CHANGELOG.md | 1 + 2 files changed, 164 insertions(+), 46 deletions(-) diff --git a/dashboards/vmagent.json b/dashboards/vmagent.json index da1fd70de3..8fa59ce021 100644 --- a/dashboards/vmagent.json +++ b/dashboards/vmagent.json @@ -6,7 +6,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.2.6" + "version": "9.2.7" }, { "type": "datasource", @@ -182,7 +182,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -249,7 +249,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -310,7 +310,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -379,7 +379,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -451,7 +451,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -515,7 +515,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -606,7 +606,7 @@ }, "showHeader": true }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -2372,8 +2372,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2475,8 +2474,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2581,8 +2579,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -2687,8 +2684,7 @@ "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "transparent" }, { "color": "red", @@ -2792,8 +2788,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2898,8 +2893,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3009,8 +3003,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3113,8 +3106,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3188,8 +3180,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4054,7 +4045,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4070,7 +4062,7 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 38 }, "id": 73, "links": [], @@ -4170,7 +4162,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4186,7 +4179,7 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 38 }, "id": 77, "links": [], @@ -4233,6 +4226,123 @@ ], "title": "Error rate ($instance)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Shows how many concurrent inserts are taking place.\n\nIf the number of concurrent inserts hitting the `limit` or is close to the `limit` constantly - it might be a sign of a resource shortage.\n\n If vmagent's CPU usage and remote write connection saturation are at normal level, it might be that `-maxConcurrentInserts` cmd-line flag need to be increased.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max_over_time(vm_concurrent_insert_current{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}} ({{job}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(vm_concurrent_insert_capacity{job=~\"$job\", instance=~\"$instance\"}) by(job)", + "interval": "", + "legendFormat": "limit ({{job}})", + "range": true, + "refId": "B" + } + ], + "title": "Concurrent inserts ($instance)", + "type": "timeseries" } ], "targets": [ @@ -4310,7 +4420,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4326,7 +4437,7 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 55 }, "id": 60, "options": { @@ -4412,7 +4523,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4428,7 +4540,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 55 }, "id": 66, "options": { @@ -4514,7 +4626,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4530,7 +4643,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 63 }, "id": 61, "options": { @@ -4616,7 +4729,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4632,7 +4746,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 63 }, "id": 65, "options": { @@ -4717,7 +4831,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null }, { "color": "red", @@ -4733,7 +4848,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 71 }, "id": 88, "options": { @@ -4815,7 +4930,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null }, { "color": "red", @@ -4831,7 +4947,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 71 }, "id": 84, "options": { @@ -4916,7 +5032,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null }, { "color": "red", @@ -4932,7 +5049,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 79 }, "id": 90, "options": { @@ -5605,8 +5722,8 @@ { "current": { "selected": true, - "text": "VictoriaMetrics", - "value": "VictoriaMetrics" + "text": "VictoriaMetrics - cluster", + "value": "VictoriaMetrics - cluster" }, "hide": 0, "includeAll": false, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d76839d1eb..f2f93259d3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). +* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). From 1043fc1fd94911c2940a2b9415430473b82fc81e Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 3 Aug 2023 10:21:18 +0200 Subject: [PATCH 31/69] alerts: add docs section for the full list of alerting rules The change also includes update of all references in other docs to the alerting rules. Signed-off-by: hagen1778 --- README.md | 2 +- deployment/docker/README.md | 19 +++++++++++++++++++ docs/CHANGELOG.md | 2 +- docs/Cluster-VictoriaMetrics.md | 2 +- docs/Quick-Start.md | 3 +-- docs/README.md | 2 +- docs/Single-server-VictoriaMetrics.md | 2 +- docs/Troubleshooting.md | 5 ++--- .../multi-regional-setup-dedicated-regions.md | 6 ++---- 9 files changed, 29 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 38c53cdc34..f792d14739 100644 --- a/README.md +++ b/README.md @@ -1782,7 +1782,7 @@ created by community. Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it. -We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) +We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts) via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus. VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page. diff --git a/deployment/docker/README.md b/deployment/docker/README.md index c7fabf6f0e..d4ab1a9cf1 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -109,3 +109,22 @@ Grafana is provisioned by default with following entities: * `VictoriaMetrics - vmalert` dashboard Remember to pick `VictoriaMetrics - cluster` datasource when viewing `VictoriaMetrics - cluster` dashboard. + +## Alerts + +See below a list of recommended alerting rules for various VictoriaMetrics components for running in production. +Some of the alerting rules thresholds are just recommendations and could require an adjustment. The list +of alerting rules is the following: +* [alerts-health.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml): + alerting rules related to all VictoriaMetrics components for tracking their "health" state; +* [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml): + alerting rules related to [single-server VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html) installation; +* [alerts-cluster.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml): + alerting rules related to [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html); +* [alerts-vmagent.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmagent.yml): + alerting rules related to [vmagent](https://docs.victoriametrics.com/vmagent.html) component; +* [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmalert.yml): + alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert.html) component; + +Please, also see [how to monitor](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#monitoring) +VictoriaMetrics installations. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f2f93259d3..4ec806ba76 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2139,7 +2139,7 @@ in front of VictoriaMetrics. [Contact us](mailto:sales@victoriametrics.com) if y Released at 2021-01-13 -* FEATURE: provide a sample list of alerting rules for VictoriaMetrics components. It is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml). +* FEATURE: provide a sample list of alerting rules for VictoriaMetrics components. It is available [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts). * FEATURE: disable final merge for data for the previous month at the beginning of new month, since it may result in high disk IO and CPU usage. Final merge can be enabled by setting `-finalMergeDelay` command-line flag to positive duration. * FEATURE: add `tfirst_over_time(m[d])` and `tlast_over_time(m[d])` functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) for returning timestamps for the first and the last data point in `m` over `d` duration. * FEATURE: add ability to pass multiple labels to `sort_by_label()` and `sort_by_label_desc()` functions. See . diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 06aa25cee8..051fa7bc1a 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -294,7 +294,7 @@ or Prometheus to scrape `/metrics` pages from all the cluster components, so the with [the official Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/) or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11831). Graphs on these dashboards contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it. -It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml). +It is recommended setting up alerts in [vmalert](https://docs.victoriametrics.com/vmalert.html) or in Prometheus from [this list](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts). See more details in the article [VictoriaMetrics Monitoring](https://victoriametrics.com/blog/victoriametrics-monitoring/). ## Cardinality limiter diff --git a/docs/Quick-Start.md b/docs/Quick-Start.md index e14034d29e..c48b42049c 100644 --- a/docs/Quick-Start.md +++ b/docs/Quick-Start.md @@ -145,8 +145,7 @@ VictoriaMetric team prepared a list of [Grafana dashboards](https://grafana.com/ for the main components. Each dashboard contains a lot of useful information and tips. It is recommended to have these dashboards installed and up to date. -The list of alerts for [single](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) -and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml) +Using the [recommended alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts) versions would also help to identify and notify about issues with the system. The rule of thumb is to have a separate installation of VictoriaMetrics or any other monitoring system diff --git a/docs/README.md b/docs/README.md index e7bb8d4884..448d59f560 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1785,7 +1785,7 @@ created by community. Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it. -We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) +We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts) via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus. VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 78df61bd24..b3e54edf71 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1793,7 +1793,7 @@ created by community. Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it. -We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) +We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts) via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus. VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page. diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 84e110baca..04e221ba46 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -414,9 +414,8 @@ would help identify and prevent most of the issues listed above. [Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards) contain panels reflecting the health state, resource usage and other specific metrics for VictoriaMetrics components. -Alerting rules for [single-node](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) -and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml) versions -of VictoriaMetrics will notify about issues with Victoriametrics components and provide recommendations for how to solve them. +The list of [recommended alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts) +for VictoriaMetrics components will notify about issues and provide recommendations for how to solve them. Internally, we heavily rely both on dashboards and alerts, and constantly improve them. It is important to stay up to date with such changes. diff --git a/docs/guides/multi-regional-setup-dedicated-regions.md b/docs/guides/multi-regional-setup-dedicated-regions.md index fc61580b0a..e9f284b64a 100644 --- a/docs/guides/multi-regional-setup-dedicated-regions.md +++ b/docs/guides/multi-regional-setup-dedicated-regions.md @@ -75,10 +75,8 @@ You can set up vmalert in each Ground control region that evaluates recording an For alert deduplication, please use [cluster mode in Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability). -We also recommend adopting these alerts: - -* VictoriaMetrics Single - [https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) -* VictoriaMetrics Cluster - [https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/deployment/docker/alerts.yml) +We also recommend adopting the list of [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts) +for VictoriaMetrics components. ### Monitoring From 2e4d0d0e418a1a35ebcb819e6b1a6a780f6e342c Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 3 Aug 2023 10:45:21 +0200 Subject: [PATCH 32/69] alerts: move `ConcurrentFlushesHitTheLimit` alert to health alerts The `ConcurrentFlushesHitTheLimit` could be related to components like vminsert, vmstorage, vm-single-node and vmagent. Moving this alert to the `health` section of alerts will be benefitial for all components and will remove the duplicates from single/cluster alerts. Signed-off-by: hagen1778 --- deployment/docker/alerts-cluster.yml | 12 ------------ deployment/docker/alerts-health.yml | 16 +++++++++++++++- deployment/docker/alerts.yml | 12 ------------ docs/CHANGELOG.md | 1 + 4 files changed, 16 insertions(+), 25 deletions(-) diff --git a/deployment/docker/alerts-cluster.yml b/deployment/docker/alerts-cluster.yml index 72dcd75d00..2994b6f9a2 100644 --- a/deployment/docker/alerts-cluster.yml +++ b/deployment/docker/alerts-cluster.yml @@ -80,18 +80,6 @@ groups: description: "RPC errors are interconnection errors between cluster components.\n Possible reasons for errors are misconfiguration, overload, network blips or unreachable components." - - alert: ConcurrentFlushesHitTheLimit - expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity - for: 15m - labels: - severity: warning - show_at: dashboard - annotations: - dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=133&var-instance={{ $labels.instance }}" - summary: "vmstorage on instance {{ $labels.instance }} is constantly hitting concurrent flushes limit" - description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n - When vmstorage constantly hits the limit it means that storage is overloaded and requires more CPU." - - alert: RowsRejectedOnIngestion expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0 for: 15m diff --git a/deployment/docker/alerts-health.yml b/deployment/docker/alerts-health.yml index fde2de5ae2..11ec22c1f8 100644 --- a/deployment/docker/alerts-health.yml +++ b/deployment/docker/alerts-health.yml @@ -1,4 +1,5 @@ -# File contains default list of alerts for VM components. +# File contains default list of alerts for various VM components. +# The following alerts are recommended for use for any VM installation. # The alerts below are just recommendations and may require some updates # and threshold calibration according to every specific setup. groups: @@ -73,3 +74,16 @@ groups: description: "The rate of TSID misses during query lookups is too high for \"{{ $labels.job }}\" ({{ $labels.instance }}).\n Make sure you're running VictoriaMetrics of v1.85.3 or higher.\n Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3502" + + - alert: ConcurrentInsertsHitTheLimit + expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit" + description: "The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs.\n + Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU. + In some cases for components like vmagent or vminsert the alert might trigger if there are too many clients + making write attempts. If vmagent's or vminsert's CPU usage and network saturation are at normal level, then + it might be worth adjusting `-maxConcurrentInserts` cmd-line flag." diff --git a/deployment/docker/alerts.yml b/deployment/docker/alerts.yml index 49cb4317e4..62df9af9da 100644 --- a/deployment/docker/alerts.yml +++ b/deployment/docker/alerts.yml @@ -60,18 +60,6 @@ groups: description: "Requests to path {{ $labels.path }} are receiving errors. Please verify if clients are sending correct requests." - - alert: ConcurrentFlushesHitTheLimit - expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity - for: 15m - labels: - severity: warning - show_at: dashboard - annotations: - dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=59&var-instance={{ $labels.instance }}" - summary: "VictoriaMetrics on instance {{ $labels.instance }} is constantly hitting concurrent flushes limit" - description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n - When VictoriaMetrics constantly hits the limit it means that storage is overloaded and requires more CPU." - - alert: RowsRejectedOnIngestion expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0 for: 15m diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4ec806ba76..86152d893f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. +* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). From 64e24e9e2bf9445ec1d218c0237451f55ac61cfd Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 3 Aug 2023 10:50:32 +0200 Subject: [PATCH 33/69] deployment: bump components version to 1.92.1 Signed-off-by: hagen1778 --- deployment/docker/docker-compose-cluster.yml | 12 ++++++------ deployment/docker/docker-compose.yml | 6 +++--- deployment/logs-benchmark/docker-compose.yml | 2 +- .../digitialocean/one-click-droplet/RELEASE_GUIDE.md | 2 +- .../files/etc/update-motd.d/99-one-click | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/deployment/docker/docker-compose-cluster.yml b/deployment/docker/docker-compose-cluster.yml index d66ba0c76f..1208c59e34 100644 --- a/deployment/docker/docker-compose-cluster.yml +++ b/deployment/docker/docker-compose-cluster.yml @@ -2,7 +2,7 @@ version: '3.5' services: vmagent: container_name: vmagent - image: victoriametrics/vmagent:v1.92.0 + image: victoriametrics/vmagent:v1.92.1 depends_on: - "vminsert" ports: @@ -32,7 +32,7 @@ services: vmstorage-1: container_name: vmstorage-1 - image: victoriametrics/vmstorage:v1.92.0-cluster + image: victoriametrics/vmstorage:v1.92.1-cluster ports: - 8482 - 8400 @@ -44,7 +44,7 @@ services: restart: always vmstorage-2: container_name: vmstorage-2 - image: victoriametrics/vmstorage:v1.92.0-cluster + image: victoriametrics/vmstorage:v1.92.1-cluster ports: - 8482 - 8400 @@ -56,7 +56,7 @@ services: restart: always vminsert: container_name: vminsert - image: victoriametrics/vminsert:v1.92.0-cluster + image: victoriametrics/vminsert:v1.92.1-cluster depends_on: - "vmstorage-1" - "vmstorage-2" @@ -68,7 +68,7 @@ services: restart: always vmselect: container_name: vmselect - image: victoriametrics/vmselect:v1.92.0-cluster + image: victoriametrics/vmselect:v1.92.1-cluster depends_on: - "vmstorage-1" - "vmstorage-2" @@ -82,7 +82,7 @@ services: vmalert: container_name: vmalert - image: victoriametrics/vmalert:v1.92.0 + image: victoriametrics/vmalert:v1.92.1 depends_on: - "vmselect" ports: diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index 850bfd22d9..d2a5ff16cc 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.5" services: vmagent: container_name: vmagent - image: victoriametrics/vmagent:v1.92.0 + image: victoriametrics/vmagent:v1.92.1 depends_on: - "victoriametrics" ports: @@ -18,7 +18,7 @@ services: restart: always victoriametrics: container_name: victoriametrics - image: victoriametrics/victoria-metrics:v1.92.0 + image: victoriametrics/victoria-metrics:v1.92.1 ports: - 8428:8428 - 8089:8089 @@ -56,7 +56,7 @@ services: restart: always vmalert: container_name: vmalert - image: victoriametrics/vmalert:v1.92.0 + image: victoriametrics/vmalert:v1.92.1 depends_on: - "victoriametrics" - "alertmanager" diff --git a/deployment/logs-benchmark/docker-compose.yml b/deployment/logs-benchmark/docker-compose.yml index fa28a1e22c..9d04d23c3e 100644 --- a/deployment/logs-benchmark/docker-compose.yml +++ b/deployment/logs-benchmark/docker-compose.yml @@ -105,7 +105,7 @@ services: - '--config=/config.yml' vmsingle: - image: victoriametrics/victoria-metrics:v1.92.0 + image: victoriametrics/victoria-metrics:v1.92.1 ports: - '8428:8428' command: diff --git a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md index e8e2d6f90f..21040199f3 100644 --- a/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md +++ b/deployment/marketplace/digitialocean/one-click-droplet/RELEASE_GUIDE.md @@ -8,7 +8,7 @@ 4. Set variables `DIGITALOCEAN_API_TOKEN` with `VM_VERSION` for `packer` environment and run make from example below: ```console -make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.92.0" +make release-victoria-metrics-digitalocean-oneclick-droplet DIGITALOCEAN_API_TOKEN="dop_v23_2e46f4759ceeeba0d0248" VM_VERSION="1.92.1" ``` diff --git a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click index 8a0c0744fb..f07cffca9a 100755 --- a/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click +++ b/deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click @@ -19,8 +19,8 @@ On the server: * VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface. ******************************************************************************** - # This image includes 1.92.0 version of VictoriaMetrics. - # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0 + # This image includes 1.92.1 version of VictoriaMetrics. + # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1 # Welcome to VictoriaMetrics droplet! From c47138e1b0137f453b0a01d5621b2169d670599b Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 3 Aug 2023 11:14:14 +0200 Subject: [PATCH 34/69] dashboards: add panels for absoulte value of mem and cpu usage by vmalert See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627 Signed-off-by: hagen1778 --- dashboards/vmalert.json | 318 ++++++++++++++++++++++++++++++++++------ docs/CHANGELOG.md | 1 + 2 files changed, 276 insertions(+), 43 deletions(-) diff --git a/dashboards/vmalert.json b/dashboards/vmalert.json index 966f4407e9..37963202a4 100644 --- a/dashboards/vmalert.json +++ b/dashboards/vmalert.json @@ -6,7 +6,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.2.6" + "version": "9.2.7" }, { "type": "datasource", @@ -204,7 +204,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -264,7 +264,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -324,7 +324,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -388,7 +388,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -452,7 +452,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -546,7 +546,7 @@ }, "showHeader": true }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -1182,7 +1182,7 @@ } ] }, - "pluginVersion": "9.2.6", + "pluginVersion": "9.2.7", "targets": [ { "datasource": { @@ -1243,6 +1243,230 @@ }, "id": 43, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "The precentage of used RSS memory\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 37, + "links": [ + { + "targetBlank": true, + "title": "Profiling", + "url": "https://docs.victoriametrics.com/vmagent.html#profiling" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory usage % ($instance)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "description": "Amount of used RSS memory\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 57, + "links": [ + { + "targetBlank": true, + "title": "Profiling", + "url": "https://docs.victoriametrics.com/vmagent.html#profiling" + } + ], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(job)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory usage ($instance)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -1308,7 +1532,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 41 }, "id": 35, "links": [ @@ -1362,7 +1586,7 @@ "type": "prometheus", "uid": "$ds" }, - "description": "Amount of used memory\n\nResident memory shows share which can be freed by OS when needed.\n\nAnonymous shows share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.\n\nIf you think that usage is abnormal or unexpected, please file an issue and attach memory profile if possible.", + "description": "Shows the max number of CPU cores used by a `job` and the corresponding limit.", "fieldConfig": { "defaults": { "color": { @@ -1414,7 +1638,7 @@ } ] }, - "unit": "percentunit" + "unit": "short" }, "overrides": [] }, @@ -1422,9 +1646,9 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 41 }, - "id": 37, + "id": 56, "links": [ { "targetBlank": true, @@ -1447,7 +1671,7 @@ }, "tooltip": { "mode": "multi", - "sort": "none" + "sort": "desc" } }, "pluginVersion": "9.2.6", @@ -1459,14 +1683,32 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", + "expr": "max(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", + "format": "time_series", "interval": "", - "legendFormat": "__auto", + "intervalFactor": 1, + "legendFormat": "{{job}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$ds" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}) by(job)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "limit ({{job}})", + "range": true, + "refId": "B" } ], - "title": "Memory usage % ($instance)", + "title": "CPU usage ($instance)", "type": "timeseries" }, { @@ -1535,7 +1777,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 49 }, "id": 39, "links": [], @@ -1641,7 +1883,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 49 }, "id": 41, "links": [], @@ -1754,8 +1996,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1857,8 +2098,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1960,8 +2200,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2064,8 +2303,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2164,8 +2402,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2292,8 +2529,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2395,8 +2631,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2497,8 +2732,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2620,8 +2854,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2713,8 +2946,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2776,9 +3008,9 @@ "list": [ { "current": { - "selected": true, - "text": "VictoriaMetrics", - "value": "VictoriaMetrics" + "selected": false, + "text": "VictoriaMetrics - cluster", + "value": "VictoriaMetrics - cluster" }, "hide": 0, "includeAll": false, @@ -2862,7 +3094,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": "5", "value": "5" }, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 86152d893f..c11ce37a3c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,6 +33,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. +* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627). * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). From 4c854c3ae20006d0040d6d89378f74d2fdb30f33 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 3 Aug 2023 11:48:37 +0200 Subject: [PATCH 35/69] security: bump go version from 1.20.6 to 1.20.7 (#4773) The update includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See the list of issues addressed in Go1.20.7 here: https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved Signed-off-by: hagen1778 --- .github/workflows/check-licenses.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/main.yml | 6 +++--- app/vmui/Dockerfile-web | 2 +- deployment/docker/Makefile | 2 +- docs/CHANGELOG.md | 2 ++ snap/local/Makefile | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-licenses.yml b/.github/workflows/check-licenses.yml index 717dde7712..54f5bd20b6 100644 --- a/.github/workflows/check-licenses.yml +++ b/.github/workflows/check-licenses.yml @@ -17,7 +17,7 @@ jobs: - name: Setup Go uses: actions/setup-go@main with: - go-version: 1.20.6 + go-version: 1.20.7 id: go - name: Code checkout uses: actions/checkout@master diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c7c4ebf267..9c9c0fda99 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -57,7 +57,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.6 + go-version: 1.20.7 check-latest: true cache: true if: ${{ matrix.language == 'go' }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4bbee329f9..d93517873e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: 1.20.6 + go-version: 1.20.7 check-latest: true cache: true @@ -56,7 +56,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: 1.20.6 + go-version: 1.20.7 check-latest: true cache: true @@ -81,7 +81,7 @@ jobs: id: go uses: actions/setup-go@v4 with: - go-version: 1.20.6 + go-version: 1.20.7 check-latest: true cache: true diff --git a/app/vmui/Dockerfile-web b/app/vmui/Dockerfile-web index 37e38d440e..2f77aae415 100644 --- a/app/vmui/Dockerfile-web +++ b/app/vmui/Dockerfile-web @@ -1,4 +1,4 @@ -FROM golang:1.20.6 as build-web-stage +FROM golang:1.20.7 as build-web-stage COPY build /build WORKDIR /build diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 6029bf6fcf..b8a377f71f 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -9,7 +9,7 @@ ROOT_IMAGE ?= alpine:3.18.2 # TODO: sync it with ROOT_IMAGE when it will be fixed in the new alpine releases CERTS_IMAGE := alpine:3.17.3 -GO_BUILDER_IMAGE := golang:1.20.6-alpine +GO_BUILDER_IMAGE := golang:1.20.7-alpine BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) DOCKER_BUILD ?= docker build diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c11ce37a3c..7c681a14c0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,8 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip +* SECURITY: upgrade Go builder from Go1.20.6 to Go1.20.7. The update includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See [the list of issues addressed in Go1.20.7](https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved). + * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow configuring deadline for a backend to be excluded from the rotation on errors via `-failTimeout` cmd-line flag. This feature could be useful when it is expected for backends to be not available for significant periods of time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415) for details. Thanks to @SunKyu for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4416). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead. diff --git a/snap/local/Makefile b/snap/local/Makefile index 9ad685cf08..50c0f40744 100644 --- a/snap/local/Makefile +++ b/snap/local/Makefile @@ -1,4 +1,4 @@ -GO_VERSION ?=1.20.6 +GO_VERSION ?=1.20.7 SNAP_BUILDER_IMAGE := local/snap-builder:2.0.0-$(shell echo $(GO_VERSION) | tr :/ __) From 327f63e408a90117d382b0e251e27b3d2ee6675f Mon Sep 17 00:00:00 2001 From: Karan Sharma Date: Thu, 3 Aug 2023 16:26:44 +0530 Subject: [PATCH 36/69] fix: remove healthcheck in vector docker-compose (#4772) --- .../docker/victorialogs/vector-docker/docker-compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deployment/docker/victorialogs/vector-docker/docker-compose.yml b/deployment/docker/victorialogs/vector-docker/docker-compose.yml index cf63b2c55e..39b636093a 100644 --- a/deployment/docker/victorialogs/vector-docker/docker-compose.yml +++ b/deployment/docker/victorialogs/vector-docker/docker-compose.yml @@ -20,11 +20,6 @@ services: condition: service_healthy victoriametrics: condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8686/health"] - interval: 1s - timeout: 1s - retries: 10 # Run `make package-victoria-logs` to build victoria-logs image victorialogs: From aef31f201a85e993dcae448a52927868f0fad9ea Mon Sep 17 00:00:00 2001 From: Alexander Marshalov <_@marshalov.org> Date: Thu, 3 Aug 2023 14:25:11 +0200 Subject: [PATCH 37/69] add info about `remoteWrite.sendTimeout` default value (#4776) Signed-off-by: Alexander Marshalov <_@marshalov.org> --- app/vmagent/README.md | 2 +- app/vmagent/remotewrite/client.go | 2 +- docs/vmagent.md | 2 +- docs/vmauth.md | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 4009082050..775efe3525 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -1571,7 +1571,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics Supports array of values separated by comma or specified via multiple flags. -remoteWrite.sendTimeout array - Timeout for sending a single block of data to the corresponding -remoteWrite.url + Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m) Supports array of values separated by comma or specified via multiple flags. -remoteWrite.shardByURL Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index 36add0c3f6..6340da4635 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -29,7 +29,7 @@ var ( rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+ "By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+ "is sent after temporary unavailability of the remote storage") - sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url") + sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)") proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+ "Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234") diff --git a/docs/vmagent.md b/docs/vmagent.md index 585ada0195..41d14d650b 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1582,7 +1582,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics Supports array of values separated by comma or specified via multiple flags. -remoteWrite.sendTimeout array - Timeout for sending a single block of data to the corresponding -remoteWrite.url + Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m) Supports array of values separated by comma or specified via multiple flags. -remoteWrite.shardByURL Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages diff --git a/docs/vmauth.md b/docs/vmauth.md index c7965aedb1..cf198945fd 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -362,6 +362,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Prefix for environment variables if -envflag.enable is set -eula By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -failTimeout duration + Sets a delay period for load balancing to skip a malfunctioning backend. (defaults 3s) -flagsAuthKey string Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -fs.disableMmap From d890038a941eb6f67453871dbccf3204f972c4b8 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 3 Aug 2023 16:22:50 +0200 Subject: [PATCH 38/69] dashboards: correctly calculate `Bytes per point` value Correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected. Signed-off-by: hagen1778 --- dashboards/victoriametrics-cluster.json | 2 +- dashboards/victoriametrics.json | 2 +- docs/CHANGELOG.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dashboards/victoriametrics-cluster.json b/dashboards/victoriametrics-cluster.json index d4bdce4abe..aae1bbba2f 100644 --- a/dashboards/victoriametrics-cluster.json +++ b/dashboards/victoriametrics-cluster.json @@ -624,7 +624,7 @@ "uid": "$ds" }, "exemplar": true, - "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\"})", + "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", "format": "time_series", "instant": true, "interval": "", diff --git a/dashboards/victoriametrics.json b/dashboards/victoriametrics.json index b3d0641c07..69a0d93b58 100644 --- a/dashboards/victoriametrics.json +++ b/dashboards/victoriametrics.json @@ -4434,7 +4434,7 @@ "uid": "$ds" }, "editorMode": "code", - "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\"})", + "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) \n/ sum(vm_rows{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"})", "format": "time_series", "interval": "", "intervalFactor": 1, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7c681a14c0..8f6d6d47fb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -36,6 +36,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627). +* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected. * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). From 22902521196eef5f38afa5474dc782551ceafd78 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Thu, 3 Aug 2023 19:04:19 +0400 Subject: [PATCH 39/69] docs: make phrase about dedup and evaluation interval relation less obscure (#4781) Value of `-dedup.minScrapeInterval` comand-line flag must be higher than `evaluation_interval` in order to make sure that only one sample on each evaluation will be left after deduplication. Moreover, value of `-dedup.minScrapeInterval` must be a multiple of vmalert's `evaluation_interval` in order to make sure that samples will be aligned between deduplication window periods. See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4774#issuecomment-1663940811 Signed-off-by: Zakhar Bessarab --- app/vmalert/README.md | 2 +- docs/vmalert.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 72077ebef5..f37fe962a0 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -519,7 +519,7 @@ Alertmanagers. To avoid recording rules results and alerts state duplication in VictoriaMetrics server don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication). -The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert `evaluation_interval`. +The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `evaluation_interval`. If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment` command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps. But due of backfilling (data delivered to the datasource with some delay) values of such results may differ, diff --git a/docs/vmalert.md b/docs/vmalert.md index d941caf0c4..cef7fb4ca6 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -530,7 +530,7 @@ Alertmanagers. To avoid recording rules results and alerts state duplication in VictoriaMetrics server don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication). -The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert `evaluation_interval`. +The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `evaluation_interval`. If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment` command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps. But due of backfilling (data delivered to the datasource with some delay) values of such results may differ, From fc01167962c5ee4fe0052d3a47fc18e29b24f865 Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Fri, 4 Aug 2023 00:53:47 +0800 Subject: [PATCH 40/69] Automatic update operator docs from VictoriaMetrics/operator@7e12a16 (#4783) --- docs/operator/CHANGELOG.md | 28 +++++++++++++++------------- docs/operator/vars.md | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index db501ef387..b0d8fa5944 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -4,19 +4,21 @@ ### Breaking changes -- **[vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): Field `OAuth2` in (HTTPAuth)[https://docs.victoriametrics.com/operator/api.html#httpauth] was renamed to `oauth2`. See [this issue](https://github.com/VictoriaMetrics/operator/issues/522) and [this PR](https://github.com/VictoriaMetrics/operator/pull/689) for details.** -- **[vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): Field `bearerTokenFilePath` in (BearerAuth)[https://docs.victoriametrics.com/operator/api.html#bearerauth] was renamed to `bearerTokenFile`. See [this issue](https://github.com/VictoriaMetrics/operator/issues/522) and [this PR](https://github.com/VictoriaMetrics/operator/pull/688/) for details.** -- **These changes affect following fields:** - - **`VMAlert.spec.datasource.OAuth2` -> `VMAlert.spec.datasource.oauth2`,** - - **`VMAlert.spec.notifier.OAuth2` -> `VMAlert.spec.notifier.oauth2`,** - - **`VMAlert.spec.notifiers[].OAuth2` -> `VMAlert.spec.notifiers[].oauth2`,** - - **`VMAlert.spec.remoteRead.OAuth2` -> `VMAlert.spec.remoteRead.oauth2`,** - - **`VMAlert.spec.remoteWrite.OAuth2` -> `VMAlert.spec.remoteWrite.oauth2`,** - - **`VMAlert.spec.datasource.bearerTokenFilePath` --> `VMAlert.spec.datasource.bearerTokenFile`,** - - **`VMAlert.spec.notifier.bearerTokenFilePath` --> `VMAlert.spec.notifier.bearerTokenFile`,** - - **`VMAlert.spec.notifiers[].bearerTokenFile` --> `VMAlert.spec.notifiers[].bearerTokenFile`,** - - **`VMAlert.spec.remoteRead.bearerTokenFilePath` --> `VMAlert.spec.remoteRead.bearerTokenFile`,** - - **`VMAlert.spec.remoteWrite.bearerTokenFilePath` --> `VMAlert.spec.remoteWrite.bearerTokenFile`.** +- **[vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): Field `OAuth2` was renamed to `oauth2` due to compatibility issue. If you defined `OAuth2` with below fields in vmalert objects using operator before v0.36.0, these fields must be reapplied with new tag `oauth2` after upgrading. See [this issue](https://github.com/VictoriaMetrics/operator/issues/522) and [this PR](https://github.com/VictoriaMetrics/operator/pull/689) for details.** + - **Affected fields:** + - **`VMAlert.spec.datasource.OAuth2` -> `VMAlert.spec.datasource.oauth2`,** + - **`VMAlert.spec.notifier.OAuth2` -> `VMAlert.spec.notifier.oauth2`,** + - **`VMAlert.spec.notifiers[].OAuth2` -> `VMAlert.spec.notifiers[].oauth2`,** + - **`VMAlert.spec.remoteRead.OAuth2` -> `VMAlert.spec.remoteRead.oauth2`,** + - **`VMAlert.spec.remoteWrite.OAuth2` -> `VMAlert.spec.remoteWrite.oauth2`,** + +- **[vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): Field `bearerTokenFilePath` was renamed to `bearerTokenFile` due to compatibility issue. If you defined `bearerTokenFilePath` with below fields in vmalert objects using operator before v0.36.0, these fields must be reapplied with new tag `bearerTokenFile` after upgrading. See [this issue](https://github.com/VictoriaMetrics/operator/issues/522) and [this PR](https://github.com/VictoriaMetrics/operator/pull/688/) for details.** + - **Affected fields:** + - **`VMAlert.spec.datasource.bearerTokenFilePath` --> `VMAlert.spec.datasource.bearerTokenFile`,** + - **`VMAlert.spec.notifier.bearerTokenFilePath` --> `VMAlert.spec.notifier.bearerTokenFile`,** + - **`VMAlert.spec.notifiers[].bearerTokenFile` --> `VMAlert.spec.notifiers[].bearerTokenFile`,** + - **`VMAlert.spec.remoteRead.bearerTokenFilePath` --> `VMAlert.spec.remoteRead.bearerTokenFile`,** + - **`VMAlert.spec.remoteWrite.bearerTokenFilePath` --> `VMAlert.spec.remoteWrite.bearerTokenFile`.** ### Fixes diff --git a/docs/operator/vars.md b/docs/operator/vars.md index be4340deec..9226bbbc67 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ aliases: - /operator/vars.html --- # Auto Generated vars for package config - updated at Thu Aug 3 03:30:07 UTC 2023 + updated at Thu Aug 3 16:52:44 UTC 2023 | varible name | variable default value | variable required | variable description | From 527cfa2f742e10e8fc9dc60b179fcc1a638e77b7 Mon Sep 17 00:00:00 2001 From: Haleygo Date: Mon, 7 Aug 2023 20:24:56 +0800 Subject: [PATCH 41/69] vmalert: fix uncleaned tmp files in tests (#4788) --- app/vmalert/main_test.go | 1 + app/vmalert/notifier/config_watcher_test.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go index ace1118514..ea9be43103 100644 --- a/app/vmalert/main_test.go +++ b/app/vmalert/main_test.go @@ -87,6 +87,7 @@ groups: ) f, err := os.CreateTemp("", "") + defer os.Remove(f.Name()) if err != nil { t.Fatal(err) } diff --git a/app/vmalert/notifier/config_watcher_test.go b/app/vmalert/notifier/config_watcher_test.go index b51cd3bfa2..921f2c3da1 100644 --- a/app/vmalert/notifier/config_watcher_test.go +++ b/app/vmalert/notifier/config_watcher_test.go @@ -14,6 +14,7 @@ import ( func TestConfigWatcherReload(t *testing.T) { f, err := os.CreateTemp("", "") + defer os.Remove(f.Name()) if err != nil { t.Fatal(err) } @@ -36,6 +37,7 @@ static_configs: } f2, err := os.CreateTemp("", "") + defer os.Remove(f2.Name()) if err != nil { t.Fatal(err) } @@ -63,6 +65,7 @@ func TestConfigWatcherStart(t *testing.T) { defer consulSDServer.Close() consulSDFile, err := os.CreateTemp("", "") + defer os.Remove(consulSDFile.Name()) if err != nil { t.Fatal(err) } @@ -109,6 +112,7 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) { defer consulSDServer2.Close() consulSDFile, err := os.CreateTemp("", "") + defer os.Remove(consulSDFile.Name()) if err != nil { t.Fatal(err) } @@ -125,6 +129,7 @@ consul_sd_configs: `, consulSDServer1.URL, consulSDServer2.URL)) staticAndConsulSDFile, err := os.CreateTemp("", "") + defer os.Remove(staticAndConsulSDFile.Name()) if err != nil { t.Fatal(err) } From 1d4a0796f497a9bf3a3d2b29739ac04119d20a47 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 7 Aug 2023 21:58:40 +0200 Subject: [PATCH 42/69] vmalert: cleanup config reload metrics handling (#4790) * rename `configErr` to `lastConfigErr` to reduce confusion * add tests to verify metrics and msg are set properly * fix mistake when config success metric wasn't restored after an error Signed-off-by: hagen1778 --- app/vmalert/main.go | 55 ++++++++++++++++++++-------------------- app/vmalert/main_test.go | 25 ++++++++++++++++++ app/vmalert/web.qtpl | 12 ++++----- app/vmalert/web.qtpl.go | 12 ++++----- docs/CHANGELOG.md | 1 + 5 files changed, 65 insertions(+), 40 deletions(-) diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 5a15eff229..4e81ce8383 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -319,8 +319,9 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig validateTplFn = notifier.ValidateTemplates } - // init reload metrics with positive values to improve alerting conditions - setConfigSuccess(fasttime.UnixTimestamp()) + // init metrics for config state with positive values to improve alerting conditions + setConfigSuccessAt(fasttime.UnixTimestamp()) + parseFn := config.Parse for { select { @@ -358,11 +359,12 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig } if configsEqual(newGroupsCfg, groupsCfg) { templates.Reload() - // set success to 1 since previous reload - // could have been unsuccessful + // set success to 1 since previous reload could have been unsuccessful + // do not update configTimestamp as config version remains old. configSuccess.Set(1) - setConfigError(nil) - // config didn't change - skip it + // reset the last config error since the config change was rolled back + setLastConfigErr(nil) + // config didn't change - skip iteration continue } if err := m.update(ctx, newGroupsCfg, false); err != nil { @@ -372,7 +374,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig } templates.Reload() groupsCfg = newGroupsCfg - setConfigSuccess(fasttime.UnixTimestamp()) + setConfigSuccessAt(fasttime.UnixTimestamp()) logger.Infof("Rules reloaded successfully from %q", *rulePath) } } @@ -389,39 +391,36 @@ func configsEqual(a, b []config.Group) bool { return true } -// setConfigSuccess sets config reload status to 1. -func setConfigSuccess(at uint64) { +// setConfigSuccessAt updates config related metrics as successful. +func setConfigSuccessAt(at uint64) { configSuccess.Set(1) configTimestamp.Set(at) - // reset the error if any - setConfigErr(nil) + // reset the lastConfigErr + setLastConfigErr(nil) } -// setConfigError sets config reload status to 0. +// setConfigError updates config related metrics according to the error. func setConfigError(err error) { configReloadErrors.Inc() configSuccess.Set(0) - setConfigErr(err) + setLastConfigErr(err) } var ( - configErrMu sync.RWMutex - // configErr represent the error message from the last - // config reload. - configErr error + lastConfigErrMu sync.RWMutex + // lastConfigErr represent the error message from the last config reload. + // The message is used in web UI as notification + lastConfigErr error ) -func setConfigErr(err error) { - configErrMu.Lock() - configErr = err - configErrMu.Unlock() +func setLastConfigErr(err error) { + lastConfigErrMu.Lock() + lastConfigErr = err + lastConfigErrMu.Unlock() } -func configError() error { - configErrMu.RLock() - defer configErrMu.RUnlock() - if configErr != nil { - return configErr - } - return nil +func getLastConfigError() error { + lastConfigErrMu.RLock() + defer lastConfigErrMu.RUnlock() + return lastConfigErr } diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go index ea9be43103..2c7e0254da 100644 --- a/app/vmalert/main_test.go +++ b/app/vmalert/main_test.go @@ -118,7 +118,29 @@ groups: return len(m.groups) } + checkCfg := func(err error) { + cErr := getLastConfigError() + cfgSuc := configSuccess.Get() + if err != nil { + if cErr == nil { + t.Fatalf("expected to have config error %s; got nil instead", cErr) + } + if cfgSuc != 0 { + t.Fatalf("expected to have metric configSuccess to be set to 0; got %d instead", cfgSuc) + } + return + } + + if cErr != nil { + t.Fatalf("unexpected config error: %s", cErr) + } + if cfgSuc != 1 { + t.Fatalf("expected to have metric configSuccess to be set to 1; got %d instead", cfgSuc) + } + } + time.Sleep(*configCheckInterval * 2) + checkCfg(nil) groupsLen := lenLocked(m) if groupsLen != 1 { t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen) @@ -126,6 +148,7 @@ groups: writeToFile(t, f.Name(), rules2) time.Sleep(*configCheckInterval * 2) + checkCfg(nil) groupsLen = lenLocked(m) if groupsLen != 2 { fmt.Println(m.groups) @@ -135,6 +158,7 @@ groups: writeToFile(t, f.Name(), rules1) procutil.SelfSIGHUP() time.Sleep(*configCheckInterval / 2) + checkCfg(nil) groupsLen = lenLocked(m) if groupsLen != 1 { t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen) @@ -143,6 +167,7 @@ groups: writeToFile(t, f.Name(), `corrupted`) procutil.SelfSIGHUP() time.Sleep(*configCheckInterval / 2) + checkCfg(fmt.Errorf("config error")) groupsLen = lenLocked(m) if groupsLen != 1 { // should remain unchanged t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen) diff --git a/app/vmalert/web.qtpl b/app/vmalert/web.qtpl index 2cbf0a2ce7..425ea6ea5f 100644 --- a/app/vmalert/web.qtpl +++ b/app/vmalert/web.qtpl @@ -12,7 +12,7 @@ {% func Welcome(r *http.Request) %} - {%= tpl.Header(r, navItems, "vmalert", configError()) %} + {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}

API:
{% for _, p := range apiLinks %} @@ -40,7 +40,7 @@ btn-primary {% func ListGroups(r *http.Request, originGroups []APIGroup) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "Groups", configError()) %} + {%= tpl.Header(r, navItems, "Groups", getLastConfigError()) %} {%code filter := r.URL.Query().Get("filter") rOk := make(map[string]int) @@ -168,7 +168,7 @@ btn-primary {% func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "Alerts", configError()) %} + {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %} {% if len(groupAlerts) > 0 %} Collapse All Expand All @@ -255,7 +255,7 @@ btn-primary {% endfunc %} {% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %} - {%= tpl.Header(r, navItems, "Notifiers", configError()) %} + {%= tpl.Header(r, navItems, "Notifiers", getLastConfigError()) %} {% if len(targets) > 0 %} Collapse All Expand All @@ -312,7 +312,7 @@ btn-primary {% func Alert(r *http.Request, alert *APIAlert) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "", configError()) %} + {%= tpl.Header(r, navItems, "", getLastConfigError()) %} {%code var labelKeys []string for k := range alert.Labels { @@ -399,7 +399,7 @@ btn-primary {% func RuleDetails(r *http.Request, rule APIRule) %} {%code prefix := utils.Prefix(r.URL.Path) %} - {%= tpl.Header(r, navItems, "", configError()) %} + {%= tpl.Header(r, navItems, "", getLastConfigError()) %} {%code var labelKeys []string for k := range rule.Labels { diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go index 0fa83c873c..bb95cc89b6 100644 --- a/app/vmalert/web.qtpl.go +++ b/app/vmalert/web.qtpl.go @@ -34,7 +34,7 @@ func StreamWelcome(qw422016 *qt422016.Writer, r *http.Request) { qw422016.N().S(` `) //line app/vmalert/web.qtpl:15 - tpl.StreamHeader(qw422016, r, navItems, "vmalert", configError()) + tpl.StreamHeader(qw422016, r, navItems, "vmalert", getLastConfigError()) //line app/vmalert/web.qtpl:15 qw422016.N().S(`

@@ -207,7 +207,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, originGroups [ qw422016.N().S(` `) //line app/vmalert/web.qtpl:43 - tpl.StreamHeader(qw422016, r, navItems, "Groups", configError()) + tpl.StreamHeader(qw422016, r, navItems, "Groups", getLastConfigError()) //line app/vmalert/web.qtpl:43 qw422016.N().S(` `) @@ -647,7 +647,7 @@ func StreamListAlerts(qw422016 *qt422016.Writer, r *http.Request, groupAlerts [] qw422016.N().S(` `) //line app/vmalert/web.qtpl:171 - tpl.StreamHeader(qw422016, r, navItems, "Alerts", configError()) + tpl.StreamHeader(qw422016, r, navItems, "Alerts", getLastConfigError()) //line app/vmalert/web.qtpl:171 qw422016.N().S(` `) @@ -922,7 +922,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n qw422016.N().S(` `) //line app/vmalert/web.qtpl:258 - tpl.StreamHeader(qw422016, r, navItems, "Notifiers", configError()) + tpl.StreamHeader(qw422016, r, navItems, "Notifiers", getLastConfigError()) //line app/vmalert/web.qtpl:258 qw422016.N().S(` `) @@ -1102,7 +1102,7 @@ func StreamAlert(qw422016 *qt422016.Writer, r *http.Request, alert *APIAlert) { qw422016.N().S(` `) //line app/vmalert/web.qtpl:315 - tpl.StreamHeader(qw422016, r, navItems, "", configError()) + tpl.StreamHeader(qw422016, r, navItems, "", getLastConfigError()) //line app/vmalert/web.qtpl:315 qw422016.N().S(` `) @@ -1311,7 +1311,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule) qw422016.N().S(` `) //line app/vmalert/web.qtpl:402 - tpl.StreamHeader(qw422016, r, navItems, "", configError()) + tpl.StreamHeader(qw422016, r, navItems, "", getLastConfigError()) //line app/vmalert/web.qtpl:402 qw422016.N().S(` `) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8f6d6d47fb..f05553ee89 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -40,6 +40,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543). * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). From 262c888d4bf6440423396997ed3bcc4384697a18 Mon Sep 17 00:00:00 2001 From: Haleygo Date: Thu, 10 Aug 2023 03:16:00 +0800 Subject: [PATCH 43/69] vmalert: fix redundant clean up move (#4803) Follow-up after https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4788/commits/55ae2c2d57b8f298b38edea346963492b0449396 --- app/vmalert/main_test.go | 2 +- app/vmalert/notifier/config_watcher_test.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/app/vmalert/main_test.go b/app/vmalert/main_test.go index 2c7e0254da..afa27bace3 100644 --- a/app/vmalert/main_test.go +++ b/app/vmalert/main_test.go @@ -87,10 +87,10 @@ groups: ) f, err := os.CreateTemp("", "") - defer os.Remove(f.Name()) if err != nil { t.Fatal(err) } + defer func() { _ = os.Remove(f.Name()) }() writeToFile(t, f.Name(), rules1) *configCheckInterval = 200 * time.Millisecond diff --git a/app/vmalert/notifier/config_watcher_test.go b/app/vmalert/notifier/config_watcher_test.go index 921f2c3da1..b51cd3bfa2 100644 --- a/app/vmalert/notifier/config_watcher_test.go +++ b/app/vmalert/notifier/config_watcher_test.go @@ -14,7 +14,6 @@ import ( func TestConfigWatcherReload(t *testing.T) { f, err := os.CreateTemp("", "") - defer os.Remove(f.Name()) if err != nil { t.Fatal(err) } @@ -37,7 +36,6 @@ static_configs: } f2, err := os.CreateTemp("", "") - defer os.Remove(f2.Name()) if err != nil { t.Fatal(err) } @@ -65,7 +63,6 @@ func TestConfigWatcherStart(t *testing.T) { defer consulSDServer.Close() consulSDFile, err := os.CreateTemp("", "") - defer os.Remove(consulSDFile.Name()) if err != nil { t.Fatal(err) } @@ -112,7 +109,6 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) { defer consulSDServer2.Close() consulSDFile, err := os.CreateTemp("", "") - defer os.Remove(consulSDFile.Name()) if err != nil { t.Fatal(err) } @@ -129,7 +125,6 @@ consul_sd_configs: `, consulSDServer1.URL, consulSDServer2.URL)) staticAndConsulSDFile, err := os.CreateTemp("", "") - defer os.Remove(staticAndConsulSDFile.Name()) if err != nil { t.Fatal(err) } From bc5065fd14be0853b1abb6e163fe2234adbda878 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Wed, 9 Aug 2023 22:02:32 +0200 Subject: [PATCH 44/69] vmalert: mention `vmalert_iteration_duration_seconds` metric in README Signed-off-by: hagen1778 --- app/vmalert/README.md | 4 ++-- docs/vmalert.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index f37fe962a0..342ae7a784 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -117,11 +117,11 @@ name: [ limit: | default = 0 ] # How many rules execute at once within a group. Increasing concurrency may speed -# up round execution speed. +# up group's evaluation duration (exposed via `vmalert_iteration_duration_seconds` metric). [ concurrency: | default = 1 ] # Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus". -# By default "prometheus" type is used. +# By default, "prometheus" type is used. [ type: ] # Optional list of HTTP URL parameters diff --git a/docs/vmalert.md b/docs/vmalert.md index cef7fb4ca6..ff47ec2603 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -128,11 +128,11 @@ name: [ limit: | default = 0 ] # How many rules execute at once within a group. Increasing concurrency may speed -# up round execution speed. +# up group's evaluation duration (exposed via `vmalert_iteration_duration_seconds` metric). [ concurrency: | default = 1 ] # Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus". -# By default "prometheus" type is used. +# By default, "prometheus" type is used. [ type: ] # Optional list of HTTP URL parameters From 835c03fb471dc2c2043e5a72e4dc3beb518fa43a Mon Sep 17 00:00:00 2001 From: Abirdcfly Date: Thu, 10 Aug 2023 10:51:44 +0800 Subject: [PATCH 45/69] vmalert: fix `vmalert_remotewrite_send_duration_seconds_total` metric value (#4801) The deferred call's arguments are evaluated immediately, but the function call is not executed until the surrounding function returns. Signed-off-by: Abirdcfly --- app/vmalert/remotewrite/remotewrite.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go index 8d4f935c10..f217bd78b6 100644 --- a/app/vmalert/remotewrite/remotewrite.go +++ b/app/vmalert/remotewrite/remotewrite.go @@ -216,7 +216,9 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) { retryInterval = maxRetryInterval } timeStart := time.Now() - defer sendDuration.Add(time.Since(timeStart).Seconds()) + defer func() { + sendDuration.Add(time.Since(timeStart).Seconds()) + }() L: for attempts := 0; ; attempts++ { err := c.send(ctx, b) From bd8ecfb5518936687c8f24cbc82a72069e50ea6b Mon Sep 17 00:00:00 2001 From: Haleygo Date: Thu, 10 Aug 2023 14:26:55 +0800 Subject: [PATCH 46/69] docs: add changelog for 4c815ed59b2996b57b0aedb42f2e2e91314ae8b1 (#4805) --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f05553ee89..70a06e3aa1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -41,6 +41,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543). +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). From 0a93abfeed5a926736ae9431714f7d6e9fee887f Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Thu, 10 Aug 2023 16:38:19 +0800 Subject: [PATCH 47/69] Automatic update operator docs from VictoriaMetrics/operator@ca27728 (#4802) --- docs/operator/CHANGELOG.md | 4 ++++ docs/operator/api.md | 8 +++++--- docs/operator/vars.md | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index b0d8fa5944..7aa19622b3 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -31,6 +31,10 @@ ### Features - [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): add [example config](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmagent_stateful_with_sharding.yaml) for vmagent statefulmode. +- [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent)/[vmsingle](https://docs.victoriametrics.com/operator/api.html#vmsingle): adapt new features in streaming aggregation: + - support `streamAggr.dropInput`, see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4243) for details; + - support list for `match` parameter, see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4635) for details; + - support `staleness_interval`, see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4667) for details. - [vmcluster](https://docs.victoriametrics.com/operator/api.html#vmagent): add [example config](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmcluster_with_additional_claim.yaml) for cluster with custom storage claims. - [vmrule](https://docs.victoriametrics.com/operator/api.html#vmrule): support `update_entries_limit` field in rules, refer to [alerting rules](https://docs.victoriametrics.com/vmalert.html#alerting-rules). See [this PR](https://github.com/VictoriaMetrics/operator/pull/691) for details. - [vmrule](https://docs.victoriametrics.com/operator/api.html#vmrule): support `keep_firing_for` field in rules, refer to [alerting rules](https://docs.victoriametrics.com/vmalert.html#alerting-rules). See [this PR](https://github.com/VictoriaMetrics/operator/pull/711) for details. diff --git a/docs/operator/api.md b/docs/operator/api.md index f592a2362a..813726aca3 100644 --- a/docs/operator/api.md +++ b/docs/operator/api.md @@ -810,7 +810,7 @@ VMAgentSpec defines the desired state of VMAgent ## VMAgentStatus -VMAgentStatus defines the observed state of VmAgent +VMAgentStatus defines the observed state of VMAgent | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | @@ -985,6 +985,7 @@ StreamAggrConfig defines the stream aggregation config | ----- | ----------- | ------ | -------- | | rules | Stream aggregation rules | [][StreamAggrRule](#streamaggrrule) | true | | keepInput | Allows writing both raw and aggregate data | bool | false | +| dropInput | Allow drop all the input samples after the aggregation | bool | false | | dedupInterval | Allows setting different de-duplication intervals per each configured remote storage | string | false | [Back to TOC](#table-of-contents) @@ -995,8 +996,9 @@ StreamAggrRule defines the rule in stream aggregation config | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| match | Match is a label selector for filtering time series for the given selector.\n\nIf the match isn't set, then all the input time series are processed. | string | false | +| match | Match is a label selector for filtering time series for the given selector.\n\nIf the match isn't set, then all the input time series are processed. | Match | false | | interval | Interval is the interval between aggregations. | string | true | +| staleness_interval | StalenessInterval defines an interval after which the series state will be reset if no samples have been sent during it. | string | false | | outputs | Outputs is a list of output aggregate functions to produce.\n\nThe following names are allowed:\n\n- total - aggregates input counters - increase - counts the increase over input counters - count_series - counts the input series - count_samples - counts the input samples - sum_samples - sums the input samples - last - the last biggest sample value - min - the minimum sample value - max - the maximum sample value - avg - the average value across all the samples - stddev - standard deviation across all the samples - stdvar - standard variance across all the samples - histogram_bucket - creates VictoriaMetrics histogram for input samples - quantiles(phi1, ..., phiN) - quantiles' estimation for phi in the range [0..1]\n\nThe output time series will have the following names:\n\n input_name:aggr_<interval>_<output> | []string | true | | by | By is an optional list of labels for grouping input series.\n\nSee also Without.\n\nIf neither By nor Without are set, then the Outputs are calculated individually per each input time series. | []string | false | | without | Without is an optional list of labels, which must be excluded when grouping input series.\n\nSee also By.\n\nIf neither By nor Without are set, then the Outputs are calculated individually per each input time series. | []string | false | @@ -1161,7 +1163,7 @@ VMAlertSpec defines the desired state of VMAlert ## VMAlertStatus -VMAlertStatus defines the observed state of VmAlert +VMAlertStatus defines the observed state of VMAlert | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | diff --git a/docs/operator/vars.md b/docs/operator/vars.md index 9226bbbc67..006fdf525e 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ aliases: - /operator/vars.html --- # Auto Generated vars for package config - updated at Thu Aug 3 16:52:44 UTC 2023 + updated at Wed Aug 9 14:55:29 UTC 2023 | varible name | variable default value | variable required | variable description | From cc7bfaca6c8d4039aa3e633a5cbfe8b61c515ec6 Mon Sep 17 00:00:00 2001 From: Yury Molodov Date: Thu, 10 Aug 2023 11:34:25 +0200 Subject: [PATCH 48/69] vmui: allow displaying the full error message on click (#4760) https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4719 --- .../DateTimeInput/DateTimeInput.tsx | 2 +- .../components/Main/TextField/TextField.tsx | 8 +-- .../Main/TextField/TextFieldError.tsx | 56 +++++++++++++++++++ .../src/components/Main/TextField/style.scss | 27 +++++---- .../src/components/Main/Tooltip/Tooltip.tsx | 2 +- app/vmui/packages/vmui/src/styles/core.scss | 16 +++--- docs/CHANGELOG.md | 1 + 7 files changed, 86 insertions(+), 26 deletions(-) create mode 100644 app/vmui/packages/vmui/src/components/Main/TextField/TextFieldError.tsx diff --git a/app/vmui/packages/vmui/src/components/Main/DatePicker/DateTimeInput/DateTimeInput.tsx b/app/vmui/packages/vmui/src/components/Main/DatePicker/DateTimeInput/DateTimeInput.tsx index 6299c01442..7986e582fc 100644 --- a/app/vmui/packages/vmui/src/components/Main/DatePicker/DateTimeInput/DateTimeInput.tsx +++ b/app/vmui/packages/vmui/src/components/Main/DatePicker/DateTimeInput/DateTimeInput.tsx @@ -36,7 +36,7 @@ const DateTimeInput: FC = ({ const [maskedValue, setMaskedValue] = useState(formatStringDate(value)); const [focusToTime, setFocusToTime] = useState(false); const [awaitChangeForEnter, setAwaitChangeForEnter] = useState(false); - const error = dayjs(maskedValue).isValid() ? "" : "Expected format: YYYY-MM-DD HH:mm:ss"; + const error = dayjs(maskedValue).isValid() ? "" : "Invalid date format"; const handleMaskedChange = (e: ChangeEvent) => { setMaskedValue(e.currentTarget.value); diff --git a/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx b/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx index 935b2ace01..2514bfb9da 100644 --- a/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx +++ b/app/vmui/packages/vmui/src/components/Main/TextField/TextField.tsx @@ -3,6 +3,7 @@ import classNames from "classnames"; import { useMemo } from "preact/compat"; import { useAppState } from "../../../state/common/StateContext"; import useDeviceDetect from "../../../hooks/useDeviceDetect"; +import TextFieldError from "./TextFieldError"; import "./style.scss"; interface TextFieldProps { @@ -132,12 +133,7 @@ const TextField: FC = ({ ) } {label && {label}} - - {error} - + {helperText && !error && ( {helperText} diff --git a/app/vmui/packages/vmui/src/components/Main/TextField/TextFieldError.tsx b/app/vmui/packages/vmui/src/components/Main/TextField/TextFieldError.tsx new file mode 100644 index 0000000000..3d3b1a95ac --- /dev/null +++ b/app/vmui/packages/vmui/src/components/Main/TextField/TextFieldError.tsx @@ -0,0 +1,56 @@ +import React, { FC, useEffect, useRef, useState } from "react"; +import useEventListener from "../../../hooks/useEventListener"; +import classNames from "classnames"; +import "./style.scss"; + +interface TextFieldErrorProps { + error: string; +} + +const TextFieldError: FC = ({ error }) => { + const errorRef = useRef(null); + const [isErrorTruncated, setIsErrorTruncated] = useState(false); + const [showFull, setShowFull] = useState(false); + + const checkIfTextTruncated = () => { + const el = errorRef.current; + if (el) { + const { offsetWidth, scrollWidth, offsetHeight, scrollHeight } = el; + // The "+1" is for the scrollbar in Firefox + const overflowed = (offsetWidth + 1) < scrollWidth || (offsetHeight + 1) < scrollHeight; + setIsErrorTruncated(overflowed); + } else { + setIsErrorTruncated(false); + } + }; + + const handleClickError = () => { + if (!isErrorTruncated) return; + setShowFull(true); + setIsErrorTruncated(false); + }; + + useEffect(() => { + setShowFull(false); + checkIfTextTruncated(); + }, [errorRef, error]); + + useEventListener("resize", checkIfTextTruncated); + + return ( + + {error} + + ); +}; + +export default TextFieldError; diff --git a/app/vmui/packages/vmui/src/components/Main/TextField/style.scss b/app/vmui/packages/vmui/src/components/Main/TextField/style.scss index 70b4636451..44a854bbfe 100644 --- a/app/vmui/packages/vmui/src/components/Main/TextField/style.scss +++ b/app/vmui/packages/vmui/src/components/Main/TextField/style.scss @@ -40,14 +40,9 @@ overflow: hidden; text-overflow: ellipsis; display: -webkit-box; - -webkit-line-clamp: 2; /* number of lines to show */ - line-clamp: 2; + -webkit-line-clamp: 1; /* number of lines to show */ + line-clamp: 1; -webkit-box-orient: vertical; - - @media (max-width: 500px) { - -webkit-line-clamp: 1; /* number of lines to show */ - line-clamp: 1; - } } &__label { @@ -56,10 +51,22 @@ } &__error { - top: calc((100% - ($font-size-small/2)) - 2px); + position: relative; + top: calc($font-size-small/-2); + width: fit-content; + overflow-wrap: anywhere; color: $color-error; pointer-events: auto; user-select: text; + + &_full { + display: block; + overflow: visible; + } + + &_overflowed { + cursor: pointer; + } } &__helper-text { @@ -117,9 +124,9 @@ align-items: center; justify-content: center; max-width: 15px; - top: auto; + top: 0; left: $padding-small; - height: 100%; + height: 40px; position: absolute; color: $color-text-secondary; } diff --git a/app/vmui/packages/vmui/src/components/Main/Tooltip/Tooltip.tsx b/app/vmui/packages/vmui/src/components/Main/Tooltip/Tooltip.tsx index 29aed95375..dad669e0e3 100644 --- a/app/vmui/packages/vmui/src/components/Main/Tooltip/Tooltip.tsx +++ b/app/vmui/packages/vmui/src/components/Main/Tooltip/Tooltip.tsx @@ -41,7 +41,7 @@ const Tooltip: FC = ({ return () => { window.removeEventListener("scroll", onScrollWindow); }; - }, [isOpen]); + }, [isOpen, title]); const popperStyle = useMemo(() => { // eslint-disable-next-line @typescript-eslint/ban-ts-comment diff --git a/app/vmui/packages/vmui/src/styles/core.scss b/app/vmui/packages/vmui/src/styles/core.scss index 5e6be91b6f..4d41f28297 100644 --- a/app/vmui/packages/vmui/src/styles/core.scss +++ b/app/vmui/packages/vmui/src/styles/core.scss @@ -52,6 +52,7 @@ input[type=number]::-webkit-outer-spin-button { left: $padding-global; bottom: $padding-global; z-index: 999; + animation: vm-slide-snackbar 150ms cubic-bezier(0.280, 0.840, 0.420, 1.1); &-content { display: grid; @@ -71,15 +72,14 @@ input[type=number]::-webkit-outer-spin-button { bottom: 0; left: 0; right: 0; - animation: vm-slide-snackbar 150ms cubic-bezier(0.280, 0.840, 0.420, 1.1); + } - @keyframes vm-slide-snackbar { - 0% { - transform: translateY(100%); - } - 100% { - transform: translateY(0); - } + @keyframes vm-slide-snackbar { + 0% { + transform: translateY(100%); + } + 100% { + transform: translateY(0); } } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 70a06e3aa1..ea6aa1d31b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,6 +34,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). +* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): allow displaying the full error message on click for trimmed error messages in vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4719). * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627). * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected. From 86f1459ca65e72416321d86db75bdc0a744779b3 Mon Sep 17 00:00:00 2001 From: Yury Molodov Date: Thu, 10 Aug 2023 11:42:13 +0200 Subject: [PATCH 49/69] vmui: hide "Logs Explorer" for the base build (#4761) LogsExplorer should be a part of VictoriaLogs binaries, as well as VMUI is now part of VictoriaMetrics binaries. --- app/vmui/packages/vmui/src/App.tsx | 5 ----- app/vmui/packages/vmui/src/constants/navigation.ts | 4 ---- 2 files changed, 9 deletions(-) diff --git a/app/vmui/packages/vmui/src/App.tsx b/app/vmui/packages/vmui/src/App.tsx index a5087f34d9..4e3fc2a89f 100644 --- a/app/vmui/packages/vmui/src/App.tsx +++ b/app/vmui/packages/vmui/src/App.tsx @@ -13,7 +13,6 @@ import ExploreMetrics from "./pages/ExploreMetrics"; import PreviewIcons from "./components/Main/Icons/PreviewIcons"; import WithTemplate from "./pages/WithTemplate"; import Relabel from "./pages/Relabel"; -import ExploreLogs from "./pages/ExploreLogs/ExploreLogs"; import ActiveQueries from "./pages/ActiveQueries"; const App: FC = () => { @@ -70,10 +69,6 @@ const App: FC = () => { path={router.icons} element={} /> - } - /> )} diff --git a/app/vmui/packages/vmui/src/constants/navigation.ts b/app/vmui/packages/vmui/src/constants/navigation.ts index 8639c5e694..5974e91d60 100644 --- a/app/vmui/packages/vmui/src/constants/navigation.ts +++ b/app/vmui/packages/vmui/src/constants/navigation.ts @@ -38,10 +38,6 @@ export const defaultNavigation: NavigationItem[] = [ label: routerOptions[router.activeQueries].title, value: router.activeQueries, }, - { - label: routerOptions[router.logs].title, - value: router.logs, - }, ] }, { From 252643d10069efddcd133aa8a58111227caf6034 Mon Sep 17 00:00:00 2001 From: Yury Molodov Date: Thu, 10 Aug 2023 12:27:28 +0200 Subject: [PATCH 50/69] vmui: change the response for active queries (#4782) * fix: change the response to a valid json (#4676) * vmui/docs: fix response of active queries https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676 --- app/vmselect/promql/active_queries.go | 7 ++++++- docs/CHANGELOG.md | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/vmselect/promql/active_queries.go b/app/vmselect/promql/active_queries.go index d66058e53b..2ef261b49b 100644 --- a/app/vmselect/promql/active_queries.go +++ b/app/vmselect/promql/active_queries.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "sort" + "strings" "sync" "sync/atomic" "time" @@ -23,8 +24,12 @@ func ActiveQueriesHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, `{"status":"ok","data":[`) for i, aqe := range aqes { d := now.Sub(aqe.startTime) + addr := aqe.quotedRemoteAddr + if n := strings.IndexByte(aqe.quotedRemoteAddr, ','); n != -1 { + addr = aqe.quotedRemoteAddr[:n] + } fmt.Fprintf(w, `{"duration":"%.3fs","id":"%016X","remote_addr":%s,"query":%q,"start":%d,"end":%d,"step":%d}`, - d.Seconds(), aqe.qid, aqe.quotedRemoteAddr, aqe.q, aqe.start, aqe.end, aqe.step) + d.Seconds(), aqe.qid, addr, aqe.q, aqe.start, aqe.end, aqe.step) if i+1 < len(aqes) { fmt.Fprintf(w, `,`) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ea6aa1d31b..1bed6f8f4c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -44,6 +44,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). +* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the response of active queries to valid JSON. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4782). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From a4a1884237dfe0360397f932856c01d6a6f32098 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Thu, 10 Aug 2023 16:27:21 +0400 Subject: [PATCH 51/69] {vmagent/remotewrite,vminsert/common}: fix dropInput and keepInput flags inconsistency (#4809) {vmagent/remotewrite,vminsert/common}: fix dropInput and keepInput flags inconsistency Sync behavior for dropInput and keepInput flags between single-node and vmagent. Fix vmagent not respecting dropInput flag and reverse logic for keepInput. --- app/vmagent/remotewrite/remotewrite.go | 13 ++++++++----- app/vminsert/common/insert_ctx.go | 2 +- docs/CHANGELOG.md | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index aae13354b8..bb2bccd4ac 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -12,6 +12,8 @@ import ( "github.com/cespare/xxhash/v2" + "github.com/VictoriaMetrics/metrics" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" @@ -27,7 +29,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" - "github.com/VictoriaMetrics/metrics" ) var ( @@ -705,11 +706,13 @@ var matchIdxsPool bytesutil.ByteBufferPool func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries { dst := src[:0] - for i, match := range matchIdxs { - if match == 0 { - continue + if !dropInput { + for i, match := range matchIdxs { + if match == 1 { + continue + } + dst = append(dst, src[i]) } - dst = append(dst, src[i]) } tail := src[len(dst):] _ = prompbmarshal.ResetTimeSeries(tail) diff --git a/app/vminsert/common/insert_ctx.go b/app/vminsert/common/insert_ctx.go index 7e55e0430d..ef4ded1905 100644 --- a/app/vminsert/common/insert_ctx.go +++ b/app/vminsert/common/insert_ctx.go @@ -171,7 +171,7 @@ func (ctx *InsertCtx) dropAggregatedRows(matchIdxs []byte) { src := ctx.mrs if !*streamAggrDropInput { for idx, match := range matchIdxs { - if match != 0 { + if match == 1 { continue } dst = append(dst, src[idx]) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1bed6f8f4c..ce167f9875 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -45,6 +45,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the response of active queries to valid JSON. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4782). +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series when `remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From c36259fca5ae9c8e58e9d6c56512cdcbedd091c3 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Thu, 10 Aug 2023 14:46:39 +0200 Subject: [PATCH 52/69] docs: mention `honor_timestamps` change in changelog https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697 Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ce167f9875..32ae73d72f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,11 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip +**Update note**: starting from this release, VictoriaMetrics single-server and vmagent +set `honor_timestamps: false` by default in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) +if this options isn't set explicitly. The change supposed to significantly improve staleness detection, compression +and query performance when scraping `cadvisor` metrics. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697). + * SECURITY: upgrade Go builder from Go1.20.6 to Go1.20.7. The update includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See [the list of issues addressed in Go1.20.7](https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). From 64111911583f2a91377b0731b351694b3e451b67 Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:33:30 +0800 Subject: [PATCH 53/69] Automatic update operator docs from VictoriaMetrics/operator@2a03bde (#4812) --- docs/operator/CHANGELOG.md | 3 ++- docs/operator/vars.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index 7aa19622b3..b4f7152c02 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -38,7 +38,8 @@ - [vmcluster](https://docs.victoriametrics.com/operator/api.html#vmagent): add [example config](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmcluster_with_additional_claim.yaml) for cluster with custom storage claims. - [vmrule](https://docs.victoriametrics.com/operator/api.html#vmrule): support `update_entries_limit` field in rules, refer to [alerting rules](https://docs.victoriametrics.com/vmalert.html#alerting-rules). See [this PR](https://github.com/VictoriaMetrics/operator/pull/691) for details. - [vmrule](https://docs.victoriametrics.com/operator/api.html#vmrule): support `keep_firing_for` field in rules, refer to [alerting rules](https://docs.victoriametrics.com/vmalert.html#alerting-rules). See [this PR](https://github.com/VictoriaMetrics/operator/pull/711) for details. -- [vmoperator parameters](https://docs.victoriametrics.com/operator/vars.html): Add option `VM_ENABLESTRICTSECURITY` and enable strict security context by default. See [this issue](https://github.com/VictoriaMetrics/operator/issues/637) and [this PR](https://github.com/VictoriaMetrics/operator/pull/692/) for details. +- [vmoperator parameters](https://docs.victoriametrics.com/operator/vars.html): Add option `VM_ENABLESTRICTSECURITY` and enable strict security context by default. See [this issue](https://github.com/VictoriaMetrics/operator/issues/637), [this](https://github.com/VictoriaMetrics/operator/pull/692/) and [this](https://github.com/VictoriaMetrics/operator/pull/712) PR for details. + ## [v0.35.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.35.1) - 12 Jul 2023 diff --git a/docs/operator/vars.md b/docs/operator/vars.md index 006fdf525e..ded4abf0c4 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,7 +10,7 @@ aliases: - /operator/vars.html --- # Auto Generated vars for package config - updated at Wed Aug 9 14:55:29 UTC 2023 + updated at Thu Aug 10 14:32:26 UTC 2023 | varible name | variable default value | variable required | variable description | @@ -127,4 +127,4 @@ aliases: | VM_PODWAITREADYINTERVALCHECK | 5s | false | - | | VM_PODWAITREADYINITDELAY | 10s | false | - | | VM_FORCERESYNCINTERVAL | 60s | false | configures force resync interval for VMAgent, VMAlert, VMAlertmanager and VMAuth. | -| VM_ENABLESTRICTSECURITY | true | false | EnableStrictSecurity will add default `securityContext` to pods and containers created by operatorDefault PodSecurityContext include:1. RunAsNonRoot: true2. RunAsUser/RunAsGroup/FSGroup: 65534'65534' refers to 'nobody' in all the used default images like alpine, busybox.If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext.Default container SecurityContext include:1. AllowPrivilegeEscalation: false2. ReadOnlyRootFilesystem: true | +| VM_ENABLESTRICTSECURITY | true | false | EnableStrictSecurity will add default `securityContext` to pods and containers created by operatorDefault PodSecurityContext include:1. RunAsNonRoot: true2. RunAsUser/RunAsGroup/FSGroup: 65534'65534' refers to 'nobody' in all the used default images like alpine, busybox.If you're using customize image, please make sure '65534' is a valid uid in there or specify SecurityContext.3. FSGroupChangePolicy: &onRootMismatchIf KubeVersion>=1.20, use `FSGroupChangePolicy="onRootMismatch"` to skip the recursive permission changewhen the root of the volume already has the correct permissions4. SeccompProfile:type: RuntimeDefaultUse `RuntimeDefault` seccomp profile by default, which is defined by the container runtime,instead of using the Unconfined (seccomp disabled) mode.Default container SecurityContext include:1. AllowPrivilegeEscalation: false2. ReadOnlyRootFilesystem: true3. Capabilities:drop:- all | From 7602b95bcbe966352a4c0d73d6dfca5224696f8e Mon Sep 17 00:00:00 2001 From: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:36:10 +0800 Subject: [PATCH 54/69] Automatic update operator docs from VictoriaMetrics/operator@9ea59e2 (#4813) Co-authored-by: Alexander Marshalov <_@marshalov.org> --- docs/operator/CHANGELOG.md | 67 +++++++++++++++++++------------------- docs/operator/vars.md | 3 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/operator/CHANGELOG.md b/docs/operator/CHANGELOG.md index b4f7152c02..1df8d24b97 100644 --- a/docs/operator/CHANGELOG.md +++ b/docs/operator/CHANGELOG.md @@ -24,6 +24,7 @@ - operator set resource requests for config-reloader container by default. See [this PR](https://github.com/VictoriaMetrics/operator/pull/695/) for details. - fix `attachMetadata` value miscovert for scrape objects. See [this issue](https://github.com/VictoriaMetrics/operator/issues/697) and [this PR](https://github.com/VictoriaMetrics/operator/pull/698) for details. +- fix volumeClaimTemplates change check for objects that generate statefulset, like vmstorage, vmselect. Before, the statefulset won't be recreated if additional `claimTemplates` object changed. See [this issue](https://github.com/VictoriaMetrics/operator/issues/507) and [this PR](https://github.com/VictoriaMetrics/operator/pull/719) for details. - [vmalert](https://docs.victoriametrics.com/operator/api.html#vmalert): fix `tlsCAFile` argument value generation when using secret or configMap. See [this issue](https://github.com/VictoriaMetrics/operator/issues/699) and [this PR](https://github.com/VictoriaMetrics/operator/issues/699) for details. - [vmalertmanager](https://docs.victoriametrics.com/operator/api.html#vmalertmanager): fix default request memory and apply default resources if not set. See [this issue](https://github.com/VictoriaMetrics/operator/issues/706) and [this PR](https://github.com/VictoriaMetrics/operator/pull/710) for details. - [vmagent](https://docs.victoriametrics.com/operator/api.html#vmagent): fix missing additional VolumeClaimTemplates when using `ClaimTemplates` under StatefulMode. @@ -94,18 +95,18 @@ ### Fixes -* [VMNodeScrape]: fixed selectors for Exists and NotExists operators with empty label Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/646 -* [VMRule] Add config for vmrule in validating webhook Thanks in https://github.com/VictoriaMetrics/operator/pull/650 -* [VMAgent]: skips misconfigured objects with missed secret references: https://github.com/VictoriaMetrics/operator/issues/648 -* [VMAgent]: correctly renders initContainer for configuration download: https://github.com/VictoriaMetrics/operator/issues/649 +- [VMNodeScrape]: fixed selectors for Exists and NotExists operators with empty label Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/646 +- [VMRule]: Add config for vmrule in validating webhook Thanks in https://github.com/VictoriaMetrics/operator/pull/650 +- [VMAgent]: skips misconfigured objects with missed secret references: https://github.com/VictoriaMetrics/operator/issues/648 +- [VMAgent]: correctly renders initContainer for configuration download: https://github.com/VictoriaMetrics/operator/issues/649 ### Features -* [VMAlertmanager]: Bump alertmanager to v0.25.0 Thanks [@tamcore](https://github.com/tamcore) in https://github.com/VictoriaMetrics/operator/pull/636 -* [VMCluster] added `clusterNativePort` field to VMSelect/VMInsert for multi-level cluster setup ([#634](https://github.com/VictoriaMetrics/operator/issues/634)) Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/639 -* [VMRule]: add notifierHeader field in vmrule spec Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/622 -* [VMPodScrape]: adds FilterRunning option as prometheus does in https://github.com/VictoriaMetrics/operator/pull/640 -* [VMAuth]: adds latest features in https://github.com/VictoriaMetrics/operator/pull/642 +- [VMAlertmanager]: Bump alertmanager to v0.25.0 Thanks [@tamcore](https://github.com/tamcore) in https://github.com/VictoriaMetrics/operator/pull/636 +- [VMCluster]: added `clusterNativePort` field to VMSelect/VMInsert for multi-level cluster setup ([#634](https://github.com/VictoriaMetrics/operator/issues/634)) Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/639 +- [VMRule]: add notifierHeader field in vmrule spec Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/622 +- [VMPodScrape]: adds FilterRunning option as prometheus does in https://github.com/VictoriaMetrics/operator/pull/640 +- [VMAuth]: adds latest features in https://github.com/VictoriaMetrics/operator/pull/642 [Changes][v0.34.0] @@ -115,22 +116,22 @@ ### Fixes -* [VMAlert]: skip bad rules and improve logging for rules exceed max configmap size https://github.com/VictoriaMetrics/operator/commit/bb754d5c20bb371a197cd6ff5afac1ba86a4d92b -* [VMAlertmanagerConfig]: fixed error with headers in VMAlertmanagerConfig.Receivers.EmailConfigs.Headers unmarshalling. Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/610 -* [VMAgent]: fixed keepInput setting for streaming aggregation. Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/618 -* [VMAlertmanagerConfig]: fix webhook config maxAlerts not work. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/625 -* [VMAgent]: Remove single quotes from remote write headers. Thanks [@axelsccp](https://github.com/axelsccp) in https://github.com/VictoriaMetrics/operator/pull/613 -* [VMAlertmanagerConfig]: fix parse route error and some comments. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/630 -* [VMUser]: properly removes finalizers for objects https://github.com/VictoriaMetrics/operator/commit/8f10113920a353f21fbcc8637076905f2e57bb34 +- [VMAlert]: skip bad rules and improve logging for rules exceed max configmap size https://github.com/VictoriaMetrics/operator/commit/bb754d5c20bb371a197cd6ff5afac1ba86a4d92b +- [VMAlertmanagerConfig]: fixed error with headers in VMAlertmanagerConfig.Receivers.EmailConfigs.Headers unmarshalling. Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/610 +- [VMAgent]: fixed keepInput setting for streaming aggregation. Thanks [@Amper](https://github.com/Amper) in https://github.com/VictoriaMetrics/operator/pull/618 +- [VMAlertmanagerConfig]: fix webhook config maxAlerts not work. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/625 +- [VMAgent]: Remove single quotes from remote write headers. Thanks [@axelsccp](https://github.com/axelsccp) in https://github.com/VictoriaMetrics/operator/pull/613 +- [VMAlertmanagerConfig]: fix parse route error and some comments. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/630 +- [VMUser]: properly removes finalizers for objects https://github.com/VictoriaMetrics/operator/commit/8f10113920a353f21fbcc8637076905f2e57bb34 ### Features -* [VMAlertmanager]: add option to disable route continue enforce. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/621 -* [VMAlertmanagerConfig]: support set require_tls to false. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/624 -* [VMAlertmanagerConfig]: add sanity check. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/627 -* Makefile: bump Alpine base image to latest v3.17.3. Thanks [@denisgolius](https://github.com/denisgolius) in https://github.com/VictoriaMetrics/operator/pull/628 -* [VMAlertmanagerConfig]: support sound field in pushover config. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/631 -* [VMAgent/VMAuth]: download initial config with initContainer https://github.com/VictoriaMetrics/operator/commit/612e7c8f40659731e7938ef9556eb088c67eb4b7 +- [VMAlertmanager]: add option to disable route continue enforce. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/621 +- [VMAlertmanagerConfig]: support set require_tls to false. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/624 +- [VMAlertmanagerConfig]: add sanity check. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/627 +- Makefile: bump Alpine base image to latest v3.17.3. Thanks [@denisgolius](https://github.com/denisgolius) in https://github.com/VictoriaMetrics/operator/pull/628 +- [VMAlertmanagerConfig]: support sound field in pushover config. Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/631 +- [VMAgent/VMAuth]: download initial config with initContainer https://github.com/VictoriaMetrics/operator/commit/612e7c8f40659731e7938ef9556eb088c67eb4b7 [Changes][v0.33.0] @@ -140,8 +141,8 @@ ### Fixes -* [config]: fixes typo at default vm apps version https://github.com/VictoriaMetrics/operator/issues/608 -* [vmsingle]: conditionally adds stream aggregation config https://github.com/VictoriaMetrics/operator/commit/4a0ca54113afcde439ca4c77e22d3ef1c0d36241 +- [config]: fixes typo at default vm apps version https://github.com/VictoriaMetrics/operator/issues/608 +- [vmsingle]: conditionally adds stream aggregation config https://github.com/VictoriaMetrics/operator/commit/4a0ca54113afcde439ca4c77e22d3ef1c0d36241 [Changes][v0.32.1] @@ -151,14 +152,14 @@ ### Fixes -* [security]: builds docker image with latest `alpine` base image and go `v1.20`. +- [security]: builds docker image with latest `alpine` base image and go `v1.20`. ### Features -* [vmauth]: automatically configures `proxy-protocol` client and `reloadAuthKey` for `config-reloader` container. https://github.com/VictoriaMetrics/operator/commit/611819233bf595a4dbd04b07d7be24b7e994379c -* [vmagent]: adds `scrapeTimeout` global configuration for `VMAgent` https://github.com/VictoriaMetrics/operator/commit/d1d5024c6befa0961f8d56c82a0554935a4b1878 -* [vmagent]: adds [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation.html) for `remoteWrite` targets https://github.com/VictoriaMetrics/operator/commit/b8baa6c2b72bdda64ebfcc9c3d86d846cd9b3c98 Thanks [@Amper](https://github.com/Amper) -* [vmsingle]: adds [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation.html) as global configuration for database https://github.com/VictoriaMetrics/operator/commit/b8baa6c2b72bdda64ebfcc9c3d86d846cd9b3c98 Thanks [@Amper](https://github.com/Amper) +- [vmauth]: automatically configures `proxy-protocol` client and `reloadAuthKey` for `config-reloader` container. https://github.com/VictoriaMetrics/operator/commit/611819233bf595a4dbd04b07d7be24b7e994379c +- [vmagent]: adds `scrapeTimeout` global configuration for `VMAgent` https://github.com/VictoriaMetrics/operator/commit/d1d5024c6befa0961f8d56c82a0554935a4b1878 +- [vmagent]: adds [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation.html) for `remoteWrite` targets https://github.com/VictoriaMetrics/operator/commit/b8baa6c2b72bdda64ebfcc9c3d86d846cd9b3c98 Thanks [@Amper](https://github.com/Amper) +- [vmsingle]: adds [streaming aggregation](https://docs.victoriametrics.com/stream-aggregation.html) as global configuration for database https://github.com/VictoriaMetrics/operator/commit/b8baa6c2b72bdda64ebfcc9c3d86d846cd9b3c98 Thanks [@Amper](https://github.com/Amper) [Changes][v0.32.0] @@ -168,13 +169,13 @@ ### Fixes -* [hpa]: Fix hpa object since v2beta deprecated in 1.26+ Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/593 -* [api]: adds missing generated client CRD entities https://github.com/VictoriaMetrics/operator/issues/599 +- [hpa]: Fix hpa object since v2beta deprecated in 1.26+ Thanks [@Haleygo](https://github.com/Haleygo) in https://github.com/VictoriaMetrics/operator/pull/593 +- [api]: adds missing generated client CRD entities https://github.com/VictoriaMetrics/operator/issues/599 ### Features -* [vmalertmanager]: Add support of vmalertmanager.spec.templates and autoreload dirs for templates and configmaps thanks [@Amper](https://github.com/Amper) https://github.com/VictoriaMetrics/operator/issues/590 https://github.com/VictoriaMetrics/operator/issues/592 -* [vmalertmanager]: Add support "%SHARD_NUM%" placeholder for vmagent sts/deployment Thanks [@Amper](https://github.com/Amper) https://github.com/VictoriaMetrics/operator/issues/508 +- [vmalertmanager]: Add support of vmalertmanager.spec.templates and autoreload dirs for templates and configmaps thanks [@Amper](https://github.com/Amper) https://github.com/VictoriaMetrics/operator/issues/590 https://github.com/VictoriaMetrics/operator/issues/592 +- [vmalertmanager]: Add support "%SHARD_NUM%" placeholder for vmagent sts/deployment Thanks [@Amper](https://github.com/Amper) https://github.com/VictoriaMetrics/operator/issues/508 [Changes][v0.31.0] diff --git a/docs/operator/vars.md b/docs/operator/vars.md index ded4abf0c4..f9b4bef55e 100644 --- a/docs/operator/vars.md +++ b/docs/operator/vars.md @@ -10,8 +10,7 @@ aliases: - /operator/vars.html --- # Auto Generated vars for package config - updated at Thu Aug 10 14:32:26 UTC 2023 - + updated at Thu Aug 10 14:33:38 UTC 2023 | varible name | variable default value | variable required | variable description | | --- | --- | --- | --- | From f2df8ad480c649d25edc93d10836a2d8bfda5591 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Tue, 1 Aug 2023 11:18:04 +0400 Subject: [PATCH 55/69] vmbackupmanager: fixes for windows compatibility (#641) * app/vmbackupmanager/storage: fix path join for windows See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704 Signed-off-by: Zakhar Bessarab * lib/backup: fixes for windows support - close dir before running os.RemoveAll. Windows FS does not allow to delete directory before all handles will be closed. - add path "normalization" for local FS to use the same format of paths for both *unix and Windows See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704 Signed-off-by: Zakhar Bessarab --------- Signed-off-by: Zakhar Bessarab --- lib/backup/fscommon/fscommon.go | 10 +++++----- lib/backup/fsremote/fsremote.go | 5 +++++ lib/backup/fsremote/normilize_path_bsd.go | 12 ++++++++++++ lib/backup/fsremote/normilize_path_darwin.go | 9 +++++++++ lib/backup/fsremote/normilize_path_linux.go | 9 +++++++++ lib/backup/fsremote/normilize_path_solaris.go | 9 +++++++++ lib/backup/fsremote/normilize_path_windows.go | 11 +++++++++++ 7 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 lib/backup/fsremote/normilize_path_bsd.go create mode 100644 lib/backup/fsremote/normilize_path_darwin.go create mode 100644 lib/backup/fsremote/normilize_path_linux.go create mode 100644 lib/backup/fsremote/normilize_path_solaris.go create mode 100644 lib/backup/fsremote/normilize_path_windows.go diff --git a/lib/backup/fscommon/fscommon.go b/lib/backup/fscommon/fscommon.go index e232e6ed69..0ba3e856ac 100644 --- a/lib/backup/fscommon/fscommon.go +++ b/lib/backup/fscommon/fscommon.go @@ -52,7 +52,7 @@ func appendFilesInternal(dst []string, d *os.File) ([]string, error) { // Process directory dst, err = AppendFiles(dst, path) if err != nil { - return nil, fmt.Errorf("cannot list %q: %w", path, err) + return nil, fmt.Errorf("cannot append files %q: %w", path, err) } continue } @@ -124,9 +124,6 @@ func removeEmptyDirs(dir string) (bool, error) { return false, err } ok, err := removeEmptyDirsInternal(d) - if err1 := d.Close(); err1 != nil { - err = err1 - } if err != nil { return false, err } @@ -157,7 +154,7 @@ func removeEmptyDirsInternal(d *os.File) (bool, error) { // Process directory ok, err := removeEmptyDirs(path) if err != nil { - return false, fmt.Errorf("cannot list %q: %w", path, err) + return false, fmt.Errorf("cannot remove empty dirs %q: %w", path, err) } if !ok { dirEntries++ @@ -221,6 +218,9 @@ func removeEmptyDirsInternal(d *os.File) (bool, error) { if dirEntries > 0 { return false, nil } + if err := d.Close(); err != nil { + return false, fmt.Errorf("cannot close %q: %w", dir, err) + } // Use os.RemoveAll() instead of os.Remove(), since the dir may contain special files such as flock.lock and backupnames.RestoreInProgressFilename, // which must be ignored. if err := os.RemoveAll(dir); err != nil { diff --git a/lib/backup/fsremote/fsremote.go b/lib/backup/fsremote/fsremote.go index 116d8d9990..45d5ff8f84 100644 --- a/lib/backup/fsremote/fsremote.go +++ b/lib/backup/fsremote/fsremote.go @@ -68,6 +68,7 @@ func (fs *FS) ListParts() ([]common.Part, error) { return nil, fmt.Errorf("cannot stat file %q for part %q: %w", file, p.Path, err) } p.ActualSize = uint64(fi.Size()) + p.Path = pathToCanonical(p.Path) parts = append(parts, p) } return parts, nil @@ -75,6 +76,7 @@ func (fs *FS) ListParts() ([]common.Part, error) { // DeletePart deletes the given part p from fs. func (fs *FS) DeletePart(p common.Part) error { + p.Path = canonicalPathToLocal(p.Path) path := fs.path(p) if err := os.Remove(path); err != nil { return fmt.Errorf("cannot remove %q: %w", path, err) @@ -95,6 +97,7 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error { if !ok { return fmt.Errorf("cannot perform server-side copying from %s to %s: both of them must be fsremote", srcFS, fs) } + p.Path = canonicalPathToLocal(p.Path) srcPath := src.path(p) dstPath := fs.path(p) if err := fs.mkdirAll(dstPath); err != nil { @@ -139,6 +142,7 @@ func (fs *FS) CopyPart(srcFS common.OriginFS, p common.Part) error { // DownloadPart download part p from fs to w. func (fs *FS) DownloadPart(p common.Part, w io.Writer) error { + p.Path = canonicalPathToLocal(p.Path) path := fs.path(p) r, err := os.Open(path) if err != nil { @@ -201,6 +205,7 @@ func (fs *FS) path(p common.Part) string { // // The function does nothing if the filePath doesn't exist. func (fs *FS) DeleteFile(filePath string) error { + filePath = canonicalPathToLocal(filePath) path := filepath.Join(fs.Dir, filePath) err := os.Remove(path) if err != nil && !os.IsNotExist(err) { diff --git a/lib/backup/fsremote/normilize_path_bsd.go b/lib/backup/fsremote/normilize_path_bsd.go new file mode 100644 index 0000000000..4fb42fee95 --- /dev/null +++ b/lib/backup/fsremote/normilize_path_bsd.go @@ -0,0 +1,12 @@ +//go:build freebsd || openbsd || dragonfly || netbsd +// +build freebsd openbsd dragonfly netbsd + +package fsremote + +func pathToCanonical(path string) string { + return path +} + +func canonicalPathToLocal(path string) string { + return path +} diff --git a/lib/backup/fsremote/normilize_path_darwin.go b/lib/backup/fsremote/normilize_path_darwin.go new file mode 100644 index 0000000000..5e9590684b --- /dev/null +++ b/lib/backup/fsremote/normilize_path_darwin.go @@ -0,0 +1,9 @@ +package fsremote + +func pathToCanonical(path string) string { + return path +} + +func canonicalPathToLocal(path string) string { + return path +} diff --git a/lib/backup/fsremote/normilize_path_linux.go b/lib/backup/fsremote/normilize_path_linux.go new file mode 100644 index 0000000000..5e9590684b --- /dev/null +++ b/lib/backup/fsremote/normilize_path_linux.go @@ -0,0 +1,9 @@ +package fsremote + +func pathToCanonical(path string) string { + return path +} + +func canonicalPathToLocal(path string) string { + return path +} diff --git a/lib/backup/fsremote/normilize_path_solaris.go b/lib/backup/fsremote/normilize_path_solaris.go new file mode 100644 index 0000000000..5e9590684b --- /dev/null +++ b/lib/backup/fsremote/normilize_path_solaris.go @@ -0,0 +1,9 @@ +package fsremote + +func pathToCanonical(path string) string { + return path +} + +func canonicalPathToLocal(path string) string { + return path +} diff --git a/lib/backup/fsremote/normilize_path_windows.go b/lib/backup/fsremote/normilize_path_windows.go new file mode 100644 index 0000000000..c1dfe71afa --- /dev/null +++ b/lib/backup/fsremote/normilize_path_windows.go @@ -0,0 +1,11 @@ +package fsremote + +import "strings" + +func pathToCanonical(path string) string { + return strings.ReplaceAll(path, "\\", "/") +} + +func canonicalPathToLocal(path string) string { + return strings.ReplaceAll(path, "/", "\\") +} From e49e4f372b7712960107b86169f60f4e2affcdf0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 03:06:34 -0700 Subject: [PATCH 56/69] docs/CHANGELOG.md: clarify the change at e3ef3df938fb8f6afe8694ad9308e22e5f0b8ba8 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697 --- docs/CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 32ae73d72f..5bee4704f6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,10 +24,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components ## tip -**Update note**: starting from this release, VictoriaMetrics single-server and vmagent -set `honor_timestamps: false` by default in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) -if this options isn't set explicitly. The change supposed to significantly improve staleness detection, compression -and query performance when scraping `cadvisor` metrics. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697). +**Update note**: starting from this release, [vmagent](https://docs.victoriametrics.com/vmagent.html) ignores timestamps provided by scrape targets by default - it associates scraped metrics with local timestamps instead. Set `honor_timestamps: true` in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) if timestamps provided by scrape targets must be used instead. This change helps removing gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) such as `container_memory_usage_bytes`. This also improves data compression and query performance over metrics collected from `cadvisor`. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697). * SECURITY: upgrade Go builder from Go1.20.6 to Go1.20.7. The update includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See [the list of issues addressed in Go1.20.7](https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved). From efb81185a768465ffee91186a551f10d0c3e53ed Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 03:10:10 -0700 Subject: [PATCH 57/69] docs/CHANGELOG.md: remove superflouos information from the line, which describes the upgrade from Go1.20.6 to Go1.20.7 --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5bee4704f6..e11c64c35c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -26,7 +26,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components **Update note**: starting from this release, [vmagent](https://docs.victoriametrics.com/vmagent.html) ignores timestamps provided by scrape targets by default - it associates scraped metrics with local timestamps instead. Set `honor_timestamps: true` in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) if timestamps provided by scrape targets must be used instead. This change helps removing gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) such as `container_memory_usage_bytes`. This also improves data compression and query performance over metrics collected from `cadvisor`. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697). -* SECURITY: upgrade Go builder from Go1.20.6 to Go1.20.7. The update includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See [the list of issues addressed in Go1.20.7](https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved). +* SECURITY: upgrade Go builder from Go1.20.6 to Go1.20.7. See [the list of issues addressed in Go1.20.7](https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow configuring deadline for a backend to be excluded from the rotation on errors via `-failTimeout` cmd-line flag. This feature could be useful when it is expected for backends to be not available for significant periods of time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415) for details. Thanks to @SunKyu for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4416). From 6d0163cca9ea94370459824d11ed36ad308a7262 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 10 Aug 2023 12:18:29 +0200 Subject: [PATCH 58/69] app/vminsert: adds note for dropSamplesOnOverload flag (#4797) Adds note for dropSamplesOnOverload flag that are samples dropped before replication --- docs/Cluster-VictoriaMetrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 051fa7bc1a..2ad4e854ee 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -844,7 +844,7 @@ Below is the output for `/path/to/vminsert -help`: -disableRerouting Whether to disable re-routing when some of vmstorage nodes accept incoming data at slower speed compared to other storage nodes. Disabled re-routing limits the ingestion rate by the slowest vmstorage node. On the other side, disabled re-routing minimizes the number of active time series in the cluster during rolling restarts and during spikes in series churn rate. See also -dropSamplesOnOverload (default true) -dropSamplesOnOverload - Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded + Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded. The drop of samples happens before the replication, so it's not recommended to use this flag with -replicationFactor enabled. -enableTCP6 Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used -envflag.enable From b50ed5ddd12c43f1978b22e9de0d855bdd9efc03 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 03:46:08 -0700 Subject: [PATCH 59/69] app/vmctl: follow-up after 5aed369132b3e38d85bb29709b23d861503d9ff0 - Fix default value for --remote-read-disable-path-append - Clarify description for the change at docs/CHANGELOG.md Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655 TODO: address the comment at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4744 --- app/vmctl/flags.go | 4 ++-- docs/CHANGELOG.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index da88ba3f1c..92c582f29e 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -556,8 +556,8 @@ var ( }, &cli.BoolFlag{ Name: remoteReadDisablePathAppend, - Usage: "Whether to disable automatic appending of the path to the remote storage.", - Value: true, + Usage: "Whether to disable automatic appending of the /api/v1/read suffix to --remote-read-src-addr", + Value: false, }, } ) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e11c64c35c..b45c226f0a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,7 +34,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). -* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): do not add `/api/v1/read` suffix to remote read storage address defined by `--remote-read-src-addr` if a `--remote-read-disable-path-append` command-line flag is set. It allows an overriding path for remote-read API via `--remote-read-src-addr`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow specifying custom full url at `--remote-read-src-addr` command-line flag if `--remote-read-disable-path-append` command-line flag is set. This allows importing data from urls, which do not end with `/api/v1/read`. For example, from Promscale. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): allow displaying the full error message on click for trimmed error messages in vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4719). * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dasbhoard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit. From 7ec4ccc005be7d28a93de9c8abf703e9b825d8a3 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 04:11:11 -0700 Subject: [PATCH 60/69] docs/CaseStudies.md: update Naver case study after 0c3d61b211b9016f4519362e0ba9c9ea8975ca9d - Fix a link to case study at the top of docs/CaseStudies.md - Remove non-essential text - Add Naver video and slides about VictoriaMetrics to docs/Articles.md Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4755 --- docs/Articles.md | 1 + docs/CaseStudies.md | 17 +++++------------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docs/Articles.md b/docs/Articles.md index 1c070e6cc1..eb648b3829 100644 --- a/docs/Articles.md +++ b/docs/Articles.md @@ -73,6 +73,7 @@ See also [case studies](https://docs.victoriametrics.com/CaseStudies.html). * [VictoriaMetrics static scraper](https://blog.differentpla.net/blog/2022/10/16/victoria-metrics-static-scraper/) * [VictoriaMetrics and Open Cosmos boldly takes edge computing to the edge of space](https://www.iot-now.com/2022/07/19/122423-victoriametrics-and-open-cosmos-boldly-takes-edge-computing-to-the-edge-of-space/) * [Evaluating Backend Options For Prometheus Metrics](https://www.techetio.com/2022/08/21/evaluating-backend-options-for-prometheus-metrics/) +* [Time Series in the Multiverse of Madness (in Korean)](https://www.youtube.com/watch?v=OUyXPgVcdw4), plus [these slides](https://deview.kr/data/deview/session/attach/%5B2B4%5DVictoriaMetrics_%E1%84%89%E1%85%B5%E1%84%80%E1%85%A8%E1%84%8B%E1%85%A7%E1%86%AF_%E1%84%83%E1%85%A6%E1%84%8B%E1%85%B5%E1%84%90%E1%85%A5_%E1%84%83%E1%85%A2%E1%84%92%E1%85%A9%E1%86%AB%E1%84%83%E1%85%A9%E1%86%AB%E1%84%8B%E1%85%B4_%E1%84%86%E1%85%A5%E1%86%AF%E1%84%90%E1%85%B5%E1%84%87%E1%85%A5%E1%84%89%E1%85%B3_Kor+Eng.pdf) ## Our articles diff --git a/docs/CaseStudies.md b/docs/CaseStudies.md index 5d5eaffead..1d46b05f05 100644 --- a/docs/CaseStudies.md +++ b/docs/CaseStudies.md @@ -30,7 +30,7 @@ where you can chat with VictoriaMetrics users to get additional references, revi - [Groove X](#groove-x) - [Idealo.de](#idealode) - [MHI Vestas Offshore Wind](#mhi-vestas-offshore-wind) - - [Naver][#naver] + - [Naver](#naver) - [Percona](#percona) - [Razorpay](#razorpay) - [Roblox](#roblox) @@ -421,19 +421,12 @@ Numbers with current, limited roll out: - Data size on disk: 800 GiB - Retention period: 3 years -## Naver +## Naver -See [our](https://www.navercorp.com/en/) video ["Time Series in the Multiverse of Madness" (in Korean)](https://www.youtube.com/watch?v=OUyXPgVcdw4) about the comparison of Time Series Database, why we have chosen VictoriaMetrics -We also covered the internals of the VictoriaMetrics data model and Cluster. -The key areas: +[Naver](https://www.navercorp.com/en/) is a global tech platform that enables wide access to advanced technologies for SMEs, creators and partners, +fueling their greater growth around the world. -* Explanation of the importance and role of Monitoring for NaverCorp -* History overview of Time Series Databases -* VictoriaMetrics Data model - read and write paths, index structure, compression and the crucial role of the churn rate -* Time series in the Multiverse Madness -* HA and Fault Tolerance - write without data loss, read for no downtime, management of Multiverse - -[Slides](https://deview.kr/data/deview/session/attach/%5B2B4%5DVictoriaMetrics_%E1%84%89%E1%85%B5%E1%84%80%E1%85%A8%E1%84%8B%E1%85%A7%E1%86%AF_%E1%84%83%E1%85%A6%E1%84%8B%E1%85%B5%E1%84%90%E1%85%A5_%E1%84%83%E1%85%A2%E1%84%92%E1%85%A9%E1%86%AB%E1%84%83%E1%85%A9%E1%86%AB%E1%84%8B%E1%85%B4_%E1%84%86%E1%85%A5%E1%86%AF%E1%84%90%E1%85%B5%E1%84%87%E1%85%A5%E1%84%89%E1%85%B3_Kor+Eng.pdf) in English and Korean +See [this video](https://www.youtube.com/watch?v=OUyXPgVcdw4) and [these slides](https://deview.kr/data/deview/session/attach/%5B2B4%5DVictoriaMetrics_%E1%84%89%E1%85%B5%E1%84%80%E1%85%A8%E1%84%8B%E1%85%A7%E1%86%AF_%E1%84%83%E1%85%A6%E1%84%8B%E1%85%B5%E1%84%90%E1%85%A5_%E1%84%83%E1%85%A2%E1%84%92%E1%85%A9%E1%86%AB%E1%84%83%E1%85%A9%E1%86%AB%E1%84%8B%E1%85%B4_%E1%84%86%E1%85%A5%E1%86%AF%E1%84%90%E1%85%B5%E1%84%87%E1%85%A5%E1%84%89%E1%85%B3_Kor+Eng.pdf) on why and how Naver uses VictoriaMetrics. ## Percona From 38e72ea344ba7d7f0694cf351056b7f1d12e950c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 04:29:04 -0700 Subject: [PATCH 61/69] docs/Release-Guide.md: document that changes must be synced between branches immediately after the commit in any branch --- docs/Release-Guide.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/Release-Guide.md b/docs/Release-Guide.md index ab195951c5..179b26638a 100644 --- a/docs/Release-Guide.md +++ b/docs/Release-Guide.md @@ -48,18 +48,22 @@ Bumping the limits may significantly improve build speed. ## Release version and Docker images -1. Make sure that the release commits have no security issues. -1. Document all the changes for new release in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) and update version if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/SECURITY.md) +1. Make sure all the changes are documented in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md). + Ideally, every change must be documented in the commit with the change. Alternatively, the change must be documented immediately + after the commit, which adds the change. +1. Make sure all the changes are synced between `master`, `cluster`, `enterprise-single-node` and `enteprise-cluster` branches. + Changes in these branches must be synced immediately after they are commited in at least a single branch. +1. Make sure that the release branches have no security issues. +1. Update release versions if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/SECURITY.md). 1. Add `(available starting from v1.xx.y)` line to feature docs introduced in the upcoming release. 1. Cut new version in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md) -and make it merged. See example in this [commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b771152039d23b5ccd637a23ea748bc44a9511a7). -1. Cherry-pick all changes from `master` to `cluster` branch, and to ENT branches `enterprise-single-node`, `enterprise-cluster`. + and make it merged. See example in this [commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b771152039d23b5ccd637a23ea748bc44a9511a7). 1. Cherry-pick bug fixes relevant for LTS releases. 1. Make sure you get all changes fetched `git fetch --all`. 1. Create the following release tags: * `git tag -s v1.xx.y` in `master` branch * `git tag -s v1.xx.y-cluster` in `cluster` branch - * `git tag -s v1.xx.y-enterprise` in `enterprise` branch + * `git tag -s v1.xx.y-enterprise` in `enterprise-single-node` branch * `git tag -s v1.xx.y-enterprise-cluster` in `enterprise-cluster` branch 1. Run `TAG=v1.xx.y make publish-release`. This command performs the following tasks: a) Build and package binaries in `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory. From 2328e4cabc06b550baf233bde609dc5af77cb64d Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 04:51:57 -0700 Subject: [PATCH 62/69] app/vmagent/remotewrite: keep in sync the default value for -remoteWrite.sendTimeout option in the description with the actually used timeout This is a follow-up for aef31f201a85e993dcae448a52927868f0fad9ea Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4776 --- app/vmagent/remotewrite/client.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index 6340da4635..5b2826f516 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -29,8 +29,9 @@ var ( rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+ "By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+ "is sent after temporary unavailability of the remote storage") - sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)") - proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+ + sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url (default "+ + defaultSendTimeout.String()+")") + proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+ "Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234") tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url") @@ -72,6 +73,8 @@ var ( awsSecretKey = flagutil.NewArrayString("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set") ) +const defaultSendTimeout = time.Minute + type client struct { sanitizedURL string remoteWriteURL string @@ -134,7 +137,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste } hc := &http.Client{ Transport: tr, - Timeout: sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute), + Timeout: sendTimeout.GetOptionalArgOrDefault(argIdx, defaultSendTimeout), } c := &client{ sanitizedURL: sanitizedURL, From ac0b7e042152572f47e72d70de5293ed81149f8b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 05:04:13 -0700 Subject: [PATCH 63/69] Revert "vmui: change the response for active queries (#4782)" This reverts commit 252643d10069efddcd133aa8a58111227caf6034. Reason for revert: the commit incorrectly fixes the the issue. The `remoteAddr` must be properly quoted inside lib/httpserver.GetQuotedRemoteAddr(). It isn't quoted properly if the request contains X-Forwarded-For header. The proper fix will be included in the follow-up commit. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676 --- app/vmselect/promql/active_queries.go | 7 +------ docs/CHANGELOG.md | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/app/vmselect/promql/active_queries.go b/app/vmselect/promql/active_queries.go index 2ef261b49b..d66058e53b 100644 --- a/app/vmselect/promql/active_queries.go +++ b/app/vmselect/promql/active_queries.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "sort" - "strings" "sync" "sync/atomic" "time" @@ -24,12 +23,8 @@ func ActiveQueriesHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, `{"status":"ok","data":[`) for i, aqe := range aqes { d := now.Sub(aqe.startTime) - addr := aqe.quotedRemoteAddr - if n := strings.IndexByte(aqe.quotedRemoteAddr, ','); n != -1 { - addr = aqe.quotedRemoteAddr[:n] - } fmt.Fprintf(w, `{"duration":"%.3fs","id":"%016X","remote_addr":%s,"query":%q,"start":%d,"end":%d,"step":%d}`, - d.Seconds(), aqe.qid, addr, aqe.q, aqe.start, aqe.end, aqe.step) + d.Seconds(), aqe.qid, aqe.quotedRemoteAddr, aqe.q, aqe.start, aqe.end, aqe.step) if i+1 < len(aqes) { fmt.Fprintf(w, `,`) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b45c226f0a..eb89962775 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -46,7 +46,6 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). -* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the response of active queries to valid JSON. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4782). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series when `remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804). From be5c4818f51a9eb426dae121cd3dc13218be947b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 05:19:44 -0700 Subject: [PATCH 64/69] lib/httpserver: properly quote the returned address from GetQuotedRemoteAddr() for requests with X-Forwarded-For header Make sure that the quoted address can be used as JSON string. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676#issuecomment-1663203424 This is a follow up for 252643d10069efddcd133aa8a58111227caf6034 and ac0b7e042152572f47e72d70de5293ed81149f8b Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676 --- docs/CHANGELOG.md | 1 + lib/httpserver/httpserver.go | 7 +++--- lib/httpserver/httpserver_test.go | 36 +++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 lib/httpserver/httpserver_test.go diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index eb89962775..8c651089d4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -47,6 +47,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series when `remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804). +* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly handle client address with `X-Forwarded-For` part. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676#issuecomment-1663203424). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index 9a9675bafc..75b5940e0d 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -436,11 +436,12 @@ var ( // GetQuotedRemoteAddr returns quoted remote address. func GetQuotedRemoteAddr(r *http.Request) string { - remoteAddr := strconv.Quote(r.RemoteAddr) // quote remoteAddr and X-Forwarded-For, since they may contain untrusted input + remoteAddr := r.RemoteAddr if addr := r.Header.Get("X-Forwarded-For"); addr != "" { - remoteAddr += ", X-Forwarded-For: " + strconv.Quote(addr) + remoteAddr += ", X-Forwarded-For: " + addr } - return remoteAddr + // quote remoteAddr and X-Forwarded-For, since they may contain untrusted input + return strconv.Quote(remoteAddr) } // Errorf writes formatted error message to w and to logger. diff --git a/lib/httpserver/httpserver_test.go b/lib/httpserver/httpserver_test.go new file mode 100644 index 0000000000..9311cbfccf --- /dev/null +++ b/lib/httpserver/httpserver_test.go @@ -0,0 +1,36 @@ +package httpserver + +import ( + "encoding/json" + "net/http" + "testing" +) + +func TestGetQuotedRemoteAddr(t *testing.T) { + f := func(remoteAddr, xForwardedFor, expectedAddr string) { + t.Helper() + + req := &http.Request{ + RemoteAddr: remoteAddr, + } + if xForwardedFor != "" { + req.Header = map[string][]string{ + "X-Forwarded-For": {xForwardedFor}, + } + } + addr := GetQuotedRemoteAddr(req) + if addr != expectedAddr { + t.Fatalf("unexpected remote addr;\ngot\n%s\nwant\n%s", addr, expectedAddr) + } + + // Verify that the addr can be unmarshaled as JSON string + var s string + if err := json.Unmarshal([]byte(addr), &s); err != nil { + t.Fatalf("cannot unmarshal addr: %s", err) + } + } + + f("1.2.3.4", "", `"1.2.3.4"`) + f("1.2.3.4", "foo.bar", `"1.2.3.4, X-Forwarded-For: foo.bar"`) + f("1.2\n\"3.4", "foo\nb\"ar", `"1.2\n\"3.4, X-Forwarded-For: foo\nb\"ar"`) +} From 271743f892590ed45db7f7c2ac05136c5ea4d865 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 05:26:16 -0700 Subject: [PATCH 65/69] docs/CHANGELOG.md: add missing context to the description of the fix at be5c4818f51a9eb426dae121cd3dc13218be947b --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8c651089d4..e9ff56efc1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -47,7 +47,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series when `remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804). -* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly handle client address with `X-Forwarded-For` part. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676#issuecomment-1663203424). +* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly handle client address with `X-Forwarded-For` part at the [Active queries](https://docs.victoriametrics.com/#active-queries) page. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676#issuecomment-1663203424). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1) From 4c4bcdf0b10c408b683a1acdbbf6315243d631bd Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 05:38:28 -0700 Subject: [PATCH 66/69] docs/CHANGELOG.md: add a link to stream aggregation for the description of the bugfix at a4a1884237dfe0360397f932856c01d6a6f32098 This makes the description more clear. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804 --- app/vmagent/remotewrite/remotewrite.go | 4 +--- docs/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index bb2bccd4ac..c8391810ff 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -11,9 +11,6 @@ import ( "time" "github.com/cespare/xxhash/v2" - - "github.com/VictoriaMetrics/metrics" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" @@ -29,6 +26,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" + "github.com/VictoriaMetrics/metrics" ) var ( diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e9ff56efc1..0097eaa0cc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -46,7 +46,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). -* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series when `remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804). +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): keep unmatched series at [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) when `-remoteWrite.streamAggr.dropInput` is set to `false` to match intended behaviour introduced at [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4804). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly handle client address with `X-Forwarded-For` part at the [Active queries](https://docs.victoriametrics.com/#active-queries) page. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4676#issuecomment-1663203424). From a19a65b3a52e36b2cd1f793aa707352dbcb59f8a Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 06:23:00 -0700 Subject: [PATCH 67/69] app/vmagent/remotewrite: go fmt --- app/vmagent/remotewrite/remotewrite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index c8391810ff..616a173cc3 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -10,7 +10,6 @@ import ( "sync/atomic" "time" - "github.com/cespare/xxhash/v2" "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" @@ -27,6 +26,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr" "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/metrics" + "github.com/cespare/xxhash/v2" ) var ( From e0017b4d471e3f07b5a1abe27fcdd105fb35ed3e Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 11 Aug 2023 06:25:51 -0700 Subject: [PATCH 68/69] all: update Go builder from Go1.20.7 to Go1.21.0 See https://tip.golang.org/doc/go1.21 and https://go.dev/blog/go1.21 --- .github/workflows/check-licenses.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/main.yml | 6 +++--- app/vmui/Dockerfile-web | 2 +- deployment/docker/Makefile | 2 +- docs/CHANGELOG.md | 2 +- snap/local/Makefile | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check-licenses.yml b/.github/workflows/check-licenses.yml index 54f5bd20b6..70d3e87f7a 100644 --- a/.github/workflows/check-licenses.yml +++ b/.github/workflows/check-licenses.yml @@ -17,7 +17,7 @@ jobs: - name: Setup Go uses: actions/setup-go@main with: - go-version: 1.20.7 + go-version: 1.21.0 id: go - name: Code checkout uses: actions/checkout@master diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9c9c0fda99..73ef32c3df 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -57,7 +57,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.7 + go-version: 1.21.0 check-latest: true cache: true if: ${{ matrix.language == 'go' }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d93517873e..268033ca91 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: 1.20.7 + go-version: 1.21.0 check-latest: true cache: true @@ -56,7 +56,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: 1.20.7 + go-version: 1.21.0 check-latest: true cache: true @@ -81,7 +81,7 @@ jobs: id: go uses: actions/setup-go@v4 with: - go-version: 1.20.7 + go-version: 1.21.0 check-latest: true cache: true diff --git a/app/vmui/Dockerfile-web b/app/vmui/Dockerfile-web index 2f77aae415..2c03eb2c2e 100644 --- a/app/vmui/Dockerfile-web +++ b/app/vmui/Dockerfile-web @@ -1,4 +1,4 @@ -FROM golang:1.20.7 as build-web-stage +FROM golang:1.21.0 as build-web-stage COPY build /build WORKDIR /build diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index b8a377f71f..3d3536d602 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -9,7 +9,7 @@ ROOT_IMAGE ?= alpine:3.18.2 # TODO: sync it with ROOT_IMAGE when it will be fixed in the new alpine releases CERTS_IMAGE := alpine:3.17.3 -GO_BUILDER_IMAGE := golang:1.20.7-alpine +GO_BUILDER_IMAGE := golang:1.21.0-alpine BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __) DOCKER_BUILD ?= docker build diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0097eaa0cc..7c16d44a1c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -26,7 +26,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components **Update note**: starting from this release, [vmagent](https://docs.victoriametrics.com/vmagent.html) ignores timestamps provided by scrape targets by default - it associates scraped metrics with local timestamps instead. Set `honor_timestamps: true` in [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) if timestamps provided by scrape targets must be used instead. This change helps removing gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) such as `container_memory_usage_bytes`. This also improves data compression and query performance over metrics collected from `cadvisor`. See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697). -* SECURITY: upgrade Go builder from Go1.20.6 to Go1.20.7. See [the list of issues addressed in Go1.20.7](https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved). +* SECURITY: upgrade Go builder from Go1.20.6 to Go1.21.0 in order to fix [this issue](https://github.com/golang/go/issues/61460). * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow configuring deadline for a backend to be excluded from the rotation on errors via `-failTimeout` cmd-line flag. This feature could be useful when it is expected for backends to be not available for significant periods of time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415) for details. Thanks to @SunKyu for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4416). diff --git a/snap/local/Makefile b/snap/local/Makefile index 50c0f40744..898a0368b6 100644 --- a/snap/local/Makefile +++ b/snap/local/Makefile @@ -1,4 +1,4 @@ -GO_VERSION ?=1.20.7 +GO_VERSION ?=1.21.0 SNAP_BUILDER_IMAGE := local/snap-builder:2.0.0-$(shell echo $(GO_VERSION) | tr :/ __) From 1bd7637fe1ad361c6f63f711e08b081d979cb13d Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Fri, 11 Aug 2023 17:37:48 +0400 Subject: [PATCH 69/69] lib/promrelabel: fix relabeling if clause (#4816) * lib/promrelabel: fix relabeling if clause being applied to labels outside of current context Relabeling is applied to each metric row separately, but in order to lower amount of memory allocations it is reusing labels. Functions which are working on current metric row labels are supposed to use only current metric labels by using provided offset, but if clause matcher was using the whole labels set instead of local metrics. This leaded to invalid relabeling results such as one described here: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806 Signed-off-by: Zakhar Bessarab * docs/CHANGELOG.md: document the bugfix Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1998 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806 --------- Signed-off-by: Zakhar Bessarab Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 1 + lib/promrelabel/relabel.go | 2 +- lib/promrelabel/relabel_test.go | 46 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7c16d44a1c..646db5dfce 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -42,6 +42,7 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected. * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-health.yml) as it could be related to many VictoriaMetrics components. +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly apply `if` filters during [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements). Previously the `if` filter could improperly work. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4806) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4816). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly set `vmalert_config_last_reload_successful` value on configuration updates or rollbacks. The bug was introduced in [v1.92.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.0) in [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4543). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix `vmalert_remotewrite_send_duration_seconds_total` value, before it didn't count in the real time spending on remote write requests. See [this pr](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4801) for details. diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 35a47e74de..c6c17b4c96 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -158,7 +158,7 @@ func FinalizeLabels(dst, src []prompbmarshal.Label) []prompbmarshal.Label { // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label { src := labels[labelsOffset:] - if !prc.If.Match(labels) { + if !prc.If.Match(src) { if prc.Action == "keep" { // Drop the target on `if` mismatch for `action: keep` return labels[:labelsOffset] diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index a71bb5a6c0..987374252c 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -915,3 +915,49 @@ func newTestRegexRelabelConfig(pattern string) *parsedRelabelConfig { } return prc } + +func TestParsedRelabelConfigsApplyForMultipleSeries(t *testing.T) { + f := func(config string, metrics []string, resultExpected []string) { + t.Helper() + pcs, err := ParseRelabelConfigsData([]byte(config)) + if err != nil { + t.Fatalf("cannot parse %q: %s", config, err) + } + + totalLabels := 0 + var labels []prompbmarshal.Label + for _, metric := range metrics { + labels = append(labels, promutils.MustNewLabelsFromString(metric).GetLabels()...) + resultLabels := pcs.Apply(labels, totalLabels) + SortLabels(resultLabels) + totalLabels += len(resultLabels) + labels = resultLabels + } + + var result []string + for i := range labels { + result = append(result, LabelsToString(labels[i:i+1])) + } + + if len(result) != len(resultExpected) { + t.Fatalf("unexpected number of results; got\n%q\nwant\n%q", result, resultExpected) + } + + for i := range result { + if result[i] != resultExpected[i] { + t.Fatalf("unexpected result[%d]; got\n%q\nwant\n%q", i, result[i], resultExpected[i]) + } + } + } + + t.Run("drops one of series", func(t *testing.T) { + f(` +- action: drop + if: '{__name__!~"smth"}' +`, []string{`smth`, `notthis`}, []string{`smth`}) + f(` +- action: drop + if: '{__name__!~"smth"}' +`, []string{`notthis`, `smth`}, []string{`smth`}) + }) +}