From 1d352b92c7ebbb5620b836bdf846ca9e90922e59 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Fri, 18 Oct 2024 12:35:23 +0300 Subject: [PATCH] lib/promscrape: fixed reload on max_scrape_size change (#7282) ### Describe Your Changes fixed reload on max_scrape_size change https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7260 ### Checklist The following checks are **mandatory**: - [x] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Signed-off-by: hagen1778 Co-authored-by: hagen1778 (cherry picked from commit 965a33c8932992bca6bfd7768acc4f2440666965) --- docs/changelog/CHANGELOG.md | 1 + lib/promscrape/client.go | 2 +- lib/promscrape/scraper_test.go | 81 ++++++++++++++++++++++++++++++++++ lib/promscrape/scrapework.go | 8 ++-- 4 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 lib/promscrape/scraper_test.go diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 9e852467a..55c42b646 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -35,6 +35,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly apply replication factor when storage node groups are used and replication factor is configured via global value such as `-replicationFactor=2`. Previously, global replication factor was ignored for storage node groups. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#vmstorage-groups-at-vmselect) for more information about storage groups configuration. * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly process response in [multi-level cluster setup](https://docs.victoriametrics.com/cluster-victoriametrics/#multi-level-cluster-setup). Before, vmselect could return no data in multi-level setup. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7270) for details. The issue was introduced in [v1.104.0](https://docs.victoriametrics.com/changelog/#v11040). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): properly apply configuration changes during hot-reload to rule groups that haven't started yet. Previously, configuration updates to such groups could have resulted into blocking all evaluations within the group, until vmalert restart. +* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmagent](https://docs.victoriametrics.com/vmagent/): properly update `max_scrape_size` param change during [hot-reload](https://docs.victoriametrics.com/vmagent/#configuration-update). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7260). ## [v1.104.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.104.0) diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go index 10d32636f..d692061b0 100644 --- a/lib/promscrape/client.go +++ b/lib/promscrape/client.go @@ -171,7 +171,7 @@ func (c *client) ReadData(dst *bytesutil.ByteBuffer) error { maxScrapeSizeExceeded.Inc() return fmt.Errorf("the response from %q exceeds -promscrape.maxScrapeSize or max_scrape_size in the scrape config (%d bytes). "+ "Possible solutions are: reduce the response size for the target, increase -promscrape.maxScrapeSize command-line flag, "+ - "increase max_scrape_size value in scrape config for the given target", c.scrapeURL, maxScrapeSize.N) + "increase max_scrape_size value in scrape config for the given target", c.scrapeURL, c.maxScrapeSize) } return nil } diff --git a/lib/promscrape/scraper_test.go b/lib/promscrape/scraper_test.go new file mode 100644 index 000000000..45a770dc1 --- /dev/null +++ b/lib/promscrape/scraper_test.go @@ -0,0 +1,81 @@ +package promscrape + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" +) + +func TestScraperReload(t *testing.T) { + f := func(oldCfgData, newCfgData string, reloadExpected bool) { + pushData := func(_ *auth.Token, _ *prompbmarshal.WriteRequest) {} + globalStopChan = make(chan struct{}) + defer close(globalStopChan) + + randName := rand.Int() + sg := newScraperGroup(fmt.Sprintf("static_configs_%d", randName), pushData, globalStopChan) + defer sg.stop() + + scrapeConfigPath := "test-scrape.yaml" + var oldCfg, newCfg Config + if err := oldCfg.parseData([]byte(oldCfgData), scrapeConfigPath); err != nil { + t.Fatalf("cannot create old config: %s", err) + } + oldSws := oldCfg.getStaticScrapeWork() + sg.update(oldSws) + oldChangesCount := sg.changesCount.Get() + + if err := newCfg.parseData([]byte(newCfgData), scrapeConfigPath); err != nil { + t.Fatalf("cannot create new config: %s", err) + } + doReload := (&newCfg).mustRestart(&oldCfg) + if doReload != reloadExpected { + t.Errorf("unexpected reload behaviour:\nexpected: %t\nactual: %t\n", reloadExpected, doReload) + } + newSws := newCfg.getStaticScrapeWork() + sg.update(newSws) + newChangesCount := sg.changesCount.Get() + if (newChangesCount != oldChangesCount) != reloadExpected { + t.Errorf("expected reload behaviour:\nexpected reload happen: %t\nactual reload happen: %t", reloadExpected, newChangesCount != oldChangesCount) + } + } + f(` +scrape_configs: +- job_name: node-exporter + static_configs: + - targets: + - localhost:8429`, ` +scrape_configs: +- job_name: node-exporter + static_configs: + - targets: + - localhost:8429`, false) + f(` +scrape_configs: +- job_name: node-exporter + static_configs: + - targets: + - localhost:8429`, ` +scrape_configs: +- job_name: node-exporter + static_configs: + - targets: + - localhost:8429 + - localhost:8428`, true) + f(` +scrape_configs: +- job_name: node-exporter + max_scrape_size: 1KiB + static_configs: + - targets: + - localhost:8429`, ` +scrape_configs: +- job_name: node-exporter + max_scrape_size: 2KiB + static_configs: + - targets: + - localhost:8429`, true) +} diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 8e4fb799e..0774dba50 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -163,13 +163,13 @@ func (sw *ScrapeWork) key() string { // Do not take into account OriginalLabels, since they can be changed with relabeling. // Do not take into account RelabelConfigs, since it is already applied to Labels. // Take into account JobNameOriginal in order to capture the case when the original job_name is changed via relabeling. - key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+ - "ExternalLabels=%s, "+ + key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, "+ + "HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, ExternalLabels=%s, MaxScrapeSize=%d, "+ "ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%q, "+ "SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+ "ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v", - sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(), - sw.ExternalLabels.String(), + sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, + sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(), sw.ExternalLabels.String(), sw.MaxScrapeSize, sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers)