lib/promscrape: fixed reload on max_scrape_size change (#7282)

### Describe Your Changes

fixed reload on max_scrape_size change
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7260

### Checklist

The following checks are **mandatory**:

- [x] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
Andrii Chubatiuk 2024-10-18 12:35:23 +03:00 committed by GitHub
parent c4fe23794a
commit 965a33c893
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 87 additions and 5 deletions

View file

@ -35,6 +35,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly apply replication factor when storage node groups are used and replication factor is configured via global value such as `-replicationFactor=2`. Previously, global replication factor was ignored for storage node groups. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#vmstorage-groups-at-vmselect) for more information about storage groups configuration. * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly apply replication factor when storage node groups are used and replication factor is configured via global value such as `-replicationFactor=2`. Previously, global replication factor was ignored for storage node groups. See [these docs](https://docs.victoriametrics.com/cluster-victoriametrics/#vmstorage-groups-at-vmselect) for more information about storage groups configuration.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly process response in [multi-level cluster setup](https://docs.victoriametrics.com/cluster-victoriametrics/#multi-level-cluster-setup). Before, vmselect could return no data in multi-level setup. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7270) for details. The issue was introduced in [v1.104.0](https://docs.victoriametrics.com/changelog/#v11040). * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly process response in [multi-level cluster setup](https://docs.victoriametrics.com/cluster-victoriametrics/#multi-level-cluster-setup). Before, vmselect could return no data in multi-level setup. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7270) for details. The issue was introduced in [v1.104.0](https://docs.victoriametrics.com/changelog/#v11040).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): properly apply configuration changes during hot-reload to rule groups that haven't started yet. Previously, configuration updates to such groups could have resulted into blocking all evaluations within the group, until vmalert restart. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): properly apply configuration changes during hot-reload to rule groups that haven't started yet. Previously, configuration updates to such groups could have resulted into blocking all evaluations within the group, until vmalert restart.
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmagent](https://docs.victoriametrics.com/vmagent/): properly update `max_scrape_size` param change during [hot-reload](https://docs.victoriametrics.com/vmagent/#configuration-update). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7260).
## [v1.104.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.104.0) ## [v1.104.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.104.0)

View file

@ -171,7 +171,7 @@ func (c *client) ReadData(dst *bytesutil.ByteBuffer) error {
maxScrapeSizeExceeded.Inc() maxScrapeSizeExceeded.Inc()
return fmt.Errorf("the response from %q exceeds -promscrape.maxScrapeSize or max_scrape_size in the scrape config (%d bytes). "+ return fmt.Errorf("the response from %q exceeds -promscrape.maxScrapeSize or max_scrape_size in the scrape config (%d bytes). "+
"Possible solutions are: reduce the response size for the target, increase -promscrape.maxScrapeSize command-line flag, "+ "Possible solutions are: reduce the response size for the target, increase -promscrape.maxScrapeSize command-line flag, "+
"increase max_scrape_size value in scrape config for the given target", c.scrapeURL, maxScrapeSize.N) "increase max_scrape_size value in scrape config for the given target", c.scrapeURL, c.maxScrapeSize)
} }
return nil return nil
} }

View file

@ -0,0 +1,81 @@
package promscrape
import (
"fmt"
"math/rand"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
)
func TestScraperReload(t *testing.T) {
f := func(oldCfgData, newCfgData string, reloadExpected bool) {
pushData := func(_ *auth.Token, _ *prompbmarshal.WriteRequest) {}
globalStopChan = make(chan struct{})
defer close(globalStopChan)
randName := rand.Int()
sg := newScraperGroup(fmt.Sprintf("static_configs_%d", randName), pushData, globalStopChan)
defer sg.stop()
scrapeConfigPath := "test-scrape.yaml"
var oldCfg, newCfg Config
if err := oldCfg.parseData([]byte(oldCfgData), scrapeConfigPath); err != nil {
t.Fatalf("cannot create old config: %s", err)
}
oldSws := oldCfg.getStaticScrapeWork()
sg.update(oldSws)
oldChangesCount := sg.changesCount.Get()
if err := newCfg.parseData([]byte(newCfgData), scrapeConfigPath); err != nil {
t.Fatalf("cannot create new config: %s", err)
}
doReload := (&newCfg).mustRestart(&oldCfg)
if doReload != reloadExpected {
t.Errorf("unexpected reload behaviour:\nexpected: %t\nactual: %t\n", reloadExpected, doReload)
}
newSws := newCfg.getStaticScrapeWork()
sg.update(newSws)
newChangesCount := sg.changesCount.Get()
if (newChangesCount != oldChangesCount) != reloadExpected {
t.Errorf("expected reload behaviour:\nexpected reload happen: %t\nactual reload happen: %t", reloadExpected, newChangesCount != oldChangesCount)
}
}
f(`
scrape_configs:
- job_name: node-exporter
static_configs:
- targets:
- localhost:8429`, `
scrape_configs:
- job_name: node-exporter
static_configs:
- targets:
- localhost:8429`, false)
f(`
scrape_configs:
- job_name: node-exporter
static_configs:
- targets:
- localhost:8429`, `
scrape_configs:
- job_name: node-exporter
static_configs:
- targets:
- localhost:8429
- localhost:8428`, true)
f(`
scrape_configs:
- job_name: node-exporter
max_scrape_size: 1KiB
static_configs:
- targets:
- localhost:8429`, `
scrape_configs:
- job_name: node-exporter
max_scrape_size: 2KiB
static_configs:
- targets:
- localhost:8429`, true)
}

View file

@ -163,13 +163,13 @@ func (sw *ScrapeWork) key() string {
// Do not take into account OriginalLabels, since they can be changed with relabeling. // Do not take into account OriginalLabels, since they can be changed with relabeling.
// Do not take into account RelabelConfigs, since it is already applied to Labels. // Do not take into account RelabelConfigs, since it is already applied to Labels.
// Take into account JobNameOriginal in order to capture the case when the original job_name is changed via relabeling. // Take into account JobNameOriginal in order to capture the case when the original job_name is changed via relabeling.
key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+ key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, "+
"ExternalLabels=%s, "+ "HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, ExternalLabels=%s, MaxScrapeSize=%d, "+
"ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%q, "+ "ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%q, "+
"SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+ "SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+
"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v", "ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v",
sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(), sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels,
sw.ExternalLabels.String(), sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(), sw.ExternalLabels.String(), sw.MaxScrapeSize,
sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(),
sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse,
sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers) sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers)