From 76af32d8699ff7fe68c3094cabe3c63c7091ca4c Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Mon, 18 Sep 2023 17:06:37 +0200
Subject: [PATCH] lib/promscrape/discovery/kubernetes: follow-up after
eeb862f3ffc91585c48e5b358807a88d958328d0
- Move the bugfix description to the correct place in docs/CHANGELOG.md
- Prevent from logging of 'context canceled' errors after the url watcher is stopped,
since these errors are expected and may confuse users.
- Remove unused urlWatcher.refCount field.
- Remove unused urlWatcher.close() method.
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4850
---
docs/CHANGELOG.md | 2 +-
.../discovery/kubernetes/api_watcher.go | 23 +++++++++----------
2 files changed, 12 insertions(+), 13 deletions(-)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 9649e89632..3a2dc9bd35 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -51,6 +51,7 @@ ssue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4825) and [these
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the bug causing render looping when switching to heatmap.
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): properly copy `parts.json` files inside `<-storageDataPath>/{data,indexdb}` folders during [incremental backups](https://docs.victoriametrics.com/vmbackup.html#incremental-backups). Previously the new `parts.json` could be skipped during incremental backups, which could lead to inability to restore from the backup. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5005). This issue has been introduced in [v1.90.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.90.0).
* BUGFIX: [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html) validate `-dedup.minScrapeInterval` value and `-downsampling.period` intervals are multiples of each other. See [these docs](https://docs.victoriametrics.com/#downsampling).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly close connections to Kubernetes API server after the change in `selectors` or `namespaces` sections of [kubernetes_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs). Previously `vmagent` could continue polling Kubernetes API server with the old `selectors` or `namespaces` configs additionally to polling new configs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4850).
## [v1.93.4](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.4)
@@ -105,7 +106,6 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
* BUGFIX: properly replace `:` chars in label names with `_` when `-usePromCompatibleNaming` command-line flag is passed to `vmagent`, `vminsert` or single-node VictoriaMetrics. This addresses [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3113#issuecomment-1275077071).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): correctly check if specified `-dst` belongs to specified `-storageDataPath`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4837).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): don't interrupt the migration process if no metrics were found for a specific tenant. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4796).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix the thread leak that occurs during the polling of the Kubernetes API server after a configuration reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4850).
## [v1.93.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.0)
diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go
index 60f91cea4b..d49795942c 100644
--- a/lib/promscrape/discovery/kubernetes/api_watcher.go
+++ b/lib/promscrape/discovery/kubernetes/api_watcher.go
@@ -504,9 +504,8 @@ func newURLWatcher(role, apiURL string, gw *groupWatcher) *urlWatcher {
apiURL: apiURL,
gw: gw,
- refCount: 0,
- ctx: ctx,
- cancel: cancel,
+ ctx: ctx,
+ cancel: cancel,
parseObject: parseObject,
parseObjectList: parseObjectList,
@@ -604,7 +603,9 @@ func (uw *urlWatcher) reloadObjects() string {
requestURL := apiURL + delimiter + "resourceVersion=0&resourceVersionMatch=NotOlderThan"
resp, err := uw.gw.doRequest(uw.ctx, requestURL)
if err != nil {
- logger.Errorf("cannot perform request to %q: %s", requestURL, err)
+ if !errors.Is(err, context.Canceled) {
+ logger.Errorf("cannot perform request to %q: %s", requestURL, err)
+ }
return ""
}
if resp.StatusCode != http.StatusOK {
@@ -681,9 +682,10 @@ func (uw *urlWatcher) watchForUpdates() {
delimiter := getQueryArgsDelimiter(apiURL)
timeoutSeconds := time.Duration(0.9 * float64(uw.gw.client.Timeout)).Seconds()
apiURL += delimiter + "watch=1&allowWatchBookmarks=true&timeoutSeconds=" + strconv.Itoa(int(timeoutSeconds))
+ stopCh := uw.ctx.Done()
for {
select {
- case <-uw.ctx.Done():
+ case <-stopCh:
return
default:
}
@@ -696,8 +698,10 @@ func (uw *urlWatcher) watchForUpdates() {
requestURL := apiURL + "&resourceVersion=" + url.QueryEscape(resourceVersion)
resp, err := uw.gw.doRequest(uw.ctx, requestURL)
if err != nil {
- logger.Errorf("cannot perform request to %q: %s", requestURL, err)
- backoffSleep()
+ if !errors.Is(err, context.Canceled) {
+ logger.Errorf("cannot perform request to %q: %s", requestURL, err)
+ backoffSleep()
+ }
continue
}
if resp.StatusCode != http.StatusOK {
@@ -844,11 +848,6 @@ func (uw *urlWatcher) maybeUpdateDependedScrapeWorksLocked() {
}
}
-// close cancels context used for API polling
-func (uw *urlWatcher) close() {
- uw.cancel()
-}
-
// Bookmark is a bookmark message from Kubernetes Watch API.
// See https://kubernetes.io/docs/reference/using-api/api-concepts/#watch-bookmarks
type Bookmark struct {