From bc37b279aadac7455ec603c640c1c763c15bb768 Mon Sep 17 00:00:00 2001 From: Hui Wang Date: Thu, 20 Jun 2024 19:20:40 +0800 Subject: [PATCH] =?UTF-8?q?vmalert:=20exit=20replay=20mode=20with=20non-ze?= =?UTF-8?q?ro=20code=20if=20generated=20samples=20are=E2=80=A6=20(#6513)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … not successfully written into remoteWrite url address https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6512 --- app/vmalert/main.go | 1 + app/vmalert/remotewrite/client.go | 5 +++++ app/vmalert/remotewrite/debug_client.go | 5 +++++ app/vmalert/remotewrite/remotewrite.go | 2 ++ app/vmalert/replay.go | 9 +++++---- app/vmalert/replay_test.go | 3 ++- docs/CHANGELOG.md | 1 + 7 files changed, 21 insertions(+), 5 deletions(-) diff --git a/app/vmalert/main.go b/app/vmalert/main.go index f7ff36ef4..f6e05a00f 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -153,6 +153,7 @@ func main() { if err := replay(groupsCfg, q, rw); err != nil { logger.Fatalf("replay failed: %s", err) } + logger.Infof("replay succeed!") return } diff --git a/app/vmalert/remotewrite/client.go b/app/vmalert/remotewrite/client.go index d3ce6b3df..1ddec30e2 100644 --- a/app/vmalert/remotewrite/client.go +++ b/app/vmalert/remotewrite/client.go @@ -148,6 +148,11 @@ func (c *Client) Close() error { return nil } +// CloseAndReport closes the client and report dropped rows +func (c *Client) CloseAndReport() (int64, error) { + return int64(droppedRows.Get()), c.Close() +} + func (c *Client) run(ctx context.Context) { ticker := time.NewTicker(c.flushInterval) wr := &prompbmarshal.WriteRequest{} diff --git a/app/vmalert/remotewrite/debug_client.go b/app/vmalert/remotewrite/debug_client.go index 17443c731..52e0237d8 100644 --- a/app/vmalert/remotewrite/debug_client.go +++ b/app/vmalert/remotewrite/debug_client.go @@ -60,6 +60,11 @@ func (c *DebugClient) Close() error { return nil } +// CloseAndReport closes the client and reports dropped rows +func (c *DebugClient) CloseAndReport() (int64, error) { + return 0, c.Close() +} + func (c *DebugClient) send(data []byte) error { b := snappy.Encode(nil, data) r := bytes.NewReader(b) diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go index 27d505e48..957acd804 100644 --- a/app/vmalert/remotewrite/remotewrite.go +++ b/app/vmalert/remotewrite/remotewrite.go @@ -10,4 +10,6 @@ type RWClient interface { Push(s prompbmarshal.TimeSeries) error // Close stops the client. Client can't be reused after Close call. Close() error + // CloseAndReport stops the client and reports dropped rows + CloseAndReport() (int64, error) } diff --git a/app/vmalert/replay.go b/app/vmalert/replay.go index ca85a72f9..0679fd415 100644 --- a/app/vmalert/replay.go +++ b/app/vmalert/replay.go @@ -68,9 +68,10 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels) total += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar) } - logger.Infof("replay finished! Imported %d samples", total) - if rw != nil { - return rw.Close() + logger.Infof("replay evaluation finished, generated %d samples", total) + droppedRows, err := rw.CloseAndReport() + if err == nil && droppedRows != 0 { + return fmt.Errorf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, total) } - return nil + return err } diff --git a/app/vmalert/replay_test.go b/app/vmalert/replay_test.go index 4379c05c1..f4b0e504f 100644 --- a/app/vmalert/replay_test.go +++ b/app/vmalert/replay_test.go @@ -8,6 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" ) @@ -161,7 +162,7 @@ func TestReplay(t *testing.T) { *replayFrom = tc.from *replayTo = tc.to *replayMaxDatapoints = tc.maxDP - if err := replay(tc.cfg, tc.qb, nil); err != nil { + if err := replay(tc.cfg, tc.qb, &remotewrite.DebugClient{}); err != nil { t.Fatalf("replay failed: %s", err) } if len(tc.qb.registry) > 0 { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 75289344f..1b6e4a879 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -49,6 +49,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): add `--disable-progress-bar` global command-line flag. It can be used for disabling dynamic progress bar for all migration modes. `--vm-disable-progress-bar` command-line flag is deprecated and will be removed in the future releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6367). * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): prevent [rate_sum](https://docs.victoriametrics.com/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/stream-aggregation/#rate_avg) producing `NaN` results for stale time series. Before, when series matched for aggregation became stale or weren't updated during aggregation interval, the `rate_sum` or `rate_avg` could produce data point with `NaN` value. During visualization, such aggregation results would be displayed as gaps in time series. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): fix path for system links printed on default vmalert's UI page when `-http.pathPrefix` is set. +* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): exit replay mode with non-zero code if generated samples are not successfully written into remoteWrite url. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6512). * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/): properly configure authentication with S3 when `-configFilePath` cmd-line flag is specified. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/) enterprise: properly configure authentication with S3 when `-s3.configFilePath` cmd-line flag is specified for reading rule configs. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): properly specify oauth2 `ClientSecret` when configuring authentication for `notifier.url`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6471) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6478).