vmalert: exit replay mode with non-zero code if generated samples are… (#6513)

… not successfully written into remoteWrite url

address https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6512

app/vmalert/main.go

@@ -153,6 +153,7 @@ func main() {
 		if err := replay(groupsCfg, q, rw); err != nil {
 			logger.Fatalf("replay failed: %s", err)
 		}
+		logger.Infof("replay succeed!")
 		return
 	}
 

app/vmalert/remotewrite/client.go

@@ -148,6 +148,11 @@ func (c *Client) Close() error {
 	return nil
 }
 
+// CloseAndReport closes the client and report dropped rows
+func (c *Client) CloseAndReport() (int64, error) {
+	return int64(droppedRows.Get()), c.Close()
+}
+
 func (c *Client) run(ctx context.Context) {
 	ticker := time.NewTicker(c.flushInterval)
 	wr := &prompbmarshal.WriteRequest{}

app/vmalert/remotewrite/debug_client.go

@@ -60,6 +60,11 @@ func (c *DebugClient) Close() error {
 	return nil
 }
 
+// CloseAndReport closes the client and reports dropped rows
+func (c *DebugClient) CloseAndReport() (int64, error) {
+	return 0, c.Close()
+}
+
 func (c *DebugClient) send(data []byte) error {
 	b := snappy.Encode(nil, data)
 	r := bytes.NewReader(b)

app/vmalert/remotewrite/remotewrite.go

@@ -10,4 +10,6 @@ type RWClient interface {
 	Push(s prompbmarshal.TimeSeries) error
 	// Close stops the client. Client can't be reused after Close call.
 	Close() error
+	// CloseAndReport stops the client and reports dropped rows
+	CloseAndReport() (int64, error)
 }

app/vmalert/replay.go

@@ -68,9 +68,10 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
 		ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
 		total += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar)
 	}
-	logger.Infof("replay finished! Imported %d samples", total)
-	if rw != nil {
-		return rw.Close()
+	logger.Infof("replay evaluation finished, generated %d samples", total)
+	droppedRows, err := rw.CloseAndReport()
+	if err == nil && droppedRows != 0 {
+		return fmt.Errorf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, total)
 	}
-	return nil
+	return err
 }

app/vmalert/replay_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -161,7 +162,7 @@ func TestReplay(t *testing.T) {
 			*replayFrom = tc.from
 			*replayTo = tc.to
 			*replayMaxDatapoints = tc.maxDP
-			if err := replay(tc.cfg, tc.qb, nil); err != nil {
+			if err := replay(tc.cfg, tc.qb, &remotewrite.DebugClient{}); err != nil {
 				t.Fatalf("replay failed: %s", err)
 			}
 			if len(tc.qb.registry) > 0 {

docs/CHANGELOG.md

@@ -49,6 +49,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): add `--disable-progress-bar` global command-line flag. It can be used for disabling dynamic progress bar for all migration modes. `--vm-disable-progress-bar`  command-line flag is deprecated and will be removed in the future releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6367).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): prevent [rate_sum](https://docs.victoriametrics.com/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/stream-aggregation/#rate_avg) producing `NaN` results for stale time series. Before, when series matched for aggregation became stale or weren't updated during aggregation interval, the `rate_sum` or `rate_avg` could produce data point with `NaN` value. During visualization, such aggregation results would be displayed as gaps in time series.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): fix path for system links printed on default vmalert's UI page when `-http.pathPrefix` is set.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): exit replay mode with non-zero code if generated samples are not successfully written into remoteWrite url. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6512).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/): properly configure authentication with S3 when `-configFilePath` cmd-line flag is specified.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/) enterprise: properly configure authentication with S3 when `-s3.configFilePath` cmd-line flag is specified for reading rule configs.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): properly specify oauth2 `ClientSecret` when configuring authentication for `notifier.url`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6471) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6478).