app/vmalert: add vmalert_alerts_send_duration_seconds metric (#8468)

### Describe Your Changes

Add `vmalert_alerts_send_latency_seconds` metric for
alertmanager.notifier.

To measure the time for alertmanager calls to send alerts per notifier.
This is needed to see the latency for each notifier from vmalert calls.

### Checklist

The following checks are **mandatory**:

- [x] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).

---------

Signed-off-by: emreya <e.yazici1990@gmail.com>
Co-authored-by: Hui Wang <haley@victoriametrics.com>
Co-authored-by: Roman Khavronenko <hagen1778@gmail.com>

app/vmalert/notifier/alertmanager.go

@@ -37,8 +37,9 @@ type AlertManager struct {
 type notifierMetrics struct {
 	set *metrics.Set
 
-	alertsSent       *metrics.Counter
-	alertsSendErrors *metrics.Counter
+	alertsSent         *metrics.Counter
+	alertsSendErrors   *metrics.Counter
+	alertsSendDuration *metrics.Histogram
 }
 
 func newNotifierMetrics(addr string) *notifierMetrics {
@@ -46,9 +47,10 @@ func newNotifierMetrics(addr string) *notifierMetrics {
 	metrics.RegisterSet(set)
 
 	return &notifierMetrics{
-		set:              set,
-		alertsSent:       set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
-		alertsSendErrors: set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+		set:                set,
+		alertsSent:         set.NewCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors:   set.NewCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+		alertsSendDuration: set.NewHistogram(fmt.Sprintf("vmalert_alerts_send_duration_seconds{addr=%q}", addr)),
 	}
 }
 
@@ -72,7 +74,9 @@ func (am AlertManager) Addr() string {
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	am.metrics.alertsSent.Add(len(alerts))
+	startTime := time.Now()
 	err := am.send(ctx, alerts, headers)
+	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
 		am.metrics.alertsSendErrors.Add(len(alerts))
 	}

docs/changelog/CHANGELOG.md

@@ -21,6 +21,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * SECURITY: upgrade Go builder from Go1.24.0 to Go1.24.1. See the list of issues addressed in [Go1.24.1](https://github.com/golang/go/issues?q=milestone%3AGo1.24.1+label%3ACherryPickApproved).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/): use original unmodified by [relabeling](https://docs.victoriametrics.com/#relabeling) job name as `scrape_pool` value for [/api/v1/targets](https://docs.victoriametrics.com/#prometheus-querying-api-usage) responses. This change fixes discrepancy with Prometheus as mentioned in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5343). Thanks to @evkuzin for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8457).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): add command-line flag `-search.maxTSDBStatusTopNSeries` to allow setting a limit for topN argument accepted by `/api/v1/status/tsdb` endpoint.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): expose `vmalert_alerts_send_duration_seconds` metric to measure the time taken to send alerts to the specified `-notifier.url`. Thanks to @eyazici90 for [the pull reuqest](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8468).
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation): fix panic on `rate` output. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8469).