From 72edc31ffb75c38fc07b0831cd9dbcdabe8c9ec9 Mon Sep 17 00:00:00 2001
From: Roman Khavronenko <roman@victoriametrics.com>
Date: Tue, 27 Jun 2023 20:15:17 +0200
Subject: [PATCH] vmauth: expose latency metrics per user (#4525)
expose `vmauth_user_request_duration_seconds`
and `vmauth_unauthorized_user_request_duration_seconds` summary metrics
for measuring requests latency per user.
Signed-off-by: hagen1778 <roman@victoriametrics.com>
---
app/vmauth/README.md | 12 ++++++++++--
app/vmauth/auth_config.go | 6 +++++-
app/vmauth/main.go | 3 +++
docs/CHANGELOG.md | 1 +
docs/vmauth.md | 12 ++++++++++--
5 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/app/vmauth/README.md b/app/vmauth/README.md
index 30782cf53e..e62757d844 100644
--- a/app/vmauth/README.md
+++ b/app/vmauth/README.md
@@ -251,7 +251,12 @@ It is recommended protecting the following endpoints with authKeys:
`vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
-`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric
+and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric
+with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file.
+It is possible to override or hide the value in the label by specifying `name` field.
+For example, the following config will result in `vmauth_user_requests_total{username="foobar"}`
+instead of `vmauth_user_requests_total{username="secret_user"}`:
```yml
users:
@@ -260,7 +265,10 @@ users:
# other config options here
```
-For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` metric without label (if `unauthorized_user` section of config is used).
+For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total`
+[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and
+`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary)
+metric without label (if `unauthorized_user` section of config is used).
## How to build from sources
diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go
index 8e9dad4673..3c7ce00ed6 100644
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@@ -50,7 +50,8 @@ type UserInfo struct {
concurrencyLimitCh chan struct{}
concurrencyLimitReached *metrics.Counter
- requests *metrics.Counter
+ requests *metrics.Counter
+ requestsDuration *metrics.Summary
}
func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -378,6 +379,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
ui := ac.UnauthorizedUser
if ui != nil {
ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
+ ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
@@ -441,9 +443,11 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
}
ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+ ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
}
if ui.Username != "" {
ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+ ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
}
mcr := ui.getMaxConcurrentRequests()
ui.concurrencyLimitCh = make(chan struct{}, mcr)
diff --git a/app/vmauth/main.go b/app/vmauth/main.go
index ceae82a1f1..c859e913cf 100644
--- a/app/vmauth/main.go
+++ b/app/vmauth/main.go
@@ -123,6 +123,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
}
func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
+ startTime := time.Now()
+ defer ui.requestsDuration.UpdateDuration(startTime)
+
ui.requests.Inc()
// Limit the concurrency of requests to backends
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5560b3efcc..91b5daa460 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -35,6 +35,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: accept timestamps in milliseconds at `start`, `end` and `time` query args in [Prometheus querying API](https://docs.victoriametrics.com/#prometheus-querying-api-usage). See [these docs](https://docs.victoriametrics.com/#timestamp-formats) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4459).
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): update retry policy for pushing data to `-remoteWrite.url`. By default, vmalert will make multiple retry attempts with exponential delay. The total time spent during retry attempts shouldn't exceed `-remoteWrite.retryMaxTime` (default is 30s). When retry time is exceeded vmalert drops the data dedicated for `-remoteWrite.url`. Before, vmalert dropped data after 5 retry attempts with 1s delay between attempts (not configurable). See `-remoteWrite.retryMinInterval` and `-remoteWrite.retryMaxTime` cmd-line flags.
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): expose `vmalert_remotewrite_send_duration_seconds_total` counter, which can be used for determining high saturation of every connection to remote storage with an alerting query `sum(rate(vmalert_remotewrite_send_duration_seconds_total[5m])) by(job, instance) > 0.9 * max(vmalert_remotewrite_concurrency) by(job, instance)`. This query triggers when a connection is saturated by more than 90%. This usually means that `-remoteWrite.concurrency` command-line flag must be increased in order to increase the number of concurrent writings into remote endpoint. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4516).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_duration_seconds` and `vmauth_unauthorized_user_request_duration_seconds` summary metrics for measuring requests latency per user.
* BUGFIX: add the following command-line flags, which can be used for limiting Graphite API calls:
`--search.maxGraphiteTagKeys` for limiting the number of tag keys returned from Graphite `/tags`, `/tags/autoComplete/*`, `/tags/findSeries` API.
diff --git a/docs/vmauth.md b/docs/vmauth.md
index a4adb5f455..bfb9d19f0c 100644
--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@@ -262,7 +262,12 @@ It is recommended protecting the following endpoints with authKeys:
`vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
-`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric
+and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric
+with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file.
+It is possible to override or hide the value in the label by specifying `name` field.
+For example, the following config will result in `vmauth_user_requests_total{username="foobar"}`
+instead of `vmauth_user_requests_total{username="secret_user"}`:
```yml
users:
@@ -271,7 +276,10 @@ users:
# other config options here
```
-For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` metric without label (if `unauthorized_user` section of config is used).
+For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total`
+[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and
+`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary)
+metric without label (if `unauthorized_user` section of config is used).
## How to build from sources