diff --git a/app/vmauth/README.md b/app/vmauth/README.md index 30782cf53..e62757d84 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -251,7 +251,12 @@ It is recommended protecting the following endpoints with authKeys: `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later. -`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`: +`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric +and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric +with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. +It is possible to override or hide the value in the label by specifying `name` field. +For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` +instead of `vmauth_user_requests_total{username="secret_user"}`: ```yml users: @@ -260,7 +265,10 @@ users: # other config options here ``` -For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` metric without label (if `unauthorized_user` section of config is used). +For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` +[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and +`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) +metric without label (if `unauthorized_user` section of config is used). ## How to build from sources diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 8e9dad467..3c7ce00ed 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -50,7 +50,8 @@ type UserInfo struct { concurrencyLimitCh chan struct{} concurrencyLimitReached *metrics.Counter - requests *metrics.Counter + requests *metrics.Counter + requestsDuration *metrics.Summary } func (ui *UserInfo) beginConcurrencyLimit() error { @@ -378,6 +379,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) { ui := ac.UnauthorizedUser if ui != nil { ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`) + ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`) ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests()) ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`) _ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 { @@ -441,9 +443,11 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) { return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken) } ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name)) + ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name)) } if ui.Username != "" { ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name)) + ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name)) } mcr := ui.getMaxConcurrentRequests() ui.concurrencyLimitCh = make(chan struct{}, mcr) diff --git a/app/vmauth/main.go b/app/vmauth/main.go index ceae82a1f..c859e913c 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -123,6 +123,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { } func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) { + startTime := time.Now() + defer ui.requestsDuration.UpdateDuration(startTime) + ui.requests.Inc() // Limit the concurrency of requests to backends diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 44aa0b0cf..d5fcd1077 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,6 +34,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: accept timestamps in milliseconds at `start`, `end` and `time` query args in [Prometheus querying API](https://docs.victoriametrics.com/#prometheus-querying-api-usage). See [these docs](https://docs.victoriametrics.com/#timestamp-formats) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4459). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): update retry policy for pushing data to `-remoteWrite.url`. By default, vmalert will make multiple retry attempts with exponential delay. The total time spent during retry attempts shouldn't exceed `-remoteWrite.retryMaxTime` (default is 30s). When retry time is exceeded vmalert drops the data dedicated for `-remoteWrite.url`. Before, vmalert dropped data after 5 retry attempts with 1s delay between attempts (not configurable). See `-remoteWrite.retryMinInterval` and `-remoteWrite.retryMaxTime` cmd-line flags. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): expose `vmalert_remotewrite_send_duration_seconds_total` counter, which can be used for determining high saturation of every connection to remote storage with an alerting query `sum(rate(vmalert_remotewrite_send_duration_seconds_total[5m])) by(job, instance) > 0.9 * max(vmalert_remotewrite_concurrency) by(job, instance)`. This query triggers when a connection is saturated by more than 90%. This usually means that `-remoteWrite.concurrency` command-line flag must be increased in order to increase the number of concurrent writings into remote endpoint. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4516). +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_duration_seconds` and `vmauth_unauthorized_user_request_duration_seconds` summary metrics for measuring requests latency per user. * FEATURE: add ability to fine-tune Graphite API limits via the following command-line flags: `-search.maxGraphiteTagKeys` for limiting the number of tag keys returned from [Graphite API for tags](https://docs.victoriametrics.com/#graphite-tags-api-usage) `-search.maxGraphiteTagValues` for limiting the number of tag values returned from [Graphite API for tag values](https://docs.victoriametrics.com/#graphite-tags-api-usage) diff --git a/docs/vmauth.md b/docs/vmauth.md index a4adb5f45..bfb9d19f0 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -262,7 +262,12 @@ It is recommended protecting the following endpoints with authKeys: `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later. -`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`: +`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric +and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric +with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. +It is possible to override or hide the value in the label by specifying `name` field. +For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` +instead of `vmauth_user_requests_total{username="secret_user"}`: ```yml users: @@ -271,7 +276,10 @@ users: # other config options here ``` -For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` metric without label (if `unauthorized_user` section of config is used). +For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` +[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and +`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) +metric without label (if `unauthorized_user` section of config is used). ## How to build from sources