From 9c19719ad6b7736173a21dbf2e87971f68280435 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 7 Jul 2021 13:25:16 +0300
Subject: [PATCH] app/{vminsert,vmselect}: export
 vminsert_request_duration_seconds and vmselect_request_duration_seconds
 histograms

---
 app/vminsert/main.go | 5 +++++
 app/vmselect/main.go | 3 +++
 docs/CHANGELOG.md    | 1 +
 3 files changed, 9 insertions(+)

diff --git a/app/vminsert/main.go b/app/vminsert/main.go
index 774fd8aba..a1b184be5 100644
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -154,6 +154,9 @@ func main() {
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	startTime := time.Now()
+	defer requestDuration.UpdateDuration(startTime)
+
 	if r.URL.Path == "/" {
 		if r.Method != "GET" {
 			return false
@@ -242,6 +245,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 var (
+	requestDuration = metrics.NewHistogram(`vminsert_request_duration_seconds`)
+
 	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/prometheus/", protocol="promremotewrite"}`)
 	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/prometheus/", protocol="promremotewrite"}`)
 
diff --git a/app/vmselect/main.go b/app/vmselect/main.go
index ce47cb188..7306785f8 100644
--- a/app/vmselect/main.go
+++ b/app/vmselect/main.go
@@ -137,6 +137,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	startTime := time.Now()
+	defer requestDuration.UpdateDuration(startTime)
 	// Limit the number of concurrent queries.
 	select {
 	case concurrencyCh <- struct{}{}:
@@ -516,6 +517,8 @@ func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 }
 
 var (
+	requestDuration = metrics.NewHistogram(`vmselect_request_duration_seconds`)
+
 	labelValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/label/{}/values"}`)
 	labelValuesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/label/{}/values"}`)
 
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 2aba59349..51a24c20b 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -9,6 +9,7 @@ sort: 15
 * FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
 * FEATURE: reduce memory usage when performing heavy queries over big number of time series.
 * FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors.
+* FEATURE: export `vmselect_request_duration_seconds` and `vminsert_request_duration_seconds` [VictoriaMetrics histograms](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) at `/metrics` page. These histograms can be used for determining latency distribution for the served requests.
 
 * BUGFIX: vmagent: remove `{ %space %}` typo in `/targets` output. The typo has been introduced in v1.62.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1408).
 * BUGFIX: vmagent: fix CSS styles on `/targets` page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1422).