From 7bc728bf533dc96319ba1e05434b745bb4f0f919 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Wed, 17 Mar 2021 01:12:28 +0200
Subject: [PATCH] app/vmselect: add `vm_index_search_duration_seconds`
 histogram for monitoring the performance of index search

---
 app/vmselect/netstorage/netstorage.go | 7 +++++++
 docs/CHANGELOG.md                     | 1 +
 2 files changed, 8 insertions(+)

diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go
index 35122909a..8f1d35374 100644
--- a/app/vmselect/netstorage/netstorage.go
+++ b/app/vmselect/netstorage/netstorage.go
@@ -9,6 +9,7 @@ import (
 	"sort"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -749,7 +750,9 @@ func ExportBlocks(sq *storage.SearchQuery, deadline searchutils.Deadline, f func
 
 	sr := getStorageSearch()
 	defer putStorageSearch(sr)
+	startTime := time.Now()
 	sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
+	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
 	gomaxprocs := cgroup.AvailableCPUs()
@@ -882,7 +885,9 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
+	startTime := time.Now()
 	maxSeriesCount := sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch, deadline.Deadline())
+	indexSearchDuration.UpdateDuration(startTime)
 	m := make(map[string][]blockRef, maxSeriesCount)
 	orderedMetricNames := make([]string, 0, maxSeriesCount)
 	blocksRead := 0
@@ -950,6 +955,8 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline search
 	return &rss, nil
 }
 
+var indexSearchDuration = metrics.NewHistogram(`vm_index_search_duration_seconds`)
+
 type blockRef struct {
 	partRef storage.PartRef
 	addr    tmpBlockAddr
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 937a4fa2d..b2149650d 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -8,6 +8,7 @@
   - `histogram_stddev(buckets)` - returns standard deviation for the given buckets.
 * FEATURE: reduce median query duration by up to 2x. See https://github.com/VictoriaMetrics/VictoriaMetrics/commit/18fe0ff14bc78860c5569e2b70de1db78fac61be
 * FEATURE: export `vm_available_memory_bytes` and `vm_available_cpu_cores` metrics, which show the number of available RAM and available CPU cores for VictoriaMetrics apps.
+* FEATURE: export `vm_index_search_duration_seconds` histogram, which can be used for troubleshooting time series search performance.
 * FEATURE: vmagent: add ability to replicate scrape targets among `vmagent` instances in the cluster with `-promscrape.cluster.replicationFactor` command-line flag. See [these docs](https://victoriametrics.github.io/vmagent.html#scraping-big-number-of-targets).
 * FEATURE: vmagent: accept `scrape_offset` option at `scrape_config`. This option may be useful when scrapes must start at the specified offset of every scrape interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details.
 * FEATURE: vmagent: support `proxy_tls_config`, `proxy_basic_auth`, `proxy_bearer_token` and `proxy_bearer_token_file` options at `scrape_config` section for configuring proxies specified via `proxy_url`. See [these docs](https://victoriametrics.github.io/vmagent.html#scraping-targets-via-a-proxy).