From 3b1c2f03c3be3082ceb2e04d3d9beb9f80eb909b Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sun, 28 Feb 2021 19:40:16 +0200
Subject: [PATCH] app/vmselect/querystats: show the number of matching queries
 in the top by average duration and in the top by summary duration

This should help debugging slow queries.
---
 app/vmselect/querystats/querystats.go | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/app/vmselect/querystats/querystats.go b/app/vmselect/querystats/querystats.go
index acc050974..9478faad1 100644
--- a/app/vmselect/querystats/querystats.go
+++ b/app/vmselect/querystats/querystats.go
@@ -108,8 +108,8 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apFilte
 	fmt.Fprintf(w, `],"topByAvgDuration":[`)
 	topByAvgDuration := qst.getTopByAvgDuration(topN, apFilter, maxLifetime)
 	for i, r := range topByAvgDuration {
-		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%q,"timeRangeSeconds":%d,"avgDurationSeconds":%.3f}`,
-			r.accountID, r.projectID, r.query, r.timeRangeSecs, r.duration.Seconds())
+		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%q,"timeRangeSeconds":%d,"avgDurationSeconds":%.3f,"count":%d}`,
+			r.accountID, r.projectID, r.query, r.timeRangeSecs, r.duration.Seconds(), r.count)
 		if i+1 < len(topByAvgDuration) {
 			fmt.Fprintf(w, `,`)
 		}
@@ -117,8 +117,8 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apFilte
 	fmt.Fprintf(w, `],"topBySumDuration":[`)
 	topBySumDuration := qst.getTopBySumDuration(topN, apFilter, maxLifetime)
 	for i, r := range topBySumDuration {
-		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%q,"timeRangeSeconds":%d,"sumDurationSeconds":%.3f}`,
-			r.accountID, r.projectID, r.query, r.timeRangeSecs, r.duration.Seconds())
+		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%q,"timeRangeSeconds":%d,"sumDurationSeconds":%.3f,"count":%d}`,
+			r.accountID, r.projectID, r.query, r.timeRangeSecs, r.duration.Seconds(), r.count)
 		if i+1 < len(topBySumDuration) {
 			fmt.Fprintf(w, `,`)
 		}
@@ -236,6 +236,7 @@ func (qst *queryStatsTracker) getTopByAvgDuration(topN int, apFilter *accountPro
 			query:         k.query,
 			timeRangeSecs: k.timeRangeSecs,
 			duration:      ks.sum / time.Duration(ks.count),
+			count:         ks.count,
 		})
 	}
 	sort.Slice(a, func(i, j int) bool {
@@ -253,28 +254,37 @@ type queryStatByDuration struct {
 	query         string
 	timeRangeSecs int64
 	duration      time.Duration
+	count         int
 }
 
 func (qst *queryStatsTracker) getTopBySumDuration(topN int, apFilter *accountProjectFilter, maxLifetime time.Duration) []queryStatByDuration {
 	currentTime := time.Now()
 	qst.mu.Lock()
-	m := make(map[queryStatKey]time.Duration)
+	type countDuration struct {
+		count int
+		sum   time.Duration
+	}
+	m := make(map[queryStatKey]countDuration)
 	for _, r := range qst.a {
 		if r.matches(apFilter, currentTime, maxLifetime) {
 			k := r.key()
-			m[k] = m[k] + r.duration
+			kd := m[k]
+			kd.count++
+			kd.sum += r.duration
+			m[k] = kd
 		}
 	}
 	qst.mu.Unlock()
 
 	var a []queryStatByDuration
-	for k, d := range m {
+	for k, kd := range m {
 		a = append(a, queryStatByDuration{
 			accountID:     k.accountID,
 			projectID:     k.projectID,
 			query:         k.query,
 			timeRangeSecs: k.timeRangeSecs,
-			duration:      d,
+			duration:      kd.sum,
+			count:         kd.count,
 		})
 	}
 	sort.Slice(a, func(i, j int) bool {