app/vmselect: export per-tenant stats on the number of requests and the cumulative request duration

The metrics are:
- vm_vmselect_http_requests_total{accountID="...",projectID="..."} - the total number of select requests per each tenant
- vm_vmselect_http_duration_ms_total{accountID="...",projectID="..."} - the total duration in milliseconds for per-tenant select requests

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/932

app/vmselect/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -203,6 +204,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request, p *httpserver.Path, at *auth.Token) bool {
+	defer func() {
+		// Count per-tenant cumulative durations and total requests
+		httpRequests.Get(at).Inc()
+		httpRequestsDuration.Get(at).Add(int(time.Since(startTime).Milliseconds()))
+	}()
 	if strings.HasPrefix(p.Suffix, "prometheus/api/v1/label/") {
 		s := p.Suffix[len("prometheus/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
@@ -568,6 +574,9 @@ var (
 	rulesRequests    = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/rules"}`)
 	alertsRequests   = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/alerts"}`)
 	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/metadata"}`)
+
+	httpRequests         = tenantmetrics.NewCounterMap(`vm_vmselect_http_requests_total`)
+	httpRequestsDuration = tenantmetrics.NewCounterMap(`vm_vmselect_http_requests_duration_ms_total`)
 )
 
 func usage() {

lib/tenantmetrics/counter_map.go

@@ -9,9 +9,10 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-type apKey struct {
-	accountID uint32
-	projectID uint32
+// TenantID defines metric tenant.
+type TenantID struct {
+	AccountID uint32
+	ProjectID uint32
 }
 
 // CounterMap is a map of counters keyed by tenant.
@@ -20,50 +21,52 @@ type CounterMap struct {
 	m      atomic.Value
 }
 
-// NewCounterMap creates new CounterMap for the given metricTemplate.
-//
+// NewCounterMap creates new CounterMap for the given metric.
 func NewCounterMap(metric string) *CounterMap {
 	cm := &CounterMap{
 		metric: metric,
 	}
-	cm.m.Store(make(map[apKey]*metrics.Counter))
+	cm.m.Store(make(map[TenantID]*metrics.Counter))
 	return cm
 }
 
-// Get returns counter for the given at.
-//
-// It always returns non-nil counter.
+// Get returns counter for the given at
 func (cm *CounterMap) Get(at *auth.Token) *metrics.Counter {
-	key := apKey{
-		accountID: at.AccountID,
-		projectID: at.ProjectID,
+	key := TenantID{
+		AccountID: at.AccountID,
+		ProjectID: at.ProjectID,
 	}
-	m := cm.m.Load().(map[apKey]*metrics.Counter)
+	return cm.GetByTenant(key)
+}
+
+// GetByTenant returns counter for the given key.
+func (cm *CounterMap) GetByTenant(key TenantID) *metrics.Counter {
+	m := cm.m.Load().(map[TenantID]*metrics.Counter)
 	if c := m[key]; c != nil {
 		// Fast path - the counter for k already exists.
 		return c
 	}
 
 	// Slow path - create missing counter for k and re-create m.
-	newM := make(map[apKey]*metrics.Counter, len(m)+1)
+	newM := make(map[TenantID]*metrics.Counter, len(m)+1)
 	for k, c := range m {
 		newM[k] = c
 	}
-	metricName := createMetricName(cm.metric, at)
+	metricName := createMetricName(cm.metric, key)
 	c := metrics.GetOrCreateCounter(metricName)
 	newM[key] = c
 	cm.m.Store(newM)
 	return c
 }
 
-func createMetricName(metric string, at *auth.Token) string {
+func createMetricName(metric string, key TenantID) string {
 	if len(metric) == 0 {
 		logger.Panicf("BUG: metric cannot be empty")
 	}
 	if metric[len(metric)-1] != '}' {
 		// Metric without labels.
-		return fmt.Sprintf(`%s{accountID="%d",projectID="%d"}`, metric, at.AccountID, at.ProjectID)
+		return fmt.Sprintf(`%s{accountID="%d",projectID="%d"}`, metric, key.AccountID, key.ProjectID)
 	}
 	// Metric with labels.
-	return fmt.Sprintf(`%s,accountID="%d",projectID="%d"}`, metric[:len(metric)-1], at.AccountID, at.ProjectID)
+	return fmt.Sprintf(`%s,accountID="%d",projectID="%d"}`, metric[:len(metric)-1], key.AccountID, key.ProjectID)
 }

lib/tenantmetrics/counter_map_test.go

@@ -14,13 +14,16 @@ func TestCreateMetricNameError(t *testing.T) {
 			t.Fatal("expecting non-nil panic")
 		}
 	}()
-	_ = createMetricName("", &auth.Token{})
+	_ = createMetricName("", TenantID{})
 }
 
 func TestCreateMetricNameSuccess(t *testing.T) {
 	f := func(s string, at *auth.Token, metricExpected string) {
 		t.Helper()
-		metric := createMetricName(s, at)
+		metric := createMetricName(s, TenantID{
+			AccountID: at.AccountID,
+			ProjectID: at.ProjectID,
+		})
 		if metric != metricExpected {
 			t.Fatalf("unexpected result for createMetricName(%q, %v); got %q; want %q", s, at, metric, metricExpected)
 		}