app/vmselect: export seriesFetched stat for /query responses (#3925)

The change adds a new field `seriesFetched` to EvalConfig object.
Since EvalConfig object can be copied inside `Exec`,
`seriesFetched` is a pointer which can be updated by all copied
objects.

The reason for having stats is that other components, like vmalert,
could benefit from this information.

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

app/vmalert/web.qtpl.go

@@ -227,7 +227,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 //line app/vmalert/web.qtpl:55
 			qw422016.N().FPrec(g.Interval, 0)
 //line app/vmalert/web.qtpl:55
-			qw422016.N().S(`s)  #</a>
+			qw422016.N().S(`s) #</a>
                  `)
 //line app/vmalert/web.qtpl:56
 			if rNotOk[g.ID] > 0 {

app/vmselect/prometheus/prometheus.go

@@ -786,7 +786,8 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	qtDone := func() {
 		qt.Donef("query=%s, time=%d: series=%d", query, start, len(result))
 	}
-	WriteQueryResponse(bw, result, qt, qtDone)
+
+	WriteQueryResponse(bw, result, qt, qtDone, ec.SeriesFetched())
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}

app/vmselect/prometheus/query_response.qtpl

@@ -6,7 +6,7 @@
 {% stripspace %}
 QueryResponse generates response for /api/v1/query.
 See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
-{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %}
+{% func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) %}
 {
 	{% code seriesCount := len(rs) %}
 	"status":"success",
@@ -28,6 +28,9 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 				{% endfor %}
 			{% endif %}
 		]
+	},
+	"stats":{
+	    "seriesFetched": "{%d seriesFetched %}"
 	}
 	{% code
 		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/query_response.qtpl.go

@@ -26,7 +26,7 @@ var (
 )
 
 //line app/vmselect/prometheus/query_response.qtpl:9
-func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
+func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) {
 //line app/vmselect/prometheus/query_response.qtpl:9
 	qw422016.N().S(`{`)
 //line app/vmselect/prometheus/query_response.qtpl:11
@@ -69,40 +69,44 @@ func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *
 //line app/vmselect/prometheus/query_response.qtpl:29
 	}
 //line app/vmselect/prometheus/query_response.qtpl:29
-	qw422016.N().S(`]}`)
+	qw422016.N().S(`]},"stats":{"seriesFetched": "`)
 //line app/vmselect/prometheus/query_response.qtpl:33
+	qw422016.N().D(seriesFetched)
+//line app/vmselect/prometheus/query_response.qtpl:33
+	qw422016.N().S(`"}`)
+//line app/vmselect/prometheus/query_response.qtpl:36
 	qt.Printf("generate /api/v1/query response for series=%d", seriesCount)
 	qtDone()
 
-//line app/vmselect/prometheus/query_response.qtpl:36
+//line app/vmselect/prometheus/query_response.qtpl:39
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/query_response.qtpl:36
+//line app/vmselect/prometheus/query_response.qtpl:39
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
-func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) {
+func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) {
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
-	StreamQueryResponse(qw422016, rs, qt, qtDone)
+	StreamQueryResponse(qw422016, rs, qt, qtDone, seriesFetched)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 }
 
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
-func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string {
+func QueryResponse(rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) string {
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
-	WriteQueryResponse(qb422016, rs, qt, qtDone)
+	WriteQueryResponse(qb422016, rs, qt, qtDone, seriesFetched)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 	return qs422016
-//line app/vmselect/prometheus/query_response.qtpl:38
+//line app/vmselect/prometheus/query_response.qtpl:41
 }

app/vmselect/promql/eval.go

@@ -134,6 +134,12 @@ type EvalConfig struct {
 
 	timestamps     []int64
 	timestampsOnce sync.Once
+
+	// seriesFetched stores the number of time series fetched
+	// from the storage during the evaluation.
+	// It is defined as a pointer since EvalConfig can be forked
+	// during the evaluation but we want to keep its state.
+	seriesFetched *int
 }
 
 // copyEvalConfig returns src copy.
@@ -151,10 +157,28 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
 	ec.GetRequestURI = src.GetRequestURI
 
+	ec.seriesFetched = src.seriesFetched
+
 	// do not copy src.timestamps - they must be generated again.
 	return &ec
 }
 
+func (ec *EvalConfig) addStats(series int) {
+	if ec.seriesFetched == nil {
+		ec.seriesFetched = new(int)
+	}
+	*ec.seriesFetched += series
+}
+
+// SeriesFetched returns the number of series fetched from storages
+// during the evaluation.
+func (ec *EvalConfig) SeriesFetched() int {
+	if ec.seriesFetched == nil {
+		return 0
+	}
+	return *ec.seriesFetched
+}
+
 func (ec *EvalConfig) validate() {
 	if ec.Start > ec.End {
 		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", ec.Start, ec.End)
@@ -1077,6 +1101,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 		tss := mergeTimeseries(tssCached, nil, start, ec)
 		return tss, nil
 	}
+	ec.addStats(rssLen)
 
 	// Verify timeseries fit available memory after the rollup.
 	// Take into account points from tssCached.

app/vmselect/promql/eval_test.go

@@ -76,3 +76,18 @@ func TestValidateMaxPointsPerSeriesSuccess(t *testing.T) {
 	f(1659962171908, 1659966077742, 5000, 800)
 	f(1659962150000, 1659966070000, 10000, 393)
 }
+
+func TestEvalConfig_SeriesFetched(t *testing.T) {
+	ec := &EvalConfig{}
+	ec.addStats(1)
+
+	if ec.SeriesFetched() != 1 {
+		t.Fatalf("expected to get 1; got %d instead", ec.SeriesFetched())
+	}
+
+	ecNew := copyEvalConfig(ec)
+	ecNew.addStats(3)
+	if ec.SeriesFetched() != 4 {
+		t.Fatalf("expected to get 4; got %d instead", ec.SeriesFetched())
+	}
+}