From 9387793f47bd0ed1b321ab51719270f7167ba0d9 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 27 Mar 2023 15:11:40 -0700 Subject: [PATCH] app/vmselect: follow-up for 10ab086366fd3cd84a961a3635b77ba32897535c - Expose stats.seriesFetched at `/api/v1/query_range` responses too for the sake of consistency. - Initialize QueryStats when it is needed and pass it to EvalConfig then. This guarantees that the QueryStats is properly collected when the query contains some subqueries. --- app/vmalert/web.qtpl.go | 2 +- app/vmselect/prometheus/prometheus.go | 16 +- .../prometheus/query_range_response.qtpl | 6 +- .../prometheus/query_range_response.qtpl.go | 159 +++++++++--------- app/vmselect/prometheus/query_response.qtpl | 5 +- .../prometheus/query_response.qtpl.go | 107 ++++++------ app/vmselect/promql/eval.go | 35 ++-- app/vmselect/promql/eval_test.go | 19 ++- 8 files changed, 181 insertions(+), 168 deletions(-) diff --git a/app/vmalert/web.qtpl.go b/app/vmalert/web.qtpl.go index 3cde8a7c76..0b584a94d4 100644 --- a/app/vmalert/web.qtpl.go +++ b/app/vmalert/web.qtpl.go @@ -227,7 +227,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr //line app/vmalert/web.qtpl:55 qw422016.N().FPrec(g.Interval, 0) //line app/vmalert/web.qtpl:55 - qw422016.N().S(`s) # + qw422016.N().S(`s) # `) //line app/vmalert/web.qtpl:56 if rNotOk[g.ID] > 0 { diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 61e8c9637f..cf17e692c4 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -842,7 +842,8 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w } else { queryOffset = 0 } - ec := promql.EvalConfig{ + qs := &promql.QueryStats{} + ec := &promql.EvalConfig{ AuthToken: at, Start: start, End: start, @@ -860,8 +861,9 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w }, DenyPartialResponse: searchutils.GetDenyPartialResponse(r), + QueryStats: qs, } - result, err := promql.Exec(qt, &ec, query, true) + result, err := promql.Exec(qt, ec, query, true) if err != nil { return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err) } @@ -885,7 +887,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w qt.Donef("query=%s, time=%d: series=%d", query, start, len(result)) } - WriteQueryResponse(bw, ec.IsPartialResponse.Load(), result, qt, qtDone, ec.SeriesFetched()) + WriteQueryResponse(bw, ec.IsPartialResponse.Load(), result, qt, qtDone, qs) if err := bw.Flush(); err != nil { return fmt.Errorf("cannot flush query response to remote client: %w", err) } @@ -950,7 +952,8 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok start, end = promql.AdjustStartEnd(start, end, step) } - ec := promql.EvalConfig{ + qs := &promql.QueryStats{} + ec := &promql.EvalConfig{ AuthToken: at, Start: start, End: end, @@ -968,8 +971,9 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok }, DenyPartialResponse: searchutils.GetDenyPartialResponse(r), + QueryStats: qs, } - result, err := promql.Exec(qt, &ec, query, false) + result, err := promql.Exec(qt, ec, query, false) if err != nil { return err } @@ -993,7 +997,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok qtDone := func() { qt.Donef("start=%d, end=%d, step=%d, query=%q: series=%d", start, end, step, query, len(result)) } - WriteQueryRangeResponse(bw, ec.IsPartialResponse.Load(), result, qt, qtDone) + WriteQueryRangeResponse(bw, ec.IsPartialResponse.Load(), result, qt, qtDone, qs) if err := bw.Flush(); err != nil { return fmt.Errorf("cannot send query range response to remote client: %w", err) } diff --git a/app/vmselect/prometheus/query_range_response.qtpl b/app/vmselect/prometheus/query_range_response.qtpl index 2d6f0cf90d..85ac5438a1 100644 --- a/app/vmselect/prometheus/query_range_response.qtpl +++ b/app/vmselect/prometheus/query_range_response.qtpl @@ -1,12 +1,13 @@ {% import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" ) %} {% stripspace %} QueryRangeResponse generates response for /api/v1/query_range. See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries -{% func QueryRangeResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) %} +{% func QueryRangeResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) %} { {% code seriesCount := len(rs) @@ -27,6 +28,9 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries {% endfor %} {% endif %} ] + }, + "stats":{ + "seriesFetched": "{%d qs.SeriesFetched %}" } {% code qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount) diff --git a/app/vmselect/prometheus/query_range_response.qtpl.go b/app/vmselect/prometheus/query_range_response.qtpl.go index 54bdbd8aa6..118a40a8f2 100644 --- a/app/vmselect/prometheus/query_range_response.qtpl.go +++ b/app/vmselect/prometheus/query_range_response.qtpl.go @@ -7,145 +7,150 @@ package prometheus //line app/vmselect/prometheus/query_range_response.qtpl:1 import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" ) // QueryRangeResponse generates response for /api/v1/query_range.See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries -//line app/vmselect/prometheus/query_range_response.qtpl:9 +//line app/vmselect/prometheus/query_range_response.qtpl:10 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmselect/prometheus/query_range_response.qtpl:9 +//line app/vmselect/prometheus/query_range_response.qtpl:10 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmselect/prometheus/query_range_response.qtpl:9 -func StreamQueryRangeResponse(qw422016 *qt422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) { -//line app/vmselect/prometheus/query_range_response.qtpl:9 +//line app/vmselect/prometheus/query_range_response.qtpl:10 +func StreamQueryRangeResponse(qw422016 *qt422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) { +//line app/vmselect/prometheus/query_range_response.qtpl:10 qw422016.N().S(`{`) -//line app/vmselect/prometheus/query_range_response.qtpl:12 +//line app/vmselect/prometheus/query_range_response.qtpl:13 seriesCount := len(rs) pointsCount := 0 -//line app/vmselect/prometheus/query_range_response.qtpl:14 +//line app/vmselect/prometheus/query_range_response.qtpl:15 qw422016.N().S(`"status":"success","isPartial":`) -//line app/vmselect/prometheus/query_range_response.qtpl:16 +//line app/vmselect/prometheus/query_range_response.qtpl:17 if isPartial { -//line app/vmselect/prometheus/query_range_response.qtpl:16 +//line app/vmselect/prometheus/query_range_response.qtpl:17 qw422016.N().S(`true`) -//line app/vmselect/prometheus/query_range_response.qtpl:16 +//line app/vmselect/prometheus/query_range_response.qtpl:17 } else { -//line app/vmselect/prometheus/query_range_response.qtpl:16 +//line app/vmselect/prometheus/query_range_response.qtpl:17 qw422016.N().S(`false`) -//line app/vmselect/prometheus/query_range_response.qtpl:16 +//line app/vmselect/prometheus/query_range_response.qtpl:17 } -//line app/vmselect/prometheus/query_range_response.qtpl:16 +//line app/vmselect/prometheus/query_range_response.qtpl:17 qw422016.N().S(`,"data":{"resultType":"matrix","result":[`) -//line app/vmselect/prometheus/query_range_response.qtpl:20 - if len(rs) > 0 { //line app/vmselect/prometheus/query_range_response.qtpl:21 - streamqueryRangeLine(qw422016, &rs[0]) + if len(rs) > 0 { //line app/vmselect/prometheus/query_range_response.qtpl:22 + streamqueryRangeLine(qw422016, &rs[0]) +//line app/vmselect/prometheus/query_range_response.qtpl:23 pointsCount += len(rs[0].Values) -//line app/vmselect/prometheus/query_range_response.qtpl:23 +//line app/vmselect/prometheus/query_range_response.qtpl:24 rs = rs[1:] -//line app/vmselect/prometheus/query_range_response.qtpl:24 - for i := range rs { -//line app/vmselect/prometheus/query_range_response.qtpl:24 - qw422016.N().S(`,`) //line app/vmselect/prometheus/query_range_response.qtpl:25 - streamqueryRangeLine(qw422016, &rs[i]) + for i := range rs { +//line app/vmselect/prometheus/query_range_response.qtpl:25 + qw422016.N().S(`,`) //line app/vmselect/prometheus/query_range_response.qtpl:26 + streamqueryRangeLine(qw422016, &rs[i]) +//line app/vmselect/prometheus/query_range_response.qtpl:27 pointsCount += len(rs[i].Values) -//line app/vmselect/prometheus/query_range_response.qtpl:27 +//line app/vmselect/prometheus/query_range_response.qtpl:28 } -//line app/vmselect/prometheus/query_range_response.qtpl:28 +//line app/vmselect/prometheus/query_range_response.qtpl:29 } -//line app/vmselect/prometheus/query_range_response.qtpl:28 - qw422016.N().S(`]}`) -//line app/vmselect/prometheus/query_range_response.qtpl:32 +//line app/vmselect/prometheus/query_range_response.qtpl:29 + qw422016.N().S(`]},"stats":{"seriesFetched": "`) +//line app/vmselect/prometheus/query_range_response.qtpl:33 + qw422016.N().D(qs.SeriesFetched) +//line app/vmselect/prometheus/query_range_response.qtpl:33 + qw422016.N().S(`"}`) +//line app/vmselect/prometheus/query_range_response.qtpl:36 qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount) qtDone() -//line app/vmselect/prometheus/query_range_response.qtpl:35 +//line app/vmselect/prometheus/query_range_response.qtpl:39 streamdumpQueryTrace(qw422016, qt) -//line app/vmselect/prometheus/query_range_response.qtpl:35 - qw422016.N().S(`}`) -//line app/vmselect/prometheus/query_range_response.qtpl:37 -} - -//line app/vmselect/prometheus/query_range_response.qtpl:37 -func WriteQueryRangeResponse(qq422016 qtio422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) { -//line app/vmselect/prometheus/query_range_response.qtpl:37 - qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/query_range_response.qtpl:37 - StreamQueryRangeResponse(qw422016, isPartial, rs, qt, qtDone) -//line app/vmselect/prometheus/query_range_response.qtpl:37 - qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/query_range_response.qtpl:37 -} - -//line app/vmselect/prometheus/query_range_response.qtpl:37 -func QueryRangeResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func()) string { -//line app/vmselect/prometheus/query_range_response.qtpl:37 - qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/query_range_response.qtpl:37 - WriteQueryRangeResponse(qb422016, isPartial, rs, qt, qtDone) -//line app/vmselect/prometheus/query_range_response.qtpl:37 - qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/query_range_response.qtpl:37 - qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/query_range_response.qtpl:37 - return qs422016 -//line app/vmselect/prometheus/query_range_response.qtpl:37 -} - //line app/vmselect/prometheus/query_range_response.qtpl:39 + qw422016.N().S(`}`) +//line app/vmselect/prometheus/query_range_response.qtpl:41 +} + +//line app/vmselect/prometheus/query_range_response.qtpl:41 +func WriteQueryRangeResponse(qq422016 qtio422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) { +//line app/vmselect/prometheus/query_range_response.qtpl:41 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/prometheus/query_range_response.qtpl:41 + StreamQueryRangeResponse(qw422016, isPartial, rs, qt, qtDone, qs) +//line app/vmselect/prometheus/query_range_response.qtpl:41 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/prometheus/query_range_response.qtpl:41 +} + +//line app/vmselect/prometheus/query_range_response.qtpl:41 +func QueryRangeResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) string { +//line app/vmselect/prometheus/query_range_response.qtpl:41 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/prometheus/query_range_response.qtpl:41 + WriteQueryRangeResponse(qb422016, isPartial, rs, qt, qtDone, qs) +//line app/vmselect/prometheus/query_range_response.qtpl:41 + qs422016 := string(qb422016.B) +//line app/vmselect/prometheus/query_range_response.qtpl:41 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/prometheus/query_range_response.qtpl:41 + return qs422016 +//line app/vmselect/prometheus/query_range_response.qtpl:41 +} + +//line app/vmselect/prometheus/query_range_response.qtpl:43 func streamqueryRangeLine(qw422016 *qt422016.Writer, r *netstorage.Result) { -//line app/vmselect/prometheus/query_range_response.qtpl:39 +//line app/vmselect/prometheus/query_range_response.qtpl:43 qw422016.N().S(`{"metric":`) -//line app/vmselect/prometheus/query_range_response.qtpl:41 +//line app/vmselect/prometheus/query_range_response.qtpl:45 streammetricNameObject(qw422016, &r.MetricName) -//line app/vmselect/prometheus/query_range_response.qtpl:41 +//line app/vmselect/prometheus/query_range_response.qtpl:45 qw422016.N().S(`,"values":`) -//line app/vmselect/prometheus/query_range_response.qtpl:42 +//line app/vmselect/prometheus/query_range_response.qtpl:46 streamvaluesWithTimestamps(qw422016, r.Values, r.Timestamps) -//line app/vmselect/prometheus/query_range_response.qtpl:42 +//line app/vmselect/prometheus/query_range_response.qtpl:46 qw422016.N().S(`}`) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 } -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 func writequeryRangeLine(qq422016 qtio422016.Writer, r *netstorage.Result) { -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 streamqueryRangeLine(qw422016, r) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 } -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 func queryRangeLine(r *netstorage.Result) string { -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 writequeryRangeLine(qb422016, r) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 return qs422016 -//line app/vmselect/prometheus/query_range_response.qtpl:44 +//line app/vmselect/prometheus/query_range_response.qtpl:48 } diff --git a/app/vmselect/prometheus/query_response.qtpl b/app/vmselect/prometheus/query_response.qtpl index 7da8655a60..a2e5e6e7a1 100644 --- a/app/vmselect/prometheus/query_response.qtpl +++ b/app/vmselect/prometheus/query_response.qtpl @@ -1,12 +1,13 @@ {% import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" ) %} {% stripspace %} QueryResponse generates response for /api/v1/query. See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries -{% func QueryResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) %} +{% func QueryResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) %} { {% code seriesCount := len(rs) %} "status":"success", @@ -31,7 +32,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries ] }, "stats":{ - "seriesFetched": "{%d seriesFetched %}" + "seriesFetched": "{%d qs.SeriesFetched %}" } {% code qt.Printf("generate /api/v1/query response for series=%d", seriesCount) diff --git a/app/vmselect/prometheus/query_response.qtpl.go b/app/vmselect/prometheus/query_response.qtpl.go index bc30752ade..a5f776029f 100644 --- a/app/vmselect/prometheus/query_response.qtpl.go +++ b/app/vmselect/prometheus/query_response.qtpl.go @@ -7,118 +7,119 @@ package prometheus //line app/vmselect/prometheus/query_response.qtpl:1 import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" ) // QueryResponse generates response for /api/v1/query.See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries -//line app/vmselect/prometheus/query_response.qtpl:9 +//line app/vmselect/prometheus/query_response.qtpl:10 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmselect/prometheus/query_response.qtpl:9 +//line app/vmselect/prometheus/query_response.qtpl:10 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmselect/prometheus/query_response.qtpl:9 -func StreamQueryResponse(qw422016 *qt422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) { -//line app/vmselect/prometheus/query_response.qtpl:9 +//line app/vmselect/prometheus/query_response.qtpl:10 +func StreamQueryResponse(qw422016 *qt422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) { +//line app/vmselect/prometheus/query_response.qtpl:10 qw422016.N().S(`{`) -//line app/vmselect/prometheus/query_response.qtpl:11 +//line app/vmselect/prometheus/query_response.qtpl:12 seriesCount := len(rs) -//line app/vmselect/prometheus/query_response.qtpl:11 +//line app/vmselect/prometheus/query_response.qtpl:12 qw422016.N().S(`"status":"success","isPartial":`) -//line app/vmselect/prometheus/query_response.qtpl:13 +//line app/vmselect/prometheus/query_response.qtpl:14 if isPartial { -//line app/vmselect/prometheus/query_response.qtpl:13 +//line app/vmselect/prometheus/query_response.qtpl:14 qw422016.N().S(`true`) -//line app/vmselect/prometheus/query_response.qtpl:13 +//line app/vmselect/prometheus/query_response.qtpl:14 } else { -//line app/vmselect/prometheus/query_response.qtpl:13 +//line app/vmselect/prometheus/query_response.qtpl:14 qw422016.N().S(`false`) -//line app/vmselect/prometheus/query_response.qtpl:13 +//line app/vmselect/prometheus/query_response.qtpl:14 } -//line app/vmselect/prometheus/query_response.qtpl:13 +//line app/vmselect/prometheus/query_response.qtpl:14 qw422016.N().S(`,"data":{"resultType":"vector","result":[`) -//line app/vmselect/prometheus/query_response.qtpl:17 +//line app/vmselect/prometheus/query_response.qtpl:18 if len(rs) > 0 { -//line app/vmselect/prometheus/query_response.qtpl:17 +//line app/vmselect/prometheus/query_response.qtpl:18 qw422016.N().S(`{"metric":`) -//line app/vmselect/prometheus/query_response.qtpl:19 +//line app/vmselect/prometheus/query_response.qtpl:20 streammetricNameObject(qw422016, &rs[0].MetricName) -//line app/vmselect/prometheus/query_response.qtpl:19 +//line app/vmselect/prometheus/query_response.qtpl:20 qw422016.N().S(`,"value":`) -//line app/vmselect/prometheus/query_response.qtpl:20 +//line app/vmselect/prometheus/query_response.qtpl:21 streammetricRow(qw422016, rs[0].Timestamps[0], rs[0].Values[0]) -//line app/vmselect/prometheus/query_response.qtpl:20 +//line app/vmselect/prometheus/query_response.qtpl:21 qw422016.N().S(`}`) -//line app/vmselect/prometheus/query_response.qtpl:22 +//line app/vmselect/prometheus/query_response.qtpl:23 rs = rs[1:] -//line app/vmselect/prometheus/query_response.qtpl:23 - for i := range rs { //line app/vmselect/prometheus/query_response.qtpl:24 + for i := range rs { +//line app/vmselect/prometheus/query_response.qtpl:25 r := &rs[i] -//line app/vmselect/prometheus/query_response.qtpl:24 +//line app/vmselect/prometheus/query_response.qtpl:25 qw422016.N().S(`,{"metric":`) -//line app/vmselect/prometheus/query_response.qtpl:26 +//line app/vmselect/prometheus/query_response.qtpl:27 streammetricNameObject(qw422016, &r.MetricName) -//line app/vmselect/prometheus/query_response.qtpl:26 +//line app/vmselect/prometheus/query_response.qtpl:27 qw422016.N().S(`,"value":`) -//line app/vmselect/prometheus/query_response.qtpl:27 +//line app/vmselect/prometheus/query_response.qtpl:28 streammetricRow(qw422016, r.Timestamps[0], r.Values[0]) -//line app/vmselect/prometheus/query_response.qtpl:27 +//line app/vmselect/prometheus/query_response.qtpl:28 qw422016.N().S(`}`) -//line app/vmselect/prometheus/query_response.qtpl:29 +//line app/vmselect/prometheus/query_response.qtpl:30 } -//line app/vmselect/prometheus/query_response.qtpl:30 +//line app/vmselect/prometheus/query_response.qtpl:31 } -//line app/vmselect/prometheus/query_response.qtpl:30 +//line app/vmselect/prometheus/query_response.qtpl:31 qw422016.N().S(`]},"stats":{"seriesFetched": "`) -//line app/vmselect/prometheus/query_response.qtpl:34 - qw422016.N().D(seriesFetched) -//line app/vmselect/prometheus/query_response.qtpl:34 +//line app/vmselect/prometheus/query_response.qtpl:35 + qw422016.N().D(qs.SeriesFetched) +//line app/vmselect/prometheus/query_response.qtpl:35 qw422016.N().S(`"}`) -//line app/vmselect/prometheus/query_response.qtpl:37 +//line app/vmselect/prometheus/query_response.qtpl:38 qt.Printf("generate /api/v1/query response for series=%d", seriesCount) qtDone() -//line app/vmselect/prometheus/query_response.qtpl:40 +//line app/vmselect/prometheus/query_response.qtpl:41 streamdumpQueryTrace(qw422016, qt) -//line app/vmselect/prometheus/query_response.qtpl:40 +//line app/vmselect/prometheus/query_response.qtpl:41 qw422016.N().S(`}`) -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 } -//line app/vmselect/prometheus/query_response.qtpl:42 -func WriteQueryResponse(qq422016 qtio422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) { -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 +func WriteQueryResponse(qq422016 qtio422016.Writer, isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) { +//line app/vmselect/prometheus/query_response.qtpl:43 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/query_response.qtpl:42 - StreamQueryResponse(qw422016, isPartial, rs, qt, qtDone, seriesFetched) -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 + StreamQueryResponse(qw422016, isPartial, rs, qt, qtDone, qs) +//line app/vmselect/prometheus/query_response.qtpl:43 qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 } -//line app/vmselect/prometheus/query_response.qtpl:42 -func QueryResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), seriesFetched int) string { -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 +func QueryResponse(isPartial bool, rs []netstorage.Result, qt *querytracer.Tracer, qtDone func(), qs *promql.QueryStats) string { +//line app/vmselect/prometheus/query_response.qtpl:43 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/query_response.qtpl:42 - WriteQueryResponse(qb422016, isPartial, rs, qt, qtDone, seriesFetched) -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 + WriteQueryResponse(qb422016, isPartial, rs, qt, qtDone, qs) +//line app/vmselect/prometheus/query_response.qtpl:43 qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 return qs422016 -//line app/vmselect/prometheus/query_response.qtpl:42 +//line app/vmselect/prometheus/query_response.qtpl:43 } diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index 1284079486..7f7dc90a0a 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -140,14 +140,14 @@ type EvalConfig struct { // IsPartialResponse is set during query execution and can be used by Exec caller after query execution. IsPartialResponse atomic.Bool + // QueryStats contains various stats for the currently executed query. + // + // The caller must initialize the QueryStats if it needs the stats. + // Otherwise the stats isn't collected. + QueryStats *QueryStats + timestamps []int64 timestampsOnce sync.Once - - // seriesFetched stores the number of time series fetched - // from the storage during the evaluation. - // It is defined as a pointer since EvalConfig can be forked - // during the evaluation but we want to keep its state. - seriesFetched *int } // copyEvalConfig returns src copy. @@ -167,27 +167,22 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig { ec.GetRequestURI = src.GetRequestURI ec.DenyPartialResponse = src.DenyPartialResponse ec.IsPartialResponse.Store(src.IsPartialResponse.Load()) - - ec.seriesFetched = src.seriesFetched + ec.QueryStats = src.QueryStats // do not copy src.timestamps - they must be generated again. return &ec } -func (ec *EvalConfig) addStats(series int) { - if ec.seriesFetched == nil { - ec.seriesFetched = new(int) - } - *ec.seriesFetched += series +// QueryStats contains various stats for the query. +type QueryStats struct { + // SeriesFetched contains the number of series fetched from storage during the query evaluation. + SeriesFetched int } -// SeriesFetched returns the number of series fetched from storages -// during the evaluation. -func (ec *EvalConfig) SeriesFetched() int { - if ec.seriesFetched == nil { - return 0 +func (qs *QueryStats) addSeriesFetched(n int) { + if qs != nil { + qs.SeriesFetched += n } - return *ec.seriesFetched } func (ec *EvalConfig) updateIsPartialResponse(isPartialResponse bool) { @@ -1119,7 +1114,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa tss := mergeTimeseries(tssCached, nil, start, ec) return tss, nil } - ec.addStats(rssLen) + ec.QueryStats.addSeriesFetched(rssLen) // Verify timeseries fit available memory after the rollup. // Take into account points from tssCached. diff --git a/app/vmselect/promql/eval_test.go b/app/vmselect/promql/eval_test.go index 8ade7f0ae3..46e54eda13 100644 --- a/app/vmselect/promql/eval_test.go +++ b/app/vmselect/promql/eval_test.go @@ -77,17 +77,20 @@ func TestValidateMaxPointsPerSeriesSuccess(t *testing.T) { f(1659962150000, 1659966070000, 10000, 393) } -func TestEvalConfig_SeriesFetched(t *testing.T) { - ec := &EvalConfig{} - ec.addStats(1) +func TestQueryStats_addSeriesFetched(t *testing.T) { + qs := &QueryStats{} + ec := &EvalConfig{ + QueryStats: qs, + } + ec.QueryStats.addSeriesFetched(1) - if ec.SeriesFetched() != 1 { - t.Fatalf("expected to get 1; got %d instead", ec.SeriesFetched()) + if qs.SeriesFetched != 1 { + t.Fatalf("expected to get 1; got %d instead", qs.SeriesFetched) } ecNew := copyEvalConfig(ec) - ecNew.addStats(3) - if ec.SeriesFetched() != 4 { - t.Fatalf("expected to get 4; got %d instead", ec.SeriesFetched()) + ecNew.QueryStats.addSeriesFetched(3) + if qs.SeriesFetched != 4 { + t.Fatalf("expected to get 4; got %d instead", qs.SeriesFetched) } }