github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-02-09 15:27:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmselect: propagate errors from vmstorage to response to the client if -search.denyPartialResponse command-line flag is set

Browse source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/891

This commit also adds `"isPartial":{true|false}` field to `/api/v1/*` responses. `"isPartial":true` is set when the response
is based on a partial data because some of vmstorage nodes weren't available during query processing.
This commit is contained in:
Aliaksandr Valialkin 2020-11-14 12:36:21 +02:00
parent 882e2e2099
commit 2ac5f00d98
27 changed files with 618 additions and 507 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
app/vmselect/graphite/graphite.go
View file

@ -76,13 +76,11 @@ func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit
MinTimestamp: from,
MaxTimestamp: until,
}
paths, isPartial, err := metricsFind(at, tr, label, query, delimiter[0], false, deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, query, delimiter[0], false, deadline)
if err != nil {
return err
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
if leavesOnly {
paths = filterLeaves(paths, delimiter)
}
@ -95,7 +93,7 @@ func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit
w.Header().Set("Content-Type", contentType)
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteMetricsFindResponse(bw, paths, delimiter, format, wildcards, jsonp)
WriteMetricsFindResponse(bw, isPartial, paths, delimiter, format, wildcards, jsonp)
if err := bw.Flush(); err != nil {
return err
}
@ -160,13 +158,15 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr
MaxTimestamp: until,
}
m := make(map[string][]string, len(queries))
isPartialResponse := false
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
for _, query := range queries {
paths, isPartial, err := metricsFind(at, tr, label, query, delimiter[0], true, deadline)
paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, query, delimiter[0], true, deadline)
if err != nil {
return err
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
if isPartial {
isPartialResponse = true
}
if leavesOnly {
paths = filterLeaves(paths, delimiter)
@ -182,7 +182,7 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr
for _, paths := range m {
sortPaths(paths, delimiter)
}
WriteMetricsExpandResponseByQuery(w, m, jsonp)
WriteMetricsExpandResponseByQuery(w, isPartialResponse, m, jsonp)
return nil
}
paths := m[queries[0]]
@ -201,7 +201,7 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr
sortPaths(paths, delimiter)
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteMetricsExpandResponseFlat(bw, paths, jsonp)
WriteMetricsExpandResponseFlat(bw, isPartialResponse, paths, jsonp)
if err := bw.Flush(); err != nil {
return err
}
@ -218,13 +218,11 @@ func MetricsIndexHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
return fmt.Errorf("cannot parse form values: %w", err)
}
jsonp := r.FormValue("jsonp")
metricNames, isPartial, err := netstorage.GetLabelValues(at, "__name__", deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
metricNames, isPartial, err := netstorage.GetLabelValues(at, denyPartialResponse, "__name__", deadline)
if err != nil {
return fmt.Errorf(`cannot obtain metric names: %w`, err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
contentType := "application/json; charset=utf-8"
if jsonp != "" {
contentType = "text/javascript; charset=utf-8"
@ -232,7 +230,7 @@ func MetricsIndexHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
w.Header().Set("Content-Type", contentType)
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteMetricsIndexResponse(bw, metricNames, jsonp)
WriteMetricsIndexResponse(bw, isPartial, metricNames, jsonp)
if err := bw.Flush(); err != nil {
return err
}
@ -241,14 +239,15 @@ func MetricsIndexHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
}
// metricsFind searches for label values that match the given query.
func metricsFind(at *auth.Token, tr storage.TimeRange, label, query string, delimiter byte, isExpand bool, deadline searchutils.Deadline) ([]string, bool, error) {
func metricsFind(at *auth.Token, denyPartialResponse bool, tr storage.TimeRange, label, query string, delimiter byte,
isExpand bool, deadline searchutils.Deadline) ([]string, bool, error) {
n := strings.IndexAny(query, "*{[")
if n < 0 || n == len(query)-1 && strings.HasSuffix(query, "*") {
expandTail := n >= 0
if expandTail {
query = query[:len(query)-1]
}
suffixes, isPartial, err := netstorage.GetTagValueSuffixes(at, tr, label, query, delimiter, deadline)
suffixes, isPartial, err := netstorage.GetTagValueSuffixes(at, denyPartialResponse, tr, label, query, delimiter, deadline)
if err != nil {
return nil, false, err
}
@ -267,7 +266,7 @@ func metricsFind(at *auth.Token, tr storage.TimeRange, label, query string, deli
return results, isPartial, nil
}
subquery := query[:n] + "*"
paths, isPartial, err := metricsFind(at, tr, label, subquery, delimiter, isExpand, deadline)
paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, subquery, delimiter, isExpand, deadline)
if err != nil {
return nil, false, err
}
@ -292,7 +291,7 @@ func metricsFind(at *auth.Token, tr storage.TimeRange, label, query string, deli
continue
}
subquery := path + tail
fullPaths, isPartialLocal, err := metricsFind(at, tr, label, subquery, delimiter, isExpand, deadline)
fullPaths, isPartialLocal, err := metricsFind(at, denyPartialResponse, tr, label, subquery, delimiter, isExpand, deadline)
if err != nil {
return nil, false, err
}

4
app/vmselect/graphite/metrics_expand_response.qtpl
View file

@ -2,7 +2,7 @@
MetricsExpandResponseByQuery generates response for /metrics/expand?groupByExpr=1 .
See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
{% func MetricsExpandResponseByQuery(m map[string][]string, jsonp string) %}
{% func MetricsExpandResponseByQuery(isPartial bool, m map[string][]string, jsonp string) %}
{% if jsonp != "" %}{%s= jsonp %}({% endif %}
{
"results":{
@ -20,7 +20,7 @@ See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
MetricsExpandResponseFlat generates response for /metrics/expand?groupByExpr=0 .
See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
{% func MetricsExpandResponseFlat(paths []string, jsonp string) %}
{% func MetricsExpandResponseFlat(isPartial bool, paths []string, jsonp string) %}
{% if jsonp != "" %}{%s= jsonp %}({% endif %}
{%= metricPaths(paths) %}
{% if jsonp != "" %}){% endif %}

20
app/vmselect/graphite/metrics_expand_response.qtpl.go
View file

@ -20,7 +20,7 @@ var (
)
//line app/vmselect/graphite/metrics_expand_response.qtpl:5
func StreamMetricsExpandResponseByQuery(qw422016 *qt422016.Writer, m map[string][]string, jsonp string) {
func StreamMetricsExpandResponseByQuery(qw422016 *qt422016.Writer, isPartial bool, m map[string][]string, jsonp string) {
//line app/vmselect/graphite/metrics_expand_response.qtpl:6
if jsonp != "" {
//line app/vmselect/graphite/metrics_expand_response.qtpl:6
@ -65,22 +65,22 @@ func StreamMetricsExpandResponseByQuery(qw422016 *qt422016.Writer, m map[string]
}
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
func WriteMetricsExpandResponseByQuery(qq422016 qtio422016.Writer, m map[string][]string, jsonp string) {
func WriteMetricsExpandResponseByQuery(qq422016 qtio422016.Writer, isPartial bool, m map[string][]string, jsonp string) {
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
StreamMetricsExpandResponseByQuery(qw422016, m, jsonp)
StreamMetricsExpandResponseByQuery(qw422016, isPartial, m, jsonp)
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
}
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
func MetricsExpandResponseByQuery(m map[string][]string, jsonp string) string {
func MetricsExpandResponseByQuery(isPartial bool, m map[string][]string, jsonp string) string {
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
WriteMetricsExpandResponseByQuery(qb422016, m, jsonp)
WriteMetricsExpandResponseByQuery(qb422016, isPartial, m, jsonp)
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
qs422016 := string(qb422016.B)
//line app/vmselect/graphite/metrics_expand_response.qtpl:18
@ -93,7 +93,7 @@ func MetricsExpandResponseByQuery(m map[string][]string, jsonp string) string {
// MetricsExpandResponseFlat generates response for /metrics/expand?groupByExpr=0 .See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
//line app/vmselect/graphite/metrics_expand_response.qtpl:23
func StreamMetricsExpandResponseFlat(qw422016 *qt422016.Writer, paths []string, jsonp string) {
func StreamMetricsExpandResponseFlat(qw422016 *qt422016.Writer, isPartial bool, paths []string, jsonp string) {
//line app/vmselect/graphite/metrics_expand_response.qtpl:24
if jsonp != "" {
//line app/vmselect/graphite/metrics_expand_response.qtpl:24
@ -114,22 +114,22 @@ func StreamMetricsExpandResponseFlat(qw422016 *qt422016.Writer, paths []string,
}
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
func WriteMetricsExpandResponseFlat(qq422016 qtio422016.Writer, paths []string, jsonp string) {
func WriteMetricsExpandResponseFlat(qq422016 qtio422016.Writer, isPartial bool, paths []string, jsonp string) {
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
StreamMetricsExpandResponseFlat(qw422016, paths, jsonp)
StreamMetricsExpandResponseFlat(qw422016, isPartial, paths, jsonp)
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
}
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
func MetricsExpandResponseFlat(paths []string, jsonp string) string {
func MetricsExpandResponseFlat(isPartial bool, paths []string, jsonp string) string {
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
WriteMetricsExpandResponseFlat(qb422016, paths, jsonp)
WriteMetricsExpandResponseFlat(qb422016, isPartial, paths, jsonp)
//line app/vmselect/graphite/metrics_expand_response.qtpl:27
qs422016 := string(qb422016.B)
//line app/vmselect/graphite/metrics_expand_response.qtpl:27

10
app/vmselect/graphite/metrics_find_response.qtpl
View file

@ -9,20 +9,20 @@
MetricsFindResponse generates response for /metrics/find .
See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
{% func MetricsFindResponse(paths []string, delimiter, format string, addWildcards bool, jsonp string) %}
{% func MetricsFindResponse(isPartial bool, paths []string, delimiter, format string, addWildcards bool, jsonp string) %}
{% if jsonp != "" %}{%s= jsonp %}({% endif %}
{% switch format %}
{% case "completer" %}
{%= metricsFindResponseCompleter(paths, delimiter, addWildcards) %}
{%= metricsFindResponseCompleter(isPartial, paths, delimiter, addWildcards) %}
{% case "treejson" %}
{%= metricsFindResponseTreeJSON(paths, delimiter, addWildcards) %}
{%= metricsFindResponseTreeJSON(isPartial, paths, delimiter, addWildcards) %}
{% default %}
{% code logger.Panicf("BUG: unexpected format=%q", format) %}
{% endswitch %}
{% if jsonp != "" %}){% endif %}
{% endfunc %}
{% func metricsFindResponseCompleter(paths []string, delimiter string, addWildcards bool) %}
{% func metricsFindResponseCompleter(isPartial bool, paths []string, delimiter string, addWildcards bool) %}
{
"metrics":[
{% for i, path := range paths %}
@ -42,7 +42,7 @@ See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
}
{% endfunc %}
{% func metricsFindResponseTreeJSON(paths []string, delimiter string, addWildcards bool) %}
{% func metricsFindResponseTreeJSON(isPartial bool, paths []string, delimiter string, addWildcards bool) %}
[
{% code
if len(paths) > 1 {

34
app/vmselect/graphite/metrics_find_response.qtpl.go
View file

@ -28,7 +28,7 @@ var (
)
//line app/vmselect/graphite/metrics_find_response.qtpl:12
func StreamMetricsFindResponse(qw422016 *qt422016.Writer, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
func StreamMetricsFindResponse(qw422016 *qt422016.Writer, isPartial bool, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
//line app/vmselect/graphite/metrics_find_response.qtpl:13
if jsonp != "" {
//line app/vmselect/graphite/metrics_find_response.qtpl:13
@ -42,11 +42,11 @@ func StreamMetricsFindResponse(qw422016 *qt422016.Writer, paths []string, delimi
//line app/vmselect/graphite/metrics_find_response.qtpl:15
case "completer":
//line app/vmselect/graphite/metrics_find_response.qtpl:16
streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
streammetricsFindResponseCompleter(qw422016, isPartial, paths, delimiter, addWildcards)
//line app/vmselect/graphite/metrics_find_response.qtpl:17
case "treejson":
//line app/vmselect/graphite/metrics_find_response.qtpl:18
streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
streammetricsFindResponseTreeJSON(qw422016, isPartial, paths, delimiter, addWildcards)
//line app/vmselect/graphite/metrics_find_response.qtpl:19
default:
//line app/vmselect/graphite/metrics_find_response.qtpl:20
@ -64,22 +64,22 @@ func StreamMetricsFindResponse(qw422016 *qt422016.Writer, paths []string, delimi
}
//line app/vmselect/graphite/metrics_find_response.qtpl:23
func WriteMetricsFindResponse(qq422016 qtio422016.Writer, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
func WriteMetricsFindResponse(qq422016 qtio422016.Writer, isPartial bool, paths []string, delimiter, format string, addWildcards bool, jsonp string) {
//line app/vmselect/graphite/metrics_find_response.qtpl:23
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/graphite/metrics_find_response.qtpl:23
StreamMetricsFindResponse(qw422016, paths, delimiter, format, addWildcards, jsonp)
StreamMetricsFindResponse(qw422016, isPartial, paths, delimiter, format, addWildcards, jsonp)
//line app/vmselect/graphite/metrics_find_response.qtpl:23
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/graphite/metrics_find_response.qtpl:23
}
//line app/vmselect/graphite/metrics_find_response.qtpl:23
func MetricsFindResponse(paths []string, delimiter, format string, addWildcards bool, jsonp string) string {
func MetricsFindResponse(isPartial bool, paths []string, delimiter, format string, addWildcards bool, jsonp string) string {
//line app/vmselect/graphite/metrics_find_response.qtpl:23
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/graphite/metrics_find_response.qtpl:23
WriteMetricsFindResponse(qb422016, paths, delimiter, format, addWildcards, jsonp)
WriteMetricsFindResponse(qb422016, isPartial, paths, delimiter, format, addWildcards, jsonp)
//line app/vmselect/graphite/metrics_find_response.qtpl:23
qs422016 := string(qb422016.B)
//line app/vmselect/graphite/metrics_find_response.qtpl:23
@ -90,7 +90,7 @@ func MetricsFindResponse(paths []string, delimiter, format string, addWildcards
}
//line app/vmselect/graphite/metrics_find_response.qtpl:25
func streammetricsFindResponseCompleter(qw422016 *qt422016.Writer, paths []string, delimiter string, addWildcards bool) {
func streammetricsFindResponseCompleter(qw422016 *qt422016.Writer, isPartial bool, paths []string, delimiter string, addWildcards bool) {
//line app/vmselect/graphite/metrics_find_response.qtpl:25
qw422016.N().S(`{"metrics":[`)
//line app/vmselect/graphite/metrics_find_response.qtpl:28
@ -137,22 +137,22 @@ func streammetricsFindResponseCompleter(qw422016 *qt422016.Writer, paths []strin
}
//line app/vmselect/graphite/metrics_find_response.qtpl:43
func writemetricsFindResponseCompleter(qq422016 qtio422016.Writer, paths []string, delimiter string, addWildcards bool) {
func writemetricsFindResponseCompleter(qq422016 qtio422016.Writer, isPartial bool, paths []string, delimiter string, addWildcards bool) {
//line app/vmselect/graphite/metrics_find_response.qtpl:43
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/graphite/metrics_find_response.qtpl:43
streammetricsFindResponseCompleter(qw422016, paths, delimiter, addWildcards)
streammetricsFindResponseCompleter(qw422016, isPartial, paths, delimiter, addWildcards)
//line app/vmselect/graphite/metrics_find_response.qtpl:43
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/graphite/metrics_find_response.qtpl:43
}
//line app/vmselect/graphite/metrics_find_response.qtpl:43
func metricsFindResponseCompleter(paths []string, delimiter string, addWildcards bool) string {
func metricsFindResponseCompleter(isPartial bool, paths []string, delimiter string, addWildcards bool) string {
//line app/vmselect/graphite/metrics_find_response.qtpl:43
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/graphite/metrics_find_response.qtpl:43
writemetricsFindResponseCompleter(qb422016, paths, delimiter, addWildcards)
writemetricsFindResponseCompleter(qb422016, isPartial, paths, delimiter, addWildcards)
//line app/vmselect/graphite/metrics_find_response.qtpl:43
qs422016 := string(qb422016.B)
//line app/vmselect/graphite/metrics_find_response.qtpl:43
@ -163,7 +163,7 @@ func metricsFindResponseCompleter(paths []string, delimiter string, addWildcards
}
//line app/vmselect/graphite/metrics_find_response.qtpl:45
func streammetricsFindResponseTreeJSON(qw422016 *qt422016.Writer, paths []string, delimiter string, addWildcards bool) {
func streammetricsFindResponseTreeJSON(qw422016 *qt422016.Writer, isPartial bool, paths []string, delimiter string, addWildcards bool) {
//line app/vmselect/graphite/metrics_find_response.qtpl:45
qw422016.N().S(`[`)
//line app/vmselect/graphite/metrics_find_response.qtpl:48
@ -286,22 +286,22 @@ func streammetricsFindResponseTreeJSON(qw422016 *qt422016.Writer, paths []string
}
//line app/vmselect/graphite/metrics_find_response.qtpl:123
func writemetricsFindResponseTreeJSON(qq422016 qtio422016.Writer, paths []string, delimiter string, addWildcards bool) {
func writemetricsFindResponseTreeJSON(qq422016 qtio422016.Writer, isPartial bool, paths []string, delimiter string, addWildcards bool) {
//line app/vmselect/graphite/metrics_find_response.qtpl:123
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/graphite/metrics_find_response.qtpl:123
streammetricsFindResponseTreeJSON(qw422016, paths, delimiter, addWildcards)
streammetricsFindResponseTreeJSON(qw422016, isPartial, paths, delimiter, addWildcards)
//line app/vmselect/graphite/metrics_find_response.qtpl:123
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/graphite/metrics_find_response.qtpl:123
}
//line app/vmselect/graphite/metrics_find_response.qtpl:123
func metricsFindResponseTreeJSON(paths []string, delimiter string, addWildcards bool) string {
func metricsFindResponseTreeJSON(isPartial bool, paths []string, delimiter string, addWildcards bool) string {
//line app/vmselect/graphite/metrics_find_response.qtpl:123
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/graphite/metrics_find_response.qtpl:123
writemetricsFindResponseTreeJSON(qb422016, paths, delimiter, addWildcards)
writemetricsFindResponseTreeJSON(qb422016, isPartial, paths, delimiter, addWildcards)
//line app/vmselect/graphite/metrics_find_response.qtpl:123
qs422016 := string(qb422016.B)
//line app/vmselect/graphite/metrics_find_response.qtpl:123

2
app/vmselect/graphite/metrics_index_response.qtpl
View file

@ -2,7 +2,7 @@
MetricsIndexResponse generates response for /metrics/index.json .
See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
{% func MetricsIndexResponse(metricNames []string, jsonp string) %}
{% func MetricsIndexResponse(isPartial bool, metricNames []string, jsonp string) %}
{% if jsonp != "" %}{%s= jsonp %}({% endif %}
{%= metricPaths(metricNames) %}
{% if jsonp != "" %}){% endif %}

10
app/vmselect/graphite/metrics_index_response.qtpl.go
View file

@ -20,7 +20,7 @@ var (
)
//line app/vmselect/graphite/metrics_index_response.qtpl:5
func StreamMetricsIndexResponse(qw422016 *qt422016.Writer, metricNames []string, jsonp string) {
func StreamMetricsIndexResponse(qw422016 *qt422016.Writer, isPartial bool, metricNames []string, jsonp string) {
//line app/vmselect/graphite/metrics_index_response.qtpl:6
if jsonp != "" {
//line app/vmselect/graphite/metrics_index_response.qtpl:6
@ -41,22 +41,22 @@ func StreamMetricsIndexResponse(qw422016 *qt422016.Writer, metricNames []string,
}
//line app/vmselect/graphite/metrics_index_response.qtpl:9
func WriteMetricsIndexResponse(qq422016 qtio422016.Writer, metricNames []string, jsonp string) {
func WriteMetricsIndexResponse(qq422016 qtio422016.Writer, isPartial bool, metricNames []string, jsonp string) {
//line app/vmselect/graphite/metrics_index_response.qtpl:9
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/graphite/metrics_index_response.qtpl:9
StreamMetricsIndexResponse(qw422016, metricNames, jsonp)
StreamMetricsIndexResponse(qw422016, isPartial, metricNames, jsonp)
//line app/vmselect/graphite/metrics_index_response.qtpl:9
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/graphite/metrics_index_response.qtpl:9
}
//line app/vmselect/graphite/metrics_index_response.qtpl:9
func MetricsIndexResponse(metricNames []string, jsonp string) string {
func MetricsIndexResponse(isPartial bool, metricNames []string, jsonp string) string {
//line app/vmselect/graphite/metrics_index_response.qtpl:9
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/graphite/metrics_index_response.qtpl:9
WriteMetricsIndexResponse(qb422016, metricNames, jsonp)
WriteMetricsIndexResponse(qb422016, isPartial, metricNames, jsonp)
//line app/vmselect/graphite/metrics_index_response.qtpl:9
qs422016 := string(qb422016.B)
//line app/vmselect/graphite/metrics_index_response.qtpl:9

133
app/vmselect/netstorage/netstorage.go
View file

@ -490,7 +490,7 @@ func DeleteSeries(at *auth.Token, sq *storage.SearchQuery, deadline searchutils.
}
// GetLabelsOnTimeRange returns labels for the given tr until the given deadline.
func GetLabelsOnTimeRange(at *auth.Token, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, bool, error) {
func GetLabelsOnTimeRange(at *auth.Token, denyPartialResponse bool, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -528,7 +528,7 @@ func GetLabelsOnTimeRange(at *auth.Token, tr storage.TimeRange, deadline searchu
}
labels = append(labels, nr.labels...)
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -539,9 +539,10 @@ func GetLabelsOnTimeRange(at *auth.Token, tr storage.TimeRange, deadline searchu
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialLabelsOnTimeRangeResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching labels on time range: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
// Deduplicate labels
labels = deduplicateStrings(labels)
@ -553,11 +554,11 @@ func GetLabelsOnTimeRange(at *auth.Token, tr storage.TimeRange, deadline searchu
}
// Sort labels like Prometheus does
sort.Strings(labels)
return labels, isPartialResult, nil
return labels, isPartial, nil
}
// GetLabels returns labels until the given deadline.
func GetLabels(at *auth.Token, deadline searchutils.Deadline) ([]string, bool, error) {
func GetLabels(at *auth.Token, denyPartialResponse bool, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -595,7 +596,7 @@ func GetLabels(at *auth.Token, deadline searchutils.Deadline) ([]string, bool, e
}
labels = append(labels, nr.labels...)
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -606,9 +607,10 @@ func GetLabels(at *auth.Token, deadline searchutils.Deadline) ([]string, bool, e
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialLabelsResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching labels: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
// Deduplicate labels
labels = deduplicateStrings(labels)
@ -620,12 +622,12 @@ func GetLabels(at *auth.Token, deadline searchutils.Deadline) ([]string, bool, e
}
// Sort labels like Prometheus does
sort.Strings(labels)
return labels, isPartialResult, nil
return labels, isPartial, nil
}
// GetLabelValuesOnTimeRange returns label values for the given labelName on the given tr
// until the given deadline.
func GetLabelValuesOnTimeRange(at *auth.Token, labelName string, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, bool, error) {
func GetLabelValuesOnTimeRange(at *auth.Token, denyPartialResponse bool, labelName string, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -667,7 +669,7 @@ func GetLabelValuesOnTimeRange(at *auth.Token, labelName string, tr storage.Time
}
labelValues = append(labelValues, nr.labelValues...)
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -678,21 +680,22 @@ func GetLabelValuesOnTimeRange(at *auth.Token, labelName string, tr storage.Time
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialLabelValuesOnTimeRangeResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching label values on time range: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
// Deduplicate label values
labelValues = deduplicateStrings(labelValues)
// Sort labelValues like Prometheus does
sort.Strings(labelValues)
return labelValues, isPartialResult, nil
return labelValues, isPartial, nil
}
// GetLabelValues returns label values for the given labelName
// until the given deadline.
func GetLabelValues(at *auth.Token, labelName string, deadline searchutils.Deadline) ([]string, bool, error) {
func GetLabelValues(at *auth.Token, denyPartialResponse bool, labelName string, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -734,7 +737,7 @@ func GetLabelValues(at *auth.Token, labelName string, deadline searchutils.Deadl
}
labelValues = append(labelValues, nr.labelValues...)
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -745,22 +748,24 @@ func GetLabelValues(at *auth.Token, labelName string, deadline searchutils.Deadl
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialLabelValuesResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching label values: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
// Deduplicate label values
labelValues = deduplicateStrings(labelValues)
// Sort labelValues like Prometheus does
sort.Strings(labelValues)
return labelValues, isPartialResult, nil
return labelValues, isPartial, nil
}
// GetTagValueSuffixes returns tag value suffixes for the given tagKey and the given tagValuePrefix.
//
// It can be used for implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
func GetTagValueSuffixes(at *auth.Token, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, deadline searchutils.Deadline) ([]string, bool, error) {
func GetTagValueSuffixes(at *auth.Token, denyPartialResponse bool, tr storage.TimeRange, tagKey, tagValuePrefix string,
delimiter byte, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -801,7 +806,7 @@ func GetTagValueSuffixes(at *auth.Token, tr storage.TimeRange, tagKey, tagValueP
m[suffix] = struct{}{}
}
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -813,19 +818,20 @@ func GetTagValueSuffixes(at *auth.Token, tr storage.TimeRange, tagKey, tagValueP
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialLabelEntriesResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching tag value suffixes: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
suffixes := make([]string, 0, len(m))
for suffix := range m {
suffixes = append(suffixes, suffix)
}
return suffixes, isPartialResult, nil
return suffixes, isPartial, nil
}
// GetLabelEntries returns all the label entries for at until the given deadline.
func GetLabelEntries(at *auth.Token, deadline searchutils.Deadline) ([]storage.TagEntry, bool, error) {
func GetLabelEntries(at *auth.Token, denyPartialResponse bool, deadline searchutils.Deadline) ([]storage.TagEntry, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -863,7 +869,7 @@ func GetLabelEntries(at *auth.Token, deadline searchutils.Deadline) ([]storage.T
}
labelEntries = append(labelEntries, nr.labelEntries...)
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -874,9 +880,10 @@ func GetLabelEntries(at *auth.Token, deadline searchutils.Deadline) ([]storage.T
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialLabelEntriesResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching label entries: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
// Substitute "" with "__name__"
@ -899,7 +906,7 @@ func GetLabelEntries(at *auth.Token, deadline searchutils.Deadline) ([]storage.T
return labelEntries[i].Key > labelEntries[j].Key
})
return labelEntries, isPartialResult, nil
return labelEntries, isPartial, nil
}
func deduplicateLabelEntries(src []storage.TagEntry) []storage.TagEntry {
@ -933,7 +940,7 @@ func deduplicateStrings(a []string) []string {
}
// GetTSDBStatusForDate returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
func GetTSDBStatusForDate(at *auth.Token, deadline searchutils.Deadline, date uint64, topN int) (*storage.TSDBStatus, bool, error) {
func GetTSDBStatusForDate(at *auth.Token, denyPartialResponse bool, deadline searchutils.Deadline, date uint64, topN int) (*storage.TSDBStatus, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -971,7 +978,7 @@ func GetTSDBStatusForDate(at *auth.Token, deadline searchutils.Deadline, date ui
}
statuses = append(statuses, nr.status)
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -981,13 +988,14 @@ func GetTSDBStatusForDate(at *auth.Token, deadline searchutils.Deadline, date ui
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialTSDBStatusResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching tsdb stats: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return nil, true, errors[0]
}
isPartial = true
}
status := mergeTSDBStatuses(statuses, topN)
return status, isPartialResult, nil
return status, isPartial, nil
}
func mergeTSDBStatuses(statuses []*storage.TSDBStatus, topN int) *storage.TSDBStatus {
@ -1037,7 +1045,7 @@ func toTopHeapEntries(m map[string]uint64, topN int) []storage.TopHeapEntry {
}
// GetSeriesCount returns the number of unique series for the given at.
func GetSeriesCount(at *auth.Token, deadline searchutils.Deadline) (uint64, bool, error) {
func GetSeriesCount(at *auth.Token, denyPartialResponse bool, deadline searchutils.Deadline) (uint64, bool, error) {
if deadline.Exceeded() {
return 0, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -1075,7 +1083,7 @@ func GetSeriesCount(at *auth.Token, deadline searchutils.Deadline) (uint64, bool
}
n += nr.n
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -1085,12 +1093,13 @@ func GetSeriesCount(at *auth.Token, deadline searchutils.Deadline) (uint64, bool
// This allows gracefully degrade vmselect in the case
// if certain storageNodes are temporarily unavailable.
partialSeriesCountResults.Inc()
// Log only the first error, since it has no sense in returning all errors.
logger.Errorf("certain storageNodes are unhealthy when fetching series count: %s", errors[0])
isPartialResult = true
if denyPartialResponse {
return 0, true, errors[0]
}
isPartial = true
}
return n, isPartialResult, nil
return n, isPartial, nil
}
type tmpBlocksFileWrapper struct {
@ -1148,9 +1157,9 @@ var metricNamePool = &sync.Pool{
// Data processing is immediately stopped if f returns non-nil error.
// It is the responsibility of f to call b.UnmarshalData before reading timestamps and values from the block.
// It is the responsibility of f to filter blocks according to the given tr.
func ExportBlocks(at *auth.Token, sq *storage.SearchQuery, deadline searchutils.Deadline, f func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error) (bool, error) {
func ExportBlocks(at *auth.Token, sq *storage.SearchQuery, deadline searchutils.Deadline, f func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error) error {
if deadline.Exceeded() {
return false, fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
}
tr := storage.TimeRange{
MinTimestamp: sq.MinTimestamp,
@ -1168,17 +1177,17 @@ func ExportBlocks(at *auth.Token, sq *storage.SearchQuery, deadline searchutils.
metricNamePool.Put(mn)
return nil
}
isPartialResult, err := processSearchQuery(at, sq, true, processBlock, deadline)
_, err := processSearchQuery(at, true, sq, true, processBlock, deadline)
if err != nil {
return true, fmt.Errorf("error occured during export: %w", err)
return fmt.Errorf("error occured during export: %w", err)
}
return isPartialResult, nil
return nil
}
// ProcessSearchQuery performs sq until the given deadline.
//
// Results.RunParallel or Results.Cancel must be called on the returned Results.
func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool, deadline searchutils.Deadline) (*Results, bool, error) {
func ProcessSearchQuery(at *auth.Token, denyPartialResponse bool, sq *storage.SearchQuery, fetchData bool, deadline searchutils.Deadline) (*Results, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -1200,7 +1209,7 @@ func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool,
}
return nil
}
isPartialResult, err := processSearchQuery(at, sq, fetchData, processBlock, deadline)
isPartial, err := processSearchQuery(at, denyPartialResponse, sq, fetchData, processBlock, deadline)
if err != nil {
putTmpBlocksFile(tbfw.tbf)
return nil, false, fmt.Errorf("error occured during search: %w", err)
@ -1224,10 +1233,11 @@ func ProcessSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool,
}
}
rss.packedTimeseries = pts
return &rss, isPartialResult, nil
return &rss, isPartial, nil
}
func processSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool, processBlock func(mb *storage.MetricBlock) error, deadline searchutils.Deadline) (bool, error) {
func processSearchQuery(at *auth.Token, denyPartialResponse bool, sq *storage.SearchQuery, fetchData bool,
processBlock func(mb *storage.MetricBlock) error, deadline searchutils.Deadline) (bool, error) {
requestData := sq.Marshal(nil)
// Send the query to all the storage nodes in parallel.
@ -1255,7 +1265,7 @@ func processSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool,
continue
}
}
isPartialResult := false
isPartial := false
if len(errors) > 0 {
if len(errors) == len(storageNodes) {
// Return only the first error, since it has no sense in returning all errors.
@ -1268,9 +1278,12 @@ func processSearchQuery(at *auth.Token, sq *storage.SearchQuery, fetchData bool,
// Do not return the error, since it may spam logs on busy vmselect
// serving high amounts of requests.
partialSearchResults.Inc()
isPartialResult = true
if denyPartialResponse {
return true, errors[0]
}
isPartial = true
}
return isPartialResult, nil
return isPartial, nil
}
type storageNode struct {

3
app/vmselect/prometheus/label_values_response.qtpl
View file

@ -1,9 +1,10 @@
{% stripspace %}
LabelValuesResponse generates response for /api/v1/label/<labelName>/values .
See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
{% func LabelValuesResponse(labelValues []string) %}
{% func LabelValuesResponse(isPartial bool, labelValues []string) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":[
{% for i, labelValue := range labelValues %}
{%q= labelValue %}

72
app/vmselect/prometheus/label_values_response.qtpl.go
View file

@ -20,48 +20,60 @@ var (
)
//line app/vmselect/prometheus/label_values_response.qtpl:4
func StreamLabelValuesResponse(qw422016 *qt422016.Writer, labelValues []string) {
func StreamLabelValuesResponse(qw422016 *qt422016.Writer, isPartial bool, labelValues []string) {
//line app/vmselect/prometheus/label_values_response.qtpl:4
qw422016.N().S(`{"status":"success","data":[`)
//line app/vmselect/prometheus/label_values_response.qtpl:8
for i, labelValue := range labelValues {
//line app/vmselect/prometheus/label_values_response.qtpl:9
qw422016.N().Q(labelValue)
//line app/vmselect/prometheus/label_values_response.qtpl:10
if i+1 < len(labelValues) {
//line app/vmselect/prometheus/label_values_response.qtpl:10
qw422016.N().S(`,`)
//line app/vmselect/prometheus/label_values_response.qtpl:10
}
//line app/vmselect/prometheus/label_values_response.qtpl:11
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/label_values_response.qtpl:7
if isPartial {
//line app/vmselect/prometheus/label_values_response.qtpl:7
qw422016.N().S(`true`)
//line app/vmselect/prometheus/label_values_response.qtpl:7
} else {
//line app/vmselect/prometheus/label_values_response.qtpl:7
qw422016.N().S(`false`)
//line app/vmselect/prometheus/label_values_response.qtpl:7
}
//line app/vmselect/prometheus/label_values_response.qtpl:7
qw422016.N().S(`,"data":[`)
//line app/vmselect/prometheus/label_values_response.qtpl:9
for i, labelValue := range labelValues {
//line app/vmselect/prometheus/label_values_response.qtpl:10
qw422016.N().Q(labelValue)
//line app/vmselect/prometheus/label_values_response.qtpl:11
if i+1 < len(labelValues) {
//line app/vmselect/prometheus/label_values_response.qtpl:11
qw422016.N().S(`,`)
//line app/vmselect/prometheus/label_values_response.qtpl:11
}
//line app/vmselect/prometheus/label_values_response.qtpl:12
}
//line app/vmselect/prometheus/label_values_response.qtpl:12
qw422016.N().S(`]}`)
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
}
//line app/vmselect/prometheus/label_values_response.qtpl:14
func WriteLabelValuesResponse(qq422016 qtio422016.Writer, labelValues []string) {
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
func WriteLabelValuesResponse(qq422016 qtio422016.Writer, isPartial bool, labelValues []string) {
//line app/vmselect/prometheus/label_values_response.qtpl:15
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/label_values_response.qtpl:14
StreamLabelValuesResponse(qw422016, labelValues)
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
StreamLabelValuesResponse(qw422016, isPartial, labelValues)
//line app/vmselect/prometheus/label_values_response.qtpl:15
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
}
//line app/vmselect/prometheus/label_values_response.qtpl:14
func LabelValuesResponse(labelValues []string) string {
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
func LabelValuesResponse(isPartial bool, labelValues []string) string {
//line app/vmselect/prometheus/label_values_response.qtpl:15
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/label_values_response.qtpl:14
WriteLabelValuesResponse(qb422016, labelValues)
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
WriteLabelValuesResponse(qb422016, isPartial, labelValues)
//line app/vmselect/prometheus/label_values_response.qtpl:15
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
return qs422016
//line app/vmselect/prometheus/label_values_response.qtpl:14
//line app/vmselect/prometheus/label_values_response.qtpl:15
}

3
app/vmselect/prometheus/labels_count_response.qtpl
View file

@ -2,9 +2,10 @@
{% stripspace %}
LabelsCountResponse generates response for /api/v1/labels/count .
{% func LabelsCountResponse(labelEntries []storage.TagEntry) %}
{% func LabelsCountResponse(isPartial bool, labelEntries []storage.TagEntry) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":{
{% for i, e := range labelEntries %}
{%q= e.Key %}:{%d= len(e.Values) %}

80
app/vmselect/prometheus/labels_count_response.qtpl.go
View file

@ -23,52 +23,64 @@ var (
)
//line app/vmselect/prometheus/labels_count_response.qtpl:5
func StreamLabelsCountResponse(qw422016 *qt422016.Writer, labelEntries []storage.TagEntry) {
func StreamLabelsCountResponse(qw422016 *qt422016.Writer, isPartial bool, labelEntries []storage.TagEntry) {
//line app/vmselect/prometheus/labels_count_response.qtpl:5
qw422016.N().S(`{"status":"success","data":{`)
//line app/vmselect/prometheus/labels_count_response.qtpl:9
for i, e := range labelEntries {
//line app/vmselect/prometheus/labels_count_response.qtpl:10
qw422016.N().Q(e.Key)
//line app/vmselect/prometheus/labels_count_response.qtpl:10
qw422016.N().S(`:`)
//line app/vmselect/prometheus/labels_count_response.qtpl:10
qw422016.N().D(len(e.Values))
//line app/vmselect/prometheus/labels_count_response.qtpl:11
if i+1 < len(labelEntries) {
//line app/vmselect/prometheus/labels_count_response.qtpl:11
qw422016.N().S(`,`)
//line app/vmselect/prometheus/labels_count_response.qtpl:11
}
//line app/vmselect/prometheus/labels_count_response.qtpl:12
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/labels_count_response.qtpl:8
if isPartial {
//line app/vmselect/prometheus/labels_count_response.qtpl:8
qw422016.N().S(`true`)
//line app/vmselect/prometheus/labels_count_response.qtpl:8
} else {
//line app/vmselect/prometheus/labels_count_response.qtpl:8
qw422016.N().S(`false`)
//line app/vmselect/prometheus/labels_count_response.qtpl:8
}
//line app/vmselect/prometheus/labels_count_response.qtpl:8
qw422016.N().S(`,"data":{`)
//line app/vmselect/prometheus/labels_count_response.qtpl:10
for i, e := range labelEntries {
//line app/vmselect/prometheus/labels_count_response.qtpl:11
qw422016.N().Q(e.Key)
//line app/vmselect/prometheus/labels_count_response.qtpl:11
qw422016.N().S(`:`)
//line app/vmselect/prometheus/labels_count_response.qtpl:11
qw422016.N().D(len(e.Values))
//line app/vmselect/prometheus/labels_count_response.qtpl:12
if i+1 < len(labelEntries) {
//line app/vmselect/prometheus/labels_count_response.qtpl:12
qw422016.N().S(`,`)
//line app/vmselect/prometheus/labels_count_response.qtpl:12
}
//line app/vmselect/prometheus/labels_count_response.qtpl:13
}
//line app/vmselect/prometheus/labels_count_response.qtpl:13
qw422016.N().S(`}}`)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
}
//line app/vmselect/prometheus/labels_count_response.qtpl:15
func WriteLabelsCountResponse(qq422016 qtio422016.Writer, labelEntries []storage.TagEntry) {
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
func WriteLabelsCountResponse(qq422016 qtio422016.Writer, isPartial bool, labelEntries []storage.TagEntry) {
//line app/vmselect/prometheus/labels_count_response.qtpl:16
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
StreamLabelsCountResponse(qw422016, labelEntries)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
StreamLabelsCountResponse(qw422016, isPartial, labelEntries)
//line app/vmselect/prometheus/labels_count_response.qtpl:16
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
}
//line app/vmselect/prometheus/labels_count_response.qtpl:15
func LabelsCountResponse(labelEntries []storage.TagEntry) string {
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
func LabelsCountResponse(isPartial bool, labelEntries []storage.TagEntry) string {
//line app/vmselect/prometheus/labels_count_response.qtpl:16
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/labels_count_response.qtpl:15
WriteLabelsCountResponse(qb422016, labelEntries)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
WriteLabelsCountResponse(qb422016, isPartial, labelEntries)
//line app/vmselect/prometheus/labels_count_response.qtpl:16
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
return qs422016
//line app/vmselect/prometheus/labels_count_response.qtpl:15
//line app/vmselect/prometheus/labels_count_response.qtpl:16
}

3
app/vmselect/prometheus/labels_response.qtpl
View file

@ -1,9 +1,10 @@
{% stripspace %}
LabelsResponse generates response for /api/v1/labels .
See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
{% func LabelsResponse(labels []string) %}
{% func LabelsResponse(isPartial bool, labels []string) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":[
{% for i, label := range labels %}
{%q= label %}

72
app/vmselect/prometheus/labels_response.qtpl.go
View file

@ -20,48 +20,60 @@ var (
)
//line app/vmselect/prometheus/labels_response.qtpl:4
func StreamLabelsResponse(qw422016 *qt422016.Writer, labels []string) {
func StreamLabelsResponse(qw422016 *qt422016.Writer, isPartial bool, labels []string) {
//line app/vmselect/prometheus/labels_response.qtpl:4
qw422016.N().S(`{"status":"success","data":[`)
//line app/vmselect/prometheus/labels_response.qtpl:8
for i, label := range labels {
//line app/vmselect/prometheus/labels_response.qtpl:9
qw422016.N().Q(label)
//line app/vmselect/prometheus/labels_response.qtpl:10
if i+1 < len(labels) {
//line app/vmselect/prometheus/labels_response.qtpl:10
qw422016.N().S(`,`)
//line app/vmselect/prometheus/labels_response.qtpl:10
}
//line app/vmselect/prometheus/labels_response.qtpl:11
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/labels_response.qtpl:7
if isPartial {
//line app/vmselect/prometheus/labels_response.qtpl:7
qw422016.N().S(`true`)
//line app/vmselect/prometheus/labels_response.qtpl:7
} else {
//line app/vmselect/prometheus/labels_response.qtpl:7
qw422016.N().S(`false`)
//line app/vmselect/prometheus/labels_response.qtpl:7
}
//line app/vmselect/prometheus/labels_response.qtpl:7
qw422016.N().S(`,"data":[`)
//line app/vmselect/prometheus/labels_response.qtpl:9
for i, label := range labels {
//line app/vmselect/prometheus/labels_response.qtpl:10
qw422016.N().Q(label)
//line app/vmselect/prometheus/labels_response.qtpl:11
if i+1 < len(labels) {
//line app/vmselect/prometheus/labels_response.qtpl:11
qw422016.N().S(`,`)
//line app/vmselect/prometheus/labels_response.qtpl:11
}
//line app/vmselect/prometheus/labels_response.qtpl:12
}
//line app/vmselect/prometheus/labels_response.qtpl:12
qw422016.N().S(`]}`)
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
}
//line app/vmselect/prometheus/labels_response.qtpl:14
func WriteLabelsResponse(qq422016 qtio422016.Writer, labels []string) {
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
func WriteLabelsResponse(qq422016 qtio422016.Writer, isPartial bool, labels []string) {
//line app/vmselect/prometheus/labels_response.qtpl:15
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/labels_response.qtpl:14
StreamLabelsResponse(qw422016, labels)
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
StreamLabelsResponse(qw422016, isPartial, labels)
//line app/vmselect/prometheus/labels_response.qtpl:15
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
}
//line app/vmselect/prometheus/labels_response.qtpl:14
func LabelsResponse(labels []string) string {
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
func LabelsResponse(isPartial bool, labels []string) string {
//line app/vmselect/prometheus/labels_response.qtpl:15
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/labels_response.qtpl:14
WriteLabelsResponse(qb422016, labels)
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
WriteLabelsResponse(qb422016, isPartial, labels)
//line app/vmselect/prometheus/labels_response.qtpl:15
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
return qs422016
//line app/vmselect/prometheus/labels_response.qtpl:14
//line app/vmselect/prometheus/labels_response.qtpl:15
}

103
app/vmselect/prometheus/prometheus.go
View file

@ -87,13 +87,13 @@ func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter,
MaxTimestamp: end,
TagFilterss: tagFilterss,
}
rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, true, deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
rss, isPartial, err := netstorage.ProcessSearchQuery(at, denyPartialResponse, sq, true, deadline)
if err != nil {
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
rss.Cancel()
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
if isPartial {
return fmt.Errorf("cannot export federated metrics, because some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
@ -168,7 +168,7 @@ func ExportCSVHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter
resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
doneCh := make(chan error)
go func() {
isPartial, err := netstorage.ExportBlocks(at, sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
err := netstorage.ExportBlocks(at, sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
if err := bw.Error(); err != nil {
return err
}
@ -187,9 +187,6 @@ func ExportCSVHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter
exportBlockPool.Put(xb)
return nil
})
if err == nil && isPartial && searchutils.GetDenyPartialResponse(r) {
err = fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
close(resultsCh)
doneCh <- err
}()
@ -258,7 +255,7 @@ func ExportNativeHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
_, _ = bw.Write(trBuf)
// Marshal native blocks.
isPartial, err := netstorage.ExportBlocks(at, sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
err = netstorage.ExportBlocks(at, sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
if err := bw.Error(); err != nil {
return err
}
@ -286,9 +283,6 @@ func ExportNativeHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
bbPool.Put(dstBuf)
return err
})
if err == nil && isPartial && searchutils.GetDenyPartialResponse(r) {
err = fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
if err != nil {
return err
}
@ -414,13 +408,13 @@ func exportHandler(at *auth.Token, w http.ResponseWriter, r *http.Request, match
resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
doneCh := make(chan error)
if !reduceMemUsage {
rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, true, deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
rss, isPartial, err := netstorage.ProcessSearchQuery(at, denyPartialResponse, sq, true, deadline)
if err != nil {
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
rss.Cancel()
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
if isPartial {
return fmt.Errorf("cannot export data, because some of vmstorage nodes are unhealthy")
}
go func() {
err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error {
@ -441,7 +435,7 @@ func exportHandler(at *auth.Token, w http.ResponseWriter, r *http.Request, match
}()
} else {
go func() {
isPartial, err := netstorage.ExportBlocks(at, sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
err := netstorage.ExportBlocks(at, sq, deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
if err := bw.Error(); err != nil {
return err
}
@ -458,9 +452,6 @@ func exportHandler(at *auth.Token, w http.ResponseWriter, r *http.Request, match
exportBlockPool.Put(xb)
return nil
})
if err == nil && isPartial && searchutils.GetDenyPartialResponse(r) {
err = fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
close(resultsCh)
doneCh <- err
}()
@ -578,10 +569,11 @@ func LabelValuesHandler(startTime time.Time, at *auth.Token, labelName string, w
}
var labelValues []string
var isPartial bool
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
if len(r.Form["match[]"]) == 0 {
if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
var err error
labelValues, isPartial, err = netstorage.GetLabelValues(at, labelName, deadline)
labelValues, isPartial, err = netstorage.GetLabelValues(at, denyPartialResponse, labelName, deadline)
if err != nil {
return fmt.Errorf(`cannot obtain label values for %q: %w`, labelName, err)
}
@ -599,7 +591,7 @@ func LabelValuesHandler(startTime time.Time, at *auth.Token, labelName string, w
MinTimestamp: start,
MaxTimestamp: end,
}
labelValues, isPartial, err = netstorage.GetLabelValuesOnTimeRange(at, labelName, tr, deadline)
labelValues, isPartial, err = netstorage.GetLabelValuesOnTimeRange(at, denyPartialResponse, labelName, tr, deadline)
if err != nil {
return fmt.Errorf(`cannot obtain label values on time range for %q: %w`, labelName, err)
}
@ -622,19 +614,16 @@ func LabelValuesHandler(startTime time.Time, at *auth.Token, labelName string, w
if err != nil {
return err
}
labelValues, isPartial, err = labelValuesWithMatches(at, labelName, matches, start, end, deadline)
labelValues, isPartial, err = labelValuesWithMatches(at, denyPartialResponse, labelName, matches, start, end, deadline)
if err != nil {
return fmt.Errorf("cannot obtain label values for %q, match[]=%q, start=%d, end=%d: %w", labelName, matches, start, end, err)
}
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteLabelValuesResponse(bw, labelValues)
WriteLabelValuesResponse(bw, isPartial, labelValues)
if err := bw.Flush(); err != nil {
return err
}
@ -642,7 +631,7 @@ func LabelValuesHandler(startTime time.Time, at *auth.Token, labelName string, w
return nil
}
func labelValuesWithMatches(at *auth.Token, labelName string, matches []string, start, end int64, deadline searchutils.Deadline) ([]string, bool, error) {
func labelValuesWithMatches(at *auth.Token, denyPartialResponse bool, labelName string, matches []string, start, end int64, deadline searchutils.Deadline) ([]string, bool, error) {
if len(matches) == 0 {
logger.Panicf("BUG: matches must be non-empty")
}
@ -673,7 +662,7 @@ func labelValuesWithMatches(at *auth.Token, labelName string, matches []string,
MaxTimestamp: end,
TagFilterss: tagFilterss,
}
rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, false, deadline)
rss, isPartial, err := netstorage.ProcessSearchQuery(at, denyPartialResponse, sq, false, deadline)
if err != nil {
return nil, false, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
@ -707,17 +696,16 @@ var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
// LabelsCountHandler processes /api/v1/labels/count request.
func LabelsCountHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
labelEntries, isPartial, err := netstorage.GetLabelEntries(at, deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
labelEntries, isPartial, err := netstorage.GetLabelEntries(at, denyPartialResponse, deadline)
if err != nil {
return fmt.Errorf(`cannot obtain label entries: %w`, err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteLabelsCountResponse(bw, labelEntries)
WriteLabelsCountResponse(bw, isPartial, labelEntries)
if err := bw.Flush(); err != nil {
return err
}
@ -761,17 +749,16 @@ func TSDBStatusHandler(startTime time.Time, at *auth.Token, w http.ResponseWrite
}
topN = n
}
status, isPartial, err := netstorage.GetTSDBStatusForDate(at, deadline, date, topN)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
status, isPartial, err := netstorage.GetTSDBStatusForDate(at, denyPartialResponse, deadline, date, topN)
if err != nil {
return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteTSDBStatusResponse(bw, status)
WriteTSDBStatusResponse(bw, isPartial, status)
if err := bw.Flush(); err != nil {
return err
}
@ -791,10 +778,11 @@ func LabelsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
}
var labels []string
var isPartial bool
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
if len(r.Form["match[]"]) == 0 {
if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
var err error
labels, isPartial, err = netstorage.GetLabels(at, deadline)
labels, isPartial, err = netstorage.GetLabels(at, denyPartialResponse, deadline)
if err != nil {
return fmt.Errorf("cannot obtain labels: %w", err)
}
@ -812,7 +800,7 @@ func LabelsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
MinTimestamp: start,
MaxTimestamp: end,
}
labels, isPartial, err = netstorage.GetLabelsOnTimeRange(at, tr, deadline)
labels, isPartial, err = netstorage.GetLabelsOnTimeRange(at, denyPartialResponse, tr, deadline)
if err != nil {
return fmt.Errorf("cannot obtain labels on time range: %w", err)
}
@ -833,19 +821,16 @@ func LabelsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
if err != nil {
return err
}
labels, isPartial, err = labelsWithMatches(at, matches, start, end, deadline)
labels, isPartial, err = labelsWithMatches(at, denyPartialResponse, matches, start, end, deadline)
if err != nil {
return fmt.Errorf("cannot obtain labels for match[]=%q, start=%d, end=%d: %w", matches, start, end, err)
}
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteLabelsResponse(bw, labels)
WriteLabelsResponse(bw, isPartial, labels)
if err := bw.Flush(); err != nil {
return err
}
@ -853,7 +838,7 @@ func LabelsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
return nil
}
func labelsWithMatches(at *auth.Token, matches []string, start, end int64, deadline searchutils.Deadline) ([]string, bool, error) {
func labelsWithMatches(at *auth.Token, denyPartialResponse bool, matches []string, start, end int64, deadline searchutils.Deadline) ([]string, bool, error) {
if len(matches) == 0 {
logger.Panicf("BUG: matches must be non-empty")
}
@ -871,7 +856,7 @@ func labelsWithMatches(at *auth.Token, matches []string, start, end int64, deadl
MaxTimestamp: end,
TagFilterss: tagFilterss,
}
rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, false, deadline)
rss, isPartial, err := netstorage.ProcessSearchQuery(at, denyPartialResponse, sq, false, deadline)
if err != nil {
return nil, false, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
@ -906,17 +891,16 @@ var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
// SeriesCountHandler processes /api/v1/series/count request.
func SeriesCountHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
n, isPartial, err := netstorage.GetSeriesCount(at, deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
n, isPartial, err := netstorage.GetSeriesCount(at, denyPartialResponse, deadline)
if err != nil {
return fmt.Errorf("cannot obtain series count: %w", err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteSeriesCountResponse(bw, n)
WriteSeriesCountResponse(bw, isPartial, n)
if err := bw.Flush(); err != nil {
return err
}
@ -967,14 +951,11 @@ func SeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
MaxTimestamp: end,
TagFilterss: tagFilterss,
}
rss, isPartial, err := netstorage.ProcessSearchQuery(at, sq, false, deadline)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
rss, isPartial, err := netstorage.ProcessSearchQuery(at, denyPartialResponse, sq, false, deadline)
if err != nil {
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
if isPartial && searchutils.GetDenyPartialResponse(r) {
rss.Cancel()
return fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
@ -995,7 +976,7 @@ func SeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
doneCh <- err
}()
// WriteSeriesResponse must consume all the data from resultsCh.
WriteSeriesResponse(bw, resultsCh)
WriteSeriesResponse(bw, isPartial, resultsCh)
if err := bw.Flush(); err != nil {
return err
}
@ -1119,7 +1100,7 @@ func QueryHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteQueryResponse(bw, result)
WriteQueryResponse(bw, ec.IsPartialResponse, result)
if err := bw.Flush(); err != nil {
return err
}
@ -1221,7 +1202,7 @@ func queryRangeHandler(startTime time.Time, at *auth.Token, w http.ResponseWrite
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
WriteQueryRangeResponse(bw, result)
WriteQueryRangeResponse(bw, ec.IsPartialResponse, result)
if err := bw.Flush(); err != nil {
return err
}

3
app/vmselect/prometheus/query_range_response.qtpl
View file

@ -5,9 +5,10 @@
{% stripspace %}
QueryRangeResponse generates response for /api/v1/query_range.
See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
{% func QueryRangeResponse(rs []netstorage.Result) %}
{% func QueryRangeResponse(isPartial bool, rs []netstorage.Result) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":{
"resultType":"matrix",
"result":[

108
app/vmselect/prometheus/query_range_response.qtpl.go
View file

@ -25,94 +25,106 @@ var (
)
//line app/vmselect/prometheus/query_range_response.qtpl:8
func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result) {
func StreamQueryRangeResponse(qw422016 *qt422016.Writer, isPartial bool, rs []netstorage.Result) {
//line app/vmselect/prometheus/query_range_response.qtpl:8
qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
//line app/vmselect/prometheus/query_range_response.qtpl:14
if len(rs) > 0 {
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/query_range_response.qtpl:11
if isPartial {
//line app/vmselect/prometheus/query_range_response.qtpl:11
qw422016.N().S(`true`)
//line app/vmselect/prometheus/query_range_response.qtpl:11
} else {
//line app/vmselect/prometheus/query_range_response.qtpl:11
qw422016.N().S(`false`)
//line app/vmselect/prometheus/query_range_response.qtpl:11
}
//line app/vmselect/prometheus/query_range_response.qtpl:11
qw422016.N().S(`,"data":{"resultType":"matrix","result":[`)
//line app/vmselect/prometheus/query_range_response.qtpl:15
streamqueryRangeLine(qw422016, &rs[0])
if len(rs) > 0 {
//line app/vmselect/prometheus/query_range_response.qtpl:16
streamqueryRangeLine(qw422016, &rs[0])
//line app/vmselect/prometheus/query_range_response.qtpl:17
rs = rs[1:]
//line app/vmselect/prometheus/query_range_response.qtpl:17
for i := range rs {
//line app/vmselect/prometheus/query_range_response.qtpl:17
qw422016.N().S(`,`)
//line app/vmselect/prometheus/query_range_response.qtpl:18
streamqueryRangeLine(qw422016, &rs[i])
for i := range rs {
//line app/vmselect/prometheus/query_range_response.qtpl:18
qw422016.N().S(`,`)
//line app/vmselect/prometheus/query_range_response.qtpl:19
streamqueryRangeLine(qw422016, &rs[i])
//line app/vmselect/prometheus/query_range_response.qtpl:20
}
//line app/vmselect/prometheus/query_range_response.qtpl:20
//line app/vmselect/prometheus/query_range_response.qtpl:21
}
//line app/vmselect/prometheus/query_range_response.qtpl:20
//line app/vmselect/prometheus/query_range_response.qtpl:21
qw422016.N().S(`]}}`)
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
}
//line app/vmselect/prometheus/query_range_response.qtpl:24
func WriteQueryRangeResponse(qq422016 qtio422016.Writer, rs []netstorage.Result) {
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
func WriteQueryRangeResponse(qq422016 qtio422016.Writer, isPartial bool, rs []netstorage.Result) {
//line app/vmselect/prometheus/query_range_response.qtpl:25
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/query_range_response.qtpl:24
StreamQueryRangeResponse(qw422016, rs)
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
StreamQueryRangeResponse(qw422016, isPartial, rs)
//line app/vmselect/prometheus/query_range_response.qtpl:25
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
}
//line app/vmselect/prometheus/query_range_response.qtpl:24
func QueryRangeResponse(rs []netstorage.Result) string {
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
func QueryRangeResponse(isPartial bool, rs []netstorage.Result) string {
//line app/vmselect/prometheus/query_range_response.qtpl:25
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/query_range_response.qtpl:24
WriteQueryRangeResponse(qb422016, rs)
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
WriteQueryRangeResponse(qb422016, isPartial, rs)
//line app/vmselect/prometheus/query_range_response.qtpl:25
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
return qs422016
//line app/vmselect/prometheus/query_range_response.qtpl:24
//line app/vmselect/prometheus/query_range_response.qtpl:25
}
//line app/vmselect/prometheus/query_range_response.qtpl:26
//line app/vmselect/prometheus/query_range_response.qtpl:27
func streamqueryRangeLine(qw422016 *qt422016.Writer, r *netstorage.Result) {
//line app/vmselect/prometheus/query_range_response.qtpl:26
//line app/vmselect/prometheus/query_range_response.qtpl:27
qw422016.N().S(`{"metric":`)
//line app/vmselect/prometheus/query_range_response.qtpl:28
//line app/vmselect/prometheus/query_range_response.qtpl:29
streammetricNameObject(qw422016, &r.MetricName)
//line app/vmselect/prometheus/query_range_response.qtpl:28
//line app/vmselect/prometheus/query_range_response.qtpl:29
qw422016.N().S(`,"values":`)
//line app/vmselect/prometheus/query_range_response.qtpl:29
//line app/vmselect/prometheus/query_range_response.qtpl:30
streamvaluesWithTimestamps(qw422016, r.Values, r.Timestamps)
//line app/vmselect/prometheus/query_range_response.qtpl:29
//line app/vmselect/prometheus/query_range_response.qtpl:30
qw422016.N().S(`}`)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
}
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
func writequeryRangeLine(qq422016 qtio422016.Writer, r *netstorage.Result) {
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
streamqueryRangeLine(qw422016, r)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
}
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
func queryRangeLine(r *netstorage.Result) string {
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
writequeryRangeLine(qb422016, r)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
return qs422016
//line app/vmselect/prometheus/query_range_response.qtpl:31
//line app/vmselect/prometheus/query_range_response.qtpl:32
}

3
app/vmselect/prometheus/query_response.qtpl
View file

@ -5,9 +5,10 @@
{% stripspace %}
QueryResponse generates response for /api/v1/query.
See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
{% func QueryResponse(rs []netstorage.Result) %}
{% func QueryResponse(isPartial bool, rs []netstorage.Result) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":{
"resultType":"vector",
"result":[

84
app/vmselect/prometheus/query_response.qtpl.go
View file

@ -25,70 +25,82 @@ var (
)
//line app/vmselect/prometheus/query_response.qtpl:8
func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result) {
func StreamQueryResponse(qw422016 *qt422016.Writer, isPartial bool, rs []netstorage.Result) {
//line app/vmselect/prometheus/query_response.qtpl:8
qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
//line app/vmselect/prometheus/query_response.qtpl:14
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/query_response.qtpl:11
if isPartial {
//line app/vmselect/prometheus/query_response.qtpl:11
qw422016.N().S(`true`)
//line app/vmselect/prometheus/query_response.qtpl:11
} else {
//line app/vmselect/prometheus/query_response.qtpl:11
qw422016.N().S(`false`)
//line app/vmselect/prometheus/query_response.qtpl:11
}
//line app/vmselect/prometheus/query_response.qtpl:11
qw422016.N().S(`,"data":{"resultType":"vector","result":[`)
//line app/vmselect/prometheus/query_response.qtpl:15
if len(rs) > 0 {
//line app/vmselect/prometheus/query_response.qtpl:14
//line app/vmselect/prometheus/query_response.qtpl:15
qw422016.N().S(`{"metric":`)
//line app/vmselect/prometheus/query_response.qtpl:16
//line app/vmselect/prometheus/query_response.qtpl:17
streammetricNameObject(qw422016, &rs[0].MetricName)
//line app/vmselect/prometheus/query_response.qtpl:16
//line app/vmselect/prometheus/query_response.qtpl:17
qw422016.N().S(`,"value":`)
//line app/vmselect/prometheus/query_response.qtpl:17
//line app/vmselect/prometheus/query_response.qtpl:18
streammetricRow(qw422016, rs[0].Timestamps[0], rs[0].Values[0])
//line app/vmselect/prometheus/query_response.qtpl:17
//line app/vmselect/prometheus/query_response.qtpl:18
qw422016.N().S(`}`)
//line app/vmselect/prometheus/query_response.qtpl:19
//line app/vmselect/prometheus/query_response.qtpl:20
rs = rs[1:]
//line app/vmselect/prometheus/query_response.qtpl:20
for i := range rs {
//line app/vmselect/prometheus/query_response.qtpl:21
for i := range rs {
//line app/vmselect/prometheus/query_response.qtpl:22
r := &rs[i]
//line app/vmselect/prometheus/query_response.qtpl:21
//line app/vmselect/prometheus/query_response.qtpl:22
qw422016.N().S(`,{"metric":`)
//line app/vmselect/prometheus/query_response.qtpl:23
//line app/vmselect/prometheus/query_response.qtpl:24
streammetricNameObject(qw422016, &r.MetricName)
//line app/vmselect/prometheus/query_response.qtpl:23
//line app/vmselect/prometheus/query_response.qtpl:24
qw422016.N().S(`,"value":`)
//line app/vmselect/prometheus/query_response.qtpl:24
//line app/vmselect/prometheus/query_response.qtpl:25
streammetricRow(qw422016, r.Timestamps[0], r.Values[0])
//line app/vmselect/prometheus/query_response.qtpl:24
//line app/vmselect/prometheus/query_response.qtpl:25
qw422016.N().S(`}`)
//line app/vmselect/prometheus/query_response.qtpl:26
//line app/vmselect/prometheus/query_response.qtpl:27
}
//line app/vmselect/prometheus/query_response.qtpl:27
//line app/vmselect/prometheus/query_response.qtpl:28
}
//line app/vmselect/prometheus/query_response.qtpl:27
//line app/vmselect/prometheus/query_response.qtpl:28
qw422016.N().S(`]}}`)
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
}
//line app/vmselect/prometheus/query_response.qtpl:31
func WriteQueryResponse(qq422016 qtio422016.Writer, rs []netstorage.Result) {
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
func WriteQueryResponse(qq422016 qtio422016.Writer, isPartial bool, rs []netstorage.Result) {
//line app/vmselect/prometheus/query_response.qtpl:32
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/query_response.qtpl:31
StreamQueryResponse(qw422016, rs)
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
StreamQueryResponse(qw422016, isPartial, rs)
//line app/vmselect/prometheus/query_response.qtpl:32
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
}
//line app/vmselect/prometheus/query_response.qtpl:31
func QueryResponse(rs []netstorage.Result) string {
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
func QueryResponse(isPartial bool, rs []netstorage.Result) string {
//line app/vmselect/prometheus/query_response.qtpl:32
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/query_response.qtpl:31
WriteQueryResponse(qb422016, rs)
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
WriteQueryResponse(qb422016, isPartial, rs)
//line app/vmselect/prometheus/query_response.qtpl:32
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
return qs422016
//line app/vmselect/prometheus/query_response.qtpl:31
//line app/vmselect/prometheus/query_response.qtpl:32
}

3
app/vmselect/prometheus/series_count_response.qtpl
View file

@ -1,8 +1,9 @@
{% stripspace %}
SeriesCountResponse generates response for /api/v1/series/count .
{% func SeriesCountResponse(n uint64) %}
{% func SeriesCountResponse(isPartial bool, n uint64) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":[{%dl int64(n) %}]
}
{% endfunc %}

52
app/vmselect/prometheus/series_count_response.qtpl.go
View file

@ -20,38 +20,50 @@ var (
)
//line app/vmselect/prometheus/series_count_response.qtpl:3
func StreamSeriesCountResponse(qw422016 *qt422016.Writer, n uint64) {
func StreamSeriesCountResponse(qw422016 *qt422016.Writer, isPartial bool, n uint64) {
//line app/vmselect/prometheus/series_count_response.qtpl:3
qw422016.N().S(`{"status":"success","data":[`)
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/series_count_response.qtpl:6
if isPartial {
//line app/vmselect/prometheus/series_count_response.qtpl:6
qw422016.N().S(`true`)
//line app/vmselect/prometheus/series_count_response.qtpl:6
} else {
//line app/vmselect/prometheus/series_count_response.qtpl:6
qw422016.N().S(`false`)
//line app/vmselect/prometheus/series_count_response.qtpl:6
}
//line app/vmselect/prometheus/series_count_response.qtpl:6
qw422016.N().S(`,"data":[`)
//line app/vmselect/prometheus/series_count_response.qtpl:7
qw422016.N().DL(int64(n))
//line app/vmselect/prometheus/series_count_response.qtpl:6
//line app/vmselect/prometheus/series_count_response.qtpl:7
qw422016.N().S(`]}`)
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
}
//line app/vmselect/prometheus/series_count_response.qtpl:8
func WriteSeriesCountResponse(qq422016 qtio422016.Writer, n uint64) {
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
func WriteSeriesCountResponse(qq422016 qtio422016.Writer, isPartial bool, n uint64) {
//line app/vmselect/prometheus/series_count_response.qtpl:9
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/series_count_response.qtpl:8
StreamSeriesCountResponse(qw422016, n)
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
StreamSeriesCountResponse(qw422016, isPartial, n)
//line app/vmselect/prometheus/series_count_response.qtpl:9
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
}
//line app/vmselect/prometheus/series_count_response.qtpl:8
func SeriesCountResponse(n uint64) string {
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
func SeriesCountResponse(isPartial bool, n uint64) string {
//line app/vmselect/prometheus/series_count_response.qtpl:9
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/series_count_response.qtpl:8
WriteSeriesCountResponse(qb422016, n)
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
WriteSeriesCountResponse(qb422016, isPartial, n)
//line app/vmselect/prometheus/series_count_response.qtpl:9
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
return qs422016
//line app/vmselect/prometheus/series_count_response.qtpl:8
//line app/vmselect/prometheus/series_count_response.qtpl:9
}

3
app/vmselect/prometheus/series_response.qtpl
View file

@ -5,9 +5,10 @@
{% stripspace %}
SeriesResponse generates response for /api/v1/series.
See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers
{% func SeriesResponse(resultsCh <-chan *quicktemplate.ByteBuffer) %}
{% func SeriesResponse(isPartial bool, resultsCh <-chan *quicktemplate.ByteBuffer) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":[
{% code bb, ok := <-resultsCh %}
{% if ok %}

74
app/vmselect/prometheus/series_response.qtpl.go
View file

@ -25,59 +25,71 @@ var (
)
//line app/vmselect/prometheus/series_response.qtpl:8
func StreamSeriesResponse(qw422016 *qt422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
func StreamSeriesResponse(qw422016 *qt422016.Writer, isPartial bool, resultsCh <-chan *quicktemplate.ByteBuffer) {
//line app/vmselect/prometheus/series_response.qtpl:8
qw422016.N().S(`{"status":"success","data":[`)
//line app/vmselect/prometheus/series_response.qtpl:12
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/series_response.qtpl:11
if isPartial {
//line app/vmselect/prometheus/series_response.qtpl:11
qw422016.N().S(`true`)
//line app/vmselect/prometheus/series_response.qtpl:11
} else {
//line app/vmselect/prometheus/series_response.qtpl:11
qw422016.N().S(`false`)
//line app/vmselect/prometheus/series_response.qtpl:11
}
//line app/vmselect/prometheus/series_response.qtpl:11
qw422016.N().S(`,"data":[`)
//line app/vmselect/prometheus/series_response.qtpl:13
bb, ok := <-resultsCh
//line app/vmselect/prometheus/series_response.qtpl:13
if ok {
//line app/vmselect/prometheus/series_response.qtpl:14
qw422016.N().Z(bb.B)
if ok {
//line app/vmselect/prometheus/series_response.qtpl:15
qw422016.N().Z(bb.B)
//line app/vmselect/prometheus/series_response.qtpl:16
quicktemplate.ReleaseByteBuffer(bb)
//line app/vmselect/prometheus/series_response.qtpl:16
for bb := range resultsCh {
//line app/vmselect/prometheus/series_response.qtpl:16
qw422016.N().S(`,`)
//line app/vmselect/prometheus/series_response.qtpl:17
qw422016.N().Z(bb.B)
for bb := range resultsCh {
//line app/vmselect/prometheus/series_response.qtpl:17
qw422016.N().S(`,`)
//line app/vmselect/prometheus/series_response.qtpl:18
qw422016.N().Z(bb.B)
//line app/vmselect/prometheus/series_response.qtpl:19
quicktemplate.ReleaseByteBuffer(bb)
//line app/vmselect/prometheus/series_response.qtpl:19
//line app/vmselect/prometheus/series_response.qtpl:20
}
//line app/vmselect/prometheus/series_response.qtpl:20
//line app/vmselect/prometheus/series_response.qtpl:21
}
//line app/vmselect/prometheus/series_response.qtpl:20
//line app/vmselect/prometheus/series_response.qtpl:21
qw422016.N().S(`]}`)
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
}
//line app/vmselect/prometheus/series_response.qtpl:23
func WriteSeriesResponse(qq422016 qtio422016.Writer, resultsCh <-chan *quicktemplate.ByteBuffer) {
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
func WriteSeriesResponse(qq422016 qtio422016.Writer, isPartial bool, resultsCh <-chan *quicktemplate.ByteBuffer) {
//line app/vmselect/prometheus/series_response.qtpl:24
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/series_response.qtpl:23
StreamSeriesResponse(qw422016, resultsCh)
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
StreamSeriesResponse(qw422016, isPartial, resultsCh)
//line app/vmselect/prometheus/series_response.qtpl:24
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
}
//line app/vmselect/prometheus/series_response.qtpl:23
func SeriesResponse(resultsCh <-chan *quicktemplate.ByteBuffer) string {
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
func SeriesResponse(isPartial bool, resultsCh <-chan *quicktemplate.ByteBuffer) string {
//line app/vmselect/prometheus/series_response.qtpl:24
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/series_response.qtpl:23
WriteSeriesResponse(qb422016, resultsCh)
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
WriteSeriesResponse(qb422016, isPartial, resultsCh)
//line app/vmselect/prometheus/series_response.qtpl:24
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
return qs422016
//line app/vmselect/prometheus/series_response.qtpl:23
//line app/vmselect/prometheus/series_response.qtpl:24
}

3
app/vmselect/prometheus/tsdb_status_response.qtpl
View file

@ -2,9 +2,10 @@
{% stripspace %}
TSDBStatusResponse generates response for /api/v1/status/tsdb .
{% func TSDBStatusResponse(status *storage.TSDBStatus) %}
{% func TSDBStatusResponse(isPartial bool, status *storage.TSDBStatus) %}
{
"status":"success",
"isPartial":{% if isPartial %}true{% else %}false{% endif %},
"data":{
"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
"labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %},

182
app/vmselect/prometheus/tsdb_status_response.qtpl.go
View file

@ -23,101 +23,113 @@ var (
)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBStatus) {
func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, isPartial bool, status *storage.TSDBStatus) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
qw422016.N().S(`{"status":"success","data":{"seriesCountByMetricName":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
qw422016.N().S(`,"labelValueCountByLabelName":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
qw422016.N().S(`,"seriesCountByLabelValuePair":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
qw422016.N().S(`}}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
StreamTSDBStatusResponse(qw422016, status)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
func TSDBStatusResponse(status *storage.TSDBStatus) string {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
WriteTSDBStatusResponse(qb422016, status)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
return qs422016
//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
qw422016.N().S(`[`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
for i, e := range a {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
qw422016.N().S(`{"name":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
qw422016.N().Q(e.Name)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
qw422016.N().S(`,"value":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
qw422016.N().D(int(e.Count))
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
qw422016.N().S(`}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
if i+1 < len(a) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
qw422016.N().S(`,`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
qw422016.N().S(`{"status":"success","isPartial":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
if isPartial {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
qw422016.N().S(`true`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
} else {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
qw422016.N().S(`false`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
qw422016.N().S(`]`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
qw422016.N().S(`,"data":{"seriesCountByMetricName":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
qw422016.N().S(`,"labelValueCountByLabelName":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
qw422016.N().S(`,"seriesCountByLabelValuePair":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:12
streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:12
qw422016.N().S(`}}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, isPartial bool, status *storage.TSDBStatus) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
streamtsdbStatusEntries(qw422016, a)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
StreamTSDBStatusResponse(qw422016, isPartial, status)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
func tsdbStatusEntries(a []storage.TopHeapEntry) string {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
func TSDBStatusResponse(isPartial bool, status *storage.TSDBStatus) string {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
writetsdbStatusEntries(qb422016, a)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
WriteTSDBStatusResponse(qb422016, isPartial, status)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
return qs422016
//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
qw422016.N().S(`[`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
for i, e := range a {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
qw422016.N().S(`{"name":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
qw422016.N().Q(e.Name)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
qw422016.N().S(`,"value":`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qw422016.N().D(int(e.Count))
//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
qw422016.N().S(`}`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
if i+1 < len(a) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
qw422016.N().S(`,`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
qw422016.N().S(`]`)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
streamtsdbStatusEntries(qw422016, a)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
}
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
func tsdbStatusEntries(a []storage.TopHeapEntry) string {
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
writetsdbStatusEntries(qb422016, a)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
return qs422016
//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
}

19
app/vmselect/promql/eval.go
View file

@ -100,6 +100,9 @@ type EvalConfig struct {
DenyPartialResponse bool
// IsPartialResponse is set during query execution and can be used by Exec caller after query execution.
IsPartialResponse bool
timestamps []int64
timestampsOnce sync.Once
}
@ -115,11 +118,18 @@ func newEvalConfig(src *EvalConfig) *EvalConfig {
ec.MayCache = src.MayCache
ec.LookbackDelta = src.LookbackDelta
ec.DenyPartialResponse = src.DenyPartialResponse
ec.IsPartialResponse = src.IsPartialResponse
// do not copy src.timestamps - they must be generated again.
return &ec
}
func (ec *EvalConfig) updateIsPartialResponse(isPartialResponse bool) {
if !ec.IsPartialResponse {
ec.IsPartialResponse = isPartialResponse
}
}
func (ec *EvalConfig) validate() {
if ec.Start > ec.End {
logger.Panicf("BUG: start cannot exceed end; got %d vs %d", ec.Start, ec.End)
@ -475,6 +485,7 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.E
if err != nil {
return nil, err
}
ec.updateIsPartialResponse(ecNew.IsPartialResponse)
if offset != 0 && len(rvs) > 0 {
// Make a copy of timestamps, since they may be used in other values.
srcTimestamps := rvs[0].Timestamps
@ -523,6 +534,7 @@ func evalRollupFuncWithSubquery(ec *EvalConfig, name string, rf rollupFunc, expr
if err != nil {
return nil, err
}
ec.updateIsPartialResponse(ecSQ.IsPartialResponse)
if len(tssSQ) == 0 {
if name == "absent_over_time" {
tss := evalNumber(ec, 1)
@ -666,14 +678,11 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
MaxTimestamp: ec.End,
TagFilterss: [][]storage.TagFilter{tfs},
}
rss, isPartial, err := netstorage.ProcessSearchQuery(ec.AuthToken, sq, true, ec.Deadline)
rss, isPartial, err := netstorage.ProcessSearchQuery(ec.AuthToken, ec.DenyPartialResponse, sq, true, ec.Deadline)
if err != nil {
return nil, err
}
if isPartial && ec.DenyPartialResponse {
rss.Cancel()
return nil, fmt.Errorf("cannot return full response, since some of vmstorage nodes are unavailable")
}
ec.updateIsPartialResponse(isPartial)
rssLen := rss.Len()
if rssLen == 0 {
rss.Cancel()

5
docs/CHANGELOG.md
View file

@ -2,6 +2,11 @@
# tip
* FEATURE: vmselect: return the original error from `vmstorage` node in query response if `-search.denyPartialResponse` is set.
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/891
* FEATURE: vmselect: add `"isPartial":{true|false}` field in JSON output for `/api/v1/*` functions
from [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/). `"isPartial":true` is set if the response contains partial data
because of a part of `vmstorage` nodes were unavailable during query processing.
* FEATURE: vmagent: reduce memory usage when service discovery detects big number of scrape targets and the set of discovered targets changes over time.
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/825
* FEATURE: vmagent: add `-promscrape.dropOriginalLabels` command-line option, which can be used for reducing memory usage when scraping big number of targets.