From 99cb1a70cfa495f77fafdca45086f0b1be19c698 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Nov 2020 01:25:38 +0200 Subject: [PATCH] app/vmselect/graphite: add `/tags` handler from Graphite Tags API See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags --- README.md | 1 + .../graphite/{graphite.go => metrics_api.go} | 0 .../{graphite_test.go => metrics_api_test.go} | 0 app/vmselect/graphite/tags_api.go | 67 +++++++++++++++++ app/vmselect/graphite/tags_response.qtpl | 16 +++++ app/vmselect/graphite/tags_response.qtpl.go | 71 +++++++++++++++++++ app/vmselect/main.go | 11 +++ app/vmselect/netstorage/netstorage.go | 20 ++++++ docs/Cluster-VictoriaMetrics.md | 1 + docs/Single-server-VictoriaMetrics.md | 1 + 10 files changed, 188 insertions(+) rename app/vmselect/graphite/{graphite.go => metrics_api.go} (100%) rename app/vmselect/graphite/{graphite_test.go => metrics_api_test.go} (100%) create mode 100644 app/vmselect/graphite/tags_api.go create mode 100644 app/vmselect/graphite/tags_response.qtpl create mode 100644 app/vmselect/graphite/tags_response.qtpl.go diff --git a/README.md b/README.md index ef0575dd5..6e94e61b3 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr - `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find). - `metrics/expand` - expands Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand). - `metrics/index.json` - returns all the metric names. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json). + - `tags` - returns tag names. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags). * URL for time series deletion: `http://:8481/delete//prometheus/api/v1/admin/tsdb/delete_series?match[]=`. Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't diff --git a/app/vmselect/graphite/graphite.go b/app/vmselect/graphite/metrics_api.go similarity index 100% rename from app/vmselect/graphite/graphite.go rename to app/vmselect/graphite/metrics_api.go diff --git a/app/vmselect/graphite/graphite_test.go b/app/vmselect/graphite/metrics_api_test.go similarity index 100% rename from app/vmselect/graphite/graphite_test.go rename to app/vmselect/graphite/metrics_api_test.go diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go new file mode 100644 index 000000000..2615ba2bf --- /dev/null +++ b/app/vmselect/graphite/tags_api.go @@ -0,0 +1,67 @@ +package graphite + +import ( + "fmt" + "net/http" + "regexp" + "strconv" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + "github.com/VictoriaMetrics/metrics" +) + +// TagsHandler implements handler for /tags endpoint. +// +// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags +func TagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { + deadline := searchutils.GetDeadlineForQuery(r, startTime) + if err := r.ParseForm(); err != nil { + return fmt.Errorf("cannot parse form values: %w", err) + } + limit := 0 + if limitStr := r.FormValue("limit"); len(limitStr) > 0 { + var err error + limit, err = strconv.Atoi(limitStr) + if err != nil { + return fmt.Errorf("cannot parse limit=%q: %w", limit, err) + } + } + denyPartialResponse := searchutils.GetDenyPartialResponse(r) + labels, isPartial, err := netstorage.GetGraphiteTags(at, denyPartialResponse, limit, deadline) + if err != nil { + return err + } + filter := r.FormValue("filter") + if len(filter) > 0 { + // Anchor filter regexp to the beginning of the string as Graphite does. + // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157 + filter = "^(?:" + filter + ")" + re, err := regexp.Compile(filter) + if err != nil { + return fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err) + } + dst := labels[:0] + for _, label := range labels { + if re.MatchString(label) { + dst = append(dst, label) + } + } + labels = dst + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + bw := bufferedwriter.Get(w) + defer bufferedwriter.Put(bw) + WriteTagsResponse(bw, isPartial, labels) + if err := bw.Flush(); err != nil { + return err + } + tagsDuration.UpdateDuration(startTime) + return nil +} + +var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`) diff --git a/app/vmselect/graphite/tags_response.qtpl b/app/vmselect/graphite/tags_response.qtpl new file mode 100644 index 000000000..233c2df62 --- /dev/null +++ b/app/vmselect/graphite/tags_response.qtpl @@ -0,0 +1,16 @@ +{% stripspace %} + +Tags generates response for /tags handler +See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags +{% func TagsResponse(isPartial bool, tags []string) %} +[ + {% for i, tag := range tags %} + { + "tag":{%q= tag %} + } + {% if i+1 < len(tags) %},{% endif %} + {% endfor %} +] +{% endfunc %} + +{% endstripspace %} diff --git a/app/vmselect/graphite/tags_response.qtpl.go b/app/vmselect/graphite/tags_response.qtpl.go new file mode 100644 index 000000000..abc091c6a --- /dev/null +++ b/app/vmselect/graphite/tags_response.qtpl.go @@ -0,0 +1,71 @@ +// Code generated by qtc from "tags_response.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +// Tags generates response for /tags handlerSee https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags + +//line app/vmselect/graphite/tags_response.qtpl:5 +package graphite + +//line app/vmselect/graphite/tags_response.qtpl:5 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line app/vmselect/graphite/tags_response.qtpl:5 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line app/vmselect/graphite/tags_response.qtpl:5 +func StreamTagsResponse(qw422016 *qt422016.Writer, isPartial bool, tags []string) { +//line app/vmselect/graphite/tags_response.qtpl:5 + qw422016.N().S(`[`) +//line app/vmselect/graphite/tags_response.qtpl:7 + for i, tag := range tags { +//line app/vmselect/graphite/tags_response.qtpl:7 + qw422016.N().S(`{"tag":`) +//line app/vmselect/graphite/tags_response.qtpl:9 + qw422016.N().Q(tag) +//line app/vmselect/graphite/tags_response.qtpl:9 + qw422016.N().S(`}`) +//line app/vmselect/graphite/tags_response.qtpl:11 + if i+1 < len(tags) { +//line app/vmselect/graphite/tags_response.qtpl:11 + qw422016.N().S(`,`) +//line app/vmselect/graphite/tags_response.qtpl:11 + } +//line app/vmselect/graphite/tags_response.qtpl:12 + } +//line app/vmselect/graphite/tags_response.qtpl:12 + qw422016.N().S(`]`) +//line app/vmselect/graphite/tags_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_response.qtpl:14 +func WriteTagsResponse(qq422016 qtio422016.Writer, isPartial bool, tags []string) { +//line app/vmselect/graphite/tags_response.qtpl:14 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/graphite/tags_response.qtpl:14 + StreamTagsResponse(qw422016, isPartial, tags) +//line app/vmselect/graphite/tags_response.qtpl:14 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/graphite/tags_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_response.qtpl:14 +func TagsResponse(isPartial bool, tags []string) string { +//line app/vmselect/graphite/tags_response.qtpl:14 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/graphite/tags_response.qtpl:14 + WriteTagsResponse(qb422016, isPartial, tags) +//line app/vmselect/graphite/tags_response.qtpl:14 + qs422016 := string(qb422016.B) +//line app/vmselect/graphite/tags_response.qtpl:14 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/graphite/tags_response.qtpl:14 + return qs422016 +//line app/vmselect/graphite/tags_response.qtpl:14 +} diff --git a/app/vmselect/main.go b/app/vmselect/main.go index c2a41ac9c..31063e682 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -336,6 +336,14 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request, return true } return true + case "graphite/tags": + graphiteTagsRequests.Inc() + if err := graphite.TagsHandler(startTime, at, w, r); err != nil { + graphiteTagsErrors.Inc() + httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) + return true + } + return true case "prometheus/api/v1/rules": // Return dumb placeholder rulesRequests.Inc() @@ -439,6 +447,9 @@ var ( graphiteMetricsIndexRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/graphite/metrics/index.json"}`) graphiteMetricsIndexErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/graphite/metrics/index.json"}`) + graphiteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/graphite/tags"}`) + graphiteTagsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/graphite/tags"}`) + rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/alerts"}`) metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/metadata"}`) diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index 18da7a503..2a42f7c6f 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -557,6 +557,26 @@ func GetLabelsOnTimeRange(at *auth.Token, denyPartialResponse bool, tr storage.T return labels, isPartial, nil } +// GetGraphiteTags returns Graphite tags until the given deadline. +func GetGraphiteTags(at *auth.Token, denyPartialResponse bool, limit int, deadline searchutils.Deadline) ([]string, bool, error) { + labels, isPartial, err := GetLabels(at, denyPartialResponse, deadline) + if err != nil { + return nil, false, err + } + if limit < len(labels) { + labels = labels[:limit] + } + // Convert __name__ to name in labels according to Graphite tags specs. + // See https://graphite.readthedocs.io/en/stable/tags.html#querying + for i, label := range labels { + if label == "__name__" { + labels[i] = "name" + break + } + } + return labels, isPartial, nil +} + // GetLabels returns labels until the given deadline. func GetLabels(at *auth.Token, denyPartialResponse bool, deadline searchutils.Deadline) ([]string, bool, error) { if deadline.Exceeded() { diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index ef0575dd5..6e94e61b3 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -205,6 +205,7 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr - `metrics/find` - searches Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find). - `metrics/expand` - expands Graphite metrics. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand). - `metrics/index.json` - returns all the metric names. See [these docs](https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json). + - `tags` - returns tag names. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags). * URL for time series deletion: `http://:8481/delete//prometheus/api/v1/admin/tsdb/delete_series?match[]=`. Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index e316e46ad..2f0385969 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -548,6 +548,7 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/ * [/tags/tagSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb) * [/tags/tagMultiSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb) +* [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) ### How to build from sources