From 414cd3965985b67fa27314094367474ba275d143 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Nov 2020 03:58:12 +0200 Subject: [PATCH] app/vmselect/graphite: apply filter then limit --- app/vmselect/graphite/tags_api.go | 58 ++++++-------------- app/vmselect/netstorage/netstorage.go | 76 +++++++++++++++++---------- 2 files changed, 63 insertions(+), 71 deletions(-) diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index d1e7d53d6..d7efd35fd 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -3,7 +3,6 @@ package graphite import ( "fmt" "net/http" - "regexp" "strconv" "time" @@ -21,24 +20,14 @@ func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter if err := r.ParseForm(); err != nil { return fmt.Errorf("cannot parse form values: %w", err) } - limit := 0 - if limitStr := r.FormValue("limit"); len(limitStr) > 0 { - var err error - limit, err = strconv.Atoi(limitStr) - if err != nil { - return fmt.Errorf("cannot parse limit=%q: %w", limit, err) - } - } - tagValues, err := netstorage.GetGraphiteTagValues(tagName, limit, deadline) + limit, err := getInt(r, "limit") if err != nil { return err } filter := r.FormValue("filter") - if len(filter) > 0 { - tagValues, err = applyRegexpFilter(filter, tagValues) - if err != nil { - return err - } + tagValues, err := netstorage.GetGraphiteTagValues(tagName, filter, limit, deadline) + if err != nil { + return err } w.Header().Set("Content-Type", "application/json; charset=utf-8") @@ -62,24 +51,14 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er if err := r.ParseForm(); err != nil { return fmt.Errorf("cannot parse form values: %w", err) } - limit := 0 - if limitStr := r.FormValue("limit"); len(limitStr) > 0 { - var err error - limit, err = strconv.Atoi(limitStr) - if err != nil { - return fmt.Errorf("cannot parse limit=%q: %w", limit, err) - } - } - labels, err := netstorage.GetGraphiteTags(limit, deadline) + limit, err := getInt(r, "limit") if err != nil { return err } filter := r.FormValue("filter") - if len(filter) > 0 { - labels, err = applyRegexpFilter(filter, labels) - if err != nil { - return err - } + labels, err := netstorage.GetGraphiteTags(filter, limit, deadline) + if err != nil { + return err } w.Header().Set("Content-Type", "application/json; charset=utf-8") @@ -95,19 +74,14 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`) -func applyRegexpFilter(filter string, ss []string) ([]string, error) { - // Anchor filter regexp to the beginning of the string as Graphite does. - // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157 - filter = "^(?:" + filter + ")" - re, err := regexp.Compile(filter) +func getInt(r *http.Request, argName string) (int, error) { + argValue := r.FormValue(argName) + if len(argValue) == 0 { + return 0, nil + } + n, err := strconv.Atoi(argValue) if err != nil { - return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err) + return 0, fmt.Errorf("cannot parse %q=%q: %w", argName, argValue, err) } - dst := ss[:0] - for _, s := range ss { - if re.MatchString(s) { - dst = append(dst, s) - } - } - return dst, nil + return n, nil } diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index f9ba19849..eb7ed9634 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "regexp" "runtime" "sort" "sync" @@ -474,29 +475,30 @@ func GetLabelsOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) ( } // GetGraphiteTags returns Graphite tags until the given deadline. -func GetGraphiteTags(limit int, deadline searchutils.Deadline) ([]string, error) { +func GetGraphiteTags(filter string, limit int, deadline searchutils.Deadline) ([]string, error) { if deadline.Exceeded() { return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) } - if limit <= 0 { - limit = *maxTagKeysPerSearch - } - if limit > *maxTagKeysPerSearch { - return nil, fmt.Errorf("limit=%d exceeds -search.maxTagKeys=%d; either decrease limit or increase -search.maxTagKeys command-line flag value", - limit, *maxTagKeysPerSearch) - } - labels, err := vmstorage.SearchTagKeys(limit, deadline.Deadline()) + labels, err := GetLabels(deadline) if err != nil { - return nil, fmt.Errorf("error during tags search: %w", err) + return nil, err } - // Substitute "" with "name" for Graphite compatibility - for i := range labels { - if labels[i] == "" { - labels[i] = "name" + if len(filter) > 0 { + labels, err = applyGraphiteRegexpFilter(filter, labels) + if err != nil { + return nil, err } } - // Sort labels like Graphite does - sort.Strings(labels) + // Substitute "__name__" with "name" for Graphite compatibility + for i := range labels { + if labels[i] == "__name__" { + labels[i] = "name" + break + } + } + if limit > 0 && limit < len(labels) { + labels = labels[:limit] + } return labels, nil } @@ -540,27 +542,26 @@ func GetLabelValuesOnTimeRange(labelName string, tr storage.TimeRange, deadline } // GetGraphiteTagValues returns tag values for the given tagName until the given deadline. -func GetGraphiteTagValues(tagName string, limit int, deadline searchutils.Deadline) ([]string, error) { +func GetGraphiteTagValues(tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) { if deadline.Exceeded() { return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) } if tagName == "name" { tagName = "" } - if limit <= 0 { - limit = *maxTagValuesPerSearch - } - if limit > *maxTagValuesPerSearch { - return nil, fmt.Errorf("limit=%d exceeds -search.maxTagValues=%d; either reduce limit or increase -search.maxTagValues command-line flag value", - limit, *maxTagValuesPerSearch) - } - // Search for tag values - tagValues, err := vmstorage.SearchTagValues([]byte(tagName), limit, deadline.Deadline()) + tagValues, err := GetLabelValues(tagName, deadline) if err != nil { - return nil, fmt.Errorf("error during tag values search for tagName=%q: %w", tagName, err) + return nil, err + } + if len(filter) > 0 { + tagValues, err = applyGraphiteRegexpFilter(filter, tagValues) + if err != nil { + return nil, err + } + } + if limit > 0 && limit < len(tagValues) { + tagValues = tagValues[:limit] } - // Sort tagValues like Graphite does - sort.Strings(tagValues) return tagValues, nil } @@ -888,3 +889,20 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error) } return tfss, nil } + +func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) { + // Anchor filter regexp to the beginning of the string as Graphite does. + // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157 + filter = "^(?:" + filter + ")" + re, err := regexp.Compile(filter) + if err != nil { + return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err) + } + dst := ss[:0] + for _, s := range ss { + if re.MatchString(s) { + dst = append(dst, s) + } + } + return dst, nil +}