github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-21 14:44:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmselect/graphite: apply filter then limit

Browse source
This commit is contained in:
Aliaksandr Valialkin 2020-11-16 03:58:12 +02:00
parent d100341394
commit 414cd39659
2 changed files with 63 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
app/vmselect/graphite/tags_api.go
View file

@ -3,7 +3,6 @@ package graphite
import (
"fmt"
"net/http"
"regexp"
"strconv"
"time"
@ -21,25 +20,15 @@ func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse form values: %w", err)
}
limit := 0
if limitStr := r.FormValue("limit"); len(limitStr) > 0 {
var err error
limit, err = strconv.Atoi(limitStr)
if err != nil {
return fmt.Errorf("cannot parse limit=%q: %w", limit, err)
}
}
tagValues, err := netstorage.GetGraphiteTagValues(tagName, limit, deadline)
limit, err := getInt(r, "limit")
if err != nil {
return err
}
filter := r.FormValue("filter")
if len(filter) > 0 {
tagValues, err = applyRegexpFilter(filter, tagValues)
tagValues, err := netstorage.GetGraphiteTagValues(tagName, filter, limit, deadline)
if err != nil {
return err
}
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
@ -62,25 +51,15 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse form values: %w", err)
}
limit := 0
if limitStr := r.FormValue("limit"); len(limitStr) > 0 {
var err error
limit, err = strconv.Atoi(limitStr)
if err != nil {
return fmt.Errorf("cannot parse limit=%q: %w", limit, err)
}
}
labels, err := netstorage.GetGraphiteTags(limit, deadline)
limit, err := getInt(r, "limit")
if err != nil {
return err
}
filter := r.FormValue("filter")
if len(filter) > 0 {
labels, err = applyRegexpFilter(filter, labels)
labels, err := netstorage.GetGraphiteTags(filter, limit, deadline)
if err != nil {
return err
}
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
@ -95,19 +74,14 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er
var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`)
func applyRegexpFilter(filter string, ss []string) ([]string, error) {
// Anchor filter regexp to the beginning of the string as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
filter = "^(?:" + filter + ")"
re, err := regexp.Compile(filter)
func getInt(r *http.Request, argName string) (int, error) {
argValue := r.FormValue(argName)
if len(argValue) == 0 {
return 0, nil
}
n, err := strconv.Atoi(argValue)
if err != nil {
return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
return 0, fmt.Errorf("cannot parse %q=%q: %w", argName, argValue, err)
}
dst := ss[:0]
for _, s := range ss {
if re.MatchString(s) {
dst = append(dst, s)
}
}
return dst, nil
return n, nil
}

72
app/vmselect/netstorage/netstorage.go
View file

@ -5,6 +5,7 @@ import (
"errors"
"flag"
"fmt"
"regexp"
"runtime"
"sort"
"sync"
@ -474,29 +475,30 @@ func GetLabelsOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) (
}
// GetGraphiteTags returns Graphite tags until the given deadline.
func GetGraphiteTags(limit int, deadline searchutils.Deadline) ([]string, error) {
func GetGraphiteTags(filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
if deadline.Exceeded() {
return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
if limit <= 0 {
limit = *maxTagKeysPerSearch
}
if limit > *maxTagKeysPerSearch {
return nil, fmt.Errorf("limit=%d exceeds -search.maxTagKeys=%d; either decrease limit or increase -search.maxTagKeys command-line flag value",
limit, *maxTagKeysPerSearch)
}
labels, err := vmstorage.SearchTagKeys(limit, deadline.Deadline())
labels, err := GetLabels(deadline)
if err != nil {
return nil, fmt.Errorf("error during tags search: %w", err)
return nil, err
}
// Substitute "" with "name" for Graphite compatibility
if len(filter) > 0 {
labels, err = applyGraphiteRegexpFilter(filter, labels)
if err != nil {
return nil, err
}
}
// Substitute "__name__" with "name" for Graphite compatibility
for i := range labels {
if labels[i] == "" {
if labels[i] == "__name__" {
labels[i] = "name"
break
}
}
// Sort labels like Graphite does
sort.Strings(labels)
if limit > 0 && limit < len(labels) {
labels = labels[:limit]
}
return labels, nil
}
@ -540,27 +542,26 @@ func GetLabelValuesOnTimeRange(labelName string, tr storage.TimeRange, deadline
}
// GetGraphiteTagValues returns tag values for the given tagName until the given deadline.
func GetGraphiteTagValues(tagName string, limit int, deadline searchutils.Deadline) ([]string, error) {
func GetGraphiteTagValues(tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
if deadline.Exceeded() {
return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
if tagName == "name" {
tagName = ""
}
if limit <= 0 {
limit = *maxTagValuesPerSearch
}
if limit > *maxTagValuesPerSearch {
return nil, fmt.Errorf("limit=%d exceeds -search.maxTagValues=%d; either reduce limit or increase -search.maxTagValues command-line flag value",
limit, *maxTagValuesPerSearch)
}
// Search for tag values
tagValues, err := vmstorage.SearchTagValues([]byte(tagName), limit, deadline.Deadline())
tagValues, err := GetLabelValues(tagName, deadline)
if err != nil {
return nil, fmt.Errorf("error during tag values search for tagName=%q: %w", tagName, err)
return nil, err
}
if len(filter) > 0 {
tagValues, err = applyGraphiteRegexpFilter(filter, tagValues)
if err != nil {
return nil, err
}
}
if limit > 0 && limit < len(tagValues) {
tagValues = tagValues[:limit]
}
// Sort tagValues like Graphite does
sort.Strings(tagValues)
return tagValues, nil
}
@ -888,3 +889,20 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
}
return tfss, nil
}
func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
// Anchor filter regexp to the beginning of the string as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
filter = "^(?:" + filter + ")"
re, err := regexp.Compile(filter)
if err != nil {
return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
}
dst := ss[:0]
for _, s := range ss {
if re.MatchString(s) {
dst = append(dst, s)
}
}
return dst, nil
}