github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-20 15:16:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/{graphite,netstorage,prometheus}: fix graphite search tags api limits, remove redudant limit from SeriesHandler handler (#4352)

Browse source 
* app/{graphite,netstorage,prometheus}: fix graphite search tags api limits, remove unused limit from SeriesHandler handler,

* app/{graphite,netstorage,prometheus}: use search.maxTagValues for Graphite

* app/{graphite,netstorage,prometheus}: update CHANGELOG.md

* app/{graphite,netstorage,prometheus}: use own flags for Graphite API

* app/{graphite,netstorage,prometheus}: cleanup

* app/{graphite,netstorage,prometheus}: cleanup

* app/{graphite,netstorage,prometheus}: update docs

---------

Co-authored-by: Nikolay <nik@victoriametrics.com>

(cherry picked from commit c7884f8686)
Signed-off-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
Dmytro Kozlov 2023-06-02 14:34:04 +02:00 committed by hagen1778
parent dfb05c884b
commit c5debee3f4
No known key found for this signature in database
GPG key ID: 3BF75F3741CA9640
7 changed files with 40 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -1144,6 +1144,10 @@ Below is the output for `/path/to/vmselect -help`:
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons
-search.maxMemoryPerQuery size

26
app/vmselect/graphite/tags_api.go
View file

@ -1,6 +1,7 @@
package graphite
import (
"flag"
"fmt"
"net/http"
"regexp"
@ -18,6 +19,11 @@ import (
"github.com/VictoriaMetrics/metrics"
)
var (
maxGraphiteTagKeysPerSearch = flag.Int("search.maxGraphiteTagKeys", 100e3, "The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API")
maxGraphiteTagValuesPerSearch = flag.Int("search.maxGraphiteTagValues", 100e3, "The maximum number of tag values returned Graphite /tags/<tag_name> API")
)
// TagsDelSeriesHandler implements /tags/delSeries handler.
//
// See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb
@ -183,13 +189,13 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.R
// Escape special chars in tagPrefix as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228
filter := regexp.QuoteMeta(valuePrefix)
tagValues, isPartial, err = netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tag, filter, limit, deadline)
tagValues, isPartial, err = netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tag, filter, *maxGraphiteTagKeysPerSearch, deadline)
if err != nil {
return err
}
} else {
// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
sq, err := getSearchQueryForExprs(startTime, at, etfs, exprs, limit*10)
sq, err := getSearchQueryForExprs(startTime, at, etfs, exprs, *maxGraphiteTagKeysPerSearch)
if err != nil {
return err
}
@ -273,13 +279,13 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res
// Escape special chars in tagPrefix as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181
filter := regexp.QuoteMeta(tagPrefix)
labels, isPartial, err = netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, limit, deadline)
labels, isPartial, err = netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch, deadline)
if err != nil {
return err
}
} else {
// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
sq, err := getSearchQueryForExprs(startTime, at, etfs, exprs, limit*10)
sq, err := getSearchQueryForExprs(startTime, at, etfs, exprs, *maxGraphiteTagKeysPerSearch)
if err != nil {
return err
}
@ -348,7 +354,7 @@ func TagsFindSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseW
if err != nil {
return fmt.Errorf("cannot setup tag filters: %w", err)
}
sq, err := getSearchQueryForExprs(startTime, at, etfs, exprs, limit*10)
sq, err := getSearchQueryForExprs(startTime, at, etfs, exprs, *maxGraphiteTagKeysPerSearch)
if err != nil {
return err
}
@ -418,11 +424,14 @@ func TagValuesHandler(startTime time.Time, at *auth.Token, tagName string, w htt
}
filter := r.FormValue("filter")
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
tagValues, isPartial, err := netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tagName, filter, limit, deadline)
tagValues, isPartial, err := netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tagName, filter, *maxGraphiteTagValuesPerSearch, deadline)
if err != nil {
return err
}
if limit > 0 && limit < len(tagValues) {
tagValues = tagValues[:limit]
}
w.Header().Set("Content-Type", "application/json")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
@ -447,11 +456,14 @@ func TagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *
}
filter := r.FormValue("filter")
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
labels, isPartial, err := netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, limit, deadline)
labels, isPartial, err := netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch, deadline)
if err != nil {
return err
}
if limit > 0 && limit < len(labels) {
labels = labels[:limit]
}
w.Header().Set("Content-Type", "application/json")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)

6
app/vmselect/prometheus/prometheus.go
View file

@ -727,11 +727,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token,
return err
}
minLimit := *maxSeriesLimit
if limit > 0 && limit < *maxSeriesLimit {
minLimit = limit
}
sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, minLimit)
sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, *maxSeriesLimit)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
metricNames, isPartial, err := netstorage.SearchMetricNames(qt, denyPartialResponse, sq, cp.deadline)
if err != nil {

4
docs/CHANGELOG.md
View file

@ -25,6 +25,10 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix nil map assignment panic in runtime introduced in this [change](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4341).
* BUGFIX: add the following command-line flags, which can be used for limiting Graphite API calls:
`--search.maxGraphiteTagKeys` for limiting the number of tag keys returned from Graphite `/tags`, `/tags/autoComplete/*`, `/tags/findSeries` API.
`--search.maxGraphiteTagValues` for limiting the number of tag values returned Graphite `/tags/<tag_name>` API.
Remove redundant limit from [Prometheus api/v1/series](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prometheus-querying-api-usage). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4339).
## [v1.91.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.91.1)

4
docs/Cluster-VictoriaMetrics.md
View file

@ -1156,6 +1156,10 @@ Below is the output for `/path/to/vmselect -help`:
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons
-search.maxMemoryPerQuery size

4
docs/README.md
View file

@ -2501,6 +2501,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
-search.maxMemoryPerQuery size

4
docs/Single-server-VictoriaMetrics.md
View file

@ -2509,6 +2509,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
-search.maxMemoryPerQuery size