diff --git a/README.md b/README.md index 4bdb24c5e..1335ca513 100644 --- a/README.md +++ b/README.md @@ -1420,6 +1420,7 @@ Use [vmctl](https://docs.victoriametrics.com/vmctl.html) for data migration. It * From Prometheus to VictoriaMetrics * From InfluxDB to VictoriaMetrics * From VictoriaMetrics to VictoriaMetrics +* From OpenTSDB to VictoriaMetrics See [vmctl docs](https://docs.victoriametrics.com/vmctl.html) for more details. @@ -1719,6 +1720,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control -promscrape.disableKeepAlive Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets + -promscrape.noStaleMarkers + Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode -promscrape.discovery.concurrency int The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) -promscrape.discovery.concurrentWaitTime duration @@ -1808,6 +1811,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum number of unique time series each search can scan. This option allows limiting memory usage (default 300000) -search.minStalenessInterval duration The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval' + -search.noStaleMarkers + Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets -search.queryStats.lastQueriesCount int Query stats for /api/v1/status/top_queries is tracked on this number of last queries. Zero value disables query stats tracking (default 20000) -search.queryStats.minQueryDuration duration diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 5812bc6e8..d661e2a3b 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -244,6 +244,11 @@ You can read more about relabeling in the following articles: * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs) +## Prometheus staleness markers + +Starting from [v1.64.0](https://docs.victoriametrics.com/CHANGELOG.html#v1640), `vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) for scraped metrics when the scrape target is removed from the list of targets. Prometheus staleness markers aren't sent in [stream parsing mode](#stream-parsing-mode) or if `-promscrape.noStaleMarkers` command-line is set. + + ## Stream parsing mode By default `vmagent` reads the full response from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics. Stream parsing mode may be enabled either globally for all of the scrape targets by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example: @@ -643,6 +648,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control -promscrape.disableKeepAlive Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets + -promscrape.noStaleMarkers + Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode -promscrape.discovery.concurrency int The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) -promscrape.discovery.concurrentWaitTime duration diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 1e1f1b1b3..64692b650 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -107,11 +107,11 @@ expression and then act according to the Rule type. There are two types of Rules: * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - -Alerting rules allows to define alert conditions via `expr` field and to send notifications +Alerting rules allows to define alert conditions via `expr` field and to send notifications [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty. * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) - Recording rules allows to define `expr` which result will be than backfilled to configured -`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally +`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally expensive expressions and save their result as a new set of time series. `vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels @@ -189,26 +189,26 @@ The state stored to the configured address on every rule evaluation. from configured address by querying time series with name `ALERTS_FOR_STATE`. Both flags are required for the proper state restoring. Restore process may fail if time series are missing -in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`) +in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`) or received state doesn't match current `vmalert` rules configuration. ### Multitenancy -There are the following approaches for alerting and recording rules across +There are the following approaches for alerting and recording rules across [multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy): -* To run a separate `vmalert` instance per each tenant. - The corresponding tenant must be specified in `-datasource.url` command-line flag - according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). +* To run a separate `vmalert` instance per each tenant. + The corresponding tenant must be specified in `-datasource.url` command-line flag + according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). For example, `/path/to/vmalert -datasource.url=http://vmselect:8481/select/123/prometheus` - would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line - flag must contain the url for the specific tenant as well. - For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording + would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line + flag must contain the url for the specific tenant as well. + For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording rules to `AccountID=123`. -* To specify `tenant` parameter per each alerting and recording group if - [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used +* To specify `tenant` parameter per each alerting and recording group if + [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used with `-clusterMode` command-line flag. For example: ```yaml @@ -224,12 +224,12 @@ groups: # Rules for accountID=456, projectID=789 ``` -If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must -contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`. +If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must +contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`. `vmselect` automatically adds the specified tenant to urls per each recording rule in this case. -The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files -at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise` +The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files +at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise` tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags). @@ -484,6 +484,8 @@ The shortlist of configuration flags is the following: Optional basic auth username for -remoteWrite.url -remoteWrite.concurrency int Defines number of writers for concurrent writing into remote querier (default 1) + -remoteWrite.disablePathAppend + Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url. -remoteWrite.flushInterval duration Defines interval of flushes to remote write endpoint (default 5s) -remoteWrite.maxBatchSize int @@ -501,7 +503,7 @@ The shortlist of configuration flags is the following: -remoteWrite.tlsServerName string Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used -remoteWrite.url string - Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428 + Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend -replay.maxDatapointsPerQuery int Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000) -replay.ruleRetryAttempts int diff --git a/app/vmalert/remotewrite/init.go b/app/vmalert/remotewrite/init.go index 50614fa0d..1b4bb5979 100644 --- a/app/vmalert/remotewrite/init.go +++ b/app/vmalert/remotewrite/init.go @@ -11,7 +11,8 @@ import ( var ( addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+ - "and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428") + "and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+ + "then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend") basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url") basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url") @@ -27,6 +28,7 @@ var ( "By default system CA is used") tlsServerName = flag.String("remoteWrite.tlsServerName", "", "Optional TLS server name to use for connections to -remoteWrite.url. "+ "By default the server name from -remoteWrite.url is used") + disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.") ) // Init creates Client object from given flags. @@ -42,13 +44,14 @@ func Init(ctx context.Context) (*Client, error) { } return NewClient(ctx, Config{ - Addr: *addr, - Concurrency: *concurrency, - MaxQueueSize: *maxQueueSize, - MaxBatchSize: *maxBatchSize, - FlushInterval: *flushInterval, - BasicAuthUser: *basicAuthUsername, - BasicAuthPass: *basicAuthPassword, - Transport: t, + Addr: *addr, + Concurrency: *concurrency, + MaxQueueSize: *maxQueueSize, + MaxBatchSize: *maxBatchSize, + FlushInterval: *flushInterval, + BasicAuthUser: *basicAuthUsername, + BasicAuthPass: *basicAuthPassword, + DisablePathAppend: *disablePathAppend, + Transport: t, }) } diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go index 492a7eac6..3647f4f32 100644 --- a/app/vmalert/remotewrite/remotewrite.go +++ b/app/vmalert/remotewrite/remotewrite.go @@ -19,13 +19,14 @@ import ( // Client is an asynchronous HTTP client for writing // timeseries via remote write protocol. type Client struct { - addr string - c *http.Client - input chan prompbmarshal.TimeSeries - baUser, baPass string - flushInterval time.Duration - maxBatchSize int - maxQueueSize int + addr string + c *http.Client + input chan prompbmarshal.TimeSeries + baUser, baPass string + flushInterval time.Duration + maxBatchSize int + maxQueueSize int + disablePathAppend bool wg sync.WaitGroup doneCh chan struct{} @@ -56,6 +57,8 @@ type Config struct { WriteTimeout time.Duration // Transport will be used by the underlying http.Client Transport *http.Transport + // DisablePathAppend can be used to not automatically append '/api/v1/write' to the remote write url + DisablePathAppend bool } const ( @@ -94,14 +97,15 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) { Timeout: cfg.WriteTimeout, Transport: cfg.Transport, }, - addr: strings.TrimSuffix(cfg.Addr, "/"), - baUser: cfg.BasicAuthUser, - baPass: cfg.BasicAuthPass, - flushInterval: cfg.FlushInterval, - maxBatchSize: cfg.MaxBatchSize, - maxQueueSize: cfg.MaxQueueSize, - doneCh: make(chan struct{}), - input: make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize), + addr: strings.TrimSuffix(cfg.Addr, "/"), + baUser: cfg.BasicAuthUser, + baPass: cfg.BasicAuthPass, + flushInterval: cfg.FlushInterval, + maxBatchSize: cfg.MaxBatchSize, + maxQueueSize: cfg.MaxQueueSize, + doneCh: make(chan struct{}), + input: make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize), + disablePathAppend: cfg.DisablePathAppend, } cc := defaultConcurrency if cfg.Concurrency > 0 { @@ -231,7 +235,9 @@ func (c *Client) send(ctx context.Context, data []byte) error { if c.baPass != "" { req.SetBasicAuth(c.baUser, c.baPass) } - req.URL.Path += writePath + if !c.disablePathAppend { + req.URL.Path += writePath + } resp, err := c.c.Do(req.WithContext(ctx)) if err != nil { return fmt.Errorf("error while sending request to %s: %w; Data len %d(%d)", diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index 8bbfcb0c1..0445b7d73 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -23,6 +23,7 @@ var ( maxPointsPerTimeseries = flag.Int("search.maxPointsPerTimeseries", 30e3, "The maximum points per a single timeseries returned from /api/v1/query_range. "+ "This option doesn't limit the number of scanned raw samples in the database. The main purpose of this option is to limit the number of per-series points "+ "returned to graphing UI such as Grafana. There is no sense in setting this limit to values bigger than the horizontal resolution of the graph") + noStaleMarkers = flag.Bool("search.noStaleMarkers", false, "Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets") ) // The minimum number of points per timeseries for enabling time rounding. @@ -764,10 +765,7 @@ func getRollupMemoryLimiter() *memoryLimiter { func evalRollupWithIncrementalAggregate(name string, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig, preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64, removeMetricGroup bool) ([]*timeseries, error) { err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error { - if name != "default_rollup" { - // Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values. - rs.Values, rs.Timestamps = dropStaleNaNs(rs.Values, rs.Timestamps) - } + rs.Values, rs.Timestamps = dropStaleNaNs(name, rs.Values, rs.Timestamps) preFunc(rs.Values, rs.Timestamps) ts := getTimeseries() defer putTimeseries(ts) @@ -801,10 +799,7 @@ func evalRollupNoIncrementalAggregate(name string, rss *netstorage.Results, rcs tss := make([]*timeseries, 0, rss.Len()*len(rcs)) var tssLock sync.Mutex err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) error { - if name != "default_rollup" { - // Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values. - rs.Values, rs.Timestamps = dropStaleNaNs(rs.Values, rs.Timestamps) - } + rs.Values, rs.Timestamps = dropStaleNaNs(name, rs.Values, rs.Timestamps) preFunc(rs.Values, rs.Timestamps) for _, rc := range rcs { if tsm := newTimeseriesMap(name, sharedTimestamps, &rs.MetricName); tsm != nil { @@ -901,7 +896,13 @@ func toTagFilter(dst *storage.TagFilter, src *metricsql.LabelFilter) { dst.IsNegative = src.IsNegative } -func dropStaleNaNs(values []float64, timestamps []int64) ([]float64, []int64) { +func dropStaleNaNs(name string, values []float64, timestamps []int64) ([]float64, []int64) { + if *noStaleMarkers || name == "default_rollup" { + // Do not drop Prometheus staleness marks (aka stale NaNs) for default_rollup() function, + // since it uses them for Prometheus-style staleness detection. + return values, timestamps + } + // Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values. hasStaleSamples := false for _, v := range values { if decimal.IsStaleNaN(v) { diff --git a/app/vmselect/promql/exec_test.go b/app/vmselect/promql/exec_test.go index 9e7830381..8f1bcc715 100644 --- a/app/vmselect/promql/exec_test.go +++ b/app/vmselect/promql/exec_test.go @@ -132,6 +132,72 @@ func TestExecSuccess(t *testing.T) { resultExpected := []netstorage.Result{r} f(q, resultExpected) }) + t.Run("bitmap_and(0xB3, 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_and(0xB3, 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{17, 17, 17, 17, 17, 17}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_and(time(), 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_and(time(), 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{0, 16, 16, 0, 0, 16}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_or(0xA2, 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_or(0xA2, 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{179, 179, 179, 179, 179, 179}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_or(time(), 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_or(time(), 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1017, 1201, 1401, 1617, 1817, 2001}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_xor(0xB3, 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_xor(0xB3, 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{162, 162, 162, 162, 162, 162}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) + t.Run("bitmap_xor(time(), 0x11)", func(t *testing.T) { + t.Parallel() + q := `bitmap_xor(time(), 0x11)` + r := netstorage.Result{ + MetricName: metricNameExpected, + Values: []float64{1017, 1185, 1385, 1617, 1817, 1985}, + Timestamps: timestampsExpected, + } + resultExpected := []netstorage.Result{r} + f(q, resultExpected) + }) t.Run("timezone_offset(UTC)", func(t *testing.T) { t.Parallel() q := `timezone_offset("UTC")` @@ -6891,6 +6957,10 @@ func TestExecError(t *testing.T) { f(`count_gt_over_time()`) f(`count_eq_over_time()`) f(`count_ne_over_time()`) + f(`timezone_offset()`) + f(`bitmap_and()`) + f(`bitmap_or()`) + f(`bitmap_xor()`) // Invalid argument type f(`median_over_time({}, 2)`) diff --git a/app/vmselect/promql/transform.go b/app/vmselect/promql/transform.go index 9ef2fefd7..9a0840673 100644 --- a/app/vmselect/promql/transform.go +++ b/app/vmselect/promql/transform.go @@ -124,6 +124,9 @@ var transformFuncs = map[string]transformFunc{ "sort_by_label": newTransformFuncSortByLabel(false), "sort_by_label_desc": newTransformFuncSortByLabel(true), "timezone_offset": transformTimezoneOffset, + "bitmap_and": newTransformBitmap(bitmapAnd), + "bitmap_or": newTransformBitmap(bitmapOr), + "bitmap_xor": newTransformBitmap(bitmapXor), } func getTransformFunc(s string) transformFunc { @@ -1914,6 +1917,37 @@ func transformPi(tfa *transformFuncArg) ([]*timeseries, error) { return evalNumber(tfa.ec, math.Pi), nil } +func bitmapAnd(a, b uint64) uint64 { + return a & b +} + +func bitmapOr(a, b uint64) uint64 { + return a | b +} + +func bitmapXor(a, b uint64) uint64 { + return a ^ b +} + +func newTransformBitmap(bitmapFunc func(a, b uint64) uint64) func(tfa *transformFuncArg) ([]*timeseries, error) { + return func(tfa *transformFuncArg) ([]*timeseries, error) { + args := tfa.args + if err := expectTransformArgsNum(args, 2); err != nil { + return nil, err + } + ns, err := getScalar(args[1], 1) + if err != nil { + return nil, err + } + tf := func(values []float64) { + for i, v := range values { + values[i] = float64(bitmapFunc(uint64(v), uint64(ns[i]))) + } + } + return doTransformValues(args[0], tf, tfa.fe) + } +} + func transformTimezoneOffset(tfa *transformFuncArg) ([]*timeseries, error) { args := tfa.args if err := expectTransformArgsNum(args, 1); err != nil { diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index f9c62064c..f7eb25ca2 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -2,8 +2,8 @@ DOCKER_NAMESPACE := victoriametrics -ROOT_IMAGE ?= alpine:3.14.0 -CERTS_IMAGE := alpine:3.14.0 +ROOT_IMAGE ?= alpine:3.14.1 +CERTS_IMAGE := alpine:3.14.1 GO_BUILDER_IMAGE := golang:1.16.7 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr : _) BASE_IMAGE := local/base:1.1.3-$(shell echo $(ROOT_IMAGE) | tr : _)-$(shell echo $(CERTS_IMAGE) | tr : _) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f721296ee..7600db231 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,18 @@ sort: 15 ## tip +## [v1.64.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.64.1) + +* FEATURE: add `bitmap_and(q, mask)`, `bitmap_or(q, mask)` and `bitmak_xor(q, mask)` functions to [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). These functions allow performing bitwise operations over data points in time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1541). +* FEATURE: vmalert: add `-remoteWrite.disablePathAppend` command-line flag, which can be used when custom `-remoteWrite.url` must be specified. For example, `./vmalert -disablePathAppend -remoteWrite.url='http://foo.bar/a/b/c?d=e'` would write data to `http://foo.bar/a/b/c?d=e` instead of `http://foo.bar/a/b/c?d=e/api/v1/write`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1536). +* FEATURE: vmagent: add `-promscrape.noStaleMarkers` command-line flag for disabling sending Prometheus stale markers for metrics from disappeared scrape targets. This option may be used for reducing memory usage when scraping big number of metrics with big number of labels and when stale markers aren't needed. +* FEATURE: vmselect: add `-search.noStaleMarkers` command-line flag for stale markers handling in queries. + +* BUGFIX: vmagent: stop scrapers for deleted targets before starting scrapers for added targets. This should prevent from possible time series overlap when old targets are substituted by new targets (for example, during new deployment in Kubernetes). The overlap could lead to incorrect query results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1509). +* BUGFIX: vmagent: send Prometheus stale markers for the previously scraped metrics on failed scrapes like Prometheus does. See [this article](https://www.robustperception.io/staleness-and-promql). +* BUGFIX: upgrade base Docker image from Alpine 3.14.0 to Alpine 3.14.1 . This fixes potential security issues - see [Alpine 3.14.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.14.1-released.html). + + ## [v1.64.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.64.0) * FEATURE: add support for Prometheus staleness markers. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1526). diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index a01c17582..739f477f2 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -668,6 +668,8 @@ Below is the output for `/path/to/vmselect -help`: The maximum step when /api/v1/query_range handler adjusts points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data (default 1m0s) -search.minStalenessInterval duration The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval' + -search.noStaleMarkers + Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets -search.queryStats.lastQueriesCount int Query stats for /api/v1/status/top_queries is tracked on this number of last queries. Zero value disables query stats tracking (default 20000) -search.queryStats.minQueryDuration duration diff --git a/docs/MetricsQL.md b/docs/MetricsQL.md index b966db012..436994be8 100644 --- a/docs/MetricsQL.md +++ b/docs/MetricsQL.md @@ -160,3 +160,6 @@ This functionality can be tried at [an editable Grafana dashboard](http://play-g - `zscore(q) by (group)` - returns independent [z-score](https://en.wikipedia.org/wiki/Standard_score) values for every point in every `group` of `q`. Useful for detecting anomalies in the group of related time series. - `timezone_offset("tz")` - returns offset in seconds for the given timezone `tz` relative to UTC. This can be useful when combining with datetime-related functions. For example, `day_of_week(time()+timezone_offset("America/Los_Angeles"))` would return weekdays for `America/Los_Angeles` time zone. Special `Local` time zone can be used for returning an offset for the time zone set on the host where VictoriaMetrics runs. See [the list of supported timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). +- `bitmap_and(q, mask)` - calculates bitwise `v & mask` for every `v` point returned from `q`. +- `bitmap_or(q, mask)` - calculates bitwise `v | mask` for every `v` point returned from `q`. +- `bitmap_xor(q, mask)` - calculates bitwise `v ^ mask` for every `v` point returned from `q`. diff --git a/docs/README.md b/docs/README.md index 4bdb24c5e..1335ca513 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1420,6 +1420,7 @@ Use [vmctl](https://docs.victoriametrics.com/vmctl.html) for data migration. It * From Prometheus to VictoriaMetrics * From InfluxDB to VictoriaMetrics * From VictoriaMetrics to VictoriaMetrics +* From OpenTSDB to VictoriaMetrics See [vmctl docs](https://docs.victoriametrics.com/vmctl.html) for more details. @@ -1719,6 +1720,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control -promscrape.disableKeepAlive Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets + -promscrape.noStaleMarkers + Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode -promscrape.discovery.concurrency int The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) -promscrape.discovery.concurrentWaitTime duration @@ -1808,6 +1811,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum number of unique time series each search can scan. This option allows limiting memory usage (default 300000) -search.minStalenessInterval duration The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval' + -search.noStaleMarkers + Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets -search.queryStats.lastQueriesCount int Query stats for /api/v1/status/top_queries is tracked on this number of last queries. Zero value disables query stats tracking (default 20000) -search.queryStats.minQueryDuration duration diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index e1a039c10..ee024df16 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1424,6 +1424,7 @@ Use [vmctl](https://docs.victoriametrics.com/vmctl.html) for data migration. It * From Prometheus to VictoriaMetrics * From InfluxDB to VictoriaMetrics * From VictoriaMetrics to VictoriaMetrics +* From OpenTSDB to VictoriaMetrics See [vmctl docs](https://docs.victoriametrics.com/vmctl.html) for more details. @@ -1723,6 +1724,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control -promscrape.disableKeepAlive Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets + -promscrape.noStaleMarkers + Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode -promscrape.discovery.concurrency int The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) -promscrape.discovery.concurrentWaitTime duration @@ -1812,6 +1815,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum number of unique time series each search can scan. This option allows limiting memory usage (default 300000) -search.minStalenessInterval duration The minimum interval for staleness calculations. This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. See also '-search.maxStalenessInterval' + -search.noStaleMarkers + Set this flag to true if the database doesn't contain Prometheus stale markers, so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets -search.queryStats.lastQueriesCount int Query stats for /api/v1/status/top_queries is tracked on this number of last queries. Zero value disables query stats tracking (default 20000) -search.queryStats.minQueryDuration duration diff --git a/docs/vmagent.md b/docs/vmagent.md index 3f37e99b4..5a03c453a 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -248,6 +248,11 @@ You can read more about relabeling in the following articles: * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs) +## Prometheus staleness markers + +Starting from [v1.64.0](https://docs.victoriametrics.com/CHANGELOG.html#v1640), `vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) for scraped metrics when the scrape target is removed from the list of targets. Prometheus staleness markers aren't sent in [stream parsing mode](#stream-parsing-mode) or if `-promscrape.noStaleMarkers` command-line is set. + + ## Stream parsing mode By default `vmagent` reads the full response from scrape target into memory, then parses it, applies [relabeling](#relabeling) and then pushes the resulting metrics to the configured `-remoteWrite.url`. This mode works good for the majority of cases when the scrape target exposes small number of metrics (e.g. less than 10 thousand). But this mode may take big amounts of memory when the scrape target exposes big number of metrics. In this case it is recommended enabling stream parsing mode. When this mode is enabled, then `vmagent` reads response from scrape target in chunks, then immediately processes every chunk and pushes the processed metrics to remote storage. This allows saving memory when scraping targets that expose millions of metrics. Stream parsing mode may be enabled either globally for all of the scrape targets by passing `-promscrape.streamParse` command-line flag or on a per-scrape target basis with `stream_parse: true` option. For example: @@ -647,6 +652,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control -promscrape.disableKeepAlive Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets + -promscrape.noStaleMarkers + Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode -promscrape.discovery.concurrency int The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100) -promscrape.discovery.concurrentWaitTime duration diff --git a/docs/vmalert.md b/docs/vmalert.md index ba5a6c55f..f939bd99c 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -111,11 +111,11 @@ expression and then act according to the Rule type. There are two types of Rules: * [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - -Alerting rules allows to define alert conditions via `expr` field and to send notifications +Alerting rules allows to define alert conditions via `expr` field and to send notifications [Alertmanager](https://github.com/prometheus/alertmanager) if execution result is not empty. * [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) - Recording rules allows to define `expr` which result will be than backfilled to configured -`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally +`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally expensive expressions and save their result as a new set of time series. `vmalert` forbids to define duplicates - rules with the same combination of name, expression and labels @@ -193,26 +193,26 @@ The state stored to the configured address on every rule evaluation. from configured address by querying time series with name `ALERTS_FOR_STATE`. Both flags are required for the proper state restoring. Restore process may fail if time series are missing -in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`) +in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`) or received state doesn't match current `vmalert` rules configuration. ### Multitenancy -There are the following approaches for alerting and recording rules across +There are the following approaches for alerting and recording rules across [multiple tenants](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy): -* To run a separate `vmalert` instance per each tenant. - The corresponding tenant must be specified in `-datasource.url` command-line flag - according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). +* To run a separate `vmalert` instance per each tenant. + The corresponding tenant must be specified in `-datasource.url` command-line flag + according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format). For example, `/path/to/vmalert -datasource.url=http://vmselect:8481/select/123/prometheus` - would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line - flag must contain the url for the specific tenant as well. - For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording + would run alerts against `AccountID=123`. For recording rules the `-remoteWrite.url` command-line + flag must contain the url for the specific tenant as well. + For example, `-remoteWrite.url=http://vminsert:8480/insert/123/prometheus` would write recording rules to `AccountID=123`. -* To specify `tenant` parameter per each alerting and recording group if - [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used +* To specify `tenant` parameter per each alerting and recording group if + [enterprise version of vmalert](https://victoriametrics.com/enterprise.html) is used with `-clusterMode` command-line flag. For example: ```yaml @@ -228,12 +228,12 @@ groups: # Rules for accountID=456, projectID=789 ``` -If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must -contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`. +If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must +contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`. `vmselect` automatically adds the specified tenant to urls per each recording rule in this case. -The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files -at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise` +The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz` files +at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) and in `*-enterprise` tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags). @@ -488,6 +488,8 @@ The shortlist of configuration flags is the following: Optional basic auth username for -remoteWrite.url -remoteWrite.concurrency int Defines number of writers for concurrent writing into remote querier (default 1) + -remoteWrite.disablePathAppend + Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url. -remoteWrite.flushInterval duration Defines interval of flushes to remote write endpoint (default 5s) -remoteWrite.maxBatchSize int @@ -505,7 +507,7 @@ The shortlist of configuration flags is the following: -remoteWrite.tlsServerName string Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used -remoteWrite.url string - Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. E.g. http://127.0.0.1:8428 + Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend -replay.maxDatapointsPerQuery int Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000) -replay.ruleRetryAttempts int diff --git a/go.mod b/go.mod index 9bae011d9..7e69f37c2 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.16 github.com/VictoriaMetrics/metrics v1.17.3 - github.com/VictoriaMetrics/metricsql v0.17.0 + github.com/VictoriaMetrics/metricsql v0.18.0 github.com/VividCortex/ewma v1.2.0 // indirect - github.com/aws/aws-sdk-go v1.40.21 + github.com/aws/aws-sdk-go v1.40.22 github.com/cespare/xxhash/v2 v2.1.1 github.com/cheggaaa/pb/v3 v3.0.8 github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect @@ -31,7 +31,7 @@ require ( github.com/valyala/fastjson v1.6.3 github.com/valyala/fastrand v1.0.0 github.com/valyala/fasttemplate v1.2.1 - github.com/valyala/gozstd v1.11.0 + github.com/valyala/gozstd v1.12.0 github.com/valyala/histogram v1.1.2 github.com/valyala/quicktemplate v1.6.3 go.uber.org/atomic v1.9.0 // indirect @@ -39,7 +39,7 @@ require ( golang.org/x/oauth2 v0.0.0-20210810183815-faf39c7919d5 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e golang.org/x/text v0.3.7 // indirect - google.golang.org/api v0.53.0 + google.golang.org/api v0.54.0 google.golang.org/grpc v1.40.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 3637119c8..c0a80736c 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.16/go.mod h1:s9o5H4T58Kt4CTrdyJp4RorBKC github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.17.3 h1:QPUakR6JRy8BhL2C2kOgYKLuoPDwtJQ+7iKIZSjt1A4= github.com/VictoriaMetrics/metrics v1.17.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.17.0 h1:OdOkx5GK2p1+EnHn/+zpLWW6ze96L1AmMV5X1HOzyfY= -github.com/VictoriaMetrics/metricsql v0.17.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.18.0 h1:BLNCjJPK305rfD4fjpWu7GV9snFj+zlhY8ispLB6rDs= +github.com/VictoriaMetrics/metricsql v0.18.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -152,8 +152,8 @@ github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.35.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.40.21 h1:QsZ49jnpwPDqh8UoJbr15ItN5oltCyo+sUj/Fl8558w= -github.com/aws/aws-sdk-go v1.40.21/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.40.22 h1:iit4tJ1hjL2GlNCrbE4aJza6jTmvEE2pDTnShct/yyY= +github.com/aws/aws-sdk-go v1.40.22/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.5.0/go.mod h1:acH3+MQoiMzozT/ivU+DbRg7Ooo2298RdRaWcOv+4vM= @@ -926,8 +926,8 @@ github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/gozstd v1.11.0 h1:VV6qQFt+4sBBj9OJ7eKVvsFAMy59Urcs9Lgd+o5FOw0= -github.com/valyala/gozstd v1.11.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/valyala/gozstd v1.12.0 h1:CVG/hZKv3VqgiesiDrFrkgTIwDr5+9yaRaFDgMso5lI= +github.com/valyala/gozstd v1.12.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.1.2 h1:vOk5VrGjMBIoPR5k6wA8vBaC8toeJ8XO0yfRjFEc1h8= github.com/valyala/histogram v1.1.2/go.mod h1:CZAr6gK9dbD7hYx2s8WSPh0p5x5wETjC+2b3PJVtEdg= github.com/valyala/quicktemplate v1.6.3 h1:O7EuMwuH7Q94U2CXD6sOX8AYHqQqWtmIk690IhmpkKA= @@ -1375,8 +1375,8 @@ google.golang.org/api v0.49.0/go.mod h1:BECiH72wsfwUvOVn3+btPD5WHi0LzavZReBndi42 google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.52.0/go.mod h1:Him/adpjt0sxtkWViy0b6xyKW/SD71CwdJ7HqJo7SrU= -google.golang.org/api v0.53.0 h1:tR42CQpqOvZcatWtP2TRJdQCQaD0SVxTDIv+vCphrZs= -google.golang.org/api v0.53.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.54.0 h1:ECJUVngj71QI6XEm7b1sAf8BljU5inEhMbKPR8Lxhhk= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index 8509636d6..20e9502ae 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -301,6 +301,7 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) { additionsCount := 0 deletionsCount := 0 swsMap := make(map[string][]prompbmarshal.Label, len(sws)) + var swsToStart []*ScrapeWork for _, sw := range sws { key := sw.key() originalLabels := swsMap[key] @@ -320,33 +321,47 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) { // The scraper for the given key already exists. continue } + swsToStart = append(swsToStart, sw) + } - // Start a scraper for the missing key. + // Stop deleted scrapers before starting new scrapers in order to prevent + // series overlap when old scrape target is substituted by new scrape target. + var stoppedChs []<-chan struct{} + for key, sc := range sg.m { + if _, ok := swsMap[key]; !ok { + close(sc.stopCh) + stoppedChs = append(stoppedChs, sc.stoppedCh) + delete(sg.m, key) + deletionsCount++ + } + } + // Wait until all the deleted scrapers are stopped before starting new scrapers. + for _, ch := range stoppedChs { + <-ch + } + + // Start new scrapers only after the deleted scrapers are stopped. + for _, sw := range swsToStart { sc := newScraper(sw, sg.name, sg.pushData) sg.activeScrapers.Inc() sg.scrapersStarted.Inc() sg.wg.Add(1) tsmGlobal.Register(sw) go func(sw *ScrapeWork) { - defer sg.wg.Done() + defer func() { + sg.wg.Done() + close(sc.stoppedCh) + }() sc.sw.run(sc.stopCh) tsmGlobal.Unregister(sw) sg.activeScrapers.Dec() sg.scrapersStopped.Inc() }(sw) + key := sw.key() sg.m[key] = sc additionsCount++ } - // Stop deleted scrapers, which are missing in sws. - for key, sc := range sg.m { - if _, ok := swsMap[key]; !ok { - close(sc.stopCh) - delete(sg.m, key) - deletionsCount++ - } - } - if additionsCount > 0 || deletionsCount > 0 { sg.changesCount.Add(additionsCount + deletionsCount) logger.Infof("%s: added targets: %d, removed targets: %d; total targets: %d", sg.name, additionsCount, deletionsCount, len(sg.m)) @@ -354,13 +369,19 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) { } type scraper struct { - sw scrapeWork + sw scrapeWork + + // stopCh is unblocked when the given scraper must be stopped. stopCh chan struct{} + + // stoppedCh is unblocked when the given scraper is stopped. + stoppedCh chan struct{} } func newScraper(sw *ScrapeWork, group string, pushData func(wr *prompbmarshal.WriteRequest)) *scraper { sc := &scraper{ - stopCh: make(chan struct{}), + stopCh: make(chan struct{}), + stoppedCh: make(chan struct{}), } c := newClient(sw) sc.sw.Config = sw diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 1fff64ecf..87bc291a4 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -27,6 +27,7 @@ import ( var ( suppressScrapeErrors = flag.Bool("promscrape.suppressScrapeErrors", false, "Whether to suppress scrape errors logging. "+ "The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed") + noStaleMarkers = flag.Bool("promscrape.noStaleMarkers", false, "Whether to disable seding Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. See also https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode") ) // ScrapeWork represents a unit of work for scraping Prometheus metrics. @@ -238,7 +239,7 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}) { timestamp += scrapeInterval.Milliseconds() select { case <-stopCh: - sw.sendStaleMarkers() + sw.sendStaleMarkers(false) return case tt := <-ticker.C: t := tt.UnixNano() / 1e6 @@ -321,6 +322,9 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error sw.addAutoTimeseries(wc, "scrape_samples_scraped", float64(samplesScraped), scrapeTimestamp) sw.addAutoTimeseries(wc, "scrape_samples_post_metric_relabeling", float64(samplesPostRelabeling), scrapeTimestamp) sw.addAutoTimeseries(wc, "scrape_series_added", float64(seriesAdded), scrapeTimestamp) + if up == 0 { + sw.sendStaleMarkers(true) + } sw.updateActiveSeries(wc) sw.pushData(&wc.writeRequest) sw.prevLabelsLen = len(wc.labels) @@ -333,6 +337,19 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error return err } +func isAutogenSeries(name string) bool { + switch name { + case "up", + "scrape_duration_seconds", + "scrape_samples_scraped", + "scrape_samples_post_metric_relabeling", + "scrape_series_added": + return true + default: + return false + } +} + func (sw *scrapeWork) pushData(wr *prompbmarshal.WriteRequest) { startTime := time.Now() sw.PushData(wr) @@ -486,6 +503,9 @@ func (sw *scrapeWork) updateSeriesAdded(wc *writeRequestCtx) { } func (sw *scrapeWork) updateActiveSeries(wc *writeRequestCtx) { + if *noStaleMarkers { + return + } b := sw.activeSeriesBuf[:0] as := sw.activeSeries[:0] for _, ts := range wc.writeRequest.Timeseries { @@ -500,7 +520,7 @@ func (sw *scrapeWork) updateActiveSeries(wc *writeRequestCtx) { sw.activeSeries = as } -func (sw *scrapeWork) sendStaleMarkers() { +func (sw *scrapeWork) sendStaleMarkers(skipAutogenSeries bool) { series := make([]prompbmarshal.TimeSeries, 0, len(sw.activeSeries)) staleMarkSamples := []prompbmarshal.Sample{ { @@ -510,6 +530,7 @@ func (sw *scrapeWork) sendStaleMarkers() { } for _, b := range sw.activeSeries { var labels []prompbmarshal.Label + skipSeries := false for len(b) > 0 { tail, name, err := encoding.UnmarshalBytes(b) if err != nil { @@ -521,16 +542,25 @@ func (sw *scrapeWork) sendStaleMarkers() { logger.Panicf("BUG: cannot unmarshal label value from activeSeries: %s", err) } b = tail + if skipAutogenSeries && string(name) == "__name__" && isAutogenSeries(bytesutil.ToUnsafeString(value)) { + skipSeries = true + } labels = append(labels, prompbmarshal.Label{ Name: bytesutil.ToUnsafeString(name), Value: bytesutil.ToUnsafeString(value), }) } + if skipSeries { + continue + } series = append(series, prompbmarshal.TimeSeries{ Labels: labels, Samples: staleMarkSamples, }) } + if len(series) == 0 { + return + } wr := &prompbmarshal.WriteRequest{ Timeseries: series, } diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go index 7d2eeced6..c98bb0fb9 100644 --- a/lib/promscrape/scrapework_test.go +++ b/lib/promscrape/scrapework_test.go @@ -128,6 +128,9 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) { if pushDataCalls == 0 { t.Fatalf("missing pushData calls") } + if len(timeseriesExpected) != 0 { + t.Fatalf("%d series weren't pushed", len(timeseriesExpected)) + } } f(``, &ScrapeWork{}, ` diff --git a/vendor/github.com/VictoriaMetrics/metricsql/transform.go b/vendor/github.com/VictoriaMetrics/metricsql/transform.go index bfeb53401..81f0afb34 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/transform.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/transform.go @@ -89,6 +89,9 @@ var transformFuncs = map[string]bool{ "sort_by_label": true, "sort_by_label_desc": true, "timezone_offset": true, + "bitmap_and": true, + "bitmap_or": true, + "bitmap_xor": true, } // IsTransformFunc returns whether funcName is known transform function. diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index 17aaed005..6a798888b 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.40.21" +const SDKVersion = "1.40.22" diff --git a/vendor/github.com/valyala/gozstd/gozstd.go b/vendor/github.com/valyala/gozstd/gozstd.go index 03954caea..4287778d5 100644 --- a/vendor/github.com/valyala/gozstd/gozstd.go +++ b/vendor/github.com/valyala/gozstd/gozstd.go @@ -66,8 +66,6 @@ func CompressDict(dst, src []byte, cd *CDict) []byte { } func compressDictLevel(dst, src []byte, cd *CDict, compressionLevel int) []byte { - concurrencyLimitCh <- struct{}{} - var cctx, cctxDict *cctxWrapper if cd == nil { cctx = cctxPool.Get().(*cctxWrapper) @@ -82,9 +80,6 @@ func compressDictLevel(dst, src []byte, cd *CDict, compressionLevel int) []byte } else { cctxDictPool.Put(cctxDict) } - - <-concurrencyLimitCh - return dst } @@ -189,8 +184,6 @@ func Decompress(dst, src []byte) ([]byte, error) { // // The given dictionary dd is used for the decompression. func DecompressDict(dst, src []byte, dd *DDict) ([]byte, error) { - concurrencyLimitCh <- struct{}{} - var dctx, dctxDict *dctxWrapper if dd == nil { dctx = dctxPool.Get().(*dctxWrapper) @@ -206,9 +199,6 @@ func DecompressDict(dst, src []byte, dd *DDict) ([]byte, error) { } else { dctxDictPool.Put(dctxDict) } - - <-concurrencyLimitCh - return dst, err } @@ -312,11 +302,6 @@ func decompressInternal(dctx, dctxDict *dctxWrapper, dst, src []byte, dd *DDict) return n } -var concurrencyLimitCh = func() chan struct{} { - gomaxprocs := runtime.GOMAXPROCS(-1) - return make(chan struct{}, gomaxprocs) -}() - func errStr(result C.size_t) string { errCode := C.ZSTD_getErrorCode(result) errCStr := C.ZSTD_getErrorString(errCode) diff --git a/vendor/google.golang.org/api/storage/v1/storage-gen.go b/vendor/google.golang.org/api/storage/v1/storage-gen.go index b90f92046..de066a0c2 100644 --- a/vendor/google.golang.org/api/storage/v1/storage-gen.go +++ b/vendor/google.golang.org/api/storage/v1/storage-gen.go @@ -2454,7 +2454,7 @@ func (c *BucketAccessControlsDeleteCall) Header() http.Header { func (c *BucketAccessControlsDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -2607,7 +2607,7 @@ func (c *BucketAccessControlsGetCall) Header() http.Header { func (c *BucketAccessControlsGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -2776,7 +2776,7 @@ func (c *BucketAccessControlsInsertCall) Header() http.Header { func (c *BucketAccessControlsInsertCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -2951,7 +2951,7 @@ func (c *BucketAccessControlsListCall) Header() http.Header { func (c *BucketAccessControlsListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -3117,7 +3117,7 @@ func (c *BucketAccessControlsPatchCall) Header() http.Header { func (c *BucketAccessControlsPatchCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -3296,7 +3296,7 @@ func (c *BucketAccessControlsUpdateCall) Header() http.Header { func (c *BucketAccessControlsUpdateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -3484,7 +3484,7 @@ func (c *BucketsDeleteCall) Header() http.Header { func (c *BucketsDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -3665,7 +3665,7 @@ func (c *BucketsGetCall) Header() http.Header { func (c *BucketsGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -3873,7 +3873,7 @@ func (c *BucketsGetIamPolicyCall) Header() http.Header { func (c *BucketsGetIamPolicyCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -4092,7 +4092,7 @@ func (c *BucketsInsertCall) Header() http.Header { func (c *BucketsInsertCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -4351,7 +4351,7 @@ func (c *BucketsListCall) Header() http.Header { func (c *BucketsListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -4565,7 +4565,7 @@ func (c *BucketsLockRetentionPolicyCall) Header() http.Header { func (c *BucketsLockRetentionPolicyCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -4802,7 +4802,7 @@ func (c *BucketsPatchCall) Header() http.Header { func (c *BucketsPatchCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -5033,7 +5033,7 @@ func (c *BucketsSetIamPolicyCall) Header() http.Header { func (c *BucketsSetIamPolicyCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -5211,7 +5211,7 @@ func (c *BucketsTestIamPermissionsCall) Header() http.Header { func (c *BucketsTestIamPermissionsCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -5453,7 +5453,7 @@ func (c *BucketsUpdateCall) Header() http.Header { func (c *BucketsUpdateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -5665,7 +5665,7 @@ func (c *ChannelsStopCall) Header() http.Header { func (c *ChannelsStopCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -5787,7 +5787,7 @@ func (c *DefaultObjectAccessControlsDeleteCall) Header() http.Header { func (c *DefaultObjectAccessControlsDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -5940,7 +5940,7 @@ func (c *DefaultObjectAccessControlsGetCall) Header() http.Header { func (c *DefaultObjectAccessControlsGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -6110,7 +6110,7 @@ func (c *DefaultObjectAccessControlsInsertCall) Header() http.Header { func (c *DefaultObjectAccessControlsInsertCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -6302,7 +6302,7 @@ func (c *DefaultObjectAccessControlsListCall) Header() http.Header { func (c *DefaultObjectAccessControlsListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -6480,7 +6480,7 @@ func (c *DefaultObjectAccessControlsPatchCall) Header() http.Header { func (c *DefaultObjectAccessControlsPatchCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -6659,7 +6659,7 @@ func (c *DefaultObjectAccessControlsUpdateCall) Header() http.Header { func (c *DefaultObjectAccessControlsUpdateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -6834,7 +6834,7 @@ func (c *NotificationsDeleteCall) Header() http.Header { func (c *NotificationsDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -6985,7 +6985,7 @@ func (c *NotificationsGetCall) Header() http.Header { func (c *NotificationsGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -7157,7 +7157,7 @@ func (c *NotificationsInsertCall) Header() http.Header { func (c *NotificationsInsertCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -7334,7 +7334,7 @@ func (c *NotificationsListCall) Header() http.Header { func (c *NotificationsListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -7514,7 +7514,7 @@ func (c *ObjectAccessControlsDeleteCall) Header() http.Header { func (c *ObjectAccessControlsDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -7693,7 +7693,7 @@ func (c *ObjectAccessControlsGetCall) Header() http.Header { func (c *ObjectAccessControlsGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -7888,7 +7888,7 @@ func (c *ObjectAccessControlsInsertCall) Header() http.Header { func (c *ObjectAccessControlsInsertCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -8089,7 +8089,7 @@ func (c *ObjectAccessControlsListCall) Header() http.Header { func (c *ObjectAccessControlsListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -8281,7 +8281,7 @@ func (c *ObjectAccessControlsPatchCall) Header() http.Header { func (c *ObjectAccessControlsPatchCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -8486,7 +8486,7 @@ func (c *ObjectAccessControlsUpdateCall) Header() http.Header { func (c *ObjectAccessControlsUpdateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -8729,7 +8729,7 @@ func (c *ObjectsComposeCall) Header() http.Header { func (c *ObjectsComposeCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -9085,7 +9085,7 @@ func (c *ObjectsCopyCall) Header() http.Header { func (c *ObjectsCopyCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -9417,7 +9417,7 @@ func (c *ObjectsDeleteCall) Header() http.Header { func (c *ObjectsDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -9654,7 +9654,7 @@ func (c *ObjectsGetCall) Header() http.Header { func (c *ObjectsGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -9908,7 +9908,7 @@ func (c *ObjectsGetIamPolicyCall) Header() http.Header { func (c *ObjectsGetIamPolicyCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -10228,7 +10228,7 @@ func (c *ObjectsInsertCall) Header() http.Header { func (c *ObjectsInsertCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -10603,7 +10603,7 @@ func (c *ObjectsListCall) Header() http.Header { func (c *ObjectsListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -10924,7 +10924,7 @@ func (c *ObjectsPatchCall) Header() http.Header { func (c *ObjectsPatchCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -11329,7 +11329,7 @@ func (c *ObjectsRewriteCall) Header() http.Header { func (c *ObjectsRewriteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -11636,7 +11636,7 @@ func (c *ObjectsSetIamPolicyCall) Header() http.Header { func (c *ObjectsSetIamPolicyCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -11841,7 +11841,7 @@ func (c *ObjectsTestIamPermissionsCall) Header() http.Header { func (c *ObjectsTestIamPermissionsCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -12106,7 +12106,7 @@ func (c *ObjectsUpdateCall) Header() http.Header { func (c *ObjectsUpdateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -12426,7 +12426,7 @@ func (c *ObjectsWatchAllCall) Header() http.Header { func (c *ObjectsWatchAllCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -12645,7 +12645,7 @@ func (c *ProjectsHmacKeysCreateCall) Header() http.Header { func (c *ProjectsHmacKeysCreateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -12798,7 +12798,7 @@ func (c *ProjectsHmacKeysDeleteCall) Header() http.Header { func (c *ProjectsHmacKeysDeleteCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -12937,7 +12937,7 @@ func (c *ProjectsHmacKeysGetCall) Header() http.Header { func (c *ProjectsHmacKeysGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -13139,7 +13139,7 @@ func (c *ProjectsHmacKeysListCall) Header() http.Header { func (c *ProjectsHmacKeysListCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -13338,7 +13338,7 @@ func (c *ProjectsHmacKeysUpdateCall) Header() http.Header { func (c *ProjectsHmacKeysUpdateCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } @@ -13517,7 +13517,7 @@ func (c *ProjectsServiceAccountGetCall) Header() http.Header { func (c *ProjectsServiceAccountGetCall) doRequest(alt string) (*http.Response, error) { reqHeaders := make(http.Header) - reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210810") + reqHeaders.Set("x-goog-api-client", "gl-go/"+gensupport.GoVersion()+" gdcl/20210812") for k, v := range c.header_ { reqHeaders[k] = v } diff --git a/vendor/modules.txt b/vendor/modules.txt index cc977c614..515942d8b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,14 +21,14 @@ github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.17.3 ## explicit github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.17.0 +# github.com/VictoriaMetrics/metricsql v0.18.0 ## explicit github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/VividCortex/ewma v1.2.0 ## explicit github.com/VividCortex/ewma -# github.com/aws/aws-sdk-go v1.40.21 +# github.com/aws/aws-sdk-go v1.40.22 ## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn @@ -205,7 +205,7 @@ github.com/valyala/fastrand # github.com/valyala/fasttemplate v1.2.1 ## explicit github.com/valyala/fasttemplate -# github.com/valyala/gozstd v1.11.0 +# github.com/valyala/gozstd v1.12.0 ## explicit github.com/valyala/gozstd # github.com/valyala/histogram v1.1.2 @@ -293,7 +293,7 @@ golang.org/x/tools/internal/typeparams # golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 golang.org/x/xerrors golang.org/x/xerrors/internal -# google.golang.org/api v0.53.0 +# google.golang.org/api v0.54.0 ## explicit google.golang.org/api/googleapi google.golang.org/api/googleapi/transport