From e6ba2af7a1b0a7a702d7af34597835b4428db6a8 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 6 Jul 2022 13:25:17 +0300 Subject: [PATCH 1/2] lib/promscrape: fix a test after c66f676f3b3ddcbd649b209fff575b994fb196d8 --- lib/promscrape/scrapework_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go index c7f59085c..91784b611 100644 --- a/lib/promscrape/scrapework_test.go +++ b/lib/promscrape/scrapework_test.go @@ -364,6 +364,7 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) { scrape_samples_scraped 2 123 scrape_duration_seconds 0 123 scrape_samples_post_metric_relabeling 2 123 + scrape_samples_limit 1 123 scrape_series_added 0 123 scrape_timeout_seconds 42 123 `) From bd5b20445e06ad707e023799fa0f33d09d76e271 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 6 Jul 2022 13:19:45 +0300 Subject: [PATCH 2/2] app/vmselect: add ability to query `vmselect` from another `vmselect` --- app/vmselect/searchutils/searchutils.go | 6 +++++- docs/CHANGELOG.md | 2 +- docs/Cluster-VictoriaMetrics.md | 7 +++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index 1ea6cb1c0..f82d5c550 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -208,7 +208,11 @@ func (d *Deadline) Deadline() uint64 { func (d *Deadline) String() string { startTime := time.Unix(int64(d.deadline), 0).Add(-d.timeout) elapsed := time.Since(startTime) - return fmt.Sprintf("%.3f seconds (elapsed %.3f seconds); the timeout can be adjusted with `%s` command-line flag", d.timeout.Seconds(), elapsed.Seconds(), d.flagHint) + msg := fmt.Sprintf("%.3f seconds (elapsed %.3f seconds)", d.timeout.Seconds(), elapsed.Seconds()) + if d.flagHint != "" { + msg += fmt.Sprintf("; the timeout can be adjusted with `%s` command-line flag", d.flagHint) + } + return msg } // GetExtraTagFilters returns additional label filters from request. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d96d7d7ad..ab488bb0a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,7 +18,7 @@ The following tip changes can be tested by building VictoriaMetrics components f **Update notes:** this release introduces backwards-incompatible changes to `vm_partial_results_total` metric by changing its labels to be consistent with `vm_requests_total` metric. If you use alerting rules or Grafana dashboards, which rely on this metric, then they must be updated. The official dashboards for VictoriaMetrics don't use this metric. -* FEATURE: [single-node VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html): allow accessing [vmalert's](https://docs.victoriametrics.com/vmalert.html) UI when `-vmalert.proxyURL` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmalert) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825). +* FEATURE: [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add support for querying lower-level `vmselect` nodes from upper-level `vmselect` nodes. This makes possible to build multi-level cluster setups for global querying view and HA purposes without the need to use [Promxy](https://github.com/jacksontj/promxy). See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multi-level-cluster-setup) for details. * FEATURE: add `-search.setLookbackToStep` command-line flag, which enables InfluxDB-like gap filling during querying. See [these docs](https://docs.victoriametrics.com/guides/migrate-from-influx.html) for details. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add an UI for [query tracing](https://docs.victoriametrics.com/#query-tracing). It can be enabled by clicking `enable query tracing` checkbox and re-running the query. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2703). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.headers` command-line option for specifying optional HTTP headers to send to the configured `-remoteWrite.url`. For example, `-remoteWrite.headers='Foo:Bar^^Baz:x'` would send `Foo: Bar` and `Baz: x` HTTP headers with every request to `-remoteWrite.url`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2805). diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 19841de85..d909082e7 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -370,12 +370,14 @@ error rates comparing the network inside a single AZ. If you need multi-AZ setup, then it is recommended running independed clusters in each AZ and setting up [vmagent](https://docs.victoriametrics.com/vmagent.html) in front of these clusters, so it could replicate incoming data into all the cluster - see [these docs](https://docs.victoriametrics.com/vmagent.html#multitenancy) for details. -Then [promxy](https://github.com/jacksontj/promxy) could be used for querying the data from multiple clusters. +Then an additional `vmselect` nodes can be configured for reading the data from multiple clusters according to [these docs](#multi-level-cluster-setup). ## Multi-level cluster setup -`vminsert` nodes can accept data from another `vminsert` nodes starting from [v1.60.0](https://docs.victoriametrics.com/CHANGELOG.html#v1600) if `-clusternativeListenAddr` command-line flag is set. For example, if `vminsert` is started with `-clusternativeListenAddr=:8400` command-line flag, then it can accept data from another `vminsert` nodes at TCP port 8400 in the same way as `vmstorage` nodes do. This allows chaining `vminsert` nodes and building multi-level cluster topologies with flexible configs. For example, the top level of `vminsert` nodes can replicate data among the second level of `vminsert` nodes located in distinct availability zones (AZ), while the second-level `vminsert` nodes can spread the data among `vmstorage` nodes located in the same AZ. Such setup guarantees cluster availability if some AZ becomes unavailable. The data from all the `vmstorage` nodes in all the AZs can be read via `vmselect` nodes, which are configured to query all the `vmstorage` nodes in all the availability zones (e.g. all the `vmstorage` addresses are passed via `-storageNode` command-line flag to `vmselect` nodes). +`vmselect` nodes can be queried by other `vmselect` nodes if they run with `-clusternativeListenAddr` command-line flag. For example, if `vmselect` is started with `-clusternativeListenAddr=:8401`, then it can accept queries from another `vmselect` nodes at TCP port 8401 in the same way as `vmstorage` nodes do. This allows chaining `vmselect` nodes and building multi-level cluster topologies. For example, the top-level `vmselect` node can query second-level `vmselect` nodes in different availability zones (AZ), while the second-level `vmselect` nodes can query `vmstorage` nodes in local AZ. + +`vminsert` nodes can accept data from another `vminsert` nodes if they run with `-clusternativeListenAddr` command-line flag. For example, if `vminsert` is started with `-clusternativeListenAddr=:8400`, then it can accept data from another `vminsert` nodes at TCP port 8400 in the same way as `vmstorage` nodes do. This allows chaining `vminsert` nodes and building multi-level cluster topologies. For example, the top-level `vminsert` node can replicate data among the second level of `vminsert` nodes located in distinct availability zones (AZ), while the second-level `vminsert` nodes can spread the data among `vmstorage` nodes in local AZ. The multi-level cluster setup for `vminsert` nodes has the following shortcomings because of synchronous replication and data sharding: @@ -384,6 +386,7 @@ The multi-level cluster setup for `vminsert` nodes has the following shortcoming These issues are addressed by [vmagent](https://docs.victoriametrics.com/vmagent.html) when it runs in [multitenancy mode](https://docs.victoriametrics.com/vmagent.html#multitenancy). `vmagent` buffers data, which must be sent to a particular AZ, when this AZ is temporarily unavailable. The buffer is stored on disk. The buffered data is sent to AZ as soon as it becomes available. + ## Helm Helm chart simplifies managing cluster version of VictoriaMetrics in Kubernetes.