From 7ad54041feb0a8e65a37b43823d36b1546e959c7 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 15 Sep 2021 18:04:28 +0300 Subject: [PATCH] app/{vminsert,vmselect}: automatically add missing port in `-storageNode` lists passed to `vminsert` and `vmselect` This should simplify manual setup of the cluster according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup --- README.md | 14 +++++++------- app/vminsert/Makefile | 2 +- app/vminsert/main.go | 2 +- app/vminsert/netstorage/netstorage.go | 5 +++++ app/vmselect/Makefile | 2 +- app/vmselect/main.go | 2 +- app/vmselect/netstorage/netstorage.go | 4 ++++ app/vmselect/prometheus/prometheus.go | 7 ++++++- docs/CHANGELOG.md | 2 ++ docs/Cluster-VictoriaMetrics.md | 16 +++++++++------- 10 files changed, 37 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 379f0b266..c1e18623f 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ ROOT_IMAGE=scratch make package A minimal cluster must contain the following nodes: * a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags -* a single `vminsert` node with `-storageNode=:8400` -* a single `vmselect` node with `-storageNode=:8401` +* a single `vminsert` node with `-storageNode=` +* a single `vmselect` node with `-storageNode=` It is recommended to run at least two nodes for each service for high availability purposes. @@ -267,8 +267,8 @@ Cluster performance and capacity scales with adding new nodes. Steps to add `vmstorage` node: 1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster. -2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `:8401`. -3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `:8400`. +2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing ``. +3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing ``. ## Updating / reconfiguring cluster nodes @@ -563,7 +563,7 @@ Below is the output for `/path/to/vminsert -help`: -sortLabels Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit -storageNode array - Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400 + Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN Supports an array of values separated by comma or specified via multiple flags. -tls Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set @@ -677,10 +677,10 @@ Below is the output for `/path/to/vmselect -help`: -search.treatDotsAsIsInRegexps Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter -selectNode array - Addresses of vmselect nodes; usage: -selectNode=vmselect-host1:8481 -selectNode=vmselect-host2:8481 + Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN Supports an array of values separated by comma or specified via multiple flags. -storageNode array - Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401 + Comma-separated dddresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN Supports an array of values separated by comma or specified via multiple flags. -tls Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set diff --git a/app/vminsert/Makefile b/app/vminsert/Makefile index aca86da68..55107932c 100644 --- a/app/vminsert/Makefile +++ b/app/vminsert/Makefile @@ -1,7 +1,7 @@ # All these commands must run from repository root. run-vminsert: - APP_NAME=vminsert ARGS='-storageNode=localhost:8400' $(MAKE) run-via-docker + APP_NAME=vminsert ARGS='-storageNode=localhost' $(MAKE) run-via-docker vminsert: APP_NAME=vminsert $(MAKE) app-local diff --git a/app/vminsert/main.go b/app/vminsert/main.go index b9b3589ee..8ba3c530e 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -54,7 +54,7 @@ var ( opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty") httpListenAddr = flag.String("httpListenAddr", ":8480", "Address to listen for http connections") maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped") - storageNodes = flagutil.NewArray("storageNode", "Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400") + storageNodes = flagutil.NewArray("storageNode", "Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN") ) var ( diff --git a/app/vminsert/netstorage/netstorage.go b/app/vminsert/netstorage/netstorage.go index 8df57501c..7ee5ec496 100644 --- a/app/vminsert/netstorage/netstorage.go +++ b/app/vminsert/netstorage/netstorage.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "io" + "net" "sort" "sync" "sync/atomic" @@ -412,6 +413,10 @@ func InitStorageNodes(addrs []string) { storageNodes = storageNodes[:0] for _, addr := range addrs { + if _, _, err := net.SplitHostPort(addr); err != nil { + // Automatically add missing port. + addr += ":8400" + } sn := &storageNode{ dialer: netutil.NewTCPDialer("vminsert", addr), diff --git a/app/vmselect/Makefile b/app/vmselect/Makefile index ffe7dd4ba..e8ae99f55 100644 --- a/app/vmselect/Makefile +++ b/app/vmselect/Makefile @@ -4,7 +4,7 @@ run-vmselect: mkdir -p vmselect-cache DOCKER_OPTS='-v $(shell pwd)/vmselect-cache:/cache' \ APP_NAME=vmselect \ - ARGS='-storageNode=localhost:8401 -selectNode=localhost:8481 -cacheDataPath=/cache' \ + ARGS='-storageNode=localhost -selectNode=localhost -cacheDataPath=/cache' \ $(MAKE) run-via-docker vmselect: diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 9f8203208..c13de7163 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -41,7 +41,7 @@ var ( "equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details") resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call") logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging") - storageNodes = flagutil.NewArray("storageNode", "Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401") + storageNodes = flagutil.NewArray("storageNode", "Comma-separated dddresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN") ) var slowQueries = metrics.NewCounter(`vm_slow_queries_total`) diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index 57257d5be..b8a10f1bc 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -2578,6 +2578,10 @@ func InitStorageNodes(addrs []string) { } for _, addr := range addrs { + if _, _, err := net.SplitHostPort(addr); err != nil { + // Automatically add missing port. + addr += ":8401" + } sn := &storageNode{ // There is no need in requests compression, since they are usually very small. connPool: netutil.NewConnPool("vmselect", addr, handshake.VMSelectClient, 0), diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 91492d94a..df84bafdc 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "math" + "net" "net/http" "sort" "strconv" @@ -43,7 +44,7 @@ var ( "See also '-search.maxLookback' flag, which has the same meaning due to historical reasons") maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+ "points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data") - selectNodes = flagutil.NewArray("selectNode", "Addresses of vmselect nodes; usage: -selectNode=vmselect-host1:8481 -selectNode=vmselect-host2:8481") + selectNodes = flagutil.NewArray("selectNode", "Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN") ) // Default step used if not set. @@ -492,6 +493,10 @@ func resetRollupResultCaches() { return } for _, selectNode := range *selectNodes { + if _, _, err := net.SplitHostPort(selectNode); err != nil { + // Add missing port + selectNode += ":8481" + } callURL := fmt.Sprintf("http://%s/internal/resetRollupResultCache", selectNode) resp, err := httpClient.Get(callURL) if err != nil { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3d3bfc077..1a9c10d56 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -22,6 +22,8 @@ sort: 15 * FEATURE: allow splitting long `regex` in relabeling filters into an array of shorter regexps, which can be put into multiple lines for better readability and maintainability. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. * FEATURE: optimize performance for queries with regexp filters on metric name like `{__name__=~"metric1|...|metricN"}`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1610) from @faceair. * FEATURE: vmui: use Prometheus-compatible query args, so `vmui` could be accessed from graph editor in Grafana. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1619). Thanks to @Loori-R. +* FEATURE: vmselect: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8401,vmstorage2:8401`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup). +* FEATURE: vminsert: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8400,vmstorage2:8400`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup). * BUGFIX: properly handle queries with multiple filters matching empty labels such as `metric{label1=~"foo|",label2="bar|"}`. This filter must match the following series: `metric`, `metric{label1="foo"}`, `metric{label2="bar"}` and `metric{label1="foo",label2="bar"}`. Previously it was matching only `metric{label1="foo",label2="bar"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1601). * BUGFIX: vmselect: reset connection timeouts after each request to `vmstorage`. This should prevent from `cannot read data in 0.000 seconds: unexpected EOF` warning in logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1562). Thanks to @mxlxm . diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 6b9736542..e5919079a 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -132,8 +132,8 @@ ROOT_IMAGE=scratch make package A minimal cluster must contain the following nodes: * a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags -* a single `vminsert` node with `-storageNode=:8400` -* a single `vmselect` node with `-storageNode=:8401` +* a single `vminsert` node with `-storageNode=` +* a single `vmselect` node with `-storageNode=` It is recommended to run at least two nodes for each service for high availability purposes. @@ -271,8 +271,8 @@ Cluster performance and capacity scales with adding new nodes. Steps to add `vmstorage` node: 1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster. -2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `:8401`. -3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `:8400`. +2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing ``. +3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing ``. ## Updating / reconfiguring cluster nodes @@ -284,6 +284,8 @@ with new configs. Cluster should remain in working state if at least a single node of each type remains available during the update process. See [cluster availability](#cluster-availability) section for details. +See also more advanced [cardinality limiter in vmagent](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter). + ## Cluster availability @@ -565,7 +567,7 @@ Below is the output for `/path/to/vminsert -help`: -sortLabels Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit -storageNode array - Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400 + Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN Supports an array of values separated by comma or specified via multiple flags. -tls Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set @@ -679,10 +681,10 @@ Below is the output for `/path/to/vmselect -help`: -search.treatDotsAsIsInRegexps Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter -selectNode array - Addresses of vmselect nodes; usage: -selectNode=vmselect-host1:8481 -selectNode=vmselect-host2:8481 + Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN Supports an array of values separated by comma or specified via multiple flags. -storageNode array - Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401 + Comma-separated dddresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN Supports an array of values separated by comma or specified via multiple flags. -tls Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set