app/{vminsert,vmselect}: automatically add missing port in -storageNode lists passed to vminsert and vmselect

This should simplify manual setup of the cluster according to https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup

README.md

@@ -128,8 +128,8 @@ ROOT_IMAGE=scratch make package
 A minimal cluster must contain the following nodes:
 
 * a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags
-* a single `vminsert` node with `-storageNode=<vmstorage_host>:8400`
-* a single `vmselect` node with `-storageNode=<vmstorage_host>:8401`
+* a single `vminsert` node with `-storageNode=<vmstorage_host>`
+* a single `vmselect` node with `-storageNode=<vmstorage_host>`
 
 It is recommended to run at least two nodes for each service
 for high availability purposes.
@@ -267,8 +267,8 @@ Cluster performance and capacity scales with adding new nodes.
 Steps to add `vmstorage` node:
 
 1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster.
-2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8401`.
-3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8400`.
+2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>`.
+3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>`.
 
 
 ## Updating / reconfiguring cluster nodes
@@ -563,7 +563,7 @@ Below is the output for `/path/to/vminsert -help`:
   -sortLabels
     	Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
   -storageNode array
-    	Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400
+    	Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN
     	Supports an array of values separated by comma or specified via multiple flags.
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
@@ -677,10 +677,10 @@ Below is the output for `/path/to/vmselect -help`:
   -search.treatDotsAsIsInRegexps
     	Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter
   -selectNode array
-    	Addresses of vmselect nodes; usage: -selectNode=vmselect-host1:8481 -selectNode=vmselect-host2:8481
+    	Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN
     	Supports an array of values separated by comma or specified via multiple flags.
   -storageNode array
-    	Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401
+    	Comma-separated dddresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN
     	Supports an array of values separated by comma or specified via multiple flags.
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set

app/vminsert/Makefile

@@ -1,7 +1,7 @@
 # All these commands must run from repository root.
 
 run-vminsert:
-	APP_NAME=vminsert ARGS='-storageNode=localhost:8400' $(MAKE) run-via-docker
+	APP_NAME=vminsert ARGS='-storageNode=localhost' $(MAKE) run-via-docker
 
 vminsert:
 	APP_NAME=vminsert $(MAKE) app-local

app/vminsert/main.go

@@ -54,7 +54,7 @@ var (
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
 	httpListenAddr         = flag.String("httpListenAddr", ":8480", "Address to listen for http connections")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
-	storageNodes           = flagutil.NewArray("storageNode", "Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400")
+	storageNodes           = flagutil.NewArray("storageNode", "Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN")
 )
 
 var (

app/vminsert/netstorage/netstorage.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -412,6 +413,10 @@ func InitStorageNodes(addrs []string) {
 
 	storageNodes = storageNodes[:0]
 	for _, addr := range addrs {
+		if _, _, err := net.SplitHostPort(addr); err != nil {
+			// Automatically add missing port.
+			addr += ":8400"
+		}
 		sn := &storageNode{
 			dialer: netutil.NewTCPDialer("vminsert", addr),
 

app/vmselect/Makefile

@@ -4,7 +4,7 @@ run-vmselect:
 	mkdir -p vmselect-cache
 	DOCKER_OPTS='-v $(shell pwd)/vmselect-cache:/cache' \
 	APP_NAME=vmselect \
-	ARGS='-storageNode=localhost:8401 -selectNode=localhost:8481 -cacheDataPath=/cache' \
+	ARGS='-storageNode=localhost -selectNode=localhost -cacheDataPath=/cache' \
 	$(MAKE) run-via-docker
 
 vmselect:

app/vmselect/main.go

@@ -41,7 +41,7 @@ var (
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details")
 	resetCacheAuthKey    = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging")
-	storageNodes         = flagutil.NewArray("storageNode", "Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401")
+	storageNodes         = flagutil.NewArray("storageNode", "Comma-separated dddresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN")
 )
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)

app/vmselect/netstorage/netstorage.go

@@ -2578,6 +2578,10 @@ func InitStorageNodes(addrs []string) {
 	}
 
 	for _, addr := range addrs {
+		if _, _, err := net.SplitHostPort(addr); err != nil {
+			// Automatically add missing port.
+			addr += ":8401"
+		}
 		sn := &storageNode{
 			// There is no need in requests compression, since they are usually very small.
 			connPool: netutil.NewConnPool("vmselect", addr, handshake.VMSelectClient, 0),

app/vmselect/prometheus/prometheus.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"math"
+	"net"
 	"net/http"
 	"sort"
 	"strconv"
@@ -43,7 +44,7 @@ var (
 		"See also '-search.maxLookback' flag, which has the same meaning due to historical reasons")
 	maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+
 		"points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data")
-	selectNodes = flagutil.NewArray("selectNode", "Addresses of vmselect nodes; usage: -selectNode=vmselect-host1:8481 -selectNode=vmselect-host2:8481")
+	selectNodes = flagutil.NewArray("selectNode", "Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN")
 )
 
 // Default step used if not set.
@@ -492,6 +493,10 @@ func resetRollupResultCaches() {
 		return
 	}
 	for _, selectNode := range *selectNodes {
+		if _, _, err := net.SplitHostPort(selectNode); err != nil {
+			// Add missing port
+			selectNode += ":8481"
+		}
 		callURL := fmt.Sprintf("http://%s/internal/resetRollupResultCache", selectNode)
 		resp, err := httpClient.Get(callURL)
 		if err != nil {

docs/CHANGELOG.md

@@ -22,6 +22,8 @@ sort: 15
 * FEATURE: allow splitting long `regex` in relabeling filters into an array of shorter regexps, which can be put into multiple lines for better readability and maintainability. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
 * FEATURE: optimize performance for queries with regexp filters on metric name like `{__name__=~"metric1|...|metricN"}`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1610) from @faceair.
 * FEATURE: vmui: use Prometheus-compatible query args, so `vmui` could be accessed from graph editor in Grafana. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1619). Thanks to @Loori-R.
+* FEATURE: vmselect: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8401,vmstorage2:8401`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup).
+* FEATURE: vminsert: automatically add missing port to `-storageNode` hostnames. For example, `-storageNode=vmstorage1,vmstorage2` is automatically translated to `-storageNode=vmstorage1:8400,vmstorage2:8400`. This simplifies [manual setup of VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-setup).
 
 * BUGFIX: properly handle queries with multiple filters matching empty labels such as `metric{label1=~"foo|",label2="bar|"}`. This filter must match the following series: `metric`, `metric{label1="foo"}`, `metric{label2="bar"}` and `metric{label1="foo",label2="bar"}`. Previously it was matching only `metric{label1="foo",label2="bar"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1601).
 * BUGFIX: vmselect: reset connection timeouts after each request to `vmstorage`. This should prevent from `cannot read data in 0.000 seconds: unexpected EOF` warning in logs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1562). Thanks to @mxlxm .

docs/Cluster-VictoriaMetrics.md

@@ -132,8 +132,8 @@ ROOT_IMAGE=scratch make package
 A minimal cluster must contain the following nodes:
 
 * a single `vmstorage` node with `-retentionPeriod` and `-storageDataPath` flags
-* a single `vminsert` node with `-storageNode=<vmstorage_host>:8400`
-* a single `vmselect` node with `-storageNode=<vmstorage_host>:8401`
+* a single `vminsert` node with `-storageNode=<vmstorage_host>`
+* a single `vmselect` node with `-storageNode=<vmstorage_host>`
 
 It is recommended to run at least two nodes for each service
 for high availability purposes.
@@ -271,8 +271,8 @@ Cluster performance and capacity scales with adding new nodes.
 Steps to add `vmstorage` node:
 
 1. Start new `vmstorage` node with the same `-retentionPeriod` as existing nodes in the cluster.
-2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8401`.
-3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>:8400`.
+2. Gradually restart all the `vmselect` nodes with new `-storageNode` arg containing `<new_vmstorage_host>`.
+3. Gradually restart all the `vminsert` nodes with new `-storageNode` arg containing `<new_vmstorage_host>`.
 
 
 ## Updating / reconfiguring cluster nodes
@@ -284,6 +284,8 @@ with new configs.
 Cluster should remain in working state if at least a single node of each type remains available during
 the update process. See [cluster availability](#cluster-availability) section for details.
 
+See also more advanced [cardinality limiter in vmagent](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter).
+
 
 ## Cluster availability
 
@@ -565,7 +567,7 @@ Below is the output for `/path/to/vminsert -help`:
   -sortLabels
     	Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
   -storageNode array
-    	Address of vmstorage nodes; usage: -storageNode=vmstorage-host1:8400 -storageNode=vmstorage-host2:8400
+    	Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN
     	Supports an array of values separated by comma or specified via multiple flags.
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set
@@ -679,10 +681,10 @@ Below is the output for `/path/to/vmselect -help`:
   -search.treatDotsAsIsInRegexps
     	Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter
   -selectNode array
-    	Addresses of vmselect nodes; usage: -selectNode=vmselect-host1:8481 -selectNode=vmselect-host2:8481
+    	Comma-serparated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN
     	Supports an array of values separated by comma or specified via multiple flags.
   -storageNode array
-    	Addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1:8401 -storageNode=vmstorage-host2:8401
+    	Comma-separated dddresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN
     	Supports an array of values separated by comma or specified via multiple flags.
   -tls
     	Whether to enable TLS (aka HTTPS) for incoming requests. -tlsCertFile and -tlsKeyFile must be set if -tls is set