diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index cdd7de0dd6..6ae13e81c5 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -50,8 +50,8 @@ var ( "By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages") shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+ "among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+ - "even distribution of series over the specified -remoteWrite.url systems") - shardByURLInverseLabels = flag.Bool("remoteWrite.shardByURL.inverseLabels", false, "Inverse the behavior of remoteWrite.shardByURL.labels so that series are sharded using all labels except the ones specified in remoteWrite.shardByURL.labels.") + "even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.inverseLabels.") + shardByURLLabelsInverse = flag.Bool("remoteWrite.shardByURL.inverseLabels", false, "Inverse the behavior of -remoteWrite.shardByURL.labels so that series are sharded using all labels except the ones specified in -remoteWrite.shardByURL.labels.") tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+ "See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue") keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+ @@ -538,18 +538,8 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar // Shard the data among rwctxs tssByURL := make([][]prompbmarshal.TimeSeries, len(rwctxs)) tmpLabels := promutils.GetLabels() - inverseLabels := *shardByURLInverseLabels for _, ts := range tssBlock { - hashLabels := ts.Labels - if len(shardByURLLabelsMap) > 0 { - hashLabels = tmpLabels.Labels[:0] - for _, label := range ts.Labels { - _, ok := shardByURLLabelsMap[label.Name] - if ok && !inverseLabels || !ok && inverseLabels { - hashLabels = append(hashLabels, label) - } - } - } + hashLabels := extractShardingLabels(tmpLabels.Labels, ts.Labels, *shardByURLLabelsInverse) h := getLabelsHash(hashLabels) idx := h % uint64(len(tssByURL)) tssByURL[idx] = append(tssByURL[idx], ts) @@ -595,6 +585,20 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar return !anyPushFailed.Load() } +func extractShardingLabels(dst, src []prompbmarshal.Label, inverse bool) []prompbmarshal.Label { + if len(shardByURLLabelsMap) < 1 { + return src + } + dst = dst[:0] + for _, label := range src { + _, ok := shardByURLLabelsMap[label.Name] + if ok && !inverse || !ok && inverse { + dst = append(dst, label) + } + } + return dst +} + // sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set. func sortLabelsIfNeeded(tss []prompbmarshal.TimeSeries) { if !*sortLabels { diff --git a/app/vmagent/remotewrite/remotewrite_test.go b/app/vmagent/remotewrite/remotewrite_test.go new file mode 100644 index 0000000000..b5634772ab --- /dev/null +++ b/app/vmagent/remotewrite/remotewrite_test.go @@ -0,0 +1,38 @@ +package remotewrite + +import ( + "reflect" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" +) + +func TestExtractShardingLabels(t *testing.T) { + shardByURLLabelsMap = make(map[string]struct{}) + shardByURLLabelsMap["instance"] = struct{}{} + shardByURLLabelsMap["job"] = struct{}{} + defer func() { + shardByURLLabelsMap = nil + }() + + f := func(in, exp []prompbmarshal.Label, inverse bool) { + t.Helper() + var got []prompbmarshal.Label + got = extractShardingLabels(got, in, inverse) + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected to get \n%#v; \ngot \n%#v instead", exp, got) + } + } + + f(nil, nil, true) + f(nil, nil, false) + + f([]prompbmarshal.Label{{Name: "foo"}}, nil, false) + f([]prompbmarshal.Label{{Name: "foo"}}, []prompbmarshal.Label{{Name: "foo"}}, true) + + f([]prompbmarshal.Label{{Name: "foo"}, {Name: "job"}}, []prompbmarshal.Label{{Name: "job"}}, false) + f([]prompbmarshal.Label{{Name: "foo"}, {Name: "job"}}, []prompbmarshal.Label{{Name: "foo"}}, true) + + f([]prompbmarshal.Label{{Name: "foo"}, {Name: "instance"}, {Name: "job"}}, []prompbmarshal.Label{{Name: "instance"}, {Name: "job"}}, false) + f([]prompbmarshal.Label{{Name: "foo"}, {Name: "instance"}, {Name: "job"}}, []prompbmarshal.Label{{Name: "foo"}}, true) +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6c9c6744b9..ed077327b0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -52,6 +52,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): expose `vm_streamaggr_flush_timeouts_total` and `vm_streamaggr_dedup_flush_timeouts_total` [counters](https://docs.victoriametrics.com/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/#monitoring), which can be used for detecting flush timeouts for stream aggregation states. Expose also `vm_streamaggr_flush_duration_seconds` and `vm_streamaggr_dedup_flush_duration_seconds` [histograms](https://docs.victoriametrics.com/keyconcepts/#histogram) for monitoring the real flush durations of stream aggregation states. * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve trace display for better visual separation of branches. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5926). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): use the provided `-remoteWrite.tlsServerName` as `Host` header in requests to `-remoteWrite.url`. This allows sending data to https remote storage by IP address instead of hostname. Thanks to @minor-fixes for initial idea and [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5802). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.shardByURL.inverseLabels` cmd-line flag to enhance sharding logic across configured `-remoteWrite.url` URLs when `-remoteWrite.shardByURL.labels` is set. Thanks to @edma2 for the idea and [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5938). * FEATURE: optimize [`/api/v1/labels`](https://docs.victoriametrics.com/url-examples/#apiv1labels) and [`/api/v1/label/.../values`](https://docs.victoriametrics.com/url-examples/#apiv1labelvalues) when `match[]` filters contains metric name. For example, `/api/v1/label/instance/values?match[]=up` now works much faster than before. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5055). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [native protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-from-victoriametrics). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5824). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for VictoriaMetrics destination specified via `--vm-*` cmd-line flags used in [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol), [OpenTSDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-opentsdb), [Prometheus](https://docs.victoriametrics.com/vmctl/#migrating-data-from-prometheus) and [Promscale](https://docs.victoriametrics.com/vmctl/#migrating-data-from-promscale) migration modes. diff --git a/docs/vmagent.md b/docs/vmagent.md index f24ff2a71a..c19b09077b 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -191,7 +191,8 @@ among remote storage systems specified in `-remoteWrite.url`. Sometimes it may b set of labels for sharding. For example, it may be needed to route all the metrics with the same `instance` label to the same `-remoteWrite.url`. In this case you can specify comma-separated list of these labels in the `-remoteWrite.shardByURL.labels` command-line flag. For example, `-remoteWrite.shardByURL.labels=instance,__name__` would shard metrics with the same name and `instance` -label to the same `-remoteWrite.url`. +label to the same `-remoteWrite.url`. The sharding logic can be inverted by specifying `-remoteWrite.shardByURL.inverseLabels`, +so that metrics are sharded using all labels except the ones specified in `-remoteWrite.shardByURL.labels`. See also [how to scrape big number of targets](#scraping-big-number-of-targets). @@ -2105,8 +2106,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Empty values are set to default value. -remoteWrite.shardByURL Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages + -remoteWrite.shardByURL.inverseLabels + Inverse the behavior of -remoteWrite.shardByURL.labels so that series are sharded using all labels except the ones specified in -remoteWrite.shardByURL.labels. -remoteWrite.shardByURL.labels array - Optional list of labels, which must be used for sharding outgoing samples among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain even distribution of series over the specified -remoteWrite.url systems + Optional list of labels, which must be used for sharding outgoing samples among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.inverseLabels. Supports an array of values separated by comma or specified via multiple flags. Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -remoteWrite.showURL