From 3b841fe9cea3e75829ac5ef69a428e9d6cdd2d66 Mon Sep 17 00:00:00 2001
From: hagen1778 <roman@victoriametrics.com>
Date: Tue, 12 Dec 2023 13:44:21 +0100
Subject: [PATCH] app/vmctl: follow-up after
6af732b6f7bc1e995bcfc747af31ddc31920423d
Make docs more clear about new feature.
Signed-off-by: hagen1778 <roman@victoriametrics.com>
(cherry picked from commit 242472086b815bde8155ebed21fcb4bb5e13def2)
---
app/vmctl/flags.go | 14 +++++++-------
docs/CHANGELOG.md | 2 +-
docs/vmctl.md | 23 ++++++++---------------
3 files changed, 16 insertions(+), 23 deletions(-)
diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go
index 5b8617e8a4..ef0e0c66aa 100644
--- a/app/vmctl/flags.go
+++ b/app/vmctl/flags.go
@@ -363,13 +363,13 @@ var (
},
&cli.StringFlag{
Name: vmNativeStepInterval,
- Usage: fmt.Sprintf("Split export data into chunks. By default, the migration will start from the oldest to the newest intervals. See also '--vm-native-filter-time-reverse'. Requires setting --%s. Valid values are '%s','%s','%s','%s','%s'.", vmNativeFilterTimeStart,
- stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
+ Usage: fmt.Sprintf("The time interval to split the migration into steps. For example, to migrate 1y of data with '--%s=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--%s'. Requires setting '--%s'. Valid values are '%s','%s','%s','%s','%s'.",
+ vmNativeStepInterval, vmNativeFilterTimeReverse, vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
Value: stepper.StepMonth,
},
&cli.BoolFlag{
Name: vmNativeFilterTimeReverse,
- Usage: "Whether to reverse order of time intervals split by '--vm-native-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest intervals.",
+ Usage: fmt.Sprintf("Whether to reverse the order of time intervals split by '--%s' cmd-line flag. When set, the migration will start from the newest to the oldest data.", vmNativeStepInterval),
Value: false,
},
&cli.BoolFlag{
@@ -527,13 +527,13 @@ var (
Value: false,
},
&cli.StringFlag{
- Name: remoteReadStepInterval,
- Usage: fmt.Sprintf("Split export data into chunks. By default, the migration will start from the oldest to the newest intervals. See also '--remote-read-filter-time-reverse'. Requires setting --%s. Valid values are %q,%q,%q,%q.", remoteReadFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
- Required: true,
+ Name: remoteReadStepInterval,
+ Usage: fmt.Sprintf("The time interval to split the migration into steps. For example, to migrate 1y of data with '--%s=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--%s'. Requires setting '--%s'. Valid values are '%s','%s','%s','%s','%s'.",
+ remoteReadStepInterval, remoteReadFilterTimeReverse, remoteReadFilterTimeStart, stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute), Required: true,
},
&cli.BoolFlag{
Name: remoteReadFilterTimeReverse,
- Usage: "Whether to reverse order of time intervals split by '--remote-read-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest intervals.",
+ Usage: fmt.Sprintf("Whether to reverse the order of time intervals split by '--%s' cmd-line flag. When set, the migration will start from the newest to the oldest data.", remoteReadStepInterval),
Value: false,
},
&cli.StringFlag{
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 5ff03b9ef4..f1d26bddf5 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -56,7 +56,7 @@ The sandbox cluster installation is running under the constant load generated by
* FEATURE: add field `version` to the response for `/api/v1/status/buildinfo` API for using more efficient API in Grafana for receiving label values. Add additional info about setup Grafana datasource. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5370) and [these docs](https://docs.victoriametrics.com/#grafana-setup) for details.
* FEATURE: add `-search.maxResponseSeries` command-line flag for limiting the number of time series a single query to [`/api/v1/query`](https://docs.victoriametrics.com/keyConcepts.html#instant-query) or [`/api/v1/query_range`](https://docs.victoriametrics.com/keyConcepts.html#range-query) can return. This limit can protect Grafana from high memory usage when the query returns too many series. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5372).
* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): ease aggregation for certain alerting rules to keep more useful labels for the context. Before, all extra labels except `job` and `instance` were ignored. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5429) and this [follow-up commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/8fb68152e67712ed2c16dcfccf7cf4d0af140835). Thanks to @7840vz.
-* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html) add `vm-native-filter-time-reverse` command-line flag for `native` mode and `remote-read-filter-time-reverse` command-line flag for `remote-read` mode which allows reversing order of migrated data chunks. See: https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow reversing the migrating order from the newest to the oldest data for [vm-native](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics) and [remote-read](https://docs.victoriametrics.com/vmctl.html#migrating-data-by-remote-read-protocol) modes via `--vm-native-filter-time-reverse` and `--remote-read-filter-time-reverse` command-line flags respectively. See: https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly calculate values for the first point on the graph for queries, which do not use [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions). For example, previously `count(up)` could return lower than expected values for the first point on the graph. This also could result in lower than expected values in the middle of the graph like in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388) when the response caching isn't disabled. The issue has been introduced in [v1.95.0](https://docs.victoriametrics.com/CHANGELOG.html#v1950).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): prevent from `FATAL: cannot flush metainfo` panic when [`-remoteWrite.multitenantURL`](https://docs.victoriametrics.com/vmagent.html#multitenancy) command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5357).
diff --git a/docs/vmctl.md b/docs/vmctl.md
index 8f2e100041..57949f7f37 100644
--- a/docs/vmctl.md
+++ b/docs/vmctl.md
@@ -497,7 +497,7 @@ Remote read API has two implementations of remote read API: default (`SAMPLES`)
Streamed version is more efficient but has lower adoption (e.g. [Promscale](#migrating-data-from-promscale)
doesn't support it).
-See `./vmctl remote-read --help` for details and full list of flags.
+See `./vmctl remote-read --help` for details and the full list of flags.
To start the migration process configure the following flags:
@@ -878,21 +878,14 @@ Importing tips:
### Using time-based chunking of migration
-It is possible split migration process into set of smaller batches based on time. This is especially useful when
-migrating large volumes of data as this adds indication of progress and ability to restore process from certain point
-in case of failure.
+It is possible to split the migration process into steps based on time via `--vm-native-step-interval` cmd-line flag.
+Supported values are: `month`, `week`, `day`, `hour`, `minute`. For example, when migrating 1 year of data with
+`--vm-native-step-interval=month` vmctl will execute it in 12 separate requests from the beginning of the interval
+to its end. To reverse the order set `--vm-native-filter-time-reverse` and migration will start from the newest to the
+oldest data. `--vm-native-filter-time-start` is required to be set when using `--vm-native-step-interval`.
-To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `week`, `day`, `hour`, `minute`.
-Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for
-export process.
-
-Every range is being processed independently, which means that:
-- after range processing is finished all data within range is migrated
-- if process fails on one of stages it is guaranteed that data of prior stages is already written,
-so it is possible to restart process starting from failed range.
-
-It is recommended using the `month` step when migrating the data over multiple months,
-since the migration with `week`, `day` and `hour` steps may take longer time to complete because of additional overhead.
+It is recommended using default `month` step when migrating the data over the long time intervals. If you hit complexity
+limits on `--vm-native-src-addr` and can't or don't want to change them, try lowering the step interval to `week`, `day` or `hour`.
Usage example:
```console