From 242472086b815bde8155ebed21fcb4bb5e13def2 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Tue, 12 Dec 2023 13:44:21 +0100 Subject: [PATCH] app/vmctl: follow-up after 6af732b6f7bc1e995bcfc747af31ddc31920423d Make docs more clear about new feature. Signed-off-by: hagen1778 --- app/vmctl/flags.go | 14 +++++++------- docs/CHANGELOG.md | 2 +- docs/vmctl.md | 23 ++++++++--------------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index 5b8617e8a4..ef0e0c66aa 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -363,13 +363,13 @@ var ( }, &cli.StringFlag{ Name: vmNativeStepInterval, - Usage: fmt.Sprintf("Split export data into chunks. By default, the migration will start from the oldest to the newest intervals. See also '--vm-native-filter-time-reverse'. Requires setting --%s. Valid values are '%s','%s','%s','%s','%s'.", vmNativeFilterTimeStart, - stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute), + Usage: fmt.Sprintf("The time interval to split the migration into steps. For example, to migrate 1y of data with '--%s=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--%s'. Requires setting '--%s'. Valid values are '%s','%s','%s','%s','%s'.", + vmNativeStepInterval, vmNativeFilterTimeReverse, vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute), Value: stepper.StepMonth, }, &cli.BoolFlag{ Name: vmNativeFilterTimeReverse, - Usage: "Whether to reverse order of time intervals split by '--vm-native-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest intervals.", + Usage: fmt.Sprintf("Whether to reverse the order of time intervals split by '--%s' cmd-line flag. When set, the migration will start from the newest to the oldest data.", vmNativeStepInterval), Value: false, }, &cli.BoolFlag{ @@ -527,13 +527,13 @@ var ( Value: false, }, &cli.StringFlag{ - Name: remoteReadStepInterval, - Usage: fmt.Sprintf("Split export data into chunks. By default, the migration will start from the oldest to the newest intervals. See also '--remote-read-filter-time-reverse'. Requires setting --%s. Valid values are %q,%q,%q,%q.", remoteReadFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute), - Required: true, + Name: remoteReadStepInterval, + Usage: fmt.Sprintf("The time interval to split the migration into steps. For example, to migrate 1y of data with '--%s=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--%s'. Requires setting '--%s'. Valid values are '%s','%s','%s','%s','%s'.", + remoteReadStepInterval, remoteReadFilterTimeReverse, remoteReadFilterTimeStart, stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute), Required: true, }, &cli.BoolFlag{ Name: remoteReadFilterTimeReverse, - Usage: "Whether to reverse order of time intervals split by '--remote-read-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest intervals.", + Usage: fmt.Sprintf("Whether to reverse the order of time intervals split by '--%s' cmd-line flag. When set, the migration will start from the newest to the oldest data.", remoteReadStepInterval), Value: false, }, &cli.StringFlag{ diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5ff03b9ef4..f1d26bddf5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -56,7 +56,7 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: add field `version` to the response for `/api/v1/status/buildinfo` API for using more efficient API in Grafana for receiving label values. Add additional info about setup Grafana datasource. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5370) and [these docs](https://docs.victoriametrics.com/#grafana-setup) for details. * FEATURE: add `-search.maxResponseSeries` command-line flag for limiting the number of time series a single query to [`/api/v1/query`](https://docs.victoriametrics.com/keyConcepts.html#instant-query) or [`/api/v1/query_range`](https://docs.victoriametrics.com/keyConcepts.html#range-query) can return. This limit can protect Grafana from high memory usage when the query returns too many series. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5372). * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): ease aggregation for certain alerting rules to keep more useful labels for the context. Before, all extra labels except `job` and `instance` were ignored. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5429) and this [follow-up commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/8fb68152e67712ed2c16dcfccf7cf4d0af140835). Thanks to @7840vz. -* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html) add `vm-native-filter-time-reverse` command-line flag for `native` mode and `remote-read-filter-time-reverse` command-line flag for `remote-read` mode which allows reversing order of migrated data chunks. See: https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow reversing the migrating order from the newest to the oldest data for [vm-native](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics) and [remote-read](https://docs.victoriametrics.com/vmctl.html#migrating-data-by-remote-read-protocol) modes via `--vm-native-filter-time-reverse` and `--remote-read-filter-time-reverse` command-line flags respectively. See: https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly calculate values for the first point on the graph for queries, which do not use [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions). For example, previously `count(up)` could return lower than expected values for the first point on the graph. This also could result in lower than expected values in the middle of the graph like in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388) when the response caching isn't disabled. The issue has been introduced in [v1.95.0](https://docs.victoriametrics.com/CHANGELOG.html#v1950). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): prevent from `FATAL: cannot flush metainfo` panic when [`-remoteWrite.multitenantURL`](https://docs.victoriametrics.com/vmagent.html#multitenancy) command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5357). diff --git a/docs/vmctl.md b/docs/vmctl.md index 8f2e100041..57949f7f37 100644 --- a/docs/vmctl.md +++ b/docs/vmctl.md @@ -497,7 +497,7 @@ Remote read API has two implementations of remote read API: default (`SAMPLES`) Streamed version is more efficient but has lower adoption (e.g. [Promscale](#migrating-data-from-promscale) doesn't support it). -See `./vmctl remote-read --help` for details and full list of flags. +See `./vmctl remote-read --help` for details and the full list of flags. To start the migration process configure the following flags: @@ -878,21 +878,14 @@ Importing tips: ### Using time-based chunking of migration -It is possible split migration process into set of smaller batches based on time. This is especially useful when -migrating large volumes of data as this adds indication of progress and ability to restore process from certain point -in case of failure. +It is possible to split the migration process into steps based on time via `--vm-native-step-interval` cmd-line flag. +Supported values are: `month`, `week`, `day`, `hour`, `minute`. For example, when migrating 1 year of data with +`--vm-native-step-interval=month` vmctl will execute it in 12 separate requests from the beginning of the interval +to its end. To reverse the order set `--vm-native-filter-time-reverse` and migration will start from the newest to the +oldest data. `--vm-native-filter-time-start` is required to be set when using `--vm-native-step-interval`. -To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `week`, `day`, `hour`, `minute`. -Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for -export process. - -Every range is being processed independently, which means that: -- after range processing is finished all data within range is migrated -- if process fails on one of stages it is guaranteed that data of prior stages is already written, -so it is possible to restart process starting from failed range. - -It is recommended using the `month` step when migrating the data over multiple months, -since the migration with `week`, `day` and `hour` steps may take longer time to complete because of additional overhead. +It is recommended using default `month` step when migrating the data over the long time intervals. If you hit complexity +limits on `--vm-native-src-addr` and can't or don't want to change them, try lowering the step interval to `week`, `day` or `hour`. Usage example: ```console