diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go
index 7a71d4ab01..5b8617e8a4 100644
--- a/app/vmctl/flags.go
+++ b/app/vmctl/flags.go
@@ -320,10 +320,11 @@ var (
)
const (
- vmNativeFilterMatch = "vm-native-filter-match"
- vmNativeFilterTimeStart = "vm-native-filter-time-start"
- vmNativeFilterTimeEnd = "vm-native-filter-time-end"
- vmNativeStepInterval = "vm-native-step-interval"
+ vmNativeFilterMatch = "vm-native-filter-match"
+ vmNativeFilterTimeStart = "vm-native-filter-time-start"
+ vmNativeFilterTimeEnd = "vm-native-filter-time-end"
+ vmNativeFilterTimeReverse = "vm-native-filter-time-reverse"
+ vmNativeStepInterval = "vm-native-step-interval"
vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol"
vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive"
@@ -362,10 +363,15 @@ var (
},
&cli.StringFlag{
Name: vmNativeStepInterval,
- Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s','%s'.", vmNativeFilterTimeStart,
+ Usage: fmt.Sprintf("Split export data into chunks. By default, the migration will start from the oldest to the newest intervals. See also '--vm-native-filter-time-reverse'. Requires setting --%s. Valid values are '%s','%s','%s','%s','%s'.", vmNativeFilterTimeStart,
stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
Value: stepper.StepMonth,
},
+ &cli.BoolFlag{
+ Name: vmNativeFilterTimeReverse,
+ Usage: "Whether to reverse order of time intervals split by '--vm-native-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest intervals.",
+ Value: false,
+ },
&cli.BoolFlag{
Name: vmNativeDisableHTTPKeepAlive,
Usage: "Disable HTTP persistent connections for requests made to VictoriaMetrics components during export",
@@ -469,6 +475,7 @@ const (
remoteReadConcurrency = "remote-read-concurrency"
remoteReadFilterTimeStart = "remote-read-filter-time-start"
remoteReadFilterTimeEnd = "remote-read-filter-time-end"
+ remoteReadFilterTimeReverse = "remote-read-filter-time-reverse"
remoteReadFilterLabel = "remote-read-filter-label"
remoteReadFilterLabelValue = "remote-read-filter-label-value"
remoteReadStepInterval = "remote-read-step-interval"
@@ -521,9 +528,14 @@ var (
},
&cli.StringFlag{
Name: remoteReadStepInterval,
- Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are %q,%q,%q,%q.", remoteReadFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
+ Usage: fmt.Sprintf("Split export data into chunks. By default, the migration will start from the oldest to the newest intervals. See also '--remote-read-filter-time-reverse'. Requires setting --%s. Valid values are %q,%q,%q,%q.", remoteReadFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
Required: true,
},
+ &cli.BoolFlag{
+ Name: remoteReadFilterTimeReverse,
+ Usage: "Whether to reverse order of time intervals split by '--remote-read-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest intervals.",
+ Value: false,
+ },
&cli.StringFlag{
Name: remoteReadSrcAddr,
Usage: "Remote read address to perform read from.",
diff --git a/app/vmctl/main.go b/app/vmctl/main.go
index 383869d002..9ee47b058d 100644
--- a/app/vmctl/main.go
+++ b/app/vmctl/main.go
@@ -12,11 +12,12 @@ import (
"syscall"
"time"
+ "github.com/urfave/cli/v2"
+
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
- "github.com/urfave/cli/v2"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
@@ -150,9 +151,10 @@ func main() {
src: rr,
dst: importer,
filter: remoteReadFilter{
- timeStart: c.Timestamp(remoteReadFilterTimeStart),
- timeEnd: c.Timestamp(remoteReadFilterTimeEnd),
- chunk: c.String(remoteReadStepInterval),
+ timeStart: c.Timestamp(remoteReadFilterTimeStart),
+ timeEnd: c.Timestamp(remoteReadFilterTimeEnd),
+ chunk: c.String(remoteReadStepInterval),
+ timeReverse: c.Bool(remoteReadFilterTimeReverse),
},
cc: c.Int(remoteReadConcurrency),
isSilent: c.Bool(globalSilent),
@@ -234,10 +236,11 @@ func main() {
rateLimit: c.Int64(vmRateLimit),
interCluster: c.Bool(vmInterCluster),
filter: native.Filter{
- Match: c.String(vmNativeFilterMatch),
- TimeStart: c.String(vmNativeFilterTimeStart),
- TimeEnd: c.String(vmNativeFilterTimeEnd),
- Chunk: c.String(vmNativeStepInterval),
+ Match: c.String(vmNativeFilterMatch),
+ TimeStart: c.String(vmNativeFilterTimeStart),
+ TimeEnd: c.String(vmNativeFilterTimeEnd),
+ Chunk: c.String(vmNativeStepInterval),
+ TimeReverse: c.Bool(vmNativeFilterTimeReverse),
},
src: &native.Client{
AuthCfg: srcAuthConfig,
diff --git a/app/vmctl/native/filter.go b/app/vmctl/native/filter.go
index a038ba3caf..7082bacba8 100644
--- a/app/vmctl/native/filter.go
+++ b/app/vmctl/native/filter.go
@@ -4,10 +4,11 @@ import "fmt"
// Filter represents request filter
type Filter struct {
- Match string
- TimeStart string
- TimeEnd string
- Chunk string
+ Match string
+ TimeStart string
+ TimeEnd string
+ Chunk string
+ TimeReverse bool
}
func (f Filter) String() string {
diff --git a/app/vmctl/remoteread.go b/app/vmctl/remoteread.go
index 8531c6cd6a..8632369c6f 100644
--- a/app/vmctl/remoteread.go
+++ b/app/vmctl/remoteread.go
@@ -7,11 +7,12 @@ import (
"sync"
"time"
+ "github.com/cheggaaa/pb/v3"
+
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
- "github.com/cheggaaa/pb/v3"
)
type remoteReadProcessor struct {
@@ -26,9 +27,10 @@ type remoteReadProcessor struct {
}
type remoteReadFilter struct {
- timeStart *time.Time
- timeEnd *time.Time
- chunk string
+ timeStart *time.Time
+ timeEnd *time.Time
+ chunk string
+ timeReverse bool
}
func (rrp *remoteReadProcessor) run(ctx context.Context) error {
@@ -41,7 +43,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
rrp.cc = 1
}
- ranges, err := stepper.SplitDateRange(*rrp.filter.timeStart, *rrp.filter.timeEnd, rrp.filter.chunk)
+ ranges, err := stepper.SplitDateRange(*rrp.filter.timeStart, *rrp.filter.timeEnd, rrp.filter.chunk, rrp.filter.timeReverse)
if err != nil {
return fmt.Errorf("failed to create date ranges for the given time filters: %v", err)
}
diff --git a/app/vmctl/stepper/split.go b/app/vmctl/stepper/split.go
index 104b38ad84..6e54014cdc 100644
--- a/app/vmctl/stepper/split.go
+++ b/app/vmctl/stepper/split.go
@@ -2,6 +2,7 @@ package stepper
import (
"fmt"
+ "sort"
"time"
)
@@ -20,7 +21,7 @@ const (
// SplitDateRange splits start-end range in a subset of ranges respecting the given step
// Ranges with granularity of StepMonth are aligned to 1st of each month in order to improve export efficiency at block transfer level
-func SplitDateRange(start, end time.Time, step string) ([][]time.Time, error) {
+func SplitDateRange(start, end time.Time, step string, timeReverse bool) ([][]time.Time, error) {
if start.After(end) {
return nil, fmt.Errorf("start time %q should come before end time %q", start.Format(time.RFC3339), end.Format(time.RFC3339))
@@ -71,6 +72,11 @@ func SplitDateRange(start, end time.Time, step string) ([][]time.Time, error) {
ranges = append(ranges, []time.Time{s, e})
currentStep = e
}
+ if timeReverse {
+ sort.SliceStable(ranges, func(i, j int) bool {
+ return ranges[i][0].After(ranges[j][0])
+ })
+ }
return ranges, nil
}
diff --git a/app/vmctl/stepper/split_test.go b/app/vmctl/stepper/split_test.go
index 062ffbffad..a5f10cc8a5 100644
--- a/app/vmctl/stepper/split_test.go
+++ b/app/vmctl/stepper/split_test.go
@@ -252,7 +252,280 @@ func Test_splitDateRange(t *testing.T) {
start := mustParseDatetime(tt.args.start)
end := mustParseDatetime(tt.args.end)
- got, err := SplitDateRange(start, end, tt.args.granularity)
+ got, err := SplitDateRange(start, end, tt.args.granularity, false)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("splitDateRange() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+
+ var testExpectedResults [][]time.Time
+ if tt.want != nil {
+ testExpectedResults = make([][]time.Time, 0)
+ for _, dr := range tt.want {
+ testExpectedResults = append(testExpectedResults, []time.Time{
+ mustParseDatetime(dr[0]),
+ mustParseDatetime(dr[1]),
+ })
+ }
+ }
+
+ if !reflect.DeepEqual(got, testExpectedResults) {
+ t.Errorf("splitDateRange() got = %v, want %v", got, testExpectedResults)
+ }
+ })
+ }
+}
+
+func Test_splitDateRange_reverse(t *testing.T) {
+ type args struct {
+ start string
+ end string
+ granularity string
+ timeReverse bool
+ }
+ tests := []struct {
+ name string
+ args args
+ want []testTimeRange
+ wantErr bool
+ }{
+ {
+ name: "validates start is before end",
+ args: args{
+ start: "2022-02-01T00:00:00Z",
+ end: "2022-01-01T00:00:00Z",
+ granularity: StepMonth,
+ timeReverse: true,
+ },
+ want: nil,
+ wantErr: true,
+ },
+ {
+ name: "validates granularity value",
+ args: args{
+ start: "2022-01-01T00:00:00Z",
+ end: "2022-02-01T00:00:00Z",
+ granularity: "non-existent-format",
+ timeReverse: true,
+ },
+ want: nil,
+ wantErr: true,
+ },
+ {
+ name: "month chunking",
+ args: args{
+ start: "2022-01-03T11:11:11Z",
+ end: "2022-03-03T12:12:12Z",
+ granularity: StepMonth,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2022-03-01T00:00:00Z",
+ "2022-03-03T12:12:12Z",
+ },
+ {
+ "2022-02-01T00:00:00Z",
+ "2022-02-28T23:59:59.999999999Z",
+ },
+ {
+ "2022-01-03T11:11:11Z",
+ "2022-01-31T23:59:59.999999999Z",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "daily chunking",
+ args: args{
+ start: "2022-01-03T11:11:11Z",
+ end: "2022-01-05T12:12:12Z",
+ granularity: StepDay,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2022-01-05T11:11:11Z",
+ "2022-01-05T12:12:12Z",
+ },
+ {
+ "2022-01-04T11:11:11Z",
+ "2022-01-05T11:11:11Z",
+ },
+ {
+ "2022-01-03T11:11:11Z",
+ "2022-01-04T11:11:11Z",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "hourly chunking",
+ args: args{
+ start: "2022-01-03T11:11:11Z",
+ end: "2022-01-03T14:14:14Z",
+ granularity: StepHour,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2022-01-03T14:11:11Z",
+ "2022-01-03T14:14:14Z",
+ },
+ {
+ "2022-01-03T13:11:11Z",
+ "2022-01-03T14:11:11Z",
+ },
+ {
+ "2022-01-03T12:11:11Z",
+ "2022-01-03T13:11:11Z",
+ },
+ {
+ "2022-01-03T11:11:11Z",
+ "2022-01-03T12:11:11Z",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "month chunking with one day time range",
+ args: args{
+ start: "2022-01-03T11:11:11Z",
+ end: "2022-01-04T12:12:12Z",
+ granularity: StepMonth,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2022-01-03T11:11:11Z",
+ "2022-01-04T12:12:12Z",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "month chunking with same day time range",
+ args: args{
+ start: "2022-01-03T11:11:11Z",
+ end: "2022-01-03T12:12:12Z",
+ granularity: StepMonth,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2022-01-03T11:11:11Z",
+ "2022-01-03T12:12:12Z",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "month chunking with one month and two days range",
+ args: args{
+ start: "2022-01-03T11:11:11Z",
+ end: "2022-02-03T00:00:00Z",
+ granularity: StepMonth,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2022-02-01T00:00:00Z",
+ "2022-02-03T00:00:00Z",
+ },
+ {
+ "2022-01-03T11:11:11Z",
+ "2022-01-31T23:59:59.999999999Z",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "week chunking with not full week",
+ args: args{
+ start: "2023-07-30T00:00:00Z",
+ end: "2023-08-05T23:59:59.999999999Z",
+ granularity: StepWeek,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2023-07-30T00:00:00Z",
+ "2023-08-05T23:59:59.999999999Z",
+ },
+ },
+ },
+ {
+ name: "week chunking with start of the week and end of the week",
+ args: args{
+ start: "2023-07-30T00:00:00Z",
+ end: "2023-08-06T00:00:00Z",
+ granularity: StepWeek,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2023-07-30T00:00:00Z",
+ "2023-08-06T00:00:00Z",
+ },
+ },
+ },
+ {
+ name: "week chunking with next one day week",
+ args: args{
+ start: "2023-07-30T00:00:00Z",
+ end: "2023-08-07T01:12:00Z",
+ granularity: StepWeek,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2023-08-06T00:00:00Z",
+ "2023-08-07T01:12:00Z",
+ },
+ {
+ "2023-07-30T00:00:00Z",
+ "2023-08-06T00:00:00Z",
+ },
+ },
+ },
+ {
+ name: "week chunking with month and not full week representation",
+ args: args{
+ start: "2023-07-30T00:00:00Z",
+ end: "2023-09-01T01:12:00Z",
+ granularity: StepWeek,
+ timeReverse: true,
+ },
+ want: []testTimeRange{
+ {
+ "2023-08-27T00:00:00Z",
+ "2023-09-01T01:12:00Z",
+ },
+ {
+ "2023-08-20T00:00:00Z",
+ "2023-08-27T00:00:00Z",
+ },
+ {
+ "2023-08-13T00:00:00Z",
+ "2023-08-20T00:00:00Z",
+ },
+ {
+ "2023-08-06T00:00:00Z",
+ "2023-08-13T00:00:00Z",
+ },
+ {
+ "2023-07-30T00:00:00Z",
+ "2023-08-06T00:00:00Z",
+ },
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ start := mustParseDatetime(tt.args.start)
+ end := mustParseDatetime(tt.args.end)
+
+ got, err := SplitDateRange(start, end, tt.args.granularity, tt.args.timeReverse)
if (err != nil) != tt.wantErr {
t.Errorf("splitDateRange() error = %v, wantErr %v", err, tt.wantErr)
return
diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go
index 31bdb6b380..49fbcda806 100644
--- a/app/vmctl/vm_native.go
+++ b/app/vmctl/vm_native.go
@@ -9,6 +9,8 @@ import (
"sync"
"time"
+ "github.com/cheggaaa/pb/v3"
+
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -18,7 +20,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
- "github.com/cheggaaa/pb/v3"
)
type vmNativeProcessor struct {
@@ -67,12 +68,11 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
ranges := [][]time.Time{{start, end}}
if p.filter.Chunk != "" {
- ranges, err = stepper.SplitDateRange(start, end, p.filter.Chunk)
+ ranges, err = stepper.SplitDateRange(start, end, p.filter.Chunk, p.filter.TimeReverse)
if err != nil {
return fmt.Errorf("failed to create date ranges for the given time filters: %w", err)
}
}
-
tenants := []string{""}
if p.interCluster {
log.Printf("Discovering tenants...")
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 51aff12e3a..5ff03b9ef4 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -56,6 +56,7 @@ The sandbox cluster installation is running under the constant load generated by
* FEATURE: add field `version` to the response for `/api/v1/status/buildinfo` API for using more efficient API in Grafana for receiving label values. Add additional info about setup Grafana datasource. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5370) and [these docs](https://docs.victoriametrics.com/#grafana-setup) for details.
* FEATURE: add `-search.maxResponseSeries` command-line flag for limiting the number of time series a single query to [`/api/v1/query`](https://docs.victoriametrics.com/keyConcepts.html#instant-query) or [`/api/v1/query_range`](https://docs.victoriametrics.com/keyConcepts.html#range-query) can return. This limit can protect Grafana from high memory usage when the query returns too many series. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5372).
* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): ease aggregation for certain alerting rules to keep more useful labels for the context. Before, all extra labels except `job` and `instance` were ignored. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5429) and this [follow-up commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/8fb68152e67712ed2c16dcfccf7cf4d0af140835). Thanks to @7840vz.
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html) add `vm-native-filter-time-reverse` command-line flag for `native` mode and `remote-read-filter-time-reverse` command-line flag for `remote-read` mode which allows reversing order of migrated data chunks. See: https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly calculate values for the first point on the graph for queries, which do not use [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions). For example, previously `count(up)` could return lower than expected values for the first point on the graph. This also could result in lower than expected values in the middle of the graph like in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388) when the response caching isn't disabled. The issue has been introduced in [v1.95.0](https://docs.victoriametrics.com/CHANGELOG.html#v1950).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): prevent from `FATAL: cannot flush metainfo` panic when [`-remoteWrite.multitenantURL`](https://docs.victoriametrics.com/vmagent.html#multitenancy) command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5357).