app/vmctl: add support for the different time format in the native binary protocol (#4189)

* app/vmctl: add support for the different time format in the native binary protocol

* app/vmctl: update flag description, update CHANGELOG.md

* app/vmctl: add comment to exported function

app/vmctl/flags.go

@@ -352,12 +352,12 @@ var (
 		},
 		&cli.StringFlag{
 			Name:     vmNativeFilterTimeStart,
-			Usage:    "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
+			Usage:    "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats",
 			Required: true,
 		},
 		&cli.StringFlag{
 			Name:  vmNativeFilterTimeEnd,
-			Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
+			Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeStepInterval,

app/vmctl/utils/time.go

@@ -0,0 +1,105 @@
+package utils
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+const (
+	// These values prevent from overflow when storing msec-precision time in int64.
+	minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
+	maxTimeMsecs = int64(1<<63-1) / 1e6
+)
+
+func parseTime(s string) (float64, error) {
+	if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') {
+		// Parse duration relative to the current time
+		d, err := promutils.ParseDuration(s)
+		if err != nil {
+			return 0, err
+		}
+		if d > 0 {
+			d = -d
+		}
+		t := time.Now().Add(d)
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 4 {
+		// Parse YYYY
+		t, err := time.Parse("2006", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if !strings.Contains(s, "-") {
+		// Parse the timestamp in milliseconds
+		return strconv.ParseFloat(s, 64)
+	}
+	if len(s) == 7 {
+		// Parse YYYY-MM
+		t, err := time.Parse("2006-01", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 10 {
+		// Parse YYYY-MM-DD
+		t, err := time.Parse("2006-01-02", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 13 {
+		// Parse YYYY-MM-DDTHH
+		t, err := time.Parse("2006-01-02T15", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 16 {
+		// Parse YYYY-MM-DDTHH:MM
+		t, err := time.Parse("2006-01-02T15:04", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 19 {
+		// Parse YYYY-MM-DDTHH:MM:SS
+		t, err := time.Parse("2006-01-02T15:04:05", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return 0, err
+	}
+	return float64(t.UnixNano()) / 1e9, nil
+}
+
+// GetTime  returns time from the given string.
+func GetTime(s string) (time.Time, error) {
+	secs, err := parseTime(s)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err)
+	}
+	msecs := int64(secs * 1e3)
+	if msecs < minTimeMsecs {
+		msecs = 0
+	}
+	if msecs > maxTimeMsecs {
+		msecs = maxTimeMsecs
+	}
+
+	return time.Unix(0, msecs*int64(time.Millisecond)), nil
+}

app/vmctl/utils/time_test.go

@@ -0,0 +1,182 @@
+package utils
+
+import (
+	"testing"
+	"time"
+)
+
+func TestGetTime(t *testing.T) {
+	l, _ := time.LoadLocation("UTC")
+	tests := []struct {
+		name    string
+		s       string
+		want    func() time.Time
+		wantErr bool
+	}{
+		{
+			name:    "empty string",
+			s:       "",
+			want:    func() time.Time { return time.Time{} },
+			wantErr: true,
+		},
+		{
+			name: "only year",
+			s:    "2019",
+			want: func() time.Time {
+				t := time.Date(2019, 1, 1, 0, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "year and month",
+			s:    "2019-01",
+			want: func() time.Time {
+				t := time.Date(2019, 1, 1, 0, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "year and not first month",
+			s:    "2019-02",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 1, 0, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "year, month and day",
+			s:    "2019-02-01",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 1, 0, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "year, month and not first day",
+			s:    "2019-02-10",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 10, 0, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "year, month, day and time",
+			s:    "2019-02-02T00",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 2, 0, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "year, month, day and one hour time",
+			s:    "2019-02-02T01",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 2, 1, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "time with zero minutes",
+			s:    "2019-02-02T01:00",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 2, 1, 0, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "time with one minute",
+			s:    "2019-02-02T01:01",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 2, 1, 1, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "time with zero seconds",
+			s:    "2019-02-02T01:01:00",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 2, 1, 1, 0, 0, l)
+				return t
+			},
+		},
+		{
+			name: "timezone with one second",
+			s:    "2019-02-02T01:01:01",
+			want: func() time.Time {
+				t := time.Date(2019, 2, 2, 1, 1, 1, 0, l)
+				return t
+			},
+		},
+		{
+			name: "time with two second and timezone",
+			s:    "2019-07-07T20:01:02Z",
+			want: func() time.Time {
+				t := time.Date(2019, 7, 7, 20, 1, 02, 0, l)
+				return t
+			},
+		},
+		{
+			name: "time with seconds and timezone",
+			s:    "2019-07-07T20:47:40+03:00",
+			want: func() time.Time {
+				l, _ = time.LoadLocation("Europe/Kiev")
+				t := time.Date(2019, 7, 7, 20, 47, 40, 0, l)
+				return t
+			},
+		},
+		{
+			name:    "negative time",
+			s:       "-292273086-05-16T16:47:06Z",
+			want:    func() time.Time { return time.Time{} },
+			wantErr: true,
+		},
+		{
+			name: "float timestamp representation",
+			s:    "1562529662.324",
+			want: func() time.Time {
+				t := time.Date(2019, 7, 7, 23, 01, 02, 324, l)
+				return t
+			},
+		},
+		{
+			name: "negative timestamp",
+			s:    "-9223372036.855",
+			want: func() time.Time {
+				l, _ = time.LoadLocation("Europe/Kiev")
+				return time.Date(1970, 01, 01, 03, 00, 00, 00, l)
+			},
+			wantErr: false,
+		},
+		{
+			name: "big timestamp",
+			s:    "9223372036.855",
+			want: func() time.Time {
+				l, _ = time.LoadLocation("Europe/Kiev")
+				t := time.Date(2262, 04, 12, 02, 47, 16, 855, l)
+				return t
+			},
+			wantErr: false,
+		},
+		{
+			name: "duration time",
+			s:    "1h5m",
+			want: func() time.Time {
+				t := time.Now().Add(-1 * time.Hour).Add(-5 * time.Minute)
+				return t
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetTime(tt.s)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseTime() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			w := tt.want()
+			if got.Unix() != w.Unix() {
+				t.Errorf("ParseTime() got = %v, want %v", got, w)
+			}
+		})
+	}
+}

app/vmctl/vm_native.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -48,18 +49,16 @@ func (p *vmNativeProcessor) run(ctx context.Context, silent bool) error {
 		startTime: time.Now(),
 	}
 
-	start, err := time.Parse(time.RFC3339, p.filter.TimeStart)
+	start, err := utils.GetTime(p.filter.TimeStart)
 	if err != nil {
-		return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
+		return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeStart, p.filter.TimeStart, err)
-			vmNativeFilterTimeStart, p.filter.TimeStart, time.RFC3339, err)
 	}
 
 	end := time.Now().In(start.Location())
 	if p.filter.TimeEnd != "" {
-		end, err = time.Parse(time.RFC3339, p.filter.TimeEnd)
+		end, err = utils.GetTime(p.filter.TimeEnd)
 		if err != nil {
-			return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
+			return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeEnd, p.filter.TimeEnd, err)
-				vmNativeFilterTimeEnd, p.filter.TimeEnd, time.RFC3339, err)
 		}
 	}
 

docs/CHANGELOG.md

@@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details.
 * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case.
 * FEATURE: introduce `-http.maxConcurrentRequests` command-line flag to protect VM components from resource exhaustion during unexpected spikes of HTTP requests. By default, the new flag's value is set to 0 which means no limits are applied.
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support for the different time formats for `--vm-native-filter-time-start` and `--vm-native-filter-time-end` flags if the native binary protocol is used for migration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4091).
 
 * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).