From e2053baf326b4b75183b2b8a15e1de8c3e9e000d Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Mon, 24 Apr 2023 19:33:30 +0300 Subject: [PATCH] app/vmctl: add support for the different time format in the native binary protocol (#4189) * app/vmctl: add support for the different time format in the native binary protocol * app/vmctl: update flag description, update CHANGELOG.md * app/vmctl: add comment to exported function --- app/vmctl/flags.go | 4 +- app/vmctl/utils/time.go | 105 ++++++++++++++++++++ app/vmctl/utils/time_test.go | 182 +++++++++++++++++++++++++++++++++++ app/vmctl/vm_native.go | 11 +-- docs/CHANGELOG.md | 1 + 5 files changed, 295 insertions(+), 8 deletions(-) create mode 100644 app/vmctl/utils/time.go create mode 100644 app/vmctl/utils/time_test.go diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index 86e4dca07..0edced6a4 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -352,12 +352,12 @@ var ( }, &cli.StringFlag{ Name: vmNativeFilterTimeStart, - Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'", + Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats", Required: true, }, &cli.StringFlag{ Name: vmNativeFilterTimeEnd, - Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'", + Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats", }, &cli.StringFlag{ Name: vmNativeStepInterval, diff --git a/app/vmctl/utils/time.go b/app/vmctl/utils/time.go new file mode 100644 index 000000000..541a84f45 --- /dev/null +++ b/app/vmctl/utils/time.go @@ -0,0 +1,105 @@ +package utils + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +const ( + // These values prevent from overflow when storing msec-precision time in int64. + minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time + maxTimeMsecs = int64(1<<63-1) / 1e6 +) + +func parseTime(s string) (float64, error) { + if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') { + // Parse duration relative to the current time + d, err := promutils.ParseDuration(s) + if err != nil { + return 0, err + } + if d > 0 { + d = -d + } + t := time.Now().Add(d) + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 4 { + // Parse YYYY + t, err := time.Parse("2006", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if !strings.Contains(s, "-") { + // Parse the timestamp in milliseconds + return strconv.ParseFloat(s, 64) + } + if len(s) == 7 { + // Parse YYYY-MM + t, err := time.Parse("2006-01", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 10 { + // Parse YYYY-MM-DD + t, err := time.Parse("2006-01-02", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 13 { + // Parse YYYY-MM-DDTHH + t, err := time.Parse("2006-01-02T15", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 16 { + // Parse YYYY-MM-DDTHH:MM + t, err := time.Parse("2006-01-02T15:04", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 19 { + // Parse YYYY-MM-DDTHH:MM:SS + t, err := time.Parse("2006-01-02T15:04:05", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil +} + +// GetTime returns time from the given string. +func GetTime(s string) (time.Time, error) { + secs, err := parseTime(s) + if err != nil { + return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err) + } + msecs := int64(secs * 1e3) + if msecs < minTimeMsecs { + msecs = 0 + } + if msecs > maxTimeMsecs { + msecs = maxTimeMsecs + } + + return time.Unix(0, msecs*int64(time.Millisecond)), nil +} diff --git a/app/vmctl/utils/time_test.go b/app/vmctl/utils/time_test.go new file mode 100644 index 000000000..55ed73e5f --- /dev/null +++ b/app/vmctl/utils/time_test.go @@ -0,0 +1,182 @@ +package utils + +import ( + "testing" + "time" +) + +func TestGetTime(t *testing.T) { + l, _ := time.LoadLocation("UTC") + tests := []struct { + name string + s string + want func() time.Time + wantErr bool + }{ + { + name: "empty string", + s: "", + want: func() time.Time { return time.Time{} }, + wantErr: true, + }, + { + name: "only year", + s: "2019", + want: func() time.Time { + t := time.Date(2019, 1, 1, 0, 0, 0, 0, l) + return t + }, + }, + { + name: "year and month", + s: "2019-01", + want: func() time.Time { + t := time.Date(2019, 1, 1, 0, 0, 0, 0, l) + return t + }, + }, + { + name: "year and not first month", + s: "2019-02", + want: func() time.Time { + t := time.Date(2019, 2, 1, 0, 0, 0, 0, l) + return t + }, + }, + { + name: "year, month and day", + s: "2019-02-01", + want: func() time.Time { + t := time.Date(2019, 2, 1, 0, 0, 0, 0, l) + return t + }, + }, + { + name: "year, month and not first day", + s: "2019-02-10", + want: func() time.Time { + t := time.Date(2019, 2, 10, 0, 0, 0, 0, l) + return t + }, + }, + { + name: "year, month, day and time", + s: "2019-02-02T00", + want: func() time.Time { + t := time.Date(2019, 2, 2, 0, 0, 0, 0, l) + return t + }, + }, + { + name: "year, month, day and one hour time", + s: "2019-02-02T01", + want: func() time.Time { + t := time.Date(2019, 2, 2, 1, 0, 0, 0, l) + return t + }, + }, + { + name: "time with zero minutes", + s: "2019-02-02T01:00", + want: func() time.Time { + t := time.Date(2019, 2, 2, 1, 0, 0, 0, l) + return t + }, + }, + { + name: "time with one minute", + s: "2019-02-02T01:01", + want: func() time.Time { + t := time.Date(2019, 2, 2, 1, 1, 0, 0, l) + return t + }, + }, + { + name: "time with zero seconds", + s: "2019-02-02T01:01:00", + want: func() time.Time { + t := time.Date(2019, 2, 2, 1, 1, 0, 0, l) + return t + }, + }, + { + name: "timezone with one second", + s: "2019-02-02T01:01:01", + want: func() time.Time { + t := time.Date(2019, 2, 2, 1, 1, 1, 0, l) + return t + }, + }, + { + name: "time with two second and timezone", + s: "2019-07-07T20:01:02Z", + want: func() time.Time { + t := time.Date(2019, 7, 7, 20, 1, 02, 0, l) + return t + }, + }, + { + name: "time with seconds and timezone", + s: "2019-07-07T20:47:40+03:00", + want: func() time.Time { + l, _ = time.LoadLocation("Europe/Kiev") + t := time.Date(2019, 7, 7, 20, 47, 40, 0, l) + return t + }, + }, + { + name: "negative time", + s: "-292273086-05-16T16:47:06Z", + want: func() time.Time { return time.Time{} }, + wantErr: true, + }, + { + name: "float timestamp representation", + s: "1562529662.324", + want: func() time.Time { + t := time.Date(2019, 7, 7, 23, 01, 02, 324, l) + return t + }, + }, + { + name: "negative timestamp", + s: "-9223372036.855", + want: func() time.Time { + l, _ = time.LoadLocation("Europe/Kiev") + return time.Date(1970, 01, 01, 03, 00, 00, 00, l) + }, + wantErr: false, + }, + { + name: "big timestamp", + s: "9223372036.855", + want: func() time.Time { + l, _ = time.LoadLocation("Europe/Kiev") + t := time.Date(2262, 04, 12, 02, 47, 16, 855, l) + return t + }, + wantErr: false, + }, + { + name: "duration time", + s: "1h5m", + want: func() time.Time { + t := time.Now().Add(-1 * time.Hour).Add(-5 * time.Minute) + return t + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTime(tt.s) + if (err != nil) != tt.wantErr { + t.Errorf("ParseTime() error = %v, wantErr %v", err, tt.wantErr) + return + } + w := tt.want() + if got.Unix() != w.Unix() { + t.Errorf("ParseTime() got = %v, want %v", got, w) + } + }) + } +} diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go index 440f2f21d..9f2696387 100644 --- a/app/vmctl/vm_native.go +++ b/app/vmctl/vm_native.go @@ -13,6 +13,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -48,18 +49,16 @@ func (p *vmNativeProcessor) run(ctx context.Context, silent bool) error { startTime: time.Now(), } - start, err := time.Parse(time.RFC3339, p.filter.TimeStart) + start, err := utils.GetTime(p.filter.TimeStart) if err != nil { - return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w", - vmNativeFilterTimeStart, p.filter.TimeStart, time.RFC3339, err) + return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeStart, p.filter.TimeStart, err) } end := time.Now().In(start.Location()) if p.filter.TimeEnd != "" { - end, err = time.Parse(time.RFC3339, p.filter.TimeEnd) + end, err = utils.GetTime(p.filter.TimeEnd) if err != nil { - return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w", - vmNativeFilterTimeEnd, p.filter.TimeEnd, time.RFC3339, err) + return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeEnd, p.filter.TimeEnd, err) } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 087ee6a1e..1cd915dae 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details. * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case. * FEATURE: introduce `-http.maxConcurrentRequests` command-line flag to protect VM components from resource exhaustion during unexpected spikes of HTTP requests. By default, the new flag's value is set to 0 which means no limits are applied. +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support for the different time formats for `--vm-native-filter-time-start` and `--vm-native-filter-time-end` flags if the native binary protocol is used for migration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4091). * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores. * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).