diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index 782acbfd9..dc9a130e0 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -50,30 +50,18 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { if len(argValue) == 0 { return roundToSeconds(defaultMs), nil } - secs, err := strconv.ParseFloat(argValue, 64) + // Handle Prometheus'-provided minTime and maxTime. + // See https://github.com/prometheus/client_golang/issues/614 + switch argValue { + case prometheusMinTimeFormatted: + return minTimeMsecs, nil + case prometheusMaxTimeFormatted: + return maxTimeMsecs, nil + } + // Parse argValue + secs, err := parseTime(argValue) if err != nil { - // Try parsing string format - t, err := time.Parse(time.RFC3339, argValue) - if err != nil { - // Handle Prometheus'-provided minTime and maxTime. - // See https://github.com/prometheus/client_golang/issues/614 - switch argValue { - case prometheusMinTimeFormatted: - return minTimeMsecs, nil - case prometheusMaxTimeFormatted: - return maxTimeMsecs, nil - } - // Try parsing duration relative to the current time - d, err1 := promutils.ParseDuration(argValue) - if err1 != nil { - return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) - } - if d > 0 { - d = -d - } - t = time.Now().Add(d) - } - secs = float64(t.UnixNano()) / 1e9 + return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err) } msecs := int64(secs * 1e3) if msecs < minTimeMsecs { @@ -85,6 +73,78 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { return msecs, nil } +func parseTime(s string) (float64, error) { + if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') { + // Parse duration relative to the current time + d, err := promutils.ParseDuration(s) + if err != nil { + return 0, err + } + if d > 0 { + d = -d + } + t := time.Now().Add(d) + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 4 { + // Parse YYYY + t, err := time.Parse("2006", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if !strings.Contains(s, "-") { + // Parse the timestamp in milliseconds + return strconv.ParseFloat(s, 64) + } + if len(s) == 7 { + // Parse YYYY-MM + t, err := time.Parse("2006-01", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 10 { + // Parse YYYY-MM-DD + t, err := time.Parse("2006-01-02", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 13 { + // Parse YYYY-MM-DDTHH + t, err := time.Parse("2006-01-02T15", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 16 { + // Parse YYYY-MM-DDTHH:MM + t, err := time.Parse("2006-01-02T15:04", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + if len(s) == 19 { + // Parse YYYY-MM-DDTHH:MM:SS + t, err := time.Parse("2006-01-02T15:04:05", s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return 0, err + } + return float64(t.UnixNano()) / 1e9, nil +} + var ( // These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442 // See https://github.com/prometheus/client_golang/issues/614 for details. diff --git a/app/vmselect/searchutils/searchutils_test.go b/app/vmselect/searchutils/searchutils_test.go index ccdb9b2c6..8090da898 100644 --- a/app/vmselect/searchutils/searchutils_test.go +++ b/app/vmselect/searchutils/searchutils_test.go @@ -11,6 +11,69 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) +func TestGetDurationSuccess(t *testing.T) { + f := func(s string, dExpected int64) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + // Verify defaultValue + d, err := GetDuration(r, "foo", 123456) + if err != nil { + t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err) + } + if d != 123456 { + t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456) + } + + // Verify dExpected + d, err = GetDuration(r, "s", 123) + if err != nil { + t.Fatalf("unexpected error in GetDuration(%q): %s", s, err) + } + if d != dExpected { + t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected) + } + } + + f("1.234", 1234) + f("1.23ms", 1) + f("1.23s", 1230) + f("2s56ms", 2056) + f("2s-5ms", 1995) + f("5m3.5s", 303500) + f("2h", 7200000) + f("1d", 24*3600*1000) + f("7d5h4m3s534ms", 623043534) +} + +func TestGetDurationError(t *testing.T) { + f := func(s string) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + if _, err := GetDuration(r, "s", 123); err == nil { + t.Fatalf("expecting non-nil error in GetDuration(%q)", s) + } + } + + // Negative durations aren't supported + f("-1.234") + + // Invalid duration + f("foo") + + // Invalid suffix + f("1md") +} + func TestGetTimeSuccess(t *testing.T) { f := func(s string, timestampExpected int64) { t.Helper() @@ -39,6 +102,17 @@ func TestGetTimeSuccess(t *testing.T) { } } + f("2019", 1546300800000) + f("2019-01", 1546300800000) + f("2019-02", 1548979200000) + f("2019-02-01", 1548979200000) + f("2019-02-02", 1549065600000) + f("2019-02-02T00", 1549065600000) + f("2019-02-02T01", 1549069200000) + f("2019-02-02T01:00", 1549069200000) + f("2019-02-02T01:01", 1549069260000) + f("2019-02-02T01:01:00", 1549069260000) + f("2019-02-02T01:01:01", 1549069261000) f("2019-07-07T20:01:02Z", 1562529662000) f("2019-07-07T20:47:40+03:00", 1562521660000) f("-292273086-05-16T16:47:06Z", minTimeMsecs) @@ -58,27 +132,26 @@ func TestGetTimeError(t *testing.T) { t.Fatalf("unexpected error in NewRequest: %s", err) } - // Verify defaultValue - ts, err := GetTime(r, "foo", 123456) - if err != nil { - t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err) - } - if ts != 123000 { - t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000) - } - - // Verify timestampExpected - _, err = GetTime(r, "s", 123) - if err == nil { + if _, err := GetTime(r, "s", 123); err == nil { t.Fatalf("expecting non-nil error in GetTime(%q)", s) } } f("foo") + f("foo1") + f("1245-5") + f("2022-x7") + f("2022-02-x7") + f("2022-02-02Tx7") + f("2022-02-02T00:x7") + f("2022-02-02T00:00:x7") + f("2022-02-02T00:00:00a") f("2019-07-07T20:01:02Zisdf") f("2019-07-07T20:47:40+03:00123") f("-292273086-05-16T16:47:07Z") f("292277025-08-18T07:12:54.999999998Z") + f("123md") + f("-12.3md") } func TestGetExtraTagFilters(t *testing.T) { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d2b502b70..7d36cc029 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386). +* FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). * FEATURE: [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling): add support for `keepequal` and `dropequal` relabeling actions, which are supported by Prometheus starting from [v2.41.0](https://github.com/prometheus/prometheus/releases/tag/v2.41.0). These relabeling actions are almost identical to `keep_if_equal` and `drop_if_equal` relabeling actions supported by VictoriaMetrics since `v1.38.0` - see [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) - so it is recommended sticking to `keep_if_equal` and `drop_if_equal` actions instead of switching to `keepequal` and `dropequal`. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513). diff --git a/docs/README.md b/docs/README.md index 999827c26..3ee3106da 100644 --- a/docs/README.md +++ b/docs/README.md @@ -715,8 +715,7 @@ VictoriaMetrics accepts optional `extra_label==` query VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example, `/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given ``. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies. -VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). -For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`. +VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats). VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point. @@ -741,6 +740,18 @@ Additionally, VictoriaMetrics provides the following handlers: For example, request to `/api/v1/status/top_queries?topN=5&maxLifetime=30s` would return up to 5 queries per list, which were executed during the last 30 seconds. VictoriaMetrics tracks the last `-search.queryStats.lastQueriesCount` queries with durations at least `-search.queryStats.minQueryDuration`. +### Timestamp formats + +VictoriaMetrics accepts the following formats for `time`, `start` and `end` query args +in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and +in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). + +- Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. +- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. + + ## Graphite API usage VictoriaMetrics supports data ingestion in Graphite protocol - see [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details. @@ -960,8 +971,9 @@ Each JSON line contains samples for a single time series. An example output: {"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]} ``` -Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either -unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/api/v1/export -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' @@ -1008,8 +1020,9 @@ where: * `` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for metrics to export. -Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either -unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/api/v1/export/csv -d 'format=' -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' @@ -1035,8 +1048,9 @@ wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq # relaunch victoriametrics with search.maxExportSeries more than value from previous command ``` -Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either -unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/api/v1/export/native -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' @@ -1272,7 +1286,8 @@ VictoriaMetrics exports [Prometheus-compatible federation data](https://promethe at `http://:8428/federate?match[]=`. Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval. -`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/federate -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 3f7f17f5b..8046d5b71 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -718,8 +718,7 @@ VictoriaMetrics accepts optional `extra_label==` query VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example, `/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given ``. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies. -VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). -For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`. +VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats). VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point. @@ -744,6 +743,18 @@ Additionally, VictoriaMetrics provides the following handlers: For example, request to `/api/v1/status/top_queries?topN=5&maxLifetime=30s` would return up to 5 queries per list, which were executed during the last 30 seconds. VictoriaMetrics tracks the last `-search.queryStats.lastQueriesCount` queries with durations at least `-search.queryStats.minQueryDuration`. +### Timestamp formats + +VictoriaMetrics accepts the following formats for `time`, `start` and `end` query args +in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and +in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). + +- Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. +- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. + + ## Graphite API usage VictoriaMetrics supports data ingestion in Graphite protocol - see [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details. @@ -963,8 +974,9 @@ Each JSON line contains samples for a single time series. An example output: {"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]} ``` -Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either -unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/api/v1/export -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' @@ -1011,8 +1023,9 @@ where: * `` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for metrics to export. -Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either -unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/api/v1/export/csv -d 'format=' -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' @@ -1038,8 +1051,9 @@ wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq # relaunch victoriametrics with search.maxExportSeries more than value from previous command ``` -Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either -unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/api/v1/export/native -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' @@ -1275,7 +1289,8 @@ VictoriaMetrics exports [Prometheus-compatible federation data](https://promethe at `http://:8428/federate?match[]=`. Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval. -`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values. +See [allowed formats](#timestamp-formats) for these args. + For example: ```console curl http://:8428/federate -d 'match[]=' -d 'start=1654543486' -d 'end=1654543486' diff --git a/docs/keyConcepts.md b/docs/keyConcepts.md index 117d62693..3673ef679 100644 --- a/docs/keyConcepts.md +++ b/docs/keyConcepts.md @@ -453,11 +453,7 @@ Params: * `query` - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. * `time` - optional timestamp when to evaluate the `query`. If `time` is skipped, then the current timestamp is used. - The `time` param can be specified in the following formats: - * [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) such as `2022-08-10T12:45:43.000Z`. - * [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) in seconds. It can contain a fractional part for millisecond precision. - * [Relative duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) - compared to the current timestamp. For example, `-1h` means `one hour before the current time`. + The `time` param can be specified in [multiple allowed formats](https://docs.victoriametrics.com/#timestamp-formats). * `step` - optional max lookback window for searching for raw samples when executing the `query`. If `step` is skipped, then it is set to `5m` (5 minutes) by default. @@ -548,11 +544,6 @@ GET | POST /api/v1/query_range?query=...&start=...&end=...&step=... Params: * `query` - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression. * `start` - the starting timestamp of the time range for `query` evaluation. - The `start` param can be specified in the following formats: - * [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) such as `2022-08-10T12:45:43.000Z`. - * [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) in seconds. It can contain a fractional part for millisecond precision. - * [Relative duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) - compared to the current timestamp. For example, `-1h` means `one hour before the current time`. * `end` - the ending timestamp of the time range for `query` evaluation. If the `end` isn't set, then the `end` is automatically set to the current time. * `step` - the [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) between datapoints, @@ -560,6 +551,8 @@ Params: The `query` is executed at `start`, `start+step`, `start+2*step`, ..., `end` timestamps. If the `step` isn't set, then it is automatically set to `5m` (5 minutes). +The `start` and `end` params can be specified in [multiple allowed formats](https://docs.victoriametrics.com/#timestamp-formats). + To get the values of `foo_bar` on the time range from `2022-05-10 09:59:00` to `2022-05-10 10:17:00`, in VictoriaMetrics we need to issue a range query: