app/vmselect/searchutils: accept partial RFC3339 values at time, start and end query args

This simplifies manual usage of the APIs. For example, the following query
would return the results over the 2022 year.

  /api/v1/query_range?start=2022&end=2023&step=1d&query=...

This is equivalent to:

  /api/v1/query_range?start=2022-01-01T00:00:00Z&end=2023-01-01T00:00:00Z&step=1d&query=...
This commit is contained in:
Aliaksandr Valialkin 2022-12-28 19:41:51 -08:00
parent 6f21435d2d
commit f27bb19213
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
6 changed files with 220 additions and 63 deletions

View file

@ -50,11 +50,6 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
if len(argValue) == 0 {
return roundToSeconds(defaultMs), nil
}
secs, err := strconv.ParseFloat(argValue, 64)
if err != nil {
// Try parsing string format
t, err := time.Parse(time.RFC3339, argValue)
if err != nil {
// Handle Prometheus'-provided minTime and maxTime.
// See https://github.com/prometheus/client_golang/issues/614
switch argValue {
@ -63,17 +58,10 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
case prometheusMaxTimeFormatted:
return maxTimeMsecs, nil
}
// Try parsing duration relative to the current time
d, err1 := promutils.ParseDuration(argValue)
if err1 != nil {
return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
}
if d > 0 {
d = -d
}
t = time.Now().Add(d)
}
secs = float64(t.UnixNano()) / 1e9
// Parse argValue
secs, err := parseTime(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err)
}
msecs := int64(secs * 1e3)
if msecs < minTimeMsecs {
@ -85,6 +73,78 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
return msecs, nil
}
func parseTime(s string) (float64, error) {
if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') {
// Parse duration relative to the current time
d, err := promutils.ParseDuration(s)
if err != nil {
return 0, err
}
if d > 0 {
d = -d
}
t := time.Now().Add(d)
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 4 {
// Parse YYYY
t, err := time.Parse("2006", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if !strings.Contains(s, "-") {
// Parse the timestamp in milliseconds
return strconv.ParseFloat(s, 64)
}
if len(s) == 7 {
// Parse YYYY-MM
t, err := time.Parse("2006-01", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 10 {
// Parse YYYY-MM-DD
t, err := time.Parse("2006-01-02", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 13 {
// Parse YYYY-MM-DDTHH
t, err := time.Parse("2006-01-02T15", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 16 {
// Parse YYYY-MM-DDTHH:MM
t, err := time.Parse("2006-01-02T15:04", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 19 {
// Parse YYYY-MM-DDTHH:MM:SS
t, err := time.Parse("2006-01-02T15:04:05", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
var (
// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
// See https://github.com/prometheus/client_golang/issues/614 for details.

View file

@ -11,6 +11,69 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
)
func TestGetDurationSuccess(t *testing.T) {
f := func(s string, dExpected int64) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest("GET", urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
// Verify defaultValue
d, err := GetDuration(r, "foo", 123456)
if err != nil {
t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err)
}
if d != 123456 {
t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456)
}
// Verify dExpected
d, err = GetDuration(r, "s", 123)
if err != nil {
t.Fatalf("unexpected error in GetDuration(%q): %s", s, err)
}
if d != dExpected {
t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected)
}
}
f("1.234", 1234)
f("1.23ms", 1)
f("1.23s", 1230)
f("2s56ms", 2056)
f("2s-5ms", 1995)
f("5m3.5s", 303500)
f("2h", 7200000)
f("1d", 24*3600*1000)
f("7d5h4m3s534ms", 623043534)
}
func TestGetDurationError(t *testing.T) {
f := func(s string) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest("GET", urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
if _, err := GetDuration(r, "s", 123); err == nil {
t.Fatalf("expecting non-nil error in GetDuration(%q)", s)
}
}
// Negative durations aren't supported
f("-1.234")
// Invalid duration
f("foo")
// Invalid suffix
f("1md")
}
func TestGetTimeSuccess(t *testing.T) {
f := func(s string, timestampExpected int64) {
t.Helper()
@ -39,6 +102,17 @@ func TestGetTimeSuccess(t *testing.T) {
}
}
f("2019", 1546300800000)
f("2019-01", 1546300800000)
f("2019-02", 1548979200000)
f("2019-02-01", 1548979200000)
f("2019-02-02", 1549065600000)
f("2019-02-02T00", 1549065600000)
f("2019-02-02T01", 1549069200000)
f("2019-02-02T01:00", 1549069200000)
f("2019-02-02T01:01", 1549069260000)
f("2019-02-02T01:01:00", 1549069260000)
f("2019-02-02T01:01:01", 1549069261000)
f("2019-07-07T20:01:02Z", 1562529662000)
f("2019-07-07T20:47:40+03:00", 1562521660000)
f("-292273086-05-16T16:47:06Z", minTimeMsecs)
@ -58,27 +132,26 @@ func TestGetTimeError(t *testing.T) {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
// Verify defaultValue
ts, err := GetTime(r, "foo", 123456)
if err != nil {
t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
}
if ts != 123000 {
t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
}
// Verify timestampExpected
_, err = GetTime(r, "s", 123)
if err == nil {
if _, err := GetTime(r, "s", 123); err == nil {
t.Fatalf("expecting non-nil error in GetTime(%q)", s)
}
}
f("foo")
f("foo1")
f("1245-5")
f("2022-x7")
f("2022-02-x7")
f("2022-02-02Tx7")
f("2022-02-02T00:x7")
f("2022-02-02T00:00:x7")
f("2022-02-02T00:00:00a")
f("2019-07-07T20:01:02Zisdf")
f("2019-07-07T20:47:40+03:00123")
f("-292273086-05-16T16:47:07Z")
f("292277025-08-18T07:12:54.999999998Z")
f("123md")
f("-12.3md")
}
func TestGetExtraTagFilters(t *testing.T) {

View file

@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386).
* FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats).
* FEATURE: [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling): add support for `keepequal` and `dropequal` relabeling actions, which are supported by Prometheus starting from [v2.41.0](https://github.com/prometheus/prometheus/releases/tag/v2.41.0). These relabeling actions are almost identical to `keep_if_equal` and `drop_if_equal` relabeling actions supported by VictoriaMetrics since `v1.38.0` - see [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) - so it is recommended sticking to `keep_if_equal` and `drop_if_equal` actions instead of switching to `keepequal` and `dropequal`.
* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513).

View file

@ -715,8 +715,7 @@ VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query
VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
`/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt).
For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats).
VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
@ -741,6 +740,18 @@ Additionally, VictoriaMetrics provides the following handlers:
For example, request to `/api/v1/status/top_queries?topN=5&maxLifetime=30s` would return up to 5 queries per list, which were executed during the last 30 seconds.
VictoriaMetrics tracks the last `-search.queryStats.lastQueriesCount` queries with durations at least `-search.queryStats.minQueryDuration`.
### Timestamp formats
VictoriaMetrics accepts the following formats for `time`, `start` and `end` query args
in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and
in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series).
- Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`.
- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`.
- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`.
- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`.
## Graphite API usage
VictoriaMetrics supports data ingestion in Graphite protocol - see [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
@ -960,8 +971,9 @@ Each JSON line contains samples for a single time series. An example output:
{"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
```
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@ -1008,8 +1020,9 @@ where:
* `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
for metrics to export.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@ -1035,8 +1048,9 @@ wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq
# relaunch victoriametrics with search.maxExportSeries more than value from previous command
```
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@ -1272,7 +1286,8 @@ VictoriaMetrics exports [Prometheus-compatible federation data](https://promethe
at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'

View file

@ -718,8 +718,7 @@ VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query
VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
`/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt).
For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats).
VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
@ -744,6 +743,18 @@ Additionally, VictoriaMetrics provides the following handlers:
For example, request to `/api/v1/status/top_queries?topN=5&maxLifetime=30s` would return up to 5 queries per list, which were executed during the last 30 seconds.
VictoriaMetrics tracks the last `-search.queryStats.lastQueriesCount` queries with durations at least `-search.queryStats.minQueryDuration`.
### Timestamp formats
VictoriaMetrics accepts the following formats for `time`, `start` and `end` query args
in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and
in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series).
- Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`.
- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`.
- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`.
- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`.
## Graphite API usage
VictoriaMetrics supports data ingestion in Graphite protocol - see [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
@ -963,8 +974,9 @@ Each JSON line contains samples for a single time series. An example output:
{"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
```
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@ -1011,8 +1023,9 @@ where:
* `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
for metrics to export.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@ -1038,8 +1051,9 @@ wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq
# relaunch victoriametrics with search.maxExportSeries more than value from previous command
```
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@ -1275,7 +1289,8 @@ VictoriaMetrics exports [Prometheus-compatible federation data](https://promethe
at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
See [allowed formats](#timestamp-formats) for these args.
For example:
```console
curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'

View file

@ -453,11 +453,7 @@ Params:
* `query` - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression.
* `time` - optional timestamp when to evaluate the `query`. If `time` is skipped, then the current timestamp is used.
The `time` param can be specified in the following formats:
* [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) such as `2022-08-10T12:45:43.000Z`.
* [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) in seconds. It can contain a fractional part for millisecond precision.
* [Relative duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations)
compared to the current timestamp. For example, `-1h` means `one hour before the current time`.
The `time` param can be specified in [multiple allowed formats](https://docs.victoriametrics.com/#timestamp-formats).
* `step` - optional max lookback window for searching for raw samples when executing the `query`.
If `step` is skipped, then it is set to `5m` (5 minutes) by default.
@ -548,11 +544,6 @@ GET | POST /api/v1/query_range?query=...&start=...&end=...&step=...
Params:
* `query` - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html) expression.
* `start` - the starting timestamp of the time range for `query` evaluation.
The `start` param can be specified in the following formats:
* [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) such as `2022-08-10T12:45:43.000Z`.
* [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) in seconds. It can contain a fractional part for millisecond precision.
* [Relative duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations)
compared to the current timestamp. For example, `-1h` means `one hour before the current time`.
* `end` - the ending timestamp of the time range for `query` evaluation.
If the `end` isn't set, then the `end` is automatically set to the current time.
* `step` - the [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) between datapoints,
@ -560,6 +551,8 @@ Params:
The `query` is executed at `start`, `start+step`, `start+2*step`, ..., `end` timestamps.
If the `step` isn't set, then it is automatically set to `5m` (5 minutes).
The `start` and `end` params can be specified in [multiple allowed formats](https://docs.victoriametrics.com/#timestamp-formats).
To get the values of `foo_bar` on the time range from `2022-05-10 09:59:00` to `2022-05-10 10:17:00`, in VictoriaMetrics we
need to issue a range query: