diff --git a/app/vmselect/clusternative/vmselect.go b/app/vmselect/clusternative/vmselect.go index c79126a60..29791a58d 100644 --- a/app/vmselect/clusternative/vmselect.go +++ b/app/vmselect/clusternative/vmselect.go @@ -8,6 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi" @@ -47,7 +48,7 @@ func NewVMSelectServer(addr string) (*vmselectapi.Server, error) { type vmstorageAPI struct{} func (api *vmstorageAPI) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) bi := newBlockIterator(qt, denyPartialResponse, sq, dl) return bi, nil @@ -59,14 +60,14 @@ func (api *vmstorageAPI) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, d } func (api *vmstorageAPI) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) metricNames, _, err := netstorage.SearchMetricNames(qt, denyPartialResponse, sq, dl) return metricNames, err } func (api *vmstorageAPI) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) labelValues, _, err := netstorage.LabelValues(qt, denyPartialResponse, labelName, sq, maxLabelValues, dl) return labelValues, err @@ -74,28 +75,28 @@ func (api *vmstorageAPI) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQ func (api *vmstorageAPI) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) suffixes, _, err := netstorage.TagValueSuffixes(qt, accountID, projectID, denyPartialResponse, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, dl) return suffixes, err } func (api *vmstorageAPI) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) labelNames, _, err := netstorage.LabelNames(qt, denyPartialResponse, sq, maxLabelNames, dl) return labelNames, err } func (api *vmstorageAPI) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) seriesCount, _, err := netstorage.SeriesCount(qt, accountID, projectID, denyPartialResponse, dl) return seriesCount, err } func (api *vmstorageAPI) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) { - denyPartialResponse := searchutils.GetDenyPartialResponse(nil) + denyPartialResponse := httputils.GetDenyPartialResponse(nil) dl := searchutils.DeadlineFromTimestamp(deadline) tsdbStatus, _, err := netstorage.TSDBStatus(qt, denyPartialResponse, sq, focusLabel, topN, dl) return tsdbStatus, err diff --git a/app/vmselect/graphite/functions_api.go b/app/vmselect/graphite/functions_api.go index a8140b3e1..2a39933d6 100644 --- a/app/vmselect/graphite/functions_api.go +++ b/app/vmselect/graphite/functions_api.go @@ -8,14 +8,14 @@ import ( "net/http" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" ) // FunctionsHandler implements /functions handler. // // See https://graphite.readthedocs.io/en/latest/functions.html#function-api func FunctionsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { - grouped := searchutils.GetBool(r, "grouped") + grouped := httputils.GetBool(r, "grouped") group := r.FormValue("group") result := make(map[string]interface{}) for funcName, fi := range funcs { diff --git a/app/vmselect/graphite/metrics_api.go b/app/vmselect/graphite/metrics_api.go index 6034b872d..7209407d6 100644 --- a/app/vmselect/graphite/metrics_api.go +++ b/app/vmselect/graphite/metrics_api.go @@ -10,10 +10,11 @@ import ( "sync" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" @@ -47,7 +48,7 @@ func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit if len(delimiter) > 1 { return fmt.Errorf("`delimiter` query arg must contain only a single char") } - if searchutils.GetBool(r, "automatic_variants") { + if httputils.GetBool(r, "automatic_variants") { // See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L152 query = addAutomaticVariants(query, delimiter) } @@ -58,19 +59,19 @@ func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit query += "*" } } - leavesOnly := searchutils.GetBool(r, "leavesOnly") - wildcards := searchutils.GetBool(r, "wildcards") + leavesOnly := httputils.GetBool(r, "leavesOnly") + wildcards := httputils.GetBool(r, "wildcards") label := r.FormValue("label") if label == "__name__" { label = "" } jsonp := r.FormValue("jsonp") - from, err := searchutils.GetTime(r, "from", 0) + from, err := httputils.GetTime(r, "from", 0) if err != nil { return err } ct := startTime.UnixNano() / 1e6 - until, err := searchutils.GetTime(r, "until", ct) + until, err := httputils.GetTime(r, "until", ct) if err != nil { return err } @@ -78,7 +79,7 @@ func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit MinTimestamp: from, MaxTimestamp: until, } - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, "", query, delimiter[0], false, deadline) if err != nil { return err @@ -126,8 +127,8 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr if len(queries) == 0 { return fmt.Errorf("missing `query` arg") } - groupByExpr := searchutils.GetBool(r, "groupByExpr") - leavesOnly := searchutils.GetBool(r, "leavesOnly") + groupByExpr := httputils.GetBool(r, "groupByExpr") + leavesOnly := httputils.GetBool(r, "leavesOnly") label := r.FormValue("label") if label == "__name__" { label = "" @@ -140,12 +141,12 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr return fmt.Errorf("`delimiter` query arg must contain only a single char") } jsonp := r.FormValue("jsonp") - from, err := searchutils.GetTime(r, "from", 0) + from, err := httputils.GetTime(r, "from", 0) if err != nil { return err } ct := startTime.UnixNano() / 1e6 - until, err := searchutils.GetTime(r, "until", ct) + until, err := httputils.GetTime(r, "until", ct) if err != nil { return err } @@ -155,7 +156,7 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr } m := make(map[string][]string, len(queries)) isPartialResponse := false - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) for _, query := range queries { paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, "", query, delimiter[0], true, deadline) if err != nil { @@ -208,7 +209,7 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr func MetricsIndexHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) jsonp := r.FormValue("jsonp") - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, 0, 0, nil, 0) metricNames, isPartial, err := netstorage.LabelValues(nil, denyPartialResponse, "__name__", sq, 0, deadline) if err != nil { diff --git a/app/vmselect/graphite/render_api.go b/app/vmselect/graphite/render_api.go index 1b2d9e891..ab000ff22 100644 --- a/app/vmselect/graphite/render_api.go +++ b/app/vmselect/graphite/render_api.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" "github.com/VictoriaMetrics/metrics" ) diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index 0f79fe4ce..f26a63b51 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -9,10 +9,11 @@ import ( "strings" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" @@ -165,7 +166,7 @@ var ( // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -180,7 +181,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.R valuePrefix := r.FormValue("valuePrefix") exprs := r.Form["expr"] var tagValues []string - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) etfs, err := searchutils.GetExtraTagFilters(r) if err != nil { return fmt.Errorf("cannot setup tag filters: %w", err) @@ -258,7 +259,7 @@ var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_sec // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -268,7 +269,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res } tagPrefix := r.FormValue("tagPrefix") exprs := r.Form["expr"] - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) etfs, err := searchutils.GetExtraTagFilters(r) if err != nil { return fmt.Errorf("cannot setup tag filters: %w", err) @@ -344,7 +345,7 @@ var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_secon // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags func TagsFindSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -360,7 +361,7 @@ func TagsFindSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseW if err != nil { return err } - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) metricNames, isPartial, err := netstorage.SearchMetricNames(nil, denyPartialResponse, sq, deadline) if err != nil { return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err) @@ -420,12 +421,12 @@ var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{pat // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags func TagValuesHandler(startTime time.Time, at *auth.Token, tagName string, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } filter := r.FormValue("filter") - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) tagValues, isPartial, err := netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tagName, filter, *maxGraphiteTagValuesPerSearch, deadline) if err != nil { return err @@ -452,12 +453,12 @@ var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/t // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags func TagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } filter := r.FormValue("filter") - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) labels, isPartial, err := netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch, deadline) if err != nil { return err diff --git a/app/vmselect/main.go b/app/vmselect/main.go index bf551d9bc..0efc4550d 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -25,6 +25,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" @@ -191,7 +192,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { startTime := time.Now() defer requestDuration.UpdateDuration(startTime) - tracerEnabled := searchutils.GetBool(r, "trace") + tracerEnabled := httputils.GetBool(r, "trace") qt := querytracer.New(tracerEnabled, r.URL.Path) // Limit the number of concurrent queries. @@ -216,7 +217,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { remoteAddr := httpserver.GetQuotedRemoteAddr(r) requestURI := httpserver.GetRequestURI(r) logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q", - d.Seconds(), remoteAddr, requestURI) + time.Since(startTime).Seconds(), remoteAddr, requestURI) return true case <-t.C: timerpool.Put(t) diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index e98591659..f510517f9 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -13,17 +13,18 @@ import ( "sync/atomic" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/auth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" @@ -97,7 +98,7 @@ func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, cp.start = cp.end - lookbackDelta } sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, *maxFederateSeries) - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) rss, isPartial, err := netstorage.ProcessSearchQuery(nil, denyPartialResponse, sq, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch data for %q: %w", sq, err) @@ -140,7 +141,7 @@ func ExportCSVHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter return fmt.Errorf("missing `format` arg; see https://docs.victoriametrics.com/#how-to-export-csv-data") } fieldNames := strings.Split(format, ",") - reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage") + reduceMemUsage := httputils.GetBool(r, "reduce_mem_usage") sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, *maxExportSeries) w.Header().Set("Content-Type", "text/csv; charset=utf-8") @@ -280,7 +281,7 @@ func ExportHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r } format := r.FormValue("format") maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line"))) - reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage") + reduceMemUsage := httputils.GetBool(r, "reduce_mem_usage") if err := exportHandler(nil, at, w, cp, format, maxRowsPerLine, reduceMemUsage); err != nil { return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", cp.start, cp.end, err) } @@ -525,12 +526,12 @@ var httpClient = &http.Client{ // Tenants processes /admin/tenants request. func Tenants(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForStatusRequest(r, startTime) - start, err := searchutils.GetTime(r, "start", 0) + start, err := httputils.GetTime(r, "start", 0) if err != nil { return err } ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) + end, err := httputils.GetTime(r, "end", ct) if err != nil { return err } @@ -562,11 +563,11 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.To if err != nil { return err } - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, *maxUniqueTimeseries) labelValues, isPartial, err := netstorage.LabelValues(qt, denyPartialResponse, labelName, sq, limit, cp.deadline) if err != nil { @@ -630,7 +631,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok } topN = n } - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) start := int64(date*secsPerDay) * 1000 end := int64((date+1)*secsPerDay)*1000 - 1 sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, start, end, cp.filterss, *maxTSDBStatusSeries) @@ -661,11 +662,11 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, if err != nil { return err } - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, *maxUniqueTimeseries) labels, isPartial, err := netstorage.LabelNames(qt, denyPartialResponse, sq, limit, cp.deadline) if err != nil { @@ -689,7 +690,7 @@ func SeriesCountHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit defer seriesCountDuration.UpdateDuration(startTime) deadline := searchutils.GetDeadlineForStatusRequest(r, startTime) - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) n, isPartial, err := netstorage.SeriesCount(nil, at.AccountID, at.ProjectID, denyPartialResponse, deadline) if err != nil { return fmt.Errorf("cannot obtain series count: %w", err) @@ -722,13 +723,13 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, if err != nil { return err } - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, *maxSeriesLimit) - denyPartialResponse := searchutils.GetDenyPartialResponse(r) + denyPartialResponse := httputils.GetDenyPartialResponse(r) metricNames, isPartial, err := netstorage.SearchMetricNames(qt, denyPartialResponse, sq, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch time series for %q: %w", sq, err) @@ -759,12 +760,12 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w ct := startTime.UnixNano() / 1e6 deadline := searchutils.GetDeadlineForQuery(r, startTime) - mayCache := !searchutils.GetBool(r, "nocache") + mayCache := !httputils.GetBool(r, "nocache") query := r.FormValue("query") if len(query) == 0 { return fmt.Errorf("missing `query` arg") } - start, err := searchutils.GetTime(r, "time", ct) + start, err := httputils.GetTime(r, "time", ct) if err != nil { return err } @@ -772,7 +773,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w if err != nil { return err } - step, err := searchutils.GetDuration(r, "step", lookbackDelta) + step, err := httputils.GetDuration(r, "step", lookbackDelta) if err != nil { return err } @@ -836,7 +837,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w if err != nil { return err } - if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset { + if !httputils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset { // Adjust start time only if `nocache` arg isn't set. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241 startPrev := start @@ -863,7 +864,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w return httpserver.GetRequestURI(r) }, - DenyPartialResponse: searchutils.GetDenyPartialResponse(r), + DenyPartialResponse: httputils.GetDenyPartialResponse(r), QueryStats: qs, } result, err := promql.Exec(qt, ec, query, true) @@ -910,15 +911,15 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok if len(query) == 0 { return fmt.Errorf("missing `query` arg") } - start, err := searchutils.GetTime(r, "start", ct-defaultStep) + start, err := httputils.GetTime(r, "start", ct-defaultStep) if err != nil { return err } - end, err := searchutils.GetTime(r, "end", ct) + end, err := httputils.GetTime(r, "end", ct) if err != nil { return err } - step, err := searchutils.GetDuration(r, "step", defaultStep) + step, err := httputils.GetDuration(r, "step", defaultStep) if err != nil { return err } @@ -935,7 +936,7 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - mayCache := !searchutils.GetBool(r, "nocache") + mayCache := !httputils.GetBool(r, "nocache") lookbackDelta, err := getMaxLookback(r) if err != nil { return err @@ -973,7 +974,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok return httpserver.GetRequestURI(r) }, - DenyPartialResponse: searchutils.GetDenyPartialResponse(r), + DenyPartialResponse: httputils.GetDenyPartialResponse(r), QueryStats: qs, } result, err := promql.Exec(qt, ec, query, false) @@ -1087,13 +1088,13 @@ func getMaxLookback(r *http.Request) (int64, error) { if d == 0 { d = maxStalenessInterval.Milliseconds() } - maxLookback, err := searchutils.GetDuration(r, "max_lookback", d) + maxLookback, err := httputils.GetDuration(r, "max_lookback", d) if err != nil { return 0, err } d = maxLookback if *setLookbackToStep { - step, err := searchutils.GetDuration(r, "step", d) + step, err := httputils.GetDuration(r, "step", d) if err != nil { return 0, err } @@ -1133,7 +1134,7 @@ func getLatencyOffsetMilliseconds(r *http.Request) (int64, error) { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836 d = 0 } - return searchutils.GetDuration(r, "latency_offset", d) + return httputils.GetDuration(r, "latency_offset", d) } // QueryStatsHandler returns query stats at `/api/v1/status/top_queries` @@ -1149,7 +1150,7 @@ func QueryStatsHandler(startTime time.Time, at *auth.Token, w http.ResponseWrite } topN = n } - maxLifetimeMsecs, err := searchutils.GetDuration(r, "maxLifetime", 10*60*1000) + maxLifetimeMsecs, err := httputils.GetDuration(r, "maxLifetime", 10*60*1000) if err != nil { return fmt.Errorf("cannot parse `maxLifetime` arg: %w", err) } @@ -1223,12 +1224,12 @@ func getCommonParamsWithDefaultDuration(r *http.Request, startTime time.Time, re // - extra_filters[] func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) { deadline := searchutils.GetDeadlineForQuery(r, startTime) - start, err := searchutils.GetTime(r, "start", 0) + start, err := httputils.GetTime(r, "start", 0) if err != nil { return nil, err } ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) + end, err := httputils.GetTime(r, "end", ct) if err != nil { return nil, err } diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index eca5f3df7..bbf5c6d1c 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -3,14 +3,12 @@ package searchutils import ( "flag" "fmt" - "math" "net/http" - "strconv" "strings" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metricsql" ) @@ -19,100 +17,11 @@ var ( maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call") maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution") maxStatusRequestDuration = flag.Duration("search.maxStatusRequestDuration", time.Minute*5, "The maximum duration for /api/v1/status/* requests") - denyPartialResponse = flag.Bool("search.denyPartialResponse", false, "Whether to deny partial responses if a part of -storageNode instances fail to perform queries; "+ - "this trades availability over consistency; see also -search.maxQueryDuration") ) -func roundToSeconds(ms int64) int64 { - return ms - ms%1000 -} - -// GetInt returns integer value from the given argKey. -func GetInt(r *http.Request, argKey string) (int, error) { - argValue := r.FormValue(argKey) - if len(argValue) == 0 { - return 0, nil - } - n, err := strconv.Atoi(argValue) - if err != nil { - return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err) - } - return n, nil -} - -// GetTime returns time from the given argKey query arg. -// -// If argKey is missing in r, then defaultMs rounded to seconds is returned. -// The rounding is needed in order to align query results in Grafana -// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 -func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { - argValue := r.FormValue(argKey) - if len(argValue) == 0 { - return roundToSeconds(defaultMs), nil - } - // Handle Prometheus'-provided minTime and maxTime. - // See https://github.com/prometheus/client_golang/issues/614 - switch argValue { - case prometheusMinTimeFormatted: - return minTimeMsecs, nil - case prometheusMaxTimeFormatted: - return maxTimeMsecs, nil - } - // Parse argValue - secs, err := promutils.ParseTime(argValue) - if err != nil { - return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err) - } - msecs := int64(secs * 1e3) - if msecs < minTimeMsecs { - msecs = 0 - } - if msecs > maxTimeMsecs { - msecs = maxTimeMsecs - } - return msecs, nil -} - -var ( - // These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442 - // See https://github.com/prometheus/client_golang/issues/614 for details. - prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano) - prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano) -) - -const ( - // These values prevent from overflow when storing msec-precision time in int64. - minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time - maxTimeMsecs = int64(1<<63-1) / 1e6 -) - -// GetDuration returns duration from the given argKey query arg. -func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) { - argValue := r.FormValue(argKey) - if len(argValue) == 0 { - return defaultValue, nil - } - secs, err := strconv.ParseFloat(argValue, 64) - if err != nil { - // Try parsing string format - d, err := promutils.ParseDuration(argValue) - if err != nil { - return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) - } - secs = d.Seconds() - } - msecs := int64(secs * 1e3) - if msecs <= 0 || msecs > maxDurationMsecs { - return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs)) - } - return msecs, nil -} - -const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000 - // GetMaxQueryDuration returns the maximum duration for query from r. func GetMaxQueryDuration(r *http.Request) time.Duration { - dms, err := GetDuration(r, "timeout", 0) + dms, err := httputils.GetDuration(r, "timeout", 0) if err != nil { dms = 0 } @@ -142,7 +51,7 @@ func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline { } func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline { - d, err := GetDuration(r, "timeout", 0) + d, err := httputils.GetDuration(r, "timeout", 0) if err != nil { d = 0 } @@ -153,28 +62,6 @@ func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64 return NewDeadline(startTime, timeout, flagHint) } -// GetBool returns boolean value from the given argKey query arg. -func GetBool(r *http.Request, argKey string) bool { - argValue := r.FormValue(argKey) - switch strings.ToLower(argValue) { - case "", "0", "f", "false", "no": - return false - default: - return true - } -} - -// GetDenyPartialResponse returns whether partial responses are denied. -func GetDenyPartialResponse(r *http.Request) bool { - if *denyPartialResponse { - return true - } - if r == nil { - return false - } - return GetBool(r, "deny_partial_response") -} - // Deadline contains deadline with the corresponding timeout for pretty error messages. type Deadline struct { deadline uint64 diff --git a/app/vmselect/searchutils/searchutils_test.go b/app/vmselect/searchutils/searchutils_test.go index 73544392f..b763b8b76 100644 --- a/app/vmselect/searchutils/searchutils_test.go +++ b/app/vmselect/searchutils/searchutils_test.go @@ -11,149 +11,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) -func TestGetDurationSuccess(t *testing.T) { - f := func(s string, dExpected int64) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - // Verify defaultValue - d, err := GetDuration(r, "foo", 123456) - if err != nil { - t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err) - } - if d != 123456 { - t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456) - } - - // Verify dExpected - d, err = GetDuration(r, "s", 123) - if err != nil { - t.Fatalf("unexpected error in GetDuration(%q): %s", s, err) - } - if d != dExpected { - t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected) - } - } - - f("1.234", 1234) - f("1.23ms", 1) - f("1.23s", 1230) - f("2s56ms", 2056) - f("2s-5ms", 1995) - f("5m3.5s", 303500) - f("2h", 7200000) - f("1d", 24*3600*1000) - f("7d5h4m3s534ms", 623043534) -} - -func TestGetDurationError(t *testing.T) { - f := func(s string) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - if _, err := GetDuration(r, "s", 123); err == nil { - t.Fatalf("expecting non-nil error in GetDuration(%q)", s) - } - } - - // Negative durations aren't supported - f("-1.234") - - // Invalid duration - f("foo") - - // Invalid suffix - f("1md") -} - -func TestGetTimeSuccess(t *testing.T) { - f := func(s string, timestampExpected int64) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - // Verify defaultValue - ts, err := GetTime(r, "foo", 123456) - if err != nil { - t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err) - } - if ts != 123000 { - t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000) - } - - // Verify timestampExpected - ts, err = GetTime(r, "s", 123) - if err != nil { - t.Fatalf("unexpected error in GetTime(%q): %s", s, err) - } - if ts != timestampExpected { - t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected) - } - } - - f("2019", 1546300800000) - f("2019-01", 1546300800000) - f("2019-02", 1548979200000) - f("2019-02-01", 1548979200000) - f("2019-02-02", 1549065600000) - f("2019-02-02T00", 1549065600000) - f("2019-02-02T01", 1549069200000) - f("2019-02-02T01:00", 1549069200000) - f("2019-02-02T01:01", 1549069260000) - f("2019-02-02T01:01:00", 1549069260000) - f("2019-02-02T01:01:01", 1549069261000) - f("2019-07-07T20:01:02Z", 1562529662000) - f("2019-07-07T20:47:40+03:00", 1562521660000) - f("-292273086-05-16T16:47:06Z", minTimeMsecs) - f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs) - f("1562529662.324", 1562529662324) - f("-9223372036.854", minTimeMsecs) - f("-9223372036.855", minTimeMsecs) - f("9223372036.855", maxTimeMsecs) -} - -func TestGetTimeError(t *testing.T) { - f := func(s string) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - if _, err := GetTime(r, "s", 123); err == nil { - t.Fatalf("expecting non-nil error in GetTime(%q)", s) - } - } - - f("foo") - f("foo1") - f("1245-5") - f("2022-x7") - f("2022-02-x7") - f("2022-02-02Tx7") - f("2022-02-02T00:x7") - f("2022-02-02T00:00:x7") - f("2022-02-02T00:00:00a") - f("2019-07-07T20:01:02Zisdf") - f("2019-07-07T20:47:40+03:00123") - f("-292273086-05-16T16:47:07Z") - f("292277025-08-18T07:12:54.999999998Z") - f("123md") - f("-12.3md") -} - func TestGetExtraTagFilters(t *testing.T) { httpReqWithForm := func(qs string) *http.Request { q, err := url.ParseQuery(qs) diff --git a/app/vmselect/bufferedwriter/bufferedwriter.go b/lib/bufferedwriter/bufferedwriter.go similarity index 100% rename from app/vmselect/bufferedwriter/bufferedwriter.go rename to lib/bufferedwriter/bufferedwriter.go diff --git a/lib/httputils/bool.go b/lib/httputils/bool.go new file mode 100644 index 000000000..321ceb80a --- /dev/null +++ b/lib/httputils/bool.go @@ -0,0 +1,34 @@ +package httputils + +import ( + "flag" + "net/http" + "strings" +) + +var ( + denyPartialResponse = flag.Bool("search.denyPartialResponse", false, "Whether to deny partial responses if a part of -storageNode instances fail to perform queries; "+ + "this trades availability over consistency; see also -search.maxQueryDuration") +) + +// GetBool returns boolean value from the given argKey query arg. +func GetBool(r *http.Request, argKey string) bool { + argValue := r.FormValue(argKey) + switch strings.ToLower(argValue) { + case "", "0", "f", "false", "no": + return false + default: + return true + } +} + +// GetDenyPartialResponse returns whether partial responses are denied. +func GetDenyPartialResponse(r *http.Request) bool { + if *denyPartialResponse { + return true + } + if r == nil { + return false + } + return GetBool(r, "deny_partial_response") +} diff --git a/lib/httputils/duration.go b/lib/httputils/duration.go new file mode 100644 index 000000000..89d31feb5 --- /dev/null +++ b/lib/httputils/duration.go @@ -0,0 +1,33 @@ +package httputils + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +// GetDuration returns duration in milliseconds from the given argKey query arg. +func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) { + argValue := r.FormValue(argKey) + if len(argValue) == 0 { + return defaultValue, nil + } + secs, err := strconv.ParseFloat(argValue, 64) + if err != nil { + // Try parsing string format + d, err := promutils.ParseDuration(argValue) + if err != nil { + return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) + } + secs = d.Seconds() + } + msecs := int64(secs * 1e3) + if msecs <= 0 || msecs > maxDurationMsecs { + return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs)) + } + return msecs, nil +} + +const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000 diff --git a/lib/httputils/duration_test.go b/lib/httputils/duration_test.go new file mode 100644 index 000000000..7cb3dc631 --- /dev/null +++ b/lib/httputils/duration_test.go @@ -0,0 +1,71 @@ +package httputils + +import ( + "fmt" + "net/http" + "net/url" + "testing" +) + +func TestGetDurationSuccess(t *testing.T) { + f := func(s string, dExpected int64) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + // Verify defaultValue + d, err := GetDuration(r, "foo", 123456) + if err != nil { + t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err) + } + if d != 123456 { + t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456) + } + + // Verify dExpected + d, err = GetDuration(r, "s", 123) + if err != nil { + t.Fatalf("unexpected error in GetDuration(%q): %s", s, err) + } + if d != dExpected { + t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected) + } + } + + f("1.234", 1234) + f("1.23ms", 1) + f("1.23s", 1230) + f("2s56ms", 2056) + f("2s-5ms", 1995) + f("5m3.5s", 303500) + f("2h", 7200000) + f("1d", 24*3600*1000) + f("7d5h4m3s534ms", 623043534) +} + +func TestGetDurationError(t *testing.T) { + f := func(s string) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + if _, err := GetDuration(r, "s", 123); err == nil { + t.Fatalf("expecting non-nil error in GetDuration(%q)", s) + } + } + + // Negative durations aren't supported + f("-1.234") + + // Invalid duration + f("foo") + + // Invalid suffix + f("1md") +} diff --git a/lib/httputils/int.go b/lib/httputils/int.go new file mode 100644 index 000000000..2547cbf16 --- /dev/null +++ b/lib/httputils/int.go @@ -0,0 +1,20 @@ +package httputils + +import ( + "fmt" + "net/http" + "strconv" +) + +// GetInt returns integer value from the given argKey. +func GetInt(r *http.Request, argKey string) (int, error) { + argValue := r.FormValue(argKey) + if len(argValue) == 0 { + return 0, nil + } + n, err := strconv.Atoi(argValue) + if err != nil { + return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err) + } + return n, nil +} diff --git a/lib/httputils/time.go b/lib/httputils/time.go new file mode 100644 index 000000000..e55591251 --- /dev/null +++ b/lib/httputils/time.go @@ -0,0 +1,60 @@ +package httputils + +import ( + "fmt" + "math" + "net/http" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +// GetTime returns time in milliseconds from the given argKey query arg. +// +// If argKey is missing in r, then defaultMs rounded to seconds is returned. +// The rounding is needed in order to align query results in Grafana +// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 +func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { + argValue := r.FormValue(argKey) + if len(argValue) == 0 { + return roundToSeconds(defaultMs), nil + } + // Handle Prometheus'-provided minTime and maxTime. + // See https://github.com/prometheus/client_golang/issues/614 + switch argValue { + case prometheusMinTimeFormatted: + return minTimeMsecs, nil + case prometheusMaxTimeFormatted: + return maxTimeMsecs, nil + } + // Parse argValue + secs, err := promutils.ParseTime(argValue) + if err != nil { + return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err) + } + msecs := int64(secs * 1e3) + if msecs < minTimeMsecs { + msecs = 0 + } + if msecs > maxTimeMsecs { + msecs = maxTimeMsecs + } + return msecs, nil +} + +var ( + // These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442 + // See https://github.com/prometheus/client_golang/issues/614 for details. + prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano) + prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano) +) + +const ( + // These values prevent from overflow when storing msec-precision time in int64. + minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time + maxTimeMsecs = int64(1<<63-1) / 1e6 +) + +func roundToSeconds(ms int64) int64 { + return ms - ms%1000 +} diff --git a/lib/httputils/time_test.go b/lib/httputils/time_test.go new file mode 100644 index 000000000..8b5701f96 --- /dev/null +++ b/lib/httputils/time_test.go @@ -0,0 +1,88 @@ +package httputils + +import ( + "fmt" + "net/http" + "net/url" + "testing" +) + +func TestGetTimeSuccess(t *testing.T) { + f := func(s string, timestampExpected int64) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + // Verify defaultValue + ts, err := GetTime(r, "foo", 123456) + if err != nil { + t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err) + } + if ts != 123000 { + t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000) + } + + // Verify timestampExpected + ts, err = GetTime(r, "s", 123) + if err != nil { + t.Fatalf("unexpected error in GetTime(%q): %s", s, err) + } + if ts != timestampExpected { + t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected) + } + } + + f("2019", 1546300800000) + f("2019-01", 1546300800000) + f("2019-02", 1548979200000) + f("2019-02-01", 1548979200000) + f("2019-02-02", 1549065600000) + f("2019-02-02T00", 1549065600000) + f("2019-02-02T01", 1549069200000) + f("2019-02-02T01:00", 1549069200000) + f("2019-02-02T01:01", 1549069260000) + f("2019-02-02T01:01:00", 1549069260000) + f("2019-02-02T01:01:01", 1549069261000) + f("2019-07-07T20:01:02Z", 1562529662000) + f("2019-07-07T20:47:40+03:00", 1562521660000) + f("-292273086-05-16T16:47:06Z", minTimeMsecs) + f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs) + f("1562529662.324", 1562529662324) + f("-9223372036.854", minTimeMsecs) + f("-9223372036.855", minTimeMsecs) + f("9223372036.855", maxTimeMsecs) +} + +func TestGetTimeError(t *testing.T) { + f := func(s string) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + if _, err := GetTime(r, "s", 123); err == nil { + t.Fatalf("expecting non-nil error in GetTime(%q)", s) + } + } + + f("foo") + f("foo1") + f("1245-5") + f("2022-x7") + f("2022-02-x7") + f("2022-02-02Tx7") + f("2022-02-02T00:x7") + f("2022-02-02T00:00:x7") + f("2022-02-02T00:00:00a") + f("2019-07-07T20:01:02Zisdf") + f("2019-07-07T20:47:40+03:00123") + f("-292273086-05-16T16:47:07Z") + f("292277025-08-18T07:12:54.999999998Z") + f("123md") + f("-12.3md") +}