From 78eaa056c03d0c7c3c69a7d86dd0134d2ed1895e Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 19 Jun 2023 22:31:57 -0700 Subject: [PATCH] app/vmselect: move common http functionality from app/vmselect/searchutils to lib/httputils While at it, move app/vmselect/bufferedwriter to lib/bufferedwriter, since it is going to be used in VictoriaLogs --- app/vmselect/graphite/functions_api.go | 4 +- app/vmselect/graphite/metrics_api.go | 21 +-- app/vmselect/graphite/render_api.go | 2 +- app/vmselect/graphite/tags_api.go | 13 +- app/vmselect/main.go | 5 +- app/vmselect/prometheus/prometheus.go | 41 ++--- app/vmselect/searchutils/searchutils.go | 106 +------------ app/vmselect/searchutils/searchutils_test.go | 143 ------------------ .../bufferedwriter/bufferedwriter.go | 0 lib/httputils/bool.go | 17 +++ lib/httputils/duration.go | 33 ++++ lib/httputils/duration_test.go | 71 +++++++++ lib/httputils/int.go | 20 +++ lib/httputils/time.go | 60 ++++++++ lib/httputils/time_test.go | 88 +++++++++++ 15 files changed, 337 insertions(+), 287 deletions(-) rename {app/vmselect => lib}/bufferedwriter/bufferedwriter.go (100%) create mode 100644 lib/httputils/bool.go create mode 100644 lib/httputils/duration.go create mode 100644 lib/httputils/duration_test.go create mode 100644 lib/httputils/int.go create mode 100644 lib/httputils/time.go create mode 100644 lib/httputils/time_test.go diff --git a/app/vmselect/graphite/functions_api.go b/app/vmselect/graphite/functions_api.go index a8140b3e1..2a39933d6 100644 --- a/app/vmselect/graphite/functions_api.go +++ b/app/vmselect/graphite/functions_api.go @@ -8,14 +8,14 @@ import ( "net/http" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" ) // FunctionsHandler implements /functions handler. // // See https://graphite.readthedocs.io/en/latest/functions.html#function-api func FunctionsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { - grouped := searchutils.GetBool(r, "grouped") + grouped := httputils.GetBool(r, "grouped") group := r.FormValue("group") result := make(map[string]interface{}) for funcName, fi := range funcs { diff --git a/app/vmselect/graphite/metrics_api.go b/app/vmselect/graphite/metrics_api.go index 5128c2e8b..6f50cd7b4 100644 --- a/app/vmselect/graphite/metrics_api.go +++ b/app/vmselect/graphite/metrics_api.go @@ -10,9 +10,10 @@ import ( "sync" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" @@ -46,7 +47,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ if len(delimiter) > 1 { return fmt.Errorf("`delimiter` query arg must contain only a single char") } - if searchutils.GetBool(r, "automatic_variants") { + if httputils.GetBool(r, "automatic_variants") { // See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L152 query = addAutomaticVariants(query, delimiter) } @@ -57,19 +58,19 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ query += "*" } } - leavesOnly := searchutils.GetBool(r, "leavesOnly") - wildcards := searchutils.GetBool(r, "wildcards") + leavesOnly := httputils.GetBool(r, "leavesOnly") + wildcards := httputils.GetBool(r, "wildcards") label := r.FormValue("label") if label == "__name__" { label = "" } jsonp := r.FormValue("jsonp") - from, err := searchutils.GetTime(r, "from", 0) + from, err := httputils.GetTime(r, "from", 0) if err != nil { return err } ct := startTime.UnixNano() / 1e6 - until, err := searchutils.GetTime(r, "until", ct) + until, err := httputils.GetTime(r, "until", ct) if err != nil { return err } @@ -124,8 +125,8 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re if len(queries) == 0 { return fmt.Errorf("missing `query` arg") } - groupByExpr := searchutils.GetBool(r, "groupByExpr") - leavesOnly := searchutils.GetBool(r, "leavesOnly") + groupByExpr := httputils.GetBool(r, "groupByExpr") + leavesOnly := httputils.GetBool(r, "leavesOnly") label := r.FormValue("label") if label == "__name__" { label = "" @@ -138,12 +139,12 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re return fmt.Errorf("`delimiter` query arg must contain only a single char") } jsonp := r.FormValue("jsonp") - from, err := searchutils.GetTime(r, "from", 0) + from, err := httputils.GetTime(r, "from", 0) if err != nil { return err } ct := startTime.UnixNano() / 1e6 - until, err := searchutils.GetTime(r, "until", ct) + until, err := httputils.GetTime(r, "until", ct) if err != nil { return err } diff --git a/app/vmselect/graphite/render_api.go b/app/vmselect/graphite/render_api.go index f5c576c36..dcc6f3378 100644 --- a/app/vmselect/graphite/render_api.go +++ b/app/vmselect/graphite/render_api.go @@ -8,8 +8,8 @@ import ( "strings" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" "github.com/VictoriaMetrics/metrics" ) diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index adbe3a886..c31b4266f 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -9,10 +9,11 @@ import ( "strings" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" @@ -164,7 +165,7 @@ var ( // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -254,7 +255,7 @@ var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_sec // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -337,7 +338,7 @@ var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_secon // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -412,7 +413,7 @@ var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{pat // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -443,7 +444,7 @@ var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/t // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 5c0b0d57c..baf798e9a 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -20,6 +20,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" @@ -95,7 +96,7 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles)) func RequestHandler(w http.ResponseWriter, r *http.Request) bool { startTime := time.Now() defer requestDuration.UpdateDuration(startTime) - tracerEnabled := searchutils.GetBool(r, "trace") + tracerEnabled := httputils.GetBool(r, "trace") qt := querytracer.New(tracerEnabled, r.URL.Path) // Limit the number of concurrent queries. @@ -120,7 +121,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { remoteAddr := httpserver.GetQuotedRemoteAddr(r) requestURI := httpserver.GetRequestURI(r) logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q", - d.Seconds(), remoteAddr, requestURI) + time.Since(startTime).Seconds(), remoteAddr, requestURI) return true case <-t.C: timerpool.Put(t) diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index dc9749805..1f47a18bc 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -12,16 +12,17 @@ import ( "sync/atomic" "time" - "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" @@ -132,7 +133,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques return fmt.Errorf("missing `format` arg; see https://docs.victoriametrics.com/#how-to-export-csv-data") } fieldNames := strings.Split(format, ",") - reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage") + reduceMemUsage := httputils.GetBool(r, "reduce_mem_usage") sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries) w.Header().Set("Content-Type", "text/csv; charset=utf-8") @@ -269,7 +270,7 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) } format := r.FormValue("format") maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line"))) - reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage") + reduceMemUsage := httputils.GetBool(r, "reduce_mem_usage") if err := exportHandler(nil, w, cp, format, maxRowsPerLine, reduceMemUsage); err != nil { return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", cp.start, cp.end, err) } @@ -473,7 +474,7 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s if err != nil { return err } - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -570,7 +571,7 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW if err != nil { return err } - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -628,7 +629,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW if err != nil { return err } - limit, err := searchutils.GetInt(r, "limit") + limit, err := httputils.GetInt(r, "limit") if err != nil { return err } @@ -664,12 +665,12 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr ct := startTime.UnixNano() / 1e6 deadline := searchutils.GetDeadlineForQuery(r, startTime) - mayCache := !searchutils.GetBool(r, "nocache") + mayCache := !httputils.GetBool(r, "nocache") query := r.FormValue("query") if len(query) == 0 { return fmt.Errorf("missing `query` arg") } - start, err := searchutils.GetTime(r, "time", ct) + start, err := httputils.GetTime(r, "time", ct) if err != nil { return err } @@ -677,7 +678,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr if err != nil { return err } - step, err := searchutils.GetDuration(r, "step", lookbackDelta) + step, err := httputils.GetDuration(r, "step", lookbackDelta) if err != nil { return err } @@ -741,7 +742,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr if err != nil { return err } - if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset { + if !httputils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset { // Adjust start time only if `nocache` arg isn't set. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241 startPrev := start @@ -813,15 +814,15 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo if len(query) == 0 { return fmt.Errorf("missing `query` arg") } - start, err := searchutils.GetTime(r, "start", ct-defaultStep) + start, err := httputils.GetTime(r, "start", ct-defaultStep) if err != nil { return err } - end, err := searchutils.GetTime(r, "end", ct) + end, err := httputils.GetTime(r, "end", ct) if err != nil { return err } - step, err := searchutils.GetDuration(r, "step", defaultStep) + step, err := httputils.GetDuration(r, "step", defaultStep) if err != nil { return err } @@ -838,7 +839,7 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error { deadline := searchutils.GetDeadlineForQuery(r, startTime) - mayCache := !searchutils.GetBool(r, "nocache") + mayCache := !httputils.GetBool(r, "nocache") lookbackDelta, err := getMaxLookback(r) if err != nil { return err @@ -988,13 +989,13 @@ func getMaxLookback(r *http.Request) (int64, error) { if d == 0 { d = maxStalenessInterval.Milliseconds() } - maxLookback, err := searchutils.GetDuration(r, "max_lookback", d) + maxLookback, err := httputils.GetDuration(r, "max_lookback", d) if err != nil { return 0, err } d = maxLookback if *setLookbackToStep { - step, err := searchutils.GetDuration(r, "step", d) + step, err := httputils.GetDuration(r, "step", d) if err != nil { return 0, err } @@ -1034,7 +1035,7 @@ func getLatencyOffsetMilliseconds(r *http.Request) (int64, error) { // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836 d = 0 } - return searchutils.GetDuration(r, "latency_offset", d) + return httputils.GetDuration(r, "latency_offset", d) } // QueryStatsHandler returns query stats at `/api/v1/status/top_queries` @@ -1050,7 +1051,7 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque } topN = n } - maxLifetimeMsecs, err := searchutils.GetDuration(r, "maxLifetime", 10*60*1000) + maxLifetimeMsecs, err := httputils.GetDuration(r, "maxLifetime", 10*60*1000) if err != nil { return fmt.Errorf("cannot parse `maxLifetime` arg: %w", err) } @@ -1120,12 +1121,12 @@ func getCommonParamsWithDefaultDuration(r *http.Request, startTime time.Time, re // - extra_filters[] func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) { deadline := searchutils.GetDeadlineForQuery(r, startTime) - start, err := searchutils.GetTime(r, "start", 0) + start, err := httputils.GetTime(r, "start", 0) if err != nil { return nil, err } ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) + end, err := httputils.GetTime(r, "end", ct) if err != nil { return nil, err } diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index d2adcd97b..8d301b346 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -3,14 +3,12 @@ package searchutils import ( "flag" "fmt" - "math" "net/http" - "strconv" "strings" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metricsql" ) @@ -21,96 +19,9 @@ var ( maxStatusRequestDuration = flag.Duration("search.maxStatusRequestDuration", time.Minute*5, "The maximum duration for /api/v1/status/* requests") ) -func roundToSeconds(ms int64) int64 { - return ms - ms%1000 -} - -// GetInt returns integer value from the given argKey. -func GetInt(r *http.Request, argKey string) (int, error) { - argValue := r.FormValue(argKey) - if len(argValue) == 0 { - return 0, nil - } - n, err := strconv.Atoi(argValue) - if err != nil { - return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err) - } - return n, nil -} - -// GetTime returns time from the given argKey query arg. -// -// If argKey is missing in r, then defaultMs rounded to seconds is returned. -// The rounding is needed in order to align query results in Grafana -// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 -func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { - argValue := r.FormValue(argKey) - if len(argValue) == 0 { - return roundToSeconds(defaultMs), nil - } - // Handle Prometheus'-provided minTime and maxTime. - // See https://github.com/prometheus/client_golang/issues/614 - switch argValue { - case prometheusMinTimeFormatted: - return minTimeMsecs, nil - case prometheusMaxTimeFormatted: - return maxTimeMsecs, nil - } - // Parse argValue - secs, err := promutils.ParseTime(argValue) - if err != nil { - return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err) - } - msecs := int64(secs * 1e3) - if msecs < minTimeMsecs { - msecs = 0 - } - if msecs > maxTimeMsecs { - msecs = maxTimeMsecs - } - return msecs, nil -} - -var ( - // These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442 - // See https://github.com/prometheus/client_golang/issues/614 for details. - prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano) - prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano) -) - -const ( - // These values prevent from overflow when storing msec-precision time in int64. - minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time - maxTimeMsecs = int64(1<<63-1) / 1e6 -) - -// GetDuration returns duration from the given argKey query arg. -func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) { - argValue := r.FormValue(argKey) - if len(argValue) == 0 { - return defaultValue, nil - } - secs, err := strconv.ParseFloat(argValue, 64) - if err != nil { - // Try parsing string format - d, err := promutils.ParseDuration(argValue) - if err != nil { - return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) - } - secs = d.Seconds() - } - msecs := int64(secs * 1e3) - if msecs <= 0 || msecs > maxDurationMsecs { - return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs)) - } - return msecs, nil -} - -const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000 - // GetMaxQueryDuration returns the maximum duration for query from r. func GetMaxQueryDuration(r *http.Request) time.Duration { - dms, err := GetDuration(r, "timeout", 0) + dms, err := httputils.GetDuration(r, "timeout", 0) if err != nil { dms = 0 } @@ -140,7 +51,7 @@ func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline { } func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline { - d, err := GetDuration(r, "timeout", 0) + d, err := httputils.GetDuration(r, "timeout", 0) if err != nil { d = 0 } @@ -151,17 +62,6 @@ func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64 return NewDeadline(startTime, timeout, flagHint) } -// GetBool returns boolean value from the given argKey query arg. -func GetBool(r *http.Request, argKey string) bool { - argValue := r.FormValue(argKey) - switch strings.ToLower(argValue) { - case "", "0", "f", "false", "no": - return false - default: - return true - } -} - // Deadline contains deadline with the corresponding timeout for pretty error messages. type Deadline struct { deadline uint64 diff --git a/app/vmselect/searchutils/searchutils_test.go b/app/vmselect/searchutils/searchutils_test.go index 73544392f..b763b8b76 100644 --- a/app/vmselect/searchutils/searchutils_test.go +++ b/app/vmselect/searchutils/searchutils_test.go @@ -11,149 +11,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) -func TestGetDurationSuccess(t *testing.T) { - f := func(s string, dExpected int64) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - // Verify defaultValue - d, err := GetDuration(r, "foo", 123456) - if err != nil { - t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err) - } - if d != 123456 { - t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456) - } - - // Verify dExpected - d, err = GetDuration(r, "s", 123) - if err != nil { - t.Fatalf("unexpected error in GetDuration(%q): %s", s, err) - } - if d != dExpected { - t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected) - } - } - - f("1.234", 1234) - f("1.23ms", 1) - f("1.23s", 1230) - f("2s56ms", 2056) - f("2s-5ms", 1995) - f("5m3.5s", 303500) - f("2h", 7200000) - f("1d", 24*3600*1000) - f("7d5h4m3s534ms", 623043534) -} - -func TestGetDurationError(t *testing.T) { - f := func(s string) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - if _, err := GetDuration(r, "s", 123); err == nil { - t.Fatalf("expecting non-nil error in GetDuration(%q)", s) - } - } - - // Negative durations aren't supported - f("-1.234") - - // Invalid duration - f("foo") - - // Invalid suffix - f("1md") -} - -func TestGetTimeSuccess(t *testing.T) { - f := func(s string, timestampExpected int64) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - // Verify defaultValue - ts, err := GetTime(r, "foo", 123456) - if err != nil { - t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err) - } - if ts != 123000 { - t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000) - } - - // Verify timestampExpected - ts, err = GetTime(r, "s", 123) - if err != nil { - t.Fatalf("unexpected error in GetTime(%q): %s", s, err) - } - if ts != timestampExpected { - t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected) - } - } - - f("2019", 1546300800000) - f("2019-01", 1546300800000) - f("2019-02", 1548979200000) - f("2019-02-01", 1548979200000) - f("2019-02-02", 1549065600000) - f("2019-02-02T00", 1549065600000) - f("2019-02-02T01", 1549069200000) - f("2019-02-02T01:00", 1549069200000) - f("2019-02-02T01:01", 1549069260000) - f("2019-02-02T01:01:00", 1549069260000) - f("2019-02-02T01:01:01", 1549069261000) - f("2019-07-07T20:01:02Z", 1562529662000) - f("2019-07-07T20:47:40+03:00", 1562521660000) - f("-292273086-05-16T16:47:06Z", minTimeMsecs) - f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs) - f("1562529662.324", 1562529662324) - f("-9223372036.854", minTimeMsecs) - f("-9223372036.855", minTimeMsecs) - f("9223372036.855", maxTimeMsecs) -} - -func TestGetTimeError(t *testing.T) { - f := func(s string) { - t.Helper() - urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) - r, err := http.NewRequest(http.MethodGet, urlStr, nil) - if err != nil { - t.Fatalf("unexpected error in NewRequest: %s", err) - } - - if _, err := GetTime(r, "s", 123); err == nil { - t.Fatalf("expecting non-nil error in GetTime(%q)", s) - } - } - - f("foo") - f("foo1") - f("1245-5") - f("2022-x7") - f("2022-02-x7") - f("2022-02-02Tx7") - f("2022-02-02T00:x7") - f("2022-02-02T00:00:x7") - f("2022-02-02T00:00:00a") - f("2019-07-07T20:01:02Zisdf") - f("2019-07-07T20:47:40+03:00123") - f("-292273086-05-16T16:47:07Z") - f("292277025-08-18T07:12:54.999999998Z") - f("123md") - f("-12.3md") -} - func TestGetExtraTagFilters(t *testing.T) { httpReqWithForm := func(qs string) *http.Request { q, err := url.ParseQuery(qs) diff --git a/app/vmselect/bufferedwriter/bufferedwriter.go b/lib/bufferedwriter/bufferedwriter.go similarity index 100% rename from app/vmselect/bufferedwriter/bufferedwriter.go rename to lib/bufferedwriter/bufferedwriter.go diff --git a/lib/httputils/bool.go b/lib/httputils/bool.go new file mode 100644 index 000000000..295528cbb --- /dev/null +++ b/lib/httputils/bool.go @@ -0,0 +1,17 @@ +package httputils + +import ( + "net/http" + "strings" +) + +// GetBool returns boolean value from the given argKey query arg. +func GetBool(r *http.Request, argKey string) bool { + argValue := r.FormValue(argKey) + switch strings.ToLower(argValue) { + case "", "0", "f", "false", "no": + return false + default: + return true + } +} diff --git a/lib/httputils/duration.go b/lib/httputils/duration.go new file mode 100644 index 000000000..89d31feb5 --- /dev/null +++ b/lib/httputils/duration.go @@ -0,0 +1,33 @@ +package httputils + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +// GetDuration returns duration in milliseconds from the given argKey query arg. +func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) { + argValue := r.FormValue(argKey) + if len(argValue) == 0 { + return defaultValue, nil + } + secs, err := strconv.ParseFloat(argValue, 64) + if err != nil { + // Try parsing string format + d, err := promutils.ParseDuration(argValue) + if err != nil { + return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err) + } + secs = d.Seconds() + } + msecs := int64(secs * 1e3) + if msecs <= 0 || msecs > maxDurationMsecs { + return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs)) + } + return msecs, nil +} + +const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000 diff --git a/lib/httputils/duration_test.go b/lib/httputils/duration_test.go new file mode 100644 index 000000000..7cb3dc631 --- /dev/null +++ b/lib/httputils/duration_test.go @@ -0,0 +1,71 @@ +package httputils + +import ( + "fmt" + "net/http" + "net/url" + "testing" +) + +func TestGetDurationSuccess(t *testing.T) { + f := func(s string, dExpected int64) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + // Verify defaultValue + d, err := GetDuration(r, "foo", 123456) + if err != nil { + t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err) + } + if d != 123456 { + t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456) + } + + // Verify dExpected + d, err = GetDuration(r, "s", 123) + if err != nil { + t.Fatalf("unexpected error in GetDuration(%q): %s", s, err) + } + if d != dExpected { + t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected) + } + } + + f("1.234", 1234) + f("1.23ms", 1) + f("1.23s", 1230) + f("2s56ms", 2056) + f("2s-5ms", 1995) + f("5m3.5s", 303500) + f("2h", 7200000) + f("1d", 24*3600*1000) + f("7d5h4m3s534ms", 623043534) +} + +func TestGetDurationError(t *testing.T) { + f := func(s string) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + if _, err := GetDuration(r, "s", 123); err == nil { + t.Fatalf("expecting non-nil error in GetDuration(%q)", s) + } + } + + // Negative durations aren't supported + f("-1.234") + + // Invalid duration + f("foo") + + // Invalid suffix + f("1md") +} diff --git a/lib/httputils/int.go b/lib/httputils/int.go new file mode 100644 index 000000000..2547cbf16 --- /dev/null +++ b/lib/httputils/int.go @@ -0,0 +1,20 @@ +package httputils + +import ( + "fmt" + "net/http" + "strconv" +) + +// GetInt returns integer value from the given argKey. +func GetInt(r *http.Request, argKey string) (int, error) { + argValue := r.FormValue(argKey) + if len(argValue) == 0 { + return 0, nil + } + n, err := strconv.Atoi(argValue) + if err != nil { + return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err) + } + return n, nil +} diff --git a/lib/httputils/time.go b/lib/httputils/time.go new file mode 100644 index 000000000..e55591251 --- /dev/null +++ b/lib/httputils/time.go @@ -0,0 +1,60 @@ +package httputils + +import ( + "fmt" + "math" + "net/http" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +// GetTime returns time in milliseconds from the given argKey query arg. +// +// If argKey is missing in r, then defaultMs rounded to seconds is returned. +// The rounding is needed in order to align query results in Grafana +// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720 +func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) { + argValue := r.FormValue(argKey) + if len(argValue) == 0 { + return roundToSeconds(defaultMs), nil + } + // Handle Prometheus'-provided minTime and maxTime. + // See https://github.com/prometheus/client_golang/issues/614 + switch argValue { + case prometheusMinTimeFormatted: + return minTimeMsecs, nil + case prometheusMaxTimeFormatted: + return maxTimeMsecs, nil + } + // Parse argValue + secs, err := promutils.ParseTime(argValue) + if err != nil { + return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err) + } + msecs := int64(secs * 1e3) + if msecs < minTimeMsecs { + msecs = 0 + } + if msecs > maxTimeMsecs { + msecs = maxTimeMsecs + } + return msecs, nil +} + +var ( + // These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442 + // See https://github.com/prometheus/client_golang/issues/614 for details. + prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano) + prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano) +) + +const ( + // These values prevent from overflow when storing msec-precision time in int64. + minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time + maxTimeMsecs = int64(1<<63-1) / 1e6 +) + +func roundToSeconds(ms int64) int64 { + return ms - ms%1000 +} diff --git a/lib/httputils/time_test.go b/lib/httputils/time_test.go new file mode 100644 index 000000000..8b5701f96 --- /dev/null +++ b/lib/httputils/time_test.go @@ -0,0 +1,88 @@ +package httputils + +import ( + "fmt" + "net/http" + "net/url" + "testing" +) + +func TestGetTimeSuccess(t *testing.T) { + f := func(s string, timestampExpected int64) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + // Verify defaultValue + ts, err := GetTime(r, "foo", 123456) + if err != nil { + t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err) + } + if ts != 123000 { + t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000) + } + + // Verify timestampExpected + ts, err = GetTime(r, "s", 123) + if err != nil { + t.Fatalf("unexpected error in GetTime(%q): %s", s, err) + } + if ts != timestampExpected { + t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected) + } + } + + f("2019", 1546300800000) + f("2019-01", 1546300800000) + f("2019-02", 1548979200000) + f("2019-02-01", 1548979200000) + f("2019-02-02", 1549065600000) + f("2019-02-02T00", 1549065600000) + f("2019-02-02T01", 1549069200000) + f("2019-02-02T01:00", 1549069200000) + f("2019-02-02T01:01", 1549069260000) + f("2019-02-02T01:01:00", 1549069260000) + f("2019-02-02T01:01:01", 1549069261000) + f("2019-07-07T20:01:02Z", 1562529662000) + f("2019-07-07T20:47:40+03:00", 1562521660000) + f("-292273086-05-16T16:47:06Z", minTimeMsecs) + f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs) + f("1562529662.324", 1562529662324) + f("-9223372036.854", minTimeMsecs) + f("-9223372036.855", minTimeMsecs) + f("9223372036.855", maxTimeMsecs) +} + +func TestGetTimeError(t *testing.T) { + f := func(s string) { + t.Helper() + urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s)) + r, err := http.NewRequest(http.MethodGet, urlStr, nil) + if err != nil { + t.Fatalf("unexpected error in NewRequest: %s", err) + } + + if _, err := GetTime(r, "s", 123); err == nil { + t.Fatalf("expecting non-nil error in GetTime(%q)", s) + } + } + + f("foo") + f("foo1") + f("1245-5") + f("2022-x7") + f("2022-02-x7") + f("2022-02-02Tx7") + f("2022-02-02T00:x7") + f("2022-02-02T00:00:x7") + f("2022-02-02T00:00:00a") + f("2019-07-07T20:01:02Zisdf") + f("2019-07-07T20:47:40+03:00123") + f("-292273086-05-16T16:47:07Z") + f("292277025-08-18T07:12:54.999999998Z") + f("123md") + f("-12.3md") +}