app/vmselect: move common http functionality from app/vmselect/searchutils to lib/httputils

While at it, move app/vmselect/bufferedwriter to lib/bufferedwriter, since it is going to be used in VictoriaLogs
This commit is contained in:
Aliaksandr Valialkin 2023-06-19 22:31:57 -07:00
parent b49d04b3dc
commit 78eaa056c0
No known key found for this signature in database
GPG key ID: A72BEC6CD3D0DED1
15 changed files with 337 additions and 287 deletions

View file

@ -8,14 +8,14 @@ import (
"net/http"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
)
// FunctionsHandler implements /functions handler.
//
// See https://graphite.readthedocs.io/en/latest/functions.html#function-api
func FunctionsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
grouped := searchutils.GetBool(r, "grouped")
grouped := httputils.GetBool(r, "grouped")
group := r.FormValue("group")
result := make(map[string]interface{})
for funcName, fi := range funcs {

View file

@ -10,9 +10,10 @@ import (
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metrics"
@ -46,7 +47,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
if len(delimiter) > 1 {
return fmt.Errorf("`delimiter` query arg must contain only a single char")
}
if searchutils.GetBool(r, "automatic_variants") {
if httputils.GetBool(r, "automatic_variants") {
// See https://github.com/graphite-project/graphite-web/blob/bb9feb0e6815faa73f538af6ed35adea0fb273fd/webapp/graphite/metrics/views.py#L152
query = addAutomaticVariants(query, delimiter)
}
@ -57,19 +58,19 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
query += "*"
}
}
leavesOnly := searchutils.GetBool(r, "leavesOnly")
wildcards := searchutils.GetBool(r, "wildcards")
leavesOnly := httputils.GetBool(r, "leavesOnly")
wildcards := httputils.GetBool(r, "wildcards")
label := r.FormValue("label")
if label == "__name__" {
label = ""
}
jsonp := r.FormValue("jsonp")
from, err := searchutils.GetTime(r, "from", 0)
from, err := httputils.GetTime(r, "from", 0)
if err != nil {
return err
}
ct := startTime.UnixNano() / 1e6
until, err := searchutils.GetTime(r, "until", ct)
until, err := httputils.GetTime(r, "until", ct)
if err != nil {
return err
}
@ -124,8 +125,8 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
if len(queries) == 0 {
return fmt.Errorf("missing `query` arg")
}
groupByExpr := searchutils.GetBool(r, "groupByExpr")
leavesOnly := searchutils.GetBool(r, "leavesOnly")
groupByExpr := httputils.GetBool(r, "groupByExpr")
leavesOnly := httputils.GetBool(r, "leavesOnly")
label := r.FormValue("label")
if label == "__name__" {
label = ""
@ -138,12 +139,12 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re
return fmt.Errorf("`delimiter` query arg must contain only a single char")
}
jsonp := r.FormValue("jsonp")
from, err := searchutils.GetTime(r, "from", 0)
from, err := httputils.GetTime(r, "from", 0)
if err != nil {
return err
}
ct := startTime.UnixNano() / 1e6
until, err := searchutils.GetTime(r, "until", ct)
until, err := httputils.GetTime(r, "until", ct)
if err != nil {
return err
}

View file

@ -8,8 +8,8 @@ import (
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
"github.com/VictoriaMetrics/metrics"
)

View file

@ -9,10 +9,11 @@ import (
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
graphiteparser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@ -164,7 +165,7 @@ var (
// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -254,7 +255,7 @@ var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_sec
// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -337,7 +338,7 @@ var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_secon
// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -412,7 +413,7 @@ var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{pat
// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -443,7 +444,7 @@ var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/t
// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}

View file

@ -20,6 +20,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
@ -95,7 +96,7 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
startTime := time.Now()
defer requestDuration.UpdateDuration(startTime)
tracerEnabled := searchutils.GetBool(r, "trace")
tracerEnabled := httputils.GetBool(r, "trace")
qt := querytracer.New(tracerEnabled, r.URL.Path)
// Limit the number of concurrent queries.
@ -120,7 +121,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
requestURI := httpserver.GetRequestURI(r)
logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
d.Seconds(), remoteAddr, requestURI)
time.Since(startTime).Seconds(), remoteAddr, requestURI)
return true
case <-t.C:
timerpool.Put(t)

View file

@ -12,16 +12,17 @@ import (
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metrics"
@ -132,7 +133,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
return fmt.Errorf("missing `format` arg; see https://docs.victoriametrics.com/#how-to-export-csv-data")
}
fieldNames := strings.Split(format, ",")
reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
reduceMemUsage := httputils.GetBool(r, "reduce_mem_usage")
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries)
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
@ -269,7 +270,7 @@ func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
}
format := r.FormValue("format")
maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line")))
reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
reduceMemUsage := httputils.GetBool(r, "reduce_mem_usage")
if err := exportHandler(nil, w, cp, format, maxRowsPerLine, reduceMemUsage); err != nil {
return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", cp.start, cp.end, err)
}
@ -473,7 +474,7 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
if err != nil {
return err
}
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -570,7 +571,7 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
if err != nil {
return err
}
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -628,7 +629,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
if err != nil {
return err
}
limit, err := searchutils.GetInt(r, "limit")
limit, err := httputils.GetInt(r, "limit")
if err != nil {
return err
}
@ -664,12 +665,12 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
ct := startTime.UnixNano() / 1e6
deadline := searchutils.GetDeadlineForQuery(r, startTime)
mayCache := !searchutils.GetBool(r, "nocache")
mayCache := !httputils.GetBool(r, "nocache")
query := r.FormValue("query")
if len(query) == 0 {
return fmt.Errorf("missing `query` arg")
}
start, err := searchutils.GetTime(r, "time", ct)
start, err := httputils.GetTime(r, "time", ct)
if err != nil {
return err
}
@ -677,7 +678,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
if err != nil {
return err
}
step, err := searchutils.GetDuration(r, "step", lookbackDelta)
step, err := httputils.GetDuration(r, "step", lookbackDelta)
if err != nil {
return err
}
@ -741,7 +742,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
if err != nil {
return err
}
if !searchutils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
if !httputils.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
// Adjust start time only if `nocache` arg isn't set.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
startPrev := start
@ -813,15 +814,15 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
if len(query) == 0 {
return fmt.Errorf("missing `query` arg")
}
start, err := searchutils.GetTime(r, "start", ct-defaultStep)
start, err := httputils.GetTime(r, "start", ct-defaultStep)
if err != nil {
return err
}
end, err := searchutils.GetTime(r, "end", ct)
end, err := httputils.GetTime(r, "end", ct)
if err != nil {
return err
}
step, err := searchutils.GetDuration(r, "step", defaultStep)
step, err := httputils.GetDuration(r, "step", defaultStep)
if err != nil {
return err
}
@ -838,7 +839,7 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, query string,
start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
mayCache := !searchutils.GetBool(r, "nocache")
mayCache := !httputils.GetBool(r, "nocache")
lookbackDelta, err := getMaxLookback(r)
if err != nil {
return err
@ -988,13 +989,13 @@ func getMaxLookback(r *http.Request) (int64, error) {
if d == 0 {
d = maxStalenessInterval.Milliseconds()
}
maxLookback, err := searchutils.GetDuration(r, "max_lookback", d)
maxLookback, err := httputils.GetDuration(r, "max_lookback", d)
if err != nil {
return 0, err
}
d = maxLookback
if *setLookbackToStep {
step, err := searchutils.GetDuration(r, "step", d)
step, err := httputils.GetDuration(r, "step", d)
if err != nil {
return 0, err
}
@ -1034,7 +1035,7 @@ func getLatencyOffsetMilliseconds(r *http.Request) (int64, error) {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2061#issuecomment-1299109836
d = 0
}
return searchutils.GetDuration(r, "latency_offset", d)
return httputils.GetDuration(r, "latency_offset", d)
}
// QueryStatsHandler returns query stats at `/api/v1/status/top_queries`
@ -1050,7 +1051,7 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
}
topN = n
}
maxLifetimeMsecs, err := searchutils.GetDuration(r, "maxLifetime", 10*60*1000)
maxLifetimeMsecs, err := httputils.GetDuration(r, "maxLifetime", 10*60*1000)
if err != nil {
return fmt.Errorf("cannot parse `maxLifetime` arg: %w", err)
}
@ -1120,12 +1121,12 @@ func getCommonParamsWithDefaultDuration(r *http.Request, startTime time.Time, re
// - extra_filters[]
func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) {
deadline := searchutils.GetDeadlineForQuery(r, startTime)
start, err := searchutils.GetTime(r, "start", 0)
start, err := httputils.GetTime(r, "start", 0)
if err != nil {
return nil, err
}
ct := startTime.UnixNano() / 1e6
end, err := searchutils.GetTime(r, "end", ct)
end, err := httputils.GetTime(r, "end", ct)
if err != nil {
return nil, err
}

View file

@ -3,14 +3,12 @@ package searchutils
import (
"flag"
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metricsql"
)
@ -21,96 +19,9 @@ var (
maxStatusRequestDuration = flag.Duration("search.maxStatusRequestDuration", time.Minute*5, "The maximum duration for /api/v1/status/* requests")
)
func roundToSeconds(ms int64) int64 {
return ms - ms%1000
}
// GetInt returns integer value from the given argKey.
func GetInt(r *http.Request, argKey string) (int, error) {
argValue := r.FormValue(argKey)
if len(argValue) == 0 {
return 0, nil
}
n, err := strconv.Atoi(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err)
}
return n, nil
}
// GetTime returns time from the given argKey query arg.
//
// If argKey is missing in r, then defaultMs rounded to seconds is returned.
// The rounding is needed in order to align query results in Grafana
// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720
func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
argValue := r.FormValue(argKey)
if len(argValue) == 0 {
return roundToSeconds(defaultMs), nil
}
// Handle Prometheus'-provided minTime and maxTime.
// See https://github.com/prometheus/client_golang/issues/614
switch argValue {
case prometheusMinTimeFormatted:
return minTimeMsecs, nil
case prometheusMaxTimeFormatted:
return maxTimeMsecs, nil
}
// Parse argValue
secs, err := promutils.ParseTime(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err)
}
msecs := int64(secs * 1e3)
if msecs < minTimeMsecs {
msecs = 0
}
if msecs > maxTimeMsecs {
msecs = maxTimeMsecs
}
return msecs, nil
}
var (
// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
// See https://github.com/prometheus/client_golang/issues/614 for details.
prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano)
prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano)
)
const (
// These values prevent from overflow when storing msec-precision time in int64.
minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
maxTimeMsecs = int64(1<<63-1) / 1e6
)
// GetDuration returns duration from the given argKey query arg.
func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) {
argValue := r.FormValue(argKey)
if len(argValue) == 0 {
return defaultValue, nil
}
secs, err := strconv.ParseFloat(argValue, 64)
if err != nil {
// Try parsing string format
d, err := promutils.ParseDuration(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
}
secs = d.Seconds()
}
msecs := int64(secs * 1e3)
if msecs <= 0 || msecs > maxDurationMsecs {
return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs))
}
return msecs, nil
}
const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
// GetMaxQueryDuration returns the maximum duration for query from r.
func GetMaxQueryDuration(r *http.Request) time.Duration {
dms, err := GetDuration(r, "timeout", 0)
dms, err := httputils.GetDuration(r, "timeout", 0)
if err != nil {
dms = 0
}
@ -140,7 +51,7 @@ func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline {
}
func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline {
d, err := GetDuration(r, "timeout", 0)
d, err := httputils.GetDuration(r, "timeout", 0)
if err != nil {
d = 0
}
@ -151,17 +62,6 @@ func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64
return NewDeadline(startTime, timeout, flagHint)
}
// GetBool returns boolean value from the given argKey query arg.
func GetBool(r *http.Request, argKey string) bool {
argValue := r.FormValue(argKey)
switch strings.ToLower(argValue) {
case "", "0", "f", "false", "no":
return false
default:
return true
}
}
// Deadline contains deadline with the corresponding timeout for pretty error messages.
type Deadline struct {
deadline uint64

View file

@ -11,149 +11,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
)
func TestGetDurationSuccess(t *testing.T) {
f := func(s string, dExpected int64) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
// Verify defaultValue
d, err := GetDuration(r, "foo", 123456)
if err != nil {
t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err)
}
if d != 123456 {
t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456)
}
// Verify dExpected
d, err = GetDuration(r, "s", 123)
if err != nil {
t.Fatalf("unexpected error in GetDuration(%q): %s", s, err)
}
if d != dExpected {
t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected)
}
}
f("1.234", 1234)
f("1.23ms", 1)
f("1.23s", 1230)
f("2s56ms", 2056)
f("2s-5ms", 1995)
f("5m3.5s", 303500)
f("2h", 7200000)
f("1d", 24*3600*1000)
f("7d5h4m3s534ms", 623043534)
}
func TestGetDurationError(t *testing.T) {
f := func(s string) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
if _, err := GetDuration(r, "s", 123); err == nil {
t.Fatalf("expecting non-nil error in GetDuration(%q)", s)
}
}
// Negative durations aren't supported
f("-1.234")
// Invalid duration
f("foo")
// Invalid suffix
f("1md")
}
func TestGetTimeSuccess(t *testing.T) {
f := func(s string, timestampExpected int64) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
// Verify defaultValue
ts, err := GetTime(r, "foo", 123456)
if err != nil {
t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
}
if ts != 123000 {
t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
}
// Verify timestampExpected
ts, err = GetTime(r, "s", 123)
if err != nil {
t.Fatalf("unexpected error in GetTime(%q): %s", s, err)
}
if ts != timestampExpected {
t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected)
}
}
f("2019", 1546300800000)
f("2019-01", 1546300800000)
f("2019-02", 1548979200000)
f("2019-02-01", 1548979200000)
f("2019-02-02", 1549065600000)
f("2019-02-02T00", 1549065600000)
f("2019-02-02T01", 1549069200000)
f("2019-02-02T01:00", 1549069200000)
f("2019-02-02T01:01", 1549069260000)
f("2019-02-02T01:01:00", 1549069260000)
f("2019-02-02T01:01:01", 1549069261000)
f("2019-07-07T20:01:02Z", 1562529662000)
f("2019-07-07T20:47:40+03:00", 1562521660000)
f("-292273086-05-16T16:47:06Z", minTimeMsecs)
f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs)
f("1562529662.324", 1562529662324)
f("-9223372036.854", minTimeMsecs)
f("-9223372036.855", minTimeMsecs)
f("9223372036.855", maxTimeMsecs)
}
func TestGetTimeError(t *testing.T) {
f := func(s string) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
if _, err := GetTime(r, "s", 123); err == nil {
t.Fatalf("expecting non-nil error in GetTime(%q)", s)
}
}
f("foo")
f("foo1")
f("1245-5")
f("2022-x7")
f("2022-02-x7")
f("2022-02-02Tx7")
f("2022-02-02T00:x7")
f("2022-02-02T00:00:x7")
f("2022-02-02T00:00:00a")
f("2019-07-07T20:01:02Zisdf")
f("2019-07-07T20:47:40+03:00123")
f("-292273086-05-16T16:47:07Z")
f("292277025-08-18T07:12:54.999999998Z")
f("123md")
f("-12.3md")
}
func TestGetExtraTagFilters(t *testing.T) {
httpReqWithForm := func(qs string) *http.Request {
q, err := url.ParseQuery(qs)

17
lib/httputils/bool.go Normal file
View file

@ -0,0 +1,17 @@
package httputils
import (
"net/http"
"strings"
)
// GetBool returns boolean value from the given argKey query arg.
func GetBool(r *http.Request, argKey string) bool {
argValue := r.FormValue(argKey)
switch strings.ToLower(argValue) {
case "", "0", "f", "false", "no":
return false
default:
return true
}
}

33
lib/httputils/duration.go Normal file
View file

@ -0,0 +1,33 @@
package httputils
import (
"fmt"
"net/http"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
)
// GetDuration returns duration in milliseconds from the given argKey query arg.
func GetDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) {
argValue := r.FormValue(argKey)
if len(argValue) == 0 {
return defaultValue, nil
}
secs, err := strconv.ParseFloat(argValue, 64)
if err != nil {
// Try parsing string format
d, err := promutils.ParseDuration(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
}
secs = d.Seconds()
}
msecs := int64(secs * 1e3)
if msecs <= 0 || msecs > maxDurationMsecs {
return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs))
}
return msecs, nil
}
const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000

View file

@ -0,0 +1,71 @@
package httputils
import (
"fmt"
"net/http"
"net/url"
"testing"
)
func TestGetDurationSuccess(t *testing.T) {
f := func(s string, dExpected int64) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
// Verify defaultValue
d, err := GetDuration(r, "foo", 123456)
if err != nil {
t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err)
}
if d != 123456 {
t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456)
}
// Verify dExpected
d, err = GetDuration(r, "s", 123)
if err != nil {
t.Fatalf("unexpected error in GetDuration(%q): %s", s, err)
}
if d != dExpected {
t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected)
}
}
f("1.234", 1234)
f("1.23ms", 1)
f("1.23s", 1230)
f("2s56ms", 2056)
f("2s-5ms", 1995)
f("5m3.5s", 303500)
f("2h", 7200000)
f("1d", 24*3600*1000)
f("7d5h4m3s534ms", 623043534)
}
func TestGetDurationError(t *testing.T) {
f := func(s string) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
if _, err := GetDuration(r, "s", 123); err == nil {
t.Fatalf("expecting non-nil error in GetDuration(%q)", s)
}
}
// Negative durations aren't supported
f("-1.234")
// Invalid duration
f("foo")
// Invalid suffix
f("1md")
}

20
lib/httputils/int.go Normal file
View file

@ -0,0 +1,20 @@
package httputils
import (
"fmt"
"net/http"
"strconv"
)
// GetInt returns integer value from the given argKey.
func GetInt(r *http.Request, argKey string) (int, error) {
argValue := r.FormValue(argKey)
if len(argValue) == 0 {
return 0, nil
}
n, err := strconv.Atoi(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse integer %q=%q: %w", argKey, argValue, err)
}
return n, nil
}

60
lib/httputils/time.go Normal file
View file

@ -0,0 +1,60 @@
package httputils
import (
"fmt"
"math"
"net/http"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
)
// GetTime returns time in milliseconds from the given argKey query arg.
//
// If argKey is missing in r, then defaultMs rounded to seconds is returned.
// The rounding is needed in order to align query results in Grafana
// executed at different times. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/720
func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
argValue := r.FormValue(argKey)
if len(argValue) == 0 {
return roundToSeconds(defaultMs), nil
}
// Handle Prometheus'-provided minTime and maxTime.
// See https://github.com/prometheus/client_golang/issues/614
switch argValue {
case prometheusMinTimeFormatted:
return minTimeMsecs, nil
case prometheusMaxTimeFormatted:
return maxTimeMsecs, nil
}
// Parse argValue
secs, err := promutils.ParseTime(argValue)
if err != nil {
return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err)
}
msecs := int64(secs * 1e3)
if msecs < minTimeMsecs {
msecs = 0
}
if msecs > maxTimeMsecs {
msecs = maxTimeMsecs
}
return msecs, nil
}
var (
// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
// See https://github.com/prometheus/client_golang/issues/614 for details.
prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano)
prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano)
)
const (
// These values prevent from overflow when storing msec-precision time in int64.
minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
maxTimeMsecs = int64(1<<63-1) / 1e6
)
func roundToSeconds(ms int64) int64 {
return ms - ms%1000
}

View file

@ -0,0 +1,88 @@
package httputils
import (
"fmt"
"net/http"
"net/url"
"testing"
)
func TestGetTimeSuccess(t *testing.T) {
f := func(s string, timestampExpected int64) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
// Verify defaultValue
ts, err := GetTime(r, "foo", 123456)
if err != nil {
t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
}
if ts != 123000 {
t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
}
// Verify timestampExpected
ts, err = GetTime(r, "s", 123)
if err != nil {
t.Fatalf("unexpected error in GetTime(%q): %s", s, err)
}
if ts != timestampExpected {
t.Fatalf("unexpected timestamp for GetTime(%q); got %d; want %d", s, ts, timestampExpected)
}
}
f("2019", 1546300800000)
f("2019-01", 1546300800000)
f("2019-02", 1548979200000)
f("2019-02-01", 1548979200000)
f("2019-02-02", 1549065600000)
f("2019-02-02T00", 1549065600000)
f("2019-02-02T01", 1549069200000)
f("2019-02-02T01:00", 1549069200000)
f("2019-02-02T01:01", 1549069260000)
f("2019-02-02T01:01:00", 1549069260000)
f("2019-02-02T01:01:01", 1549069261000)
f("2019-07-07T20:01:02Z", 1562529662000)
f("2019-07-07T20:47:40+03:00", 1562521660000)
f("-292273086-05-16T16:47:06Z", minTimeMsecs)
f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs)
f("1562529662.324", 1562529662324)
f("-9223372036.854", minTimeMsecs)
f("-9223372036.855", minTimeMsecs)
f("9223372036.855", maxTimeMsecs)
}
func TestGetTimeError(t *testing.T) {
f := func(s string) {
t.Helper()
urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
r, err := http.NewRequest(http.MethodGet, urlStr, nil)
if err != nil {
t.Fatalf("unexpected error in NewRequest: %s", err)
}
if _, err := GetTime(r, "s", 123); err == nil {
t.Fatalf("expecting non-nil error in GetTime(%q)", s)
}
}
f("foo")
f("foo1")
f("1245-5")
f("2022-x7")
f("2022-02-x7")
f("2022-02-02Tx7")
f("2022-02-02T00:x7")
f("2022-02-02T00:00:x7")
f("2022-02-02T00:00:00a")
f("2019-07-07T20:01:02Zisdf")
f("2019-07-07T20:47:40+03:00123")
f("-292273086-05-16T16:47:07Z")
f("292277025-08-18T07:12:54.999999998Z")
f("123md")
f("-12.3md")
}