github-mirrors/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-03-11 15:34:56 +00:00
Code Issues Projects Releases Packages Wiki Activity

app/vmselect: add -search.storageTimeout command-line flag for limiting the maximum duration of query execution per each -storageNode

Browse source 
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711
This commit is contained in:
Aliaksandr Valialkin 2020-09-16 21:32:03 +03:00
parent 9705ac5d7a
commit 9793008734
2 changed files with 21 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
app/vmselect/netstorage/netstorage.go
View file

@ -1319,9 +1319,21 @@ func (sn *storageNode) execOnConn(rpcName string, f func(bc *handshake.BufferedC
d := time.Unix(int64(deadline.Deadline()), 0) d := time.Unix(int64(deadline.Deadline()), 0)
nowSecs := fasttime.UnixTimestamp() nowSecs := fasttime.UnixTimestamp()
currentTime := time.Unix(int64(nowSecs), 0) currentTime := time.Unix(int64(nowSecs), 0)
storageTimeout := *searchutils.StorageTimeout
if storageTimeout > 0 {
dd := currentTime.Add(storageTimeout)
if dd.Sub(d) < 0 {
// Limit the remote deadline to storageTimeout,
// so slow vmstorage nodes may stop processing the request.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711 .
// The local deadline remains the same, so data obtained from
// the remaining vmstorage nodes could be processed locally.
d = dd
}
}
timeout := d.Sub(currentTime) timeout := d.Sub(currentTime)
if timeout <= 0 { if timeout <= 0 {
return fmt.Errorf("request timeout reached: %s", deadline.String()) return fmt.Errorf("request timeout reached: %s or -search.storageTimeout=%s", deadline.String(), storageTimeout.String())
} }
bc, err := sn.connPool.Get() bc, err := sn.connPool.Get()
if err != nil { if err != nil {

10
app/vmselect/searchutils/searchutils.go
View file

@ -15,9 +15,15 @@ import (
var ( var (
maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call") maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution") maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution; see also -search.storageTimeout")
denyPartialResponse = flag.Bool("search.denyPartialResponse", false, "Whether to deny partial responses if a part of -storageNode instances fail to perform queries; "+ denyPartialResponse = flag.Bool("search.denyPartialResponse", false, "Whether to deny partial responses if a part of -storageNode instances fail to perform queries; "+
"this trades availability over consistency; see also -search.maxQueryDuration") "this trades availability over consistency; see also -search.maxQueryDuration and -search.storageTimeout")
// StorageTimeout limits the duration of query execution on every vmstorage node.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711
StorageTimeout = flag.Duration("search.storageTimeout", 0, "The timeout for per-storage query processing; "+
"this allows returning partial responses if certain -storageNode instances slowly process the query; "+
"see also -search.maxQueryDuration and -search.denyPartialResponse command-line flags")
) )
// GetTime returns time from the given argKey query arg. // GetTime returns time from the given argKey query arg.