diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index 452ad5eddd..e4ebc7ae5b 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -1319,9 +1319,21 @@ func (sn *storageNode) execOnConn(rpcName string, f func(bc *handshake.BufferedC d := time.Unix(int64(deadline.Deadline()), 0) nowSecs := fasttime.UnixTimestamp() currentTime := time.Unix(int64(nowSecs), 0) + storageTimeout := *searchutils.StorageTimeout + if storageTimeout > 0 { + dd := currentTime.Add(storageTimeout) + if dd.Sub(d) < 0 { + // Limit the remote deadline to storageTimeout, + // so slow vmstorage nodes may stop processing the request. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711 . + // The local deadline remains the same, so data obtained from + // the remaining vmstorage nodes could be processed locally. + d = dd + } + } timeout := d.Sub(currentTime) if timeout <= 0 { - return fmt.Errorf("request timeout reached: %s", deadline.String()) + return fmt.Errorf("request timeout reached: %s or -search.storageTimeout=%s", deadline.String(), storageTimeout.String()) } bc, err := sn.connPool.Get() if err != nil { diff --git a/app/vmselect/searchutils/searchutils.go b/app/vmselect/searchutils/searchutils.go index 084cb73935..aa9140fd5a 100644 --- a/app/vmselect/searchutils/searchutils.go +++ b/app/vmselect/searchutils/searchutils.go @@ -15,9 +15,15 @@ import ( var ( maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call") - maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution") + maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution; see also -search.storageTimeout") denyPartialResponse = flag.Bool("search.denyPartialResponse", false, "Whether to deny partial responses if a part of -storageNode instances fail to perform queries; "+ - "this trades availability over consistency; see also -search.maxQueryDuration") + "this trades availability over consistency; see also -search.maxQueryDuration and -search.storageTimeout") + + // StorageTimeout limits the duration of query execution on every vmstorage node. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711 + StorageTimeout = flag.Duration("search.storageTimeout", 0, "The timeout for per-storage query processing; "+ + "this allows returning partial responses if certain -storageNode instances slowly process the query; "+ + "see also -search.maxQueryDuration and -search.denyPartialResponse command-line flags") ) // GetTime returns time from the given argKey query arg.