all: return 503 http error if service is temporarily unavailable

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/156

app/vminsert/concurrencylimiter/concurrencylimiter.go

@@ -3,9 +3,11 @@ package concurrencylimiter
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"runtime"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -53,7 +55,10 @@ func Do(f func() error) error {
 	case <-t.C:
 		timerpool.Put(t)
 		concurrencyLimitTimeout.Inc()
-		return fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch))
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch)),
+			StatusCode: http.StatusServiceUnavailable,
+		}
 	}
 }
 

app/vminsert/netstorage/insert_ctx.go

@@ -2,11 +2,13 @@ package netstorage
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consts"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	xxhash "github.com/cespare/xxhash/v2"
@@ -33,7 +35,10 @@ func (br *bufRows) pushTo(sn *storageNode) error {
 	br.buf = br.buf[:0]
 	br.rows = 0
 	if err != nil {
-		return fmt.Errorf("cannot send %d bytes to storageNode %q: %s", bufLen, sn.dialer.Addr(), err)
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot send %d bytes to storageNode %q: %s", bufLen, sn.dialer.Addr(), err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
 	}
 	return nil
 }

app/vminsert/opentsdbhttp/request_handler.go

@@ -107,7 +107,9 @@ func insertHandlerInternal(at *auth.Token, req *http.Request, maxSize int64) err
 			tag := &r.Tags[j]
 			ic.AddLabel(tag.Key, tag.Value)
 		}
-		ic.WriteDataPoint(at, ic.Labels, r.Timestamp, r.Value)
+		if err := ic.WriteDataPoint(at, ic.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
 	}
 	rowsInserted.Get(at).Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))

app/vmselect/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"runtime"
 	"strings"
@@ -108,7 +109,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		case <-t.C:
 			timerpool.Put(t)
 			concurrencyLimitTimeout.Inc()
-			httpserver.Errorf(w, "cannot handle more than %d concurrent requests", cap(concurrencyCh))
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("cannot handle more than %d concurrent requests", cap(concurrencyCh)),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, "%s", err)
 			return true
 		}
 	}
@@ -252,7 +257,10 @@ func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 	logger.Errorf("error in %q: %s", r.URL.Path, err)
 
 	w.Header().Set("Content-Type", "application/json")
-	statusCode := 422
+	statusCode := http.StatusUnprocessableEntity
+	if esc, ok := err.(*httpserver.ErrorWithStatusCode); ok {
+		statusCode = esc.StatusCode
+	}
 	w.WriteHeader(statusCode)
 	prometheus.WriteErrorResponse(w, statusCode, err)
 }

lib/httpserver/httpserver.go

@@ -357,7 +357,29 @@ var (
 func Errorf(w http.ResponseWriter, format string, args ...interface{}) {
 	errStr := fmt.Sprintf(format, args...)
 	logger.Errorf("%s", errStr)
-	http.Error(w, errStr, http.StatusBadRequest)
+
+	// Extract statusCode from args
+	statusCode := http.StatusBadRequest
+	for _, arg := range args {
+		if esc, ok := arg.(*ErrorWithStatusCode); ok {
+			statusCode = esc.StatusCode
+			break
+		}
+	}
+	http.Error(w, errStr, statusCode)
+}
+
+// ErrorWithStatusCode is error with HTTP status code.
+//
+// The given StatusCode is sent to client when the error is passed to Errorf.
+type ErrorWithStatusCode struct {
+	Err        error
+	StatusCode int
+}
+
+// Error implements error interface.
+func (e *ErrorWithStatusCode) Error() string {
+	return e.Err.Error()
 }
 
 func isTrivialNetworkError(err error) bool {