lib/httpserver: print full requestURI in httpserver.Errorf

This should simplify debugging.

app/vmagent/main.go

@@ -165,7 +165,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -174,7 +174,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(r); err != nil {
 			vmimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -183,7 +183,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(r); err != nil {
 			csvimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -192,7 +192,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		prometheusimportRequests.Inc()
 		if err := prometheusimport.InsertHandler(r); err != nil {
 			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -201,7 +201,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(r); err != nil {
 			nativeimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -210,7 +210,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(r); err != nil {
 			influxWriteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)

app/vmalert/web.go

@@ -34,7 +34,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/groups":
 		data, err := rh.listGroups()
 		if err != nil {
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
@@ -43,7 +43,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/alerts":
 		data, err := rh.listAlerts()
 		if err != nil {
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
@@ -61,7 +61,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		// /api/v1/<groupName>/<alertID>/status
 		data, err := rh.alert(r.URL.Path)
 		if err != nil {
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")

app/vminsert/main.go

@@ -181,7 +181,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -190,7 +190,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(at, r); err != nil {
 			vmimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -199,7 +199,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(at, r); err != nil {
 			csvimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -208,7 +208,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		prometheusimportRequests.Inc()
 		if err := prometheusimport.InsertHandler(at, r); err != nil {
 			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -217,7 +217,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(at, r); err != nil {
 			nativeimportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -226,7 +226,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(at, r); err != nil {
 			influxWriteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)

app/vmselect/main.go

@@ -174,7 +174,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			d := time.Since(actualStartTime)
 			if d >= *logSlowQueryDuration {
 				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-				requestURI := getRequestURI(r)
+				requestURI := httpserver.GetRequestURI(r)
 				logger.Warnf("slow query according to -search.logSlowQueryDuration=%s: remoteAddr=%s, duration=%.3f seconds; requestURI: %q",
 					*logSlowQueryDuration, remoteAddr, d.Seconds(), requestURI)
 				slowQueries.Inc()
@@ -247,7 +247,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		graphiteTagValuesRequests.Inc()
 		if err := graphite.TagValuesHandler(startTime, at, tagName, w, r); err != nil {
 			graphiteTagValuesErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -332,7 +332,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		exportRequests.Inc()
 		if err := prometheus.ExportHandler(startTime, at, w, r); err != nil {
 			exportErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -340,7 +340,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		exportNativeRequests.Inc()
 		if err := prometheus.ExportNativeHandler(startTime, at, w, r); err != nil {
 			exportNativeErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -348,7 +348,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		exportCSVRequests.Inc()
 		if err := prometheus.ExportCSVHandler(startTime, at, w, r); err != nil {
 			exportCSVErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -356,7 +356,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		federateRequests.Inc()
 		if err := prometheus.FederateHandler(startTime, at, w, r); err != nil {
 			federateErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -365,7 +365,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		httpserver.EnableCORS(w, r)
 		if err := graphite.MetricsFindHandler(startTime, at, w, r); err != nil {
 			graphiteMetricsFindErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -374,7 +374,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		httpserver.EnableCORS(w, r)
 		if err := graphite.MetricsExpandHandler(startTime, at, w, r); err != nil {
 			graphiteMetricsExpandErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -383,7 +383,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		httpserver.EnableCORS(w, r)
 		if err := graphite.MetricsIndexHandler(startTime, at, w, r); err != nil {
 			graphiteMetricsIndexErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -391,7 +391,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		graphiteTagsTagSeriesRequests.Inc()
 		if err := graphite.TagsTagSeriesHandler(startTime, at, w, r); err != nil {
 			graphiteTagsTagSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -399,7 +399,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		graphiteTagsTagMultiSeriesRequests.Inc()
 		if err := graphite.TagsTagMultiSeriesHandler(startTime, at, w, r); err != nil {
 			graphiteTagsTagMultiSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -407,7 +407,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		graphiteTagsRequests.Inc()
 		if err := graphite.TagsHandler(startTime, at, w, r); err != nil {
 			graphiteTagsErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -415,7 +415,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		graphiteTagsFindSeriesRequests.Inc()
 		if err := graphite.TagsFindSeriesHandler(startTime, at, w, r); err != nil {
 			graphiteTagsFindSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -424,7 +424,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		httpserver.EnableCORS(w, r)
 		if err := graphite.TagsAutoCompleteTagsHandler(startTime, at, w, r); err != nil {
 			graphiteTagsAutoCompleteTagsErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -433,7 +433,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		httpserver.EnableCORS(w, r)
 		if err := graphite.TagsAutoCompleteValuesHandler(startTime, at, w, r); err != nil {
 			graphiteTagsAutoCompleteValuesErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -441,7 +441,7 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		graphiteTagsDelSeriesRequests.Inc()
 		if err := graphite.TagsDelSeriesHandler(startTime, at, w, r); err != nil {
 			graphiteTagsDelSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		return true
@@ -480,7 +480,7 @@ func deleteHandler(startTime time.Time, w http.ResponseWriter, r *http.Request,
 		deleteRequests.Inc()
 		if err := prometheus.DeleteHandler(startTime, at, r); err != nil {
 			deleteErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
@@ -503,7 +503,7 @@ func isGraphiteTagsPath(path string) bool {
 }
 
 func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
-	logger.Warnf("error in %q: %s", r.RequestURI, err)
+	logger.Warnf("error in %q: %s", httpserver.GetRequestURI(r), err)
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	statusCode := http.StatusUnprocessableEntity
@@ -515,23 +515,6 @@ func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 	prometheus.WriteErrorResponse(w, statusCode, err)
 }
 
-func getRequestURI(r *http.Request) string {
-	requestURI := r.RequestURI
-	if r.Method != "POST" {
-		return requestURI
-	}
-	_ = r.ParseForm()
-	queryArgs := r.PostForm.Encode()
-	if len(queryArgs) == 0 {
-		return requestURI
-	}
-	delimiter := "?"
-	if strings.Contains(requestURI, delimiter) {
-		delimiter = "&"
-	}
-	return requestURI + delimiter + queryArgs
-}
-
 var (
 	labelValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/label/{}/values"}`)
 	labelValuesErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/prometheus/api/v1/label/{}/values"}`)

docs/CHANGELOG.md

@@ -8,6 +8,7 @@ sort: 15
 
 * FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
 * FEATURE: reduce memory usage when performing heavy queries over big number of time series.
+* FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors.
 
 * BUGFIX: vmagent: remove `{ %space %}` typo in `/targets` output. The typo has been introduced in v1.62.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1408).
 * BUGFIX: vmagent: fix CSS styles on `/targets` page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1422).

lib/httpserver/httpserver.go

@@ -539,7 +539,8 @@ func GetQuotedRemoteAddr(r *http.Request) string {
 func Errorf(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
 	errStr := fmt.Sprintf(format, args...)
 	remoteAddr := GetQuotedRemoteAddr(r)
-	errStr = fmt.Sprintf("remoteAddr: %s; %s", remoteAddr, errStr)
+	requestURI := GetRequestURI(r)
+	errStr = fmt.Sprintf("remoteAddr: %s; requestURI: %s; %s", remoteAddr, requestURI, errStr)
 	logger.WarnfSkipframes(1, "%s", errStr)
 
 	// Extract statusCode from args
@@ -600,3 +601,21 @@ func WriteAPIHelp(w io.Writer, pathList [][2]string) {
 		fmt.Fprintf(w, "<a href=%q>%s</a> - %s<br/>", p, p, doc)
 	}
 }
+
+// GetRequestURI returns requestURI for r.
+func GetRequestURI(r *http.Request) string {
+	requestURI := r.RequestURI
+	if r.Method != "POST" {
+		return requestURI
+	}
+	_ = r.ParseForm()
+	queryArgs := r.PostForm.Encode()
+	if len(queryArgs) == 0 {
+		return requestURI
+	}
+	delimiter := "?"
+	if strings.Contains(requestURI, delimiter) {
+		delimiter = "&"
+	}
+	return requestURI + delimiter + queryArgs
+}

lib/ingestserver/opentsdbhttp/server.go

@@ -89,7 +89,7 @@ func newRequestHandler(insertHandler func(r *http.Request) error) http.Handler {
 		writeRequests.Inc()
 		if err := insertHandler(r); err != nil {
 			writeErrors.Inc()
-			httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, r, "%s", err)
 			return
 		}
 		w.WriteHeader(http.StatusNoContent)