From 47a038401b703d2c2648413c93c51782288112a0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 13 Nov 2020 10:25:39 +0200 Subject: [PATCH] all: consistently return text-based HTTP responses with charset=utf-8 This is a follow-up for https://github.com/VictoriaMetrics/VictoriaMetrics/pull/897 --- app/vmagent/main.go | 2 +- app/vmalert/datasource/vm.go | 2 +- app/vmalert/notifier/alertmanager.go | 2 +- app/vmalert/web.go | 6 +++--- app/vminsert/main.go | 6 +++--- app/vmselect/graphite/graphite.go | 12 ++++++------ app/vmselect/main.go | 8 ++++---- app/vmselect/prometheus/prometheus.go | 24 ++++++++++++------------ app/vmstorage/main.go | 8 ++++---- docs/CHANGELOG.md | 2 ++ lib/httpserver/httpserver.go | 6 +++--- 11 files changed, 40 insertions(+), 38 deletions(-) diff --git a/app/vmagent/main.go b/app/vmagent/main.go index 1f2e28249..04c6cd8b2 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -228,7 +228,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy) http.Error(w, errMsg, http.StatusTooEarly) } else { - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) } diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go index bf57a04be..7e0bc4ce4 100644 --- a/app/vmalert/datasource/vm.go +++ b/app/vmalert/datasource/vm.go @@ -82,7 +82,7 @@ func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) { if err != nil { return nil, err } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", "application/json; charset=utf-8") if s.basicAuthPass != "" { req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass) } diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go index f7dd03615..a24f5f723 100644 --- a/app/vmalert/notifier/alertmanager.go +++ b/app/vmalert/notifier/alertmanager.go @@ -28,7 +28,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert) error { if err != nil { return err } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", "application/json; charset=utf-8") req = req.WithContext(ctx) if am.basicAuthPass != "" { req.SetBasicAuth(am.basicAuthUser, am.basicAuthPass) diff --git a/app/vmalert/web.go b/app/vmalert/web.go index dbd90fa9a..a095356d9 100644 --- a/app/vmalert/web.go +++ b/app/vmalert/web.go @@ -40,7 +40,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool { httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) return true } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Write(data) return true case "/api/v1/alerts": @@ -49,7 +49,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool { httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) return true } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Write(data) return true case "/-/reload": @@ -67,7 +67,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool { httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) return true } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Write(data) return true } diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 1ab37655d..e47132551 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -155,13 +155,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { return true case "/targets": promscrapeTargetsRequests.Inc() - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") showOriginalLabels, _ := strconv.ParseBool(r.FormValue("show_original_labels")) promscrape.WriteHumanReadableTargetsStatus(w, showOriginalLabels) return true case "/api/v1/targets": promscrapeAPIV1TargetsRequests.Inc() - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") state := r.FormValue("state") promscrape.WriteAPIV1Targets(w, state) return true @@ -175,7 +175,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { errMsg := fmt.Sprintf("waiting for scrape config to init targets, configs left: %d", rdy) http.Error(w, errMsg, http.StatusTooEarly) } else { - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) } diff --git a/app/vmselect/graphite/graphite.go b/app/vmselect/graphite/graphite.go index ac736fa51..6578b7938 100644 --- a/app/vmselect/graphite/graphite.go +++ b/app/vmselect/graphite/graphite.go @@ -84,9 +84,9 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ } paths = deduplicatePaths(paths, delimiter) sortPaths(paths, delimiter) - contentType := "application/json" + contentType := "application/json; charset=utf-8" if jsonp != "" { - contentType = "text/javascript" + contentType = "text/javascript; charset=utf-8" } w.Header().Set("Content-Type", contentType) bw := bufferedwriter.Get(w) @@ -166,9 +166,9 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re } m[query] = paths } - contentType := "application/json" + contentType := "application/json; charset=utf-8" if jsonp != "" { - contentType = "text/javascript" + contentType = "text/javascript; charset=utf-8" } w.Header().Set("Content-Type", contentType) if groupByExpr { @@ -215,9 +215,9 @@ func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Req if err != nil { return fmt.Errorf(`cannot obtain metric names: %w`, err) } - contentType := "application/json" + contentType := "application/json; charset=utf-8" if jsonp != "" { - contentType = "text/javascript" + contentType = "text/javascript; charset=utf-8" } w.Header().Set("Content-Type", contentType) bw := bufferedwriter.Get(w) diff --git a/app/vmselect/main.go b/app/vmselect/main.go index d184eba77..1b2fb9a05 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -262,19 +262,19 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { case "/api/v1/rules": // Return dumb placeholder rulesRequests.Inc() - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`) return true case "/api/v1/alerts": // Return dumb placehloder alertsRequests.Inc() - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`) return true case "/api/v1/metadata": // Return dumb placeholder metadataRequests.Inc() - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`) return true case "/api/v1/admin/tsdb/delete_series": @@ -299,7 +299,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) { logger.Warnf("error in %q: %s", r.RequestURI, err) - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") statusCode := http.StatusUnprocessableEntity var esc *httpserver.ErrorWithStatusCode if errors.As(err, &esc) { diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index cdee46c67..05f316f28 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -88,7 +88,7 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request return fmt.Errorf("cannot fetch data for %q: %w", sq, err) } - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) error { @@ -151,7 +151,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques MaxTimestamp: end, TagFilterss: tagFilterss, } - w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Type", "text/csv; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -331,9 +331,9 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo WriteExportJSONLine(bb, xb) resultsCh <- bb } - contentType := "application/stream+json" + contentType := "application/stream+json; charset=utf-8" if format == "prometheus" { - contentType = "text/plain" + contentType = "text/plain; charset=utf-8" writeLineFunc = func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) { bb := quicktemplate.AcquireByteBuffer() WriteExportPrometheusLine(bb, xb) @@ -561,7 +561,7 @@ func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWr } } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteLabelValuesResponse(bw, labelValues) @@ -639,7 +639,7 @@ func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ if err != nil { return fmt.Errorf(`cannot obtain label entries: %w`, err) } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteLabelsCountResponse(bw, labelEntries) @@ -690,7 +690,7 @@ func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque if err != nil { return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err) } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteTSDBStatusResponse(bw, status) @@ -760,7 +760,7 @@ func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) } } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteLabelsResponse(bw, labels) @@ -826,7 +826,7 @@ func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ if err != nil { return fmt.Errorf("cannot obtain series count: %w", err) } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteSeriesCountResponse(bw, n) @@ -883,7 +883,7 @@ func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) return fmt.Errorf("cannot fetch data for %q: %w", sq, err) } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) resultsCh := make(chan *quicktemplate.ByteBuffer) @@ -1020,7 +1020,7 @@ func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) e } } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteQueryResponse(bw, result) @@ -1119,7 +1119,7 @@ func queryRangeHandler(startTime time.Time, w http.ResponseWriter, query string, // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153 result = removeEmptyValuesAndTimeseries(result) - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) WriteQueryRangeResponse(bw, result) diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index 25de22f46..6ccf50dd6 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -251,7 +251,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { switch path { case "/create": - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") snapshotPath, err := Storage.CreateSnapshot() if err != nil { err = fmt.Errorf("cannot create snapshot: %w", err) @@ -265,7 +265,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { } return true case "/list": - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") snapshots, err := Storage.ListSnapshots() if err != nil { err = fmt.Errorf("cannot list snapshots: %w", err) @@ -282,7 +282,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { fmt.Fprintf(w, `]}`) return true case "/delete": - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") snapshotName := r.FormValue("snapshot") if err := Storage.DeleteSnapshot(snapshotName); err != nil { err = fmt.Errorf("cannot delete snapshot %q: %w", snapshotName, err) @@ -292,7 +292,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { fmt.Fprintf(w, `{"status":"ok"}`) return true case "/delete_all": - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") snapshots, err := Storage.ListSnapshots() if err != nil { err = fmt.Errorf("cannot list snapshots: %w", err) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 759deea64..e93b9661a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,8 @@ * FEATURE: add `-search.treatDotsAsIsInRegexps` command-line flag, which can be used for automatic escaping of dots in regexp label filters used in queries. For example, if `-search.treatDotsAsIsInRegexps` is set, then the query `foo{bar=~"aaa.bb.cc|dd.eee"}` is automatically converted to `foo{bar=~"aaa\\.bb\\.cc|dd\\.eee"}`. This may be useful for querying Graphite data. +* FEATURE: consistently return text-based HTTP responses such as `plain/text` and `application/json` with `charset=utf-8`. + See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/897 * BUGFIX: do not return data points in the end of the selected time range for time series ending in the middle of the selected time range. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/887 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/845 diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index 1cf52c77a..daa8c8e59 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -208,7 +208,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques r.URL.Path = path switch r.URL.Path { case "/health": - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") deadline := atomic.LoadInt64(&s.shutdownDelayDeadline) if deadline <= 0 { w.Write([]byte("OK")) @@ -244,7 +244,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques return } startTime := time.Now() - w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") WritePrometheusMetrics(w) metricsHandlerDuration.UpdateDuration(startTime) return @@ -395,7 +395,7 @@ func (zrw *gzipResponseWriter) Write(p []byte) (int, error) { if h.Get("Content-Type") == "" { // Disable auto-detection of content-type, since it // is incorrectly detected after the compression. - h.Set("Content-Type", "text/html") + h.Set("Content-Type", "text/html; charset=utf-8") } } zrw.writeHeader()