From 2ca92aaa34c9266be3151e195e500b28bfad43dc Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Tue, 18 Apr 2023 16:47:26 +0400 Subject: [PATCH] lib/httpserver: add handler to serve `/robots.txt` and deny search indexing (#4143) This handler will instruct search engines that indexing is not allowed for the content exposed to the internet. This should help to address issues like #4128 when instances are exposed to the internet without authentication. --- docs/CHANGELOG.md | 2 ++ lib/httpserver/httpserver.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fd1ef8d45..528b0003f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## v1.79.x long-time support release (LTS) +* SECURITY: serve `/robots.txt` content to disallow indexing of the exposed instances by search engines. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4128) for details. + ## [v1.79.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.12) Released at 2023-04-06 diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index 43f8bc49f..7a219348a 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -316,6 +316,11 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1833 fmt.Fprintf(w, "VictoriaMetrics is Ready.\n") return + case "/robots.txt": + // This prevents search engines from indexing contents + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4128 + fmt.Fprintf(w, "User-agent: *\nDisallow: /\n") + return default: if strings.HasPrefix(r.URL.Path, "/debug/pprof/") { pprofRequests.Inc()