vmagent:scrape config support enable_http2 (#4295)

app/vmagent: support `enable_http2` in scrape config

This change adds HTTP2 support for scrape config
and improves compatibility with Prometheus config.

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4283

(cherry picked from commit 72c3cd47eb)

lib/promauth/config.go

@@ -126,6 +126,9 @@ type HTTPClientConfig struct {
 
 	// FollowRedirects specifies whether the client should follow HTTP 3xx redirects.
 	FollowRedirects *bool `yaml:"follow_redirects,omitempty"`
+
+	// EnableHTTP2 specifies whether the client should configure HTTP2.
+	EnableHTTP2 *bool `yaml:"enable_http2,omitempty"`
 }
 
 // ProxyClientConfig represents proxy client config.

lib/promscrape/client.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/net/http2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -160,6 +162,13 @@ func newClient(ctx context.Context, sw *ScrapeWork) *client {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047
 		Timeout: 30 * sw.ScrapeTimeout,
 	}
+	if sw.EnableHTTP2 {
+		_, err := http2.ConfigureTransports(sc.Transport.(*http.Transport))
+		if err != nil {
+			logger.Errorf("failed to configure HTTP/2 transport: %s", err)
+		}
+	}
+
 	if sw.DenyRedirects {
 		sc.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse

lib/promscrape/config.go

@@ -992,6 +992,10 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 	if sc.HTTPClientConfig.FollowRedirects != nil {
 		denyRedirects = !*sc.HTTPClientConfig.FollowRedirects
 	}
+	enableHTTP2 := false
+	if sc.HTTPClientConfig.EnableHTTP2 != nil {
+		enableHTTP2 = !*sc.HTTPClientConfig.EnableHTTP2
+	}
 	metricsPath := sc.MetricsPath
 	if metricsPath == "" {
 		metricsPath = "/metrics"
@@ -1044,6 +1048,7 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		honorLabels:          honorLabels,
 		honorTimestamps:      honorTimestamps,
 		denyRedirects:        denyRedirects,
+		enableHTTP2:          enableHTTP2,
 		externalLabels:       externalLabels,
 		relabelConfigs:       relabelConfigs,
 		metricRelabelConfigs: metricRelabelConfigs,
@@ -1074,6 +1079,7 @@ type scrapeWorkConfig struct {
 	honorLabels          bool
 	honorTimestamps      bool
 	denyRedirects        bool
+	enableHTTP2          bool
 	externalLabels       *promutils.Labels
 	relabelConfigs       *promrelabel.ParsedConfigs
 	metricRelabelConfigs *promrelabel.ParsedConfigs
@@ -1351,6 +1357,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
 		HonorLabels:          swc.honorLabels,
 		HonorTimestamps:      swc.honorTimestamps,
 		DenyRedirects:        swc.denyRedirects,
+		EnableHTTP2:          swc.enableHTTP2,
 		OriginalLabels:       originalLabels,
 		Labels:               labelsCopy,
 		ExternalLabels:       swc.externalLabels,

lib/promscrape/discovery/azure/api.go

@@ -110,7 +110,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, err
 	}
-	c, err := discoveryutils.NewClient(env.ResourceManagerEndpoint, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	c, err := discoveryutils.NewClient(env.ResourceManagerEndpoint, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create client for %q: %w", env.ResourceManagerEndpoint, err)
 	}
@@ -230,7 +230,7 @@ func getRefreshTokenFunc(sdc *SDConfig, ac, proxyAC *promauth.Config, env *cloud
 		return nil, fmt.Errorf("unsupported `authentication_method: %q` only `OAuth` and `ManagedIdentity` are supported", authenticationMethod)
 	}
 
-	authClient, err := discoveryutils.NewClient(tokenEndpoint, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	authClient, err := discoveryutils.NewClient(tokenEndpoint, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot build auth client: %w", err)
 	}

lib/promscrape/discovery/azure/machine_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
@@ -66,7 +67,7 @@ func TestGetVirtualMachinesSuccess(t *testing.T) {
 				}
 			}))
 			defer testServer.Close()
-			c, err := discoveryutils.NewClient(testServer.URL, nil, nil, nil, nil)
+			c, err := discoveryutils.NewClient(testServer.URL, nil, nil, nil, promauth.HTTPClientConfig{})
 			if err != nil {
 				t.Fatalf("unexpected error at client create: %s", err)
 			}

lib/promscrape/discovery/consul/api.go

@@ -80,7 +80,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discovery/consulagent/api.go

@@ -74,7 +74,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discovery/digitalocean/api.go

@@ -36,7 +36,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discovery/docker/api.go

@@ -50,7 +50,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(sdc.Host, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(sdc.Host, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", sdc.Host, err)
 	}

lib/promscrape/discovery/dockerswarm/api.go

@@ -49,7 +49,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(sdc.Host, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(sdc.Host, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", sdc.Host, err)
 	}

lib/promscrape/discovery/eureka/api.go

@@ -34,7 +34,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discovery/http/api.go

@@ -44,7 +44,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discovery/kuma/api.go

@@ -60,7 +60,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discovery/nomad/api.go

@@ -68,7 +68,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
 	}
-	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig.FollowRedirects)
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC, sdc.HTTPClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
 	}

lib/promscrape/discoveryutils/client.go

@@ -13,6 +13,8 @@ import (
 	"sync"
 	"time"
 
+	"golang.org/x/net/http2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -83,7 +85,7 @@ type HTTPClient struct {
 var defaultDialer = &net.Dialer{}
 
 // NewClient returns new Client for the given args.
-func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxyAC *promauth.Config, followRedirects *bool) (*Client, error) {
+func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxyAC *promauth.Config, httpCfg promauth.HTTPClientConfig) (*Client, error) {
 	u, err := url.Parse(apiServer)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse apiServer=%q: %w", apiServer, err)
@@ -139,7 +141,7 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 			ac.SetHeaders(req, true)
 		}
 	}
-	if followRedirects != nil && !*followRedirects {
+	if httpCfg.FollowRedirects != nil && !*httpCfg.FollowRedirects {
 		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
 		}
@@ -153,6 +155,12 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 			proxyURL.SetHeaders(proxyAC, req)
 		}
 	}
+	if httpCfg.EnableHTTP2 != nil && *httpCfg.EnableHTTP2 {
+		_, err := http2.ConfigureTransports(client.Transport.(*http.Transport))
+		if err != nil {
+			return nil, fmt.Errorf("failed to configure HTTP/2 transport: %s", err)
+		}
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 
 	c := &Client{

lib/promscrape/discoveryutils/client_test.go

@@ -0,0 +1,111 @@
+package discoveryutils
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
+)
+
+func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httptest.Server, error) {
+	testServer := httptest.NewUnstartedServer(http.HandlerFunc(handler))
+	testServer.Start()
+	return testServer, nil
+}
+
+func TestNewClientFromConfig(t *testing.T) {
+	allowed := true
+	notAllowed := false
+	newClientValidConfig := []struct {
+		httpCfg         promauth.HTTPClientConfig
+		handler         func(w http.ResponseWriter, r *http.Request)
+		expectedMessage string
+	}{
+		{
+			httpCfg: promauth.HTTPClientConfig{
+				FollowRedirects: &allowed,
+			},
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				switch r.URL.Path {
+				case "/redirected":
+					fmt.Fprint(w, "I'm here to serve you!!!")
+				default:
+					w.Header().Set("Location", "/redirected")
+					w.WriteHeader(http.StatusFound)
+					fmt.Fprint(w, "It should follow the redirect.")
+				}
+			},
+			expectedMessage: "I'm here to serve you!!!",
+		},
+		{
+			httpCfg: promauth.HTTPClientConfig{
+				FollowRedirects: &notAllowed,
+			},
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				switch r.URL.Path {
+				case "/redirected":
+					fmt.Fprint(w, "The redirection was followed.")
+				default:
+					w.Header().Set("Location", "/redirected")
+					w.WriteHeader(http.StatusFound)
+					fmt.Fprint(w, "I'm before redirect")
+				}
+			},
+			expectedMessage: "I'm before redirect",
+		},
+		{
+			httpCfg: promauth.HTTPClientConfig{
+				EnableHTTP2: &allowed,
+			},
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				switch r.URL.Path {
+				case "/redirected":
+					fmt.Fprint(w, "I'm here to serve you!!!")
+				default:
+					w.Header().Set("Location", "/redirected")
+					w.WriteHeader(http.StatusFound)
+					fmt.Fprint(w, "I'm before redirect")
+				}
+			},
+			expectedMessage: "I'm here to serve you!!!",
+		},
+	}
+
+	for _, validConfig := range newClientValidConfig {
+		testServer, err := newTestServer(validConfig.handler)
+		if err != nil {
+			t.Fatal(err.Error())
+		}
+		defer testServer.Close()
+
+		client, err := NewClient("http://0.0.0.0:1234", nil, &proxy.URL{}, nil, validConfig.httpCfg)
+		if err != nil {
+			t.Errorf("Can't create a client from this config: %+v", validConfig.httpCfg)
+			continue
+		}
+
+		response, err := client.client.client.Get(testServer.URL)
+		if err != nil {
+			t.Errorf("Can't connect to the test server using this config: %+v: %v", validConfig.httpCfg, err)
+			continue
+		}
+
+		message, err := io.ReadAll(response.Body)
+		response.Body.Close()
+		if err != nil {
+			t.Errorf("Can't read the server response body using this config: %+v", validConfig.httpCfg)
+			continue
+		}
+
+		trimMessage := strings.TrimSpace(string(message))
+		if validConfig.expectedMessage != trimMessage {
+			t.Errorf("The expected message (%s) differs from the obtained message (%s) using this config: %+v",
+				validConfig.expectedMessage, trimMessage, validConfig.httpCfg)
+		}
+	}
+}

lib/promscrape/scrapework.go

@@ -66,6 +66,9 @@ type ScrapeWork struct {
 	// Whether to deny redirects during requests to scrape config.
 	DenyRedirects bool
 
+	// Whether to configure HTTP2.
+	EnableHTTP2 bool
+
 	// OriginalLabels contains original labels before relabeling.
 	//
 	// These labels are needed for relabeling troubleshooting at /targets page.
@@ -399,7 +402,8 @@ func (sw *scrapeWork) mustSwitchToStreamParseMode(responseSize int) bool {
 
 // getTargetResponse() fetches response from sw target in the same way as when scraping the target.
 func (sw *scrapeWork) getTargetResponse() ([]byte, error) {
-	if *streamParse || sw.Config.StreamParse || sw.mustSwitchToStreamParseMode(sw.prevBodyLen) {
+	// use stream reader when stream mode enabled or http2 enabled
+	if *streamParse || sw.Config.StreamParse || sw.mustSwitchToStreamParseMode(sw.prevBodyLen) || sw.Config.EnableHTTP2 {
 		// Read the response in stream mode.
 		sr, err := sw.GetStreamReader()
 		if err != nil {

lib/promscrape/testdata/prometheus.yml

@@ -7,6 +7,7 @@ scrape_configs:
     honor_labels: true
     honor_timestamps: false
     follow_redirects: false
+    enable_http2: true
     static_configs:
       - targets: ["foo.bar", "aaa"]
         labels:
@@ -21,6 +22,7 @@ scrape_configs:
       - role: endpoints
         api_server: "https://localhost:1234"
         follow_redirects: true
+        enable_http2: true
         tls_config:
           cert_file: valid_cert_file
           key_file: valid_key_file