promscrape/discovery: support kubeconfig (#2533)

lib/promauth/config.go

@@ -68,8 +68,11 @@ func (s *Secret) String() string {
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config
 type TLSConfig struct {
+	CA                 []byte `yaml:"ca,omitempty"`
 	CAFile             string `yaml:"ca_file,omitempty"`
+	Cert               []byte `yaml:"cert,omitempty"`
 	CertFile           string `yaml:"cert_file,omitempty"`
+	Key                []byte `yaml:"key,omitempty"`
 	KeyFile            string `yaml:"key_file,omitempty"`
 	ServerName         string `yaml:"server_name,omitempty"`
 	InsecureSkipVerify bool   `yaml:"insecure_skip_verify,omitempty"`
@@ -456,9 +459,10 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 	if tlsConfig != nil {
 		tlsServerName = tlsConfig.ServerName
 		tlsInsecureSkipVerify = tlsConfig.InsecureSkipVerify
-		if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" {
+		if (tlsConfig.CertFile != "" || tlsConfig.KeyFile != "") || (len(tlsConfig.Key) != 0 || len(tlsConfig.Cert) != 0) {
 			getTLSCert = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
 				// Re-read TLS certificate from disk. This is needed for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420
+				if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" {
 					certPath := fs.GetFilepath(baseDir, tlsConfig.CertFile)
 					keyPath := fs.GetFilepath(baseDir, tlsConfig.KeyFile)
 					cert, err := tls.LoadX509KeyPair(certPath, keyPath)
@@ -466,6 +470,13 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 						return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err)
 					}
 					return &cert, nil
+				} else {
+					cert, err := tls.X509KeyPair(tlsConfig.Cert, tlsConfig.Key)
+					if err != nil {
+						return nil, fmt.Errorf("cannot load TLS certificate: %w", err)
+					}
+					return &cert, nil
+				}
 			}
 			// Check whether the configured TLS cert can be loaded.
 			if _, err := getTLSCert(nil); err != nil {
@@ -479,8 +490,11 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 			if err != nil {
 				return nil, fmt.Errorf("cannot read `ca_file` %q: %w", tlsConfig.CAFile, err)
 			}
+			tlsConfig.CA = data
+		}
+		if len(tlsConfig.CA) != 0 {
 			tlsRootCA = x509.NewCertPool()
-			if !tlsRootCA.AppendCertsFromPEM(data) {
+			if !tlsRootCA.AppendCertsFromPEM(tlsConfig.CA) {
 				return nil, fmt.Errorf("cannot parse data from `ca_file` %q", tlsConfig.CAFile)
 			}
 		}

lib/promscrape/discovery/kubernetes/api.go

@@ -1,7 +1,10 @@
 package kubernetes
 
 import (
+	"encoding/base64"
 	"fmt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"gopkg.in/yaml.v2"
 	"net"
 	"os"
 	"strings"
@@ -14,6 +17,159 @@ type apiConfig struct {
 	aw *apiWatcher
 }
 
+type Config struct {
+	Kind        string      `yaml:"kind,omitempty"`
+	APIVersion  string      `yaml:"apiVersion,omitempty"`
+	Preferences Preferences `yaml:"preferences"`
+	Clusters    []struct {
+		Name    string   `yaml:"name"`
+		Cluster *Cluster `yaml:"cluster"`
+	} `yaml:"clusters"`
+	AuthInfos []struct {
+		Name     string    `yaml:"name"`
+		AuthInfo *AuthInfo `yaml:"user"`
+	} `yaml:"users"`
+	Contexts []struct {
+		Name    string   `yaml:"name"`
+		Context *Context `yaml:"context"`
+	} `yaml:"contexts"`
+	CurrentContext string `yaml:"current-context"`
+}
+
+type Preferences struct {
+	Colors bool `yaml:"colors,omitempty"`
+}
+
+type Cluster struct {
+	LocationOfOrigin         string
+	Server                   string `yaml:"server"`
+	TLSServerName            string `yaml:"tls-server-name,omitempty"`
+	InsecureSkipTLSVerify    bool   `yaml:"insecure-skip-tls-verify,omitempty"`
+	CertificateAuthority     string `yaml:"certificate-authority,omitempty"`
+	CertificateAuthorityData string `yaml:"certificate-authority-data,omitempty"`
+	ProxyURL                 string `yaml:"proxy-url,omitempty"`
+}
+
+// AuthInfo contains information that describes identity information.  This is use to tell the kubernetes cluster who you are.
+type AuthInfo struct {
+	LocationOfOrigin      string
+	ClientCertificate     string   `yaml:"client-certificate,omitempty"`
+	ClientCertificateData string   `yaml:"client-certificate-data,omitempty"`
+	ClientKey             string   `yaml:"client-key,omitempty"`
+	ClientKeyData         string   `yaml:"client-key-data,omitempty"`
+	Token                 string   `yaml:"token,omitempty"`
+	TokenFile             string   `yaml:"tokenFile,omitempty"`
+	Impersonate           string   `yaml:"act-as,omitempty"`
+	ImpersonateUID        string   `yaml:"act-as-uid,omitempty"`
+	ImpersonateGroups     []string `yaml:"act-as-groups,omitempty"`
+	ImpersonateUserExtra  []string `yaml:"act-as-user-extra,omitempty"`
+	Username              string   `yaml:"username,omitempty"`
+	Password              string   `yaml:"password,omitempty"`
+}
+
+// Context is a tuple of references to a cluster (how do I communicate with a kubernetes cluster), a user (how do I identify myself), and a namespace (what subset of resources do I want to work with)
+type Context struct {
+	LocationOfOrigin string
+	Cluster          string `yaml:"cluster"`
+	AuthInfo         string `yaml:"user"`
+	Namespace        string `yaml:"namespace,omitempty"`
+}
+
+type ApiConfig struct {
+	basicAuth *promauth.BasicAuthConfig
+	server    string
+	token     string
+	tokenFile string
+	tlsConfig *promauth.TLSConfig
+}
+
+func buildConfig(sdc *SDConfig) (*ApiConfig, error) {
+
+	data, err := fs.ReadFileOrHTTP(sdc.KubeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read kubeConfig from %q: %w", sdc.KubeConfig, err)
+	}
+	var config Config
+	if err = yaml.Unmarshal(data, &config); err != nil {
+		return nil, fmt.Errorf("cannot parse %q: %w", sdc.KubeConfig, err)
+	}
+
+	var authInfos = make(map[string]*AuthInfo)
+	for _, obj := range config.AuthInfos {
+		authInfos[obj.Name] = obj.AuthInfo
+	}
+	var clusterInfos = make(map[string]*Cluster)
+	for _, obj := range config.Clusters {
+		clusterInfos[obj.Name] = obj.Cluster
+	}
+	var contexts = make(map[string]*Context)
+	for _, obj := range config.Contexts {
+		contexts[obj.Name] = obj.Context
+	}
+
+	contextName := config.CurrentContext
+	configContext, exists := contexts[contextName]
+	if !exists {
+		return nil, fmt.Errorf("context %q does not exist", contextName)
+	}
+
+	clusterInfoName := configContext.Cluster
+	configClusterInfo, exists := clusterInfos[clusterInfoName]
+	if !exists {
+		return nil, fmt.Errorf("cluster %q does not exist", clusterInfoName)
+	}
+
+	authInfoName := configContext.AuthInfo
+	configAuthInfo, exists := authInfos[authInfoName]
+	if authInfoName != "" && !exists {
+		return nil, fmt.Errorf("auth info %q does not exist", authInfoName)
+	}
+
+	var apiConfig ApiConfig
+
+	apiConfig.tlsConfig = &promauth.TLSConfig{
+		CAFile:             configClusterInfo.CertificateAuthority,
+		ServerName:         configClusterInfo.TLSServerName,
+		InsecureSkipVerify: configClusterInfo.InsecureSkipTLSVerify,
+	}
+
+	if len(configClusterInfo.CertificateAuthorityData) != 0 {
+		apiConfig.tlsConfig.CA, err = base64.StdEncoding.DecodeString(configClusterInfo.CertificateAuthorityData)
+		if err != nil {
+			return nil, fmt.Errorf("cannot base64-decode configClusterInfo.CertificateAuthorityData %q: %w", configClusterInfo.CertificateAuthorityData, err)
+		}
+	}
+
+	if configAuthInfo != nil {
+		apiConfig.tlsConfig.CertFile = configAuthInfo.ClientCertificate
+		apiConfig.tlsConfig.KeyFile = configAuthInfo.ClientKey
+		apiConfig.token = configAuthInfo.Token
+		apiConfig.tokenFile = configAuthInfo.TokenFile
+		if len(configAuthInfo.ClientCertificateData) != 0 {
+			apiConfig.tlsConfig.Cert, err = base64.StdEncoding.DecodeString(configAuthInfo.ClientCertificateData)
+			if err != nil {
+				return nil, fmt.Errorf("cannot base64-decode configAuthInfo.ClientCertificateData %q: %w", configClusterInfo.CertificateAuthorityData, err)
+			}
+		}
+		if len(configAuthInfo.ClientKeyData) != 0 {
+			apiConfig.tlsConfig.Key, err = base64.StdEncoding.DecodeString(configAuthInfo.ClientKeyData)
+			if err != nil {
+				return nil, fmt.Errorf("cannot base64-decode configAuthInfo.ClientKeyData %q: %w", configClusterInfo.CertificateAuthorityData, err)
+			}
+		}
+		if len(configAuthInfo.Username) > 0 || len(configAuthInfo.Password) > 0 {
+			apiConfig.basicAuth = &promauth.BasicAuthConfig{
+				Username: configAuthInfo.Username,
+				Password: promauth.NewSecret(configAuthInfo.Password),
+			}
+		}
+	}
+
+	apiConfig.server = configClusterInfo.Server
+
+	return &apiConfig, nil
+}
+
 func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFunc) (*apiConfig, error) {
 	role := sdc.role()
 	switch role {
@@ -26,6 +182,20 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}
 	apiServer := sdc.APIServer
+
+	if len(sdc.KubeConfig) != 0 {
+		config, err := buildConfig(sdc)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse kube config: %w", err)
+		}
+		acNew, err := promauth.NewConfig(".", nil, config.basicAuth, config.token, config.tokenFile, nil, config.tlsConfig)
+		if err != nil {
+			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
+		}
+		ac = acNew
+		apiServer = config.server
+	}
+
 	if len(apiServer) == 0 {
 		// Assume we run at k8s pod.
 		// Discover apiServer and auth config according to k8s docs.

lib/promscrape/discovery/kubernetes/api_test.go

@@ -0,0 +1,68 @@
+package kubernetes
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"reflect"
+	"testing"
+)
+
+func TestParseKubeConfig(t *testing.T) {
+
+	type testCase struct {
+		name           string
+		sdc            *SDConfig
+		expectedConfig *ApiConfig
+	}
+
+	var cases = []testCase{
+		{
+			name: "token",
+			sdc: &SDConfig{
+				KubeConfig: "testdata/kubeconfig_token.yaml",
+			},
+			expectedConfig: &ApiConfig{
+				token:     "abc",
+				tlsConfig: &promauth.TLSConfig{},
+			},
+		},
+		{
+			name: "cert",
+			sdc: &SDConfig{
+				KubeConfig: "testdata/kubeconfig_cert.yaml",
+			},
+			expectedConfig: &ApiConfig{
+				server: "localhost:8000",
+				tlsConfig: &promauth.TLSConfig{
+					CA:   []byte("authority"),
+					Cert: []byte("certificate"),
+					Key:  []byte("key"),
+				},
+			},
+		},
+		{
+			name: "basic",
+			sdc: &SDConfig{
+				KubeConfig: "testdata/kubeconfig_basic.yaml",
+			},
+			expectedConfig: &ApiConfig{
+				basicAuth: &promauth.BasicAuthConfig{
+					Password: promauth.NewSecret("secret"),
+					Username: "user1",
+				},
+				tlsConfig: &promauth.TLSConfig{},
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ac, err := buildConfig(tc.sdc)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(ac, tc.expectedConfig) {
+				t.Fatalf("unexpected result, got: %v, want: %v", ac, tc.expectedConfig)
+			}
+		})
+	}
+
+}

lib/promscrape/discovery/kubernetes/kubernetes.go

@@ -22,6 +22,7 @@ type SDConfig struct {
 
 	// Use role() function for accessing the Role field
 	Role       string `yaml:"role"`
+	KubeConfig string `yaml:"kubeconfig_file"`
 
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
 	ProxyURL         *proxy.URL                `yaml:"proxy_url,omitempty"`

lib/promscrape/discovery/kubernetes/testdata/kubeconfig_basic.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+clusters:
+  - cluster:
+      server: ""
+    name: k8s
+contexts:
+  - context:
+      cluster: k8s
+      user: user1
+    name: user1@k8s
+current-context: user1@k8s
+kind: Config
+preferences: {}
+users:
+  - name: user1
+    user:
+      username: user1
+      password: secret

lib/promscrape/discovery/kubernetes/testdata/kubeconfig_cert.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+clusters:
+  - cluster:
+      certificate-authority-data: YXV0aG9yaXR5
+      server: localhost:8000
+    name: k8s
+contexts:
+  - context:
+      cluster: k8s
+      user: user1
+    name: user1@k8s
+current-context: user1@k8s
+kind: Config
+preferences: {}
+users:
+  - name: user1
+    user:
+      client-certificate-data: Y2VydGlmaWNhdGU=
+      client-key-data: a2V5

lib/promscrape/discovery/kubernetes/testdata/kubeconfig_token.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+clusters:
+  - cluster:
+      server: ""
+    name: k8s
+contexts:
+  - context:
+      cluster: k8s
+      user: user1
+    name: user1@k8s
+current-context: user1@k8s
+kind: Config
+preferences: {}
+users:
+  - name: user1
+    user:
+      token: abc