From 006b8c7534f97cf5379c9cea8292f9ca0c0b32d6 Mon Sep 17 00:00:00 2001 From: hadesy Date: Thu, 2 Jun 2022 02:34:00 +0800 Subject: [PATCH] promscrape/discovery: support kubeconfig (#2533) --- lib/promauth/config.go | 30 +++- lib/promscrape/discovery/kubernetes/api.go | 170 ++++++++++++++++++ .../discovery/kubernetes/api_test.go | 68 +++++++ .../discovery/kubernetes/kubernetes.go | 3 +- .../kubernetes/testdata/kubeconfig_basic.yaml | 18 ++ .../kubernetes/testdata/kubeconfig_cert.yaml | 19 ++ .../kubernetes/testdata/kubeconfig_token.yaml | 17 ++ 7 files changed, 316 insertions(+), 9 deletions(-) create mode 100644 lib/promscrape/discovery/kubernetes/api_test.go create mode 100644 lib/promscrape/discovery/kubernetes/testdata/kubeconfig_basic.yaml create mode 100644 lib/promscrape/discovery/kubernetes/testdata/kubeconfig_cert.yaml create mode 100644 lib/promscrape/discovery/kubernetes/testdata/kubeconfig_token.yaml diff --git a/lib/promauth/config.go b/lib/promauth/config.go index bcacf9d11..e96f01bb6 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -68,8 +68,11 @@ func (s *Secret) String() string { // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config type TLSConfig struct { + CA []byte `yaml:"ca,omitempty"` CAFile string `yaml:"ca_file,omitempty"` + Cert []byte `yaml:"cert,omitempty"` CertFile string `yaml:"cert_file,omitempty"` + Key []byte `yaml:"key,omitempty"` KeyFile string `yaml:"key_file,omitempty"` ServerName string `yaml:"server_name,omitempty"` InsecureSkipVerify bool `yaml:"insecure_skip_verify,omitempty"` @@ -456,16 +459,24 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be if tlsConfig != nil { tlsServerName = tlsConfig.ServerName tlsInsecureSkipVerify = tlsConfig.InsecureSkipVerify - if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" { + if (tlsConfig.CertFile != "" || tlsConfig.KeyFile != "") || (len(tlsConfig.Key) != 0 || len(tlsConfig.Cert) != 0) { getTLSCert = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { // Re-read TLS certificate from disk. This is needed for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420 - certPath := fs.GetFilepath(baseDir, tlsConfig.CertFile) - keyPath := fs.GetFilepath(baseDir, tlsConfig.KeyFile) - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err) + if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" { + certPath := fs.GetFilepath(baseDir, tlsConfig.CertFile) + keyPath := fs.GetFilepath(baseDir, tlsConfig.KeyFile) + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err) + } + return &cert, nil + } else { + cert, err := tls.X509KeyPair(tlsConfig.Cert, tlsConfig.Key) + if err != nil { + return nil, fmt.Errorf("cannot load TLS certificate: %w", err) + } + return &cert, nil } - return &cert, nil } // Check whether the configured TLS cert can be loaded. if _, err := getTLSCert(nil); err != nil { @@ -479,8 +490,11 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be if err != nil { return nil, fmt.Errorf("cannot read `ca_file` %q: %w", tlsConfig.CAFile, err) } + tlsConfig.CA = data + } + if len(tlsConfig.CA) != 0 { tlsRootCA = x509.NewCertPool() - if !tlsRootCA.AppendCertsFromPEM(data) { + if !tlsRootCA.AppendCertsFromPEM(tlsConfig.CA) { return nil, fmt.Errorf("cannot parse data from `ca_file` %q", tlsConfig.CAFile) } } diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go index 7d9c07b98..906470338 100644 --- a/lib/promscrape/discovery/kubernetes/api.go +++ b/lib/promscrape/discovery/kubernetes/api.go @@ -1,7 +1,10 @@ package kubernetes import ( + "encoding/base64" "fmt" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" + "gopkg.in/yaml.v2" "net" "os" "strings" @@ -14,6 +17,159 @@ type apiConfig struct { aw *apiWatcher } +type Config struct { + Kind string `yaml:"kind,omitempty"` + APIVersion string `yaml:"apiVersion,omitempty"` + Preferences Preferences `yaml:"preferences"` + Clusters []struct { + Name string `yaml:"name"` + Cluster *Cluster `yaml:"cluster"` + } `yaml:"clusters"` + AuthInfos []struct { + Name string `yaml:"name"` + AuthInfo *AuthInfo `yaml:"user"` + } `yaml:"users"` + Contexts []struct { + Name string `yaml:"name"` + Context *Context `yaml:"context"` + } `yaml:"contexts"` + CurrentContext string `yaml:"current-context"` +} + +type Preferences struct { + Colors bool `yaml:"colors,omitempty"` +} + +type Cluster struct { + LocationOfOrigin string + Server string `yaml:"server"` + TLSServerName string `yaml:"tls-server-name,omitempty"` + InsecureSkipTLSVerify bool `yaml:"insecure-skip-tls-verify,omitempty"` + CertificateAuthority string `yaml:"certificate-authority,omitempty"` + CertificateAuthorityData string `yaml:"certificate-authority-data,omitempty"` + ProxyURL string `yaml:"proxy-url,omitempty"` +} + +// AuthInfo contains information that describes identity information. This is use to tell the kubernetes cluster who you are. +type AuthInfo struct { + LocationOfOrigin string + ClientCertificate string `yaml:"client-certificate,omitempty"` + ClientCertificateData string `yaml:"client-certificate-data,omitempty"` + ClientKey string `yaml:"client-key,omitempty"` + ClientKeyData string `yaml:"client-key-data,omitempty"` + Token string `yaml:"token,omitempty"` + TokenFile string `yaml:"tokenFile,omitempty"` + Impersonate string `yaml:"act-as,omitempty"` + ImpersonateUID string `yaml:"act-as-uid,omitempty"` + ImpersonateGroups []string `yaml:"act-as-groups,omitempty"` + ImpersonateUserExtra []string `yaml:"act-as-user-extra,omitempty"` + Username string `yaml:"username,omitempty"` + Password string `yaml:"password,omitempty"` +} + +// Context is a tuple of references to a cluster (how do I communicate with a kubernetes cluster), a user (how do I identify myself), and a namespace (what subset of resources do I want to work with) +type Context struct { + LocationOfOrigin string + Cluster string `yaml:"cluster"` + AuthInfo string `yaml:"user"` + Namespace string `yaml:"namespace,omitempty"` +} + +type ApiConfig struct { + basicAuth *promauth.BasicAuthConfig + server string + token string + tokenFile string + tlsConfig *promauth.TLSConfig +} + +func buildConfig(sdc *SDConfig) (*ApiConfig, error) { + + data, err := fs.ReadFileOrHTTP(sdc.KubeConfig) + if err != nil { + return nil, fmt.Errorf("cannot read kubeConfig from %q: %w", sdc.KubeConfig, err) + } + var config Config + if err = yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("cannot parse %q: %w", sdc.KubeConfig, err) + } + + var authInfos = make(map[string]*AuthInfo) + for _, obj := range config.AuthInfos { + authInfos[obj.Name] = obj.AuthInfo + } + var clusterInfos = make(map[string]*Cluster) + for _, obj := range config.Clusters { + clusterInfos[obj.Name] = obj.Cluster + } + var contexts = make(map[string]*Context) + for _, obj := range config.Contexts { + contexts[obj.Name] = obj.Context + } + + contextName := config.CurrentContext + configContext, exists := contexts[contextName] + if !exists { + return nil, fmt.Errorf("context %q does not exist", contextName) + } + + clusterInfoName := configContext.Cluster + configClusterInfo, exists := clusterInfos[clusterInfoName] + if !exists { + return nil, fmt.Errorf("cluster %q does not exist", clusterInfoName) + } + + authInfoName := configContext.AuthInfo + configAuthInfo, exists := authInfos[authInfoName] + if authInfoName != "" && !exists { + return nil, fmt.Errorf("auth info %q does not exist", authInfoName) + } + + var apiConfig ApiConfig + + apiConfig.tlsConfig = &promauth.TLSConfig{ + CAFile: configClusterInfo.CertificateAuthority, + ServerName: configClusterInfo.TLSServerName, + InsecureSkipVerify: configClusterInfo.InsecureSkipTLSVerify, + } + + if len(configClusterInfo.CertificateAuthorityData) != 0 { + apiConfig.tlsConfig.CA, err = base64.StdEncoding.DecodeString(configClusterInfo.CertificateAuthorityData) + if err != nil { + return nil, fmt.Errorf("cannot base64-decode configClusterInfo.CertificateAuthorityData %q: %w", configClusterInfo.CertificateAuthorityData, err) + } + } + + if configAuthInfo != nil { + apiConfig.tlsConfig.CertFile = configAuthInfo.ClientCertificate + apiConfig.tlsConfig.KeyFile = configAuthInfo.ClientKey + apiConfig.token = configAuthInfo.Token + apiConfig.tokenFile = configAuthInfo.TokenFile + if len(configAuthInfo.ClientCertificateData) != 0 { + apiConfig.tlsConfig.Cert, err = base64.StdEncoding.DecodeString(configAuthInfo.ClientCertificateData) + if err != nil { + return nil, fmt.Errorf("cannot base64-decode configAuthInfo.ClientCertificateData %q: %w", configClusterInfo.CertificateAuthorityData, err) + } + } + if len(configAuthInfo.ClientKeyData) != 0 { + apiConfig.tlsConfig.Key, err = base64.StdEncoding.DecodeString(configAuthInfo.ClientKeyData) + if err != nil { + return nil, fmt.Errorf("cannot base64-decode configAuthInfo.ClientKeyData %q: %w", configClusterInfo.CertificateAuthorityData, err) + } + } + if len(configAuthInfo.Username) > 0 || len(configAuthInfo.Password) > 0 { + apiConfig.basicAuth = &promauth.BasicAuthConfig{ + Username: configAuthInfo.Username, + Password: promauth.NewSecret(configAuthInfo.Password), + } + } + } + + apiConfig.server = configClusterInfo.Server + + return &apiConfig, nil +} + func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFunc) (*apiConfig, error) { role := sdc.role() switch role { @@ -26,6 +182,20 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu return nil, fmt.Errorf("cannot parse auth config: %w", err) } apiServer := sdc.APIServer + + if len(sdc.KubeConfig) != 0 { + config, err := buildConfig(sdc) + if err != nil { + return nil, fmt.Errorf("cannot parse kube config: %w", err) + } + acNew, err := promauth.NewConfig(".", nil, config.basicAuth, config.token, config.tokenFile, nil, config.tlsConfig) + if err != nil { + return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err) + } + ac = acNew + apiServer = config.server + } + if len(apiServer) == 0 { // Assume we run at k8s pod. // Discover apiServer and auth config according to k8s docs. diff --git a/lib/promscrape/discovery/kubernetes/api_test.go b/lib/promscrape/discovery/kubernetes/api_test.go new file mode 100644 index 000000000..a012093c2 --- /dev/null +++ b/lib/promscrape/discovery/kubernetes/api_test.go @@ -0,0 +1,68 @@ +package kubernetes + +import ( + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "reflect" + "testing" +) + +func TestParseKubeConfig(t *testing.T) { + + type testCase struct { + name string + sdc *SDConfig + expectedConfig *ApiConfig + } + + var cases = []testCase{ + { + name: "token", + sdc: &SDConfig{ + KubeConfig: "testdata/kubeconfig_token.yaml", + }, + expectedConfig: &ApiConfig{ + token: "abc", + tlsConfig: &promauth.TLSConfig{}, + }, + }, + { + name: "cert", + sdc: &SDConfig{ + KubeConfig: "testdata/kubeconfig_cert.yaml", + }, + expectedConfig: &ApiConfig{ + server: "localhost:8000", + tlsConfig: &promauth.TLSConfig{ + CA: []byte("authority"), + Cert: []byte("certificate"), + Key: []byte("key"), + }, + }, + }, + { + name: "basic", + sdc: &SDConfig{ + KubeConfig: "testdata/kubeconfig_basic.yaml", + }, + expectedConfig: &ApiConfig{ + basicAuth: &promauth.BasicAuthConfig{ + Password: promauth.NewSecret("secret"), + Username: "user1", + }, + tlsConfig: &promauth.TLSConfig{}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ac, err := buildConfig(tc.sdc) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(ac, tc.expectedConfig) { + t.Fatalf("unexpected result, got: %v, want: %v", ac, tc.expectedConfig) + } + }) + } + +} diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go index 4be5aa783..edd69b860 100644 --- a/lib/promscrape/discovery/kubernetes/kubernetes.go +++ b/lib/promscrape/discovery/kubernetes/kubernetes.go @@ -21,7 +21,8 @@ type SDConfig struct { APIServer string `yaml:"api_server,omitempty"` // Use role() function for accessing the Role field - Role string `yaml:"role"` + Role string `yaml:"role"` + KubeConfig string `yaml:"kubeconfig_file"` HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` ProxyURL *proxy.URL `yaml:"proxy_url,omitempty"` diff --git a/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_basic.yaml b/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_basic.yaml new file mode 100644 index 000000000..a3fa85de3 --- /dev/null +++ b/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_basic.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +clusters: + - cluster: + server: "" + name: k8s +contexts: + - context: + cluster: k8s + user: user1 + name: user1@k8s +current-context: user1@k8s +kind: Config +preferences: {} +users: + - name: user1 + user: + username: user1 + password: secret \ No newline at end of file diff --git a/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_cert.yaml b/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_cert.yaml new file mode 100644 index 000000000..e14136938 --- /dev/null +++ b/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_cert.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: + - cluster: + certificate-authority-data: YXV0aG9yaXR5 + server: localhost:8000 + name: k8s +contexts: + - context: + cluster: k8s + user: user1 + name: user1@k8s +current-context: user1@k8s +kind: Config +preferences: {} +users: + - name: user1 + user: + client-certificate-data: Y2VydGlmaWNhdGU= + client-key-data: a2V5 \ No newline at end of file diff --git a/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_token.yaml b/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_token.yaml new file mode 100644 index 000000000..6c715ba38 --- /dev/null +++ b/lib/promscrape/discovery/kubernetes/testdata/kubeconfig_token.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +clusters: + - cluster: + server: "" + name: k8s +contexts: + - context: + cluster: k8s + user: user1 + name: user1@k8s +current-context: user1@k8s +kind: Config +preferences: {} +users: + - name: user1 + user: + token: abc \ No newline at end of file