From 158c50c0eead31952f7589809d174df683f55b15 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 1 Jul 2021 15:27:44 +0300 Subject: [PATCH] lib/promauth: reload TLS certificates from disk on every mTLS connection as Prometheus does This allows updating client certificates without the need to restart vmagent and/or single-node VictoriaMetrics. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/470 --- docs/CHANGELOG.md | 1 + lib/promauth/config.go | 45 +++++++++--------- lib/promscrape/config_test.go | 47 ------------------- lib/promscrape/testdata/ssl-cert-snakeoil.key | 28 ----------- lib/promscrape/testdata/ssl-cert-snakeoil.pem | 17 ------- 5 files changed, 25 insertions(+), 113 deletions(-) delete mode 100644 lib/promscrape/testdata/ssl-cert-snakeoil.key delete mode 100644 lib/promscrape/testdata/ssl-cert-snakeoil.pem diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e0ee43ab7..51dc323e8 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,7 @@ sort: 15 ## tip +* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420). * FEATURE: reduce memory usage when performing heavy queries over high number of time series. * BUGFIX: vmagent: remove `{ %space %}` typo in `/targets` output. The typo has been introduced in v1.62.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1408). diff --git a/lib/promauth/config.go b/lib/promauth/config.go index f555c77e6..32206c7a2 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -158,10 +158,12 @@ func (oi *oauth2ConfigInternal) getTokenSource() (oauth2.TokenSource, error) { type Config struct { // Optional TLS config TLSRootCA *x509.CertPool - TLSCertificate *tls.Certificate TLSServerName string TLSInsecureSkipVerify bool + getTLSCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + tlsCertDigest string + getAuthHeader func() string authHeaderLock sync.Mutex authHeader string @@ -189,7 +191,7 @@ func (ac *Config) GetAuthHeader() string { // String returns human-readable representation for ac. func (ac *Config) String() string { return fmt.Sprintf("AuthDigest=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v", - ac.authDigest, ac.tlsRootCAString(), ac.tlsCertificateString(), ac.TLSServerName, ac.TLSInsecureSkipVerify) + ac.authDigest, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify) } func (ac *Config) tlsRootCAString() string { @@ -200,13 +202,6 @@ func (ac *Config) tlsRootCAString() string { return string(bytes.Join(data, []byte("\n"))) } -func (ac *Config) tlsCertificateString() string { - if ac.TLSCertificate == nil { - return "" - } - return string(bytes.Join(ac.TLSCertificate.Certificate, []byte("\n"))) -} - // NewTLSConfig returns new TLS config for the given ac. func (ac *Config) NewTLSConfig() *tls.Config { tlsCfg := &tls.Config{ @@ -215,10 +210,7 @@ func (ac *Config) NewTLSConfig() *tls.Config { if ac == nil { return tlsCfg } - if ac.TLSCertificate != nil { - // Do not set tlsCfg.GetClientCertificate, since tlsCfg.Certificates should work OK. - tlsCfg.Certificates = []tls.Certificate{*ac.TLSCertificate} - } + tlsCfg.GetClientCertificate = ac.getTLSCert tlsCfg.RootCAs = ac.TLSRootCA tlsCfg.ServerName = ac.TLSServerName tlsCfg.InsecureSkipVerify = ac.TLSInsecureSkipVerify @@ -350,20 +342,29 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be authDigest = fmt.Sprintf("oauth2(%s)", o.String()) } var tlsRootCA *x509.CertPool - var tlsCertificate *tls.Certificate + var getTLSCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + tlsCertDigest := "" tlsServerName := "" tlsInsecureSkipVerify := false if tlsConfig != nil { tlsServerName = tlsConfig.ServerName tlsInsecureSkipVerify = tlsConfig.InsecureSkipVerify if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" { - certPath := getFilepath(baseDir, tlsConfig.CertFile) - keyPath := getFilepath(baseDir, tlsConfig.KeyFile) - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err) + getTLSCert = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + // Re-read TLS certificate from disk. This is needed for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420 + certPath := getFilepath(baseDir, tlsConfig.CertFile) + keyPath := getFilepath(baseDir, tlsConfig.KeyFile) + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err) + } + return &cert, nil } - tlsCertificate = &cert + // Check whether the configured TLS cert can be loaded. + if _, err := getTLSCert(nil); err != nil { + return nil, err + } + tlsCertDigest = fmt.Sprintf("certFile=%q, keyFile=%q", tlsConfig.CertFile, tlsConfig.KeyFile) } if tlsConfig.CAFile != "" { path := getFilepath(baseDir, tlsConfig.CAFile) @@ -379,10 +380,12 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be } ac := &Config{ TLSRootCA: tlsRootCA, - TLSCertificate: tlsCertificate, TLSServerName: tlsServerName, TLSInsecureSkipVerify: tlsInsecureSkipVerify, + getTLSCert: getTLSCert, + tlsCertDigest: tlsCertDigest, + getAuthHeader: getAuthHeader, authDigest: authDigest, } diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index f3720e0b0..6b0e5376f 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -1,7 +1,6 @@ package promscrape import ( - "crypto/tls" "fmt" "reflect" "strconv" @@ -1246,52 +1245,6 @@ scrape_configs: jobNameOriginal: "foo", }, }) - snakeoilCert, err := tls.LoadX509KeyPair("testdata/ssl-cert-snakeoil.pem", "testdata/ssl-cert-snakeoil.key") - if err != nil { - t.Fatalf("cannot load snakeoil cert: %s", err) - } - f(` -scrape_configs: -- job_name: foo - tls_config: - cert_file: testdata/ssl-cert-snakeoil.pem - key_file: testdata/ssl-cert-snakeoil.key - static_configs: - - targets: ["foo.bar:1234"] -`, []*ScrapeWork{ - { - ScrapeURL: "http://foo.bar:1234/metrics", - ScrapeInterval: defaultScrapeInterval, - ScrapeTimeout: defaultScrapeTimeout, - Labels: []prompbmarshal.Label{ - { - Name: "__address__", - Value: "foo.bar:1234", - }, - { - Name: "__metrics_path__", - Value: "/metrics", - }, - { - Name: "__scheme__", - Value: "http", - }, - { - Name: "instance", - Value: "foo.bar:1234", - }, - { - Name: "job", - Value: "foo", - }, - }, - AuthConfig: &promauth.Config{ - TLSCertificate: &snakeoilCert, - }, - ProxyAuthConfig: &promauth.Config{}, - jobNameOriginal: "foo", - }, - }) f(` global: external_labels: diff --git a/lib/promscrape/testdata/ssl-cert-snakeoil.key b/lib/promscrape/testdata/ssl-cert-snakeoil.key deleted file mode 100644 index 00a79a3b5..000000000 --- a/lib/promscrape/testdata/ssl-cert-snakeoil.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG -3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U -wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 -FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf -IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg -GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF -sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 -sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D -uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb -K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 -YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ -DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk -B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV -Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x -IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY -wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj -wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D -FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m -tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX -fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU -ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk -K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT -6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt -9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN -Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV -c257YgaWmjK9uB0Y2r2VxS0G ------END PRIVATE KEY----- diff --git a/lib/promscrape/testdata/ssl-cert-snakeoil.pem b/lib/promscrape/testdata/ssl-cert-snakeoil.pem deleted file mode 100644 index 93e77cd95..000000000 --- a/lib/promscrape/testdata/ssl-cert-snakeoil.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV -BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV -MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D -K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te -+z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij -L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 -xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY -6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG -SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 -L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 -45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li -K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 -X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI -whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd ------END CERTIFICATE-----