From f06c7e99fe29f671b94bcf69f398435f84be3364 Mon Sep 17 00:00:00 2001 From: Zhu Jiekun Date: Mon, 28 Oct 2024 03:38:34 +0800 Subject: [PATCH] lib/promscrape: adds support for PuppetDB service discovery This commit adds support for [PuppetDB](https://www.puppet.com/docs/puppetdb/8/overview.html) service discovery to the `vmagent` and `victoria-metrics-single` components. Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5744 --- docs/README.md | 2 + docs/sd_configs.md | 56 +++++++ docs/vmagent.md | 2 + lib/promscrape/config.go | 15 ++ lib/promscrape/discovery/puppetdb/api.go | 75 +++++++++ lib/promscrape/discovery/puppetdb/api_test.go | 25 +++ .../puppetdb/mock_puppetdb_server_test.go | 41 +++++ lib/promscrape/discovery/puppetdb/puppetdb.go | 48 ++++++ .../discovery/puppetdb/puppetdb_test.go | 119 ++++++++++++++ lib/promscrape/discovery/puppetdb/resource.go | 151 ++++++++++++++++++ lib/promscrape/scraper.go | 2 + 11 files changed, 536 insertions(+) create mode 100644 lib/promscrape/discovery/puppetdb/api.go create mode 100644 lib/promscrape/discovery/puppetdb/api_test.go create mode 100644 lib/promscrape/discovery/puppetdb/mock_puppetdb_server_test.go create mode 100644 lib/promscrape/discovery/puppetdb/puppetdb.go create mode 100644 lib/promscrape/discovery/puppetdb/puppetdb_test.go create mode 100644 lib/promscrape/discovery/puppetdb/resource.go diff --git a/docs/README.md b/docs/README.md index e4bb5871c..1a1905a9c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3090,6 +3090,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#openstack_sd_configs for details (default 30s) -promscrape.ovhcloudSDCheckInterval duration Interval for checking for changes in OVH Cloud VPS and dedicated server. This works only if ovhcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#ovhcloud_sd_configs for details (default 30s) + -promscrape.puppetdbSDCheckInterval duration + Interval for checking for changes in PuppetDB API. This works only if puppetdb_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#puppetdb_sd_configs for details (default 30s) -promscrape.seriesLimitPerTarget int Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter for more info -promscrape.streamParse diff --git a/docs/sd_configs.md b/docs/sd_configs.md index e7b81e5e7..a85fc1803 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -31,6 +31,7 @@ supports the following Prometheus-compatible service discovery options for Prome * `nomad_sd_configs` is for discovering and scraping targets registered in [HashiCorp Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs). * `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs). * `ovhcloud_sd_configs` is for discovering and scraping OVH Cloud VPS and dedicated server targets. See [these docs](#ovhcloud_sd_configs). +* `puppetdb_sd_configs` is for discovering and scraping PuppetDB targets. See [these docs](#puppetdb_sd_configs). * `static_configs` is for scraping statically defined targets. See [these docs](#static_configs). * `vultr_sd_configs` is for discovering and scraping [Vultr](https://www.vultr.com/) targets. See [these docs](#vultr_sd_configs). * `yandexcloud_sd_configs` is for discovering and scraping [Yandex Cloud](https://cloud.yandex.com/en/) targets. See [these docs](#yandexcloud_sd_configs). @@ -1546,6 +1547,61 @@ Dedicated servers: The list of discovered OVH Cloud targets is refreshed at the interval, which can be configured via `-promscrape.ovhcloudSDCheckInterval` command-line flag. +## puppetdb_sd_configs + +_Available from [TODO](https://docs.victoriametrics.com/changelog/#TODO) version._ + +PuppetDB SD configuration allows retrieving scrape targets from [PuppetDB](https://www.puppet.com/docs/puppetdb/8/overview.html) resources. + +This SD discovers resources and will create a target for each resource returned by the API. + +Configuration example: + +```yaml +scrape_configs: +- job_name: puppetdb_job + puppetdb_sd_configs: + # The URL of the PuppetDB root query endpoint. + - url: + + # Puppet Query Language (PQL) query. Only resources are supported. + # https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html + query: + + # Whether to include the parameters as meta labels. + # Due to the differences between parameter types and Prometheus labels, + # some parameters might not be rendered. The format of the parameters might + # also change in future releases. + # + # Note: Enabling this exposes parameters in the VMUI and API. Make sure + # that you don't have secrets exposed as parameters if you enable this. + # + # include_parameters: | default false + + # The port to scrape metrics from. + # + # port: | default = 80 + + # Additional HTTP API client options can be specified here. + # See https://docs.victoriametrics.com/sd_configs.html#http-api-client-options +``` + +The resource address is the `certname` of the resource and can be changed during relabeling. +The following meta labels are available on targets during relabeling: + +* `__meta_puppetdb_query`: the Puppet Query Language (PQL) query. +* `__meta_puppetdb_certname`: the name of the node associated with the resource. +* `__meta_puppetdb_resource`: a SHA-1 hash of the resource’s type, title, and parameters, for identification. +* `__meta_puppetdb_type`: the resource type. +* `__meta_puppetdb_title`: the resource title. +* `__meta_puppetdb_exported`: whether the resource is exported (`"true"` or `"false"`). +* `__meta_puppetdb_tags`: comma separated list of resource tags. +* `__meta_puppetdb_file`: the manifest file in which the resource was declared. +* `__meta_puppetdb_environment`: the environment of the node associated with the resource. +* `__meta_puppetdb_parameter_`: the parameters of the resource. + +The list of discovered PuppetDB targets is refreshed at the interval, which can be configured via `-promscrape.puppetdbSDCheckInterval` command-line flag. + ## static_configs A static config allows specifying a list of targets and a common label set for them. diff --git a/docs/vmagent.md b/docs/vmagent.md index 74981aca8..5525ac4c7 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -2015,6 +2015,8 @@ See the docs at https://docs.victoriametrics.com/vmagent/ . Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#openstack_sd_configs for details (default 30s) -promscrape.ovhcloudSDCheckInterval duration Interval for checking for changes in OVH Cloud VPS and dedicated server. This works only if ovhcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#ovhcloud_sd_configs for details (default 30s) + -promscrape.puppetdbSDCheckInterval duration + Interval for checking for changes in PuppetDB API. This works only if puppetdb_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs/#puppetdb_sd_configs for details (default 30s) -promscrape.seriesLimitPerTarget int Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter for more info -promscrape.streamParse diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 9221d7f43..9b1fcdbf0 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -38,6 +38,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/nomad" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ovhcloud" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/puppetdb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/vultr" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/yandexcloud" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" @@ -314,6 +315,7 @@ type ScrapeConfig struct { NomadSDConfigs []nomad.SDConfig `yaml:"nomad_sd_configs,omitempty"` OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"` OVHCloudSDConfigs []ovhcloud.SDConfig `yaml:"ovhcloud_sd_configs,omitempty"` + PuppetDBSDConfigs []puppetdb.SDConfig `yaml:"puppetdb_sd_configs,omitempty"` StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"` VultrSDConfigs []vultr.SDConfig `yaml:"vultr_configs,omitempty"` YandexCloudSDConfigs []yandexcloud.SDConfig `yaml:"yandexcloud_sd_configs,omitempty"` @@ -399,6 +401,9 @@ func (sc *ScrapeConfig) mustStop() { for i := range sc.OVHCloudSDConfigs { sc.OVHCloudSDConfigs[i].MustStop() } + for i := range sc.PuppetDBSDConfigs { + sc.PuppetDBSDConfigs[i].MustStop() + } for i := range sc.VultrSDConfigs { sc.VultrSDConfigs[i].MustStop() } @@ -772,6 +777,16 @@ func (cfg *Config) getOVHCloudSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork { return cfg.getScrapeWorkGeneric(visitConfigs, "ovhcloud_sd_config", prev) } +// getPuppetDBSDScrapeWork returns `puppetdb_sd_configs` ScrapeWork from cfg. +func (cfg *Config) getPuppetDBSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork { + visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) { + for i := range sc.PuppetDBSDConfigs { + visitor(&sc.PuppetDBSDConfigs[i]) + } + } + return cfg.getScrapeWorkGeneric(visitConfigs, "puppetdb_sd_config", prev) +} + // getVultrSDScrapeWork returns `vultr_sd_configs` ScrapeWork from cfg. func (cfg *Config) getVultrSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork { visitConfigs := func(sc *ScrapeConfig, visitor func(sdc targetLabelsGetter)) { diff --git a/lib/promscrape/discovery/puppetdb/api.go b/lib/promscrape/discovery/puppetdb/api.go new file mode 100644 index 000000000..499301c14 --- /dev/null +++ b/lib/promscrape/discovery/puppetdb/api.go @@ -0,0 +1,75 @@ +package puppetdb + +import ( + "errors" + "fmt" + "net/url" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" +) + +var configMap = discoveryutils.NewConfigMap() + +type apiConfig struct { + client *discoveryutils.Client + + query string + includeParameters bool + port int +} + +func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { + v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) }) + if err != nil { + return nil, err + } + return v.(*apiConfig), nil +} + +func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { + // the following param checks align with Prometheus + if sdc.URL == "" { + return nil, errors.New("URL is missing") + } + parsedURL, err := url.Parse(sdc.URL) + if err != nil { + return nil, fmt.Errorf("parse URL %s error: %v", sdc.URL, err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("URL %s scheme must be 'http' or 'https'", sdc.URL) + } + if parsedURL.Host == "" { + return nil, fmt.Errorf("host is missing in URL %s", sdc.URL) + } + if sdc.Query == "" { + return nil, errors.New("query missing") + } + + port := sdc.Port + if port == 0 { + port = 80 + } + + // other general checks + ac, err := sdc.HTTPClientConfig.NewConfig(baseDir) + if err != nil { + return nil, fmt.Errorf("cannot parse auth config: %w", err) + } + proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir) + if err != nil { + return nil, fmt.Errorf("cannot parse proxy auth config: %w", err) + } + + client, err := discoveryutils.NewClient(parsedURL.String(), ac, sdc.ProxyURL, proxyAC, &sdc.HTTPClientConfig) + if err != nil { + return nil, fmt.Errorf("cannot create HTTP client for %q: %w", sdc.URL, err) + } + + return &apiConfig{ + client: client, + + query: sdc.Query, + includeParameters: sdc.IncludeParameters, + port: port, + }, nil +} diff --git a/lib/promscrape/discovery/puppetdb/api_test.go b/lib/promscrape/discovery/puppetdb/api_test.go new file mode 100644 index 000000000..00de2e53b --- /dev/null +++ b/lib/promscrape/discovery/puppetdb/api_test.go @@ -0,0 +1,25 @@ +package puppetdb + +import ( + "testing" +) + +func Test_newAPIConfig(t *testing.T) { + f := func(url, query string, includeParameters bool, port int, wantErr bool) { + t.Helper() + sdc := &SDConfig{ + URL: url, + Query: query, + IncludeParameters: includeParameters, + Port: port, + } + if _, err := newAPIConfig(sdc, ""); wantErr != (err != nil) { + t.Fatalf("newAPIConfig want error = %t, but error = %v", wantErr, err) + } + } + + f("https://puppetdb.example.com", `resources { type = "Class" and title = "Prometheus::Node_exporter" }`, true, 9100, false) + f("", `resources { type = "Class" and title = "Prometheus::Node_exporter" }`, true, 9100, true) + f("https://puppetdb.example.com", ``, true, 9100, true) + f("ftp://invalid.url", `resources { type = "Class" and title = "Prometheus::Node_exporter" }`, true, 9100, true) +} diff --git a/lib/promscrape/discovery/puppetdb/mock_puppetdb_server_test.go b/lib/promscrape/discovery/puppetdb/mock_puppetdb_server_test.go new file mode 100644 index 000000000..fb8e17615 --- /dev/null +++ b/lib/promscrape/discovery/puppetdb/mock_puppetdb_server_test.go @@ -0,0 +1,41 @@ +package puppetdb + +import ( + "fmt" + "net/http" + "net/http/httptest" +) + +func newMockPuppetDBServer(jsonResponse func(path string) ([]byte, error)) *puppetdbServer { + rw := &puppetdbServer{} + rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler)) + rw.jsonResponse = jsonResponse + return rw +} + +type puppetdbServer struct { + *httptest.Server + jsonResponse func(path string) ([]byte, error) +} + +func (rw *puppetdbServer) err(w http.ResponseWriter, err error) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) +} + +func (rw *puppetdbServer) handler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + rw.err(w, fmt.Errorf("bad method %q", r.Method)) + return + } + + resp, err := rw.jsonResponse(r.RequestURI) + if err != nil { + rw.err(w, err) + return + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Write(resp) + w.WriteHeader(http.StatusOK) +} diff --git a/lib/promscrape/discovery/puppetdb/puppetdb.go b/lib/promscrape/discovery/puppetdb/puppetdb.go new file mode 100644 index 000000000..f566e7a21 --- /dev/null +++ b/lib/promscrape/discovery/puppetdb/puppetdb.go @@ -0,0 +1,48 @@ +package puppetdb + +import ( + "flag" + "fmt" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" +) + +// SDCheckInterval defines interval for targets refresh. +var SDCheckInterval = flag.Duration("promscrape.puppetdbSDCheckInterval", 30*time.Second, "Interval for checking for changes in PuppetDB API. "+ + "This works only if puppetdb_sd_configs is configured in '-promscrape.config' file. "+ + "See https://docs.victoriametrics.com/sd_configs/#puppetdb_sd_configs for details") + +// SDConfig is the configuration for PuppetDB based discovery. +type SDConfig struct { + URL string `yaml:"url"` + Query string `yaml:"query"` + IncludeParameters bool `yaml:"include_parameters"` + Port int `yaml:"port"` + + HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` + ProxyURL *proxy.URL `yaml:"proxy_url,omitempty"` + ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` +} + +// GetLabels returns labels for PuppetDB according to service discover config. +func (sdc *SDConfig) GetLabels(baseDir string) ([]*promutils.Labels, error) { + cfg, err := getAPIConfig(sdc, baseDir) + if err != nil { + return nil, fmt.Errorf("cannot get API config: %w", err) + } + + resources, err := getResourceList(cfg) + if err != nil { + return nil, err + } + + return getResourceLabels(resources, cfg), nil +} + +// MustStop stops further usage for sdc. +func (sdc *SDConfig) MustStop() { + _ = configMap.Delete(sdc) +} diff --git a/lib/promscrape/discovery/puppetdb/puppetdb_test.go b/lib/promscrape/discovery/puppetdb/puppetdb_test.go new file mode 100644 index 000000000..dc7b04dc7 --- /dev/null +++ b/lib/promscrape/discovery/puppetdb/puppetdb_test.go @@ -0,0 +1,119 @@ +package puppetdb + +import ( + "reflect" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +var jsonResponse = `[ + { + "certname": "edinburgh.example.com", + "environment": "prod", + "exported": false, + "file": "/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp", + "line": 384, + "parameters": { + "access_log": true, + "access_log_file": "ssl_access_log", + "additional_includes": [ ], + "directoryindex": "", + "docroot": "/var/www/html", + "ensure": "absent", + "options": [ + "Indexes", + "FollowSymLinks", + "MultiViews" + ], + "php_flags": { }, + "labels": { + "alias": "edinburgh" + }, + "scriptaliases": [ + { + "alias": "/cgi-bin", + "path": "/var/www/cgi-bin" + } + ], + "port": 22, + "pi": 3.141592653589793, + "buckets": [ + 0, + 2, + 5 + ], + "coordinates": [ + 60.13464726551357, + -2.0513768021728893 + ] + }, + "resource": "49af83866dc5a1518968b68e58a25319107afe11", + "tags": [ + "roles::hypervisor", + "apache", + "apache::vhost", + "class", + "default-ssl", + "profile_hypervisor", + "vhost", + "profile_apache", + "hypervisor", + "__node_regexp__edinburgh", + "roles", + "node" + ], + "title": "default-ssl", + "type": "Apache::Vhost" + } +]` + +// TestSDConfig_GetLabels test example response and expect labels are from: +// https://github.com/prometheus/prometheus/blob/685493187ec5f5734777769f595cf8418d49900d/discovery/puppetdb/puppetdb_test.go#L110C6-L110C39 +func TestSDConfig_GetLabels(t *testing.T) { + mockSvr := newMockPuppetDBServer(func(_ string) ([]byte, error) { + return []byte(jsonResponse), nil + }) + + sdConfig := &SDConfig{ + URL: mockSvr.URL, + Query: "vhosts", + Port: 9100, + IncludeParameters: true, + } + + expectLabels := &promutils.Labels{} + expectLabels.Add("__address__", "edinburgh.example.com:9100") + expectLabels.Add("__meta_puppetdb_query", "vhosts") + expectLabels.Add("__meta_puppetdb_certname", "edinburgh.example.com") + expectLabels.Add("__meta_puppetdb_environment", "prod") + expectLabels.Add("__meta_puppetdb_exported", "false") + expectLabels.Add("__meta_puppetdb_file", "/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp") + expectLabels.Add("__meta_puppetdb_parameter_access_log", "true") + expectLabels.Add("__meta_puppetdb_parameter_access_log_file", "ssl_access_log") + expectLabels.Add("__meta_puppetdb_parameter_buckets", "0,2,5") + expectLabels.Add("__meta_puppetdb_parameter_coordinates", "60.13464726551357,-2.0513768021728893") + expectLabels.Add("__meta_puppetdb_parameter_docroot", "/var/www/html") + expectLabels.Add("__meta_puppetdb_parameter_ensure", "absent") + expectLabels.Add("__meta_puppetdb_parameter_labels_alias", "edinburgh") + expectLabels.Add("__meta_puppetdb_parameter_options", "Indexes,FollowSymLinks,MultiViews") + expectLabels.Add("__meta_puppetdb_parameter_pi", "3.141592653589793") + expectLabels.Add("__meta_puppetdb_parameter_port", "22") + expectLabels.Add("__meta_puppetdb_resource", "49af83866dc5a1518968b68e58a25319107afe11") + expectLabels.Add("__meta_puppetdb_tags", ",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,") + expectLabels.Add("__meta_puppetdb_title", "default-ssl") + expectLabels.Add("__meta_puppetdb_type", "Apache::Vhost") + + result, err := sdConfig.GetLabels("") + if err != nil { + t.Fatalf("GetLabels got err: %v", err) + } + + if len(result) != 1 { + t.Fatalf("GetLabels get result len != 1") + } + + if !reflect.DeepEqual(result[0].ToMap(), expectLabels.ToMap()) { + t.Fatalf("GetLabels incorrect, want: %v, got: %v", expectLabels.ToMap(), result[0].ToMap()) + } +} diff --git a/lib/promscrape/discovery/puppetdb/resource.go b/lib/promscrape/discovery/puppetdb/resource.go new file mode 100644 index 000000000..de148b5de --- /dev/null +++ b/lib/promscrape/discovery/puppetdb/resource.go @@ -0,0 +1,151 @@ +package puppetdb + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +const ( + separator = "," +) + +type resource struct { + Certname string `json:"certname"` + Resource string `json:"resource"` + Type string `json:"type"` + Title string `json:"title"` + Exported bool `json:"exported"` + Tags []string `json:"tags"` + File string `json:"file"` + Environment string `json:"environment"` + Parameters parameters `json:"parameters"` +} + +type parameters map[string]interface{} + +// addToLabels add Parameters map to existing labels. +// See: https://github.com/prometheus/prometheus/blob/685493187ec5f5734777769f595cf8418d49900d/discovery/puppetdb/resources.go#L39 +func (p *parameters) addToLabels(keyPrefix string, m *promutils.Labels) { + if p == nil { + return + } + + for k, v := range *p { + var labelValue string + switch value := v.(type) { + case string: + labelValue = value + case bool: + labelValue = strconv.FormatBool(value) + case int64: + labelValue = strconv.FormatInt(value, 10) + case float64: + labelValue = strconv.FormatFloat(value, 'g', -1, 64) + case []string: + labelValue = separator + strings.Join(value, separator) + separator + case []interface{}: + if len(value) == 0 { + continue + } + values := make([]string, len(value)) + for i, v := range value { + switch value := v.(type) { + case string: + values[i] = value + case bool: + values[i] = strconv.FormatBool(value) + case int64: + values[i] = strconv.FormatInt(value, 10) + case float64: + values[i] = strconv.FormatFloat(value, 'g', -1, 64) + case []string: + values[i] = separator + strings.Join(value, separator) + separator + } + } + labelValue = strings.Join(values, separator) + case map[string]interface{}: + subParameter := parameters(value) + subParameter.addToLabels(keyPrefix+discoveryutils.SanitizeLabelName(k+"_"), m) + default: + continue + } + if labelValue == "" { + continue + } + name := discoveryutils.SanitizeLabelName(k) + m.Add(keyPrefix+name, labelValue) + } +} + +func getResourceList(cfg *apiConfig) ([]resource, error) { + body := struct { + Query string `json:"query"` + }{cfg.query} + + bodyBytes, err := json.Marshal(body) + if err != nil { + return nil, err + } + + modifyRequestFunc := func(request *http.Request) { + request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + request.Header.Set("Accept", "application/json") + request.Header.Set("Content-Type", "application/json") + request.Method = http.MethodPost + } + + // https://www.puppet.com/docs/puppetdb/8/api/query/v4/overview#pdbqueryv4 + resp, err := cfg.client.GetAPIResponseWithReqParams("/pdb/query/v4", modifyRequestFunc) + if err != nil { + return nil, err + } + + var resources []resource + if err = json.Unmarshal(resp, &resources); err != nil { + return nil, err + } + + return resources, nil +} + +func getResourceLabels(resources []resource, cfg *apiConfig) []*promutils.Labels { + ms := make([]*promutils.Labels, 0, len(resources)) + + for _, res := range resources { + m := promutils.NewLabels(18) + + m.Add("__address__", discoveryutils.JoinHostPort(res.Certname, cfg.port)) + m.Add("__meta_puppetdb_certname", res.Certname) + m.Add("__meta_puppetdb_environment", res.Environment) + m.Add("__meta_puppetdb_exported", fmt.Sprintf("%t", res.Exported)) + m.Add("__meta_puppetdb_file", res.File) + m.Add("__meta_puppetdb_query", cfg.query) + m.Add("__meta_puppetdb_resource", res.Resource) + m.Add("__meta_puppetdb_title", res.Title) + m.Add("__meta_puppetdb_type", res.Type) + + if len(res.Tags) > 0 { + //discoveryutils.AddTagsToLabels(m, resource.Tags, "__meta_puppetdb_tags", separator) + m.Add("__meta_puppetdb_tags", separator+strings.Join(res.Tags, separator)+separator) + } + + // Parameters are not included by default. This should only be enabled + // on select resources as it might expose secrets on the Prometheus UI + // for certain resources. + if cfg.includeParameters { + res.Parameters.addToLabels("__meta_puppetdb_parameter_", m) + } + + ms = append(ms, m) + } + + return ms +} diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index b34cf1592..d8ad2707e 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -31,6 +31,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/nomad" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ovhcloud" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/puppetdb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/vultr" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/yandexcloud" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" @@ -143,6 +144,7 @@ func runScraper(configFile string, pushData func(at *auth.Token, wr *prompbmarsh scs.add("nomad_sd_configs", *nomad.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getNomadSDScrapeWork(swsPrev) }) scs.add("openstack_sd_configs", *openstack.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) }) scs.add("ovhcloud_sd_configs", *ovhcloud.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getOVHCloudSDScrapeWork(swsPrev) }) + scs.add("puppetdb_sd_configs", *puppetdb.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getPuppetDBSDScrapeWork(swsPrev) }) scs.add("vultr_sd_configs", *vultr.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getVultrSDScrapeWork(swsPrev) }) scs.add("yandexcloud_sd_configs", *yandexcloud.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getYandexCloudSDScrapeWork(swsPrev) }) scs.add("static_configs", 0, func(cfg *Config, _ []*ScrapeWork) []*ScrapeWork { return cfg.getStaticScrapeWork() })