mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-21 14:44:00 +00:00
lib/promscrape/discovery/kubernetes: allow attaching node-level labels to role: endpoints
and role: endpointlice
targets in the same way as Prometheus does
See https://github.com/prometheus/prometheus/pull/10759
This commit is contained in:
parent
6bb4bcd30c
commit
c4cc45d7f8
3 changed files with 226 additions and 177 deletions
|
@ -23,6 +23,7 @@ If you use alerting rules or Grafana dashboards, which rely on this metric, then
|
|||
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add an UI for [query tracing](https://docs.victoriametrics.com/#query-tracing). It can be enabled by clicking `enable query tracing` checkbox and re-running the query. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2703).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.headers` command-line option for specifying optional HTTP headers to send to the configured `-remoteWrite.url`. For example, `-remoteWrite.headers='Foo:Bar^^Baz:x'` would send `Foo: Bar` and `Baz: x` HTTP headers with every request to `-remoteWrite.url`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2805).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): push per-target `scrape_samples_limit` metric to the cofigured `-remoteWrite.url` if `sample_limit` option is set for this target in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). See [this feature request](https://github.com/VictoriaMetrics/operator/issues/497).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): attach node-level labels to [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) targets if `attach_metadata: {"node": true}` is set for `role: endpoints` and `role: endpointslice`. This is a feature backport from Prometheus 2.37 - see [this pull request](https://github.com/prometheus/prometheus/pull/10759).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`:
|
||||
|
||||
```yaml
|
||||
|
|
|
@ -345,7 +345,7 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
|
|||
gw.startWatchersForRole("pod", nil)
|
||||
gw.startWatchersForRole("service", nil)
|
||||
}
|
||||
if gw.attachNodeMetadata && role == "pod" {
|
||||
if gw.attachNodeMetadata && (role == "pod" || role == "endpoints" || role == "endpointslice") {
|
||||
gw.startWatchersForRole("node", nil)
|
||||
}
|
||||
paths := getAPIPathsWithNamespaces(role, gw.namespaces, gw.selectors)
|
||||
|
@ -803,8 +803,8 @@ func (uw *urlWatcher) maybeUpdateDependedScrapeWorksLocked() {
|
|||
uwx.needRecreateScrapeWorks = true
|
||||
continue
|
||||
}
|
||||
if attachNodeMetadata && role == "node" && uwx.role == "pod" {
|
||||
// pod objects depend on node objects if attachNodeMetadata is set
|
||||
if attachNodeMetadata && role == "node" && (uwx.role == "pod" || uwx.role == "endpoints" || uwx.role == "endpointslice") {
|
||||
// pod, endpoints and enpointslices objects depend on node objects if attachNodeMetadata is set
|
||||
uwx.needRecreateScrapeWorks = true
|
||||
continue
|
||||
}
|
||||
|
|
|
@ -119,197 +119,245 @@ func TestGetEndpointLabels(t *testing.T) {
|
|||
containerPorts map[string][]ContainerPort
|
||||
endpointPorts []EndpointPort
|
||||
}
|
||||
f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
eps := Endpoints{
|
||||
Metadata: ObjectMeta{
|
||||
Name: "test-eps",
|
||||
Namespace: "default",
|
||||
},
|
||||
Subsets: []EndpointSubset{
|
||||
{
|
||||
Ports: args.endpointPorts,
|
||||
Addresses: []EndpointAddress{
|
||||
{
|
||||
IP: "10.13.15.15",
|
||||
TargetRef: ObjectReference{
|
||||
Kind: "Pod",
|
||||
Namespace: "default",
|
||||
Name: "test-pod",
|
||||
},
|
||||
f := func(t *testing.T, args testArgs, wantLabels [][]prompbmarshal.Label) {
|
||||
t.Helper()
|
||||
eps := Endpoints{
|
||||
Metadata: ObjectMeta{
|
||||
Name: "test-eps",
|
||||
Namespace: "default",
|
||||
},
|
||||
Subsets: []EndpointSubset{
|
||||
{
|
||||
Ports: args.endpointPorts,
|
||||
Addresses: []EndpointAddress{
|
||||
{
|
||||
IP: "10.13.15.15",
|
||||
TargetRef: ObjectReference{
|
||||
Kind: "Pod",
|
||||
Namespace: "default",
|
||||
Name: "test-pod",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
svc := Service{
|
||||
Metadata: ObjectMeta{
|
||||
Name: "test-eps",
|
||||
Namespace: "default",
|
||||
},
|
||||
Spec: ServiceSpec{
|
||||
Ports: []ServicePort{
|
||||
{
|
||||
Name: "test-port",
|
||||
Port: 8081,
|
||||
},
|
||||
},
|
||||
}
|
||||
svc := Service{
|
||||
Metadata: ObjectMeta{
|
||||
Name: "test-eps",
|
||||
Namespace: "default",
|
||||
},
|
||||
Spec: ServiceSpec{
|
||||
ClusterIP: "1.2.3.4",
|
||||
Type: "service-type",
|
||||
Ports: []ServicePort{
|
||||
{
|
||||
Name: "test-port",
|
||||
Port: 8081,
|
||||
},
|
||||
},
|
||||
}
|
||||
pod := Pod{
|
||||
Metadata: ObjectMeta{
|
||||
Name: "test-pod",
|
||||
Namespace: "default",
|
||||
},
|
||||
Status: PodStatus{PodIP: "192.168.15.1"},
|
||||
}
|
||||
for cn, ports := range args.containerPorts {
|
||||
pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
|
||||
}
|
||||
var gw groupWatcher
|
||||
gw.m = map[string]*urlWatcher{
|
||||
"pod": {
|
||||
role: "pod",
|
||||
objectsByKey: map[string]object{
|
||||
"default/test-pod": &pod,
|
||||
},
|
||||
}
|
||||
pod := Pod{
|
||||
Metadata: ObjectMeta{
|
||||
UID: "pod-uid",
|
||||
Name: "test-pod",
|
||||
Namespace: "default",
|
||||
},
|
||||
Spec: PodSpec{
|
||||
NodeName: "test-node",
|
||||
},
|
||||
Status: PodStatus{
|
||||
Phase: "abc",
|
||||
PodIP: "192.168.15.1",
|
||||
HostIP: "4.5.6.7",
|
||||
},
|
||||
}
|
||||
node := Node{
|
||||
Metadata: ObjectMeta{
|
||||
Labels: []prompbmarshal.Label{
|
||||
{
|
||||
Name: "node-label",
|
||||
Value: "xyz",
|
||||
},
|
||||
},
|
||||
"service": {
|
||||
role: "service",
|
||||
objectsByKey: map[string]object{
|
||||
"default/test-eps": &svc,
|
||||
},
|
||||
},
|
||||
}
|
||||
for cn, ports := range args.containerPorts {
|
||||
pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
|
||||
}
|
||||
var gw groupWatcher
|
||||
gw.m = map[string]*urlWatcher{
|
||||
"pod": {
|
||||
role: "pod",
|
||||
objectsByKey: map[string]object{
|
||||
"default/test-pod": &pod,
|
||||
},
|
||||
}
|
||||
var sortedLabelss [][]prompbmarshal.Label
|
||||
gotLabels := eps.getTargetLabels(&gw)
|
||||
for _, lbs := range gotLabels {
|
||||
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
|
||||
}
|
||||
if !areEqualLabelss(sortedLabelss, wantLabels) {
|
||||
t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
|
||||
|
||||
}
|
||||
})
|
||||
},
|
||||
"service": {
|
||||
role: "service",
|
||||
objectsByKey: map[string]object{
|
||||
"default/test-eps": &svc,
|
||||
},
|
||||
},
|
||||
"node": {
|
||||
role: "node",
|
||||
objectsByKey: map[string]object{
|
||||
"/test-node": &node,
|
||||
},
|
||||
},
|
||||
}
|
||||
gw.attachNodeMetadata = true
|
||||
var sortedLabelss [][]prompbmarshal.Label
|
||||
gotLabels := eps.getTargetLabels(&gw)
|
||||
for _, lbs := range gotLabels {
|
||||
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
|
||||
}
|
||||
if !areEqualLabelss(sortedLabelss, wantLabels) {
|
||||
t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
|
||||
}
|
||||
}
|
||||
|
||||
f("1 port from endpoint", testArgs{
|
||||
endpointPorts: []EndpointPort{
|
||||
{
|
||||
Name: "web",
|
||||
Port: 8081,
|
||||
t.Run("1 port from endpoint", func(t *testing.T) {
|
||||
f(t, testArgs{
|
||||
endpointPorts: []EndpointPort{
|
||||
{
|
||||
Name: "web",
|
||||
Port: 8081,
|
||||
Protocol: "foobar",
|
||||
},
|
||||
},
|
||||
},
|
||||
}, [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.13.15.15:8081",
|
||||
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
|
||||
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
|
||||
"__meta_kubernetes_endpoint_port_name": "web",
|
||||
"__meta_kubernetes_endpoint_port_protocol": "",
|
||||
"__meta_kubernetes_endpoint_ready": "true",
|
||||
"__meta_kubernetes_endpoints_name": "test-eps",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_pod_host_ip": "",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "",
|
||||
"__meta_kubernetes_pod_phase": "",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "",
|
||||
"__meta_kubernetes_service_cluster_ip": "",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "",
|
||||
}),
|
||||
}, [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.13.15.15:8081",
|
||||
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
|
||||
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
|
||||
"__meta_kubernetes_endpoint_port_name": "web",
|
||||
"__meta_kubernetes_endpoint_port_protocol": "foobar",
|
||||
"__meta_kubernetes_endpoint_ready": "true",
|
||||
"__meta_kubernetes_endpoints_name": "test-eps",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_node_label_node_label": "xyz",
|
||||
"__meta_kubernetes_node_labelpresent_node_label": "true",
|
||||
"__meta_kubernetes_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_phase": "abc",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "pod-uid",
|
||||
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "service-type",
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
f("1 port from endpoint and 1 from pod", testArgs{
|
||||
containerPorts: map[string][]ContainerPort{"metrics": {{
|
||||
Name: "http-metrics",
|
||||
ContainerPort: 8428,
|
||||
}}},
|
||||
endpointPorts: []EndpointPort{
|
||||
{
|
||||
Name: "web",
|
||||
Port: 8081,
|
||||
t.Run("1 port from endpoint and 1 from pod", func(t *testing.T) {
|
||||
f(t, testArgs{
|
||||
containerPorts: map[string][]ContainerPort{"metrics": {{
|
||||
Name: "http-metrics",
|
||||
ContainerPort: 8428,
|
||||
Protocol: "foobar",
|
||||
}}},
|
||||
endpointPorts: []EndpointPort{
|
||||
{
|
||||
Name: "web",
|
||||
Port: 8081,
|
||||
Protocol: "https",
|
||||
},
|
||||
},
|
||||
},
|
||||
}, [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.13.15.15:8081",
|
||||
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
|
||||
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
|
||||
"__meta_kubernetes_endpoint_port_name": "web",
|
||||
"__meta_kubernetes_endpoint_port_protocol": "",
|
||||
"__meta_kubernetes_endpoint_ready": "true",
|
||||
"__meta_kubernetes_endpoints_name": "test-eps",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_pod_host_ip": "",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "",
|
||||
"__meta_kubernetes_pod_phase": "",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "",
|
||||
"__meta_kubernetes_service_cluster_ip": "",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "",
|
||||
}),
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "192.168.15.1:8428",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_pod_container_name": "metrics",
|
||||
"__meta_kubernetes_pod_container_port_name": "http-metrics",
|
||||
"__meta_kubernetes_pod_container_port_number": "8428",
|
||||
"__meta_kubernetes_pod_container_port_protocol": "",
|
||||
"__meta_kubernetes_pod_host_ip": "",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "",
|
||||
"__meta_kubernetes_pod_phase": "",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "",
|
||||
"__meta_kubernetes_service_cluster_ip": "",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "",
|
||||
}),
|
||||
}, [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.13.15.15:8081",
|
||||
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
|
||||
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
|
||||
"__meta_kubernetes_endpoint_port_name": "web",
|
||||
"__meta_kubernetes_endpoint_port_protocol": "https",
|
||||
"__meta_kubernetes_endpoint_ready": "true",
|
||||
"__meta_kubernetes_endpoints_name": "test-eps",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_node_label_node_label": "xyz",
|
||||
"__meta_kubernetes_node_labelpresent_node_label": "true",
|
||||
"__meta_kubernetes_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_phase": "abc",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "pod-uid",
|
||||
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "service-type",
|
||||
}),
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "192.168.15.1:8428",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_node_label_node_label": "xyz",
|
||||
"__meta_kubernetes_node_labelpresent_node_label": "true",
|
||||
"__meta_kubernetes_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_container_name": "metrics",
|
||||
"__meta_kubernetes_pod_container_port_name": "http-metrics",
|
||||
"__meta_kubernetes_pod_container_port_number": "8428",
|
||||
"__meta_kubernetes_pod_container_port_protocol": "foobar",
|
||||
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_phase": "abc",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "pod-uid",
|
||||
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "service-type",
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
f("1 port from endpoint", testArgs{
|
||||
containerPorts: map[string][]ContainerPort{"metrics": {{
|
||||
Name: "web",
|
||||
ContainerPort: 8428,
|
||||
}}},
|
||||
endpointPorts: []EndpointPort{
|
||||
{
|
||||
Name: "web",
|
||||
Port: 8428,
|
||||
t.Run("1 port from endpoint", func(t *testing.T) {
|
||||
f(t, testArgs{
|
||||
containerPorts: map[string][]ContainerPort{"metrics": {{
|
||||
Name: "web",
|
||||
ContainerPort: 8428,
|
||||
Protocol: "sdc",
|
||||
}}},
|
||||
endpointPorts: []EndpointPort{
|
||||
{
|
||||
Name: "web",
|
||||
Port: 8428,
|
||||
Protocol: "xabc",
|
||||
},
|
||||
},
|
||||
},
|
||||
}, [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.13.15.15:8428",
|
||||
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
|
||||
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
|
||||
"__meta_kubernetes_endpoint_port_name": "web",
|
||||
"__meta_kubernetes_endpoint_port_protocol": "",
|
||||
"__meta_kubernetes_endpoint_ready": "true",
|
||||
"__meta_kubernetes_endpoints_name": "test-eps",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_pod_container_name": "metrics",
|
||||
"__meta_kubernetes_pod_container_port_name": "web",
|
||||
"__meta_kubernetes_pod_container_port_number": "8428",
|
||||
"__meta_kubernetes_pod_container_port_protocol": "",
|
||||
"__meta_kubernetes_pod_host_ip": "",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "",
|
||||
"__meta_kubernetes_pod_phase": "",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "",
|
||||
"__meta_kubernetes_service_cluster_ip": "",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "",
|
||||
}),
|
||||
}, [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.13.15.15:8428",
|
||||
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
|
||||
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
|
||||
"__meta_kubernetes_endpoint_port_name": "web",
|
||||
"__meta_kubernetes_endpoint_port_protocol": "xabc",
|
||||
"__meta_kubernetes_endpoint_ready": "true",
|
||||
"__meta_kubernetes_endpoints_name": "test-eps",
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_node_label_node_label": "xyz",
|
||||
"__meta_kubernetes_node_labelpresent_node_label": "true",
|
||||
"__meta_kubernetes_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_container_name": "metrics",
|
||||
"__meta_kubernetes_pod_container_port_name": "web",
|
||||
"__meta_kubernetes_pod_container_port_number": "8428",
|
||||
"__meta_kubernetes_pod_container_port_protocol": "sdc",
|
||||
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
|
||||
"__meta_kubernetes_pod_ip": "192.168.15.1",
|
||||
"__meta_kubernetes_pod_name": "test-pod",
|
||||
"__meta_kubernetes_pod_node_name": "test-node",
|
||||
"__meta_kubernetes_pod_phase": "abc",
|
||||
"__meta_kubernetes_pod_ready": "unknown",
|
||||
"__meta_kubernetes_pod_uid": "pod-uid",
|
||||
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
|
||||
"__meta_kubernetes_service_name": "test-eps",
|
||||
"__meta_kubernetes_service_type": "service-type",
|
||||
}),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue