VictoriaMetrics/lib/promscrape/config_test.go
Aliaksandr Valialkin 855436efd2
lib/promauth: refactor NewConfig in order to improve maintainability
1. Split NewConfig into smaller functions
2. Introduce Options struct for simplifying construction of the Config with various options

This commit is based on https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2684
2022-07-04 14:31:12 +03:00

1948 lines
40 KiB
Go

package promscrape
import (
"bytes"
"fmt"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
)
func TestInternStringSerial(t *testing.T) {
if err := testInternString(t); err != nil {
t.Fatalf("unexpected error: %s", err)
}
}
func TestInternStringConcurrent(t *testing.T) {
concurrency := 5
resultCh := make(chan error, concurrency)
for i := 0; i < concurrency; i++ {
go func() {
resultCh <- testInternString(t)
}()
}
timer := time.NewTimer(5 * time.Second)
for i := 0; i < concurrency; i++ {
select {
case err := <-resultCh:
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
case <-timer.C:
t.Fatalf("timeout")
}
}
}
func testInternString(t *testing.T) error {
for i := 0; i < 1000; i++ {
s := fmt.Sprintf("foo_%d", i)
s1 := internString(s)
if s != s1 {
return fmt.Errorf("unexpected string returned from internString; got %q; want %q", s1, s)
}
}
return nil
}
func TestMergeLabels(t *testing.T) {
f := func(swc *scrapeWorkConfig, target string, extraLabels, metaLabels map[string]string, resultExpected string) {
t.Helper()
var labels []prompbmarshal.Label
labels = mergeLabels(labels[:0], swc, target, extraLabels, metaLabels)
result := promLabelsString(labels)
if result != resultExpected {
t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
}
}
f(&scrapeWorkConfig{}, "foo", nil, nil, `{__address__="foo",__metrics_path__="",__scheme__="",__scrape_interval__="",__scrape_timeout__="",job=""}`)
f(&scrapeWorkConfig{}, "foo", map[string]string{"foo": "bar"}, nil, `{__address__="foo",__metrics_path__="",__scheme__="",__scrape_interval__="",__scrape_timeout__="",foo="bar",job=""}`)
f(&scrapeWorkConfig{}, "foo", map[string]string{"job": "bar"}, nil, `{__address__="foo",__metrics_path__="",__scheme__="",__scrape_interval__="",__scrape_timeout__="",job="bar"}`)
f(&scrapeWorkConfig{
jobName: "xyz",
scheme: "https",
metricsPath: "/foo/bar",
scrapeIntervalString: "15s",
scrapeTimeoutString: "10s",
externalLabels: map[string]string{
"job": "bar",
"a": "b",
},
}, "foo", nil, nil, `{__address__="foo",__metrics_path__="/foo/bar",__scheme__="https",__scrape_interval__="15s",__scrape_timeout__="10s",a="b",job="xyz"}`)
f(&scrapeWorkConfig{
jobName: "xyz",
scheme: "https",
metricsPath: "/foo/bar",
externalLabels: map[string]string{
"job": "bar",
"a": "b",
},
}, "foo", map[string]string{
"job": "extra_job",
"foo": "extra_foo",
"a": "xyz",
}, map[string]string{
"__meta_x": "y",
}, `{__address__="foo",__meta_x="y",__metrics_path__="/foo/bar",__scheme__="https",__scrape_interval__="",__scrape_timeout__="",a="xyz",foo="extra_foo",job="extra_job"}`)
}
func TestScrapeConfigUnmarshalMarshal(t *testing.T) {
f := func(data string) {
t.Helper()
var cfg Config
data = strings.TrimSpace(data)
if err := cfg.unmarshal([]byte(data), true); err != nil {
t.Fatalf("parse error: %s\ndata:\n%s", err, data)
}
resultData := string(cfg.marshal())
result := strings.TrimSpace(resultData)
if result != data {
t.Fatalf("unexpected marshaled config:\ngot\n%s\nwant\n%s", result, data)
}
}
f(`
global:
scrape_interval: 10s
`)
f(`
scrape_config_files:
- foo
- bar
`)
f(`
scrape_configs:
- job_name: foo
scrape_timeout: 1.5s
static_configs:
- targets:
- foo
- bar
labels:
foo: bar
`)
f(`
scrape_configs:
- job_name: foo
honor_labels: true
honor_timestamps: false
scheme: https
params:
foo:
- x
authorization:
type: foobar
headers:
- 'TenantID: fooBar'
- 'X: y:z'
relabel_configs:
- source_labels: [abc]
static_configs:
- targets:
- foo
relabel_debug: true
scrape_align_interval: 1h30m0s
proxy_bearer_token_file: file.txt
proxy_headers:
- 'My-Auth-Header: top-secret'
`)
}
func TestNeedSkipScrapeWork(t *testing.T) {
f := func(key string, membersCount, replicationFactor, memberNum int, needSkipExpected bool) {
t.Helper()
needSkip := needSkipScrapeWork(key, membersCount, replicationFactor, memberNum)
if needSkip != needSkipExpected {
t.Fatalf("unexpected needSkipScrapeWork(key=%q, membersCount=%d, replicationFactor=%d, memberNum=%d); got %v; want %v",
key, membersCount, replicationFactor, memberNum, needSkip, needSkipExpected)
}
}
// Disabled clustering
f("foo", 0, 0, 0, false)
// A cluster with 2 nodes with disabled replication
f("foo", 2, 0, 0, true)
f("foo", 2, 0, 1, false)
// A cluster with 2 nodes with replicationFactor=2
f("foo", 2, 2, 0, false)
f("foo", 2, 2, 1, false)
// A cluster with 3 nodes with replicationFactor=2
f("foo", 3, 2, 0, false)
f("foo", 3, 2, 1, true)
f("foo", 3, 2, 2, false)
}
func TestLoadStaticConfigs(t *testing.T) {
scs, err := loadStaticConfigs("testdata/file_sd.json")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if len(scs) == 0 {
t.Fatalf("expecting non-zero static configs")
}
// Try loading non-existing file
scs, err = loadStaticConfigs("testdata/non-exsiting-file")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if scs != nil {
t.Fatalf("unexpected non-nil static configs: %#v", scs)
}
// Try loading invalid file
scs, err = loadStaticConfigs("testdata/prometheus.yml")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if scs != nil {
t.Fatalf("unexpected non-nil static configs: %#v", scs)
}
}
func TestLoadConfig(t *testing.T) {
cfg, data, err := loadConfig("testdata/prometheus.yml")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if cfg == nil {
t.Fatalf("expecting non-nil config")
}
if data == nil {
t.Fatalf("expecting non-nil data")
}
cfg, data, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if cfg == nil {
t.Fatalf("expecting non-nil config")
}
if data == nil {
t.Fatalf("expecting non-nil data")
}
// Try loading non-existing file
cfg, data, err = loadConfig("testdata/non-existing-file")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if cfg != nil {
t.Fatalf("unexpected non-nil config: %#v", cfg)
}
if data != nil {
t.Fatalf("unexpected data wit length=%d: %q", len(data), data)
}
// Try loading invalid file
cfg, data, err = loadConfig("testdata/file_sd_1.yml")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if cfg != nil {
t.Fatalf("unexpected non-nil config: %#v", cfg)
}
if data != nil {
t.Fatalf("unexpected data wit length=%d: %q", len(data), data)
}
}
func TestBlackboxExporter(t *testing.T) {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/684
data := `
scrape_configs:
- job_name: 'blackbox'
metrics_path: /probe
params:
module: [dns_udp_example] # Look for dns response
static_configs:
- targets:
- 8.8.8.8
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: black:9115 # The blackbox exporter's real hostname:port.%
`
var cfg Config
allData, err := cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parase data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws := cfg.getStaticScrapeWork()
resetNonEssentialFields(sws)
swsExpected := []*ScrapeWork{{
ScrapeURL: "http://black:9115/probe?module=dns_udp_example&target=8.8.8.8",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "black:9115",
},
{
Name: "__metrics_path__",
Value: "/probe",
},
{
Name: "__param_module",
Value: "dns_udp_example",
},
{
Name: "__param_target",
Value: "8.8.8.8",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "8.8.8.8",
},
{
Name: "job",
Value: "blackbox",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "blackbox",
}}
if !reflect.DeepEqual(sws, swsExpected) {
t.Fatalf("unexpected scrapeWork;\ngot\n%#v\nwant\n%#v", sws, swsExpected)
}
}
func TestGetFileSDScrapeWork(t *testing.T) {
data := `
scrape_configs:
- job_name: foo
file_sd_configs:
- files: [testdata/file_sd.json]
`
var cfg Config
allData, err := cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parase data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws := cfg.getFileSDScrapeWork(nil)
if !equalStaticConfigForScrapeWorks(sws, sws) {
t.Fatalf("unexpected non-equal static configs;\nsws:\n%#v", sws)
}
// Load another static config
dataNew := `
scrape_configs:
- job_name: foo
file_sd_configs:
- files: [testdata/file_sd_1.yml]
`
var cfgNew Config
allData, err = cfgNew.parseData([]byte(dataNew), "sss")
if err != nil {
t.Fatalf("cannot parse data: %s", err)
}
if string(allData) != dataNew {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, dataNew)
}
swsNew := cfgNew.getFileSDScrapeWork(sws)
if equalStaticConfigForScrapeWorks(swsNew, sws) {
t.Fatalf("unexpected equal static configs;\nswsNew:\n%#v\nsws:\n%#v", swsNew, sws)
}
// Try loading invalid static config
data = `
scrape_configs:
- job_name: foo
file_sd_configs:
- files: [testdata/prometheus.yml]
`
allData, err = cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parse data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws = cfg.getFileSDScrapeWork(swsNew)
if len(sws) != 0 {
t.Fatalf("unexpected non-empty sws:\n%#v", sws)
}
// Empty target in static config
data = `
scrape_configs:
- job_name: foo
file_sd_configs:
- files: [testdata/empty_target_file_sd.yml]
`
allData, err = cfg.parseData([]byte(data), "sss")
if err != nil {
t.Fatalf("cannot parse data: %s", err)
}
if string(allData) != data {
t.Fatalf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
sws = cfg.getFileSDScrapeWork(swsNew)
if len(sws) != 0 {
t.Fatalf("unexpected non-empty sws:\n%#v", sws)
}
}
func getFileSDScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
var cfg Config
allData, err := cfg.parseData(data, path)
if err != nil {
return nil, fmt.Errorf("cannot parse data: %w", err)
}
if !bytes.Equal(allData, data) {
return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
return cfg.getFileSDScrapeWork(nil), nil
}
func getStaticScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
var cfg Config
allData, err := cfg.parseData(data, path)
if err != nil {
return nil, fmt.Errorf("cannot parse data: %w", err)
}
if !bytes.Equal(allData, data) {
return nil, fmt.Errorf("invalid data returned from parseData;\ngot\n%s\nwant\n%s", allData, data)
}
return cfg.getStaticScrapeWork(), nil
}
func TestGetStaticScrapeWorkFailure(t *testing.T) {
f := func(data string) {
t.Helper()
sws, err := getStaticScrapeWork([]byte(data), "non-existing-file")
if err == nil {
t.Fatalf("expecting non-nil error")
}
if sws != nil {
t.Fatalf("expecting nil sws")
}
}
// incorrect yaml
f(`foo bar baz`)
// Missing job_name
f(`
scrape_configs:
- static_configs:
- targets: ["foo"]
`)
// Duplicate job_name
f(`
scrape_configs:
- job_name: foo
static_configs:
targets: ["foo"]
- job_name: foo
static_configs:
targets: ["bar"]
`)
// Invalid scheme
f(`
scrape_configs:
- job_name: x
scheme: asdf
static_configs:
- targets: ["foo"]
`)
// Missing username in `basic_auth`
f(`
scrape_configs:
- job_name: x
basic_auth:
password: sss
static_configs:
- targets: ["a"]
`)
// Both password and password_file set in `basic_auth`
f(`
scrape_configs:
- job_name: x
basic_auth:
username: foobar
password: sss
password_file: sdfdf
static_configs:
- targets: ["a"]
`)
// Invalid password_file set in `basic_auth`
f(`
scrape_configs:
- job_name: x
basic_auth:
username: foobar
password_file: ['foobar']
static_configs:
- targets: ["a"]
`)
// Both `bearer_token` and `bearer_token_file` are set
f(`
scrape_configs:
- job_name: x
bearer_token: foo
bearer_token_file: bar
static_configs:
- targets: ["a"]
`)
// Both `basic_auth` and `bearer_token` are set
f(`
scrape_configs:
- job_name: x
bearer_token: foo
basic_auth:
username: foo
password: bar
static_configs:
- targets: ["a"]
`)
// Both `authorization` and `basic_auth` are set
f(`
scrape_configs:
- job_name: x
authorization:
credentials: foobar
basic_auth:
username: foobar
static_configs:
- targets: ["a"]
`)
// Both `authorization` and `bearer_token` are set
f(`
scrape_configs:
- job_name: x
authorization:
credentials: foobar
bearer_token: foo
static_configs:
- targets: ["a"]
`)
// Invalid `bearer_token_file`
f(`
scrape_configs:
- job_name: x
bearer_token_file: [foobar]
static_configs:
- targets: ["a"]
`)
// non-existing ca_file
f(`
scrape_configs:
- job_name: aa
tls_config:
ca_file: non/extising/file
static_configs:
- targets: ["s"]
`)
// invalid ca_file
f(`
scrape_configs:
- job_name: aa
tls_config:
ca_file: testdata/prometheus.yml
static_configs:
- targets: ["s"]
`)
// non-existing cert_file
f(`
scrape_configs:
- job_name: aa
tls_config:
cert_file: non/extising/file
static_configs:
- targets: ["s"]
`)
// non-existing key_file
f(`
scrape_configs:
- job_name: aa
tls_config:
key_file: non/extising/file
static_configs:
- targets: ["s"]
`)
// Invalid regex in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- regex: "("
source_labels: [foo]
target_label: bar
static_configs:
- targets: ["s"]
`)
// Missing target_label for action=replace in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: replace
source_labels: [foo]
static_configs:
- targets: ["s"]
`)
// Missing source_labels for action=keep in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: keep
static_configs:
- targets: ["s"]
`)
// Missing source_labels for action=drop in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: drop
static_configs:
- targets: ["s"]
`)
// Missing source_labels for action=hashmod in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: hashmod
target_label: bar
modulus: 123
static_configs:
- targets: ["s"]
`)
// Missing target for action=hashmod in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: hashmod
source_labels: [foo]
modulus: 123
static_configs:
- targets: ["s"]
`)
// Missing modulus for action=hashmod in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: hashmod
source_labels: [foo]
target_label: bar
static_configs:
- targets: ["s"]
`)
// Invalid action in relabel_configs
f(`
scrape_configs:
- job_name: aa
relabel_configs:
- action: foobar
static_configs:
- targets: ["s"]
`)
// Invalid scrape_config_files contents
f(`
scrape_config_files:
- job_name: aa
static_configs:
- targets: ["s"]
`)
}
func resetNonEssentialFields(sws []*ScrapeWork) {
for i := range sws {
sws[i].OriginalLabels = nil
}
}
// String returns human-readable representation for sw.
func (sw *ScrapeWork) String() string {
return strconv.Quote(sw.key())
}
func TestGetFileSDScrapeWorkSuccess(t *testing.T) {
f := func(data string, expectedSws []*ScrapeWork) {
t.Helper()
sws, err := getFileSDScrapeWork([]byte(data), "non-existing-file")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
resetNonEssentialFields(sws)
// Remove `__vm_filepath` label, since its value depends on the current working dir.
for _, sw := range sws {
for j := range sw.Labels {
label := &sw.Labels[j]
if label.Name == "__vm_filepath" {
label.Value = ""
}
}
}
if !reflect.DeepEqual(sws, expectedSws) {
t.Fatalf("unexpected scrapeWork; got\n%+v\nwant\n%+v", sws, expectedSws)
}
}
f(`
scrape_configs:
- job_name: foo
static_configs:
- targets: ["xxx"]
`, []*ScrapeWork{})
f(`
scrape_configs:
- job_name: foo
metrics_path: /abc/de
file_sd_configs:
- files: ["testdata/file_sd.json", "testdata/file_sd*.yml"]
`, []*ScrapeWork{
{
ScrapeURL: "http://host1:80/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "host1",
},
{
Name: "__metrics_path__",
Value: "/abc/de",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "__vm_filepath",
Value: "",
},
{
Name: "instance",
Value: "host1:80",
},
{
Name: "job",
Value: "foo",
},
{
Name: "qwe",
Value: "rty",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://host2:80/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "host2",
},
{
Name: "__metrics_path__",
Value: "/abc/de",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "__vm_filepath",
Value: "",
},
{
Name: "instance",
Value: "host2:80",
},
{
Name: "job",
Value: "foo",
},
{
Name: "qwe",
Value: "rty",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://localhost:9090/abc/de",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "localhost:9090",
},
{
Name: "__metrics_path__",
Value: "/abc/de",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "__vm_filepath",
Value: "",
},
{
Name: "instance",
Value: "localhost:9090",
},
{
Name: "job",
Value: "foo",
},
{
Name: "yml",
Value: "test",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
}
func TestGetStaticScrapeWorkSuccess(t *testing.T) {
f := func(data string, expectedSws []*ScrapeWork) {
t.Helper()
sws, err := getStaticScrapeWork([]byte(data), "non-exsiting-file")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
resetNonEssentialFields(sws)
if !reflect.DeepEqual(sws, expectedSws) {
t.Fatalf("unexpected scrapeWork; got\n%+v\nwant\n%+v", sws, expectedSws)
}
}
f(``, nil)
f(`
scrape_configs:
- job_name: foo
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "foo",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
global:
external_labels:
datacenter: foobar
jobs: xxx
scrape_configs:
- job_name: foo
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "datacenter",
Value: "foobar",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "foo",
},
{
Name: "jobs",
Value: "xxx",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
global:
scrape_interval: 8s
scrape_timeout: 34s
scrape_configs:
- job_name: foo
scrape_interval: 54s
scrape_timeout: 12s
metrics_path: /foo/bar
scheme: https
honor_labels: true
honor_timestamps: false
follow_redirects: false
params:
p: ["x&y", "="]
xaa:
proxy_url: http://foo.bar
static_configs:
- targets: ["foo.bar", "aaa"]
labels:
x: y
__scrape_timeout__: "5s"
- job_name: qwer
tls_config:
server_name: foobar
insecure_skip_verify: true
static_configs:
- targets: [1.2.3.4]
- job_name: asdf
static_configs:
- targets: [foobar]
`, []*ScrapeWork{
{
ScrapeURL: "https://foo.bar:443/foo/bar?p=x%26y&p=%3D",
ScrapeInterval: 54 * time.Second,
ScrapeTimeout: 5 * time.Second,
HonorLabels: true,
HonorTimestamps: false,
DenyRedirects: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar",
},
{
Name: "__metrics_path__",
Value: "/foo/bar",
},
{
Name: "__param_p",
Value: "x&y",
},
{
Name: "__scheme__",
Value: "https",
},
{
Name: "__scrape_interval__",
Value: "54s",
},
{
Name: "__scrape_timeout__",
Value: "5s",
},
{
Name: "instance",
Value: "foo.bar:443",
},
{
Name: "job",
Value: "foo",
},
{
Name: "x",
Value: "y",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
ProxyURL: proxy.MustNewURL("http://foo.bar"),
jobNameOriginal: "foo",
},
{
ScrapeURL: "https://aaa:443/foo/bar?p=x%26y&p=%3D",
ScrapeInterval: 54 * time.Second,
ScrapeTimeout: 5 * time.Second,
HonorLabels: true,
HonorTimestamps: false,
DenyRedirects: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "aaa",
},
{
Name: "__metrics_path__",
Value: "/foo/bar",
},
{
Name: "__param_p",
Value: "x&y",
},
{
Name: "__scheme__",
Value: "https",
},
{
Name: "__scrape_interval__",
Value: "54s",
},
{
Name: "__scrape_timeout__",
Value: "5s",
},
{
Name: "instance",
Value: "aaa:443",
},
{
Name: "job",
Value: "foo",
},
{
Name: "x",
Value: "y",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
ProxyURL: proxy.MustNewURL("http://foo.bar"),
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://1.2.3.4:80/metrics",
ScrapeInterval: 8 * time.Second,
ScrapeTimeout: 8 * time.Second,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "1.2.3.4",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "8s",
},
{
Name: "__scrape_timeout__",
Value: "8s",
},
{
Name: "instance",
Value: "1.2.3.4:80",
},
{
Name: "job",
Value: "qwer",
},
},
AuthConfig: &promauth.Config{
TLSServerName: "foobar",
TLSInsecureSkipVerify: true,
},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "qwer",
},
{
ScrapeURL: "http://foobar:80/metrics",
ScrapeInterval: 8 * time.Second,
ScrapeTimeout: 8 * time.Second,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foobar",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "8s",
},
{
Name: "__scrape_timeout__",
Value: "8s",
},
{
Name: "instance",
Value: "foobar:80",
},
{
Name: "job",
Value: "asdf",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "asdf",
},
})
f(`
scrape_configs:
- job_name: foo
relabel_configs:
- source_labels: [__scheme__, __address__]
separator: "://"
target_label: __tmp_url
- source_labels: [__tmp_url, __metrics_path__]
separator: ""
target_label: url
- action: labeldrop
regex: "job|__tmp_.+"
- action: drop
source_labels: [__address__]
regex: "drop-.*"
- action: keep
source_labels: [__param_x]
regex: keep_me
- action: labelkeep
regex: "__.*|url"
- action: labelmap
regex: "(url)"
replacement: "prefix:${1}"
- action: hashmod
modulus: 123
source_labels: [__address__]
target_label: hash
- action: replace
source_labels: [__address__]
target_label: foobar
replacement: ""
params:
x: [keep_me]
static_configs:
- targets: ["foo.bar:1234", "drop-this-target"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics?x=keep_me",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__param_x",
Value: "keep_me",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "hash",
Value: "82",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "prefix:url",
Value: "http://foo.bar:1234/metrics",
},
{
Name: "url",
Value: "http://foo.bar:1234/metrics",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
scrape_configs:
- job_name: foo
scheme: https
relabel_configs:
- action: replace
source_labels: [non-existing-label]
target_label: instance
replacement: fake.addr
- action: replace
source_labels: [__address__]
target_label: foobar
regex: "missing-regex"
replacement: aaabbb
- action: replace
source_labels: [__scheme__]
target_label: job
- action: replace
source_labels: [__scheme__]
target_label: __scheme__
replacement: mailto
- target_label: __metrics_path__
replacement: /abc.de
- target_label: __param_a
replacement: b
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "mailto://foo.bar:1234/abc.de?a=b",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/abc.de",
},
{
Name: "__param_a",
Value: "b",
},
{
Name: "__scheme__",
Value: "mailto",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "fake.addr",
},
{
Name: "job",
Value: "https",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
scrape_configs:
- job_name: foo
scheme: https
relabel_configs:
- action: keep
source_labels: [__address__]
regex: "foo\\.bar:.*"
- action: hashmod
source_labels: [job]
modulus: 4
target_label: job
- action: labeldrop
regex: "non-matching-regex"
- action: labelkeep
regex: "job|__address__"
- action: labeldrop
regex: ""
static_configs:
- targets: ["foo.bar:1234", "xyz"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "3",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
scrape_configs:
- job_name: foo
metric_relabel_configs:
- source_labels: [foo]
target_label: abc
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "foo",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
MetricRelabelConfigs: mustParseRelabelConfigs(`
- source_labels: [foo]
target_label: abc
`),
jobNameOriginal: "foo",
},
})
f(`
scrape_configs:
- job_name: foo
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "foo",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
scrape_configs:
- job_name: foo
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "foo",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
f(`
global:
external_labels:
job: foobar
foo: xx
q: qwe
__address__: aaasdf
__param_a: jlfd
scrape_configs:
- job_name: aaa
params:
a: [b, xy]
static_configs:
- targets: ["a"]
labels:
foo: bar
__param_a: c
__address__: pp
job: yyy
`, []*ScrapeWork{
{
ScrapeURL: "http://pp:80/metrics?a=c&a=xy",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "pp",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__param_a",
Value: "c",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "foo",
Value: "bar",
},
{
Name: "instance",
Value: "pp:80",
},
{
Name: "job",
Value: "yyy",
},
{
Name: "q",
Value: "qwe",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "aaa",
},
})
opts := &promauth.Options{
Headers: []string{"My-Auth: foo-Bar"},
}
ac, err := opts.NewConfig()
if err != nil {
t.Fatalf("unexpected error when creating promauth.Config: %s", err)
}
opts = &promauth.Options{
Headers: []string{"Foo:bar"},
}
proxyAC, err := opts.NewConfig()
if err != nil {
t.Fatalf("unexpected error when creating promauth.Config for proxy: %s", err)
}
f(`
scrape_configs:
- job_name: 'snmp'
sample_limit: 100
disable_keepalive: true
disable_compression: true
headers:
- "My-Auth: foo-Bar"
proxy_headers:
- "Foo: bar"
scrape_align_interval: 1s
scrape_offset: 0.5s
static_configs:
- targets:
- 192.168.1.2 # SNMP device.
metrics_path: /snmp
params:
module: [if_mib]
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: 127.0.0.1:9116 # The SNMP exporter's real hostname:port.
- target_label: __series_limit__
replacement: 1234
- target_label: __stream_parse__
replacement: true
`, []*ScrapeWork{
{
ScrapeURL: "http://127.0.0.1:9116/snmp?module=if_mib&target=192.168.1.2",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "127.0.0.1:9116",
},
{
Name: "__metrics_path__",
Value: "/snmp",
},
{
Name: "__param_module",
Value: "if_mib",
},
{
Name: "__param_target",
Value: "192.168.1.2",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "__series_limit__",
Value: "1234",
},
{
Name: "__stream_parse__",
Value: "true",
},
{
Name: "instance",
Value: "192.168.1.2",
},
{
Name: "job",
Value: "snmp",
},
},
AuthConfig: ac,
ProxyAuthConfig: proxyAC,
SampleLimit: 100,
DisableKeepAlive: true,
DisableCompression: true,
StreamParse: true,
ScrapeAlignInterval: time.Second,
ScrapeOffset: 500 * time.Millisecond,
SeriesLimit: 1234,
jobNameOriginal: "snmp",
},
})
f(`
scrape_configs:
- job_name: path wo slash
static_configs:
- targets: ["foo.bar:1234"]
relabel_configs:
- replacement: metricspath
target_label: __metrics_path__
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metricspath",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "metricspath",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "1m0s",
},
{
Name: "__scrape_timeout__",
Value: "10s",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "path wo slash",
},
},
jobNameOriginal: "path wo slash",
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
},
})
f(`
global:
scrape_timeout: 1d
scrape_configs:
- job_name: foo
scrape_interval: 1w
scrape_align_interval: 1d
scrape_offset: 2d
static_configs:
- targets: ["foo.bar:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo.bar:1234/metrics",
ScrapeInterval: time.Hour * 24 * 7,
ScrapeTimeout: time.Hour * 24,
ScrapeAlignInterval: time.Hour * 24,
ScrapeOffset: time.Hour * 24 * 2,
HonorTimestamps: true,
Labels: []prompbmarshal.Label{
{
Name: "__address__",
Value: "foo.bar:1234",
},
{
Name: "__metrics_path__",
Value: "/metrics",
},
{
Name: "__scheme__",
Value: "http",
},
{
Name: "__scrape_interval__",
Value: "168h0m0s",
},
{
Name: "__scrape_timeout__",
Value: "24h0m0s",
},
{
Name: "instance",
Value: "foo.bar:1234",
},
{
Name: "job",
Value: "foo",
},
},
AuthConfig: &promauth.Config{},
ProxyAuthConfig: &promauth.Config{},
jobNameOriginal: "foo",
},
})
}
func equalStaticConfigForScrapeWorks(a, b []*ScrapeWork) bool {
if len(a) != len(b) {
return false
}
for i := range a {
keyA := a[i].key()
keyB := b[i].key()
if keyA != keyB {
return false
}
}
return true
}
func TestScrapeConfigClone(t *testing.T) {
f := func(sc *ScrapeConfig) {
t.Helper()
scCopy := sc.clone()
if !reflect.DeepEqual(sc, scCopy) {
t.Fatalf("unexpected result after unmarshalJSON() for JSON:\n%s", sc.marshalJSON())
}
}
f(&ScrapeConfig{})
bFalse := false
var ie promrelabel.IfExpression
if err := ie.Parse(`{foo=~"bar",baz!="z"}`); err != nil {
t.Fatalf("unexpected error: %s", err)
}
f(&ScrapeConfig{
JobName: "foo",
ScrapeInterval: promutils.NewDuration(time.Second * 47),
HonorLabels: true,
HonorTimestamps: &bFalse,
Params: map[string][]string{
"foo": {"bar", "baz"},
},
HTTPClientConfig: promauth.HTTPClientConfig{
Authorization: &promauth.Authorization{
Credentials: promauth.NewSecret("foo"),
},
BasicAuth: &promauth.BasicAuthConfig{
Username: "user_x",
Password: promauth.NewSecret("pass_x"),
},
BearerToken: promauth.NewSecret("zx"),
OAuth2: &promauth.OAuth2Config{
ClientSecret: promauth.NewSecret("aa"),
Scopes: []string{"foo", "bar"},
TLSConfig: &promauth.TLSConfig{
CertFile: "foo",
},
},
TLSConfig: &promauth.TLSConfig{
KeyFile: "aaa",
},
},
ProxyURL: proxy.MustNewURL("https://foo.bar:3434/assdf/dsfd?sdf=dsf"),
RelabelConfigs: []promrelabel.RelabelConfig{{
SourceLabels: []string{"foo", "aaa"},
Regex: &promrelabel.MultiLineRegex{
S: "foo\nbar",
},
If: &ie,
}},
SampleLimit: 10,
GCESDConfigs: []gce.SDConfig{{
Project: "foo",
Zone: gce.ZoneYAML{
Zones: []string{"a", "b"},
},
}},
StreamParse: true,
ProxyClientConfig: promauth.ProxyClientConfig{
BearerTokenFile: "foo",
},
})
}