VictoriaMetrics/app/vmctl/remote_read_test.go

package main

import (
	"context"
	"net/http"
	"testing"
	"time"

	"github.com/prometheus/prometheus/prompb"

	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
)

func TestRemoteRead(t *testing.T) {
	barpool.Disable(true)
	defer func() {
		barpool.Disable(false)
	}()
	defer func() { isSilent = false }()

	var testCases = []struct {
		name             string
		remoteReadConfig remoteread.Config
		vmCfg            vm.Config
		start            string
		end              string
		numOfSamples     int64
		numOfSeries      int64
		rrp              remoteReadProcessor
		chunk            string
		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
		expectedSeries   []vm.TimeSeries
	}{
		{
			name:             "step minute on minute time range",
			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
			start:            "2022-11-26T11:23:05+02:00",
			end:              "2022-11-26T11:24:05+02:00",
			numOfSamples:     2,
			numOfSeries:      3,
			chunk:            stepper.StepMinute,
			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
			expectedSeries: []vm.TimeSeries{
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
					Timestamps: []int64{1669454585000, 1669454615000},
					Values:     []float64{0, 0},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
					Timestamps: []int64{1669454585000, 1669454615000},
					Values:     []float64{100, 100},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
					Timestamps: []int64{1669454585000, 1669454615000},
					Values:     []float64{200, 200},
				},
			},
		},
		{
			name:             "step month on month time range",
			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
			vmCfg: vm.Config{Addr: "", Concurrency: 1,
				Transport: http.DefaultTransport.(*http.Transport)},
			start:            "2022-09-26T11:23:05+02:00",
			end:              "2022-11-26T11:24:05+02:00",
			numOfSamples:     2,
			numOfSeries:      3,
			chunk:            stepper.StepMonth,
			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
			expectedSeries: []vm.TimeSeries{
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
					Timestamps: []int64{1664184185000},
					Values:     []float64{0},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
					Timestamps: []int64{1664184185000},
					Values:     []float64{100},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
					Timestamps: []int64{1664184185000},
					Values:     []float64{200},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
					Timestamps: []int64{1666819415000},
					Values:     []float64{0},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
					Timestamps: []int64{1666819415000},
					Values:     []float64{100},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
					Timestamps: []int64{1666819415000},
					Values:     []float64{200}},
			},
		},
	}

	for _, tt := range testCases {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
			defer remoteReadServer.Close()
			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
			defer remoteWriteServer.Close()

			tt.remoteReadConfig.Addr = remoteReadServer.URL()

			rr, err := remoteread.NewClient(tt.remoteReadConfig)
			if err != nil {
				t.Fatalf("error create remote read client: %s", err)
			}

			start, err := time.Parse(time.RFC3339, tt.start)
			if err != nil {
				t.Fatalf("Error parse start time: %s", err)
			}

			end, err := time.Parse(time.RFC3339, tt.end)
			if err != nil {
				t.Fatalf("Error parse end time: %s", err)
			}

			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)

			remoteReadServer.SetRemoteReadSeries(rrs)
			remoteWriteServer.ExpectedSeries(tt.expectedSeries)

			tt.vmCfg.Addr = remoteWriteServer.URL()

			b, err := backoff.New(10, 1.8, time.Second*2)
			if err != nil {
				t.Fatalf("failed to create backoff: %s", err)
			}
			tt.vmCfg.Backoff = b

			importer, err := vm.NewImporter(ctx, tt.vmCfg)
			if err != nil {
				t.Fatalf("failed to create VM importer: %s", err)
			}
			defer importer.Close()

			rmp := remoteReadProcessor{
				src: rr,
				dst: importer,
				filter: remoteReadFilter{
					timeStart: &start,
					timeEnd:   &end,
					chunk:     tt.chunk,
				},
				cc:        1,
				isVerbose: false,
			}

			err = rmp.run(ctx)
			if err != nil {
				t.Fatalf("failed to run remote read processor: %s", err)
			}
		})
	}
}

func TestSteamRemoteRead(t *testing.T) {
	barpool.Disable(true)
	defer func() {
		barpool.Disable(false)
	}()
	defer func() { isSilent = false }()

	var testCases = []struct {
		name             string
		remoteReadConfig remoteread.Config
		vmCfg            vm.Config
		start            string
		end              string
		numOfSamples     int64
		numOfSeries      int64
		rrp              remoteReadProcessor
		chunk            string
		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
		expectedSeries   []vm.TimeSeries
	}{
		{
			name:             "step minute on minute time range",
			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
			start:            "2022-11-26T11:23:05+02:00",
			end:              "2022-11-26T11:24:05+02:00",
			numOfSamples:     2,
			numOfSeries:      3,
			chunk:            stepper.StepMinute,
			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
			expectedSeries: []vm.TimeSeries{
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
					Timestamps: []int64{1669454585000, 1669454615000},
					Values:     []float64{0, 0},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
					Timestamps: []int64{1669454585000, 1669454615000},
					Values:     []float64{100, 100},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
					Timestamps: []int64{1669454585000, 1669454615000},
					Values:     []float64{200, 200},
				},
			},
		},
		{
			name:             "step month on month time range",
			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
			start:            "2022-09-26T11:23:05+02:00",
			end:              "2022-11-26T11:24:05+02:00",
			numOfSamples:     2,
			numOfSeries:      3,
			chunk:            stepper.StepMonth,
			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
			expectedSeries: []vm.TimeSeries{
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
					Timestamps: []int64{1664184185000},
					Values:     []float64{0},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
					Timestamps: []int64{1664184185000},
					Values:     []float64{100},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
					Timestamps: []int64{1664184185000},
					Values:     []float64{200},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
					Timestamps: []int64{1666819415000},
					Values:     []float64{0},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
					Timestamps: []int64{1666819415000},
					Values:     []float64{100},
				},
				{
					Name:       "vm_metric_1",
					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
					Timestamps: []int64{1666819415000},
					Values:     []float64{200}},
			},
		},
	}

	for _, tt := range testCases {
		t.Run(tt.name, func(t *testing.T) {
			ctx := context.Background()
			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
			defer remoteReadServer.Close()
			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
			defer remoteWriteServer.Close()

			tt.remoteReadConfig.Addr = remoteReadServer.URL()

			rr, err := remoteread.NewClient(tt.remoteReadConfig)
			if err != nil {
				t.Fatalf("error create remote read client: %s", err)
			}

			start, err := time.Parse(time.RFC3339, tt.start)
			if err != nil {
				t.Fatalf("Error parse start time: %s", err)
			}

			end, err := time.Parse(time.RFC3339, tt.end)
			if err != nil {
				t.Fatalf("Error parse end time: %s", err)
			}

			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)

			remoteReadServer.InitMockStorage(rrs)
			remoteWriteServer.ExpectedSeries(tt.expectedSeries)

			tt.vmCfg.Addr = remoteWriteServer.URL()

			b, err := backoff.New(10, 1.8, time.Second*2)
			if err != nil {
				t.Fatalf("failed to create backoff: %s", err)
			}

			tt.vmCfg.Backoff = b
			importer, err := vm.NewImporter(ctx, tt.vmCfg)
			if err != nil {
				t.Fatalf("failed to create VM importer: %s", err)
			}
			defer importer.Close()

			rmp := remoteReadProcessor{
				src: rr,
				dst: importer,
				filter: remoteReadFilter{
					timeStart: &start,
					timeEnd:   &end,
					chunk:     tt.chunk,
				},
				cc:        1,
				isVerbose: false,
			}

			err = rmp.run(ctx)
			if err != nil {
				t.Fatalf("failed to run remote read processor: %s", err)
			}
		})
	}
}