tests: couple applications and test suit (#7476)

* make test suite responisble for stopping apps
* reuse test suite fields to simplify function signatures

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>

(cherry picked from commit e60cce54a8)
Signed-off-by: hagen1778 <roman@victoriametrics.com>

apptest/testcase.go

@@ -11,11 +11,18 @@ import (
 type TestCase struct {
 	t   *testing.T
 	cli *Client
+
+	startedApps []Stopper
+}
+
+// Stopper is an interface of objects that needs to be stopped via Stop() call
+type Stopper interface {
+	Stop()
 }
 
 // NewTestCase creates a new test case.
 func NewTestCase(t *testing.T) *TestCase {
-	return &TestCase{t, NewClient()}
+	return &TestCase{t, NewClient(), nil}
 }
 
 // Dir returns the directory name that should be used by as the -storageDataDir.
@@ -29,14 +36,73 @@ func (tc *TestCase) Client() *Client {
 	return tc.cli
 }
 
-// Close performs the test case clean up, such as closing all client connections
+// Stop performs the test case clean up, such as closing all client connections
 // and removing the -storageDataDir directory.
 //
 // Note that the -storageDataDir is not removed in case of test case failure to
-// allow for furher manual debugging.
-func (tc *TestCase) Close() {
+// allow for further manual debugging.
+func (tc *TestCase) Stop() {
 	tc.cli.CloseConnections()
+	for _, app := range tc.startedApps {
+		app.Stop()
+	}
 	if !tc.t.Failed() {
 		fs.MustRemoveAll(tc.Dir())
 	}
 }
+
+// MustStartVmsingle is a test helper function that starts an instance of
+// vmsingle and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
+	tc.t.Helper()
+
+	app, err := StartVmsingle(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+// MustStartVmstorage is a test helper function that starts an instance of
+// vmstorage and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstorage {
+	tc.t.Helper()
+
+	app, err := StartVmstorage(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+// MustStartVmselect is a test helper function that starts an instance of
+// vmselect and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmselect(instance string, flags []string) *Vmselect {
+	tc.t.Helper()
+
+	app, err := StartVmselect(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+// MustStartVminsert is a test helper function that starts an instance of
+// vminsert and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVminsert(instance string, flags []string) *Vminsert {
+	tc.t.Helper()
+
+	app, err := StartVminsert(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(app)
+	return app
+}
+
+func (tc *TestCase) addApp(app Stopper) {
+	tc.startedApps = append(tc.startedApps, app)
+}

apptest/tests/key_concepts_test.go

@@ -1,11 +1,11 @@
 package tests
 
 import (
-	"testing"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"testing"
+	"time"
 )
 
 // Data used in examples in
@@ -30,15 +30,12 @@ var docData = []string{
 // TestSingleKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
 func TestSingleKeyConceptsQuery(t *testing.T) {
 	tc := apptest.NewTestCase(t)
-	defer tc.Close()
+	defer tc.Stop()
 
-	cli := tc.Client()
-
-	vmsingle := apptest.MustStartVmsingle(t, "vmsingle", []string{
+	vmsingle := tc.MustStartVmsingle("vmsingle", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage",
 		"-retentionPeriod=100y",
-	}, cli)
-	defer vmsingle.Stop()
+	})
 
 	opts := apptest.QueryOpts{Timeout: "5s"}
 
@@ -53,7 +50,7 @@ func TestSingleKeyConceptsQuery(t *testing.T) {
 // TestClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
 func TestClusterKeyConceptsQuery(t *testing.T) {
 	tc := apptest.NewTestCase(t)
-	defer tc.Close()
+	defer tc.Stop()
 
 	// Set up the following cluster configuration:
 	//
@@ -64,29 +61,27 @@ func TestClusterKeyConceptsQuery(t *testing.T) {
 	// - vmselect points to the two vmstorages and is expected to query both
 	//   vmstorages and build the full result out of the two partial results.
 
-	cli := tc.Client()
-
-	vmstorage1 := apptest.MustStartVmstorage(t, "vmstorage-1", []string{
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
-	}, cli)
-	defer vmstorage1.Stop()
-	vmstorage2 := apptest.MustStartVmstorage(t, "vmstorage-2", []string{
+		"-retentionPeriod=100y",
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
-	}, cli)
-	defer vmstorage2.Stop()
-	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
-	}, cli)
-	defer vminsert.Stop()
-	vmselect := apptest.MustStartVmselect(t, "vmselect", []string{
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
 		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
-	}, cli)
-	defer vmselect.Stop()
+	})
 
 	opts := apptest.QueryOpts{Timeout: "5s", Tenant: "0"}
 
 	// Insert example data from documentation.
 	vminsert.PrometheusAPIV1ImportPrometheus(t, docData, opts)
+	time.Sleep(2 * time.Second)
+
 	vmstorage1.ForceFlush(t)
 	vmstorage2.ForceFlush(t)
 

apptest/tests/multilevel_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestClusterMultilevelSelect(t *testing.T) {
 	tc := apptest.NewTestCase(t)
-	defer tc.Close()
+	defer tc.Stop()
 
 	// Set up the following multi-level cluster configuration:
 	//
@@ -20,24 +20,18 @@ func TestClusterMultilevelSelect(t *testing.T) {
 	// vmisert writes data into vmstorage.
 	// vmselect (L2) reads that data via vmselect (L1).
 
-	cli := tc.Client()
-
-	vmstorage := apptest.MustStartVmstorage(t, "vmstorage", []string{
+	vmstorage := tc.MustStartVmstorage("vmstorage", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage",
-	}, cli)
-	defer vmstorage.Stop()
-	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
 		"-storageNode=" + vmstorage.VminsertAddr(),
-	}, cli)
-	defer vminsert.Stop()
-	vmselectL1 := apptest.MustStartVmselect(t, "vmselect-level1", []string{
+	})
+	vmselectL1 := tc.MustStartVmselect("vmselect-level1", []string{
 		"-storageNode=" + vmstorage.VmselectAddr(),
-	}, cli)
-	defer vmselectL1.Stop()
-	vmselectL2 := apptest.MustStartVmselect(t, "vmselect-level2", []string{
+	})
+	vmselectL2 := tc.MustStartVmselect("vmselect-level2", []string{
 		"-storageNode=" + vmselectL1.ClusternativeListenAddr(),
-	}, cli)
-	defer vmselectL2.Stop()
+	})
 
 	// Insert 1000 unique time series.Wait for 2 seconds to let vmstorage
 	// flush pending items so they become searchable.

apptest/tests/sharding_test.go

@@ -11,7 +11,7 @@ import (
 
 func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.T) {
 	tc := apptest.NewTestCase(t)
-	defer tc.Close()
+	defer tc.Stop()
 
 	// Set up the following cluster configuration:
 	//
@@ -22,24 +22,18 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
 	// - vmselect points to the two vmstorages and is expected to query both
 	//   vmstorages and build the full result out of the two partial results.
 
-	cli := tc.Client()
-
-	vmstorage1 := apptest.MustStartVmstorage(t, "vmstorage-1", []string{
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
-	}, cli)
-	defer vmstorage1.Stop()
-	vmstorage2 := apptest.MustStartVmstorage(t, "vmstorage-2", []string{
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
 		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
-	}, cli)
-	defer vmstorage2.Stop()
-	vminsert := apptest.MustStartVminsert(t, "vminsert", []string{
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
-	}, cli)
-	defer vminsert.Stop()
-	vmselect := apptest.MustStartVmselect(t, "vmselect", []string{
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
 		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
-	}, cli)
-	defer vmselect.Stop()
+	})
 
 	// Insert 1000 unique time series and verify the that inserted data has been
 	// indeed sharded by checking various metrics exposed by vminsert and

apptest/vminsert.go

@@ -18,19 +18,6 @@ type Vminsert struct {
 	cli            *Client
 }
 
-// MustStartVminsert is a test helper function that starts an instance of
-// vminsert and fails the test if the app fails to start.
-func MustStartVminsert(t *testing.T, instance string, flags []string, cli *Client) *Vminsert {
-	t.Helper()
-
-	app, err := StartVminsert(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
-}
-
 // StartVminsert starts an instance of vminsert with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)

apptest/vmselect.go

@@ -19,19 +19,6 @@ type Vmselect struct {
 	cli                     *Client
 }
 
-// MustStartVmselect is a test helper function that starts an instance of
-// vmselect and fails the test if the app fails to start.
-func MustStartVmselect(t *testing.T, instance string, flags []string, cli *Client) *Vmselect {
-	t.Helper()
-
-	app, err := StartVmselect(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
-}
-
 // StartVmselect starts an instance of vmselect with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)

apptest/vmsingle.go

@@ -27,19 +27,6 @@ type Vmsingle struct {
 	prometheusAPIV1SeriesURL           string
 }
 
-// MustStartVmsingle is a test helper function that starts an instance of
-// vmsingle and fails the test if the app fails to start.
-func MustStartVmsingle(t *testing.T, instance string, flags []string, cli *Client) *Vmsingle {
-	t.Helper()
-
-	app, err := StartVmsingle(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
-}
-
 // StartVmsingle starts an instance of vmsingle with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr).
@@ -75,8 +62,8 @@ func StartVmsingle(instance string, flags []string, cli *Client) (*Vmsingle, err
 	}, nil
 }
 
-// ForceFlush is a test helper function that forces the flushing of insterted
-// data so it becomes available for searching immediately.
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
 func (app *Vmsingle) ForceFlush(t *testing.T) {
 	t.Helper()
 

apptest/vmstorage.go

@@ -23,19 +23,6 @@ type Vmstorage struct {
 	forceFlushURL string
 }
 
-// MustStartVmstorage is a test helper function that starts an instance of
-// vmstorage and fails the test if the app fails to start.
-func MustStartVmstorage(t *testing.T, instance string, flags []string, cli *Client) *Vmstorage {
-	t.Helper()
-
-	app, err := StartVmstorage(instance, flags, cli)
-	if err != nil {
-		t.Fatalf("Could not start %s: %v", instance, err)
-	}
-
-	return app
-}
-
 // StartVmstorage starts an instance of vmstorage with the given flags. It also
 // sets the default flags and populates the app instance state with runtime
 // values extracted from the application log (such as httpListenAddr)
@@ -85,8 +72,8 @@ func (app *Vmstorage) VmselectAddr() string {
 	return app.vmselectAddr
 }
 
-// ForceFlush is a test helper function that forces the flushing of insterted
-// data so it becomes available for searching immediately.
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
 func (app *Vmstorage) ForceFlush(t *testing.T) {
 	t.Helper()