Add ability to split integration tests into different groups

docs/test-framework-dev-guide.md

@@ -67,6 +67,16 @@ between, and it can be very specific or not very specific.
 > **_NOTE:_**  This only filters down the tests based on the platform. It will not execute a tests on a platform unless
 > the test defines as supporting it.
 
+#### Selecting specific group
+
+By default, the running will run all test groups. When working on groups of tests it's better to limit to a specific
+group of tests instead of running all tests. This can be done by using the `TEST_GROUPS="default upgrade-standalone"`
+environment variable. This variable can take multiple groups with a space between.
+
+- `TEST_GROUPS="default" mage integration:test` to execute only tests in the "default" group.
+- `TEST_GROUPS="default upgrade-standalone" mage integration:test` to execute only tests in the "default" or
+"upgrade-standalone" group.
+
 #### Passing additional go test flags
 
 When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`.
@@ -168,6 +178,17 @@ the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test
 framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools`
 package for helper utilities.
 
+### Test group
+
+Every `define.Require` must define a `Group` that it belongs too. All tests with in the same group are executed
+on the same instance. Placing similar tests in the same group allows those tests to run on its own instance
+as well as provides a way for a developer to select a specific group of tests with `TEST_GROUP="{group-name}"`.
+
+Grouping tests is another way of spreading out the testing load across multiple instances. The more groups that
+are defined the more instances will be provisioned to complete all tests. A balance between a small good set of
+groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can
+out weight the benefits of creating another group.
+
 ### Test namespaces
 
 Every test has access to its own unique namespace (a string value). This namespace can

magefile.go

@@ -1778,6 +1778,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 		StateDir:          ".integration-cache",
 		DiagnosticsDir:    diagDir,
 		Platforms:         testPlatforms(),
+		Groups:            testGroups(),
 		Matrix:            matrix,
 		SingleTest:        singleTest,
 		VerboseMode:       mg.Verbose(),
@@ -1868,6 +1869,20 @@ func testPlatforms() []string {
 	return platforms
 }
 
+func testGroups() []string {
+	groupsStr := os.Getenv("TEST_GROUPS")
+	if groupsStr == "" {
+		return nil
+	}
+	var groups []string
+	for _, g := range strings.Split(groupsStr, " ") {
+		if g != "" {
+			groups = append(groups, g)
+		}
+	}
+	return groups
+}
+
 // Pre-requisite: user must have the gcloud CLI installed
 func authGCP(ctx context.Context) error {
 	// We only need the service account token to exist.

pkg/testing/define/batch.go

@@ -41,6 +41,10 @@ var defaultOS = []OS{
 
 // Batch is a grouping of tests that all have the same requirements.
 type Batch struct {
+	// Group must be set on each test to define which group the tests belongs.
+	// Tests that are in the same group are executed on the same runner.
+	Group string `json:"shard_id,omitempty"`
+
 	// OS defines the operating systems this test batch needs.
 	OS OS `json:"os"`
 
@@ -179,11 +183,12 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch
 		var batch Batch
 		batchIdx := -1
 		if !req.Isolate {
-			batchIdx = findBatchIdx(batches, o, req.Stack)
+			batchIdx = findBatchIdx(batches, req.Group, o, req.Stack)
 		}
 		if batchIdx == -1 {
 			// new batch required
 			batch = Batch{
+				Group:     req.Group,
 				OS:        o,
 				Isolate:   req.Isolate,
 				Tests:     nil,
@@ -241,12 +246,16 @@ func appendPackageTest(tests []BatchPackageTests, pkg string, name string, stack
 	return tests
 }
 
-func findBatchIdx(batches []Batch, os OS, stack *Stack) int {
+func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int {
 	for i, b := range batches {
 		if b.Isolate {
 			// never add to an isolate batch
 			continue
 		}
+		if b.Group != group {
+			// must be in the same group
+			continue
+		}
 		if b.OS.Type != os.Type || b.OS.Arch != os.Arch {
 			// must be same type and arch both are always defined at this point
 			continue

pkg/testing/define/batch_test.go

@@ -93,6 +93,7 @@ func TestBatch(t *testing.T) {
 	}
 	expected := []Batch{
 		{
+			Group: "default",
 			OS: OS{
 				Type: Darwin,
 				Arch: AMD64,
@@ -101,6 +102,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: darwinSudoTests,
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Darwin,
 				Arch: ARM64,
@@ -109,6 +111,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: darwinSudoTests,
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Linux,
 				Arch: AMD64,
@@ -117,6 +120,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: linuxSudoTests,
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type:    Linux,
 				Arch:    ARM64,
@@ -152,6 +156,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: linuxSudoTests,
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Windows,
 				Arch: AMD64,
@@ -160,6 +165,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: windowsSudoTests,
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Darwin,
 				Arch: AMD64,
@@ -177,6 +183,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Darwin,
 				Arch: ARM64,
@@ -194,6 +201,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Linux,
 				Arch: AMD64,
@@ -211,6 +219,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Linux,
 				Arch: ARM64,
@@ -228,6 +237,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Windows,
 				Arch: AMD64,
@@ -245,6 +255,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Darwin,
 				Arch: AMD64,
@@ -262,6 +273,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Darwin,
 				Arch: ARM64,
@@ -279,6 +291,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Linux,
 				Arch: AMD64,
@@ -296,6 +309,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Linux,
 				Arch: ARM64,
@@ -313,6 +327,7 @@ func TestBatch(t *testing.T) {
 			},
 		},
 		{
+			Group: "default",
 			OS: OS{
 				Type: Windows,
 				Arch: AMD64,
@@ -329,6 +344,53 @@ func TestBatch(t *testing.T) {
 				},
 			},
 		},
+		{
+			Group: "one",
+			OS: OS{
+				Type:    Linux,
+				Arch:    ARM64,
+				Version: "20.04",
+				Distro:  "ubuntu",
+			},
+			Stack: &Stack{
+				Version: "8.8.0",
+			},
+			Tests: []BatchPackageTests{
+				{
+					Name: pkgName,
+					Tests: []BatchPackageTest{
+						{
+							Name:  "TestGroup_One_One",
+							Stack: true,
+						},
+						{
+							Name:  "TestGroup_One_Two",
+							Stack: true,
+						},
+					},
+				},
+			},
+		},
+		{
+			Group: "two",
+			OS: OS{
+				Type: Linux,
+				Arch: ARM64,
+			},
+			Tests: []BatchPackageTests{
+				{
+					Name: pkgName,
+					Tests: []BatchPackageTest{
+						{
+							Name: "TestGroup_Two_One",
+						},
+						{
+							Name: "TestGroup_Two_Two",
+						},
+					},
+				},
+			},
+		},
 	}
 
 	actual, err := DetermineBatches("testdata", "", "batch_test")

pkg/testing/define/requirements.go

@@ -82,6 +82,13 @@ type Stack struct {
 
 // Requirements defines the testing requirements for the test to run.
 type Requirements struct {
+	// Group must be set on each test to define which group the tests belongs.
+	// Tests that are in the same group are executed on the same runner.
+	//
+	// Useful when a tests take a long time to complete and sharding them across multiple
+	// host can improve the total amount of time to complete all the tests.
+	Group string `json:"group"`
+
 	// OS defines the operating systems this test can run on. In the case
 	// multiple are provided the test is ran multiple times one time on each
 	// combination.
@@ -108,6 +115,9 @@ type Requirements struct {
 
 // Validate returns an error if not valid.
 func (r Requirements) Validate() error {
+	if r.Group == "" {
+		return errors.New("group is required")
+	}
 	for i, o := range r.OS {
 		if err := o.Validate(); err != nil {
 			return fmt.Errorf("invalid os %d: %w", i, err)