Parallelize the remaining checks

OctopusSolutionsEngineering · Oct 9, 2024 · 1c5c3a3 · 1c5c3a3
1 parent ca1abb7
commit 1c5c3a3
Show file tree

Hide file tree

Showing 8 changed files with 355 additions and 176 deletions.
diff --git a/internal/checks/naming/octopus_invalid_variables_check.go b/internal/checks/naming/octopus_invalid_variables_check.go
@@ -1,6 +1,7 @@
 package naming
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"github.com/OctopusDeploy/go-octopusdeploy/v2/pkg/client"
@@ -13,7 +14,9 @@ import (
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/checks"
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/client_wrapper"
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/config"
+	"github.com/hayageek/threadsafe"
 	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
 	"regexp"
 	"strings"
 )
@@ -74,35 +77,57 @@ func (o OctopusInvalidVariableNameCheck) Execute(concurrency int) (checks.Octopu
 			checks.Naming), nil
 	}
 
-	messages := []string{}
+	g, _ := errgroup.WithContext(context.Background())
+	g.SetLimit(concurrency)
+
+	messages := threadsafe.NewSlice[string]()
+	goroutineErrors := threadsafe.NewSlice[error]()
+
 	for i, p := range projects {
-		zap.L().Debug(o.Id() + " " + fmt.Sprintf("%.2f", float32(i+1)/float32(len(projects))*100) + "% complete")
 
-		variableSet, err := o.client.Variables.GetAll(p.ID)
+		i := i
+		p := p
 
-		if err != nil {
-			if !o.errorHandler.ShouldContinue(err) {
-				return nil, err
-			}
-			continue
-		}
+		g.Go(func() error {
+			zap.L().Debug(o.Id() + " " + fmt.Sprintf("%.2f", float32(i+1)/float32(len(projects))*100) + "% complete")
 
-		for _, v := range variableSet.Variables {
-			if checks.IgnoreVariable(v.Name) {
-				continue
+			variableSet, err := o.client.Variables.GetAll(p.ID)
+
+			if err != nil {
+				if !o.errorHandler.ShouldContinue(err) {
+					goroutineErrors.Append(err)
+				}
+				return nil
 			}
 
-			if !regex.Match([]byte(v.Name)) {
-				messages = append(messages, p.Name+": "+v.Name)
+			for _, v := range variableSet.Variables {
+				if checks.IgnoreVariable(v.Name) {
+					continue
+				}
+
+				if !regex.Match([]byte(v.Name)) {
+					messages.Append(p.Name + ": " + v.Name)
+				}
+
 			}
 
-		}
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	// Treat the first error as the root cause
+	if goroutineErrors.Length() > 0 {
+		return o.errorHandler.HandleError(o.Id(), checks.Naming, goroutineErrors.Values()[0])
 	}
 
-	if len(messages) > 0 {
+	if messages.Length() > 0 {
 
 		return checks.NewOctopusCheckResultImpl(
-			"The following variables do not match the regex "+o.config.VariableNameRegex+":\n"+strings.Join(messages, "\n"),
+			"The following variables do not match the regex "+o.config.VariableNameRegex+":\n"+strings.Join(messages.Values(), "\n"),
 			o.Id(),
 			"",
 			checks.Warning,

diff --git a/internal/checks/naming/octopus_project_uses_default_step_names.go b/internal/checks/naming/octopus_project_uses_default_step_names.go
@@ -1,6 +1,7 @@
 package naming
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"github.com/OctopusDeploy/go-octopusdeploy/v2/pkg/client"
@@ -9,8 +10,10 @@ import (
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/checks"
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/client_wrapper"
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/config"
+	"github.com/hayageek/threadsafe"
 	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
+	"golang.org/x/sync/errgroup"
 	"strings"
 )
 
@@ -57,48 +60,68 @@ func (o OctopusProjectDefaultStepNames) Execute(concurrency int) (checks.Octopus
 		return o.errorHandler.HandleError(o.Id(), checks.Naming, err)
 	}
 
-	actionsWithDefaultNames := []string{}
+	g, _ := errgroup.WithContext(context.Background())
+	g.SetLimit(concurrency)
+
+	actionsWithDefaultNames := threadsafe.NewSlice[string]()
+	goroutineErrors := threadsafe.NewSlice[error]()
+
 	for i, p := range projects {
-		zap.L().Debug(o.Id() + " " + fmt.Sprintf("%.2f", float32(i+1)/float32(len(projects))*100) + "% complete")
+		i := i
+		p := p
 
-		deploymentProcess, err := o.stepsInDeploymentProcess(p.DeploymentProcessID)
+		g.Go(func() error {
+			zap.L().Debug(o.Id() + " " + fmt.Sprintf("%.2f", float32(i+1)/float32(len(projects))*100) + "% complete")
 
-		if err != nil {
-			if !o.errorHandler.ShouldContinue(err) {
-				return nil, err
+			deploymentProcess, err := o.stepsInDeploymentProcess(p.DeploymentProcessID)
+
+			if err != nil {
+				if !o.errorHandler.ShouldContinue(err) {
+					goroutineErrors.Append(err)
+				}
+				return nil
 			}
-			continue
-		}
 
-		if deploymentProcess == nil {
-			continue
-		}
+			if deploymentProcess == nil {
+				return nil
+			}
 
-		for _, s := range deploymentProcess.Steps {
-			for _, a := range s.Actions {
-				if slices.Index(checks.DefaultStepNames, a.Name) != -1 {
-					actionsWithDefaultNames = append(actionsWithDefaultNames, p.Name+"/"+a.Name)
+			for _, s := range deploymentProcess.Steps {
+				for _, a := range s.Actions {
+					if slices.Index(checks.DefaultStepNames, a.Name) != -1 {
+						actionsWithDefaultNames.Append(p.Name + "/" + a.Name)
+					}
 				}
 			}
-		}
 
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	// Treat the first error as the root cause
+	if goroutineErrors.Length() > 0 {
+		return o.errorHandler.HandleError(o.Id(), checks.Naming, goroutineErrors.Values()[0])
 	}
 
-	if len(actionsWithDefaultNames) > 0 {
+	if actionsWithDefaultNames.Length() > 0 {
 		return checks.NewOctopusCheckResultImpl(
-			"The following project actions use the default step names:\n"+strings.Join(actionsWithDefaultNames, "\n"),
+			"The following project actions use the default step names:\n"+strings.Join(actionsWithDefaultNames.Values(), "\n"),
 			o.Id(),
 			"",
 			checks.Warning,
-			checks.Organization), nil
+			checks.Naming), nil
 	}
 
 	return checks.NewOctopusCheckResultImpl(
 		"There are no project actions default step names",
 		o.Id(),
 		"",
 		checks.Ok,
-		checks.Organization), nil
+		checks.Naming), nil
 }
 
 func (o OctopusProjectDefaultStepNames) stepsInDeploymentProcess(deploymentProcessID string) (*deployments.DeploymentProcess, error) {

diff --git a/internal/checks/naming/octopus_project_uses_invalid_container_image.go b/internal/checks/naming/octopus_project_uses_invalid_container_image.go
@@ -1,6 +1,7 @@
 package naming
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"github.com/OctopusDeploy/go-octopusdeploy/v2/pkg/client"
@@ -9,7 +10,9 @@ import (
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/checks"
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/client_wrapper"
 	"github.com/OctopusSolutionsEngineering/OctopusRecommendationEngine/internal/config"
+	"github.com/hayageek/threadsafe"
 	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
 	"regexp"
 	"strings"
 )
@@ -73,52 +76,75 @@ func (o OctopusProjectContainerImageRegex) Execute(concurrency int) (checks.Octo
 		return o.errorHandler.HandleError(o.Id(), checks.Naming, err)
 	}
 
-	actionsWithInvalidImages := []string{}
+	g, _ := errgroup.WithContext(context.Background())
+	g.SetLimit(concurrency)
+
+	actionsWithInvalidImages := threadsafe.NewSlice[string]()
+	goroutineErrors := threadsafe.NewSlice[error]()
+
 	for i, p := range projects {
-		zap.L().Debug(o.Id() + " " + fmt.Sprintf("%.2f", float32(i+1)/float32(len(projects))*100) + "% complete")
 
-		deploymentProcess, err := o.stepsInDeploymentProcess(p.DeploymentProcessID)
+		i := i
+		p := p
 
-		if err != nil {
-			if !o.errorHandler.ShouldContinue(err) {
-				return nil, err
-			}
-			continue
-		}
+		g.Go(func() error {
 
-		if deploymentProcess == nil {
-			continue
-		}
+			zap.L().Debug(o.Id() + " " + fmt.Sprintf("%.2f", float32(i+1)/float32(len(projects))*100) + "% complete")
 
-		for _, s := range deploymentProcess.Steps {
-			for _, a := range s.Actions {
-				if a.Container == nil || strings.TrimSpace(a.Container.Image) == "" {
-					continue
+			deploymentProcess, err := o.stepsInDeploymentProcess(p.DeploymentProcessID)
+
+			if err != nil {
+				if !o.errorHandler.ShouldContinue(err) {
+					goroutineErrors.Append(err)
 				}
+				return nil
+			}
+
+			if deploymentProcess == nil {
+				return nil
+			}
 
-				if !regex.Match([]byte(a.Container.Image)) {
-					actionsWithInvalidImages = append(actionsWithInvalidImages, p.Name+"/"+a.Name+": "+a.Container.Image)
+			for _, s := range deploymentProcess.Steps {
+				for _, a := range s.Actions {
+					if a.Container == nil || strings.TrimSpace(a.Container.Image) == "" {
+						continue
+					}
+
+					if !regex.Match([]byte(a.Container.Image)) {
+						actionsWithInvalidImages.Append(p.Name + "/" + a.Name + ": " + a.Container.Image)
+					}
 				}
 			}
-		}
+
+			return nil
+		})
 
 	}
 
-	if len(actionsWithInvalidImages) > 0 {
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	// Treat the first error as the root cause
+	if goroutineErrors.Length() > 0 {
+		return o.errorHandler.HandleError(o.Id(), checks.Naming, goroutineErrors.Values()[0])
+	}
+
+	if actionsWithInvalidImages.Length() > 0 {
 		return checks.NewOctopusCheckResultImpl(
-			"The following project actions do not match the regex "+o.config.ContainerImageRegex+":\n"+strings.Join(actionsWithInvalidImages, "\n"),
+			"The following project actions do not match the regex "+o.config.ContainerImageRegex+":\n"+strings.Join(actionsWithInvalidImages.Values(), "\n"),
 			o.Id(),
 			"",
 			checks.Warning,
-			checks.Organization), nil
+			checks.Naming), nil
 	}
 
 	return checks.NewOctopusCheckResultImpl(
 		"There are no project actions with invalid container images",
 		o.Id(),
 		"",
 		checks.Ok,
-		checks.Organization), nil
+		checks.Naming), nil
 }
 
 func (o OctopusProjectContainerImageRegex) stepsInDeploymentProcess(deploymentProcessID string) (*deployments.DeploymentProcess, error) {