diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index adca2d027..626530e5b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -58,4 +58,10 @@ jobs: go-version-file: go.mod - name: Run the upgrade e2e test - run: make test-upgrade-e2e + run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e + + - uses: cytopia/upload-artifact-retry-action@v0.1.7 + if: failure() + with: + name: upgrade-e2e-artifacts + path: /tmp/artifacts/ diff --git a/test/e2e/cluster_extension_install_test.go b/test/e2e/cluster_extension_install_test.go index 8034a2e9b..54a76ecc3 100644 --- a/test/e2e/cluster_extension_install_test.go +++ b/test/e2e/cluster_extension_install_test.go @@ -3,18 +3,13 @@ package e2e import ( "context" "fmt" - "io" "os" - "path/filepath" - "strings" "testing" "time" "github.com/google/go-containerregistry/pkg/crane" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -24,12 +19,11 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" - kubeclient "k8s.io/client-go/kubernetes" - "k8s.io/utils/env" "sigs.k8s.io/controller-runtime/pkg/client" ocv1 "github.com/operator-framework/operator-controller/api/v1" catalogd "github.com/operator-framework/operator-controller/catalogd/api/v1" + "github.com/operator-framework/operator-controller/test/utils" ) const ( @@ -306,7 +300,7 @@ func TestClusterExtensionInstallRegistry(t *testing.T) { clusterExtension, extensionCatalog, sa, ns := testInit(t) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) clusterExtension.Spec = ocv1.ClusterExtensionSpec{ Source: ocv1.SourceConfig{ @@ -441,7 +435,7 @@ func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) { require.NoError(t, err) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) defer func(cat *catalogd.ClusterCatalog) { require.NoError(t, c.Delete(context.Background(), cat)) require.Eventually(t, func() bool { @@ -489,7 +483,7 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) { clusterExtension, extensionCatalog, sa, ns := testInit(t) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) t.Log("By creating an ClusterExtension at a specified version") clusterExtension.Spec = ocv1.ClusterExtensionSpec{ @@ -552,7 +546,7 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) { clusterExtension, extensionCatalog, sa, ns := testInit(t) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) t.Log("By creating an ClusterExtension at a specified version") clusterExtension.Spec = ocv1.ClusterExtensionSpec{ @@ -601,7 +595,7 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) { t.Log("When resolving upgrade edges") clusterExtension, extensionCatalog, sa, ns := testInit(t) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) t.Log("By creating an ClusterExtension at a specified version") clusterExtension.Spec = ocv1.ClusterExtensionSpec{ @@ -649,7 +643,7 @@ func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) { t.Log("It resolves again when a catalog is patched with new ImageRef") clusterExtension, extensionCatalog, sa, ns := testInit(t) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) clusterExtension.Spec = ocv1.ClusterExtensionSpec{ Source: ocv1.SourceConfig{ @@ -736,7 +730,7 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) { sa, err := createServiceAccount(context.Background(), types.NamespacedName{Name: clusterExtensionName, Namespace: ns.Name}, clusterExtensionName) require.NoError(t, err) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) clusterExtension.Spec = ocv1.ClusterExtensionSpec{ Source: ocv1.SourceConfig{ @@ -797,7 +791,7 @@ func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T t.Log("It resolves again when managed content is changed") clusterExtension, extensionCatalog, sa, ns := testInit(t) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) clusterExtension.Spec = ocv1.ClusterExtensionSpec{ Source: ocv1.SourceConfig{ @@ -860,7 +854,7 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes err := c.Create(context.Background(), sa) require.NoError(t, err) defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns) - defer getArtifactsOutput(t) + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) clusterExtension.Spec = ocv1.ClusterExtensionSpec{ Source: ocv1.SourceConfig{ @@ -936,131 +930,3 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes } }, pollDuration, pollInterval) } - -// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path. -// Currently it saves: -// - clusterextensions -// - pods logs -// - deployments -// - catalogsources -func getArtifactsOutput(t *testing.T) { - basePath := env.GetString("ARTIFACT_PATH", "") - if basePath == "" { - return - } - - kubeClient, err := kubeclient.NewForConfig(cfg) - require.NoError(t, err) - - // sanitize the artifact name for use as a directory name - testName := strings.ReplaceAll(strings.ToLower(t.Name()), " ", "-") - // Get the test description and sanitize it for use as a directory name - artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName) - - // Create the full artifact path - err = os.MkdirAll(artifactPath, 0755) - require.NoError(t, err) - - // Get all namespaces - namespaces := corev1.NamespaceList{} - if err := c.List(context.Background(), &namespaces); err != nil { - fmt.Printf("Failed to list namespaces: %v", err) - } - - // get all cluster extensions save them to the artifact path. - clusterExtensions := ocv1.ClusterExtensionList{} - if err := c.List(context.Background(), &clusterExtensions, client.InNamespace("")); err != nil { - fmt.Printf("Failed to list cluster extensions: %v", err) - } - for _, clusterExtension := range clusterExtensions.Items { - // Save cluster extension to artifact path - clusterExtensionYaml, err := yaml.Marshal(clusterExtension) - if err != nil { - fmt.Printf("Failed to marshal cluster extension: %v", err) - continue - } - if err := os.WriteFile(filepath.Join(artifactPath, clusterExtension.Name+"-clusterextension.yaml"), clusterExtensionYaml, 0600); err != nil { - fmt.Printf("Failed to write cluster extension to file: %v", err) - } - } - - // get all catalogsources save them to the artifact path. - catalogsources := catalogd.ClusterCatalogList{} - if err := c.List(context.Background(), &catalogsources, client.InNamespace("")); err != nil { - fmt.Printf("Failed to list catalogsources: %v", err) - } - for _, catalogsource := range catalogsources.Items { - // Save catalogsource to artifact path - catalogsourceYaml, err := yaml.Marshal(catalogsource) - if err != nil { - fmt.Printf("Failed to marshal catalogsource: %v", err) - continue - } - if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil { - fmt.Printf("Failed to write catalogsource to file: %v", err) - } - } - - for _, namespace := range namespaces.Items { - // let's ignore kube-* namespaces. - if strings.Contains(namespace.Name, "kube-") { - continue - } - - namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name) - if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil { - fmt.Printf("Failed to create namespaced artifact path: %v", err) - continue - } - - // get all deployments in the namespace and save them to the artifact path. - deployments := appsv1.DeploymentList{} - if err := c.List(context.Background(), &deployments, client.InNamespace(namespace.Name)); err != nil { - fmt.Printf("Failed to list deployments %v in namespace: %q", err, namespace.Name) - continue - } - - for _, deployment := range deployments.Items { - // Save deployment to artifact path - deploymentYaml, err := yaml.Marshal(deployment) - if err != nil { - fmt.Printf("Failed to marshal deployment: %v", err) - continue - } - if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil { - fmt.Printf("Failed to write deployment to file: %v", err) - } - } - - // Get logs from all pods in all namespaces - pods := corev1.PodList{} - if err := c.List(context.Background(), &pods, client.InNamespace(namespace.Name)); err != nil { - fmt.Printf("Failed to list pods %v in namespace: %q", err, namespace.Name) - } - for _, pod := range pods.Items { - if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { - continue - } - for _, container := range pod.Spec.Containers { - logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(context.Background()) - if err != nil { - fmt.Printf("Failed to get logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) - continue - } - defer logs.Close() - - outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt")) - if err != nil { - fmt.Printf("Failed to create file for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) - continue - } - defer outFile.Close() - - if _, err := io.Copy(outFile, logs); err != nil { - fmt.Printf("Failed to copy logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) - continue - } - } - } - } -} diff --git a/test/upgrade-e2e/post_upgrade_test.go b/test/upgrade-e2e/post_upgrade_test.go index 547a7142a..204c79330 100644 --- a/test/upgrade-e2e/post_upgrade_test.go +++ b/test/upgrade-e2e/post_upgrade_test.go @@ -20,35 +20,25 @@ import ( ocv1 "github.com/operator-framework/operator-controller/api/v1" catalogd "github.com/operator-framework/operator-controller/catalogd/api/v1" + "github.com/operator-framework/operator-controller/test/utils" +) + +const ( + artifactName = "operator-controller-upgrade-e2e" ) func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { t.Log("Starting checks after OLM upgrade") ctx := context.Background() + defer utils.CollectTestArtifacts(t, artifactName, c, cfg) - managerLabelSelector := labels.Set{"control-plane": "operator-controller-controller-manager"} + // wait for catalogd deployment to finish + t.Log("Wait for catalogd deployment to be ready") + catalogdManagerPod := waitForDeployment(t, ctx, "catalogd-controller-manager") - t.Log("Checking that the controller-manager deployment is updated") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var managerDeployments appsv1.DeploymentList - assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) - assert.Len(ct, managerDeployments.Items, 1) - managerDeployment := managerDeployments.Items[0] - - assert.True(ct, - managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, - ) - }, time.Minute, time.Second) - - var managerPods corev1.PodList - t.Log("Waiting for only one controller-manager Pod to remain") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) - assert.Len(ct, managerPods.Items, 1) - }, time.Minute, time.Second) + // wait for operator-controller deployment to finish + t.Log("Wait for operator-controller deployment to be ready") + managerPod := waitForDeployment(t, ctx, "operator-controller-controller-manager") t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it") // Make sure that after we upgrade OLM itself we can still reconcile old objects without any changes @@ -58,20 +48,32 @@ func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { "reconcile ending", fmt.Sprintf(`ClusterExtension=%q`, testClusterExtensionName), } - found, err := watchPodLogsForSubstring(logCtx, &managerPods.Items[0], "manager", substrings...) + found, err := watchPodLogsForSubstring(logCtx, managerPod, "manager", substrings...) require.NoError(t, err) require.True(t, found) - t.Log("Checking that the ClusterCatalog is serving") + t.Log("Checking that the ClusterCatalog is unpacked") require.EventuallyWithT(t, func(ct *assert.CollectT) { var clusterCatalog catalogd.ClusterCatalog assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog)) + + // check serving condition cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogd.TypeServing) - if !assert.NotNil(ct, cond) { + assert.NotNil(ct, cond) + assert.Equal(ct, metav1.ConditionTrue, cond.Status) + assert.Equal(ct, catalogd.ReasonAvailable, cond.Reason) + + // mitigation for upgrade-e2e flakiness caused by the following bug + // https://github.com/operator-framework/operator-controller/issues/1626 + // wait until the unpack time > than the catalogd controller pod creation time + cond = apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogd.TypeProgressing) + if cond == nil { return } assert.Equal(ct, metav1.ConditionTrue, cond.Status) - assert.Equal(ct, catalogd.ReasonAvailable, cond.Reason) + assert.Equal(ct, catalogd.ReasonSucceeded, cond.Reason) + + assert.True(ct, clusterCatalog.Status.LastUnpacked.After(catalogdManagerPod.CreationTimestamp.Time)) }, time.Minute, time.Second) t.Log("Checking that the ClusterExtension is installed") @@ -79,9 +81,7 @@ func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { require.EventuallyWithT(t, func(ct *assert.CollectT) { assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled) - if !assert.NotNil(ct, cond) { - return - } + assert.NotNil(ct, cond) assert.Equal(ct, metav1.ConditionTrue, cond.Status) assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) assert.Contains(ct, cond.Message, "Installed bundle") @@ -101,9 +101,7 @@ func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { require.EventuallyWithT(t, func(ct *assert.CollectT) { assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled) - if !assert.NotNil(ct, cond) { - return - } + assert.NotNil(ct, cond) assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) assert.Contains(ct, cond.Message, "Installed bundle") assert.Equal(ct, ocv1.BundleMetadata{Name: "test-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.Install.Bundle) @@ -111,6 +109,39 @@ func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { }, time.Minute, time.Second) } +// waitForDeployment checks that the updated deployment with the given control-plane label +// has reached the desired number of replicas and that the number pods matches that number +// i.e. no old pods remain. It will return a pointer to the first pod. This is only necessary +// to facilitate the mitigation put in place for https://github.com/operator-framework/operator-controller/issues/1626 +func waitForDeployment(t *testing.T, ctx context.Context, controlPlaneLabel string) *corev1.Pod { + deploymentLabelSelector := labels.Set{"control-plane": controlPlaneLabel}.AsSelector() + + t.Log("Checking that the deployment is updated") + var desiredNumReplicas int32 + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var managerDeployments appsv1.DeploymentList + assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: deploymentLabelSelector})) + assert.Len(ct, managerDeployments.Items, 1) + managerDeployment := managerDeployments.Items[0] + + assert.True(ct, + managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, + ) + desiredNumReplicas = *managerDeployment.Spec.Replicas + }, time.Minute, time.Second) + + var managerPods corev1.PodList + t.Logf("Ensure the number of remaining pods equal the desired number of replicas (%d)", desiredNumReplicas) + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: deploymentLabelSelector})) + assert.Len(ct, managerPods.Items, 1) + }, time.Minute, time.Second) + return &managerPods.Items[0] +} + func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container string, substrings ...string) (bool, error) { podLogOpts := corev1.PodLogOptions{ Follow: true, diff --git a/test/upgrade-e2e/upgrade_e2e_suite_test.go b/test/upgrade-e2e/upgrade_e2e_suite_test.go index 3283265af..7c003b6e4 100644 --- a/test/upgrade-e2e/upgrade_e2e_suite_test.go +++ b/test/upgrade-e2e/upgrade_e2e_suite_test.go @@ -6,6 +6,7 @@ import ( "testing" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,12 +22,14 @@ var ( c client.Client kclientset kubernetes.Interface + cfg *rest.Config testClusterCatalogName string testClusterExtensionName string ) func TestMain(m *testing.M) { var ok bool + cfg = ctrl.GetConfigOrDie() testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv) if !ok { fmt.Printf("%q is not set", testClusterCatalogNameEnv) diff --git a/test/utils/artifacts.go b/test/utils/artifacts.go new file mode 100644 index 000000000..02cef051b --- /dev/null +++ b/test/utils/artifacts.go @@ -0,0 +1,152 @@ +package utils + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/utils/env" + "sigs.k8s.io/controller-runtime/pkg/client" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + catalogd "github.com/operator-framework/operator-controller/catalogd/api/v1" +) + +// CollectTestArtifacts gets all the artifacts from the test run and saves them to the artifact path. +// Currently, it saves: +// - clusterextensions +// - pods logs +// - deployments +// - catalogsources +func CollectTestArtifacts(t *testing.T, artifactName string, c client.Client, cfg *rest.Config) { + basePath := env.GetString("ARTIFACT_PATH", "") + if basePath == "" { + return + } + + kubeClient, err := kubeclient.NewForConfig(cfg) + require.NoError(t, err) + + // sanitize the artifact name for use as a directory name + testName := strings.ReplaceAll(strings.ToLower(t.Name()), " ", "-") + // Get the test description and sanitize it for use as a directory name + artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName) + + // Create the full artifact path + err = os.MkdirAll(artifactPath, 0755) + require.NoError(t, err) + + // Get all namespaces + namespaces := corev1.NamespaceList{} + if err := c.List(context.Background(), &namespaces); err != nil { + fmt.Printf("Failed to list namespaces: %v", err) + } + + // get all cluster extensions save them to the artifact path. + clusterExtensions := ocv1.ClusterExtensionList{} + if err := c.List(context.Background(), &clusterExtensions, client.InNamespace("")); err != nil { + fmt.Printf("Failed to list cluster extensions: %v", err) + } + for _, clusterExtension := range clusterExtensions.Items { + // Save cluster extension to artifact path + clusterExtensionYaml, err := yaml.Marshal(clusterExtension) + if err != nil { + fmt.Printf("Failed to marshal cluster extension: %v", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, clusterExtension.Name+"-clusterextension.yaml"), clusterExtensionYaml, 0600); err != nil { + fmt.Printf("Failed to write cluster extension to file: %v", err) + } + } + + // get all catalogsources save them to the artifact path. + catalogsources := catalogd.ClusterCatalogList{} + if err := c.List(context.Background(), &catalogsources, client.InNamespace("")); err != nil { + fmt.Printf("Failed to list catalogsources: %v", err) + } + for _, catalogsource := range catalogsources.Items { + // Save catalogsource to artifact path + catalogsourceYaml, err := yaml.Marshal(catalogsource) + if err != nil { + fmt.Printf("Failed to marshal catalogsource: %v", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil { + fmt.Printf("Failed to write catalogsource to file: %v", err) + } + } + + for _, namespace := range namespaces.Items { + // let's ignore kube-* namespaces. + if strings.Contains(namespace.Name, "kube-") { + continue + } + + namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name) + if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil { + fmt.Printf("Failed to create namespaced artifact path: %v", err) + continue + } + + // get all deployments in the namespace and save them to the artifact path. + deployments := appsv1.DeploymentList{} + if err := c.List(context.Background(), &deployments, client.InNamespace(namespace.Name)); err != nil { + fmt.Printf("Failed to list deployments %v in namespace: %q", err, namespace.Name) + continue + } + + for _, deployment := range deployments.Items { + // Save deployment to artifact path + deploymentYaml, err := yaml.Marshal(deployment) + if err != nil { + fmt.Printf("Failed to marshal deployment: %v", err) + continue + } + if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil { + fmt.Printf("Failed to write deployment to file: %v", err) + } + } + + // Get logs from all pods in all namespaces + pods := corev1.PodList{} + if err := c.List(context.Background(), &pods, client.InNamespace(namespace.Name)); err != nil { + fmt.Printf("Failed to list pods %v in namespace: %q", err, namespace.Name) + } + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { + continue + } + for _, container := range pod.Spec.Containers { + logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(context.Background()) + if err != nil { + fmt.Printf("Failed to get logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) + continue + } + defer logs.Close() + + outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt")) + if err != nil { + fmt.Printf("Failed to create file for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) + continue + } + defer outFile.Close() + + if _, err := io.Copy(outFile, logs); err != nil { + fmt.Printf("Failed to copy logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) + continue + } + } + } + } +}