From 8e6ee58c1e8df74a934f49023aaedca816a1970f Mon Sep 17 00:00:00 2001 From: Per Goncalves da Silva Date: Wed, 15 Jan 2025 12:16:37 +0100 Subject: [PATCH 1/4] add artifact collection to upgrade-e2e-tests Signed-off-by: Per Goncalves da Silva --- .github/workflows/e2e.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8b104d920..81563e4b2 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -58,4 +58,10 @@ jobs: go-version-file: go.mod - name: Run the upgrade e2e test - run: make test-upgrade-e2e + run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e + + - uses: cytopia/upload-artifact-retry-action@v0.1.7 + if: failure() + with: + name: upgrade-e2e-artifacts + path: /tmp/artifacts/ From 877f85d7e6fa43ecd6d4564f1ca9f24cf9a4d8d8 Mon Sep 17 00:00:00 2001 From: Per Goncalves da Silva Date: Wed, 15 Jan 2025 13:16:30 +0100 Subject: [PATCH 2/4] split e2e and upgrade-e2e into separate ghas to make upgrade-e2e easier to re-run Signed-off-by: Per Goncalves da Silva --- .github/workflows/e2e.yaml | 9 --------- .github/workflows/upgrade-e2e.yaml | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/upgrade-e2e.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 81563e4b2..70749743d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -56,12 +56,3 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod - - - name: Run the upgrade e2e test - run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e - - - uses: cytopia/upload-artifact-retry-action@v0.1.7 - if: failure() - with: - name: upgrade-e2e-artifacts - path: /tmp/artifacts/ diff --git a/.github/workflows/upgrade-e2e.yaml b/.github/workflows/upgrade-e2e.yaml new file mode 100644 index 000000000..8ebdc5b65 --- /dev/null +++ b/.github/workflows/upgrade-e2e.yaml @@ -0,0 +1,28 @@ +name: upgrade-e2e + +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - main + +jobs: + upgrade-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run the upgrade e2e test + run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e + + - uses: cytopia/upload-artifact-retry-action@v0.1.7 + if: failure() + with: + name: upgrade-e2e-artifacts + path: /tmp/artifacts/ From 171af61e1c4395f8d0f0d0e97bad0289ab016f7f Mon Sep 17 00:00:00 2001 From: Per Goncalves da Silva Date: Wed, 15 Jan 2025 13:36:33 +0100 Subject: [PATCH 3/4] add artifact collection to upgrade-e2e Signed-off-by: Per Goncalves da Silva --- test/upgrade-e2e/post_upgrade_test.go | 139 +++++++++++++++++++++ test/upgrade-e2e/upgrade_e2e_suite_test.go | 3 + 2 files changed, 142 insertions(+) diff --git a/test/upgrade-e2e/post_upgrade_test.go b/test/upgrade-e2e/post_upgrade_test.go index 547a7142a..34eb7d223 100644 --- a/test/upgrade-e2e/post_upgrade_test.go +++ b/test/upgrade-e2e/post_upgrade_test.go @@ -4,27 +4,38 @@ import ( "bufio" "context" "fmt" + "io" + "os" + "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/utils/env" "sigs.k8s.io/controller-runtime/pkg/client" ocv1 "github.com/operator-framework/operator-controller/api/v1" catalogd "github.com/operator-framework/operator-controller/catalogd/api/v1" ) +const ( + artifactName = "operator-controller-upgrade-e2e" +) + func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { t.Log("Starting checks after OLM upgrade") ctx := context.Background() + defer getArtifactsOutput(t) managerLabelSelector := labels.Set{"control-plane": "operator-controller-controller-manager"} @@ -141,3 +152,131 @@ func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container st return false, scanner.Err() } + +// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path. +// Currently it saves: +// - clusterextensions +// - pods logs +// - deployments +// - catalogsources +func getArtifactsOutput(t *testing.T) { + basePath := env.GetString("ARTIFACT_PATH", "") + if basePath == "" { + return + } + + kubeClient, err := kubeclient.NewForConfig(cfg) + require.NoError(t, err) + + // sanitize the artifact name for use as a directory name + testName := strings.ReplaceAll(strings.ToLower(t.Name()), " ", "-") + // Get the test description and sanitize it for use as a directory name + artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName) + + // Create the full artifact path + err = os.MkdirAll(artifactPath, 0755) + require.NoError(t, err) + + // Get all namespaces + namespaces := corev1.NamespaceList{} + if err := c.List(context.Background(), &namespaces); err != nil { + fmt.Printf("Failed to list namespaces: %v", err) + } + + // get all cluster extensions save them to the artifact path. + clusterExtensions := ocv1.ClusterExtensionList{} + if err := c.List(context.Background(), &clusterExtensions, client.InNamespace("")); err != nil { + fmt.Printf("Failed to list cluster extensions: %v", err) + } + for _, clusterExtension := range clusterExtensions.Items { + // Save cluster extension to artifact path + clusterExtensionYaml, err := yaml.Marshal(clusterExtension) + if err != nil { + fmt.Printf("Failed to marshal cluster extension: %v", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, clusterExtension.Name+"-clusterextension.yaml"), clusterExtensionYaml, 0600); err != nil { + fmt.Printf("Failed to write cluster extension to file: %v", err) + } + } + + // get all catalogsources save them to the artifact path. + catalogsources := catalogd.ClusterCatalogList{} + if err := c.List(context.Background(), &catalogsources, client.InNamespace("")); err != nil { + fmt.Printf("Failed to list catalogsources: %v", err) + } + for _, catalogsource := range catalogsources.Items { + // Save catalogsource to artifact path + catalogsourceYaml, err := yaml.Marshal(catalogsource) + if err != nil { + fmt.Printf("Failed to marshal catalogsource: %v", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil { + fmt.Printf("Failed to write catalogsource to file: %v", err) + } + } + + for _, namespace := range namespaces.Items { + // let's ignore kube-* namespaces. + if strings.Contains(namespace.Name, "kube-") { + continue + } + + namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name) + if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil { + fmt.Printf("Failed to create namespaced artifact path: %v", err) + continue + } + + // get all deployments in the namespace and save them to the artifact path. + deployments := appsv1.DeploymentList{} + if err := c.List(context.Background(), &deployments, client.InNamespace(namespace.Name)); err != nil { + fmt.Printf("Failed to list deployments %v in namespace: %q", err, namespace.Name) + continue + } + + for _, deployment := range deployments.Items { + // Save deployment to artifact path + deploymentYaml, err := yaml.Marshal(deployment) + if err != nil { + fmt.Printf("Failed to marshal deployment: %v", err) + continue + } + if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil { + fmt.Printf("Failed to write deployment to file: %v", err) + } + } + + // Get logs from all pods in all namespaces + pods := corev1.PodList{} + if err := c.List(context.Background(), &pods, client.InNamespace(namespace.Name)); err != nil { + fmt.Printf("Failed to list pods %v in namespace: %q", err, namespace.Name) + } + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { + continue + } + for _, container := range pod.Spec.Containers { + logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(context.Background()) + if err != nil { + fmt.Printf("Failed to get logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) + continue + } + defer logs.Close() + + outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt")) + if err != nil { + fmt.Printf("Failed to create file for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) + continue + } + defer outFile.Close() + + if _, err := io.Copy(outFile, logs); err != nil { + fmt.Printf("Failed to copy logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err) + continue + } + } + } + } +} diff --git a/test/upgrade-e2e/upgrade_e2e_suite_test.go b/test/upgrade-e2e/upgrade_e2e_suite_test.go index 3283265af..7c003b6e4 100644 --- a/test/upgrade-e2e/upgrade_e2e_suite_test.go +++ b/test/upgrade-e2e/upgrade_e2e_suite_test.go @@ -6,6 +6,7 @@ import ( "testing" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,12 +22,14 @@ var ( c client.Client kclientset kubernetes.Interface + cfg *rest.Config testClusterCatalogName string testClusterExtensionName string ) func TestMain(m *testing.M) { var ok bool + cfg = ctrl.GetConfigOrDie() testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv) if !ok { fmt.Printf("%q is not set", testClusterCatalogNameEnv) From 5d621fdff5d3ea1c289d355e7f5e21ab4ff9c9e8 Mon Sep 17 00:00:00 2001 From: Per Goncalves da Silva Date: Wed, 15 Jan 2025 15:35:20 +0100 Subject: [PATCH 4/4] assert that last unpacked > progressing last transitioned Signed-off-by: Per Goncalves da Silva --- test/upgrade-e2e/post_upgrade_test.go | 135 ++++++++++++++------------ 1 file changed, 74 insertions(+), 61 deletions(-) diff --git a/test/upgrade-e2e/post_upgrade_test.go b/test/upgrade-e2e/post_upgrade_test.go index 34eb7d223..956b44c75 100644 --- a/test/upgrade-e2e/post_upgrade_test.go +++ b/test/upgrade-e2e/post_upgrade_test.go @@ -1,7 +1,6 @@ package upgradee2e import ( - "bufio" "context" "fmt" "io" @@ -37,52 +36,40 @@ func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { ctx := context.Background() defer getArtifactsOutput(t) - managerLabelSelector := labels.Set{"control-plane": "operator-controller-controller-manager"} + now := time.Now() - t.Log("Checking that the controller-manager deployment is updated") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var managerDeployments appsv1.DeploymentList - assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) - assert.Len(ct, managerDeployments.Items, 1) - managerDeployment := managerDeployments.Items[0] + // wait for catalogd deployment to finish + t.Log("Wait for catalogd deployment to be ready") + waitForDeployment(t, ctx, "catalogd-controller-manager") - assert.True(ct, - managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, - ) - }, time.Minute, time.Second) + // wait for operator-controller deployment to finish + t.Log("Wait for operator-controller deployment to be ready") + waitForDeployment(t, ctx, "operator-controller-controller-manager") - var managerPods corev1.PodList - t.Log("Waiting for only one controller-manager Pod to remain") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) - assert.Len(ct, managerPods.Items, 1) - }, time.Minute, time.Second) - - t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it") - // Make sure that after we upgrade OLM itself we can still reconcile old objects without any changes - logCtx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - substrings := []string{ - "reconcile ending", - fmt.Sprintf(`ClusterExtension=%q`, testClusterExtensionName), - } - found, err := watchPodLogsForSubstring(logCtx, &managerPods.Items[0], "manager", substrings...) - require.NoError(t, err) - require.True(t, found) - - t.Log("Checking that the ClusterCatalog is serving") + t.Log("Checking that the ClusterCatalog is unpacked") require.EventuallyWithT(t, func(ct *assert.CollectT) { var clusterCatalog catalogd.ClusterCatalog assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog)) + + // check serving condition cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogd.TypeServing) if !assert.NotNil(ct, cond) { return } assert.Equal(ct, metav1.ConditionTrue, cond.Status) assert.Equal(ct, catalogd.ReasonAvailable, cond.Reason) + + // check progressing condition + cond = apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogd.TypeProgressing) + if !assert.NotNil(ct, cond) { + return + } + assert.Equal(ct, metav1.ConditionTrue, cond.Status) + assert.Equal(ct, catalogd.ReasonSucceeded, cond.Reason) + + // check that the catalog was recently unpacked (after progressing is over) + t.Logf("last unpacked: %s - progressing last transitioned: %s", clusterCatalog.Status.LastUnpacked.String(), cond.LastTransitionTime.String()) + assert.True(ct, clusterCatalog.Status.LastUnpacked.After(now)) }, time.Minute, time.Second) t.Log("Checking that the ClusterExtension is installed") @@ -122,37 +109,63 @@ func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { }, time.Minute, time.Second) } -func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container string, substrings ...string) (bool, error) { - podLogOpts := corev1.PodLogOptions{ - Follow: true, - Container: container, - } - - req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) - podLogs, err := req.Stream(ctx) - if err != nil { - return false, err - } - defer podLogs.Close() +func waitForDeployment(t *testing.T, ctx context.Context, controlPlaneLabel string) { + deploymentLabelSelector := labels.Set{"control-plane": controlPlaneLabel}.AsSelector() - scanner := bufio.NewScanner(podLogs) - for scanner.Scan() { - line := scanner.Text() + t.Log("Checking that the deployment is updated") + require.EventuallyWithT(t, func(ct *assert.CollectT) { + var managerDeployments appsv1.DeploymentList + assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: deploymentLabelSelector})) + assert.Len(ct, managerDeployments.Items, 1) + managerDeployment := managerDeployments.Items[0] - foundCount := 0 - for _, substring := range substrings { - if strings.Contains(line, substring) { - foundCount++ - } - } - if foundCount == len(substrings) { - return true, nil - } - } + assert.True(ct, + managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && + managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, + ) + }, time.Minute, time.Second) - return false, scanner.Err() + var managerPods corev1.PodList + t.Log("Waiting for only one Pod to remain") + require.EventuallyWithT(t, func(ct *assert.CollectT) { + assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: deploymentLabelSelector})) + assert.Len(ct, managerPods.Items, 1) + }, time.Minute, time.Second) } +//func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container string, substrings ...string) (bool, error) { +// podLogOpts := corev1.PodLogOptions{ +// Follow: true, +// Container: container, +// } +// +// req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) +// podLogs, err := req.Stream(ctx) +// if err != nil { +// return false, err +// } +// defer podLogs.Close() +// +// scanner := bufio.NewScanner(podLogs) +// for scanner.Scan() { +// line := scanner.Text() +// +// foundCount := 0 +// for _, substring := range substrings { +// if strings.Contains(line, substring) { +// foundCount++ +// } +// } +// if foundCount == len(substrings) { +// return true, nil +// } +// } +// +// return false, scanner.Err() +//} + // getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path. // Currently it saves: // - clusterextensions