Skip to content

Commit

Permalink
🐛 mitigate upgrade-e2e test flakiness (#1627)
Browse files Browse the repository at this point in the history 
* add artifact collection to upgrade-e2e-tests

Signed-off-by: Per Goncalves da Silva <pegoncal@redhat.com>

* mitigate upgrade-e2e flakiness

Signed-off-by: Per Goncalves da Silva <pegoncal@redhat.com>

---------

Signed-off-by: Per Goncalves da Silva <pegoncal@redhat.com>
Co-authored-by: Per Goncalves da Silva <pegoncal@redhat.com>
  • Loading branch information
@perdasilva
perdasilva and Per Goncalves da Silva authored Jan 16, 2025
1 parent c8380d2 commit 1f48426
Show file tree
Hide file tree
Showing 5 changed files with 235 additions and 177 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/e2e.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,10 @@ jobs:
go-version-file: go.mod

- name: Run the upgrade e2e test
run: make test-upgrade-e2e
run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e

- uses: cytopia/upload-artifact-retry-action@v0.1.7
if: failure()
with:
name: upgrade-e2e-artifacts
path: /tmp/artifacts/
154 changes: 10 additions & 144 deletions test/e2e/cluster_extension_install_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,13 @@ package e2e
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/google/go-containerregistry/pkg/crane"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v2"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
Expand All @@ -24,12 +19,11 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/rand"
kubeclient "k8s.io/client-go/kubernetes"
"k8s.io/utils/env"
"sigs.k8s.io/controller-runtime/pkg/client"

ocv1 "github.com/operator-framework/operator-controller/api/v1"
catalogd "github.com/operator-framework/operator-controller/catalogd/api/v1"
"github.com/operator-framework/operator-controller/test/utils"
)

const (
Expand Down Expand Up @@ -306,7 +300,7 @@ func TestClusterExtensionInstallRegistry(t *testing.T) {

clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Source: ocv1.SourceConfig{
Expand Down Expand Up @@ -441,7 +435,7 @@ func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) {
require.NoError(t, err)

defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)
defer func(cat *catalogd.ClusterCatalog) {
require.NoError(t, c.Delete(context.Background(), cat))
require.Eventually(t, func() bool {
Expand Down Expand Up @@ -489,7 +483,7 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) {

clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

t.Log("By creating an ClusterExtension at a specified version")
clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Expand Down Expand Up @@ -552,7 +546,7 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) {

clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

t.Log("By creating an ClusterExtension at a specified version")
clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Expand Down Expand Up @@ -601,7 +595,7 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) {
t.Log("When resolving upgrade edges")
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

t.Log("By creating an ClusterExtension at a specified version")
clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Expand Down Expand Up @@ -649,7 +643,7 @@ func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) {
t.Log("It resolves again when a catalog is patched with new ImageRef")
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Source: ocv1.SourceConfig{
Expand Down Expand Up @@ -736,7 +730,7 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) {
sa, err := createServiceAccount(context.Background(), types.NamespacedName{Name: clusterExtensionName, Namespace: ns.Name}, clusterExtensionName)
require.NoError(t, err)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Source: ocv1.SourceConfig{
Expand Down Expand Up @@ -797,7 +791,7 @@ func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T
t.Log("It resolves again when managed content is changed")
clusterExtension, extensionCatalog, sa, ns := testInit(t)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Source: ocv1.SourceConfig{
Expand Down Expand Up @@ -860,7 +854,7 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
err := c.Create(context.Background(), sa)
require.NoError(t, err)
defer testCleanup(t, extensionCatalog, clusterExtension, sa, ns)
defer getArtifactsOutput(t)
defer utils.CollectTestArtifacts(t, artifactName, c, cfg)

clusterExtension.Spec = ocv1.ClusterExtensionSpec{
Source: ocv1.SourceConfig{
Expand Down Expand Up @@ -936,131 +930,3 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes
}
}, pollDuration, pollInterval)
}

// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path.
// Currently it saves:
// - clusterextensions
// - pods logs
// - deployments
// - catalogsources
func getArtifactsOutput(t *testing.T) {
basePath := env.GetString("ARTIFACT_PATH", "")
if basePath == "" {
return
}

kubeClient, err := kubeclient.NewForConfig(cfg)
require.NoError(t, err)

// sanitize the artifact name for use as a directory name
testName := strings.ReplaceAll(strings.ToLower(t.Name()), " ", "-")
// Get the test description and sanitize it for use as a directory name
artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName)

// Create the full artifact path
err = os.MkdirAll(artifactPath, 0755)
require.NoError(t, err)

// Get all namespaces
namespaces := corev1.NamespaceList{}
if err := c.List(context.Background(), &namespaces); err != nil {
fmt.Printf("Failed to list namespaces: %v", err)
}

// get all cluster extensions save them to the artifact path.
clusterExtensions := ocv1.ClusterExtensionList{}
if err := c.List(context.Background(), &clusterExtensions, client.InNamespace("")); err != nil {
fmt.Printf("Failed to list cluster extensions: %v", err)
}
for _, clusterExtension := range clusterExtensions.Items {
// Save cluster extension to artifact path
clusterExtensionYaml, err := yaml.Marshal(clusterExtension)
if err != nil {
fmt.Printf("Failed to marshal cluster extension: %v", err)
continue
}
if err := os.WriteFile(filepath.Join(artifactPath, clusterExtension.Name+"-clusterextension.yaml"), clusterExtensionYaml, 0600); err != nil {
fmt.Printf("Failed to write cluster extension to file: %v", err)
}
}

// get all catalogsources save them to the artifact path.
catalogsources := catalogd.ClusterCatalogList{}
if err := c.List(context.Background(), &catalogsources, client.InNamespace("")); err != nil {
fmt.Printf("Failed to list catalogsources: %v", err)
}
for _, catalogsource := range catalogsources.Items {
// Save catalogsource to artifact path
catalogsourceYaml, err := yaml.Marshal(catalogsource)
if err != nil {
fmt.Printf("Failed to marshal catalogsource: %v", err)
continue
}
if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil {
fmt.Printf("Failed to write catalogsource to file: %v", err)
}
}

for _, namespace := range namespaces.Items {
// let's ignore kube-* namespaces.
if strings.Contains(namespace.Name, "kube-") {
continue
}

namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name)
if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil {
fmt.Printf("Failed to create namespaced artifact path: %v", err)
continue
}

// get all deployments in the namespace and save them to the artifact path.
deployments := appsv1.DeploymentList{}
if err := c.List(context.Background(), &deployments, client.InNamespace(namespace.Name)); err != nil {
fmt.Printf("Failed to list deployments %v in namespace: %q", err, namespace.Name)
continue
}

for _, deployment := range deployments.Items {
// Save deployment to artifact path
deploymentYaml, err := yaml.Marshal(deployment)
if err != nil {
fmt.Printf("Failed to marshal deployment: %v", err)
continue
}
if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil {
fmt.Printf("Failed to write deployment to file: %v", err)
}
}

// Get logs from all pods in all namespaces
pods := corev1.PodList{}
if err := c.List(context.Background(), &pods, client.InNamespace(namespace.Name)); err != nil {
fmt.Printf("Failed to list pods %v in namespace: %q", err, namespace.Name)
}
for _, pod := range pods.Items {
if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed {
continue
}
for _, container := range pod.Spec.Containers {
logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(context.Background())
if err != nil {
fmt.Printf("Failed to get logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
continue
}
defer logs.Close()

outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt"))
if err != nil {
fmt.Printf("Failed to create file for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
continue
}
defer outFile.Close()

if _, err := io.Copy(outFile, logs); err != nil {
fmt.Printf("Failed to copy logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
continue
}
}
}
}
}
Loading

0 comments on commit 1f48426

Please sign in to comment.