Skip to content

Commit

Permalink
Ignore org deletion exit code in AfterSuite
Browse files Browse the repository at this point in the history
Smoke tests have been failing frequently because org deletion times out.
Previous debug output showed that the timeout is caused by org namespace
being not deleted because of app pods do not disappear and remaining
pods are in phase `Pending` (e.g. see
https://ci.korifi.cf-app.com/teams/main/pipelines/main/jobs/deploy-korifi-acceptance/builds/802)

We have found a [k8s bug
report](kubernetes/kubernetes#121435) that
pending pods cannot be deleted. We suspect that in those cases the
cluster is just being slow.

In order to workaround that bug, we simply ignore the org deletion exit
code as org deletion is not really in the focus of the smoke tests. Org
deletion is covered in details by e2e tests.

Also, delete the related debug logs as they are not needed anymore.

fixes #3061

Co-authored-by: Danail Branekov <danailster@gmail.com>
Co-authored-by: Georgi Sabev <georgethebeatle@gmail.com>
  • Loading branch information
danail-branekov and georgethebeatle committed Mar 19, 2024
1 parent c4b81d9 commit 80649e6
Show file tree
Hide file tree
Showing 3 changed files with 1 addition and 75 deletions.
1 change: 0 additions & 1 deletion api/routing/router_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,6 @@ var _ = Describe("Router", func() {
Expect(res).To(HaveHTTPBody(MatchJSON(`{"not":"allowed"}`)))
})
})

})

func mkReq(handler http.Handler, method, url string) (*http.Response, error) {
Expand Down
58 changes: 1 addition & 57 deletions tests/smoke/smoke_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,17 @@ import (
"strings"
"testing"

"code.cloudfoundry.org/korifi/api/repositories"
korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1"
"code.cloudfoundry.org/korifi/tests/helpers"
"code.cloudfoundry.org/korifi/tests/helpers/fail_handler"

"github.com/cloudfoundry/cf-test-helpers/generator"
"github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
"github.com/onsi/ginkgo/v2/types"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gexec"
gomegatypes "github.com/onsi/gomega/types"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
Expand Down Expand Up @@ -60,8 +57,6 @@ func TestSmoke(t *testing.T) {
Container: "manager",
},
})
printLeakedNamespaces(config, repositories.OrgPrefix)
printLeakedNamespaces(config, repositories.SpacePrefix)
},
}))
SetDefaultEventuallyTimeout(helpers.EventuallyTimeout())
Expand Down Expand Up @@ -101,11 +96,7 @@ var _ = BeforeSuite(func() {
})

var _ = AfterSuite(func() {
if CurrentSpecReport().State.Is(types.SpecStateFailed) {
printAppReport(buildpackAppName)
}

Expect(helpers.Cf("delete-org", orgName, "-f").Wait()).To(Exit(0))
Expect(helpers.Cf("delete-org", orgName, "-f").Wait()).To(Exit())

serviceAccountFactory.DeleteServiceAccount(cfAdmin)
helpers.RemoveUserFromKubeConfig(cfAdmin)
Expand All @@ -122,53 +113,6 @@ func sessionOutput(session *Session) (string, error) {
return strings.TrimSpace(string(session.Out.Contents())), nil
}

func printLeakedNamespaces(config *rest.Config, prefix string) {
utilruntime.Must(korifiv1alpha1.AddToScheme(scheme.Scheme))
k8sClient, err := client.New(config, client.Options{Scheme: scheme.Scheme})
if err != nil {
fmt.Fprintf(GinkgoWriter, "failed to create k8s client: %v\n", err)
return
}

namespaces := &corev1.NamespaceList{}
if err := k8sClient.List(context.Background(), namespaces); err != nil {
fmt.Fprintf(GinkgoWriter, "failed to list namespaces: %v\n", err)
return
}

fmt.Fprintf(GinkgoWriter, "\nPrinting leaked namespaces with %q prefix:\n", prefix)
for _, namespace := range namespaces.Items {
if namespace.DeletionTimestamp == nil {
continue
}

if !strings.Contains(namespace.Name, prefix) {
continue
}

if err := printObject(k8sClient, &namespace); err != nil {
fmt.Fprintf(GinkgoWriter, "failed printing namespace %s: %v\n", namespace.Name, err)
return
}

fmt.Fprintf(GinkgoWriter, "\nPrinting pods in namespace: %s\n", namespace.Name)
pods := &corev1.PodList{}
err := k8sClient.List(context.Background(), pods, client.InNamespace(namespace.Name))
if err != nil {
fmt.Fprintf(GinkgoWriter, "failed listing pods in namespace %s: %v\n", namespace.Name, err)
return
}

for _, pod := range pods.Items {
if err = printObject(k8sClient, &pod); err != nil {
fmt.Fprintf(GinkgoWriter, "failed printing pod %s: %v\n", pod.Name, err)
return
}
}

}
}

func printCfApp(config *rest.Config) {
utilruntime.Must(korifiv1alpha1.AddToScheme(scheme.Scheme))
k8sClient, err := client.New(config, client.Options{Scheme: scheme.Scheme})
Expand Down
17 changes: 0 additions & 17 deletions tests/smoke/smoke_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"crypto/tls"
"fmt"
"net/http"
"strings"

"code.cloudfoundry.org/korifi/tests/helpers"
. "code.cloudfoundry.org/korifi/tests/matchers"
Expand Down Expand Up @@ -72,22 +71,6 @@ var _ = Describe("Smoke Tests", func() {
})
})

func printAppReport(appName string) {
if appName == "" {
return
}

printAppReportBanner(fmt.Sprintf("***** APP REPORT: %s *****", appName))
Expect(helpers.Cf("app", appName, "--guid")).To(Exit())
Expect(helpers.Cf("logs", "--recent", appName)).To(Exit())
printAppReportBanner(fmt.Sprintf("*** END APP REPORT: %s ***", appName))
}

func printAppReportBanner(announcement string) {
sequence := strings.Repeat("*", len(announcement))
fmt.Fprintf(GinkgoWriter, "\n\n%s\n%s\n%s\n", sequence, announcement, sequence)
}

func appResponseShould(appName, requestPath string, matchExpectations types.GomegaMatcher) {
var httpClient http.Client
httpClient.Transport = &http.Transport{
Expand Down

0 comments on commit 80649e6

Please sign in to comment.