[wip] Helm install smoke test

.github/workflows/container.yaml

@@ -83,3 +83,78 @@ jobs:
           cache-to: type=gha,mode=max
           tags: |
             ghcr.io/spinkube/spin-operator:${{ env.VERSION }}
+
+  helm-install-smoke-test:
+    runs-on: ubuntu-latest
+    needs: docker
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.21.x"
+
+      - name: Install helm
+        uses: Azure/setup-helm@v3
+        with:
+          version: v3.14.0
+
+      - name: setup k3d
+        uses: engineerd/configurator@v0.0.10
+        with:
+          name: k3d
+          url: https://github.com/k3d-io/k3d/releases/download/v5.6.0/k3d-linux-amd64
+
+      - name: start k3d cluster
+        run: |
+          k3d cluster create wasm-cluster \
+          --image ghcr.io/deislabs/containerd-wasm-shims/examples/k3d:v0.11.0 \
+          --port  "8081:80@loadbalancer" \
+          --agents 2
+
+      - name: install crd
+        run: make install
+
+      - name: apply runtime class
+        run: kubectl apply -f spin-runtime-class.yaml
+
+      - name: helm install
+        run: |
+          helm repo add kwasm-operator http://kwasm.sh/kwasm-operator
+          helm repo add cert-manager https://charts.jetstack.io
+          helm dependency build charts/spin-operator
+          helm install spin-operator \
+            --namespace spin-operator \
+            --create-namespace \
+            --devel \
+            --wait \
+            --set controllerManager.manager.image.repository=ttl.sh/spoopy-operator-pr-${{ github.event.pull_request.number }} \
+            --set controllerManager.manager.image.tag=24h \
+            --debug \
+            charts/spin-operator
+
+      - name: debug
+        if: always()
+        run: |
+          kubectl get pods -A
+          kubectl get pods -n spin-operator
+          kubectl logs -n spin-operator $(kubectl get pods -n spin-operator | grep spin-operator-controller-manager | awk '{print $1}')
+          kubectl describe -n spin-operator pod $(kubectl get pods -n spin-operator | grep spin-operator-controller-manager | awk '{print $1}')
+
+          kubectl logs -n spin-operator $(kubectl get pods -n spin-operator | grep wait-for-webhook-svc | awk '{print $1}')
+          kubectl describe -n spin-operator pod $(kubectl get pods -n spin-operator | grep wait-for-webhook-svc | awk '{print $1}')
+
+      - name: run spin app
+        run: |
+          kubectl apply -f config/samples/simple.yaml
+          kubectl rollout status deployment simple-spinapp --timeout 90s
+
+          kubectl get pods -A
+
+          kubectl port-forward svc/simple-spinapp 8083:80 &
+          timeout 15s bash -c 'until curl -f -vvv http://localhost:8083/hello; do sleep 2; done'
+
+      - name: Verify curl
+        run: curl localhost:8083/hello

charts/spin-operator/templates/containerd-shim-spin-executor.yaml

@@ -2,4 +2,8 @@ apiVersion: core.spinoperator.dev/v1
 kind: SpinAppExecutor
 metadata:
   name: containerd-shim-spin
+  namespace: default
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
 spec:

charts/spin-operator/templates/deployment.yaml

@@ -79,4 +79,5 @@ spec:
       - name: cert
         secret:
           defaultMode: 420
-          secretName: webhook-server-cert
+          secretName: webhook-server-cert
+          optional: true

charts/spin-operator/templates/wait-for-webhook.yaml

@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "spin-operator.fullname" . }}-wait-for-webhook-svc
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "3"
+  labels:
+  {{- include "spin-operator.labels" . | nindent 4 }}
+spec:
+  template:
+    spec:
+      containers:
+      - name: curl
+        image: denolehov/curl:latest
+        command: ["curl",  "-kf", "-vvv", "https://spin-operator-webhook-service.spin-operator.svc.cluster.local/mutate-core-spinoperator-dev-v1-spinappexecutor"]
+      restartPolicy: OnFailure
+  backoffLimit: 100

cmd/main.go

@@ -116,14 +116,12 @@ func main() {
 		os.Exit(1)
 	}
 	if enableWebhooks {
-		if err = webhook.SetupSpinAppWebhookWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create webhook", "webhook", "SpinApp")
-			os.Exit(1)
-		}
-		if err = webhook.SetupSpinAppExecutorWebhookWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create webhook", "webhook", "SpinAppExecutor")
-			os.Exit(1)
-		}
+		go func() {
+			if err = webhook.LazyWebhookStarter(mgr); err != nil {
+				setupLog.Error(err, "unable to create webhook", "webhook", "SpinApp")
+				os.Exit(1)
+			}
+		}()
 	}
 	//+kubebuilder:scaffold:builder
 

documentation/content/running-on-azure-kubernetes-service.md

@@ -114,13 +114,6 @@ kubectl logs -n spin-operator -l app.kubernetes.io/name=kwasm-operator
 {"level":"info","time":"2024-02-12T11:24:00Z","message":"Job aks-nodepool1-31687461-vmss000000-provision-kwasm is Completed. Happy WASMing"}
 ```
 
-The final step for setting up Spin Operator is creating a `SpinAppExecutor`:
-
-```shell
-# Create a SpinAppExecutor
-kubectl apply -f https://github.com/spinkube/spin-operator/blob/main/config/samples/shim-executor.yaml
-```
-
 ## Deploying a Spin App to AKS
 
 To validate the Spin Operator deployment, you will deploy a simple Spin App to the AKS cluster. The following command will install a simple Spin App using the `SpinApp` CRD you provisioned in the previous section:

internal/webhook/admission.go

@@ -1,10 +1,48 @@
 package webhook
 
 import (
+	"os"
+	"time"
+
 	spinv1 "github.com/spinkube/spin-operator/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
+func LazyWebhookStarter(mgr ctrl.Manager) error {
+	ticker := time.NewTicker(2 * time.Second)
+	timeout := time.NewTimer(5 * time.Minute)
+
+	crtFile := "/tmp/k8s-webhook-server/serving-certs/tls.crt"
+	webhookSetupLog := ctrl.Log.WithName("webhook-setup")
+
+	for {
+		select {
+		case <-ticker.C:
+			_, err := os.ReadFile(crtFile)
+			if err != nil && os.IsNotExist(err) {
+				webhookSetupLog.Info("file does not exist yet")
+				continue
+			}
+
+			webhookSetupLog.Info("crtfile found, setting up webhook")
+
+			if err = SetupSpinAppWebhookWithManager(mgr); err != nil {
+				webhookSetupLog.Error(err, "unable to create webhook", "webhook", "SpinApp")
+				os.Exit(1)
+			}
+			if err = SetupSpinAppExecutorWebhookWithManager(mgr); err != nil {
+				webhookSetupLog.Error(err, "unable to create webhook", "webhook", "SpinAppExecutor")
+				os.Exit(1)
+			}
+
+			return nil
+		case <-timeout.C:
+			ticker.Stop()
+			panic("timed out while waiting for webhook to start")
+		}
+	}
+}
+
 func SetupSpinAppWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
 		For(&spinv1.SpinApp{}).