feat: InitContainer support for ACI Connector

README.md

@@ -27,7 +27,7 @@ This document details configuring the Virtual Kubelet ACI provider.
 
 Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Container Instances provide. Please check the Azure documentation accurate details on region availability, pricing and new features. The list here attempts to give an accurate reference for the features we support in ACI and the ACI provider within Virtual Kubelet.
 
-### features
+### Features
 
 * Volumes: empty dir, github repo, projection, Azure Files, Azure Files CSI drivers
 * Secure env variables, config maps
@@ -36,6 +36,7 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont
 * Basic Azure Networking support within AKS virtual node
 * [Exec support](https://docs.microsoft.com/azure/container-instances/container-instances-exec) for container instances
 * Azure Monitor integration or formally known as OMS
+* Support for init-containers ([use init containers](#Create-pod-with-init-containers))
 
 ### Limitations
 
@@ -44,7 +45,6 @@ Virtual Kubelet's ACI provider relies heavily on the feature set that Azure Cont
 * [Limitations](https://docs.microsoft.com/azure/container-instances/container-instances-vnet) with VNet
 * VNet peering
 * Argument support for exec
-* Init containers
 * [Host aliases](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) support
 * downward APIs (i.e podIP)
 
@@ -650,6 +650,28 @@ Output:
 ```
 -->
 
+### Create pod with init containers
+Multiple init containers can be specified in the podspec similar to how containers are specified
+
+```yaml
+spec:
+  initContainers:
+  - image: <INIT CONTAINER IMAGE 1>
+    name: init-container-01
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo \"Hi\"" ]
+  - image: <INIT CONTAINER IMAGE 2>
+    name: init-container-02
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo \"Hi\"" ]
+  containers:
+  - image: <CONTAINER IMAGE>
+    imagePullPolicy: Always
+    name: container
+    command: [ "/bin/sh" ]
+```
+More information on init containers can be found in [Kubernetes](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and [ACI](https://docs.microsoft.com/en-us/azure/container-instances/container-instances-init-container) documentations
+
 ## Work around for the virtual kubelet pod
 
 If your pod that's scheduled onto the Virtual Kubelet node is in a pending state please add this workaround to your Virtual Kubelet pod spec.

e2e/fixtures/initcontainers_pod.yml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vk-e2e-initcontainers
+  namespace: vk-test
+spec:
+  initContainers:
+  - image: alpine
+    name: init-container-01
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo \"Hi\"" ]
+  - image: alpine
+    name: init-container-02
+    command: [ "/bin/sh" ]
+    args: [ "-c", "echo \"Hi\"" ]
+  containers:
+  - image: alpine
+    imagePullPolicy: Always
+    name: container
+    command: [
+        "sh",
+        "-c",
+        "while sleep 10; do echo pod with init container; done;"
+    ]
+    resources:
+      requests:
+        memory: 1G
+        cpu: 1
+    volumeMounts:
+      - name: azure
+        mountPath: /mnt/azure
+  nodeSelector:
+    kubernetes.io/role: agent
+    beta.kubernetes.io/os: linux
+    type: virtual-kubelet
+  tolerations:
+  - key: virtual-kubelet.io/provider
+    operator: Exists
+  volumes:
+  - name: azure
+    csi:
+      driver: file.csi.azure.com
+      volumeAttributes:
+        secretName: csidriversecret  # required
+        shareName: vncsidriversharename  # required

e2e/pods_test.go

@@ -18,6 +18,7 @@ func TestPodLifecycle(t *testing.T) {
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatal(string(out))
 	}
+
 	cmd = kubectl("apply", "-f", "fixtures/hpa.yml", "--namespace=vk-test")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatal(string(out))
@@ -50,7 +51,65 @@ func TestPodLifecycle(t *testing.T) {
 		time.Sleep(10 * time.Second)
 	}
 
-	t.Log("clean up pod")
+	t.Log("clean up")
+	cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+}
+
+func TestPodWithInitContainers(t *testing.T) {
+	// delete the namespace first
+	cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	// create namespace
+	cmd = kubectl("apply", "-f", "fixtures/namespace.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	testStorageAccount := os.Getenv("CSI_DRIVER_STORAGE_ACCOUNT_NAME")
+	testStorageKey := os.Getenv("CSI_DRIVER_STORAGE_ACCOUNT_KEY")
+
+	cmd = kubectl("create", "secret", "generic", "csidriversecret", "--from-literal", "azurestorageaccountname="+testStorageAccount, "--from-literal", "azurestorageaccountkey="+testStorageKey, "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+
+	cmd = kubectl("apply", "-f", "fixtures/initcontainers_pod.yml")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	deadline, ok := t.Deadline()
+	timeout := time.Until(deadline)
+	if !ok {
+		timeout = 300 * time.Second
+	}
+	cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-initcontainers", "--namespace=vk-test")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatal(string(out))
+	}
+	t.Log("success create pod")
+
+	// query metrics
+	deadline = time.Now().Add(10 * time.Minute)
+	for {
+		t.Log("query metrics ....")
+		cmd = kubectl("get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/vk-test/pods/vk-e2e-initcontainers")
+		out, err := cmd.CombinedOutput()
+		if time.Now().After(deadline) {
+			t.Fatal("failed to query pod's stats from metrics server API")
+		}
+		if err == nil {
+			t.Logf("success query metrics %s", string(out))
+			break
+		}
+		time.Sleep(10 * time.Second)
+	}
+	t.Log("clean up")
 	cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatal(string(out))

hack/e2e/aks.sh

@@ -20,7 +20,7 @@ if [ "$PR_RAND" = "" ]; then
 fi
 
 : "${RESOURCE_GROUP:=vk-aci-test-$RANDOM_NUM}"
-: "${LOCATION:=westus2}"
+: "${LOCATION:=eastus2}"
 : "${CLUSTER_NAME:=${RESOURCE_GROUP}}"
 : "${NODE_COUNT:=1}"
 : "${CHART_NAME:=vk-aci-test-aks}"

pkg/provider/aci.go

@@ -303,7 +303,15 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 		return err
 
 	}
+
+	// get initContainers
+	initContainers, err := p.getInitContainers(ctx, pod)
+	if err != nil {
+		return err
+	}
+
 	// assign all the things
+	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers = &initContainers
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers = containers
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes = &volumes
 	cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.ImageRegistryCredentials = creds
@@ -802,6 +810,91 @@ func readDockerConfigJSONSecret(secret *v1.Secret, ips []azaci.ImageRegistryCred
 	return ips, err
 }
 
+//verify if Container is properly declared for the use on ACI
+func (p *ACIProvider) verifyContainer(container *v1.Container) error {
+	if len(container.Command) == 0 && len(container.Args) > 0 {
+		return errdefs.InvalidInput("ACI does not support providing args without specifying the command. Please supply both command and args to the pod spec.")
+	}
+	return nil
+}
+
+//this method is used for both initConainers and containers
+func (p *ACIProvider) getCommand(container *v1.Container) *[]string {
+	command := append(container.Command, container.Args...)
+	return &command
+}
+
+//get VolumeMounts declared on Container as []aci.VolumeMount
+func (p *ACIProvider) getVolumeMounts(container *v1.Container) *[]azaci.VolumeMount {
+	volumeMounts := make([]azaci.VolumeMount, 0, len(container.VolumeMounts))
+	for _, v := range container.VolumeMounts {
+		volumeMounts = append(volumeMounts, azaci.VolumeMount{
+			Name:      &v.Name,
+			MountPath: &v.MountPath,
+			ReadOnly:  &v.ReadOnly,
+		})
+	}
+	return &volumeMounts
+}
+
+//get EnvironmentVariables declared on Container as []aci.EnvironmentVariable
+func (p *ACIProvider) getEnvironmentVariables(container *v1.Container) *[]azaci.EnvironmentVariable {
+	environmentVariable := make([]azaci.EnvironmentVariable, 0, len(container.Env))
+	for _, e := range container.Env {
+		if e.Value != "" {
+			envVar := getACIEnvVar(e)
+			environmentVariable = append(environmentVariable, envVar)
+		}
+	}
+	return &environmentVariable
+}
+
+//get InitContainers defined in Pod as []aci.InitContainerDefinition
+func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]azaci.InitContainerDefinition, error) {
+	initContainers := make([]azaci.InitContainerDefinition, 0, len(pod.Spec.InitContainers))
+	for i, initContainer := range pod.Spec.InitContainers {
+		err := p.verifyContainer(&initContainer)
+		if err != nil {
+			log.G(ctx).Errorf("couldn't verify container %v", err)
+			return nil, err
+		}
+
+		if initContainer.Ports != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support ports")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support ports")
+		}
+		if initContainer.Resources.Requests != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support resources requests")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support resources requests")
+		}
+		if initContainer.Resources.Limits != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support resources limits")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support resources limits")
+		}
+		if initContainer.LivenessProbe != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support livenessProbe")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support livenessProbe")
+		}
+		if initContainer.ReadinessProbe != nil {
+			log.G(ctx).Errorf("azure container instances initcontainers do not support readinessProbe")
+			return nil, errdefs.InvalidInput("azure container instances initContainers do not support readinessProbe")
+		}
+
+		newInitContainer := azaci.InitContainerDefinition{
+			Name: &pod.Spec.InitContainers[i].Name,
+			InitContainerPropertiesDefinition: &azaci.InitContainerPropertiesDefinition {
+				Image: &pod.Spec.InitContainers[i].Image,
+				Command: p.getCommand(&initContainer),
+				VolumeMounts: p.getVolumeMounts(&initContainer),
+				EnvironmentVariables: p.getEnvironmentVariables(&initContainer),
+			},
+		}
+
+		initContainers = append(initContainers, newInitContainer)
+	}
+	return initContainers, nil
+}
+
 func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) {
 	containers := make([]azaci.Container, 0, len(pod.Spec.Containers))