feat(kyverno): volsync policy (#3250)

.envrc

@@ -1,5 +1,6 @@
 #shellcheck disable=SC2148,SC2155
-export KUBECONFIG="$(PWD)/kubernetes/kubeconfig"
-export SOPS_AGE_KEY_FILE="$(PWD)/age.key"
-export TALOSCONFIG="$(PWD)/talos/clusterconfig/talosconfig"
+export MINIJINJA_CONFIG_FILE="$(expand_path ./.minijinja.toml)"
+export KUBECONFIG="$(expand_path ./kubernetes/kubeconfig)"
+export SOPS_AGE_KEY_FILE="$(expand_path ./age.key)"
+export TALOSCONFIG="$(expand_path ./talos/clusterconfig/talosconfig)"
 export TASK_X_MAP_VARIABLES=0

.minijinja.toml

@@ -0,0 +1,5 @@
+autoescape = "none"
+newline = true
+trim-blocks = true
+lstrip-blocks = true
+env = true

.taskfiles/bootstrap/Taskfile.yaml

@@ -41,65 +41,17 @@ tasks:
   rook:
     internal: true
     cmds:
-      - for: { var: nodes }
-        task: rook-data
-        vars:
-          node: '{{.ITEM}}'
-      - for: { var: m0 }
-        task: rook-disk
-        vars:
-          node: m0
-          serial: '{{.ITEM}}'
-      - for: { var: m1 }
-        task: rook-disk
-        vars:
-          node: m1
-          serial: '{{.ITEM}}'
-      - for: { var: m2 }
-        task: rook-disk
-        vars:
-          node: m2
-          serial: '{{.ITEM}}'
-    vars:
-      nodes: m0 m1 m2
-      m0: S72ANJ0TC02334R
-      m1: S72ANJ0TC01288Z
-      m2: S72ANJ0TC02325Y
-
-  rook-data:
-    internal: true
-    cmds:
-      - envsubst < <(cat {{.BOOTSTRAP_RESOURCES_DIR}}/rook-data-job.tmpl.yaml) | kubectl apply -f -
-      - bash {{.BOOTSTRAP_RESOURCES_DIR}}/wait-for-job.sh {{.job}} default
-      - kubectl --namespace default wait job/{{.job}} --for condition=complete --timeout=1m
-      - kubectl --namespace default logs job/{{.job}}
-      - kubectl --namespace default delete job {{.job}}
-    env:
-      job: '{{.job}}'
-      node: '{{.node}}'
-    vars:
-      job: wipe-data-{{.node}}
-    preconditions:
-      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/wait-for-job.sh
-      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/rook-data-job.tmpl.yaml
-
-  rook-disk:
-    internal: true
-    cmds:
-      - envsubst < <(cat {{.BOOTSTRAP_RESOURCES_DIR}}/rook-disk-job.tmpl.yaml) | kubectl apply -f -
-      - bash {{.BOOTSTRAP_RESOURCES_DIR}}/wait-for-job.sh {{.job}} default
-      - kubectl --namespace default wait job/{{.job}} --for condition=complete --timeout=1m
-      - kubectl --namespace default logs job/{{.job}}
-      - kubectl --namespace default delete job {{.job}}
+      - minijinja-cli {{.BOOTSTRAP_RESOURCES_DIR}}/wipe-rook.yaml.j2 | kubectl apply --server-side --filename -
+      - until kubectl --namespace default get job/wipe-rook &>/dev/null; do sleep 5; done
+      - kubectl --namespace default wait job/wipe-rook --for=condition=complete --timeout=5m
+      - stern --namespace default job/wipe-rook --no-follow
+      - kubectl --namespace default delete job wipe-rook
     env:
-      disk: /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_{{.serial}}
-      job: '{{.job}}'
-      node: '{{.node}}'
-    vars:
-      job: wipe-disk-{{.node}}-{{.serial | lower}}
+      MODEL: SAMSUNG_MZQL23T8HCLS-00A07
+      NODE_COUNT:
+        sh: talosctl config info --output json | jq --raw-output '.nodes | length'
     preconditions:
-      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/wait-for-job.sh
-      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/rook-disk-job.tmpl.yaml
+      - test -f {{.BOOTSTRAP_RESOURCES_DIR}}/wipe-rook.yaml.j2
 
   flux:
     internal: true

.taskfiles/bootstrap/resources/wipe-rook.yaml.j2

@@ -0,0 +1,59 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: &app wipe-rook
+  namespace: default
+  labels:
+    app.kubernetes.io/name: *app
+spec:
+  parallelism: {{ ENV.NODE_COUNT }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: *app
+    spec:
+      restartPolicy: Never
+      initContainers:
+        - name: data
+          image: docker.io/library/alpine:latest
+          command: ["/bin/sh", "-c"]
+          args: ["rm -rf /mnt/host_var/lib/rook"]
+          volumeMounts:
+            - mountPath: /mnt/host_var
+              name: host-var
+          securityContext:
+            privileged: true
+          resources: {}
+      containers:
+        - name: disk
+          image: docker.io/library/alpine:latest
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              apk add --no-cache findutils nvme-cli;
+              DISK=$(find /dev/disk/by-id/ -iname "*{{ ENV.MODEL }}*" -not -name "*-part[0-9+]");
+              echo "=== Wiping $DISK ===";
+              nvme format --lbaf=1 $DISK --force;
+              nvme format --block-size=4096 $DISK --force;
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: host-dev
+              mountPath: /dev/disk/by-id
+          resources: {}
+      volumes:
+        - name: host-var
+          hostPath:
+            path: /var
+        - name: host-dev
+          hostPath:
+            path: /dev/disk/by-id
+            type: Directory
+      topologySpreadConstraints:
+        - maxSkew: 1
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: *app
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule

.taskfiles/kubernetes/Taskfile.yaml

@@ -30,8 +30,9 @@ tasks:
     desc: Run a privileged pod
     cmd: |
       kubectl run privileged-{{.node}} -i --rm --image=null \
-        --overrides="$(yq {{.KUBERNETES_RESOURCES_DIR}}/privileged-pod.tmpl.yaml -o=json | envsubst)"
+        --overrides="$(yq {{.KUBERNETES_RESOURCES_DIR}}/privileged-pod.yaml.j2 -o json | minijinja-cli)"
     env:
-      node: '{{.node}}'
+      NODE: '{{.node}}'
     preconditions:
-      - test -f {{.KUBERNETES_RESOURCES_DIR}}/privileged-pod.tmpl.yaml
+      - test -f {{.KUBERNETES_RESOURCES_DIR}}/privileged-pod.yaml.j2
+      - which kubectl minijinja-cli

.taskfiles/kubernetes/resources/privileged-pod.tmpl.yaml → .taskfiles/kubernetes/resources/privileged-pod.yaml.j2

@@ -22,7 +22,7 @@ spec:
   hostIPC: true
   hostNetwork: true
   hostPID: true
-  nodeName: ${node}
+  nodeName: "{{ ENV.NODE }}"
   restartPolicy: Never
   volumes:
     - name: rootfs