From 4756c33c4e6c67839c9d1ca1e0c397851f737e59 Mon Sep 17 00:00:00 2001 From: Steven Kreitzer Date: Thu, 2 Jan 2025 13:54:53 -0600 Subject: [PATCH] feat(volsync): volsync policy --- .taskfiles/volsync/Taskfile.yaml | 3 - .../apps/kyverno/kyverno/app/helmrelease.yaml | 37 +++-- .../apps/kyverno/kyverno/policies/gatus.yaml | 10 +- .../kyverno/policies/kustomization.yaml | 1 + .../kyverno/kyverno/policies/volsync.yaml | 130 ++++++++++++++++++ .../media/jellyseerr/app/kustomization.yaml | 5 + kubernetes/apps/media/jellyseerr/app/pvc.yaml | 13 ++ kubernetes/apps/media/jellyseerr/ks.yaml | 23 ++++ kubernetes/apps/media/kustomization.yaml | 1 + .../apps/volsync-system/volsync/ks.yaml | 1 + 10 files changed, 197 insertions(+), 27 deletions(-) create mode 100644 kubernetes/apps/kyverno/kyverno/policies/volsync.yaml create mode 100644 kubernetes/apps/media/jellyseerr/app/kustomization.yaml create mode 100644 kubernetes/apps/media/jellyseerr/app/pvc.yaml create mode 100644 kubernetes/apps/media/jellyseerr/ks.yaml diff --git a/.taskfiles/volsync/Taskfile.yaml b/.taskfiles/volsync/Taskfile.yaml index ad051acd7e..74af9ece7a 100644 --- a/.taskfiles/volsync/Taskfile.yaml +++ b/.taskfiles/volsync/Taskfile.yaml @@ -99,10 +99,7 @@ tasks: now: '{{now | date "150405"}}' ns: '{{.ns | default "default"}}' job: volsync-src-{{.app}} - controller: - sh: true && {{.VOLSYNC_RESOURCES_DIR}}/which-controller.sh {{.app}} {{.ns}} preconditions: - - test -f {{.VOLSYNC_RESOURCES_DIR}}/which-controller.sh - test -f {{.VOLSYNC_RESOURCES_DIR}}/wait-for-job.sh - kubectl -n {{.ns}} get replicationsources {{.app}} diff --git a/kubernetes/apps/kyverno/kyverno/app/helmrelease.yaml b/kubernetes/apps/kyverno/kyverno/app/helmrelease.yaml index 91aa568710..b434479c8d 100644 --- a/kubernetes/apps/kyverno/kyverno/app/helmrelease.yaml +++ b/kubernetes/apps/kyverno/kyverno/app/helmrelease.yaml @@ -31,31 +31,30 @@ spec: rbac: clusterRole: extraResources: - - apiGroups: - - "" - resources: - - pods - verbs: - - create - - update - - delete + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "update", "delete"] + - apiGroups: ["external-secrets.io"] + resources: ["externalsecrets"] + verbs: ["create", "update", "patch", "delete", "get", "list"] + - apiGroups: ["volsync.backube"] + resources: ["replicationsources", "replicationdestinations"] + verbs: ["create", "update", "patch", "delete", "get", "list"] serviceMonitor: enabled: true backgroundController: rbac: clusterRole: extraResources: - - apiGroups: - - "" - resources: - - pods - verbs: - - create - - update - - patch - - delete - - get - - list + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "update", "patch", "delete", "get", "list"] + - apiGroups: ["external-secrets.io"] + resources: ["externalsecrets"] + verbs: ["create", "update", "patch", "delete", "get", "list"] + - apiGroups: ["volsync.backube"] + resources: ["replicationsources", "replicationdestinations"] + verbs: ["create", "update", "patch", "delete", "get", "list"] resources: requests: cpu: 100m diff --git a/kubernetes/apps/kyverno/kyverno/policies/gatus.yaml b/kubernetes/apps/kyverno/kyverno/policies/gatus.yaml index b54aa6c7c6..9494348466 100644 --- a/kubernetes/apps/kyverno/kyverno/policies/gatus.yaml +++ b/kubernetes/apps/kyverno/kyverno/policies/gatus.yaml @@ -13,7 +13,6 @@ metadata: all Ingresses with the ingressClassName set to external. pod-policies.kyverno.io/autogen-controllers: none spec: - generateExisting: true rules: - name: *name match: @@ -36,21 +35,22 @@ spec: context: - name: GATUS_HOST variable: - value: '{{ request.object.metadata.annotations."gatus.io/host" || request.object.spec.rules[0].host }}' + value: "{{ request.object.metadata.annotations.\"gatus.io/host\" || request.object.spec.rules[0].host }}" jmesPath: "to_string(@)" - name: GATUS_NAME variable: - value: '{{ request.object.metadata.annotations."gatus.io/name" || request.object.metadata.name }}' + value: "{{ request.object.metadata.annotations.\"gatus.io/name\" || request.object.metadata.name }}" jmesPath: "to_string(@)" - name: GATUS_PATH variable: - value: '{{ request.object.metadata.annotations."gatus.io/path" || request.object.spec.rules[0].http.paths[0].path }}' + value: "{{ request.object.metadata.annotations.\"gatus.io/path\" || request.object.spec.rules[0].http.paths[0].path }}" jmesPath: "to_string(@)" - name: GATUS_STATUS_CODE variable: - value: '{{ request.object.metadata.annotations."gatus.io/status-code" || `200` }}' + value: "{{ request.object.metadata.annotations.\"gatus.io/status-code\" || '200' }}" jmesPath: "to_string(@)" generate: + generateExisting: true apiVersion: v1 kind: ConfigMap name: "{{ request.object.metadata.name }}-gatus-ep" diff --git a/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml b/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml index 15d774b918..a5a86b1c94 100644 --- a/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml +++ b/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml @@ -5,3 +5,4 @@ resources: - ./gatus.yaml - ./limits.yaml - ./ndots.yaml + - ./volsync.yaml diff --git a/kubernetes/apps/kyverno/kyverno/policies/volsync.yaml b/kubernetes/apps/kyverno/kyverno/policies/volsync.yaml new file mode 100644 index 0000000000..6423254f65 --- /dev/null +++ b/kubernetes/apps/kyverno/kyverno/policies/volsync.yaml @@ -0,0 +1,130 @@ +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: volsync + annotations: + policies.kyverno.io/title: Volume Synchronization + policies.kyverno.io/category: Storage + policies.kyverno.io/severity: low + policies.kyverno.io/subject: Pod + policies.kyverno.io/description: >- + This policy will automatically synchronize volumes for all Pods with + the volumeSynchronization set to true. + pod-policies.kyverno.io/autogen-controllers: none +spec: + useServerSideApply: true + rules: + - name: volsync-mutate-pvc + match: &match + resources: + kinds: + - PersistentVolumeClaim + annotations: + volsync.io/enabled: "true" + mutate: + patchStrategicMerge: + spec: + dataSourceRef: + kind: ReplicationDestination + apiGroup: volsync.backube + name: "{{ request.object.metadata.name }}" + - name: volsync-external-secret + match: *match + generate: + generateExisting: true + apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + name: "{{ request.object.metadata.name }}-restic" + namespace: "{{ request.object.metadata.namespace }}" + synchronize: true + data: + spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-connect + target: + name: "{{ request.object.metadata.name }}-restic-secret" + creationPolicy: Owner + template: + engineVersion: v2 + data: + RESTIC_REPOSITORY: "\\{{ .REPOSITORY_TEMPLATE }}/{{ request.object.metadata.name }}" + RESTIC_PASSWORD: "\\{{ .RESTIC_PASSWORD }}" + AWS_ACCESS_KEY_ID: "\\{{ .AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "\\{{ .AWS_SECRET_ACCESS_KEY }}" + dataFrom: + - extract: + key: volsync-restic-template + - name: volsync-replication-destination + match: *match + context: &context + - name: VOLSYNC_USER + variable: + value: "{{ request.object.metadata.annotations.\"volsync.io/user\" || '568' }}" + jmesPath: "to_number(@)" + - name: VOLSYNC_GROUP + variable: + value: "{{ request.object.metadata.annotations.\"volsync.io/group\" || '568' }}" + jmesPath: "to_number(@)" + - name: VOLSYNC_CACHE + variable: + value: "{{ request.object.metadata.annotations.\"volsync.io/cache\" || '8Gi' }}" + jmesPath: "to_string(@)" + generate: + generateExisting: true + apiVersion: volsync.backube/v1alpha1 + kind: ReplicationDestination + name: "{{ request.object.metadata.name }}" + namespace: "{{ request.object.metadata.namespace }}" + synchronize: true + data: + spec: + trigger: + manual: restore-once + restic: + repository: "{{ request.object.metadata.name }}-restic-secret" + copyMethod: Snapshot + accessModes: "{{ request.object.spec.accessModes }}" + storageClassName: "{{ request.object.spec.storageClassName }}" + volumeSnapshotClassName: "csi-{{ request.object.spec.storageClassName }}" + cacheAccessModes: ["ReadWriteOnce"] + cacheCapacity: "{{ VOLSYNC_CACHE }}" + cacheStorageClassName: openebs-hostpath + moverSecurityContext: + runAsUser: "{{ VOLSYNC_USER }}" + runAsGroup: "{{ VOLSYNC_GROUP }}" + fsGroup: "{{ VOLSYNC_GROUP }}" + capacity: "{{ request.object.spec.resources.requests.storage }}" + - name: volsync-replication-source + match: *match + context: *context + generate: + generateExisting: true + apiVersion: volsync.backube/v1alpha1 + kind: ReplicationSource + name: "{{ request.object.metadata.name }}" + namespace: "{{ request.object.metadata.namespace }}" + synchronize: true + data: + spec: + sourcePVC: "{{ request.object.metadata.name }}" + trigger: + schedule: "0 * * * *" + restic: + pruneIntervalDays: 14 + repository: "{{ request.object.metadata.name }}-restic-secret" + copyMethod: Snapshot + accessModes: "{{ request.object.spec.accessModes }}" + storageClassName: "{{ request.object.spec.storageClassName }}" + volumeSnapshotClassName: "csi-{{ request.object.spec.storageClassName }}" + cacheAccessModes: ["ReadWriteOnce"] + cacheCapacity: "{{ VOLSYNC_CACHE }}" + cacheStorageClassName: openebs-hostpath + moverSecurityContext: + runAsUser: "{{ VOLSYNC_USER }}" + runAsGroup: "{{ VOLSYNC_GROUP }}" + fsGroup: "{{ VOLSYNC_GROUP }}" + retain: + hourly: 24 + daily: 7 diff --git a/kubernetes/apps/media/jellyseerr/app/kustomization.yaml b/kubernetes/apps/media/jellyseerr/app/kustomization.yaml new file mode 100644 index 0000000000..23a5576057 --- /dev/null +++ b/kubernetes/apps/media/jellyseerr/app/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./pvc.yaml diff --git a/kubernetes/apps/media/jellyseerr/app/pvc.yaml b/kubernetes/apps/media/jellyseerr/app/pvc.yaml new file mode 100644 index 0000000000..24c2cf57e8 --- /dev/null +++ b/kubernetes/apps/media/jellyseerr/app/pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jellyseerr + annotations: + volsync.io/enabled: "true" +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 15Gi + storageClassName: ceph-block diff --git a/kubernetes/apps/media/jellyseerr/ks.yaml b/kubernetes/apps/media/jellyseerr/ks.yaml new file mode 100644 index 0000000000..8da36dbcb5 --- /dev/null +++ b/kubernetes/apps/media/jellyseerr/ks.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app jellyseerr + namespace: flux-system +spec: + targetNamespace: media + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: rook-ceph-cluster + - name: volsync + path: ./kubernetes/apps/media/jellyseerr/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/apps/media/kustomization.yaml b/kubernetes/apps/media/kustomization.yaml index 5e40b18e80..e9dbd1bb8e 100644 --- a/kubernetes/apps/media/kustomization.yaml +++ b/kubernetes/apps/media/kustomization.yaml @@ -6,6 +6,7 @@ resources: - ./autobrr/ks.yaml - ./bazarr/ks.yaml - ./cross-seed/ks.yaml + - ./jellyseerr/ks.yaml - ./overseerr/ks.yaml - ./plex/ks.yaml - ./prowlarr/ks.yaml diff --git a/kubernetes/apps/volsync-system/volsync/ks.yaml b/kubernetes/apps/volsync-system/volsync/ks.yaml index 5eae3d7d6e..ae9658b1e4 100644 --- a/kubernetes/apps/volsync-system/volsync/ks.yaml +++ b/kubernetes/apps/volsync-system/volsync/ks.yaml @@ -10,6 +10,7 @@ spec: labels: app.kubernetes.io/name: *app dependsOn: + - name: kyverno-policies - name: snapshot-controller path: ./kubernetes/apps/volsync-system/volsync/app prune: true