Add new validator when adding bd

cmd/node-disk-manager-webhook/main.go

@@ -8,6 +8,10 @@ import (
 	"github.com/harvester/webhook/pkg/config"
 	"github.com/harvester/webhook/pkg/server"
 	"github.com/harvester/webhook/pkg/server/admission"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
+	ctlcorev1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	ctlstorage "github.com/rancher/wrangler/v3/pkg/generated/controllers/storage"
+	ctlstoragev1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/storage/v1"
 	"github.com/rancher/wrangler/v3/pkg/kubeconfig"
 	"github.com/rancher/wrangler/v3/pkg/signals"
 	"github.com/rancher/wrangler/v3/pkg/start"
@@ -23,7 +27,9 @@ import (
 const webhookName = "harvester-node-disk-manager-webhook"
 
 type resourceCaches struct {
-	bdCache ctldiskv1.BlockDeviceCache
+	bdCache           ctldiskv1.BlockDeviceCache
+	storageClassCache ctlstoragev1.StorageClassCache
+	pvCache           ctlcorev1.PersistentVolumeCache
 }
 
 func main() {
@@ -109,7 +115,7 @@ func runWebhookServer(ctx context.Context, cfg *rest.Config, options *config.Opt
 		bdMutator,
 	}
 
-	bdValidator := blockdevice.NewBlockdeviceValidator(resourceCaches.bdCache)
+	bdValidator := blockdevice.NewBlockdeviceValidator(resourceCaches.bdCache, resourceCaches.storageClassCache, resourceCaches.pvCache)
 	var validators = []admission.Validator{
 		bdValidator,
 	}
@@ -138,9 +144,19 @@ func newCaches(ctx context.Context, cfg *rest.Config, threadiness int) (*resourc
 	if err != nil {
 		return nil, err
 	}
-	starters = append(starters, disks)
+	storageFactory, err := ctlstorage.NewFactoryFromConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	coreFactory, err := core.NewFactoryFromConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+	starters = append(starters, disks, storageFactory, coreFactory)
 	resourceCaches := &resourceCaches{
-		bdCache: disks.Harvesterhci().V1beta1().BlockDevice().Cache(),
+		bdCache:           disks.Harvesterhci().V1beta1().BlockDevice().Cache(),
+		storageClassCache: storageFactory.Storage().V1().StorageClass().Cache(),
+		pvCache:           coreFactory.Core().V1().PersistentVolume().Cache(),
 	}
 
 	if err := start.All(ctx, threadiness, starters...); err != nil {

deploy/charts/harvester-node-disk-manager/templates/rbac.yaml

@@ -50,6 +50,12 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "secrets", "configmaps" ]
     verbs: [ "*" ]
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumes" ]
+    verbs: [ "get", "watch", "list" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "storageclasses" ]
+    verbs: [ "get", "watch", "list" ]
   - apiGroups: [ "harvesterhci.io" ]
     resources: [ "blockdevices" ]
     verbs: [ "*" ]

pkg/controller/blockdevice/controller.go

@@ -218,7 +218,6 @@ func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisione
 	// upgrade case, we need to update some fields
 	if device.Spec.Provisioner == nil && device.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned {
 		device.Spec.Provision = true
-		device.Spec.FileSystem.Provisioned = false
 		provisionerLHV1 := &diskv1.LonghornProvisionerInfo{
 			EngineVersion: provisioner.TypeLonghornV1,
 		}

pkg/provisioner/longhornv1.go

@@ -97,6 +97,8 @@ func (p *LonghornV1Provisioner) Provision() (bool, error) {
 	}
 
 	if (synced && !diskv1.DiskAddedToNode.IsTrue(p.device)) || provisioned {
+		// mark `filesystem.provisioned` to true, that the mutator could work
+		p.device.Spec.FileSystem.Provisioned = true
 		logrus.Debugf("Set blockdevice CRD (%v) to provisioned", p.device)
 		msg := fmt.Sprintf("Added disk %s to longhorn node `%s` as an additional disk", p.device.Name, p.nodeObj.Name)
 		setCondDiskAddedToNodeTrue(p.device, msg, diskv1.ProvisionPhaseProvisioned)
@@ -111,6 +113,7 @@ func (p *LonghornV1Provisioner) UnProvision() (bool, error) {
 
 	// inner functions
 	updateProvisionPhaseUnprovisioned := func() {
+		p.device.Spec.FileSystem.Provisioned = false
 		msg := fmt.Sprintf("Disk not in longhorn node `%s`", p.nodeObj.Name)
 		setCondDiskAddedToNodeFalse(p.device, msg, diskv1.ProvisionPhaseUnprovisioned)
 	}

pkg/utils/utils.go

@@ -19,6 +19,10 @@ const (
 	DiskRemoveTag = "harvester-ndm-disk-remove"
 	// Harvester Namespace
 	HarvesterNS = "harvester-system"
+	// LVMCSIDriver is the LVM CSI driver name
+	LVMCSIDriver = "lvm.driver.harvesterhci.io"
+	// LVMTopologyNodeKey is the key of LVM topology node
+	LVMTopologyNodeKey = "topology.lvm.csi/node"
 )
 
 var CmdTimeoutError error

pkg/webhook/blockdevice/mutator.go

@@ -62,7 +62,7 @@ func (m *Mutator) Update(req *admission.Request, oldObj, newObj runtime.Object)
 		return patchOps, nil
 	}
 	// means we need to disable, align the .spec.filesystem.provisioned with .spec.provision -> false
-	if prevProvision && !newBd.Spec.FileSystem.Provisioned {
+	if prevProvision && !newBd.Spec.FileSystem.Provisioned && oldBd.Spec.FileSystem.Provisioned {
 		if newBd.Spec.Provision {
 			patchProvision := admission.PatchOp{
 				Op:    admission.PatchOpReplace,

pkg/webhook/blockdevice/validator.go

@@ -1,35 +1,54 @@
 package blockdevice
 
 import (
+	"fmt"
+
 	werror "github.com/harvester/webhook/pkg/error"
 	"github.com/harvester/webhook/pkg/server/admission"
+	ctlcorev1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	ctlstoragev1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/storage/v1"
+	"github.com/sirupsen/logrus"
 	admissionregv1 "k8s.io/api/admissionregistration/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 
 	diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1"
 	ctldiskv1 "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io/v1beta1"
+	"github.com/harvester/node-disk-manager/pkg/utils"
 )
 
 type Validator struct {
 	admission.DefaultValidator
 
-	BlockdeviceCache ctldiskv1.BlockDeviceCache
+	BlockdeviceCache  ctldiskv1.BlockDeviceCache
+	storageClassCache ctlstoragev1.StorageClassCache
+	pvCache           ctlcorev1.PersistentVolumeCache
 }
 
-func NewBlockdeviceValidator(blockdeviceCache ctldiskv1.BlockDeviceCache) *Validator {
+func NewBlockdeviceValidator(blockdeviceCache ctldiskv1.BlockDeviceCache, storageClassCache ctlstoragev1.StorageClassCache, pvCache ctlcorev1.PersistentVolumeCache) *Validator {
 	return &Validator{
-		BlockdeviceCache: blockdeviceCache,
+		BlockdeviceCache:  blockdeviceCache,
+		storageClassCache: storageClassCache,
+		pvCache:           pvCache,
 	}
 }
 
 func (v *Validator) Create(_ *admission.Request, newObj runtime.Object) error {
 	bd := newObj.(*diskv1.BlockDevice)
-	return v.validateProvisioner(bd)
+	if err := v.validateProvisioner(bd); err != nil {
+		return err
+	}
+	return v.validateLVMProvisioner(nil, bd)
 }
 
-func (v *Validator) Update(_ *admission.Request, _, newObj runtime.Object) error {
+func (v *Validator) Update(_ *admission.Request, oldObj, newObj runtime.Object) error {
 	newBd := newObj.(*diskv1.BlockDevice)
-	return v.validateProvisioner(newBd)
+	oldBd := oldObj.(*diskv1.BlockDevice)
+	if err := v.validateProvisioner(newBd); err != nil {
+		return err
+	}
+	return v.validateLVMProvisioner(oldBd, newBd)
 }
 
 func (v *Validator) validateProvisioner(bd *diskv1.BlockDevice) error {
@@ -43,6 +62,71 @@ func (v *Validator) validateProvisioner(bd *diskv1.BlockDevice) error {
 	return nil
 }
 
+// validateLVMProvisioner will check the blockdeivce with LVM provisioner and block
+// if there is already have any pvc created with in the target volume group
+func (v *Validator) validateLVMProvisioner(oldbd, newbd *diskv1.BlockDevice) error {
+
+	// check again, skip if no LVM provisioner
+	if newbd.Spec.Provisioner == nil || newbd.Spec.Provisioner.LVM == nil {
+		return nil
+	}
+
+	// Adding case, should not happened
+	if oldbd == nil {
+		logrus.Info("Adding blockdevice with provisioner should not happen")
+		return v.validateVGIsAlreadyUsed(newbd)
+	}
+
+	// means add or remove
+	if oldbd.Spec.Provision != newbd.Spec.Provision {
+		return v.validateVGIsAlreadyUsed(newbd)
+	}
+
+	return nil
+
+}
+
+func (v *Validator) validateVGIsAlreadyUsed(bd *diskv1.BlockDevice) error {
+	targetVGName := bd.Spec.Provisioner.LVM.VgName
+	// find what we wanted to check
+	allStorageClasses, err := v.storageClassCache.List(labels.Everything())
+	if err != nil {
+		return werror.NewBadRequest("Failed to list storage classes")
+	}
+	targetSC := ""
+	for _, sc := range allStorageClasses {
+		if sc.Provisioner != utils.LVMCSIDriver {
+			continue
+		}
+		scTargetNode := getLVMTopologyNodes(sc)
+		if scTargetNode != bd.Spec.NodeName {
+			continue
+		}
+		if sc.Parameters["vgName"] == targetVGName {
+			targetSC = sc.Name
+			break
+		}
+	}
+
+	// no related SC found, just return
+	if targetSC == "" {
+		return nil
+	}
+
+	// check if there is any PV created with the targetSC
+	pvs, err := v.pvCache.List(labels.Everything())
+	if err != nil {
+		return werror.NewBadRequest("Failed to list PVs")
+	}
+	for _, pv := range pvs {
+		if pv.Spec.StorageClassName == targetSC {
+			errStr := fmt.Sprintf("There is already a PVC created using the target volume group (%v), so we cannot add or remove the associated blockdevices", targetVGName)
+			return werror.NewBadRequest(errStr)
+		}
+	}
+	return nil
+}
+
 func (v *Validator) Resource() admission.Resource {
 	return admission.Resource{
 		Names:      []string{"blockdevices"},
@@ -56,3 +140,14 @@ func (v *Validator) Resource() admission.Resource {
 		},
 	}
 }
+
+func getLVMTopologyNodes(sc *storagev1.StorageClass) string {
+	for _, topology := range sc.AllowedTopologies {
+		for _, matchLabel := range topology.MatchLabelExpressions {
+			if matchLabel.Key == utils.LVMTopologyNodeKey {
+				return matchLabel.Values[0]
+			}
+		}
+	}
+	return ""
+}