[WIP] Test udn node scale

contrib/kind.sh

@@ -946,6 +946,7 @@ install_ovn() {
   run_kubectl apply -f k8s.ovn.org_adminpolicybasedexternalroutes.yaml
   run_kubectl apply -f k8s.ovn.org_userdefinednetworks.yaml
   run_kubectl apply -f k8s.ovn.org_clusteruserdefinednetworks.yaml
+  run_kubectl apply -f k8s.ovn.org_udnnodes.yaml
   # NOTE: When you update vendoring versions for the ANP & BANP APIs, we must update the version of the CRD we pull from in the below URL
   run_kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.5/config/crd/experimental/policy.networking.k8s.io_adminnetworkpolicies.yaml
   run_kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.5/config/crd/experimental/policy.networking.k8s.io_baselineadminnetworkpolicies.yaml

dist/images/daemonset.sh

@@ -1041,5 +1041,6 @@ cp ../templates/k8s.ovn.org_egressservices.yaml.j2 ${output_dir}/k8s.ovn.org_egr
 cp ../templates/k8s.ovn.org_adminpolicybasedexternalroutes.yaml.j2 ${output_dir}/k8s.ovn.org_adminpolicybasedexternalroutes.yaml
 cp ../templates/k8s.ovn.org_userdefinednetworks.yaml.j2 ${output_dir}/k8s.ovn.org_userdefinednetworks.yaml
 cp ../templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2 ${output_dir}/k8s.ovn.org_clusteruserdefinednetworks.yaml
+cp ../templates/k8s.ovn.org_udnnodes.yaml.j2 ${output_dir}/k8s.ovn.org_udnnodes.yaml
 
 exit 0

dist/templates/k8s.ovn.org_udnnodes.yaml.j2

@@ -0,0 +1,150 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.4
+  name: udnnodes.k8s.ovn.org
+spec:
+  group: k8s.ovn.org
+  names:
+    kind: UDNNode
+    listKind: UDNNodeList
+    plural: udnnodes
+    singular: udnnode
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .status.status
+      name: Status
+      type: string
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: UDNNode holds node specific information per network
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: UDNNodeSpec defines the desired state of UDNNode
+            properties:
+              joinSubnets:
+                description: |-
+                  JoinSubnets are used inside the OVN network topology.
+
+                  Dual-stack clusters may set 2 subnets (one for each IP family), otherwise only 1 subnet is allowed.
+                  This field is only allowed for "Primary" network.
+                  It is not recommended to set this field without explicit need and understanding of the OVN network topology.
+                  When omitted, the platform will choose a reasonable default which is subject to change over time.
+                items:
+                  type: string
+                maxItems: 2
+                minItems: 1
+                type: array
+              layer2TunnelID:
+                type: integer
+              managementPortMACAddress:
+                type: string
+              network-id:
+                type: integer
+              nodeSubnets:
+                description: |-
+                  NodeSubnets are used for the pod network across the cluster.
+
+                  Dual-stack clusters may set 2 subnets (one for each IP family), otherwise only 1 subnet is allowed.
+                  Given subnet is split into smaller subnets for every node.
+                items:
+                  type: string
+                maxItems: 2
+                minItems: 1
+                type: array
+            type: object
+          status:
+            description: UDNNodeStatus defines the observed state of UDNNode
+            properties:
+              conditions:
+                description: An array of condition objects indicating details about
+                  status of EgressQoS object.
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
+              status:
+                description: A concise indication of whether the EgressQoS resource
+                  is applied with success.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

dist/templates/rbac-ovnkube-cluster-manager.yaml.j2

@@ -86,6 +86,10 @@ rules:
           - clusteruserdefinednetworks/status
           - clusteruserdefinednetworks/finalizers
       verbs: [ "patch", "update" ]
+    - apiGroups: ["k8s.ovn.org"]
+      resources:
+           - udnnodes
+      verbs: [ "create", "delete", "get", "list", "watch", "update", "patch"]
     - apiGroups: [""]
       resources:
           - events

dist/templates/rbac-ovnkube-master.yaml.j2

@@ -106,6 +106,10 @@ rules:
           - adminnetworkpolicies/status
           - baselineadminnetworkpolicies/status
       verbs: [ "patch", "update" ]
+    - apiGroups: ["k8s.ovn.org"]
+      resources:
+          - udnnodes
+      verbs: [ "create", "delete", "get", "list", "watch", "update", "patch"]
     - apiGroups: ["k8s.ovn.org"]
       resources:
           - egressfirewalls/status

dist/templates/rbac-ovnkube-node.yaml.j2

@@ -184,6 +184,10 @@ rules:
           - userdefinednetworks
           - clusteruserdefinednetworks
       verbs: [ "get", "list", "watch" ]
+    - apiGroups: ["k8s.ovn.org"]
+      resources:
+          - udnnodes
+      verbs: [ "create", "delete", "get", "list", "watch", "update", "patch"]
     {% if ovn_enable_ovnkube_identity == "true" -%}
     - apiGroups: ["certificates.k8s.io"]
       resources:

go-controller/hack/update-codegen.sh

@@ -108,3 +108,5 @@ echo "Copying userdefinednetworks CRD"
 cp _output/crds/k8s.ovn.org_userdefinednetworks.yaml ../dist/templates/k8s.ovn.org_userdefinednetworks.yaml.j2
 echo "Copying clusteruserdefinednetworks CRD"
 cp _output/crds/k8s.ovn.org_clusteruserdefinednetworks.yaml ../dist/templates/k8s.ovn.org_clusteruserdefinednetworks.yaml.j2
+echo "Copying userdefinednodes CRD"
+cp _output/crds/k8s.ovn.org_udnnodes.yaml ../dist/templates/k8s.ovn.org_udnnodes.yaml.j2

go-controller/pkg/clustermanager/network_cluster_controller.go

@@ -90,6 +90,7 @@ func newNetworkClusterController(networkIDAllocator idallocator.NamedAllocator,
 			KClient: ovnClient.KubeClient,
 		},
 		IPAMClaimsClient: ovnClient.IPAMClaimsClient,
+		UDNNodeClient:    ovnClient.UserDefinedNodeClient,
 	}
 
 	wg := &sync.WaitGroup{}
@@ -186,34 +187,36 @@ func (ncc *networkClusterController) init() error {
 		if util.IsNetworkSegmentationSupportEnabled() && ncc.IsPrimaryNetwork() {
 			// if the network is a primary L2 UDN network, then we need to reserve
 			// the IDs used by each node in this network's pod allocator
-			nodes, err := ncc.watchFactory.GetNodes()
+			udnNodes, err := ncc.watchFactory.GetUDNNodes(ncc.NetInfo.GetNetworkName())
 			if err != nil {
-				return fmt.Errorf("failed to list node objects: %w", err)
+				return fmt.Errorf("failed to list udnNode objects: %w", err)
 			}
-			for _, node := range nodes {
-				tunnelID, err := util.ParseUDNLayer2NodeGRLRPTunnelIDs(node, ncc.GetNetworkName())
-				if err != nil {
-					if util.IsAnnotationNotSetError(err) {
-						klog.Warningf("tunnelID annotation does not exist for the node %s for network %s, err: %v; we need to allocate it...",
-							node.Name, ncc.GetNetworkName(), err)
-					} else {
-						return fmt.Errorf("failed to fetch tunnelID annotation from the node %s for network %s, err: %v",
-							node.Name, ncc.GetNetworkName(), err)
-					}
+			for _, udnNode := range udnNodes {
+				nodeName := udnNode.Labels["nodeName"]
+				if len(nodeName) == 0 {
+					klog.Errorf("UDN Node is somehow missing nodeName label!: %#v", udnNode)
+					continue
 				}
-				if tunnelID != util.InvalidID {
-					if err := ncc.tunnelIDAllocator.ReserveID(ncc.GetNetworkName()+"_"+node.Name, tunnelID); err != nil {
-						return fmt.Errorf("unable to reserve id for network %s, node %s: %w", ncc.GetNetworkName(), node.Name, err)
-					}
+				if udnNode.Spec.Layer2TunnelID == nil {
+					klog.Warningf("tunnelID does not exist for UDN Node: %s, node %s for network %s, err: %v; we need to allocate it...",
+						udnNode.Name, nodeName, ncc.GetNetworkName(), err)
+					continue
+				}
+				if *udnNode.Spec.Layer2TunnelID <= util.NoID {
+					klog.Errorf("UDN Node: %q, node: %q has invalid tunnel id: %d", udnNode.Name, nodeName, *udnNode.Spec.Layer2TunnelID)
+					continue
+				}
+				if err := ncc.tunnelIDAllocator.ReserveID(ncc.GetNetworkName()+"_"+nodeName, *udnNode.Spec.Layer2TunnelID); err != nil {
+					return fmt.Errorf("unable to reserve id for network %s, node %s: %w", ncc.GetNetworkName(), nodeName, err)
 				}
 			}
 		}
 	}
 
 	if ncc.hasNodeAllocation() {
 		ncc.retryNodes = ncc.newRetryFramework(factory.NodeType, true)
-
-		ncc.nodeAllocator = node.NewNodeAllocator(networkID, ncc.NetInfo, ncc.watchFactory.NodeCoreInformer().Lister(), ncc.kube, ncc.tunnelIDAllocator)
+		ncc.nodeAllocator = node.NewNodeAllocator(networkID, ncc.NetInfo, ncc.watchFactory.NodeCoreInformer().Lister(),
+			ncc.watchFactory.UserDefinedNodeInformer().Lister(), ncc.kube, ncc.tunnelIDAllocator, ncc.watchFactory)
 		err := ncc.nodeAllocator.Init()
 		if err != nil {
 			return fmt.Errorf("failed to initialize host subnet ip allocator: %w", err)