[release-4.15] OCPBUGS-43605: Add SDN node subnet gateway IP to host-network address_set

go-controller/pkg/ovn/base_network_controller.go

@@ -3,6 +3,7 @@ package ovn
 import (
 	"fmt"
 	"net"
+	"os"
 	"sync"
 	"time"
 
@@ -37,6 +38,8 @@ import (
 	utilnet "k8s.io/utils/net"
 )
 
+const migrationEnvVar = "NODE_CNI"
+
 // CommonNetworkControllerInfo structure is place holder for all fields shared among controllers.
 type CommonNetworkControllerInfo struct {
 	client       clientset.Interface
@@ -65,6 +68,9 @@ type CommonNetworkControllerInfo struct {
 
 	// Northbound database zone name to which this Controller is connected to - aka local zone
 	zone string
+
+	// is running in SDN live migration mode
+	inMigrationMode bool
 }
 
 // BaseNetworkController structure holds per-network fields and network specific configuration
@@ -171,6 +177,7 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO
 	if err != nil {
 		return nil, fmt.Errorf("error getting NB zone name : err - %w", err)
 	}
+	_, inMigration := os.LookupEnv(migrationEnvVar)
 	return &CommonNetworkControllerInfo{
 		client:             client,
 		kube:               kube,
@@ -183,6 +190,7 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO
 		multicastSupport:   multicastSupport,
 		svcTemplateSupport: svcTemplateSupport,
 		zone:               zone,
+		inMigrationMode:    inMigration,
 	}, nil
 }
 

go-controller/pkg/ovn/default_network_controller.go

@@ -860,9 +860,12 @@ func (h *defaultNetworkControllerEventHandler) UpdateResource(oldObj, newObj int
 		if config.HybridOverlay.Enabled {
 			if util.NoHostSubnet(newNode) && !util.NoHostSubnet(oldNode) {
 				klog.Infof("Node %s has been updated to be a remote/unmanaged hybrid overlay node", newNode.Name)
+				// need to reset the host network address set, as the address is different in ovn and sdn.
+				h.oc.syncHostNetAddrSetFailed.Store(newNode.Name, true)
 				return h.oc.addUpdateHoNodeEvent(newNode)
 			} else if !util.NoHostSubnet(newNode) && util.NoHostSubnet(oldNode) {
 				klog.Infof("Node %s has been updated to be an ovn-kubernetes managed node", newNode.Name)
+				h.oc.syncHostNetAddrSetFailed.Store(newNode.Name, true)
 				if err := h.oc.deleteHoNodeEvent(newNode); err != nil {
 					return err
 				}

go-controller/pkg/ovn/master.go

@@ -912,6 +912,38 @@ func (oc *DefaultNetworkController) deleteHoNodeEvent(node *kapi.Node) error {
 			return fmt.Errorf("failed to remove hybrid overlay static routes and route policy: %w", err)
 		}
 	}
+	if oc.inMigrationMode {
+		// Remove SDN node subnet GW IP from address_set specific to HostNetworkNamespace
+		hoHostNetworkPolicyIPs, err := oc.getHostNamespaceAddressesForHoNode(node)
+		if err != nil {
+			parsedErr := err
+			if !oc.isLocalZoneNode(node) {
+				parsedErr = types.NewSuppressedError(err)
+			}
+			return fmt.Errorf("error parsing annotation for node %s: %w", node.Name, parsedErr)
+		}
+		if len(hoHostNetworkPolicyIPs) > 0 {
+			// delete the host network IPs for this ho node from host network namespace's address set
+			if err = func() error {
+				hostNetworkNamespace := config.Kubernetes.HostNetworkNamespace
+				if hostNetworkNamespace != "" {
+					nsInfo, nsUnlock, err := oc.ensureNamespaceLocked(hostNetworkNamespace, true, nil)
+					if err != nil {
+						return fmt.Errorf("failed to ensure namespace locked: %v", err)
+					}
+					defer nsUnlock()
+					if err = nsInfo.addressSet.DeleteAddresses(util.StringSlice(hoHostNetworkPolicyIPs)); err != nil &&
+						!errors.Is(err, libovsdbclient.ErrNotFound) {
+						return err
+					}
+				}
+				return nil
+			}(); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 

go-controller/pkg/ovn/namespace.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/ovn-org/libovsdb/ovsdb"
+	hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
@@ -328,20 +329,45 @@ func (oc *DefaultNetworkController) getAllHostNamespaceAddresses() []net.IP {
 	} else {
 		ips = make([]net.IP, 0, len(existingNodes))
 		for _, node := range existingNodes {
+			var hostNetworkIPs []net.IP
 			if config.HybridOverlay.Enabled && util.NoHostSubnet(node) {
-				// skip hybrid overlay nodes
-				continue
-			}
-			hostNetworkIPs, err := oc.getHostNamespaceAddressesForNode(node)
-			if err != nil {
-				klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err)
+				if oc.inMigrationMode {
+					hostNetworkIPs, err = oc.getHostNamespaceAddressesForHoNode(node)
+					if err != nil {
+						klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err)
+					}
+				} else {
+					continue
+				}
+			} else {
+				hostNetworkIPs, err = oc.getHostNamespaceAddressesForNode(node)
+				if err != nil {
+					klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err)
+				}
 			}
 			ips = append(ips, hostNetworkIPs...)
 		}
 	}
 	return ips
 }
 
+func (oc *DefaultNetworkController) getHostNamespaceAddressesForHoNode(node *kapi.Node) ([]net.IP, error) {
+	var ips []net.IP
+	// during SDN live migration, add the SDN node GW IP to the host network address set.
+	hoSubnet, ok := node.Annotations[hotypes.HybridOverlayNodeSubnet]
+	if !ok {
+		// skip hybrid overlay nodes without per-node subnet
+		return nil, nil
+	}
+	_, subnet, err := net.ParseCIDR(hoSubnet)
+	if err != nil {
+		klog.Errorf("Error parsing hybrid overlay subnet %s for node %s: %v", hoSubnet, node.Name, err)
+	}
+	gwIP := util.GetNodeGatewayIfAddr(subnet)
+	ips = append(ips, gwIP.IP)
+	return ips, nil
+}
+
 // getHostNamespaceAddressesForNode retrives management port and gateway router LRP
 // IP of a specific node
 func (oc *DefaultNetworkController) getHostNamespaceAddressesForNode(node *kapi.Node) ([]net.IP, error) {

go-controller/pkg/ovn/namespace_test.go

@@ -348,6 +348,121 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() {
 			fakeOvn.asf.EventuallyExpectAddressSetWithIPs(hostNetworkNamespace, allowIPs)
 		})
 
+		ginkgo.It("creates an address set for hybrid overlay nodes when the host network traffic namespace is created", func() {
+			config.HybridOverlay.Enabled = true
+			config.Kubernetes.NoHostSubnetNodes, _ = metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+				MatchLabels: map[string]string{"hybrid-overlay-node": "true"},
+			})
+			config.Gateway.Mode = config.GatewayModeShared
+			config.Gateway.NodeportEnable = true
+			var err error
+			config.Default.ClusterSubnets, err = config.ParseClusterSubnetEntries(clusterCIDR)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			node1 := tNode{
+				Name:       "node1",
+				NodeIP:     "1.2.3.4",
+				NodeSubnet: "10.1.1.0/24",
+				NodeGWIP:   "10.1.1.1/24",
+			}
+			// create a test node and annotate it with host subnet
+			testNode := v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   node1.Name,
+					Labels: map[string]string{"hybrid-overlay-node": "true"},
+					Annotations: map[string]string{
+						"k8s.ovn.org/hybrid-overlay-node-subnet": node1.NodeSubnet,
+						"k8s.ovn.org/node-subnets":               fmt.Sprintf("{\"default\":\"%s\"}", node1.NodeSubnet),
+					},
+				},
+			}
+
+			hostNetworkNamespace := "test-host-network-ns"
+			config.Kubernetes.HostNetworkNamespace = hostNetworkNamespace
+
+			expectedClusterLBGroup := newLoadBalancerGroup(ovntypes.ClusterLBGroupName)
+			expectedSwitchLBGroup := newLoadBalancerGroup(ovntypes.ClusterSwitchLBGroupName)
+			expectedRouterLBGroup := newLoadBalancerGroup(ovntypes.ClusterRouterLBGroupName)
+			expectedOVNClusterRouter := newOVNClusterRouter()
+			expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID})
+			expectedClusterRouterPortGroup := newRouterPortGroup()
+			expectedClusterPortGroup := newClusterPortGroup()
+
+			fakeOvn.startWithDBSetup(
+				libovsdbtest.TestSetup{
+					NBData: []libovsdbtest.TestData{
+						newClusterJoinSwitch(),
+						expectedOVNClusterRouter,
+						expectedNodeSwitch,
+						expectedClusterRouterPortGroup,
+						expectedClusterPortGroup,
+						expectedClusterLBGroup,
+						expectedSwitchLBGroup,
+						expectedRouterLBGroup,
+					},
+				},
+				&v1.NamespaceList{
+					Items: []v1.Namespace{
+						*newNamespace(hostNetworkNamespace),
+					},
+				},
+				&v1.NodeList{
+					Items: []v1.Node{
+						testNode,
+					},
+				},
+			)
+			fakeOvn.controller.multicastSupport = false
+			fakeOvn.controller.SCTPSupport = true
+			fakeOvn.controller.inMigrationMode = true
+
+			err = fakeOvn.controller.WatchNamespaces()
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			err = fakeOvn.controller.WatchNodes()
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			err = fakeOvn.controller.StartServiceController(wg, false)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			// check the namespace again and ensure the address set
+			// being created with the right set of IPs in it.
+			ip, _, _ := net.ParseCIDR(node1.NodeGWIP)
+			allowIPs := []string{ip.String()}
+			fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs)
+
+			// switch the node from a ho node to a ovn node
+			ovn_node1 := tNode{
+				Name:                 "node1",
+				NodeIP:               "1.2.3.4",
+				NodeLRPMAC:           "0a:58:0a:01:01:01",
+				LrpIP:                "100.64.0.2",
+				LrpIPv6:              "fd98::2",
+				DrLrpIP:              "100.64.0.1",
+				PhysicalBridgeMAC:    "11:22:33:44:55:66",
+				SystemID:             "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6",
+				NodeSubnet:           "10.1.1.0/24",
+				GWRouter:             ovntypes.GWRouterPrefix + "node1",
+				GatewayRouterIPMask:  "172.16.16.2/24",
+				GatewayRouterIP:      "172.16.16.2",
+				GatewayRouterNextHop: "172.16.16.1",
+				PhysicalBridgeName:   "br-eth0",
+				NodeGWIP:             "10.1.1.1/24",
+				NodeMgmtPortIP:       "10.1.1.2",
+				NodeMgmtPortMAC:      "0a:58:0a:01:01:02",
+				DnatSnatIP:           "169.254.0.1",
+			}
+			ovnTestNode := ovn_node1.k8sNode("2")
+			ovnTestNode.Annotations["k8s.ovn.org/hybrid-overlay-node-subnet"] = testNode.Annotations["k8s.ovn.org/hybrid-overlay-node-subnet"]
+			ovnTestNode.Annotations["k8s.ovn.org/node-subnets"] = testNode.Annotations["k8s.ovn.org/node-subnets"]
+
+			fakeOvn.fakeClient.GetNodeClientset().KubeClient.CoreV1().Nodes().Update(context.TODO(), &ovnTestNode, metav1.UpdateOptions{})
+			// check the namespace again and ensure the ho node gateway IP is removed from the address_set
+			// and the ovn ips are added instead.
+			allowIPs = []string{"10.1.1.2", "100.64.0.2"}
+			fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs)
+		})
+
 		ginkgo.It("reconciles an existing namespace port group, without updating it", func() {
 			// this flag will create namespaced port group
 			config.OVNKubernetesFeature.EnableEgressFirewall = true