From 1d29d9743dd20134ef7cea1fa87030a80b8ef2d1 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Mon, 14 Oct 2024 10:46:18 +0000 Subject: [PATCH] Add SDN node subnet gateway IP to host-network address_set During live SDN migration, host-to-pod traffic originating from SDN nodes will use the first IP address of the hybrid overlay node subnet. These IPs are being added to ensure proper functionality of host network policies. Signed-off-by: Peng Liu --- .../pkg/ovn/base_network_controller.go | 8 ++ .../pkg/ovn/default_network_controller.go | 3 + go-controller/pkg/ovn/master.go | 32 +++++ go-controller/pkg/ovn/namespace.go | 38 +++++- go-controller/pkg/ovn/namespace_test.go | 115 ++++++++++++++++++ 5 files changed, 190 insertions(+), 6 deletions(-) diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index 8a36c7297f..79d1b98f20 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -3,6 +3,7 @@ package ovn import ( "fmt" "net" + "os" "sync" "time" @@ -37,6 +38,8 @@ import ( utilnet "k8s.io/utils/net" ) +const migrationEnvVar = "NODE_CNI" + // CommonNetworkControllerInfo structure is place holder for all fields shared among controllers. type CommonNetworkControllerInfo struct { client clientset.Interface @@ -65,6 +68,9 @@ type CommonNetworkControllerInfo struct { // Northbound database zone name to which this Controller is connected to - aka local zone zone string + + // is running in SDN live migration mode + inMigrationMode bool } // BaseNetworkController structure holds per-network fields and network specific configuration @@ -171,6 +177,7 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO if err != nil { return nil, fmt.Errorf("error getting NB zone name : err - %w", err) } + _, inMigration := os.LookupEnv(migrationEnvVar) return &CommonNetworkControllerInfo{ client: client, kube: kube, @@ -183,6 +190,7 @@ func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeO multicastSupport: multicastSupport, svcTemplateSupport: svcTemplateSupport, zone: zone, + inMigrationMode: inMigration, }, nil } diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index 75eef43a21..e2236b98c5 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -860,9 +860,12 @@ func (h *defaultNetworkControllerEventHandler) UpdateResource(oldObj, newObj int if config.HybridOverlay.Enabled { if util.NoHostSubnet(newNode) && !util.NoHostSubnet(oldNode) { klog.Infof("Node %s has been updated to be a remote/unmanaged hybrid overlay node", newNode.Name) + // need to reset the host network address set, as the address is different in ovn and sdn. + h.oc.syncHostNetAddrSetFailed.Store(newNode.Name, true) return h.oc.addUpdateHoNodeEvent(newNode) } else if !util.NoHostSubnet(newNode) && util.NoHostSubnet(oldNode) { klog.Infof("Node %s has been updated to be an ovn-kubernetes managed node", newNode.Name) + h.oc.syncHostNetAddrSetFailed.Store(newNode.Name, true) if err := h.oc.deleteHoNodeEvent(newNode); err != nil { return err } diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index d1720f815b..12d5995e60 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -912,6 +912,38 @@ func (oc *DefaultNetworkController) deleteHoNodeEvent(node *kapi.Node) error { return fmt.Errorf("failed to remove hybrid overlay static routes and route policy: %w", err) } } + if oc.inMigrationMode { + // Remove SDN node subnet GW IP from address_set specific to HostNetworkNamespace + hoHostNetworkPolicyIPs, err := oc.getHostNamespaceAddressesForHoNode(node) + if err != nil { + parsedErr := err + if !oc.isLocalZoneNode(node) { + parsedErr = types.NewSuppressedError(err) + } + return fmt.Errorf("error parsing annotation for node %s: %w", node.Name, parsedErr) + } + if len(hoHostNetworkPolicyIPs) > 0 { + // delete the host network IPs for this ho node from host network namespace's address set + if err = func() error { + hostNetworkNamespace := config.Kubernetes.HostNetworkNamespace + if hostNetworkNamespace != "" { + nsInfo, nsUnlock, err := oc.ensureNamespaceLocked(hostNetworkNamespace, true, nil) + if err != nil { + return fmt.Errorf("failed to ensure namespace locked: %v", err) + } + defer nsUnlock() + if err = nsInfo.addressSet.DeleteAddresses(util.StringSlice(hoHostNetworkPolicyIPs)); err != nil && + !errors.Is(err, libovsdbclient.ErrNotFound) { + return err + } + } + return nil + }(); err != nil { + return err + } + } + } + return nil } diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index 3cb62705e4..367892dab7 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ovn-org/libovsdb/ovsdb" + hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -328,13 +329,21 @@ func (oc *DefaultNetworkController) getAllHostNamespaceAddresses() []net.IP { } else { ips = make([]net.IP, 0, len(existingNodes)) for _, node := range existingNodes { + var hostNetworkIPs []net.IP if config.HybridOverlay.Enabled && util.NoHostSubnet(node) { - // skip hybrid overlay nodes - continue - } - hostNetworkIPs, err := oc.getHostNamespaceAddressesForNode(node) - if err != nil { - klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err) + if oc.inMigrationMode { + hostNetworkIPs, err = oc.getHostNamespaceAddressesForHoNode(node) + if err != nil { + klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err) + } + } else { + continue + } + } else { + hostNetworkIPs, err = oc.getHostNamespaceAddressesForNode(node) + if err != nil { + klog.Errorf("Error parsing annotation for node %s: %v", node.Name, err) + } } ips = append(ips, hostNetworkIPs...) } @@ -342,6 +351,23 @@ func (oc *DefaultNetworkController) getAllHostNamespaceAddresses() []net.IP { return ips } +func (oc *DefaultNetworkController) getHostNamespaceAddressesForHoNode(node *kapi.Node) ([]net.IP, error) { + var ips []net.IP + // during SDN live migration, add the SDN node GW IP to the host network address set. + hoSubnet, ok := node.Annotations[hotypes.HybridOverlayNodeSubnet] + if !ok { + // skip hybrid overlay nodes without per-node subnet + return nil, nil + } + _, subnet, err := net.ParseCIDR(hoSubnet) + if err != nil { + klog.Errorf("Error parsing hybrid overlay subnet %s for node %s: %v", hoSubnet, node.Name, err) + } + gwIP := util.GetNodeGatewayIfAddr(subnet) + ips = append(ips, gwIP.IP) + return ips, nil +} + // getHostNamespaceAddressesForNode retrives management port and gateway router LRP // IP of a specific node func (oc *DefaultNetworkController) getHostNamespaceAddressesForNode(node *kapi.Node) ([]net.IP, error) { diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index cb768bbe5b..256f687cab 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -348,6 +348,121 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { fakeOvn.asf.EventuallyExpectAddressSetWithIPs(hostNetworkNamespace, allowIPs) }) + ginkgo.It("creates an address set for hybrid overlay nodes when the host network traffic namespace is created", func() { + config.HybridOverlay.Enabled = true + config.Kubernetes.NoHostSubnetNodes, _ = metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: map[string]string{"hybrid-overlay-node": "true"}, + }) + config.Gateway.Mode = config.GatewayModeShared + config.Gateway.NodeportEnable = true + var err error + config.Default.ClusterSubnets, err = config.ParseClusterSubnetEntries(clusterCIDR) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1 := tNode{ + Name: "node1", + NodeIP: "1.2.3.4", + NodeSubnet: "10.1.1.0/24", + NodeGWIP: "10.1.1.1/24", + } + // create a test node and annotate it with host subnet + testNode := v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1.Name, + Labels: map[string]string{"hybrid-overlay-node": "true"}, + Annotations: map[string]string{ + "k8s.ovn.org/hybrid-overlay-node-subnet": node1.NodeSubnet, + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", node1.NodeSubnet), + }, + }, + } + + hostNetworkNamespace := "test-host-network-ns" + config.Kubernetes.HostNetworkNamespace = hostNetworkNamespace + + expectedClusterLBGroup := newLoadBalancerGroup(ovntypes.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(ovntypes.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(ovntypes.ClusterRouterLBGroupName) + expectedOVNClusterRouter := newOVNClusterRouter() + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) + expectedClusterRouterPortGroup := newRouterPortGroup() + expectedClusterPortGroup := newClusterPortGroup() + + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + newClusterJoinSwitch(), + expectedOVNClusterRouter, + expectedNodeSwitch, + expectedClusterRouterPortGroup, + expectedClusterPortGroup, + expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, + }, + }, + &v1.NamespaceList{ + Items: []v1.Namespace{ + *newNamespace(hostNetworkNamespace), + }, + }, + &v1.NodeList{ + Items: []v1.Node{ + testNode, + }, + }, + ) + fakeOvn.controller.multicastSupport = false + fakeOvn.controller.SCTPSupport = true + fakeOvn.controller.inMigrationMode = true + + err = fakeOvn.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = fakeOvn.controller.WatchNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = fakeOvn.controller.StartServiceController(wg, false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // check the namespace again and ensure the address set + // being created with the right set of IPs in it. + ip, _, _ := net.ParseCIDR(node1.NodeGWIP) + allowIPs := []string{ip.String()} + fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs) + + // switch the node from a ho node to a ovn node + ovn_node1 := tNode{ + Name: "node1", + NodeIP: "1.2.3.4", + NodeLRPMAC: "0a:58:0a:01:01:01", + LrpIP: "100.64.0.2", + LrpIPv6: "fd98::2", + DrLrpIP: "100.64.0.1", + PhysicalBridgeMAC: "11:22:33:44:55:66", + SystemID: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", + NodeSubnet: "10.1.1.0/24", + GWRouter: ovntypes.GWRouterPrefix + "node1", + GatewayRouterIPMask: "172.16.16.2/24", + GatewayRouterIP: "172.16.16.2", + GatewayRouterNextHop: "172.16.16.1", + PhysicalBridgeName: "br-eth0", + NodeGWIP: "10.1.1.1/24", + NodeMgmtPortIP: "10.1.1.2", + NodeMgmtPortMAC: "0a:58:0a:01:01:02", + DnatSnatIP: "169.254.0.1", + } + ovnTestNode := ovn_node1.k8sNode("2") + ovnTestNode.Annotations["k8s.ovn.org/hybrid-overlay-node-subnet"] = testNode.Annotations["k8s.ovn.org/hybrid-overlay-node-subnet"] + ovnTestNode.Annotations["k8s.ovn.org/node-subnets"] = testNode.Annotations["k8s.ovn.org/node-subnets"] + + fakeOvn.fakeClient.GetNodeClientset().KubeClient.CoreV1().Nodes().Update(context.TODO(), &ovnTestNode, metav1.UpdateOptions{}) + // check the namespace again and ensure the ho node gateway IP is removed from the address_set + // and the ovn ips are added instead. + allowIPs = []string{"10.1.1.2", "100.64.0.2"} + fakeOvn.asf.EventuallyExpectAddressSetWithAddresses(hostNetworkNamespace, allowIPs) + }) + ginkgo.It("reconciles an existing namespace port group, without updating it", func() { // this flag will create namespaced port group config.OVNKubernetesFeature.EnableEgressFirewall = true