Skip to content

Commit

Permalink
Merge pull request openshift#1303 from dcbw/ho-host-network
Browse files Browse the repository at this point in the history 
hybrid-overlay: enable host to access hybrid overlay nodes
  • Loading branch information
@dcbw
dcbw authored May 8, 2020
2 parents 2d69e70 + 7e074da commit d306b78
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 51 deletions.
6 changes: 4 additions & 2 deletions go-controller/hack/test-go.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ function testrun {
go test -mod vendor ${args}
}

# These packages requires root for network namespace maniuplation in unit tests
root_pkgs=("github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node" "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/controller")

i=0
for pkg in ${PKGS}; do
# This package requires root
if [[ "$USER" != root && "$pkg" == github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node ]]; then
if [[ "$USER" != root && " ${root_pkgs[@]} " =~ " $pkg " ]]; then
testfile=$(mktemp --tmpdir ovn-test.XXXXXXXX)
echo "sudo required for ${pkg}, compiling test to ${testfile}"
testrun "${i}" "${pkg}" -c -o "${testfile}"
Expand Down
64 changes: 53 additions & 11 deletions go-controller/hybrid-overlay/pkg/controller/node_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,20 @@ import (
"crypto/sha256"
"fmt"
"net"
"os"
"reflect"
"strings"
"time"

hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"

"github.com/vishvananda/netlink"

kapi "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
Expand Down Expand Up @@ -82,7 +86,7 @@ func (n *NodeController) deletePod(pod *kapi.Pod) error {

func getPodDetails(pod *kapi.Pod, nodeName string) ([]*net.IPNet, net.HardwareAddr, error) {
if pod.Spec.NodeName != nodeName {
return nil, nil, fmt.Errorf("not scheduled")
return nil, nil, fmt.Errorf("not scheduled on this node")
}

podInfo, err := util.UnmarshalPodAnnotation(pod.Annotations)
Expand Down Expand Up @@ -135,16 +139,16 @@ func (n *NodeController) syncPods(pods []interface{}) {
continue
}

parts := strings.Split(line, ",")
for _, part := range parts {
part = strings.TrimSpace(part)
fields := strings.Split(line, ",")
for _, field := range fields {
field = strings.TrimSpace(field)
// Ignore non-cookie fields and any flows with the special zero-cookie
const cookieTag string = "cookie=0x"
if !strings.HasPrefix(part, cookieTag) {
continue
}
cookie := part[len(cookieTag):]
if _, ok := kubePods[cookie]; !ok {
cookiesToRemove[cookie] = true
if strings.HasPrefix(field, cookieTag) && field != "cookie=0x0" {
cookie := field[len(cookieTag):]
if _, ok := kubePods[cookie]; !ok {
cookiesToRemove[cookie] = true
}
}
}
}
Expand Down Expand Up @@ -403,6 +407,11 @@ func (n *NodeController) ensureHybridOverlayBridge() error {
return nil
}

mgmtPortMAC, err := util.GetOVSPortMACAddress(util.K8sMgmtIntfName)
if err != nil {
return fmt.Errorf("failed to read management port MAC address: %v", mgmtPortMAC)
}

_, stderr, err := util.RunOVSVsctl("--may-exist", "add-br", extBridgeName,
"--", "set", "Bridge", extBridgeName, "fail_mode=secure")
if err != nil {
Expand All @@ -422,7 +431,7 @@ func (n *NodeController) ensureHybridOverlayBridge() error {
"error: %v", stdout, stderr, err)
}

if _, _, err = util.RunIP("link", "set", extBridgeName, "up"); err != nil {
if _, err := util.LinkSetUp(extBridgeName); err != nil {
return fmt.Errorf("failed to up %s: %v", extBridgeName, err)
}

Expand Down Expand Up @@ -490,6 +499,39 @@ func (n *NodeController) ensureHybridOverlayBridge() error {
"stderr: %q, error: %v", stderr, err)
}

if len(config.HybridOverlay.ClusterSubnets) > 0 {
// Add a route via the hybrid overlay port IP through the management port
// interface for each hybrid overlay cluster subnet
mgmtPortLink, err := netlink.LinkByName(util.K8sMgmtIntfName)
if err != nil {
return fmt.Errorf("failed to lookup link %s: %v", util.K8sMgmtIntfName, err)
}
hybridOverlayIfAddr := util.GetNodeHybridOverlayIfAddr(subnet)
for _, clusterEntry := range config.HybridOverlay.ClusterSubnets {
route := &netlink.Route{
Dst: clusterEntry.CIDR,
LinkIndex: mgmtPortLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Gw: hybridOverlayIfAddr.IP,
}
err := netlink.RouteAdd(route)
if err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to add route for subnet %s via gateway %s: %v",
route.Dst, route.Gw, err)
}
}

// Add a rule to fix up return host-network traffic
mgmtIfAddr := util.GetNodeManagementIfAddr(subnet)
_, stderr, err = util.RunOVSOfctl("add-flow", extBridgeName,
fmt.Sprintf("table=10,priority=100,ip,nw_dst=%s actions=mod_dl_src:%s,mod_dl_dst:%s,output:ext",
mgmtIfAddr.IP.String(), portMAC.String(), mgmtPortMAC.String()))
if err != nil {
return fmt.Errorf("failed to set up hybrid overlay bridge host dispatch reply rule,"+
"stderr: %q, error: %v", stderr, err)
}
}

n.drMAC = portMAC.String()

n.initialized = true
Expand Down
131 changes: 93 additions & 38 deletions go-controller/hybrid-overlay/pkg/controller/node_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,17 @@ import (
ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"

"github.com/containernetworking/plugins/pkg/ns"
"github.com/containernetworking/plugins/pkg/testutils"
"github.com/vishvananda/netlink"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

const (
testOVSMAC string = "11:22:33:44:55:66"
testMgmtMAC string = "06:05:04:03:02:01"
testDRMAC string = "00:00:00:7a:af:04"
testNodeSubnet string = "1.2.3.0/24"
testNodeIP string = "1.2.3.3"
Expand All @@ -36,6 +41,10 @@ func addNodeSetupCmds(fexec *ovntest.FakeExec, nodeName string) (string, string)
Output: testNodeSubnet,
})
addGetPortAddressesCmds(fexec, nodeName, testDRMAC, testNodeIP)
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
Cmd: "ovs-vsctl --timeout=15 --if-exists get interface ovn-k8s-mp0 mac_in_use",
Output: testMgmtMAC,
})
fexec.AddFakeCmdsNoOutputNoError([]string{
"ovs-vsctl --timeout=15 --may-exist add-br br-ext -- set Bridge br-ext fail_mode=secure",
})
Expand All @@ -45,7 +54,6 @@ func addNodeSetupCmds(fexec *ovntest.FakeExec, nodeName string) (string, string)
})
fexec.AddFakeCmdsNoOutputNoError([]string{
"ovs-vsctl --timeout=15 set bridge br-ext other-config:hwaddr=" + testOVSMAC,
"ip link set br-ext up",
"ovs-vsctl --timeout=15 --may-exist add-port br-int int -- --may-exist add-port br-ext ext -- set Interface int type=patch options:peer=ext external-ids:iface-id=int-" + nodeName + " -- set Interface ext type=patch options:peer=int",
"ovs-ofctl add-flow br-ext table=0,priority=0,actions=drop",
})
Expand Down Expand Up @@ -100,10 +108,69 @@ func createPod(namespace, name, node, podIP, podMAC string) *v1.Pod {
}
}

func addLink(name string) {
err := netlink.LinkAdd(&netlink.Dummy{
LinkAttrs: netlink.LinkAttrs{
Name: name,
},
})
Expect(err).NotTo(HaveOccurred())
origLink, err := netlink.LinkByName(name)
Expect(err).NotTo(HaveOccurred())
err = netlink.LinkSetUp(origLink)
Expect(err).NotTo(HaveOccurred())
}

func expectRouteForSubnet(routes []netlink.Route, subnet *net.IPNet, hoIfAddr net.IP) {
found := false
for _, route := range routes {
if route.Dst.String() == subnet.String() && route.Gw.String() == hoIfAddr.String() {
found = true
break
}
}
Expect(found).To(BeTrue())
}

func validateNetlinkState(nodeSubnet string) {
link, err := netlink.LinkByName(extBridgeName)
Expect(err).NotTo(HaveOccurred())
Expect(link.Attrs().Flags & net.FlagUp).To(Equal(net.FlagUp))

link, err = netlink.LinkByName(util.K8sMgmtIntfName)
Expect(err).NotTo(HaveOccurred())
Expect(link.Attrs().Flags & net.FlagUp).To(Equal(net.FlagUp))

routes, err := netlink.RouteList(link, netlink.FAMILY_ALL)
Expect(err).NotTo(HaveOccurred())

// Expect a route to the hybrid overlay CIDR via the .3 address
// through the management port
_, ipnet, err := net.ParseCIDR(nodeSubnet)
Expect(err).NotTo(HaveOccurred())
hybridOverlayIfAddr := util.GetNodeHybridOverlayIfAddr(ipnet)
for _, hoSubnet := range config.HybridOverlay.ClusterSubnets {
expectRouteForSubnet(routes, hoSubnet.CIDR, hybridOverlayIfAddr.IP)
}
}

func appRun(app *cli.App, netns ns.NetNS) {
_ = netns.Do(func(ns.NetNS) error {
defer GinkgoRecover()
err := app.Run([]string{
app.Name,
hoNodeCliArg,
})
Expect(err).NotTo(HaveOccurred())
return nil
})
}

var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
var (
app *cli.App
fexec *ovntest.FakeExec
netns ns.NetNS
)
const thisNode string = "mynode"

Expand All @@ -118,6 +185,22 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
fexec = ovntest.NewFakeExec()
err := util.SetExec(fexec)
Expect(err).NotTo(HaveOccurred())

netns, err = testutils.NewNS()
Expect(err).NotTo(HaveOccurred())

// prepare br-ext and ovn-k8s-mp0 in original namespace
_ = netns.Do(func(ns.NetNS) error {
defer GinkgoRecover()
addLink(extBridgeName)
addLink(util.K8sMgmtIntfName)
return nil
})
})

AfterEach(func() {
Expect(netns.Close()).To(Succeed())
Expect(testutils.UnmountNS(netns)).To(Succeed())
})

It("does not set up tunnels for non-hybrid-overlay nodes without annotations", func() {
Expand Down Expand Up @@ -156,12 +239,7 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
return nil
}

err := app.Run([]string{
app.Name,
hoNodeCliArg,
})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})

It("does not set up tunnels for non-hybrid-overlay nodes with subnet annotations", func() {
Expand Down Expand Up @@ -207,14 +285,10 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
Expect(err).NotTo(HaveOccurred())

Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
validateNetlinkState(node1Subnet)
return nil
}

err := app.Run([]string{
app.Name,
hoNodeCliArg,
})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})

It("sets up local node hybrid overlay bridge", func() {
Expand Down Expand Up @@ -254,12 +328,7 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)
return nil
}

err := app.Run([]string{
app.Name,
hoNodeCliArg,
})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})

It("sets up tunnels for Windows nodes", func() {
Expand Down Expand Up @@ -306,15 +375,10 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {
Expect(err).NotTo(HaveOccurred())

Expect(fexec.CalledMatchesExpected()).To(BeTrue(), fexec.ErrorDesc)

validateNetlinkState(node1Subnet)
return nil
}

err := app.Run([]string{
app.Name,
hoNodeCliArg,
})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})

It("removes stale node flows on initial sync", func() {
Expand Down Expand Up @@ -361,12 +425,7 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {

return nil
}

err := app.Run([]string{
app.Name,
hoNodeCliArg,
})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})

It("removes stale pod flows on initial sync", func() {
Expand Down Expand Up @@ -408,9 +467,7 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {

return nil
}

err := app.Run([]string{app.Name})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})

It("sets up local pod flows", func() {
Expand Down Expand Up @@ -458,8 +515,6 @@ var _ = Describe("Hybrid Overlay Node Linux Operations", func() {

return nil
}

err := app.Run([]string{app.Name})
Expect(err).NotTo(HaveOccurred())
appRun(app, netns)
})
})

0 comments on commit d306b78

Please sign in to comment.