From 8611daa99ab4f58f1bc49b8a11508f23ab7628ae Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Tue, 27 Apr 2021 10:30:38 +0200 Subject: [PATCH 1/7] create vlan subinterface on pod container This commit attempts to create vlan sub interfaces on the pod container for the given selective vlan trunks. It also configures IP based on the IPAM config given for each VLAN ID. TODO: figure out what else needed by ipam module apart from just overwriting netconf with trunk's ipam data. does it also need more changes into netconf and cni_args ? Signed-off-by: Periyasamy Palanisamy --- pkg/plugin/plugin.go | 195 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index aae11ec19..062c541a8 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -55,6 +55,7 @@ const ( type netConf struct { types.NetConf BrName string `json:"bridge,omitempty"` + IPAM *IPAM `json:"ipam,omitempty"` VlanTag *uint `json:"vlan"` MTU int `json:"mtu"` Trunk []*trunk `json:"trunk,omitempty"` @@ -67,6 +68,23 @@ type trunk struct { MinID *uint `json:"minID,omitempty"` MaxID *uint `json:"maxID,omitempty"` ID *uint `json:"id,omitempty"` + IPAM *IPAM `json:"ipam,omitempty"` +} + +type IPAM struct { + *Range + Type string `json:"type,omitempty"` + Routes []*types.Route `json:"routes,omitempty"` + Ranges []RangeSet `json:"ranges,omitempty"` +} + +type RangeSet []Range + +type Range struct { + RangeStart net.IP `json:"rangeStart,omitempty"` // The first ip, inclusive + RangeEnd net.IP `json:"rangeEnd,omitempty"` // The last ip, inclusive + Subnet types.IPNet `json:"subnet"` + Gateway net.IP `json:"gateway,omitempty"` } // EnvArgs args containing common, desired mac and ovs port name @@ -118,6 +136,17 @@ func loadNetConf(bytes []byte) (*netConf, error) { return netconf, nil } +func marshalNetConf(netconf *netConf) ([]byte, error) { + var ( + data []byte + err error + ) + if data, err = json.Marshal(*netconf); err != nil { + return nil, fmt.Errorf("failed to marshal netconf: %v", err) + } + return data, nil +} + func loadFlatNetConf(configPath string) (*netConf, error) { confFiles := getOvsConfFiles() if configPath != "" { @@ -183,6 +212,92 @@ func generateRandomMac() net.HardwareAddr { return net.HardwareAddr(append(prefix, suffix...)) } +func setupTrunkIfaces(netconf *netConf, contNetns ns.NetNS, contIfaceName string) ([]*current.Result, error) { + results := make([]*current.Result, 0) + origIPAMConfig := netconf.IPAM + err := contNetns.Do(func(hostNetns ns.NetNS) error { + for _, trunk := range netconf.Trunk { + if trunk.IPAM.Type == "" { + continue + } + subIfName := contIfaceName + "." + fmt.Sprint(*trunk.ID) + if err := createVlanLink(contIfaceName, subIfName, *trunk.ID); err != nil { + return err + } + var macAddress net.HardwareAddr + containerSubIfLink, err := netlink.LinkByName(subIfName) + // In case the MAC address is already assigned to another interface, retry + for i := 1; i <= macSetupRetries; i++ { + macAddress = generateRandomMac() + err = assignMacToLink(containerSubIfLink, macAddress, subIfName) + if err != nil && i == macSetupRetries { + return err + } + } + netconf.IPAM = trunk.IPAM + netData, err := marshalNetConf(netconf) + if err != nil { + return err + } + var result *current.Result + r, err := ipam.ExecAdd(trunk.IPAM.Type, netData) + if err != nil { + // Invoke ipam del if err to avoid ip leak + ipam.ExecDel(trunk.IPAM.Type, netData) + return fmt.Errorf("failed to set up IPAM plugin type %q for vlan %d: %v", trunk.IPAM.Type, *trunk.ID, err) + } + // Convert whatever the IPAM result was into the current Result type + result, err = current.NewResultFromResult(r) + if err != nil { + return err + } + if len(result.IPs) == 0 { + return fmt.Errorf("IPAM plugin %q returned missing IP config for vlan %d: %v", trunk.IPAM.Type, *trunk.ID, err) + } + result.Interfaces = []*current.Interface{{ + Name: subIfName, + Mac: macAddress.String(), + Sandbox: contNetns.Path(), + }} + for _, ipc := range result.IPs { + // All addresses apply to the container interface (move from host) + ipc.Interface = current.Int(0) + } + + if err := ipam.ConfigureIface(subIfName, result); err != nil { + return err + } + results = append(results, result) + } + return nil + }) + netconf.IPAM = origIPAMConfig + if err != nil { + return nil, err + } + return results, nil +} + +func cleanupTrunkIfaces(netconf *netConf) error { + origIPAMConfig := netconf.IPAM + for _, trunk := range netconf.Trunk { + if trunk.IPAM.Type == "" { + continue + } + netconf.IPAM = trunk.IPAM + netData, err := marshalNetConf(netconf) + if err != nil { + return err + } + err = ipam.ExecDel(trunk.IPAM.Type, netData) + if err != nil { + return err + } + } + netconf.IPAM = origIPAMConfig + return nil +} + func setupVeth(contNetns ns.NetNS, contIfaceName string, requestedMac string, mtu int) (*current.Interface, *current.Interface, error) { hostIface := ¤t.Interface{} contIface := ¤t.Interface{} @@ -248,6 +363,67 @@ func assignMacToLink(link netlink.Link, mac net.HardwareAddr, name string) error return nil } +func createVlanLink(parentIfName string, subIfName string, vlanID uint) error { + if vlanID > 4094 || vlanID < 1 { + return fmt.Errorf("vlan id must be between 1-4094, received: %d", vlanID) + } + if interfaceExists(subIfName) { + return nil + } + // get the parent link to attach a vlan subinterface + parentLink, err := netlink.LinkByName(parentIfName) + if err != nil { + return fmt.Errorf("failed to find master interface %s on the host: %v", parentIfName, err) + } + vlanLink := &netlink.Vlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: subIfName, + ParentIndex: parentLink.Attrs().Index, + }, + VlanId: int(vlanID), + } + // create the subinterface + if err := netlink.LinkAdd(vlanLink); err != nil { + return fmt.Errorf("failed to create %s vlan link: %v", vlanLink.Name, err) + } + // Bring the new netlink iface up + if err := netlink.LinkSetUp(vlanLink); err != nil { + return fmt.Errorf("failed to enable %s the macvlan parent link %v", vlanLink.Name, err) + } + return nil +} + +func deleteVlanLink(parentIfName string, subIfName string, vlanID uint) error { + if !interfaceExists(subIfName) { + return nil + } + // get the parent link to attach a vlan subinterface + parentLink, err := netlink.LinkByName(parentIfName) + if err != nil { + return nil + } + vlanLink := &netlink.Vlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: subIfName, + ParentIndex: parentLink.Attrs().Index, + }, + VlanId: int(vlanID), + } + // delete the subinterface + if err := netlink.LinkDel(vlanLink); err != nil { + return fmt.Errorf("failed to delete %s vlan link: %v", vlanLink.Name, err) + } + return nil +} + +func interfaceExists(ifName string) bool { + _, err := netlink.LinkByName(ifName) + if err != nil { + return false + } + return true +} + func getBridgeName(bridgeName, ovnPort string) (string, error) { if bridgeName != "" { return bridgeName, nil @@ -485,6 +661,19 @@ func CmdAdd(args *skel.CmdArgs) error { } } + if len(netconf.Trunk) > 0 { + subIfResults, err := setupTrunkIfaces(netconf, contNetns, args.IfName) + if err != nil { + return err + } + ifLength := len(result.Interfaces) + for idx, subIfresult := range subIfResults { + result.Interfaces = append(result.Interfaces, subIfresult.Interfaces[0]) + subIfresult.IPs[0].Interface = current.Int(ifLength + idx) + result.IPs = append(result.IPs, subIfresult.IPs[0]) + } + } + return types.PrintResult(result, netconf.CNIVersion) } @@ -576,6 +765,12 @@ func CmdDel(args *skel.CmdArgs) error { return err } } + if len(netconf.Trunk) > 0 { + err = cleanupTrunkIfaces(netconf) + if err != nil { + return err + } + } if args.Netns == "" { // The CNI_NETNS parameter may be empty according to version 0.4.0 From 6c68b7d1bc9c776c82b0f6bca7cb350166d51bc8 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Tue, 27 Apr 2021 16:50:50 +0200 Subject: [PATCH 2/7] add nad example for vlan with ipam configuration Signed-off-by: Periyasamy Palanisamy --- examples/ovs-net-vlan-ipam.yml | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 examples/ovs-net-vlan-ipam.yml diff --git a/examples/ovs-net-vlan-ipam.yml b/examples/ovs-net-vlan-ipam.yml new file mode 100644 index 000000000..4dc8c2a4d --- /dev/null +++ b/examples/ovs-net-vlan-ipam.yml @@ -0,0 +1,46 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: ovs-cni-net +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "ovs", + "bridge": "br0", + "ipam": { + "type": "host-local", + "subnet": "192.168.0.0/24", + "rangeStart": "192.168.0.200", + "rangeEnd": "192.168.0.216", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.0.1" + }, + "trunk": [ + { "id" : 42, + "ipam": { + "type": "host-local", + "subnet": "192.168.42.0/24", + "rangeStart": "192.168.42.200", + "rangeEnd": "192.168.42.216", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.42.1" + } + }, + { "id" : 50, + "ipam": { + "type": "host-local", + "subnet": "192.168.50.0/24", + "rangeStart": "192.168.50.200", + "rangeEnd": "192.168.50.216", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.50.1" + } + } + ] +}' From ccf4430dda797ba894105844d23751a97dfe92f2 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Tue, 27 Apr 2021 17:07:40 +0200 Subject: [PATCH 3/7] make ipam struct as private Signed-off-by: Periyasamy Palanisamy --- pkg/plugin/plugin.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 062c541a8..131f498f0 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -54,33 +54,33 @@ const ( type netConf struct { types.NetConf - BrName string `json:"bridge,omitempty"` - IPAM *IPAM `json:"ipam,omitempty"` - VlanTag *uint `json:"vlan"` - MTU int `json:"mtu"` - Trunk []*trunk `json:"trunk,omitempty"` - DeviceID string `json:"deviceID"` // PCI address of a VF in valid sysfs format - ConfigurationPath string `json:"configuration_path"` - SocketFile string `json:"socket_file"` + BrName string `json:"bridge,omitempty"` + IPAM *ipamConfig `json:"ipam,omitempty"` + VlanTag *uint `json:"vlan"` + MTU int `json:"mtu"` + Trunk []*trunk `json:"trunk,omitempty"` + DeviceID string `json:"deviceID"` // PCI address of a VF in valid sysfs format + ConfigurationPath string `json:"configuration_path"` + SocketFile string `json:"socket_file"` } type trunk struct { - MinID *uint `json:"minID,omitempty"` - MaxID *uint `json:"maxID,omitempty"` - ID *uint `json:"id,omitempty"` - IPAM *IPAM `json:"ipam,omitempty"` + MinID *uint `json:"minID,omitempty"` + MaxID *uint `json:"maxID,omitempty"` + ID *uint `json:"id,omitempty"` + IPAM *ipamConfig `json:"ipam,omitempty"` } -type IPAM struct { - *Range +type ipamConfig struct { + *ipRange Type string `json:"type,omitempty"` Routes []*types.Route `json:"routes,omitempty"` - Ranges []RangeSet `json:"ranges,omitempty"` + Ranges []rangeSet `json:"ranges,omitempty"` } -type RangeSet []Range +type rangeSet []ipRange -type Range struct { +type ipRange struct { RangeStart net.IP `json:"rangeStart,omitempty"` // The first ip, inclusive RangeEnd net.IP `json:"rangeEnd,omitempty"` // The last ip, inclusive Subnet types.IPNet `json:"subnet"` From 1f0fda31195a94152a58a5bb39bc9bd07a9fd6c2 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Wed, 28 Apr 2021 11:27:12 +0200 Subject: [PATCH 4/7] Revert "make ipam struct as private" This reverts commit ccf4430dda797ba894105844d23751a97dfe92f2. --- pkg/plugin/plugin.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 131f498f0..062c541a8 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -54,33 +54,33 @@ const ( type netConf struct { types.NetConf - BrName string `json:"bridge,omitempty"` - IPAM *ipamConfig `json:"ipam,omitempty"` - VlanTag *uint `json:"vlan"` - MTU int `json:"mtu"` - Trunk []*trunk `json:"trunk,omitempty"` - DeviceID string `json:"deviceID"` // PCI address of a VF in valid sysfs format - ConfigurationPath string `json:"configuration_path"` - SocketFile string `json:"socket_file"` + BrName string `json:"bridge,omitempty"` + IPAM *IPAM `json:"ipam,omitempty"` + VlanTag *uint `json:"vlan"` + MTU int `json:"mtu"` + Trunk []*trunk `json:"trunk,omitempty"` + DeviceID string `json:"deviceID"` // PCI address of a VF in valid sysfs format + ConfigurationPath string `json:"configuration_path"` + SocketFile string `json:"socket_file"` } type trunk struct { - MinID *uint `json:"minID,omitempty"` - MaxID *uint `json:"maxID,omitempty"` - ID *uint `json:"id,omitempty"` - IPAM *ipamConfig `json:"ipam,omitempty"` + MinID *uint `json:"minID,omitempty"` + MaxID *uint `json:"maxID,omitempty"` + ID *uint `json:"id,omitempty"` + IPAM *IPAM `json:"ipam,omitempty"` } -type ipamConfig struct { - *ipRange +type IPAM struct { + *Range Type string `json:"type,omitempty"` Routes []*types.Route `json:"routes,omitempty"` - Ranges []rangeSet `json:"ranges,omitempty"` + Ranges []RangeSet `json:"ranges,omitempty"` } -type rangeSet []ipRange +type RangeSet []Range -type ipRange struct { +type Range struct { RangeStart net.IP `json:"rangeStart,omitempty"` // The first ip, inclusive RangeEnd net.IP `json:"rangeEnd,omitempty"` // The last ip, inclusive Subnet types.IPNet `json:"subnet"` From a4bcb58a299fe8928f48c5c02618af7278281c6b Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Wed, 28 Apr 2021 11:29:52 +0200 Subject: [PATCH 5/7] fix golint error Signed-off-by: Periyasamy Palanisamy --- pkg/plugin/plugin.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 062c541a8..30499ca95 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -71,6 +71,7 @@ type trunk struct { IPAM *IPAM `json:"ipam,omitempty"` } +// IPAM ipam configuration type IPAM struct { *Range Type string `json:"type,omitempty"` @@ -78,8 +79,10 @@ type IPAM struct { Ranges []RangeSet `json:"ranges,omitempty"` } +// RangeSet ip range set type RangeSet []Range +// Range ip range configuration type Range struct { RangeStart net.IP `json:"rangeStart,omitempty"` // The first ip, inclusive RangeEnd net.IP `json:"rangeEnd,omitempty"` // The last ip, inclusive From 718b3e5af7efbd04509ee94432d23c9c79b55018 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Thu, 29 Apr 2021 10:41:39 +0200 Subject: [PATCH 6/7] fix unit test errors Signed-off-by: Periyasamy Palanisamy --- pkg/plugin/plugin.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 30499ca95..1d17e9ce0 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -220,7 +220,7 @@ func setupTrunkIfaces(netconf *netConf, contNetns ns.NetNS, contIfaceName string origIPAMConfig := netconf.IPAM err := contNetns.Do(func(hostNetns ns.NetNS) error { for _, trunk := range netconf.Trunk { - if trunk.IPAM.Type == "" { + if trunk.IPAM == nil || trunk.IPAM.Type == "" { continue } subIfName := contIfaceName + "." + fmt.Sprint(*trunk.ID) @@ -284,7 +284,7 @@ func setupTrunkIfaces(netconf *netConf, contNetns ns.NetNS, contIfaceName string func cleanupTrunkIfaces(netconf *netConf) error { origIPAMConfig := netconf.IPAM for _, trunk := range netconf.Trunk { - if trunk.IPAM.Type == "" { + if trunk.IPAM == nil || trunk.IPAM.Type == "" { continue } netconf.IPAM = trunk.IPAM @@ -590,7 +590,7 @@ func CmdAdd(args *skel.CmdArgs) error { } // run the IPAM plugin - if netconf.IPAM.Type != "" { + if netconf.IPAM != nil && netconf.IPAM.Type != "" { r, err := ipam.ExecAdd(netconf.IPAM.Type, args.StdinData) if err != nil { return fmt.Errorf("failed to set up IPAM plugin type %q: %v", netconf.IPAM.Type, err) @@ -762,7 +762,7 @@ func CmdDel(args *skel.CmdArgs) error { return err } - if netconf.IPAM.Type != "" { + if netconf.IPAM != nil && netconf.IPAM.Type != "" { err = ipam.ExecDel(netconf.IPAM.Type, args.StdinData) if err != nil { return err From 37d145fecbf2b1c86dd6f8fa2750c22ced188f5a Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Thu, 6 May 2021 13:07:54 +0200 Subject: [PATCH 7/7] correction in vlan sub interface creation and ip assignment Signed-off-by: Periyasamy Palanisamy --- examples/ovs-net-vlan-ipam.yml | 6 ------ pkg/plugin/plugin.go | 32 +++++++++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/ovs-net-vlan-ipam.yml b/examples/ovs-net-vlan-ipam.yml index 4dc8c2a4d..ef628aa4c 100644 --- a/examples/ovs-net-vlan-ipam.yml +++ b/examples/ovs-net-vlan-ipam.yml @@ -24,9 +24,6 @@ spec: "subnet": "192.168.42.0/24", "rangeStart": "192.168.42.200", "rangeEnd": "192.168.42.216", - "routes": [ - { "dst": "0.0.0.0/0" } - ], "gateway": "192.168.42.1" } }, @@ -36,9 +33,6 @@ spec: "subnet": "192.168.50.0/24", "rangeStart": "192.168.50.200", "rangeEnd": "192.168.50.216", - "routes": [ - { "dst": "0.0.0.0/0" } - ], "gateway": "192.168.50.1" } } diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 1d17e9ce0..6740fe26c 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -215,10 +215,14 @@ func generateRandomMac() net.HardwareAddr { return net.HardwareAddr(append(prefix, suffix...)) } -func setupTrunkIfaces(netconf *netConf, contNetns ns.NetNS, contIfaceName string) ([]*current.Result, error) { +func setupTrunkIfaces(netconf *netConf, contNetns ns.NetNS, contIfaceName, contIfaceMac string) ([]*current.Result, error) { results := make([]*current.Result, 0) origIPAMConfig := netconf.IPAM - err := contNetns.Do(func(hostNetns ns.NetNS) error { + macAddress, err := net.ParseMAC(contIfaceMac) + if err != nil { + return nil, fmt.Errorf("failed to parse requested MAC %q: %v", contIfaceMac, err) + } + err = contNetns.Do(func(hostNetns ns.NetNS) error { for _, trunk := range netconf.Trunk { if trunk.IPAM == nil || trunk.IPAM.Type == "" { continue @@ -227,15 +231,13 @@ func setupTrunkIfaces(netconf *netConf, contNetns ns.NetNS, contIfaceName string if err := createVlanLink(contIfaceName, subIfName, *trunk.ID); err != nil { return err } - var macAddress net.HardwareAddr containerSubIfLink, err := netlink.LinkByName(subIfName) - // In case the MAC address is already assigned to another interface, retry - for i := 1; i <= macSetupRetries; i++ { - macAddress = generateRandomMac() - err = assignMacToLink(containerSubIfLink, macAddress, subIfName) - if err != nil && i == macSetupRetries { - return err - } + if err != nil { + return err + } + err = assignMacToLink(containerSubIfLink, macAddress, subIfName) + if err != nil { + return err } netconf.IPAM = trunk.IPAM netData, err := marshalNetConf(netconf) @@ -665,15 +667,19 @@ func CmdAdd(args *skel.CmdArgs) error { } if len(netconf.Trunk) > 0 { - subIfResults, err := setupTrunkIfaces(netconf, contNetns, args.IfName) + subIfResults, err := setupTrunkIfaces(netconf, contNetns, args.IfName, contIface.Mac) if err != nil { return err } ifLength := len(result.Interfaces) for idx, subIfresult := range subIfResults { + // subIfresult has only one interface result.Interfaces = append(result.Interfaces, subIfresult.Interfaces[0]) - subIfresult.IPs[0].Interface = current.Int(ifLength + idx) - result.IPs = append(result.IPs, subIfresult.IPs[0]) + ifIdxPtr := current.Int(ifLength + idx) + for ipIndex := range subIfresult.IPs { + subIfresult.IPs[ipIndex].Interface = ifIdxPtr + result.IPs = append(result.IPs, subIfresult.IPs[ipIndex]) + } } }