From 7065af324134a85c588f53a7cc6cf2f6c5a167ef Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 4 Oct 2023 18:09:08 +0200 Subject: [PATCH 01/19] ovn-tester/ovn_workload: Extract generic methods from WorkerNode class. Signed-off-by: Frode Nordahl --- .../cms/ovn_kubernetes/ovn_kubernetes.py | 206 +++++++++++++++++- ovn-tester/ovn_workload.py | 193 ++-------------- 2 files changed, 220 insertions(+), 179 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 7dfaeed4..f6fde14d 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -1,11 +1,213 @@ +import logging +from collections import namedtuple + +import netaddr + +from randmac import RandMac + +import ovn_load_balancer as lb +import ovn_utils +import ovn_stats + from ovn_context import Context -from ovn_workload import WorkerNode from ovn_utils import DualStackSubnet +from ovn_workload import ( + ChassisNode, + DEFAULT_BACKEND_PORT, +) - +log = logging.getLogger(__name__) +ClusterBringupCfg = namedtuple('ClusterBringupCfg', ['n_pods_per_node']) OVN_HEATER_CMS_PLUGIN = 'OVNKubernetes' +class WorkerNode(ChassisNode): + def __init__( + self, + phys_node, + container, + mgmt_ip, + protocol, + int_net, + ext_net, + gw_net, + unique_id, + ): + super().__init__(phys_node, container, mgmt_ip, protocol) + self.int_net = int_net + self.ext_net = ext_net + self.gw_net = gw_net + self.id = unique_id + + def configure(self, physical_net): + self.configure_localnet(physical_net) + phys_ctl = ovn_utils.PhysCtl(self) + phys_ctl.external_host_provision( + ip=self.ext_net.reverse(2), gw=self.ext_net.reverse() + ) + + @ovn_stats.timeit + def provision(self, cluster): + self.connect(cluster.get_relay_connection_string()) + self.wait(cluster.sbctl, cluster.cluster_cfg.node_timeout_s) + + # Create a node switch and connect it to the cluster router. + self.switch = cluster.nbctl.ls_add( + f'lswitch-{self.container}', net_s=self.int_net + ) + lrp_name = f'rtr-to-node-{self.container}' + ls_rp_name = f'node-to-rtr-{self.container}' + self.rp = cluster.nbctl.lr_port_add( + cluster.router, lrp_name, RandMac(), self.int_net.reverse() + ) + self.ls_rp = cluster.nbctl.ls_port_add( + self.switch, ls_rp_name, self.rp + ) + + # Make the lrp as distributed gateway router port. + cluster.nbctl.lr_port_set_gw_chassis(self.rp, self.container) + + # Create a gw router and connect it to the cluster join switch. + self.gw_router = cluster.nbctl.lr_add(f'gwrouter-{self.container}') + cluster.nbctl.lr_set_options( + self.gw_router, + { + 'always_learn_from_arp_request': 'false', + 'dynamic_neigh_routers': 'true', + 'chassis': self.container, + 'lb_force_snat_ip': 'router_ip', + 'snat-ct-zone': 0, + }, + ) + join_grp_name = f'gw-to-join-{self.container}' + join_ls_grp_name = f'join-to-gw-{self.container}' + + gr_gw = self.gw_net.reverse(self.id + 2) + self.gw_rp = cluster.nbctl.lr_port_add( + self.gw_router, join_grp_name, RandMac(), gr_gw + ) + self.join_gw_rp = cluster.nbctl.ls_port_add( + cluster.join_switch, join_ls_grp_name, self.gw_rp + ) + + # Create an external switch connecting the gateway router to the + # physnet. + self.ext_switch = cluster.nbctl.ls_add( + f'ext-{self.container}', net_s=self.ext_net + ) + ext_lrp_name = f'gw-to-ext-{self.container}' + ext_ls_rp_name = f'ext-to-gw-{self.container}' + self.ext_rp = cluster.nbctl.lr_port_add( + self.gw_router, ext_lrp_name, RandMac(), self.ext_net.reverse() + ) + self.ext_gw_rp = cluster.nbctl.ls_port_add( + self.ext_switch, ext_ls_rp_name, self.ext_rp + ) + + # Configure physnet. + self.physnet_port = cluster.nbctl.ls_port_add( + self.ext_switch, + f'provnet-{self.container}', + localnet=True, + ) + cluster.nbctl.ls_port_set_set_type(self.physnet_port, 'localnet') + cluster.nbctl.ls_port_set_set_options( + self.physnet_port, f'network_name={cluster.brex_cfg.physical_net}' + ) + + # Route for traffic entering the cluster. + cluster.nbctl.route_add( + self.gw_router, cluster.net, self.gw_net.reverse() + ) + + # Default route to get out of cluster via physnet. + cluster.nbctl.route_add( + self.gw_router, + ovn_utils.DualStackSubnet( + netaddr.IPNetwork("0.0.0.0/0"), netaddr.IPNetwork("::/0") + ), + self.ext_net.reverse(2), + ) + + # Route for traffic that needs to exit the cluster + # (via gw router). + cluster.nbctl.route_add( + cluster.router, self.int_net, gr_gw, policy="src-ip" + ) + + # SNAT traffic leaving the cluster. + cluster.nbctl.nat_add(self.gw_router, gr_gw, cluster.net) + + @ovn_stats.timeit + def provision_port(self, cluster, passive=False): + name = f'lp-{self.id}-{self.next_lport_index}' + + log.info(f'Creating lport {name}') + lport = cluster.nbctl.ls_port_add( + self.switch, + name, + mac=str(RandMac()), + ip=self.int_net.forward(self.next_lport_index + 1), + gw=self.int_net.reverse(), + ext_gw=self.ext_net.reverse(2), + metadata=self, + passive=passive, + security=True, + ) + + self.lports.append(lport) + self.next_lport_index += 1 + return lport + + @ovn_stats.timeit + def provision_load_balancers(self, cluster, ports, global_cfg): + # Add one port IP as a backend to the cluster load balancer. + if global_cfg.run_ipv4: + port_ips = ( + f'{port.ip}:{DEFAULT_BACKEND_PORT}' + for port in ports + if port.ip is not None + ) + cluster_vips = cluster.cluster_cfg.vips.keys() + cluster.load_balancer.add_backends_to_vip(port_ips, cluster_vips) + cluster.load_balancer.add_to_switches([self.switch.name]) + cluster.load_balancer.add_to_routers([self.gw_router.name]) + + if global_cfg.run_ipv6: + port_ips6 = ( + f'[{port.ip6}]:{DEFAULT_BACKEND_PORT}' + for port in ports + if port.ip6 is not None + ) + cluster_vips6 = cluster.cluster_cfg.vips6.keys() + cluster.load_balancer6.add_backends_to_vip( + port_ips6, cluster_vips6 + ) + cluster.load_balancer6.add_to_switches([self.switch.name]) + cluster.load_balancer6.add_to_routers([self.gw_router.name]) + + # GW Load balancer has no VIPs/backends configured on it, since + # this load balancer is used for hostnetwork services. We're not + # using those right now so the load blaancer is empty. + if global_cfg.run_ipv4: + self.gw_load_balancer = lb.OvnLoadBalancer( + f'lb-{self.gw_router.name}', cluster.nbctl + ) + self.gw_load_balancer.add_to_routers([self.gw_router.name]) + if global_cfg.run_ipv6: + self.gw_load_balancer6 = lb.OvnLoadBalancer( + f'lb-{self.gw_router.name}6', cluster.nbctl + ) + self.gw_load_balancer6.add_to_routers([self.gw_router.name]) + + @ovn_stats.timeit + def ping_external(self, cluster, port): + if port.ip: + self.run_ping(cluster, 'ext-ns', port.ip) + if port.ip6: + self.run_ping(cluster, 'ext-ns', port.ip6) + + class OVNKubernetes: @staticmethod def add_cluster_worker_nodes(cluster, workers, az): diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index aeaac719..b8f3f85d 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -153,23 +153,15 @@ def enable_trim_on_compaction(self): ) -class WorkerNode(Node): +class ChassisNode(Node): def __init__( self, phys_node, container, mgmt_ip, protocol, - int_net, - ext_net, - gw_net, - unique_id, ): super().__init__(phys_node, container, mgmt_ip, protocol) - self.int_net = int_net - self.ext_net = ext_net - self.gw_net = gw_net - self.id = unique_id self.switch = None self.gw_router = None self.ext_switch = None @@ -197,13 +189,6 @@ def configure_localnet(self, physical_net): 'ovn-bridge-mappings', f'{physical_net}:br-ex' ) - def configure(self, physical_net): - self.configure_localnet(physical_net) - phys_ctl = ovn_utils.PhysCtl(self) - phys_ctl.external_host_provision( - ip=self.ext_net.reverse(2), gw=self.ext_net.reverse() - ) - @ovn_stats.timeit def wait(self, sbctl, timeout_s): for _ in range(timeout_s * 10): @@ -212,166 +197,12 @@ def wait(self, sbctl, timeout_s): time.sleep(0.1) raise ovn_exceptions.OvnChassisTimeoutException() - @ovn_stats.timeit - def provision(self, cluster): - self.connect(cluster.get_relay_connection_string()) - self.wait(cluster.sbctl, cluster.cluster_cfg.node_timeout_s) - - # Create a node switch and connect it to the cluster router. - self.switch = cluster.nbctl.ls_add( - f'lswitch-{self.container}', net_s=self.int_net - ) - lrp_name = f'rtr-to-node-{self.container}' - ls_rp_name = f'node-to-rtr-{self.container}' - self.rp = cluster.nbctl.lr_port_add( - cluster.router, lrp_name, RandMac(), self.int_net.reverse() - ) - self.ls_rp = cluster.nbctl.ls_port_add( - self.switch, ls_rp_name, self.rp - ) - - # Make the lrp as distributed gateway router port. - cluster.nbctl.lr_port_set_gw_chassis(self.rp, self.container) - - # Create a gw router and connect it to the cluster join switch. - self.gw_router = cluster.nbctl.lr_add(f'gwrouter-{self.container}') - cluster.nbctl.lr_set_options( - self.gw_router, - { - 'always_learn_from_arp_request': 'false', - 'dynamic_neigh_routers': 'true', - 'chassis': self.container, - 'lb_force_snat_ip': 'router_ip', - 'snat-ct-zone': 0, - }, - ) - join_grp_name = f'gw-to-join-{self.container}' - join_ls_grp_name = f'join-to-gw-{self.container}' - - gr_gw = self.gw_net.reverse(self.id + 2) - self.gw_rp = cluster.nbctl.lr_port_add( - self.gw_router, join_grp_name, RandMac(), gr_gw - ) - self.join_gw_rp = cluster.nbctl.ls_port_add( - cluster.join_switch, join_ls_grp_name, self.gw_rp - ) - - # Create an external switch connecting the gateway router to the - # physnet. - self.ext_switch = cluster.nbctl.ls_add( - f'ext-{self.container}', net_s=self.ext_net - ) - ext_lrp_name = f'gw-to-ext-{self.container}' - ext_ls_rp_name = f'ext-to-gw-{self.container}' - self.ext_rp = cluster.nbctl.lr_port_add( - self.gw_router, ext_lrp_name, RandMac(), self.ext_net.reverse() - ) - self.ext_gw_rp = cluster.nbctl.ls_port_add( - self.ext_switch, ext_ls_rp_name, self.ext_rp - ) - - # Configure physnet. - self.physnet_port = cluster.nbctl.ls_port_add( - self.ext_switch, - f'provnet-{self.container}', - localnet=True, - ) - cluster.nbctl.ls_port_set_set_type(self.physnet_port, 'localnet') - cluster.nbctl.ls_port_set_set_options( - self.physnet_port, f'network_name={cluster.brex_cfg.physical_net}' - ) - - # Route for traffic entering the cluster. - cluster.nbctl.route_add( - self.gw_router, cluster.net, self.gw_net.reverse() - ) - - # Default route to get out of cluster via physnet. - cluster.nbctl.route_add( - self.gw_router, - ovn_utils.DualStackSubnet( - netaddr.IPNetwork("0.0.0.0/0"), netaddr.IPNetwork("::/0") - ), - self.ext_net.reverse(2), - ) - - # Route for traffic that needs to exit the cluster - # (via gw router). - cluster.nbctl.route_add( - cluster.router, self.int_net, gr_gw, policy="src-ip" - ) - - # SNAT traffic leaving the cluster. - cluster.nbctl.nat_add(self.gw_router, gr_gw, cluster.net) - - @ovn_stats.timeit - def provision_port(self, cluster, passive=False): - name = f'lp-{self.id}-{self.next_lport_index}' - - log.info(f'Creating lport {name}') - lport = cluster.nbctl.ls_port_add( - self.switch, - name, - mac=str(RandMac()), - ip=self.int_net.forward(self.next_lport_index + 1), - gw=self.int_net.reverse(), - ext_gw=self.ext_net.reverse(2), - metadata=self, - passive=passive, - security=True, - ) - - self.lports.append(lport) - self.next_lport_index += 1 - return lport - @ovn_stats.timeit def unprovision_port(self, cluster, port): cluster.nbctl.ls_port_del(port) self.unbind_port(port) self.lports.remove(port) - @ovn_stats.timeit - def provision_load_balancers(self, cluster, ports, global_cfg): - # Add one port IP as a backend to the cluster load balancer. - if global_cfg.run_ipv4: - port_ips = ( - f'{port.ip}:{DEFAULT_BACKEND_PORT}' - for port in ports - if port.ip is not None - ) - cluster_vips = cluster.cluster_cfg.vips.keys() - cluster.load_balancer.add_backends_to_vip(port_ips, cluster_vips) - cluster.load_balancer.add_to_switches([self.switch.name]) - cluster.load_balancer.add_to_routers([self.gw_router.name]) - - if global_cfg.run_ipv6: - port_ips6 = ( - f'[{port.ip6}]:{DEFAULT_BACKEND_PORT}' - for port in ports - if port.ip6 is not None - ) - cluster_vips6 = cluster.cluster_cfg.vips6.keys() - cluster.load_balancer6.add_backends_to_vip( - port_ips6, cluster_vips6 - ) - cluster.load_balancer6.add_to_switches([self.switch.name]) - cluster.load_balancer6.add_to_routers([self.gw_router.name]) - - # GW Load balancer has no VIPs/backends configured on it, since - # this load balancer is used for hostnetwork services. We're not - # using those right now so the load blaancer is empty. - if global_cfg.run_ipv4: - self.gw_load_balancer = lb.OvnLoadBalancer( - f'lb-{self.gw_router.name}', cluster.nbctl - ) - self.gw_load_balancer.add_to_routers([self.gw_router.name]) - if global_cfg.run_ipv6: - self.gw_load_balancer6 = lb.OvnLoadBalancer( - f'lb-{self.gw_router.name}6', cluster.nbctl - ) - self.gw_load_balancer6.add_to_routers([self.gw_router.name]) - @ovn_stats.timeit def bind_port(self, port): log.info(f'Binding lport {port.name} on {self.container}') @@ -421,13 +252,6 @@ def run_ping(self, cluster, src, dest): def ping_port(self, cluster, port, dest): self.run_ping(cluster, port.name, dest) - @ovn_stats.timeit - def ping_external(self, cluster, port): - if port.ip: - self.run_ping(cluster, 'ext-ns', port.ip) - if port.ip6: - self.run_ping(cluster, 'ext-ns', port.ip6) - def ping_ports(self, cluster, ports): for port in ports: if port.ip: @@ -438,6 +262,21 @@ def ping_ports(self, cluster, ports): def get_connection_string(self, port): return f"{self.protocol}:{self.mgmt_ip}:{port}" + def configure(self, physical_net): + raise NotImplementedError + + @ovn_stats.timeit + def provision(self, cluster): + raise NotImplementedError + + @ovn_stats.timeit + def provision_port(self, cluster, passive=False): + raise NotImplementedError + + @ovn_stats.timeit + def ping_external(self, cluster, port): + raise NotImplementedError + ACL_DEFAULT_DENY_PRIO = 1 ACL_DEFAULT_ALLOW_ARP_PRIO = 2 From 8d10c59225f413cc99553cab4a961d09c2916558 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 25 Oct 2023 10:57:27 +0200 Subject: [PATCH 02/19] ovn-tester/ovn_workload: Extract generic methods from Cluster class. Commit 1380bba moved instantiation of CMS Cluster class back to the generic ovn_tester.py. The `create_cluster` function is generic and actually takes almost identical arguments as the `Cluster` class. Let's make it part of the `Cluster` constructor instead! This change also makes it apparent that a separate CMS plugin class is not necessary, so let's roll this into the CMS specifc Cluster class. Co-Authored-by: Dumitru Ceara Signed-off-by: Dumitru Ceara Signed-off-by: Frode Nordahl --- .../cms/ovn_kubernetes/ovn_kubernetes.py | 163 +++++++++++++----- ovn-tester/ovn_tester.py | 50 +----- ovn-tester/ovn_workload.py | 122 +++++-------- 3 files changed, 169 insertions(+), 166 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index f6fde14d..15fe692e 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -9,16 +9,134 @@ import ovn_utils import ovn_stats -from ovn_context import Context from ovn_utils import DualStackSubnet from ovn_workload import ( ChassisNode, + Cluster, DEFAULT_BACKEND_PORT, + DEFAULT_VIP_PORT, ) log = logging.getLogger(__name__) ClusterBringupCfg = namedtuple('ClusterBringupCfg', ['n_pods_per_node']) -OVN_HEATER_CMS_PLUGIN = 'OVNKubernetes' +OVN_HEATER_CMS_PLUGIN = 'OVNKubernetesCluster' + + +class OVNKubernetesCluster(Cluster): + def __init__(self, cluster_cfg, central, brex_cfg, az): + super().__init__(cluster_cfg, central, brex_cfg, az) + self.net = cluster_cfg.cluster_net + self.gw_net = ovn_utils.DualStackSubnet.next( + cluster_cfg.gw_net, + az * (cluster_cfg.n_workers // cluster_cfg.n_az), + ) + self.router = None + self.load_balancer = None + self.load_balancer6 = None + self.join_switch = None + self.last_selected_worker = 0 + self.n_ns = 0 + self.ts_switch = None + + def add_cluster_worker_nodes(self, workers): + cluster_cfg = self.cluster_cfg + + # Allocate worker IPs after central and relay IPs. + mgmt_ip = ( + cluster_cfg.node_net.ip + + 2 + + cluster_cfg.n_az + * (len(self.central_nodes) + len(self.relay_nodes)) + ) + + protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" + internal_net = cluster_cfg.internal_net + external_net = cluster_cfg.external_net + # Number of workers for each az + n_az_workers = cluster_cfg.n_workers // cluster_cfg.n_az + self.add_workers( + [ + WorkerNode( + workers[i % len(workers)], + f'ovn-scale-{i}', + mgmt_ip + i, + protocol, + DualStackSubnet.next(internal_net, i), + DualStackSubnet.next(external_net, i), + self.gw_net, + i, + ) + for i in range( + self.az * n_az_workers, (self.az + 1) * n_az_workers + ) + ] + ) + + def create_cluster_router(self, rtr_name): + self.router = self.nbctl.lr_add(rtr_name) + self.nbctl.lr_set_options( + self.router, + { + 'always_learn_from_arp_request': 'false', + }, + ) + + def create_cluster_load_balancer(self, lb_name, global_cfg): + if global_cfg.run_ipv4: + self.load_balancer = lb.OvnLoadBalancer( + lb_name, self.nbctl, self.cluster_cfg.vips + ) + self.load_balancer.add_vips(self.cluster_cfg.static_vips) + + if global_cfg.run_ipv6: + self.load_balancer6 = lb.OvnLoadBalancer( + f'{lb_name}6', self.nbctl, self.cluster_cfg.vips6 + ) + self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) + + def create_cluster_join_switch(self, sw_name): + self.join_switch = self.nbctl.ls_add(sw_name, net_s=self.gw_net) + + self.join_rp = self.nbctl.lr_port_add( + self.router, + f'rtr-to-{sw_name}', + RandMac(), + self.gw_net.reverse(), + ) + self.join_ls_rp = self.nbctl.ls_port_add( + self.join_switch, f'{sw_name}-to-rtr', self.join_rp + ) + + @ovn_stats.timeit + def provision_vips_to_load_balancers(self, backend_lists): + n_vips = len(self.load_balancer.vips.keys()) + vip_ip = self.cluster_cfg.vip_subnet.ip.__add__(n_vips + 1) + + vips = { + f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ + f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports + ] + for i, ports in enumerate(backend_lists) + } + self.load_balancer.add_vips(vips) + + def unprovision_vips(self): + if self.load_balancer: + self.load_balancer.clear_vips() + self.load_balancer.add_vips(self.cluster_cfg.static_vips) + if self.load_balancer6: + self.load_balancer6.clear_vips() + self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) + + def provision_lb_group(self, name='cluster-lb-group'): + self.lb_group = lb.OvnLoadBalancerGroup(name, self.nbctl) + for w in self.worker_nodes: + self.nbctl.ls_add_lbg(w.switch, self.lb_group.lbg) + self.nbctl.lr_add_lbg(w.gw_router, self.lb_group.lbg) + + def provision_lb(self, lb): + log.info(f'Creating load balancer {lb.name}') + self.lb_group.add_lb(lb) class WorkerNode(ChassisNode): @@ -206,44 +324,3 @@ def ping_external(self, cluster, port): self.run_ping(cluster, 'ext-ns', port.ip) if port.ip6: self.run_ping(cluster, 'ext-ns', port.ip6) - - -class OVNKubernetes: - @staticmethod - def add_cluster_worker_nodes(cluster, workers, az): - cluster_cfg = cluster.cluster_cfg - - # Allocate worker IPs after central and relay IPs. - mgmt_ip = ( - cluster_cfg.node_net.ip - + 2 - + cluster_cfg.n_az - * (len(cluster.central_nodes) + len(cluster.relay_nodes)) - ) - - protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - internal_net = cluster_cfg.internal_net - external_net = cluster_cfg.external_net - # Number of workers for each az - n_az_workers = cluster_cfg.n_workers // cluster_cfg.n_az - cluster.add_workers( - [ - WorkerNode( - workers[i % len(workers)], - f'ovn-scale-{i}', - mgmt_ip + i, - protocol, - DualStackSubnet.next(internal_net, i), - DualStackSubnet.next(external_net, i), - cluster.gw_net, - i, - ) - for i in range(az * n_az_workers, (az + 1) * n_az_workers) - ] - ) - - @staticmethod - def prepare_test(clusters): - with Context(clusters, 'prepare_test clusters'): - for c in clusters: - c.start() diff --git a/ovn-tester/ovn_tester.py b/ovn-tester/ovn_tester.py index e89addae..9c341096 100644 --- a/ovn-tester/ovn_tester.py +++ b/ovn-tester/ovn_tester.py @@ -10,13 +10,11 @@ import time from collections import namedtuple +from ovn_context import Context from ovn_sandbox import PhysicalNode from ovn_workload import ( BrExConfig, - CentralNode, - Cluster, ClusterConfig, - RelayNode, ) from ovn_utils import DualStackSubnet from ovs.stream import Stream @@ -178,7 +176,7 @@ def load_cms(cms_name): mod = importlib.import_module(f'cms.{cms_name}') class_name = getattr(mod, 'OVN_HEATER_CMS_PLUGIN') cls = getattr(mod, class_name) - return cls() + return cls def configure_tests(yaml, clusters, global_cfg): @@ -196,40 +194,6 @@ def configure_tests(yaml, clusters, global_cfg): return tests -def create_cluster(cluster_cfg, central, workers, brex_cfg, cms, az): - protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" - db_containers = ( - [ - f'ovn-central-az{az+1}-1', - f'ovn-central-az{az+1}-2', - f'ovn-central-az{az+1}-3', - ] - if cluster_cfg.clustered_db - else [f'ovn-central-az{az+1}-1'] - ) - - mgmt_ip = cluster_cfg.node_net.ip + 2 + az * len(db_containers) - central_nodes = [ - CentralNode(central, c, mgmt_ip + i, protocol) - for i, c in enumerate(db_containers) - ] - - mgmt_ip = ( - cluster_cfg.node_net.ip - + 2 - + cluster_cfg.n_az * len(central_nodes) - + az * cluster_cfg.n_relays - ) - relay_nodes = [ - RelayNode(central, f'ovn-relay-az{az+1}-{i+1}', mgmt_ip + i, protocol) - for i in range(cluster_cfg.n_relays) - ] - - cluster = Cluster(central_nodes, relay_nodes, cluster_cfg, brex_cfg, az) - cms.add_cluster_worker_nodes(cluster, workers, az) - return cluster - - def set_ssl_keys(cluster_cfg): Stream.ssl_set_private_key_file(cluster_cfg.ssl_private_key) Stream.ssl_set_certificate_file(cluster_cfg.ssl_cert) @@ -253,19 +217,23 @@ def set_ssl_keys(cluster_cfg): ): raise ovn_exceptions.OvnInvalidConfigException() - cms = load_cms(global_cfg.cms_name) + cms_cls = load_cms(global_cfg.cms_name) central, workers = read_physical_deployment(sys.argv[1], global_cfg) clusters = [ - create_cluster(cluster_cfg, central, workers, brex_cfg, cms, i) + cms_cls(cluster_cfg, central, brex_cfg, i) for i in range(cluster_cfg.n_az) ] + for c in clusters: + c.add_cluster_worker_nodes(workers) tests = configure_tests(config, clusters, global_cfg) if cluster_cfg.enable_ssl: set_ssl_keys(cluster_cfg) - cms.prepare_test(clusters) + with Context(clusters, 'prepare_test clusters'): + for c in clusters: + c.prepare_test() for test in tests: test.run(clusters, global_cfg) sys.exit(0) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index b8f3f85d..721f1a2b 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -8,7 +8,6 @@ import netaddr from collections import namedtuple from collections import defaultdict -from randmac import RandMac from datetime import datetime log = logging.getLogger(__name__) @@ -605,33 +604,58 @@ def provision_vips_to_load_balancers(self, backend_lists, version, az=0): class Cluster: - def __init__(self, central_nodes, relay_nodes, cluster_cfg, brex_cfg, az): + def __init__(self, cluster_cfg, central, brex_cfg, az): # In clustered mode use the first node for provisioning. - self.central_nodes = central_nodes - self.relay_nodes = relay_nodes self.worker_nodes = [] self.cluster_cfg = cluster_cfg self.brex_cfg = brex_cfg self.nbctl = None self.sbctl = None self.icnbctl = None - self.net = cluster_cfg.cluster_net - self.gw_net = ovn_utils.DualStackSubnet.next( - cluster_cfg.gw_net, - az * (cluster_cfg.n_workers // cluster_cfg.n_az), - ) self.az = az - self.router = None - self.load_balancer = None - self.load_balancer6 = None - self.join_switch = None - self.last_selected_worker = 0 - self.n_ns = 0 - self.ts_switch = None + + protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" + db_containers = ( + [ + f'ovn-central-az{self.az+1}-1', + f'ovn-central-az{self.az+1}-2', + f'ovn-central-az{self.az+1}-3', + ] + if cluster_cfg.clustered_db + else [f'ovn-central-az{self.az+1}-1'] + ) + + mgmt_ip = cluster_cfg.node_net.ip + 2 + self.az * len(db_containers) + self.central_nodes = [ + CentralNode(central, c, mgmt_ip + i, protocol) + for i, c in enumerate(db_containers) + ] + + mgmt_ip = ( + cluster_cfg.node_net.ip + + 2 + + cluster_cfg.n_az * len(self.central_nodes) + + self.az * cluster_cfg.n_relays + ) + self.relay_nodes = [ + RelayNode( + central, + f'ovn-relay-az{self.az+1}-{i+1}', + mgmt_ip + i, + protocol, + ) + for i in range(cluster_cfg.n_relays) + ] + + def add_cluster_worker_nodes(self, workers): + raise NotImplementedError def add_workers(self, worker_nodes): self.worker_nodes.extend(worker_nodes) + def prepare_test(self): + self.start() + def start(self): for c in self.central_nodes: c.start( @@ -692,41 +716,6 @@ def get_relay_connection_string(self): ) return self.get_sb_connection_string() - def create_cluster_router(self, rtr_name): - self.router = self.nbctl.lr_add(rtr_name) - self.nbctl.lr_set_options( - self.router, - { - 'always_learn_from_arp_request': 'false', - }, - ) - - def create_cluster_load_balancer(self, lb_name, global_cfg): - if global_cfg.run_ipv4: - self.load_balancer = lb.OvnLoadBalancer( - lb_name, self.nbctl, self.cluster_cfg.vips - ) - self.load_balancer.add_vips(self.cluster_cfg.static_vips) - - if global_cfg.run_ipv6: - self.load_balancer6 = lb.OvnLoadBalancer( - f'{lb_name}6', self.nbctl, self.cluster_cfg.vips6 - ) - self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) - - def create_cluster_join_switch(self, sw_name): - self.join_switch = self.nbctl.ls_add(sw_name, net_s=self.gw_net) - - self.join_rp = self.nbctl.lr_port_add( - self.router, - f'rtr-to-{sw_name}', - RandMac(), - self.gw_net.reverse(), - ) - self.join_ls_rp = self.nbctl.ls_port_add( - self.join_switch, f'{sw_name}-to-rtr', self.join_rp - ) - def provision_ports(self, n_ports, passive=False): return [ self.select_worker_for_port().provision_ports(self, 1, passive)[0] @@ -745,38 +734,7 @@ def ping_ports(self, ports): for w, ports in ports_per_worker.items(): w.ping_ports(self, ports) - @ovn_stats.timeit - def provision_vips_to_load_balancers(self, backend_lists): - n_vips = len(self.load_balancer.vips.keys()) - vip_ip = self.cluster_cfg.vip_subnet.ip.__add__(n_vips + 1) - - vips = { - f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ - f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports - ] - for i, ports in enumerate(backend_lists) - } - self.load_balancer.add_vips(vips) - - def unprovision_vips(self): - if self.load_balancer: - self.load_balancer.clear_vips() - self.load_balancer.add_vips(self.cluster_cfg.static_vips) - if self.load_balancer6: - self.load_balancer6.clear_vips() - self.load_balancer6.add_vips(self.cluster_cfg.static_vips6) - def select_worker_for_port(self): self.last_selected_worker += 1 self.last_selected_worker %= len(self.worker_nodes) return self.worker_nodes[self.last_selected_worker] - - def provision_lb_group(self, name='cluster-lb-group'): - self.lb_group = lb.OvnLoadBalancerGroup(name, self.nbctl) - for w in self.worker_nodes: - self.nbctl.ls_add_lbg(w.switch, self.lb_group.lbg) - self.nbctl.lr_add_lbg(w.gw_router, self.lb_group.lbg) - - def provision_lb(self, lb): - log.info(f'Creating load balancer {lb.name}') - self.lb_group.add_lb(lb) From 84e495f687653ba5cf73900bbc82ff132b244483 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 25 Oct 2023 11:10:30 +0200 Subject: [PATCH 03/19] ovn-tester/ovn_workload: Move Namespace class to CMS plugin. Signed-off-by: Frode Nordahl --- .../cms/ovn_kubernetes/ovn_kubernetes.py | 331 +++++++++++++++++- .../ovn_kubernetes/tests/cluster_density.py | 2 +- .../cms/ovn_kubernetes/tests/density_heavy.py | 2 +- .../cms/ovn_kubernetes/tests/density_light.py | 2 +- ovn-tester/cms/ovn_kubernetes/tests/netpol.py | 2 +- .../ovn_kubernetes/tests/netpol_cross_ns.py | 2 +- .../tests/netpol_multitenant.py | 2 +- .../cms/ovn_kubernetes/tests/service_route.py | 2 +- ovn-tester/ovn_workload.py | 327 ----------------- 9 files changed, 332 insertions(+), 340 deletions(-) diff --git a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py index 15fe692e..c52d2b2b 100644 --- a/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py +++ b/ovn-tester/cms/ovn_kubernetes/ovn_kubernetes.py @@ -10,16 +10,335 @@ import ovn_stats from ovn_utils import DualStackSubnet -from ovn_workload import ( - ChassisNode, - Cluster, - DEFAULT_BACKEND_PORT, - DEFAULT_VIP_PORT, -) +from ovn_workload import ChassisNode, Cluster log = logging.getLogger(__name__) ClusterBringupCfg = namedtuple('ClusterBringupCfg', ['n_pods_per_node']) OVN_HEATER_CMS_PLUGIN = 'OVNKubernetesCluster' +ACL_DEFAULT_DENY_PRIO = 1 +ACL_DEFAULT_ALLOW_ARP_PRIO = 2 +ACL_NETPOL_ALLOW_PRIO = 3 +DEFAULT_NS_VIP_SUBNET = netaddr.IPNetwork('30.0.0.0/16') +DEFAULT_NS_VIP_SUBNET6 = netaddr.IPNetwork('30::/32') +DEFAULT_VIP_PORT = 80 +DEFAULT_BACKEND_PORT = 8080 + + +class Namespace: + def __init__(self, clusters, name, global_cfg): + self.clusters = clusters + self.nbctl = [cluster.nbctl for cluster in clusters] + self.ports = [[] for _ in range(len(clusters))] + self.enforcing = False + self.pg_def_deny_igr = [ + nbctl.port_group_create(f'pg_deny_igr_{name}') + for nbctl in self.nbctl + ] + self.pg_def_deny_egr = [ + nbctl.port_group_create(f'pg_deny_egr_{name}') + for nbctl in self.nbctl + ] + self.pg = [ + nbctl.port_group_create(f'pg_{name}') for nbctl in self.nbctl + ] + self.addr_set4 = [ + ( + nbctl.address_set_create(f'as_{name}') + if global_cfg.run_ipv4 + else None + ) + for nbctl in self.nbctl + ] + self.addr_set6 = [ + ( + nbctl.address_set_create(f'as6_{name}') + if global_cfg.run_ipv6 + else None + ) + for nbctl in self.nbctl + ] + self.sub_as = [[] for _ in range(len(clusters))] + self.sub_pg = [[] for _ in range(len(clusters))] + self.load_balancer = None + for cluster in self.clusters: + cluster.n_ns += 1 + self.name = name + + @ovn_stats.timeit + def add_ports(self, ports, az=0): + self.ports[az].extend(ports) + # Always add port IPs to the address set but not to the PGs. + # Simulate what OpenShift does, which is: create the port groups + # when the first network policy is applied. + if self.addr_set4: + for i, nbctl in enumerate(self.nbctl): + nbctl.address_set_add_addrs( + self.addr_set4[i], [str(p.ip) for p in ports] + ) + if self.addr_set6: + for i, nbctl in enumerate(self.nbctl): + nbctl.address_set_add_addrs( + self.addr_set6[i], [str(p.ip6) for p in ports] + ) + if self.enforcing: + self.nbctl[az].port_group_add_ports( + self.pg_def_deny_igr[az], ports + ) + self.nbctl[az].port_group_add_ports( + self.pg_def_deny_egr[az], ports + ) + self.nbctl[az].port_group_add_ports(self.pg[az], ports) + + def unprovision(self): + # ACLs are garbage collected by OVSDB as soon as all the records + # referencing them are removed. + for i, cluster in enumerate(self.clusters): + cluster.unprovision_ports(self.ports[i]) + for i, nbctl in enumerate(self.nbctl): + nbctl.port_group_del(self.pg_def_deny_igr[i]) + nbctl.port_group_del(self.pg_def_deny_egr[i]) + nbctl.port_group_del(self.pg[i]) + if self.addr_set4: + nbctl.address_set_del(self.addr_set4[i]) + if self.addr_set6: + nbctl.address_set_del(self.addr_set6[i]) + nbctl.port_group_del(self.sub_pg[i]) + nbctl.address_set_del(self.sub_as[i]) + + def unprovision_ports(self, ports, az=0): + '''Unprovision a subset of ports in the namespace without having to + unprovision the entire namespace or any of its network policies.''' + + for port in ports: + self.ports[az].remove(port) + + self.clusters[az].unprovision_ports(ports) + + def enforce(self): + if self.enforcing: + return + self.enforcing = True + for i, nbctl in enumerate(self.nbctl): + nbctl.port_group_add_ports(self.pg_def_deny_igr[i], self.ports[i]) + nbctl.port_group_add_ports(self.pg_def_deny_egr[i], self.ports[i]) + nbctl.port_group_add_ports(self.pg[i], self.ports[i]) + + def create_sub_ns(self, ports, global_cfg, az=0): + n_sub_pgs = len(self.sub_pg[az]) + suffix = f'{self.name}_{n_sub_pgs}' + pg = self.nbctl[az].port_group_create(f'sub_pg_{suffix}') + self.nbctl[az].port_group_add_ports(pg, ports) + self.sub_pg[az].append(pg) + for i, nbctl in enumerate(self.nbctl): + if global_cfg.run_ipv4: + addr_set = nbctl.address_set_create(f'sub_as_{suffix}') + nbctl.address_set_add_addrs( + addr_set, [str(p.ip) for p in ports] + ) + self.sub_as[i].append(addr_set) + if global_cfg.run_ipv6: + addr_set = nbctl.address_set_create(f'sub_as_{suffix}6') + nbctl.address_set_add_addrs( + addr_set, [str(p.ip6) for p in ports] + ) + self.sub_as[i].append(addr_set) + return n_sub_pgs + + @ovn_stats.timeit + def default_deny(self, family, az=0): + self.enforce() + + addr_set = f'self.addr_set{family}.name' + self.nbctl[az].acl_add( + self.pg_def_deny_igr[az].name, + 'to-lport', + ACL_DEFAULT_DENY_PRIO, + 'port-group', + f'ip4.src == \\${addr_set} && ' + f'outport == @{self.pg_def_deny_igr[az].name}', + 'drop', + ) + self.nbctl[az].acl_add( + self.pg_def_deny_egr[az].name, + 'to-lport', + ACL_DEFAULT_DENY_PRIO, + 'port-group', + f'ip4.dst == \\${addr_set} && ' + f'inport == @{self.pg_def_deny_egr[az].name}', + 'drop', + ) + self.nbctl[az].acl_add( + self.pg_def_deny_igr[az].name, + 'to-lport', + ACL_DEFAULT_ALLOW_ARP_PRIO, + 'port-group', + f'outport == @{self.pg_def_deny_igr[az].name} && arp', + 'allow', + ) + self.nbctl[az].acl_add( + self.pg_def_deny_egr[az].name, + 'to-lport', + ACL_DEFAULT_ALLOW_ARP_PRIO, + 'port-group', + f'inport == @{self.pg_def_deny_egr[az].name} && arp', + 'allow', + ) + + @ovn_stats.timeit + def allow_within_namespace(self, family, az=0): + self.enforce() + + addr_set = f'self.addr_set{family}.name' + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.src == \\${addr_set} && ' f'outport == @{self.pg[az].name}', + 'allow-related', + ) + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.dst == \\${addr_set} && ' f'inport == @{self.pg[az].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def allow_cross_namespace(self, ns, family): + self.enforce() + + for az, nbctl in enumerate(self.nbctl): + if len(self.ports[az]) == 0: + continue + addr_set = f'self.addr_set{family}.name' + nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.src == \\${addr_set} && ' + f'outport == @{ns.pg[az].name}', + 'allow-related', + ) + ns_addr_set = f'ns.addr_set{family}.name' + nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip4.dst == \\${ns_addr_set} && ' + f'inport == @{self.pg[az].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def allow_sub_namespace(self, src, dst, family, az=0): + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip{family}.src == \\${self.sub_as[az][src].name} && ' + f'outport == @{self.sub_pg[az][dst].name}', + 'allow-related', + ) + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip{family}.dst == \\${self.sub_as[az][dst].name} && ' + f'inport == @{self.sub_pg[az][src].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def allow_from_external( + self, external_ips, include_ext_gw=False, family=4, az=0 + ): + self.enforce() + # If requested, include the ext-gw of the first port in the namespace + # so we can check that this rule is enforced. + if include_ext_gw: + assert len(self.ports) > 0 + if family == 4 and self.ports[az][0].ext_gw: + external_ips.append(self.ports[az][0].ext_gw) + elif family == 6 and self.ports[az][0].ext_gw6: + external_ips.append(self.ports[az][0].ext_gw6) + ips = [str(ip) for ip in external_ips] + self.nbctl[az].acl_add( + self.pg[az].name, + 'to-lport', + ACL_NETPOL_ALLOW_PRIO, + 'port-group', + f'ip.{family} == {{{",".join(ips)}}} && ' + f'outport == @{self.pg[az].name}', + 'allow-related', + ) + + @ovn_stats.timeit + def check_enforcing_internal(self, az=0): + # "Random" check that first pod can reach last pod in the namespace. + if len(self.ports[az]) > 1: + src = self.ports[az][0] + dst = self.ports[az][-1] + worker = src.metadata + if src.ip: + worker.ping_port(self.clusters[az], src, dst.ip) + if src.ip6: + worker.ping_port(self.clusters[az], src, dst.ip6) + + @ovn_stats.timeit + def check_enforcing_external(self, az=0): + if len(self.ports[az]) > 0: + dst = self.ports[az][0] + worker = dst.metadata + worker.ping_external(self.clusters[az], dst) + + @ovn_stats.timeit + def check_enforcing_cross_ns(self, ns, az=0): + if len(self.ports[az]) > 0 and len(ns.ports[az]) > 0: + dst = ns.ports[az][0] + src = self.ports[az][0] + worker = src.metadata + if src.ip and dst.ip: + worker.ping_port(self.clusters[az], src, dst.ip) + if src.ip6 and dst.ip6: + worker.ping_port(self.clusters[az], src, dst.ip6) + + def create_load_balancer(self, az=0): + self.load_balancer = lb.OvnLoadBalancer( + f'lb_{self.name}', self.nbctl[az] + ) + + @ovn_stats.timeit + def provision_vips_to_load_balancers(self, backend_lists, version, az=0): + vip_ns_subnet = DEFAULT_NS_VIP_SUBNET + if version == 6: + vip_ns_subnet = DEFAULT_NS_VIP_SUBNET6 + vip_net = vip_ns_subnet.next(self.clusters[az].n_ns) + n_vips = len(self.load_balancer.vips.keys()) + vip_ip = vip_net.ip.__add__(n_vips + 1) + + if version == 6: + vips = { + f'[{vip_ip + i}]:{DEFAULT_VIP_PORT}': [ + f'[{p.ip6}]:{DEFAULT_BACKEND_PORT}' for p in ports + ] + for i, ports in enumerate(backend_lists) + } + self.load_balancer.add_vips(vips) + else: + vips = { + f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ + f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports + ] + for i, ports in enumerate(backend_lists) + } + self.load_balancer.add_vips(vips) class OVNKubernetesCluster(Cluster): diff --git a/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py b/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py index b397cd04..d3ab8017 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/cluster_density.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd import ovn_exceptions diff --git a/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py b/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py index e34ca600..1c3d87f1 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/density_heavy.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd import ovn_load_balancer as lb import ovn_exceptions diff --git a/ovn-tester/cms/ovn_kubernetes/tests/density_light.py b/ovn-tester/cms/ovn_kubernetes/tests/density_light.py index 3732e3f3..a0bf135d 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/density_light.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/density_light.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol.py index 7601d862..9feda706 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd from itertools import chain import ovn_exceptions diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py index 4ddcf0a6..7677cf53 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_cross_ns.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd NpCrossNsCfg = namedtuple('NpCrossNsCfg', ['n_ns', 'pods_ns_ratio']) diff --git a/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py b/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py index 6ef719f3..9dc43061 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/netpol_multitenant.py @@ -1,7 +1,7 @@ from collections import namedtuple import netaddr from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd diff --git a/ovn-tester/cms/ovn_kubernetes/tests/service_route.py b/ovn-tester/cms/ovn_kubernetes/tests/service_route.py index 5f333a80..0b3a297b 100644 --- a/ovn-tester/cms/ovn_kubernetes/tests/service_route.py +++ b/ovn-tester/cms/ovn_kubernetes/tests/service_route.py @@ -1,6 +1,6 @@ from collections import namedtuple from ovn_context import Context -from ovn_workload import Namespace +from cms.ovn_kubernetes import Namespace from ovn_ext_cmd import ExtCmd import netaddr diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 721f1a2b..3ec04bc3 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -3,7 +3,6 @@ import ovn_sandbox import ovn_stats import ovn_utils -import ovn_load_balancer as lb import time import netaddr from collections import namedtuple @@ -277,332 +276,6 @@ def ping_external(self, cluster, port): raise NotImplementedError -ACL_DEFAULT_DENY_PRIO = 1 -ACL_DEFAULT_ALLOW_ARP_PRIO = 2 -ACL_NETPOL_ALLOW_PRIO = 3 -DEFAULT_NS_VIP_SUBNET = netaddr.IPNetwork('30.0.0.0/16') -DEFAULT_NS_VIP_SUBNET6 = netaddr.IPNetwork('30::/32') -DEFAULT_VIP_PORT = 80 -DEFAULT_BACKEND_PORT = 8080 - - -class Namespace: - def __init__(self, clusters, name, global_cfg): - self.clusters = clusters - self.nbctl = [cluster.nbctl for cluster in clusters] - self.ports = [[] for _ in range(len(clusters))] - self.enforcing = False - self.pg_def_deny_igr = [ - nbctl.port_group_create(f'pg_deny_igr_{name}') - for nbctl in self.nbctl - ] - self.pg_def_deny_egr = [ - nbctl.port_group_create(f'pg_deny_egr_{name}') - for nbctl in self.nbctl - ] - self.pg = [ - nbctl.port_group_create(f'pg_{name}') for nbctl in self.nbctl - ] - self.addr_set4 = [ - ( - nbctl.address_set_create(f'as_{name}') - if global_cfg.run_ipv4 - else None - ) - for nbctl in self.nbctl - ] - self.addr_set6 = [ - ( - nbctl.address_set_create(f'as6_{name}') - if global_cfg.run_ipv6 - else None - ) - for nbctl in self.nbctl - ] - self.sub_as = [[] for _ in range(len(clusters))] - self.sub_pg = [[] for _ in range(len(clusters))] - self.load_balancer = None - for cluster in self.clusters: - cluster.n_ns += 1 - self.name = name - - @ovn_stats.timeit - def add_ports(self, ports, az=0): - self.ports[az].extend(ports) - # Always add port IPs to the address set but not to the PGs. - # Simulate what OpenShift does, which is: create the port groups - # when the first network policy is applied. - if self.addr_set4: - for i, nbctl in enumerate(self.nbctl): - nbctl.address_set_add_addrs( - self.addr_set4[i], [str(p.ip) for p in ports] - ) - if self.addr_set6: - for i, nbctl in enumerate(self.nbctl): - nbctl.address_set_add_addrs( - self.addr_set6[i], [str(p.ip6) for p in ports] - ) - if self.enforcing: - self.nbctl[az].port_group_add_ports( - self.pg_def_deny_igr[az], ports - ) - self.nbctl[az].port_group_add_ports( - self.pg_def_deny_egr[az], ports - ) - self.nbctl[az].port_group_add_ports(self.pg[az], ports) - - def unprovision(self): - # ACLs are garbage collected by OVSDB as soon as all the records - # referencing them are removed. - for i, cluster in enumerate(self.clusters): - cluster.unprovision_ports(self.ports[i]) - for i, nbctl in enumerate(self.nbctl): - nbctl.port_group_del(self.pg_def_deny_igr[i]) - nbctl.port_group_del(self.pg_def_deny_egr[i]) - nbctl.port_group_del(self.pg[i]) - if self.addr_set4: - nbctl.address_set_del(self.addr_set4[i]) - if self.addr_set6: - nbctl.address_set_del(self.addr_set6[i]) - nbctl.port_group_del(self.sub_pg[i]) - nbctl.address_set_del(self.sub_as[i]) - - def unprovision_ports(self, ports, az=0): - '''Unprovision a subset of ports in the namespace without having to - unprovision the entire namespace or any of its network policies.''' - - for port in ports: - self.ports[az].remove(port) - - self.clusters[az].unprovision_ports(ports) - - def enforce(self): - if self.enforcing: - return - self.enforcing = True - for i, nbctl in enumerate(self.nbctl): - nbctl.port_group_add_ports(self.pg_def_deny_igr[i], self.ports[i]) - nbctl.port_group_add_ports(self.pg_def_deny_egr[i], self.ports[i]) - nbctl.port_group_add_ports(self.pg[i], self.ports[i]) - - def create_sub_ns(self, ports, global_cfg, az=0): - n_sub_pgs = len(self.sub_pg[az]) - suffix = f'{self.name}_{n_sub_pgs}' - pg = self.nbctl[az].port_group_create(f'sub_pg_{suffix}') - self.nbctl[az].port_group_add_ports(pg, ports) - self.sub_pg[az].append(pg) - for i, nbctl in enumerate(self.nbctl): - if global_cfg.run_ipv4: - addr_set = nbctl.address_set_create(f'sub_as_{suffix}') - nbctl.address_set_add_addrs( - addr_set, [str(p.ip) for p in ports] - ) - self.sub_as[i].append(addr_set) - if global_cfg.run_ipv6: - addr_set = nbctl.address_set_create(f'sub_as_{suffix}6') - nbctl.address_set_add_addrs( - addr_set, [str(p.ip6) for p in ports] - ) - self.sub_as[i].append(addr_set) - return n_sub_pgs - - @ovn_stats.timeit - def default_deny(self, family, az=0): - self.enforce() - - addr_set = f'self.addr_set{family}.name' - self.nbctl[az].acl_add( - self.pg_def_deny_igr[az].name, - 'to-lport', - ACL_DEFAULT_DENY_PRIO, - 'port-group', - f'ip4.src == \\${addr_set} && ' - f'outport == @{self.pg_def_deny_igr[az].name}', - 'drop', - ) - self.nbctl[az].acl_add( - self.pg_def_deny_egr[az].name, - 'to-lport', - ACL_DEFAULT_DENY_PRIO, - 'port-group', - f'ip4.dst == \\${addr_set} && ' - f'inport == @{self.pg_def_deny_egr[az].name}', - 'drop', - ) - self.nbctl[az].acl_add( - self.pg_def_deny_igr[az].name, - 'to-lport', - ACL_DEFAULT_ALLOW_ARP_PRIO, - 'port-group', - f'outport == @{self.pg_def_deny_igr[az].name} && arp', - 'allow', - ) - self.nbctl[az].acl_add( - self.pg_def_deny_egr[az].name, - 'to-lport', - ACL_DEFAULT_ALLOW_ARP_PRIO, - 'port-group', - f'inport == @{self.pg_def_deny_egr[az].name} && arp', - 'allow', - ) - - @ovn_stats.timeit - def allow_within_namespace(self, family, az=0): - self.enforce() - - addr_set = f'self.addr_set{family}.name' - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.src == \\${addr_set} && ' f'outport == @{self.pg[az].name}', - 'allow-related', - ) - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.dst == \\${addr_set} && ' f'inport == @{self.pg[az].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def allow_cross_namespace(self, ns, family): - self.enforce() - - for az, nbctl in enumerate(self.nbctl): - if len(self.ports[az]) == 0: - continue - addr_set = f'self.addr_set{family}.name' - nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.src == \\${addr_set} && ' - f'outport == @{ns.pg[az].name}', - 'allow-related', - ) - ns_addr_set = f'ns.addr_set{family}.name' - nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip4.dst == \\${ns_addr_set} && ' - f'inport == @{self.pg[az].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def allow_sub_namespace(self, src, dst, family, az=0): - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip{family}.src == \\${self.sub_as[az][src].name} && ' - f'outport == @{self.sub_pg[az][dst].name}', - 'allow-related', - ) - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip{family}.dst == \\${self.sub_as[az][dst].name} && ' - f'inport == @{self.sub_pg[az][src].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def allow_from_external( - self, external_ips, include_ext_gw=False, family=4, az=0 - ): - self.enforce() - # If requested, include the ext-gw of the first port in the namespace - # so we can check that this rule is enforced. - if include_ext_gw: - assert len(self.ports) > 0 - if family == 4 and self.ports[az][0].ext_gw: - external_ips.append(self.ports[az][0].ext_gw) - elif family == 6 and self.ports[az][0].ext_gw6: - external_ips.append(self.ports[az][0].ext_gw6) - ips = [str(ip) for ip in external_ips] - self.nbctl[az].acl_add( - self.pg[az].name, - 'to-lport', - ACL_NETPOL_ALLOW_PRIO, - 'port-group', - f'ip.{family} == {{{",".join(ips)}}} && ' - f'outport == @{self.pg[az].name}', - 'allow-related', - ) - - @ovn_stats.timeit - def check_enforcing_internal(self, az=0): - # "Random" check that first pod can reach last pod in the namespace. - if len(self.ports[az]) > 1: - src = self.ports[az][0] - dst = self.ports[az][-1] - worker = src.metadata - if src.ip: - worker.ping_port(self.clusters[az], src, dst.ip) - if src.ip6: - worker.ping_port(self.clusters[az], src, dst.ip6) - - @ovn_stats.timeit - def check_enforcing_external(self, az=0): - if len(self.ports[az]) > 0: - dst = self.ports[az][0] - worker = dst.metadata - worker.ping_external(self.clusters[az], dst) - - @ovn_stats.timeit - def check_enforcing_cross_ns(self, ns, az=0): - if len(self.ports[az]) > 0 and len(ns.ports[az]) > 0: - dst = ns.ports[az][0] - src = self.ports[az][0] - worker = src.metadata - if src.ip and dst.ip: - worker.ping_port(self.clusters[az], src, dst.ip) - if src.ip6 and dst.ip6: - worker.ping_port(self.clusters[az], src, dst.ip6) - - def create_load_balancer(self, az=0): - self.load_balancer = lb.OvnLoadBalancer( - f'lb_{self.name}', self.nbctl[az] - ) - - @ovn_stats.timeit - def provision_vips_to_load_balancers(self, backend_lists, version, az=0): - vip_ns_subnet = DEFAULT_NS_VIP_SUBNET - if version == 6: - vip_ns_subnet = DEFAULT_NS_VIP_SUBNET6 - vip_net = vip_ns_subnet.next(self.clusters[az].n_ns) - n_vips = len(self.load_balancer.vips.keys()) - vip_ip = vip_net.ip.__add__(n_vips + 1) - - if version == 6: - vips = { - f'[{vip_ip + i}]:{DEFAULT_VIP_PORT}': [ - f'[{p.ip6}]:{DEFAULT_BACKEND_PORT}' for p in ports - ] - for i, ports in enumerate(backend_lists) - } - self.load_balancer.add_vips(vips) - else: - vips = { - f'{vip_ip + i}:{DEFAULT_VIP_PORT}': [ - f'{p.ip}:{DEFAULT_BACKEND_PORT}' for p in ports - ] - for i, ports in enumerate(backend_lists) - } - self.load_balancer.add_vips(vips) - - class Cluster: def __init__(self, cluster_cfg, central, brex_cfg, az): # In clustered mode use the first node for provisioning. From cb528e6b654415538a7a8f7a0f070b08a1c58b66 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 22 Sep 2023 18:16:59 +0200 Subject: [PATCH 04/19] ovn_utils: Additional OVN NB command wrappers. Added more methods into OvnNbctl class that map to ovsdbapp commands: * Create 'DHCP_Options' * Set 'options' for 'DHCP_Options' * Enable 'Logical_Switch_Port' * Set IPv4 address on 'Logical_Switch_Port' Signed-off-by: Martin Kalcok --- ovn-tester/ovn_utils.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/ovn-tester/ovn_utils.py b/ovn-tester/ovn_utils.py index 10d02dbf..e84eedb7 100644 --- a/ovn-tester/ovn_utils.py +++ b/ovn-tester/ovn_utils.py @@ -5,6 +5,7 @@ import time from collections import namedtuple from functools import partial +from typing import Dict, Optional import ovsdbapp.schema.open_vswitch.impl_idl as ovs_impl_idl import ovsdbapp.schema.ovn_northbound.impl_idl as nb_impl_idl import ovsdbapp.schema.ovn_southbound.impl_idl as sb_impl_idl @@ -45,6 +46,7 @@ AddressSet = namedtuple('AddressSet', ['name']) LoadBalancer = namedtuple('LoadBalancer', ['name', 'uuid']) LoadBalancerGroup = namedtuple('LoadBalancerGroup', ['name', 'uuid']) +DhcpOptions = namedtuple("DhcpOptions", ["uuid", "cidr"]) DEFAULT_CTL_TIMEOUT = 60 @@ -555,6 +557,23 @@ def ls_port_set_set_options(self, port, options): def ls_port_set_set_type(self, port, lsp_type): self.idl.lsp_set_type(port.name, lsp_type).execute() + def ls_port_enable(self, port: LSPort) -> None: + """Set Logical Switch Port's state to 'enabled'.""" + self.idl.lsp_set_enabled(port.name, True).execute() + + def ls_port_set_ipv4_address(self, port: LSPort, addr: str) -> LSPort: + """Set Logical Switch Port's IPv4 address. + + :param port: LSPort to modify + :param addr: IPv4 address to set + :return: Modified LSPort object with updated 'ip' attribute + """ + addresses = [f"{port.mac} {addr}"] + log.info(f"Setting addresses for port {port.uuid}: {addresses}") + self.idl.lsp_set_addresses(port.uuid, addresses).execute() + + return port._replace(ip=addr) + def port_group_create(self, name): self.idl.pg_add(name).execute() return PortGroup(name=name) @@ -708,6 +727,28 @@ def lb_remove_from_switches(self, lb, switches): for s in switches: txn.add(self.idl.ls_lb_del(s, lb.uuid, if_exists=True)) + def create_dhcp_options( + self, cidr: str, ext_ids: Optional[Dict] = None + ) -> DhcpOptions: + """Create entry in DHCP_Options table. + + :param cidr: DHCP address pool (i.e. '192.168.1.0/24') + :param ext_ids: Optional entries to 'external_ids' column + :return: DhcpOptions object + """ + ext_ids = {} if ext_ids is None else ext_ids + + log.info(f"Creating DHCP Options for {cidr}. External IDs: {ext_ids}") + add_command = self.idl.dhcp_options_add(cidr, **ext_ids) + add_command.execute() + + return DhcpOptions(add_command.result.uuid, cidr) + + def dhcp_options_set_options(self, uuid_: str, options: Dict) -> None: + """Set 'options' column for 'DHCP_Options' entry.""" + log.info(f"Setting DHCP options for {uuid_}: {options}") + self.idl.dhcp_options_set_options(uuid_, **options).execute() + def sync(self, wait="hv", timeout=DEFAULT_CTL_TIMEOUT): with self.idl.transaction( check_error=True, timeout=timeout, wait_type=wait From b7d6a7842149d7d771a0dfcb1756db350dad9c1d Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 22 Sep 2023 18:35:36 +0200 Subject: [PATCH 05/19] ovn_utils: Additional parameters for OvnNbctl methods. Neutron utilizes 'external_ids' a lot. This change adds ability to set them when creating objects like routers/ports/switches. Additionaly some objects also now accept fields like 'extra_config' and 'enabled'. All these additions are optional and should not alter previous behavior of these methods if they are not supplied. misc: couple type hints for easier work with the code. Signed-off-by: Martin Kalcok --- ovn-tester/ovn_utils.py | 88 +++++++++++++++++++++++++++----------- ovn-tester/ovn_workload.py | 7 +-- 2 files changed, 68 insertions(+), 27 deletions(-) diff --git a/ovn-tester/ovn_utils.py b/ovn-tester/ovn_utils.py index e84eedb7..8915c9ca 100644 --- a/ovn-tester/ovn_utils.py +++ b/ovn-tester/ovn_utils.py @@ -5,7 +5,7 @@ import time from collections import namedtuple from functools import partial -from typing import Dict, Optional +from typing import Dict, List, Optional import ovsdbapp.schema.open_vswitch.impl_idl as ovs_impl_idl import ovsdbapp.schema.ovn_northbound.impl_idl as nb_impl_idl import ovsdbapp.schema.ovn_southbound.impl_idl as sb_impl_idl @@ -438,28 +438,53 @@ def set_inactivity_probe(self, value): ("inactivity_probe", value), ).execute() - def lr_add(self, name): + def lr_add(self, name, ext_ids: Optional[Dict] = None): + ext_ids = {} if ext_ids is None else ext_ids + log.info(f'Creating lrouter {name}') - uuid = self.uuid_transaction(partial(self.idl.lr_add, name)) + uuid = self.uuid_transaction( + partial(self.idl.lr_add, name, external_ids=ext_ids) + ) return LRouter(name=name, uuid=uuid) - def lr_port_add(self, router, name, mac, dual_ip=None): + def lr_port_add( + self, router, name, mac, dual_ip=None, ext_ids: Optional[Dict] = None + ): + ext_ids = {} if ext_ids is None else ext_ids networks = [] if dual_ip.ip4 and dual_ip.plen4: networks.append(f'{dual_ip.ip4}/{dual_ip.plen4}') if dual_ip.ip6 and dual_ip.plen6: networks.append(f'{dual_ip.ip6}/{dual_ip.plen6}') - self.idl.lrp_add(router.uuid, name, str(mac), networks).execute() + self.idl.lrp_add( + router.uuid, name, str(mac), networks, external_ids=ext_ids + ).execute() return LRPort(name=name, mac=mac, ip=dual_ip) def lr_port_set_gw_chassis(self, rp, chassis, priority=10): log.info(f'Setting gw chassis {chassis} for router port {rp.name}') self.idl.lrp_set_gateway_chassis(rp.name, chassis, priority).execute() - def ls_add(self, name, net_s): + def ls_add( + self, + name: str, + net_s: DualStackSubnet, + ext_ids: Optional[Dict] = None, + other_config: Optional[Dict] = None, + ) -> LSwitch: + ext_ids = {} if ext_ids is None else ext_ids + other_config = {} if other_config is None else other_config + log.info(f'Creating lswitch {name}') - uuid = self.uuid_transaction(partial(self.idl.ls_add, name)) + uuid = self.uuid_transaction( + partial( + self.idl.ls_add, + name, + external_ids=ext_ids, + other_config=other_config, + ) + ) return LSwitch( name=name, cidr=net_s.n4, @@ -480,17 +505,18 @@ def ls_get_uuid(self, name, timeout): def ls_port_add( self, - lswitch, - name, - router_port=None, - mac=None, - ip=None, - gw=None, - ext_gw=None, - metadata=None, - passive=False, - security=False, - localnet=False, + lswitch: LSwitch, + name: str, + router_port: Optional[LRPort] = None, + mac: Optional[str] = None, + ip: Optional[DualStackIP] = None, + gw: Optional[DualStackIP] = None, + ext_gw: Optional[DualStackIP] = None, + metadata=None, # typehint: ovn_workload.ChassisNode + passive: bool = False, + security: bool = False, + localnet: bool = False, + ext_ids: Optional[Dict] = None, ): columns = dict() if router_port: @@ -514,6 +540,9 @@ def ls_port_add( if security: columns["port_security"] = addresses + if ext_ids is not None: + columns["external_ids"] = ext_ids + uuid = self.uuid_transaction( partial(self.idl.lsp_add, lswitch.uuid, name, **columns) ) @@ -547,7 +576,15 @@ def ls_port_add( def ls_port_del(self, port): self.idl.lsp_del(port.name).execute() - def ls_port_set_set_options(self, port, options): + def ls_port_set_set_options(self, port: LSPort, options: str): + """Set 'options' column for Logical Switch Port. + + :param port: Logical Switch Port to modify + :param options: Space-separated key-value pairs that are set as + options. Keys and values are separated by '='. + i.e.: 'opt1=val1 opt2=val2' + :return: None + """ opts = dict( (k, v) for k, v in (element.split("=") for element in options.split()) @@ -574,14 +611,15 @@ def ls_port_set_ipv4_address(self, port: LSPort, addr: str) -> LSPort: return port._replace(ip=addr) - def port_group_create(self, name): - self.idl.pg_add(name).execute() + def port_group_create(self, name, ext_ids: Optional[Dict] = None): + ext_ids = {} if ext_ids is None else ext_ids + self.idl.pg_add(name, external_ids=ext_ids).execute() return PortGroup(name=name) def port_group_add(self, pg, lport): self.idl.pg_add_ports(pg.name, lport.uuid).execute() - def port_group_add_ports(self, pg, lports): + def port_group_add_ports(self, pg: PortGroup, lports: List[LSPort]): MAX_PORTS_IN_BATCH = 500 for i in range(0, len(lports), MAX_PORTS_IN_BATCH): lports_slice = lports[i : i + MAX_PORTS_IN_BATCH] @@ -620,14 +658,16 @@ def acl_add( entity="switch", match="", verdict="allow", + ext_ids: Optional[Dict] = None, ): + ext_ids = {} if ext_ids is None else ext_ids if entity == "switch": self.idl.acl_add( - name, direction, priority, match, verdict + name, direction, priority, match, verdict, **ext_ids ).execute() else: # "port-group" self.idl.pg_acl_add( - name, direction, priority, match, verdict + name, direction, priority, match, verdict, **ext_ids ).execute() def route_add(self, router, network, gw, policy="dst-ip"): diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 3ec04bc3..83782a2c 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -8,6 +8,7 @@ from collections import namedtuple from collections import defaultdict from datetime import datetime +from typing import Optional log = logging.getLogger(__name__) @@ -282,9 +283,9 @@ def __init__(self, cluster_cfg, central, brex_cfg, az): self.worker_nodes = [] self.cluster_cfg = cluster_cfg self.brex_cfg = brex_cfg - self.nbctl = None - self.sbctl = None - self.icnbctl = None + self.nbctl: Optional[ovn_utils.OvnNbctl] = None + self.sbctl: Optional[ovn_utils.OvnSbctl] = None + self.icnbctl: Optional[ovn_utils.OvnIcNbctl] = None self.az = az protocol = "ssl" if cluster_cfg.enable_ssl else "tcp" From 3b0a2bc4dc405a0cac9b2d519daf47597e7d2f88 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 22 Sep 2023 20:02:37 +0200 Subject: [PATCH 06/19] CMS: Implementation of Openstack CMS This implementation of Openstack CMS is capable of getting OVN to the state before first guest VMs are added. It provisions following: * Project Router * Project's internal network * (Optionally) External network and routing from Project's internal network via the external network. Signed-off-by: Martin Kalcok --- ovn-tester/cms/openstack/__init__.py | 3 + ovn-tester/cms/openstack/openstack.py | 555 ++++++++++++++++++++++++++ 2 files changed, 558 insertions(+) create mode 100644 ovn-tester/cms/openstack/__init__.py create mode 100644 ovn-tester/cms/openstack/openstack.py diff --git a/ovn-tester/cms/openstack/__init__.py b/ovn-tester/cms/openstack/__init__.py new file mode 100644 index 00000000..6356677b --- /dev/null +++ b/ovn-tester/cms/openstack/__init__.py @@ -0,0 +1,3 @@ +from .openstack import OpenStackCloud, OVN_HEATER_CMS_PLUGIN + +__all__ = [OpenStackCloud, OVN_HEATER_CMS_PLUGIN] diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py new file mode 100644 index 00000000..eb4a5c88 --- /dev/null +++ b/ovn-tester/cms/openstack/openstack.py @@ -0,0 +1,555 @@ +import logging +import uuid +from dataclasses import dataclass +from typing import List, Optional, Dict + +import netaddr + +from randmac import RandMac + +from ovn_sandbox import PhysicalNode +from ovn_utils import ( + DualStackSubnet, + LSwitch, + LSPort, + PortGroup, + LRouter, + LRPort, + DualStackIP, +) +from ovn_workload import ChassisNode, Cluster + +log = logging.getLogger(__name__) + +OVN_HEATER_CMS_PLUGIN = 'OpenStackCloud' + + +@dataclass +class NeutronNetwork: + """Group of OVN objects that logically belong to same Neutron network.""" + + network: LSwitch + ports: Dict[str, LSPort] + security_group: Optional[PortGroup] = None + + def __post_init__(self): + self._ip_pool = self.network.cidr.iter_hosts() + self.gateway = self.next_host_ip() + + def next_host_ip(self) -> netaddr.IPAddress: + """Return next available host IP in this network's range.""" + return next(self._ip_pool) + + +class Project: + """Represent network components of an OpenStack Project aka. Tenant.""" + + def __init__( + self, + networks: List[NeutronNetwork] = None, + router: Optional[LRouter] = None, + ): + self.networks: List[NeutronNetwork] = ( + [] if networks is None else networks + ) + self.router = router + self._id = str(uuid.uuid4()) + + @property + def uuid(self) -> str: + """Return arbitrary UUID assigned to this Openstack Project.""" + return self._id + + +class OpenStackCloud(Cluster): + """Representation of Openstack cloud/deployment.""" + + def __init__(self, cluster_cfg, central, brex_cfg, az): + super().__init__(cluster_cfg, central, brex_cfg, az) + self.router = None + self.external_port: Optional[LRPort] = None + self._projects: List[Project] = [] + + self._int_net_base = DualStackSubnet(cluster_cfg.node_net) + self._int_net_offset = 0 + + self._ext_net_pool_index = 0 + + def add_cluster_worker_nodes(self, workers: List[PhysicalNode]): + """Add OpenStack flavored worker nodes as per configuration.""" + # Allocate worker IPs after central and relay IPs. + mgmt_ip = ( + self.cluster_cfg.node_net.ip + + 2 + + len(self.central_nodes) + + len(self.relay_nodes) + ) + + protocol = "ssl" if self.cluster_cfg.enable_ssl else "tcp" + # XXX: To facilitate network nodes we'd have to make bigger change + # to introduce `n_compute_nodes` and `n_network_nodes` into the + # ClusterConfig. I tested it and sems like additional changes + # would be required in `ovn-fake-multinode` repo to handle worker + # creation in similar way as it's done for `n_workers` and + # `n_relays` right now. + network_nodes = [] + compute_nodes = [ + ComputeNode( + workers[i % len(workers)], + f"ovn-scale-{i}", + mgmt_ip + i, + protocol, + True, + ) + for i in range(self.cluster_cfg.n_workers) + ] + self.add_workers(network_nodes + compute_nodes) + + @property + def projects(self) -> List[Project]: + """Return list of Projects that exist in Openstack deployment.""" + return self._projects + + def next_external_ip(self) -> DualStackIP: + """Return next available IP address from configured 'external_net'.""" + self._ext_net_pool_index += 1 + return self.cluster_cfg.external_net.forward(self._ext_net_pool_index) + + def next_int_net(self): + """Return next subnet for project/tenant internal network. + + This method takes subnet configured as 'node_net' in cluster config as + base and each call returns next subnet in line. + """ + network = DualStackSubnet.next( + self._int_net_base, self._int_net_offset + ) + self._int_net_offset += 1 + + return network + + def new_project( + self, gw_nodes: Optional[List[ChassisNode]] = None + ) -> Project: + """Create new Project/Tenant in the Openstack cloud. + + Following things are provisioned for the Project: + * Project router + * Internal network + + If 'gw_nodes' are provided, external network is also provisioned + with default route through the gateway nodes. + + :param gw_nodes: Optional list of Chassis that act like gateways. + :return: New Project object that is automatically also added + to 'self.projects' list. + """ + project = Project() + + project.router = self._create_project_router( + f"provider-router-{project.uuid}" + ) + + if gw_nodes: + self.add_external_network_to_project(project, gw_nodes) + + self.add_internal_network_to_project(project, self.external_port) + self._projects.append(project) + return project + + def add_external_network_to_project( + self, project: Project, gw_nodes: List[ChassisNode] + ) -> None: + """Provision external net to Project that adds external connectivity. + + This method adds new network to project with prefix 'ext_net', + adds it to the Project's router and configures default route on the + router to go through this network. + + :param project: Project to which external network will be added + :param gw_nodes: List of chassis that act as network gateways. + :return: None + """ + + ext_net = self._create_project_net(f"ext_net_{project.uuid}", 1500) + ext_net_port = self._add_metadata_port(ext_net, project.uuid) + provider_port = self._add_provider_network_port(ext_net) + neutron_ext_network = NeutronNetwork( + network=ext_net, + ports={ + ext_net_port.uuid: ext_net_port, + provider_port.uuid: provider_port, + }, + ) + project.networks.append(neutron_ext_network) + + ls_port, lr_port = self._add_router_port_external_gw( + neutron_ext_network, project.router + ) + self.external_port = lr_port + gw_net = DualStackSubnet(netaddr.IPNetwork("0.0.0.0/0")) + + # XXX: ovsdbapp does not allow setting external IDs to static route + # XXX: Setting 'policy' to "" throws "constraint violation" error in + # logs because ovsdbapp does not allow not specifying policy. + # However, the route itself is created successfully with no + # policy, the same way Neutron does it. + self.nbctl.route_add(project.router, gw_net, lr_port.ip, "") + for index, chassis in enumerate(gw_nodes): + self.nbctl.lr_port_set_gw_chassis( + lr_port, chassis.container, index + 1 + ) + + def add_internal_network_to_project( + self, project: Project, snat_port: Optional[LRPort] = None + ) -> None: + """Provision internal net to the Project for guest communication. + + If 'snat_port' is provided. A SNAT rule will be configured for traffic + exiting this network. + + :param project: Project to which internal network will be added. + :param snat_port: Gateway port that should SNAT outgoing traffic. + :return: None + """ + int_net = self._create_project_net(f"int_net_{project.uuid}", 1442) + + int_net_port = self._add_metadata_port(int_net, project.uuid) + security_group = self._create_default_security_group() + neutron_int_network = NeutronNetwork( + network=int_net, + ports={int_net_port.uuid: int_net_port}, + security_group=security_group, + ) + + self._add_network_subnet(network=neutron_int_network) + self._assign_port_ips(neutron_int_network) + project.networks.append(neutron_int_network) + + self._add_router_port_internal( + neutron_int_network, project.router, project + ) + + snated_network = DualStackSubnet(int_net.cidr) + if snat_port is not None: + self.nbctl.nat_add(project.router, snat_port.ip, snated_network) + + def _create_project_net(self, net_name: str, mtu: int = 1500) -> LSwitch: + """Create Logical Switch that represents neutron network. + + This method creates Logical Switch with same parameters as Neutron + would. + """ + switch_ext_ids = { + "neutron:availability_zone_hints": "", + "neutron:mtu": str(mtu), + "neutron:network_name": net_name, + "neutron:revision_number": "1", + } + switch_config = { + "mcast_flood_unregistered": "false", + "mcast_snoop": "false", + "vlan-passthru": "false", + } + + neutron_net_name = f"neutron-{uuid.uuid4()}" + neutron_network = self.nbctl.ls_add( + neutron_net_name, + self.next_int_net(), + ext_ids=switch_ext_ids, + other_config=switch_config, + ) + + return neutron_network + + def _add_network_subnet(self, network: NeutronNetwork) -> None: + """Create DHCP subnet for a network. + + Adding subnet to a network in Neutron results in DHCP_Options being + created in OVN. + """ + external_ids = { + "neutron:revision_number": "0", + "subnet_id": str(uuid.uuid4()), + } + static_routes = ( + f"{{169.254.169.254/32,{network.gateway}, " + f"0.0.0.0/0,{network.gateway}}}" + ) + options = { + "classless_static_route": static_routes, + "dns_server": "{1.1.1.1}", + "lease_time": "43200", + "mtu": "1442", + "router": str(network.gateway), + "server_id": str(network.gateway), + "server_mac": str(RandMac()), + } + dhcp_options = self.nbctl.create_dhcp_options( + str(network.network.cidr), ext_ids=external_ids + ) + + self.nbctl.dhcp_options_set_options(dhcp_options.uuid, options) + + def _assign_port_ips(self, network: NeutronNetwork) -> None: + """Assign IPs to each LS port associated with the Network.""" + for uuid_, port in network.ports.items(): + if port.ip is None or port.ip == "unknown": + updated_port = self.nbctl.ls_port_set_ipv4_address( + port, str(network.next_host_ip()) + ) + network.ports[uuid_] = updated_port + # XXX: Unable to update external_ids + + def _add_metadata_port(self, network: LSwitch, project_id: str) -> LSPort: + """Create metadata port in LSwitch with Neutron's external IDs.""" + port_ext_ids = { + "neutron:cidrs": "", + "neutron:device_id": f"ovnmeta-{network.uuid}", + "neutron:device_owner": "network:distributed", + "neutron:network_name": network.name, + "neutron:port_name": "", + "neutron:project_id": project_id, + "neutron:revision_number": "1", + "neutron:security_group_ids": "", + "neutron:subnet_pool_addr_scope4": "", + "neutron:subnet_pool_addr_scope6": "", + } + + port = self.nbctl.ls_port_add( + lswitch=network, + name=str(uuid.uuid4()), + mac=str(RandMac()), + ext_ids=port_ext_ids, + ) + self.nbctl.ls_port_set_set_type(port, "localport") + self.nbctl.ls_port_set_set_options(port, "requested-chassis=") + self.nbctl.ls_port_enable(port) + + return port + + def _add_provider_network_port(self, network: LSwitch) -> LSPort: + """Add port to Logical Switch that represents connection to ext. net""" + options = ( + "mcast_flood=false, mcast_flood_reports=true, " + "network_name=physnet1" + ) + port = self.nbctl.ls_port_add( + lswitch=network, + name=f"provnet-{uuid.uuid4()}", + localnet=True, + ) + self.nbctl.ls_port_set_set_type(port, "localnet") + self.nbctl.ls_port_set_set_options(port, options) + + return port + + def _add_router_port_internal( + self, neutron_net: NeutronNetwork, router: LRouter, project: Project + ) -> (LSPort, LRPort): + """Add a pair of ports that connect router to the internal network. + + The Router Port is automatically assigned network's gateway IP. + + :param neutron_net: Neutron network that represents internal network + :param router: Router to which the network is connected + :param project: Project to which 'neutron_net' belongs + :return: The pair of Logical Switch Port and Logical Router Port + """ + port_ip = DualStackIP( + neutron_net.gateway, neutron_net.network.cidr.prefixlen, None, None + ) + return self._add_router_port( + neutron_net.network, router, port_ip, project.uuid, False + ) + + def _add_router_port_external_gw( + self, neutron_net: NeutronNetwork, router: LRouter + ) -> (LSPort, LRPort): + """Add a pair of ports that connect router to the external network. + + The Router Port is assigned IP form external network's range. + + :param neutron_net: Neutron network that represents external network + :param router: Router to which the network is connected + :return: The pair of Logical Switch Port and Logical Router Port + """ + port_ip = self.next_external_ip() + return self._add_router_port( + neutron_net.network, router, port_ip, "", True + ) + + def _add_router_port( + self, + network: LSwitch, + router: LRouter, + port_ip: DualStackIP, + project_id: str = "", + is_gw: bool = False, + ) -> (LSPort, LRPort): + """Add a pair of ports that connect Logical Router and Logical Switch. + + :param network: Logical Switch to which a Logical Switch Port is added + :param router: Logical Router to which a Logical Router Port is added + :param port_ip: IP assigned to the router port + :param project_id: Optional Openstack Project ID that's associated with + the network. + :param is_gw: Set to True if Router port is connected to external + network and should act as gateway. + :return: The pair of Logical Switch Port and Logical Router Port + """ + subnet_id = str(uuid.uuid4()) + port_name = uuid.uuid4() + router_port_name = f"lrp-{port_name}" + device_owner = "network:router_gateway" if is_gw else "" + # XXX: neutron adds "neutron-" prefix to router name (which is UUID), + # but for the ports' external IDs we want to extract just the + # UUID. + router_id = router.name.lstrip("neutron-") + + lsp_external_ids = { + "neutron:cidrs": str(self.cluster_cfg.external_net.n4), + "neutron:device_id": router_id, + "neutron:device_owner": device_owner, + "neutron:network_name": network.name, + "neutron:port_name": "", + "neutron:project_id": project_id, + "neutron:revision_number": "1", + "neutron:security_group_ids": "", + "neutron:subnet_pool_addr_scope4": "", + "neutron:subnet_pool_addr_scope6": "", + } + lrp_external_ids = { + "neutron:network_name": network.name, + "neutron:revision_number": "1", + "neutron:router_name": router_id, + "neutron:subnet_ids": subnet_id, + } + + lr_port = self.nbctl.lr_port_add( + router, router_port_name, str(RandMac()), port_ip, lrp_external_ids + ) + + ls_port = self.nbctl.ls_port_add( + lswitch=network, + name=str(port_name), + router_port=lr_port, + ext_ids=lsp_external_ids, + ) + self.nbctl.ls_port_enable(ls_port) + + if is_gw: + lsp_options = ( + "exclude-lb-vips-from-garp=true, " + "nat-addresses=router, " + f"router-port={router_port_name}" + ) + self.nbctl.ls_port_set_set_options(ls_port, lsp_options) + + return ls_port, lr_port + + def _create_default_security_group(self) -> PortGroup: + """Create default security group for the project. + + This method provisions Port Group and default ACL rules. + """ + neutron_security_group = str(uuid.uuid4()) + pg_name = f"pg_{neutron_security_group}".replace("-", "_") + pg_ext_ids = {"neutron:security_group_id": neutron_security_group} + port_group = self.nbctl.port_group_create(pg_name, ext_ids=pg_ext_ids) + + in_rules = [ + f"inport == @{pg_name} && ip4", + f"inport == @{pg_name} && ip6", + ] + out_rules = [ + f"outport == @{pg_name} && ip4 && ip4.src == $pg_{pg_name}_ip4", + f"outport == @{pg_name} && ip6 && ip6.src == $pg_{pg_name}_ip6", + ] + + for rule in in_rules: + ext_ids = {"neutron:security_group_rule_id": str(uuid.uuid4())} + self.nbctl.acl_add( + name=port_group.name, + direction="from-lport", + priority=1002, + entity="port-group", + match=rule, + verdict="allow-related", + ext_ids=ext_ids, + ) + + for rule in out_rules: + ext_ids = {"neutron:security_group_rule_id": str(uuid.uuid4())} + self.nbctl.acl_add( + name=port_group.name, + direction="to-lport", + priority=1002, + entity="port-group", + match=rule, + verdict="allow-related", + ext_ids=ext_ids, + ) + + return port_group + + def _create_project_router(self, name: str) -> LRouter: + """Create Logical Router in the same way as Neutron would.""" + options = { + "always_learn_from_arp_request": "false", + "dynamic_neigh_routers": "true", + } + external_ids = { + "neutron:availability_zone_hints": "", + "neutron:gw_port_id": "", + "neutron:revision_number": "1", + "neutron:router_name": name, + } + router = self.nbctl.lr_add(f"neutron-{uuid.uuid4()}", external_ids) + self.nbctl.lr_set_options(router, options) + + return router + + +class NetworkNode(ChassisNode): + """Represent a network node, a node with OVS/OVN but no VMs.""" + + def __init__( + self, + phys_node, + container, + mgmt_ip, + protocol, + is_gateway: bool = True, + ): + super().__init__(phys_node, container, mgmt_ip, protocol) + self.is_gateway = is_gateway + + def configure(self, physical_net): + pass + + def provision(self, cluster: OpenStackCloud) -> None: + """Connect Chassis to OVN Central.""" + self.connect(cluster.get_relay_connection_string()) + + +class ComputeNode(ChassisNode): + """Represent a compute node, a node with OVS/OVN as well as VMs.""" + + def __init__( + self, + phys_node, + container, + mgmt_ip, + protocol, + is_gateway: bool = True, + ): + super().__init__(phys_node, container, mgmt_ip, protocol) + self.is_gateway = is_gateway + + def configure(self, physical_net): + pass + + def provision(self, cluster: OpenStackCloud) -> None: + """Connect Chassis to OVN Central.""" + self.connect(cluster.get_relay_connection_string()) From fc8e54d0b17ce2a2035d80d0aab506d5e27e73f6 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 22 Sep 2023 20:10:02 +0200 Subject: [PATCH 07/19] CMS Openstack: Basic Openstack cloud scenario Very basic Openstack test scenario that brings up three compute nodes and provisions one Project with internal network and external connectivity. Co-Authored-by: Dumitru Ceara Signed-off-by: Dumitru Ceara Signed-off-by: Martin Kalcok --- .../cms/openstack/tests/base_openstack.py | 32 +++++++++++++++++++ test-scenarios/openstack.yml | 11 +++++++ 2 files changed, 43 insertions(+) create mode 100644 ovn-tester/cms/openstack/tests/base_openstack.py create mode 100644 test-scenarios/openstack.yml diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py new file mode 100644 index 00000000..bbcd86cf --- /dev/null +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -0,0 +1,32 @@ +import logging + +from typing import List + +from ovn_ext_cmd import ExtCmd +from ovn_context import Context +from ovn_workload import ChassisNode + +from cms.openstack import OpenStackCloud + +log = logging.getLogger(__name__) + + +class BaseOpenstack(ExtCmd): + def __init__(self, config, cluster, global_cfg): + super().__init__(config, cluster) + + def run(self, clouds: List[OpenStackCloud], global_cfg): + # create ovn topology + with Context(clouds, "base_cluster_bringup", len(clouds)) as ctx: + for i in ctx: + ovn = clouds[i] + worker_count = len(ovn.worker_nodes) + for i in range(worker_count): + worker_node: ChassisNode = ovn.worker_nodes[i] + log.info( + f"Provisioning {worker_node.__class__.__name__} " + f"({i+1}/{worker_count})" + ) + worker_node.provision(ovn) + + _ = ovn.new_project(gw_nodes=ovn.worker_nodes) diff --git a/test-scenarios/openstack.yml b/test-scenarios/openstack.yml new file mode 100644 index 00000000..70ed8b8d --- /dev/null +++ b/test-scenarios/openstack.yml @@ -0,0 +1,11 @@ +global: + log_cmds: false + cms_name: openstack + +cluster: + clustered_db: true + log_txns_db: true + n_workers: 3 + +base_openstack: + foo: bar \ No newline at end of file From 9b8d9310c890b54efb6a4aa4194bab73602489f4 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 26 Sep 2023 13:35:12 +0200 Subject: [PATCH 08/19] OS Test: Make number of projects configurable. Test config option 'n_projects' allows specifying how many projects will be created during the test. Signed-off-by: Martin Kalcok --- ovn-tester/cms/openstack/tests/base_openstack.py | 12 +++++++++++- test-scenarios/openstack.yml | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py index bbcd86cf..753acefc 100644 --- a/ovn-tester/cms/openstack/tests/base_openstack.py +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -1,5 +1,6 @@ import logging +from dataclasses import dataclass from typing import List from ovn_ext_cmd import ExtCmd @@ -11,9 +12,17 @@ log = logging.getLogger(__name__) +@dataclass +class BaseOpenstackConfig: + + n_projects: int = 1 + + class BaseOpenstack(ExtCmd): def __init__(self, config, cluster, global_cfg): super().__init__(config, cluster) + test_config = config.get("base_openstack") + self.config = BaseOpenstackConfig(**test_config) def run(self, clouds: List[OpenStackCloud], global_cfg): # create ovn topology @@ -29,4 +38,5 @@ def run(self, clouds: List[OpenStackCloud], global_cfg): ) worker_node.provision(ovn) - _ = ovn.new_project(gw_nodes=ovn.worker_nodes) + for _ in range(self.config.n_projects): + _ = ovn.new_project(gw_nodes=ovn.worker_nodes) diff --git a/test-scenarios/openstack.yml b/test-scenarios/openstack.yml index 70ed8b8d..42769a5a 100644 --- a/test-scenarios/openstack.yml +++ b/test-scenarios/openstack.yml @@ -8,4 +8,4 @@ cluster: n_workers: 3 base_openstack: - foo: bar \ No newline at end of file + n_projects: 3 \ No newline at end of file From 69cf5d805ab14f1b1fac52091b1fdcad7eef44a5 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 26 Sep 2023 15:29:34 +0200 Subject: [PATCH 09/19] CMS Openstack: Evenly distribute GW chassis Attempt to evenly distribute load when assigning Gateway Chassis to project routers. Signed-off-by: Martin Kalcok --- ovn-tester/cms/openstack/openstack.py | 53 ++++++++++++++++--- .../cms/openstack/tests/base_openstack.py | 3 +- test-scenarios/openstack.yml | 3 +- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py index eb4a5c88..c3c99087 100644 --- a/ovn-tester/cms/openstack/openstack.py +++ b/ovn-tester/cms/openstack/openstack.py @@ -1,5 +1,7 @@ import logging +import random import uuid + from dataclasses import dataclass from typing import List, Optional, Dict @@ -64,6 +66,8 @@ def uuid(self) -> str: class OpenStackCloud(Cluster): """Representation of Openstack cloud/deployment.""" + MAX_GW_PER_ROUTER = 5 + def __init__(self, cluster_cfg, central, brex_cfg, az): super().__init__(cluster_cfg, central, brex_cfg, az) self.router = None @@ -128,19 +132,19 @@ def next_int_net(self): return network - def new_project( - self, gw_nodes: Optional[List[ChassisNode]] = None - ) -> Project: + def new_project(self, gw_nodes: int = 0) -> Project: """Create new Project/Tenant in the Openstack cloud. Following things are provisioned for the Project: * Project router * Internal network - If 'gw_nodes' are provided, external network is also provisioned - with default route through the gateway nodes. + If 'gw_nodes' count is more than 0, external network is also + provisioned with default route through the gateway nodes. Gateway + nodes and their priority are selected at random from all + available 'worker_nodes'. - :param gw_nodes: Optional list of Chassis that act like gateways. + :param gw_nodes: Number of gateway chassis to use. :return: New Project object that is automatically also added to 'self.projects' list. """ @@ -151,7 +155,10 @@ def new_project( ) if gw_nodes: - self.add_external_network_to_project(project, gw_nodes) + self.add_external_network_to_project( + project, + self._get_gateway_chassis(gw_nodes), + ) self.add_internal_network_to_project(project, self.external_port) self._projects.append(project) @@ -234,6 +241,38 @@ def add_internal_network_to_project( if snat_port is not None: self.nbctl.nat_add(project.router, snat_port.ip, snated_network) + def _get_gateway_chassis(self, count: int = 1) -> List[ChassisNode]: + """Return list of Gateway Chassis with size defined by 'count'. + + Chassis are picked at random from all available 'worker_nodes' to + attempt to evenly distribute gateway loads. + + Parameter 'count' can't be larger than number of all available gateways + or larger than 5 which is hardcoded maximum of gateways per router in + Neutron. + """ + warn = "" + worker_count = len(self.worker_nodes) + if count > worker_count: + count = worker_count + warn += ( + f"{count} Gateway chassis requested but only " + f"{worker_count} available.\n" + ) + + if count > self.MAX_GW_PER_ROUTER: + count = self.MAX_GW_PER_ROUTER + warn += ( + f"Maximum number of gateways per router " + f"is {self.MAX_GW_PER_ROUTER}\n" + ) + + if warn: + warn += f"Using only {count} Gateways per router." + log.warning(warn) + + return random.sample(self.worker_nodes, count) + def _create_project_net(self, net_name: str, mtu: int = 1500) -> LSwitch: """Create Logical Switch that represents neutron network. diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py index 753acefc..c897948b 100644 --- a/ovn-tester/cms/openstack/tests/base_openstack.py +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -16,6 +16,7 @@ class BaseOpenstackConfig: n_projects: int = 1 + n_gws_per_project: int = 1 class BaseOpenstack(ExtCmd): @@ -39,4 +40,4 @@ def run(self, clouds: List[OpenStackCloud], global_cfg): worker_node.provision(ovn) for _ in range(self.config.n_projects): - _ = ovn.new_project(gw_nodes=ovn.worker_nodes) + _ = ovn.new_project(gw_nodes=self.config.n_gws_per_project) diff --git a/test-scenarios/openstack.yml b/test-scenarios/openstack.yml index 42769a5a..d159c9b7 100644 --- a/test-scenarios/openstack.yml +++ b/test-scenarios/openstack.yml @@ -8,4 +8,5 @@ cluster: n_workers: 3 base_openstack: - n_projects: 3 \ No newline at end of file + n_projects: 3 + n_gws_per_project: 3 \ No newline at end of file From 64167cb2b15111f49dd12068b860f3a14c111f5f Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Sat, 30 Sep 2023 10:47:27 +0200 Subject: [PATCH 10/19] Cluster: Add full-mesh pinging option New method adds ability to perform full-mesh pings between list of ports. Signed-off-by: Martin Kalcok --- ovn-tester/ovn_workload.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 83782a2c..c482938e 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -8,7 +8,7 @@ from collections import namedtuple from collections import defaultdict from datetime import datetime -from typing import Optional +from typing import List, Optional log = logging.getLogger(__name__) @@ -408,6 +408,24 @@ def ping_ports(self, ports): for w, ports in ports_per_worker.items(): w.ping_ports(self, ports) + def mesh_ping_ports(self, ports: List[ovn_utils.LSPort]) -> None: + """Perform full-mesh ping test between ports.""" + all_ips = [port.ip for port in ports] + + for port in ports: + chassis: Optional[ChassisNode] = port.metadata + if chassis is None: + log.error( + f"Port {port.name} is missing 'metadata' attribute. " + f"Can't perform ping." + ) + continue + + for dest_ip in all_ips: + if dest_ip == port.ip: + continue + chassis.ping_port(self, port, dest_ip) + def select_worker_for_port(self): self.last_selected_worker += 1 self.last_selected_worker %= len(self.worker_nodes) From e4e6aab84e5090f52ad9e7a0be92a399ee579f52 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Sat, 30 Sep 2023 12:15:06 +0200 Subject: [PATCH 11/19] CMS Openstack: Add ability to simulate VMs VMs can now be provisioned and assigned to projects. Co-Authored-by: Dumitru Ceara Signed-off-by: Dumitru Ceara Signed-off-by: Martin Kalcok --- ovn-tester/cms/openstack/openstack.py | 109 ++++++++++++++++-- .../cms/openstack/tests/base_openstack.py | 23 +++- test-scenarios/openstack.yml | 5 +- 3 files changed, 121 insertions(+), 16 deletions(-) diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py index c3c99087..38a3ce85 100644 --- a/ovn-tester/cms/openstack/openstack.py +++ b/ovn-tester/cms/openstack/openstack.py @@ -3,6 +3,7 @@ import uuid from dataclasses import dataclass +from itertools import cycle from typing import List, Optional, Dict import netaddr @@ -32,6 +33,7 @@ class NeutronNetwork: network: LSwitch ports: Dict[str, LSPort] + name: str security_group: Optional[PortGroup] = None def __post_init__(self): @@ -48,13 +50,14 @@ class Project: def __init__( self, - networks: List[NeutronNetwork] = None, + int_net: NeutronNetwork = None, + ext_net: NeutronNetwork = None, router: Optional[LRouter] = None, ): - self.networks: List[NeutronNetwork] = ( - [] if networks is None else networks - ) + self.int_net = int_net + self.ext_net = ext_net self.router = router + self.vm_ports: List[LSPort] = [] self._id = str(uuid.uuid4()) @property @@ -79,6 +82,17 @@ def __init__(self, cluster_cfg, central, brex_cfg, az): self._ext_net_pool_index = 0 + def add_workers(self, workers: List[PhysicalNode]): + """Expand parent method to update cycled list.""" + super().add_workers(workers) + self._compute_nodes = cycle( + [ + node + for node in self.worker_nodes + if isinstance(node, ComputeNode) + ] + ) + def add_cluster_worker_nodes(self, workers: List[PhysicalNode]): """Add OpenStack flavored worker nodes as per configuration.""" # Allocate worker IPs after central and relay IPs. @@ -132,6 +146,10 @@ def next_int_net(self): return network + def select_worker_for_port(self) -> 'ComputeNode': + """Cyclically return compute nodes available in cluster.""" + return next(self._compute_nodes) + def new_project(self, gw_nodes: int = 0) -> Project: """Create new Project/Tenant in the Openstack cloud. @@ -177,8 +195,8 @@ def add_external_network_to_project( :param gw_nodes: List of chassis that act as network gateways. :return: None """ - - ext_net = self._create_project_net(f"ext_net_{project.uuid}", 1500) + ext_net_name = f"ext_net_{project.uuid}" + ext_net = self._create_project_net(ext_net_name, 1500) ext_net_port = self._add_metadata_port(ext_net, project.uuid) provider_port = self._add_provider_network_port(ext_net) neutron_ext_network = NeutronNetwork( @@ -187,8 +205,9 @@ def add_external_network_to_project( ext_net_port.uuid: ext_net_port, provider_port.uuid: provider_port, }, + name=ext_net_name, ) - project.networks.append(neutron_ext_network) + project.ext_net = neutron_ext_network ls_port, lr_port = self._add_router_port_external_gw( neutron_ext_network, project.router @@ -219,19 +238,21 @@ def add_internal_network_to_project( :param snat_port: Gateway port that should SNAT outgoing traffic. :return: None """ - int_net = self._create_project_net(f"int_net_{project.uuid}", 1442) + int_net_name = f"int_net_{project.uuid}" + int_net = self._create_project_net(int_net_name, 1442) int_net_port = self._add_metadata_port(int_net, project.uuid) security_group = self._create_default_security_group() neutron_int_network = NeutronNetwork( network=int_net, ports={int_net_port.uuid: int_net_port}, + name=int_net_name, security_group=security_group, ) self._add_network_subnet(network=neutron_int_network) self._assign_port_ips(neutron_int_network) - project.networks.append(neutron_int_network) + project.int_net = neutron_int_network self._add_router_port_internal( neutron_int_network, project.router, project @@ -241,6 +262,15 @@ def add_internal_network_to_project( if snat_port is not None: self.nbctl.nat_add(project.router, snat_port.ip, snated_network) + def add_vm_to_project(self, project: Project, vm_name: str): + compute = self.select_worker_for_port() + vm_port = self._add_vm_port( + project.int_net, project.uuid, compute, vm_name + ) + compute.bind_port(vm_port) + + project.vm_ports.append(vm_port) + def _get_gateway_chassis(self, count: int = 1) -> List[ChassisNode]: """Return list of Gateway Chassis with size defined by 'count'. @@ -340,6 +370,65 @@ def _assign_port_ips(self, network: NeutronNetwork) -> None: network.ports[uuid_] = updated_port # XXX: Unable to update external_ids + def _add_vm_port( + self, + neutron_net: NeutronNetwork, + project_id: str, + chassis: 'ComputeNode', + vm_name: str, + ) -> LSPort: + """Create port that will represent interface of a VM. + + :param neutron_net: Network in which the port will be created. + :param project_id: ID of a project to which the VM belongs. + :param chassis: Chassis on which the port will be provisioned. + :param vm_name: VM name that's used to derive port name. Beware that + due to the port naming convention and limits on + interface name length, this name can't be longer than + 12 characters. + :return: LSPort representing VM's network interface + """ + port_name = f"lp-{vm_name}" + if len(port_name) > 15: + raise RuntimeError( + f"Maximum port name length is 15. Port {port_name} is too " + f"long. Consider using shorter VM name." + ) + port_addr = neutron_net.next_host_ip() + net_addr: netaddr.IPNetwork = neutron_net.network.cidr + port_ext_ids = { + "neutron:cidrs": f"{port_addr}/{net_addr.prefixlen}", + "neutron:device_id": str(uuid.uuid4()), + "neutron:device_owner": "compute:nova", + "neutron:network_name": neutron_net.name, + "neutron:port_name": "", + "neutron:project_id": project_id, + "neutron:revision_number": "1", + "neutron:security_group_ids": neutron_net.security_group.name, + "neutron:subnet_pool_addr_scope4": "", + "neutron:subnet_pool_addr_scope6": "", + } + port_options = ( + f"mcast_flood_reports=true " + f"requested-chassis={chassis.container}" + ) + ls_port = self.nbctl.ls_port_add( + lswitch=neutron_net.network, + name=port_name, + mac=str(RandMac()), + ip=DualStackIP(port_addr, net_addr.prefixlen, None, None), + ext_ids=port_ext_ids, + gw=DualStackIP(neutron_net.gateway, None, None, None), + metadata=chassis, + ) + self.nbctl.ls_port_set_set_options(ls_port, port_options) + self.nbctl.ls_port_enable(ls_port) + self.nbctl.port_group_add_ports( + pg=neutron_net.security_group, lports=[ls_port] + ) + + return ls_port + def _add_metadata_port(self, network: LSwitch, project_id: str) -> LSPort: """Create metadata port in LSwitch with Neutron's external IDs.""" port_ext_ids = { @@ -370,7 +459,7 @@ def _add_metadata_port(self, network: LSwitch, project_id: str) -> LSPort: def _add_provider_network_port(self, network: LSwitch) -> LSPort: """Add port to Logical Switch that represents connection to ext. net""" options = ( - "mcast_flood=false, mcast_flood_reports=true, " + "mcast_flood=false mcast_flood_reports=true " "network_name=physnet1" ) port = self.nbctl.ls_port_add( diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py index c897948b..967bba53 100644 --- a/ovn-tester/cms/openstack/tests/base_openstack.py +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -14,9 +14,9 @@ @dataclass class BaseOpenstackConfig: - n_projects: int = 1 n_gws_per_project: int = 1 + n_vms_per_project: int = 3 class BaseOpenstack(ExtCmd): @@ -27,7 +27,7 @@ def __init__(self, config, cluster, global_cfg): def run(self, clouds: List[OpenStackCloud], global_cfg): # create ovn topology - with Context(clouds, "base_cluster_bringup", len(clouds)) as ctx: + with Context(clouds, "base_openstack_bringup", len(clouds)) as ctx: for i in ctx: ovn = clouds[i] worker_count = len(ovn.worker_nodes) @@ -39,5 +39,20 @@ def run(self, clouds: List[OpenStackCloud], global_cfg): ) worker_node.provision(ovn) - for _ in range(self.config.n_projects): - _ = ovn.new_project(gw_nodes=self.config.n_gws_per_project) + with Context(clouds, "base_openstack_provision", len(clouds)) as ctx: + for i in ctx: + ovn = clouds[i] + for _ in range(self.config.n_projects): + _ = ovn.new_project(gw_nodes=self.config.n_gws_per_project) + + for project in ovn.projects: + for index in range(self.config.n_vms_per_project): + ovn.add_vm_to_project( + project, f"{project.uuid[:6]}-{index}" + ) + + with Context(clouds, "base_openstack", len(clouds)) as ctx: + for i in ctx: + ovn = clouds[i] + for project in ovn.projects: + ovn.mesh_ping_ports(project.vm_ports) diff --git a/test-scenarios/openstack.yml b/test-scenarios/openstack.yml index d159c9b7..7ea9c1ed 100644 --- a/test-scenarios/openstack.yml +++ b/test-scenarios/openstack.yml @@ -8,5 +8,6 @@ cluster: n_workers: 3 base_openstack: - n_projects: 3 - n_gws_per_project: 3 \ No newline at end of file + n_projects: 20 + n_gws_per_project: 3 + n_vms_per_project: 10 \ No newline at end of file From 1e987a14b10accd04ce03983deca504580d407be Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Mon, 2 Oct 2023 15:04:28 +0200 Subject: [PATCH 12/19] CMS Openstack: Decouple project and ext. network Based on the feedback, Openstack clouds usually contain only one (or few) external networks. This change removes automatic per-project external network creation and allows sharing external network between multiple projects. Signed-off-by: Martin Kalcok --- ovn-tester/cms/openstack/__init__.py | 12 +- ovn-tester/cms/openstack/openstack.py | 109 ++++++++++++------ .../cms/openstack/tests/base_openstack.py | 9 +- 3 files changed, 92 insertions(+), 38 deletions(-) diff --git a/ovn-tester/cms/openstack/__init__.py b/ovn-tester/cms/openstack/__init__.py index 6356677b..c8e6f9d0 100644 --- a/ovn-tester/cms/openstack/__init__.py +++ b/ovn-tester/cms/openstack/__init__.py @@ -1,3 +1,11 @@ -from .openstack import OpenStackCloud, OVN_HEATER_CMS_PLUGIN +from .openstack import ( + OpenStackCloud, + OVN_HEATER_CMS_PLUGIN, + ExternalNetworkSpec, +) -__all__ = [OpenStackCloud, OVN_HEATER_CMS_PLUGIN] +__all__ = [ + OpenStackCloud, + OVN_HEATER_CMS_PLUGIN, + ExternalNetworkSpec, +] diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py index 38a3ce85..cdb8704c 100644 --- a/ovn-tester/cms/openstack/openstack.py +++ b/ovn-tester/cms/openstack/openstack.py @@ -45,6 +45,20 @@ def next_host_ip(self) -> netaddr.IPAddress: return next(self._ip_pool) +@dataclass +class ExternalNetworkSpec: + """Information required to connect external network to Project's router. + :param neutron_net: Object of already provisioned NeutronNetwork + :param num_gw_nodes: Number of Chassis to use as gateways for this + network. Note that the number can't be greater than 5 + and can't be greater than total number of Chassis + available in the system. + """ + + neutron_net: NeutronNetwork + num_gw_nodes: int + + class Project: """Represent network components of an OpenStack Project aka. Tenant.""" @@ -74,7 +88,6 @@ class OpenStackCloud(Cluster): def __init__(self, cluster_cfg, central, brex_cfg, az): super().__init__(cluster_cfg, central, brex_cfg, az) self.router = None - self.external_port: Optional[LRPort] = None self._projects: List[Project] = [] self._int_net_base = DualStackSubnet(cluster_cfg.node_net) @@ -150,56 +163,53 @@ def select_worker_for_port(self) -> 'ComputeNode': """Cyclically return compute nodes available in cluster.""" return next(self._compute_nodes) - def new_project(self, gw_nodes: int = 0) -> Project: + def new_project(self, ext_net: Optional[ExternalNetworkSpec]) -> Project: """Create new Project/Tenant in the Openstack cloud. Following things are provisioned for the Project: * Project router * Internal network - If 'gw_nodes' count is more than 0, external network is also - provisioned with default route through the gateway nodes. Gateway - nodes and their priority are selected at random from all - available 'worker_nodes'. + If 'ext_net' is provided, this external network will be connected + to the project's router and default gateway route will be configured + through this network. - :param gw_nodes: Number of gateway chassis to use. + :param ext_net: External network to be connected to the Project. :return: New Project object that is automatically also added to 'self.projects' list. """ project = Project() + gateway_port = None project.router = self._create_project_router( f"provider-router-{project.uuid}" ) - - if gw_nodes: - self.add_external_network_to_project( - project, - self._get_gateway_chassis(gw_nodes), + if ext_net is not None: + gateway_port = self.connect_external_network_to_project( + project, ext_net ) - self.add_internal_network_to_project(project, self.external_port) + self.add_internal_network_to_project(project, gateway_port) self._projects.append(project) return project - def add_external_network_to_project( - self, project: Project, gw_nodes: List[ChassisNode] - ) -> None: - """Provision external net to Project that adds external connectivity. - - This method adds new network to project with prefix 'ext_net', - adds it to the Project's router and configures default route on the - router to go through this network. + def new_external_network( + self, provider_network: str = "physnet1" + ) -> NeutronNetwork: + """Provision external network that can be added as gw to Projects. - :param project: Project to which external network will be added - :param gw_nodes: List of chassis that act as network gateways. - :return: None + :param provider_network: Name of the provider network used when + creating provider port. + :return: NeutronNetwork object representing external network. """ - ext_net_name = f"ext_net_{project.uuid}" - ext_net = self._create_project_net(ext_net_name, 1500) - ext_net_port = self._add_metadata_port(ext_net, project.uuid) - provider_port = self._add_provider_network_port(ext_net) - neutron_ext_network = NeutronNetwork( + ext_net_uuid = uuid.uuid4() + ext_net_name = f"ext_net_{ext_net_uuid}" + ext_net = self._create_project_net(f"ext_net_{ext_net_uuid}", 1500) + ext_net_port = self._add_metadata_port(ext_net, str(ext_net_uuid)) + provider_port = self._add_provider_network_port( + ext_net, provider_network + ) + return NeutronNetwork( network=ext_net, ports={ ext_net_port.uuid: ext_net_port, @@ -207,10 +217,28 @@ def add_external_network_to_project( }, name=ext_net_name, ) - project.ext_net = neutron_ext_network + + def connect_external_network_to_project( + self, + project: Project, + external_network: ExternalNetworkSpec, + ) -> LRPort: + """Connect external net to Project that adds external connectivity. + This method takes existing Neutron network, connects it to the + Project's router and configures default route on the router to go + through this network. + + Gateway Chassis will be picked at random from available Chassis + nodes based on the number of gateways specified in 'external_network' + specification. + + :param project: Project to which external network will be added + :param external_network: External network that will be connected. + :return: Logical Router Port that connects to the external network. + """ ls_port, lr_port = self._add_router_port_external_gw( - neutron_ext_network, project.router + external_network.neutron_net, project.router ) self.external_port = lr_port gw_net = DualStackSubnet(netaddr.IPNetwork("0.0.0.0/0")) @@ -221,11 +249,17 @@ def add_external_network_to_project( # However, the route itself is created successfully with no # policy, the same way Neutron does it. self.nbctl.route_add(project.router, gw_net, lr_port.ip, "") + + gw_nodes = self._get_gateway_chassis(external_network.num_gw_nodes) for index, chassis in enumerate(gw_nodes): self.nbctl.lr_port_set_gw_chassis( lr_port, chassis.container, index + 1 ) + project.ext_net = external_network + + return lr_port + def add_internal_network_to_project( self, project: Project, snat_port: Optional[LRPort] = None ) -> None: @@ -456,11 +490,18 @@ def _add_metadata_port(self, network: LSwitch, project_id: str) -> LSPort: return port - def _add_provider_network_port(self, network: LSwitch) -> LSPort: - """Add port to Logical Switch that represents connection to ext. net""" + def _add_provider_network_port( + self, network: LSwitch, network_name: str + ) -> LSPort: + """Add port to Logical Switch that represents connection to ext. net + :param network: Network (LSwitch) in which the port will be created. + :param network_name: Name of the provider network (used when setting + provider port options) + :return: LSPort representing provider port + """ options = ( "mcast_flood=false mcast_flood_reports=true " - "network_name=physnet1" + f"network_name={network_name}" ) port = self.nbctl.ls_port_add( lswitch=network, diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py index 967bba53..e20fa778 100644 --- a/ovn-tester/cms/openstack/tests/base_openstack.py +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -7,7 +7,7 @@ from ovn_context import Context from ovn_workload import ChassisNode -from cms.openstack import OpenStackCloud +from cms.openstack import OpenStackCloud, ExternalNetworkSpec log = logging.getLogger(__name__) @@ -42,8 +42,13 @@ def run(self, clouds: List[OpenStackCloud], global_cfg): with Context(clouds, "base_openstack_provision", len(clouds)) as ctx: for i in ctx: ovn = clouds[i] + ext_net = ExternalNetworkSpec( + neutron_net=ovn.new_external_network(), + num_gw_nodes=self.config.n_gws_per_project, + ) + for _ in range(self.config.n_projects): - _ = ovn.new_project(gw_nodes=self.config.n_gws_per_project) + _ = ovn.new_project(ext_net=ext_net) for project in ovn.projects: for index in range(self.config.n_vms_per_project): From 329d03ffb2ea3f6a658b40a8e253236583212414 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 3 Oct 2023 14:21:03 +0200 Subject: [PATCH 13/19] CMS Openstack: Rename chassis config option `n_gws_per_project` renamed to `n_chassis_per_gw_lrp` to better capture effect of the option. Openstack test scenario was also renamed to be more in line with naming convention of other tests. Signed-off-by: Martin Kalcok --- ovn-tester/cms/openstack/tests/base_openstack.py | 4 ++-- .../{openstack.yml => openstack-20-projects-10-vms.yml} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename test-scenarios/{openstack.yml => openstack-20-projects-10-vms.yml} (74%) diff --git a/ovn-tester/cms/openstack/tests/base_openstack.py b/ovn-tester/cms/openstack/tests/base_openstack.py index e20fa778..e595094d 100644 --- a/ovn-tester/cms/openstack/tests/base_openstack.py +++ b/ovn-tester/cms/openstack/tests/base_openstack.py @@ -15,7 +15,7 @@ @dataclass class BaseOpenstackConfig: n_projects: int = 1 - n_gws_per_project: int = 1 + n_chassis_per_gw_lrp: int = 1 n_vms_per_project: int = 3 @@ -44,7 +44,7 @@ def run(self, clouds: List[OpenStackCloud], global_cfg): ovn = clouds[i] ext_net = ExternalNetworkSpec( neutron_net=ovn.new_external_network(), - num_gw_nodes=self.config.n_gws_per_project, + num_gw_nodes=self.config.n_chassis_per_gw_lrp, ) for _ in range(self.config.n_projects): diff --git a/test-scenarios/openstack.yml b/test-scenarios/openstack-20-projects-10-vms.yml similarity index 74% rename from test-scenarios/openstack.yml rename to test-scenarios/openstack-20-projects-10-vms.yml index 7ea9c1ed..5c793da9 100644 --- a/test-scenarios/openstack.yml +++ b/test-scenarios/openstack-20-projects-10-vms.yml @@ -9,5 +9,5 @@ cluster: base_openstack: n_projects: 20 - n_gws_per_project: 3 - n_vms_per_project: 10 \ No newline at end of file + n_chassis_per_gw_lrp: 3 + n_vms_per_project: 10 From 6138899add2be1aaa14bd71ce73d8f59fa7c6886 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 3 Oct 2023 14:32:08 +0200 Subject: [PATCH 14/19] CI: Add task to run low-scale OS test Signed-off-by: Martin Kalcok --- .cirrus.yml | 7 ++++++- test-scenarios/openstack-low-scale.yml | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 test-scenarios/openstack-low-scale.yml diff --git a/.cirrus.yml b/.cirrus.yml index 536b6408..80749c22 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -69,12 +69,17 @@ low_scale_task: upload_caches: - runtime - test_script: + test_base_ovn_script: - 'sed -i "s/^ log_cmds\: False/ log_cmds\: True/" test-scenarios/ovn-low-scale*.yml' - ./do.sh run test-scenarios/ovn-low-scale.yml low-scale - ./do.sh run test-scenarios/ovn-low-scale-ic.yml low-scale-ic + test_openstack_script: + - 'sed -i "s/^ log_cmds\: false/ log_cmds\: true/" + test-scenarios/openstack-low-scale.yml' + - ./do.sh run test-scenarios/openstack-low-scale.yml openstack-low-scale + check_logs_script: - ./utils/logs-checker.sh diff --git a/test-scenarios/openstack-low-scale.yml b/test-scenarios/openstack-low-scale.yml new file mode 100644 index 00000000..17e1073d --- /dev/null +++ b/test-scenarios/openstack-low-scale.yml @@ -0,0 +1,13 @@ +global: + log_cmds: false + cms_name: openstack + +cluster: + clustered_db: true + log_txns_db: true + n_workers: 3 + +base_openstack: + n_projects: 2 + n_chassis_per_gw_lrp: 3 + n_vms_per_project: 3 From 6b73cca8ad093e2e2e1eca55b06bf69db41229a9 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 10 Oct 2023 16:25:39 +0200 Subject: [PATCH 15/19] CI: Use CMS name in the OVN Kubernetes test. Signed-off-by: Frode Nordahl --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 80749c22..be957119 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -69,7 +69,7 @@ low_scale_task: upload_caches: - runtime - test_base_ovn_script: + test_ovn_kubernetes_script: - 'sed -i "s/^ log_cmds\: False/ log_cmds\: True/" test-scenarios/ovn-low-scale*.yml' - ./do.sh run test-scenarios/ovn-low-scale.yml low-scale From d2815bed65eae6537b183cb01bf10eea6f79dd91 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 10 Oct 2023 15:30:34 +0200 Subject: [PATCH 16/19] cms/openstack: Drop mcast_flood_reports port option. Neutron no longer applies the `mcast_flood_reports=true` option to LSPs [0]. 0: https://review.opendev.org/c/openstack/neutron/+/888127 Signed-off-by: Frode Nordahl --- ovn-tester/cms/openstack/openstack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py index cdb8704c..8c18d1dd 100644 --- a/ovn-tester/cms/openstack/openstack.py +++ b/ovn-tester/cms/openstack/openstack.py @@ -443,7 +443,6 @@ def _add_vm_port( "neutron:subnet_pool_addr_scope6": "", } port_options = ( - f"mcast_flood_reports=true " f"requested-chassis={chassis.container}" ) ls_port = self.nbctl.ls_port_add( From 181783986d5361115c25f007f2630c4383b80975 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 10 Oct 2023 15:33:16 +0200 Subject: [PATCH 17/19] cms/openstack: Drop empty requested-chassis for metadata port. While Neutron currently adds this, it has been proposed for removal as it is not needed by OVN [0]. 0: https://review.opendev.org/c/openstack/neutron/+/897489 Signed-off-by: Frode Nordahl --- ovn-tester/cms/openstack/openstack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py index 8c18d1dd..27216273 100644 --- a/ovn-tester/cms/openstack/openstack.py +++ b/ovn-tester/cms/openstack/openstack.py @@ -484,7 +484,6 @@ def _add_metadata_port(self, network: LSwitch, project_id: str) -> LSPort: ext_ids=port_ext_ids, ) self.nbctl.ls_port_set_set_type(port, "localport") - self.nbctl.ls_port_set_set_options(port, "requested-chassis=") self.nbctl.ls_port_enable(port) return port From 7a90bd76b7f1bf739baead0d0372b5a290db4f17 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 10 Oct 2023 15:52:37 +0200 Subject: [PATCH 18/19] cms/openstack: Set MTU for VM and GW ports. Signed-off-by: Frode Nordahl --- ovn-tester/cms/openstack/openstack.py | 25 ++++++++++++++------- ovn-tester/ovn_utils.py | 31 ++++++++++++++++++++++++--- ovn-tester/ovn_workload.py | 10 +++++++-- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/ovn-tester/cms/openstack/openstack.py b/ovn-tester/cms/openstack/openstack.py index 27216273..84a8bb5e 100644 --- a/ovn-tester/cms/openstack/openstack.py +++ b/ovn-tester/cms/openstack/openstack.py @@ -34,6 +34,7 @@ class NeutronNetwork: network: LSwitch ports: Dict[str, LSPort] name: str + mtu: int security_group: Optional[PortGroup] = None def __post_init__(self): @@ -204,7 +205,8 @@ def new_external_network( """ ext_net_uuid = uuid.uuid4() ext_net_name = f"ext_net_{ext_net_uuid}" - ext_net = self._create_project_net(f"ext_net_{ext_net_uuid}", 1500) + mtu = 1500 + ext_net = self._create_project_net(f"ext_net_{ext_net_uuid}", mtu) ext_net_port = self._add_metadata_port(ext_net, str(ext_net_uuid)) provider_port = self._add_provider_network_port( ext_net, provider_network @@ -216,6 +218,7 @@ def new_external_network( provider_port.uuid: provider_port, }, name=ext_net_name, + mtu=mtu, ) def connect_external_network_to_project( @@ -273,7 +276,8 @@ def add_internal_network_to_project( :return: None """ int_net_name = f"int_net_{project.uuid}" - int_net = self._create_project_net(int_net_name, 1442) + mtu = 1442 + int_net = self._create_project_net(int_net_name, mtu) int_net_port = self._add_metadata_port(int_net, project.uuid) security_group = self._create_default_security_group() @@ -281,6 +285,7 @@ def add_internal_network_to_project( network=int_net, ports={int_net_port.uuid: int_net_port}, name=int_net_name, + mtu=mtu, security_group=security_group, ) @@ -301,7 +306,7 @@ def add_vm_to_project(self, project: Project, vm_name: str): vm_port = self._add_vm_port( project.int_net, project.uuid, compute, vm_name ) - compute.bind_port(vm_port) + compute.bind_port(vm_port, mtu_request=project.int_net.mtu) project.vm_ports.append(vm_port) @@ -442,9 +447,7 @@ def _add_vm_port( "neutron:subnet_pool_addr_scope4": "", "neutron:subnet_pool_addr_scope6": "", } - port_options = ( - f"requested-chassis={chassis.container}" - ) + port_options = f"requested-chassis={chassis.container}" ls_port = self.nbctl.ls_port_add( lswitch=neutron_net.network, name=port_name, @@ -543,7 +546,7 @@ def _add_router_port_external_gw( """ port_ip = self.next_external_ip() return self._add_router_port( - neutron_net.network, router, port_ip, "", True + neutron_net.network, router, port_ip, "", True, neutron_net.mtu ) def _add_router_port( @@ -553,6 +556,7 @@ def _add_router_port( port_ip: DualStackIP, project_id: str = "", is_gw: bool = False, + mtu: Optional[int] = None, ) -> (LSPort, LRPort): """Add a pair of ports that connect Logical Router and Logical Switch. @@ -594,7 +598,12 @@ def _add_router_port( } lr_port = self.nbctl.lr_port_add( - router, router_port_name, str(RandMac()), port_ip, lrp_external_ids + router, + router_port_name, + str(RandMac()), + port_ip, + lrp_external_ids, + {"gateway_mtu": str(mtu)} if mtu else None, ) ls_port = self.nbctl.ls_port_add( diff --git a/ovn-tester/ovn_utils.py b/ovn-tester/ovn_utils.py index 8915c9ca..313afdae 100644 --- a/ovn-tester/ovn_utils.py +++ b/ovn-tester/ovn_utils.py @@ -209,7 +209,14 @@ def set_global_external_id(self, key, value): ("external_ids", {key: str(value)}), ).execute(check_error=True) - def add_port(self, port, bridge, internal=True, ifaceid=None): + def add_port( + self, + port, + bridge, + internal=True, + ifaceid=None, + mtu_request: Optional[int] = None, + ): name = port.name with self.idl.transaction(check_error=True) as txn: txn.add(self.idl.add_port(bridge, name)) @@ -221,6 +228,12 @@ def add_port(self, port, bridge, internal=True, ifaceid=None): txn.add( self.idl.iface_set_external_id(name, "iface-id", ifaceid) ) + if mtu_request: + txn.add( + self.idl.db_set( + "Interface", name, ("mtu_request", mtu_request) + ) + ) def del_port(self, port): self.idl.del_port(port.name).execute(check_error=True) @@ -448,9 +461,16 @@ def lr_add(self, name, ext_ids: Optional[Dict] = None): return LRouter(name=name, uuid=uuid) def lr_port_add( - self, router, name, mac, dual_ip=None, ext_ids: Optional[Dict] = None + self, + router, + name, + mac, + dual_ip=None, + ext_ids: Optional[Dict] = None, + options: Optional[Dict] = None, ): ext_ids = {} if ext_ids is None else ext_ids + options = {} if options is None else options networks = [] if dual_ip.ip4 and dual_ip.plen4: networks.append(f'{dual_ip.ip4}/{dual_ip.plen4}') @@ -458,7 +478,12 @@ def lr_port_add( networks.append(f'{dual_ip.ip6}/{dual_ip.plen6}') self.idl.lrp_add( - router.uuid, name, str(mac), networks, external_ids=ext_ids + router.uuid, + name, + str(mac), + networks, + external_ids=ext_ids, + options=options, ).execute() return LRPort(name=name, mac=mac, ip=dual_ip) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index c482938e..0ca0a59a 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -203,9 +203,15 @@ def unprovision_port(self, cluster, port): self.lports.remove(port) @ovn_stats.timeit - def bind_port(self, port): + def bind_port(self, port, mtu_request: Optional[int] = None): log.info(f'Binding lport {port.name} on {self.container}') - self.vsctl.add_port(port, 'br-int', internal=True, ifaceid=port.name) + self.vsctl.add_port( + port, + 'br-int', + internal=True, + ifaceid=port.name, + mtu_request=mtu_request, + ) # Skip creating a netns for "passive" ports, we won't be sending # traffic on those. if not port.passive: From 799324e4fd15691547d437c322312e0bb69df603 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 10 Oct 2023 16:28:45 +0200 Subject: [PATCH 19/19] ovn-tester/ovn_workload: Collect stats for `mesh_ping_ports`. Signed-off-by: Frode Nordahl --- ovn-tester/ovn_workload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 0ca0a59a..28f6af1a 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -414,6 +414,7 @@ def ping_ports(self, ports): for w, ports in ports_per_worker.items(): w.ping_ports(self, ports) + @ovn_stats.timeit def mesh_ping_ports(self, ports: List[ovn_utils.LSPort]) -> None: """Perform full-mesh ping test between ports.""" all_ips = [port.ip for port in ports]