Merge pull request #186 from LorenzoBianconi/ovn-ic-central-nodes-dep…

…loyment-rework

Ovn ic central nodes deployment rework

README.md

@@ -112,8 +112,8 @@ The file should contain the following mandatory sections and fields:
   - `ssh_key`: An ssh private key to install in the TESTER that can be used
     to communicate with the other machines in the cluster.
     Default: `~/.ssh/id_rsa`
-- `central-node`:
-  - `name`: the hostname (or IP) of the node that will run `ovn-central`
+- `central-nodes`:
+  - `name`: the hostnames (or IPs) of the nodes that will run `ovn-central`
     (`ovn-northd` and databases).
 - `worker-nodes`:
   - the list of worker node hostnames (or IPs). If needed, worker nodes can
@@ -134,7 +134,7 @@ In case some of the physical machines in the setup have different
 capabilities (e.g, could host more containers, or use a different ethernet
 interface), the following per-node fields can be used to customize the
 deployment. Except for `fake-nodes` which is valid only in the context of
-worker nodes, all others are valid both for the `central-node` and also for
+worker nodes, all others are valid both for the `central-nodes` and also for
 `worker-nodes`:
 - `user`: the username to be used when connecting from the tester node.
 - `internal-iface`: the name of the Ethernet interface used for DB and

ovn-fake-multinode-utils/generate-hosts.py

@@ -44,17 +44,14 @@ def generate_tester(config, internal_iface):
     )
 
 
-def generate_controller(config, internal_iface):
-    generate_node(config, internal_iface, ovn_central="true")
-
-
-def generate_workers(nodes_config, internal_iface):
+def generate_nodes(nodes_config, internal_iface, **kwargs):
     for node_config in nodes_config:
         host, node_config = helpers.get_node_config(node_config)
         iface = node_config.get('internal-iface', internal_iface)
         generate_node_string(
             host,
             internal_iface=iface,
+            **kwargs,
         )
 
 
@@ -63,16 +60,17 @@ def generate(input_file, target, repo, branch):
         config = yaml.safe_load(yaml_file)
         user = config.get('user', 'root')
         prefix = config.get('prefix', 'ovn-scale')
-        central_config = config['central-node']
         tester_config = config['tester-node']
         internal_iface = config['internal-iface']
 
         print('[tester_hosts]')
         generate_tester(tester_config, internal_iface)
         print('\n[central_hosts]')
-        generate_controller(central_config, internal_iface)
+        generate_nodes(
+            config['central-nodes'], internal_iface, ovn_central="true"
+        )
         print('\n[worker_hosts]')
-        generate_workers(config['worker-nodes'], internal_iface)
+        generate_nodes(config['worker-nodes'], internal_iface)
         print()
 
         print('[all:vars]')

ovn-fake-multinode-utils/playbooks/bringup-cluster.yml

@@ -34,34 +34,56 @@
 - name: Bring up central nodes
   hosts: central_hosts
   tasks:
+  - name: Compute central facts (standalone)
+    when: clustered_db == "no"
+    ansible.builtin.set_fact:
+      n_ips: '{{ n_relays|int + 1 }}'
+      central_ic_id: 'ovn-central-az0'
+
+  - name: Compute central facts (clustered)
+    when: clustered_db == "yes"
+    ansible.builtin.set_fact:
+      n_ips: '{{ n_relays|int + 3 }}'
+      central_ic_id: 'ovn-central-az0-1'
+
+  - name: Determine node to run IC DBs
+    ansible.builtin.set_fact: ic_db_node="{{ groups['central_hosts'] | first }}"
+
   - name: Start central containers
     environment:
-      CENTRAL_COUNT: '{{ n_az }}'
+      CENTRAL_COUNT: 1
       CHASSIS_COUNT: 0
+      CENTRAL_NAME: 'ovn-central-az{{ item|int }}'
+      CENTRAL_PREFIX: 'ovn-central-az{{ item|int }}'
       CREATE_FAKE_VMS: no
       ENABLE_ETCD: '{{ use_ovsdb_etcd }}'
       ENABLE_SSL:  '{{ enable_ssl }}'
       GW_COUNT: 0
       IP_CIDR:  "{{ node_net|ansible.utils.ipaddr('prefix') }}"
       IP_HOST:  "{{ node_net|ansible.utils.ipaddr('network') }}"
-      IP_START: "{{ node_net|ansible.utils.ipmath(2) }}"
-      OVN_DB_CLUSTER:  '{{ clustered_db }}'
-      OVN_DP_TYPE:     '{{ datapath_type }}'
-      OVN_MONITOR_ALL: '{{ monitor_all }}'
-      RELAY_COUNT:     '{{ n_relays }}'
+      IP_START: "{{ node_net|ansible.utils.ipmath(2 + item|int * n_ips|int) }}"
+      OVN_START_IC_DBS: '{{ (item|int == 0) | ansible.builtin.ternary("yes", "no") }}'
+      CENTRAL_IC_ID:    '{{ central_ic_id }}'
+      CENTRAL_IC_IP:    '{{ node_net|ansible.utils.ipmath(2) }}'
+      OVN_DB_CLUSTER:   '{{ clustered_db }}'
+      OVN_DP_TYPE:      '{{ datapath_type }}'
+      OVN_MONITOR_ALL:  '{{ monitor_all }}'
+      RELAY_COUNT:      '{{ n_relays }}'
       RUNC_CMD: podman
     ansible.builtin.shell: |
       ./ovn_cluster.sh start
     args:
       chdir: "{{ ovn_fake_multinode_target_path }}/ovn-fake-multinode"
+    loop: "{{ range(groups['central_hosts'].index(inventory_hostname),
+                    n_az|int, groups['central_hosts']|length) | list }}"
 
 
 - name: Bring up worker nodes
   hosts: worker_hosts
   tasks:
   - name: Start worker containers
     environment:
-      CENTRAL_COUNT: '{{ n_az }}'
+      CENTRAL_COUNT: 1
       CHASSIS_COUNT: 0
       CREATE_FAKE_VMS: no
       ENABLE_ETCD: '{{ use_ovsdb_etcd }}'

ovn-tester/ovn_tester.py

@@ -40,15 +40,15 @@ def read_physical_deployment(deployment, global_cfg):
     with open(deployment, 'r') as yaml_file:
         dep = yaml.safe_load(yaml_file)
 
-        central_dep = dep['central-node']
-        central_node = PhysicalNode(
-            central_dep.get('name', 'localhost'), global_cfg.log_cmds
-        )
+        central_nodes = [
+            PhysicalNode(central, global_cfg.log_cmds)
+            for central in dep['central-nodes']
+        ]
         worker_nodes = [
             PhysicalNode(worker, global_cfg.log_cmds)
             for worker in dep['worker-nodes']
         ]
-        return central_node, worker_nodes
+        return central_nodes, worker_nodes
 
 
 # SSL files are installed by ovn-fake-multinode in these locations.
@@ -218,9 +218,9 @@ def set_ssl_keys(cluster_cfg):
         raise ovn_exceptions.OvnInvalidConfigException()
 
     cms_cls = load_cms(global_cfg.cms_name)
-    central, workers = read_physical_deployment(sys.argv[1], global_cfg)
+    centrals, workers = read_physical_deployment(sys.argv[1], global_cfg)
     clusters = [
-        cms_cls(cluster_cfg, central, brex_cfg, i)
+        cms_cls(cluster_cfg, centrals[i % len(centrals)], brex_cfg, i)
         for i in range(cluster_cfg.n_az)
     ]
     for c in clusters:

ovn-tester/ovn_workload.py

@@ -297,30 +297,29 @@ def __init__(self, cluster_cfg, central, brex_cfg, az):
         protocol = "ssl" if cluster_cfg.enable_ssl else "tcp"
         db_containers = (
             [
-                f'ovn-central-az{self.az+1}-1',
-                f'ovn-central-az{self.az+1}-2',
-                f'ovn-central-az{self.az+1}-3',
+                f'ovn-central-az{self.az}-1',
+                f'ovn-central-az{self.az}-2',
+                f'ovn-central-az{self.az}-3',
             ]
             if cluster_cfg.clustered_db
-            else [f'ovn-central-az{self.az+1}-1']
+            else [f'ovn-central-az{self.az}-1']
         )
 
-        mgmt_ip = cluster_cfg.node_net.ip + 2 + self.az * len(db_containers)
+        mgmt_ip = (
+            cluster_cfg.node_net.ip
+            + 2
+            + self.az * (len(db_containers) + cluster_cfg.n_relays)
+        )
         self.central_nodes = [
             CentralNode(central, c, mgmt_ip + i, protocol)
             for i, c in enumerate(db_containers)
         ]
 
-        mgmt_ip = (
-            cluster_cfg.node_net.ip
-            + 2
-            + cluster_cfg.n_az * len(self.central_nodes)
-            + self.az * cluster_cfg.n_relays
-        )
+        mgmt_ip += len(db_containers)
         self.relay_nodes = [
             RelayNode(
                 central,
-                f'ovn-relay-az{self.az+1}-{i+1}',
+                f'ovn-relay-az{self.az}-{i+1}',
                 mgmt_ip + i,
                 protocol,
             )

physical-deployments/ci.yml

@@ -1,7 +1,7 @@
 internal-iface: lo
 
-central-node:
-  name: <host>
+central-nodes:
+  - <host>
 
 tester-node:
   name: <host>

physical-deployments/physical-deployment.yml

@@ -1,7 +1,7 @@
 internal-iface: eno1
 
-central-node:
-  name: host02.mydomain.com
+central-nodes:
+  - host02.mydomain.com
 
 tester-node:
   name: host02.mydoman.com