tests: Wait for controller exit before restart.

In some rare cases, and despite "recent" changes to wait for cleanup
before replying to exit, ovn-controller was still running when trying
to restart it.

Signed-off-by: Xavier Simonart <xsimonar@redhat.com>
Signed-off-by: Dumitru Ceara <dceara@redhat.com>
(cherry picked from commit ca52a80)

tests/ovn-macros.at

@@ -1191,6 +1191,15 @@ m4_define([OVN_CHECK_SCAPY_EDNS_CLIENT_SUBNET_SUPPORT],
     AT_SKIP_IF([! echo "from scapy.layers.dns import EDNS0ClientSubnet" | python 2>&1 > /dev/null])
 ])
 
+m4_define([OVN_CONTROLLER_EXIT],
+  [TMPPID=$(cat $1/ovn-controller.pid)
+   AT_CHECK([as $1 ovn-appctl -t ovn-controller exit --restart])
+   # Make sure ovn-controller stopped so that a future restart will not fail.
+   # Checking debug/status is running is not enough as there might be a small race condition.
+   echo "Waiting for pid $TMPPID"
+   OVS_WAIT_WHILE([kill -0 $TMPPID 2>/dev/null])
+])
+
 m4_define([OFTABLE_PHY_TO_LOG], [0])
 m4_define([OFTABLE_LOG_INGRESS_PIPELINE], [8])
 m4_define([OFTABLE_OUTPUT_LARGE_PKT_DETECT], [40])

tests/ovn.at

@@ -20733,7 +20733,7 @@ echo $expected | ovstest test-ovn expr-to-packets > expected
 OVN_CHECK_PACKETS([hv2/vif1-tx.pcap], [expected])
 
 # Stop ovn-controller on hv2 with --restart flag
-as hv2 ovs-appctl -t ovn-controller exit --restart
+OVN_CONTROLLER_EXIT([hv2])
 
 # Now send the packet again. This time, it should still arrive
 OVS_WAIT_UNTIL([as hv1 ovs-appctl -t ovn-controller inject-pkt "$packet"])
@@ -29512,7 +29512,7 @@ check test "$hvt2" -gt 0
 # Kill ovn-controller on chassis hv3, so that it won't update nb_cfg.
 # Then wait for 9 out of 10
 sleep 1
-check as hv3 ovn-appctl -t ovn-controller exit --restart
+OVN_CONTROLLER_EXIT([hv3])
 wait_for_ports_up
 ovn-nbctl --wait=sb sync
 wait_row_count Chassis_Private 9 name!=hv3 nb_cfg=2
@@ -36602,7 +36602,7 @@ check_tunnel_port hv1 br-int hv2@192.168.0.2%192.168.0.1
 check_tunnel_port hv2 br-int hv1@192.168.0.1%192.168.0.2
 
 # Stop ovn-controller on hv1
-check as hv1 ovn-appctl -t ovn-controller exit --restart
+OVN_CONTROLLER_EXIT([hv1])
 
 # The tunnel should remain intact
 check_tunnel_port hv1 br-int hv2@192.168.0.2%192.168.0.1
@@ -36631,7 +36631,7 @@ check_tunnel_port hv2 br-int1 hv1@192.168.0.1%192.168.0.2
 check grep -q "Clearing old tunnel port \"ovn-hv1-0\" (hv1@192.168.0.1%192.168.0.2) from bridge \"br-int\"" hv2/ovn-controller.log
 
 # Stop ovn-controller on hv1
-check as hv1 ovn-appctl -t ovn-controller exit --restart
+OVN_CONTROLLER_EXIT([hv1])
 
 # The tunnel should remain intact
 check_tunnel_port hv1 br-int1 hv2@192.168.0.2%192.168.0.1
@@ -36721,10 +36721,7 @@ prev_id2=$(ovs-vsctl --bare --columns _uuid find port external_ids:ovn-chassis-i
 # The hv2 is running we can remove the override file
 rm -f ${OVN_SYSCONFDIR}/system-id-override
 
-check ovn-appctl -t ovn-controller exit --restart
-
-# Make sure ovn-controller stopped before restarting it
-OVS_WAIT_UNTIL([test x$(ovn-appctl -t ovn-controller debug/status) != "xrunning"])
+OVN_CONTROLLER_EXIT([hv1])
 
 # for some reason SSL ovsdb configuration overrides CLI, so
 # delete ssl config from ovsdb to give CLI arguments priority
@@ -37440,9 +37437,7 @@ AT_CHECK([grep -c "NXT_CT_FLUSH_ZONE" hv1/ovs-vswitchd.log], [0], [dnl
 ])
 
 AS_BOX([Check conversion from UUID - restart])
-ovn-appctl -t ovn-controller exit --restart
-# Make sure ovn-controller stopped before restarting it
-OVS_WAIT_UNTIL([test "$(ovn-appctl -t ovn-controller debug/status)" != "running"])
+OVN_CONTROLLER_EXIT([hv1])
 
 replace_with_uuid lr0
 replace_with_uuid sw0