netdev-dpdk: Set vhost port maximum number of queue pairs.

This patch uses the new rte_vhost_driver_set_max_queue_num
API to set the maximum number of queue pairs supported by
the vhost-user port.

This is required for VDUSE which needs to specify the
maximum number of queue pairs at creation time. Without it
128 queue pairs metadata would be allocated.

To configure it, a new 'vhost-max-queue-pairs' option is
introduced.

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Acked-by: Eelco Chaudron <echaudro@redhat.com>
Signed-off-by: Kevin Traynor <ktraynor@redhat.com>

Documentation/topics/dpdk/vhost-user.rst

@@ -375,6 +375,21 @@ Tx retries max can be set for vhost-user-client ports::
 
   Configurable vhost tx retries are not supported with vhost-user ports.
 
+vhost-user-client max queue pairs config
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For vhost-user-client interfaces using the VDUSE backend, the maximum number of
+queue pairs the Virtio device will support can be set at port creation time. If
+not set, the default value is 1 queue pair. This value is ignored for
+Vhost-user backends.
+
+Maximum number of queue pairs can be set for vhost-user-client-ports::
+
+    $ ovs-vsctl add-port br0 vduse0 \
+        -- set Interface vduse0 type=dpdkvhostuserclient \
+            options:vhost-server-path=/dev/vduse/vduse0 \
+            options:vhost-max-queue-pairs=4
+
 .. _dpdk-testpmd:
 
 DPDK in the Guest

lib/netdev-dpdk.c

@@ -19,6 +19,7 @@
 
 #include <errno.h>
 #include <signal.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
@@ -153,6 +154,11 @@ typedef uint16_t dpdk_port_t;
 /* Legacy default value for vhost tx retries. */
 #define VHOST_ENQ_RETRY_DEF 8
 
+/* VDUSE-only, ignore for vhost-user. */
+#define VHOST_MAX_QUEUE_PAIRS_MIN 1
+#define VHOST_MAX_QUEUE_PAIRS_DEF VHOST_MAX_QUEUE_PAIRS_MIN
+#define VHOST_MAX_QUEUE_PAIRS_MAX 128
+
 #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
 
 /* List of required flags advertised by the hardware that will be used
@@ -554,6 +560,9 @@ struct netdev_dpdk {
         /* Socket ID detected when vHost device is brought up */
         int requested_socket_id;
 
+        /* Ignored by DPDK for vhost-user backends, only for VDUSE. */
+        uint8_t vhost_max_queue_pairs;
+
         /* Denotes whether vHost port is client/server mode */
         uint64_t vhost_driver_flags;
 
@@ -1606,6 +1615,8 @@ vhost_common_construct(struct netdev *netdev)
 
     atomic_init(&dev->vhost_tx_retries_max, VHOST_ENQ_RETRY_DEF);
 
+    dev->vhost_max_queue_pairs = VHOST_MAX_QUEUE_PAIRS_DEF;
+
     return common_construct(netdev, DPDK_ETH_PORT_ID_INVALID,
                             DPDK_DEV_VHOST, socket_id);
 }
@@ -2488,13 +2499,23 @@ netdev_dpdk_vhost_client_set_config(struct netdev *netdev,
     struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
     const char *path;
     int max_tx_retries, cur_max_tx_retries;
+    uint32_t max_queue_pairs;
 
     ovs_mutex_lock(&dev->mutex);
     if (!(dev->vhost_driver_flags & RTE_VHOST_USER_CLIENT)) {
         path = smap_get(args, "vhost-server-path");
         if (!nullable_string_is_equal(path, dev->vhost_id)) {
             free(dev->vhost_id);
             dev->vhost_id = nullable_xstrdup(path);
+
+            max_queue_pairs = smap_get_int(args, "vhost-max-queue-pairs",
+                                           VHOST_MAX_QUEUE_PAIRS_DEF);
+            if (max_queue_pairs < VHOST_MAX_QUEUE_PAIRS_MIN
+                || max_queue_pairs > VHOST_MAX_QUEUE_PAIRS_MAX) {
+                max_queue_pairs = VHOST_MAX_QUEUE_PAIRS_DEF;
+            }
+            dev->vhost_max_queue_pairs = max_queue_pairs;
+
             netdev_request_reconfigure(netdev);
         }
     }
@@ -6397,6 +6418,18 @@ netdev_dpdk_vhost_client_reconfigure(struct netdev *netdev)
             goto unlock;
         }
 
+        /* Setting max queue pairs is only useful and effective with VDUSE. */
+        if (strncmp(dev->vhost_id, "/dev/vduse/", 11) == 0) {
+            uint32_t max_qp = dev->vhost_max_queue_pairs;
+
+            err = rte_vhost_driver_set_max_queue_num(dev->vhost_id, max_qp);
+            if (err) {
+                VLOG_ERR("rte_vhost_driver_set_max_queue_num failed for "
+                         "vhost-user client port: %s\n", dev->up.name);
+                goto unlock;
+            }
+        }
+
         err = rte_vhost_driver_start(dev->vhost_id);
         if (err) {
             VLOG_ERR("rte_vhost_driver_start failed for vhost user "

vswitchd/vswitch.xml

@@ -3540,6 +3540,19 @@ ovs-vsctl add-port br0 p0 -- set Interface p0 type=patch options:peer=p1 \
         </p>
       </column>
 
+      <column name="options" key="vhost-max-queue-pairs"
+              type='{"type": "integer", "minInteger" : 1, "maxInteger": 128}'>
+        <p>
+          The value specifies the maximum number of queue pairs supported by
+          a vHost device. This is ignored for vhost-user backends, only VDUSE
+          is supported.
+          Only supported by dpdkvhostuserclient interfaces.
+        </p>
+        <p>
+          Default value is 1.
+        </p>
+      </column>
+
       <column name="options" key="tx-retries-max"
               type='{"type": "integer", "minInteger": 0, "maxInteger": 32}'>
         <p>