ovsdb-server: Make use of cooperative multitasking.

Initialize the cooperative multitasking module for the
ovsdb-server.

The server side schema conversion process used for storage
engines such as RAFT is time consuming, yield during
processing.

After the schema conversion is done, the processing of JSON-RPC
sessions and OVSDB monitors for reconnecting clients can
overrun the configured election timer.

The destruction of JSON objects representing the database
contents has been identified as one of the primary offenders.
Make use of yielding version of the JSON object destroy function
to mitigate.

This series has been tested by checking success of schema
conversion, ensuring no involuntary leader change occurs with
election timer configurations as low as 750 msec, on a 75MB
database with ~ 100 connected clients as produced by the
ovn-heater ocp-120-density-light test scenario.

Signed-off-by: Frode Nordahl <frode.nordahl@canonical.com>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>

NEWS

@@ -10,6 +10,8 @@ Post-v3.2.0
        remotes and database configuration, including setting options for
        connection methods for relays and active-backup replication.
        For more details see ovsdb-server(1) and ovsdb(7).
+     * Make use of cooperative multitasking to improve maintenance of RAFT
+       cluster during long running processing such as online schema conversion.
    - OpenFlow:
      * NXT_CT_FLUSH extension is updated to support flushing connections
        based on mark and labels.  'ct-flush' command of ovs-ofctl updated

ovsdb/file.c

@@ -23,6 +23,7 @@
 
 #include "bitmap.h"
 #include "column.h"
+#include "cooperative-multitasking.h"
 #include "log.h"
 #include "openvswitch/json.h"
 #include "lockfile.h"
@@ -321,6 +322,8 @@ ovsdb_convert_table(struct ovsdb_txn *txn,
         struct ovsdb_row *dst_row = ovsdb_row_create(dst_table);
         *ovsdb_row_get_uuid_rw(dst_row) = *ovsdb_row_get_uuid(src_row);
 
+        cooperative_multitasking_yield();
+
         SHASH_FOR_EACH (node, &src_table->schema->columns) {
             const struct ovsdb_column *src_column = node->data;
             const struct ovsdb_column *dst_column;

ovsdb/jsonrpc-server.c

@@ -21,6 +21,7 @@
 
 #include "bitmap.h"
 #include "column.h"
+#include "cooperative-multitasking.h"
 #include "openvswitch/dynamic-string.h"
 #include "monitor.h"
 #include "openvswitch/json.h"
@@ -694,6 +695,8 @@ ovsdb_jsonrpc_session_run_all(struct ovsdb_jsonrpc_remote *remote)
     struct ovsdb_jsonrpc_session *s;
 
     LIST_FOR_EACH_SAFE (s, node, &remote->sessions) {
+        cooperative_multitasking_yield();
+
         int error = ovsdb_jsonrpc_session_run(s);
         if (error) {
             ovsdb_jsonrpc_session_close(s);

ovsdb/monitor.c

@@ -20,8 +20,10 @@
 
 #include "bitmap.h"
 #include "column.h"
+#include "cooperative-multitasking.h"
 #include "openvswitch/dynamic-string.h"
 #include "openvswitch/json.h"
+#include "json.h"
 #include "jsonrpc.h"
 #include "ovsdb-error.h"
 #include "ovsdb-parser.h"
@@ -262,7 +264,7 @@ ovsdb_monitor_json_cache_flush(struct ovsdb_monitor *dbmon)
     struct ovsdb_monitor_json_cache_node *node;
 
     HMAP_FOR_EACH_POP(node, hmap_node, &dbmon->json_cache) {
-        json_destroy(node->json);
+        json_destroy_with_yield(node->json);
         free(node);
     }
 }
@@ -278,7 +280,7 @@ ovsdb_monitor_json_cache_destroy(struct ovsdb_monitor *dbmon,
             = ovsdb_monitor_json_cache_search(dbmon, v, change_set);
         if (node) {
             hmap_remove(&dbmon->json_cache, &node->hmap_node);
-            json_destroy(node->json);
+            json_destroy_with_yield(node->json);
             free(node);
         }
     }
@@ -1172,6 +1174,8 @@ ovsdb_monitor_compose_update(
         struct ovsdb_monitor_table *mt = mcst->mt;
 
         HMAP_FOR_EACH_SAFE (row, hmap_node, &mcst->rows) {
+            cooperative_multitasking_yield();
+
             struct json *row_json;
             row_json = (*row_update)(mt, condition, OVSDB_MONITOR_ROW, row,
                                      initial, changed, mcst->n_columns);
@@ -1217,6 +1221,8 @@ ovsdb_monitor_compose_cond_change_update(
         HMAP_FOR_EACH (row, hmap_node, &mt->table->rows) {
             struct json *row_json;
 
+            cooperative_multitasking_yield();
+
             row_json = ovsdb_monitor_compose_row_update2(mt, condition,
                                                          OVSDB_ROW, row,
                                                          false, changed,
@@ -1286,8 +1292,9 @@ ovsdb_monitor_get_update(
 
                         /* Pre-serializing the object to avoid doing this
                          * for every client. */
-                        json_serialized = json_serialized_object_create(json);
-                        json_destroy(json);
+                        json_serialized =
+                            json_serialized_object_create_with_yield(json);
+                        json_destroy_with_yield(json);
                         json = json_serialized;
                     }
                     ovsdb_monitor_json_cache_insert(dbmon, version, mcs,

ovsdb/ovsdb-server.c

@@ -24,6 +24,7 @@
 
 #include "column.h"
 #include "command-line.h"
+#include "cooperative-multitasking.h"
 #include "daemon.h"
 #include "dirs.h"
 #include "dns-resolve.h"
@@ -915,6 +916,7 @@ main(int argc, char *argv[])
     }
     dns_resolve_destroy();
     perf_counters_destroy();
+    cooperative_multitasking_destroy();
     service_stop();
     return 0;
 }

ovsdb/trigger.c

@@ -20,6 +20,7 @@
 #include <limits.h>
 #include <string.h>
 
+#include "cooperative-multitasking.h"
 #include "file.h"
 #include "openvswitch/json.h"
 #include "jsonrpc.h"
@@ -181,6 +182,8 @@ ovsdb_trigger_run(struct ovsdb *db, long long int now)
     bool disconnect_all = false;
 
     LIST_FOR_EACH_SAFE (t, node, &db->triggers) {
+        cooperative_multitasking_yield();
+
         if (run_triggers
             || now - t->created >= t->timeout_msec
             || t->progress || t->txn_forward) {