Add new vshard configuration options

.github/workflows/compatibility-test.yml

@@ -21,16 +21,10 @@ jobs:
         downgrade: [null]
         include:
           - tarantool: '1.10'
-            cartridge: '2.7.9'
-            downgrade: true
-          - tarantool: '1.10'
-            cartridge: '2.8.4'
-            downgrade: true
-          - tarantool: '2.10'
-            cartridge: '2.7.9'
+            cartridge: '2.8.6'
             downgrade: true
           - tarantool: '2.10'
-            cartridge: '2.8.4'
+            cartridge: '2.8.6'
             downgrade: true
     runs-on: ubuntu-20.04
     env:

CHANGELOG.rst

@@ -18,6 +18,15 @@ Fixed
 
 - Raft failover state transitions.
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Added
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- New VShard configuration options: ``rebalancer`` (on server/replicaset level)
+  and ``rebalancer_mode`` (on VShard config level).
+
+- ``rebalancer_enabled`` field to boxinfo GraphQL API.
+
 -------------------------------------------------------------------------------
 [2.8.6] - 2024-02-01
 -------------------------------------------------------------------------------

cartridge.lua

@@ -171,6 +171,12 @@ end
 --  env `TARANTOOL_BUCKET_COUNT`,
 --  args `--bucket-count`)
 --
+-- @tparam ?number opts.rebalancer_mode
+--  rebalancer_mode for vshard cluster. See vshard doc for more details.
+--  (default: "auto", overridden by
+--  env `TARANTOOL_REBALANCER_MODE`,
+--  args `--rebalancer-mode`)
+--
 -- @tparam ?table opts.vshard_groups
 --  vshard storage groups.
 --  `{group_name = VshardGroup, ...}`, `{'group1', 'group2', ...}` or
@@ -282,6 +288,7 @@ local function cfg(opts, box_opts)
         advertise_uri = '?string',
         cluster_cookie = '?string',
         bucket_count = '?number',
+        rebalancer_mode = '?string',
         http_port = '?string|number',
         http_host = '?string',
         http_enabled = '?boolean',
@@ -467,6 +474,14 @@ local function cfg(opts, box_opts)
         title.update(box_opts.custom_proc_title)
     end
 
+    if opts.rebalancer_mode == nil then
+        opts.rebalancer_mode = 'auto'
+    elseif not (opts.rebalancer_mode == 'auto'
+    or opts.rebalancer_mode == 'off'
+    or opts.rebalancer_mode == 'manual') then
+        return nil, CartridgeCfgError:new('Invalid rebalancer_mode %q', opts.rebalancer_mode)
+    end
+
     local vshard_groups = {}
     for k, v in pairs(opts.vshard_groups or {}) do
         local name, params
@@ -790,7 +805,7 @@ local function cfg(opts, box_opts)
         end
     end
 
-    vshard_utils.set_known_groups(vshard_groups, opts.bucket_count)
+    vshard_utils.set_known_groups(vshard_groups, opts.bucket_count, opts.rebalancer_mode)
 
     -- Set up issues
     local issue_limits, err = argparse.get_opts({

cartridge/argparse.lua

@@ -98,6 +98,7 @@ local cluster_opts = {
     console_sock = 'string', -- **string**
     auth_enabled = 'boolean', -- **boolean**
     bucket_count = 'number', -- **number**
+    rebalancer_mode = 'string', -- **string**
     upgrade_schema = 'boolean', -- **boolean**
     swim_broadcast = 'boolean', -- **boolean**
     upload_prefix = 'string', -- **string**

cartridge/logging_whitelist.lua

@@ -4,6 +4,7 @@ local cartridge_opts = {
     'auth_backend_name',
     'auth_enabled',
     'bucket_count',
+    'rebalancer_mode',
     'console_sock',
     'http_enabled',
     'http_host',

cartridge/lua-api/boxinfo.lua

@@ -79,7 +79,8 @@ local function get_info(uri)
         local rs_uuid = box_info.cluster.uuid
         local vshard_group = topology_cfg.replicasets[rs_uuid].vshard_group or 'default'
         local ok, storage_info = pcall(vshard and vshard.storage.info)
-
+        local rebalancer_enabled = vshard and vshard.storage and
+            vshard.storage.internal.rebalancer_fiber ~= nil
         if ok then
             storage_info = {
                 vshard_group = vshard_group,
@@ -89,6 +90,7 @@ local function get_info(uri)
                 buckets_garbage = storage_info.bucket.garbage,
                 buckets_pinned = storage_info.bucket.pinned,
                 buckets_sending = storage_info.bucket.sending,
+                rebalancer_enabled = rebalancer_enabled,
             }
         else
             storage_info = box.NULL

cartridge/lua-api/edit-topology.lua

@@ -32,6 +32,7 @@ local function __join_server(topology_cfg, params)
         uuid = 'string',
         zone = '?string',
         labels = '?table',
+        rebalancer = '?boolean',
         replicaset_uuid = 'string',
     })
 
@@ -59,6 +60,7 @@ local function __join_server(topology_cfg, params)
         labels = params.labels,
         disabled = false,
         electable = true,
+        rebalancer = params.rebalancer,
         replicaset_uuid = params.replicaset_uuid,
     }
 
@@ -74,6 +76,7 @@ local function __edit_server(topology_cfg, params)
         labels = '?table',
         disabled = '?boolean',
         electable = '?boolean',
+        rebalancer = '?boolean',
         expelled = '?boolean',
     })
 
@@ -108,6 +111,12 @@ local function __edit_server(topology_cfg, params)
         server.electable = params.electable
     end
 
+    if params.rebalancer ~= nil then
+        server.rebalancer = params.rebalancer
+    else
+        server.rebalancer = nil -- avoid setting it to box.NULL
+    end
+
     if params.expelled == true then
         topology_cfg.servers[params.uuid] = 'expelled'
     end
@@ -122,6 +131,7 @@ local function __edit_replicaset(topology_cfg, params)
         all_rw = '?boolean',
         roles = '?table',
         weight = '?number',
+        rebalancer = '?boolean',
         failover_priority = '?table',
         vshard_group = '?string',
         join_servers = '?table',
@@ -229,6 +239,18 @@ local function __edit_replicaset(topology_cfg, params)
         end
     until true
 
+    repeat
+        if not replicaset.roles['vshard-storage'] then
+            -- ignore unless replicaset is a storage
+            break
+        end
+        if params.rebalancer ~= nil then
+            replicaset.rebalancer = params.rebalancer
+        else
+            replicaset.rebalancer = nil -- avoid setting it to box.NULL
+        end
+    until true
+
     return true
 end
 
@@ -254,6 +276,7 @@ end
 -- @tfield ?{string,...} roles
 -- @tfield ?boolean all_rw
 -- @tfield ?number weight
+-- @tfield ?boolean rebalancer
 -- @tfield ?{string,...} failover_priority
 --   array of uuids specifying servers failover priority
 -- @tfield ?string vshard_group
@@ -276,6 +299,7 @@ end
 -- @tfield ?table labels
 -- @tfield ?boolean disabled
 -- @tfield ?boolean electable
+-- @tfield ?boolean rebalancer
 -- @tfield ?boolean expelled
 --   Expelling an instance is permanent and can't be undone.
 --   It's suitable for situations when the hardware is destroyed,
@@ -341,6 +365,7 @@ local function edit_topology(args)
         end
     end
 
+    local rebalancer_enabled = false
     for replicaset_uuid, _ in pairs(topology_cfg.replicasets) do
         local replicaset_empty = true
         for _, _, server in fun.filter(topology.not_expelled, topology_cfg.servers) do
@@ -362,6 +387,23 @@ local function edit_topology(args)
                     table.insert(replicaset.master, leader_uuid)
                 end
             end
+            if replicaset.rebalancer == true and rebalancer_enabled then
+                return nil, EditTopologyError:new(
+                    'Several rebalancer flags found in config'
+                )
+            elseif replicaset.rebalancer == true then
+                rebalancer_enabled = true
+            end
+        end
+    end
+
+    for _, server in pairs(topology_cfg.servers) do
+        if server.rebalancer == true and rebalancer_enabled then
+            return nil, EditTopologyError:new(
+                'Several rebalancer flags found in config'
+            )
+        elseif server.rebalancer == true then
+            rebalancer_enabled = true
         end
     end
 

cartridge/lua-api/get-topology.lua

@@ -28,6 +28,9 @@ local lua_api_proxy = require('cartridge.lua-api.proxy')
 -- @tfield number weight
 --   Vshard replicaset weight.
 --   Matters only if vshard-storage role is enabled.
+-- @tfield boolean rebalancer
+--   Is rebalancer enabled on this replicaset.
+--   Matters only if vshard-storage role is enabled.
 -- @tfield string vshard_group
 --   Name of vshard group the replicaset belongs to.
 -- @tfield boolean all_rw
@@ -45,6 +48,7 @@ local lua_api_proxy = require('cartridge.lua-api.proxy')
 -- @tfield string uuid
 -- @tfield boolean disabled
 -- @tfield boolean electable
+-- @tfield boolean rebalancer
 -- @tfield string status
 --   Instance health.
 -- @tfield string message
@@ -118,6 +122,7 @@ local function get_topology()
             servers = {},
             all_rw = replicaset.all_rw or false,
             alias = replicaset.alias or 'unnamed',
+            rebalancer = replicaset.rebalancer,
         }
 
         local enabled_roles = roles.get_enabled_roles(replicaset.roles)
@@ -152,6 +157,7 @@ local function get_topology()
             disabled = not topology.not_disabled(instance_uuid, server),
             electable = topology.electable(instance_uuid, server),
             zone = server.zone,
+            rebalancer = server.rebalancer,
             alias = nil,
             status = nil,
             message = nil,

cartridge/topology.lua

@@ -48,6 +48,7 @@ local e_config = errors.new_class('Invalid cluster topology config')
         --     electable = true,
         --     replicaset_uuid = 'replicaset-uuid-2',
         --     zone = nil | string,
+        --     rebalancer = nil | true,
         -- },
     },
     replicasets = {
@@ -60,6 +61,7 @@ local e_config = errors.new_class('Invalid cluster topology config')
         --     master = {'instance-uuid-2', 'instance-uuid-1'} -- new format
         --     weight = 1.0,
         --     vshard_group = 'group_name',
+        --     rebalancer = nil | true,
         -- }
     },
 }]]
@@ -258,6 +260,12 @@ local function validate_schema(field, topology)
                 '%s.zone must be a string, got %s', field, type(server.zone)
             )
 
+            e_config:assert(
+                server.rebalancer == nil or
+                type(server.rebalancer) == 'boolean',
+                '%s.rebalancer must be a string, got %s', field, type(server.rebalancer)
+            )
+
             if rawget(_G, '__cartridge_log_invalid_labels') == true then
                 local ok, err = label_utils.validate_labels(field, server)
                 if not ok then
@@ -270,6 +278,7 @@ local function validate_schema(field, topology)
                 ['disabled'] = true,
                 ['electable'] = true,
                 ['replicaset_uuid'] = true,
+                ['rebalancer'] = true,
                 ['labels'] = true,
                 ['zone'] = true,
             }
@@ -345,6 +354,11 @@ local function validate_schema(field, topology)
             '%s.vshard_group must be a string, got %s', field, type(replicaset.vshard_group)
         )
 
+        e_config:assert(
+            (replicaset.rebalancer == nil) or (type(replicaset.rebalancer) == 'boolean'),
+            '%s.rebalancer must be a boolean, got %s', field, type(replicaset.rebalancer)
+        )
+
         for k, v in pairs(replicaset.roles) do
             e_config:assert(
                 type(k) == 'string',
@@ -368,6 +382,7 @@ local function validate_schema(field, topology)
             ['roles'] = true,
             ['master'] = true,
             ['weight'] = true,
+            ['rebalancer'] = true,
             ['vshard_group'] = true,
             ['all_rw'] = true,
             ['alias'] = true,