diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0bfd2817d..5b79f1592 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,8 @@ Added - New VShard configuration options: ``rebalancer`` and ``rebalancer_mode``. +- ``rebalancer_enabled`` field to boxinfo GraphQL API. + ------------------------------------------------------------------------------- [2.8.6] - 2024-02-01 ------------------------------------------------------------------------------- diff --git a/cartridge/lua-api/boxinfo.lua b/cartridge/lua-api/boxinfo.lua index ddd3140d4..e08a2ec8b 100644 --- a/cartridge/lua-api/boxinfo.lua +++ b/cartridge/lua-api/boxinfo.lua @@ -79,7 +79,8 @@ local function get_info(uri) local rs_uuid = box_info.cluster.uuid local vshard_group = topology_cfg.replicasets[rs_uuid].vshard_group or 'default' local ok, storage_info = pcall(vshard and vshard.storage.info) - + local rebalancer_enabled = vshard and vshard.storage and + vshard.storage.internal.rebalancer_fiber ~= nil if ok then storage_info = { vshard_group = vshard_group, @@ -89,6 +90,7 @@ local function get_info(uri) buckets_garbage = storage_info.bucket.garbage, buckets_pinned = storage_info.bucket.pinned, buckets_sending = storage_info.bucket.sending, + rebalancer_enabled = rebalancer_enabled, } else storage_info = box.NULL diff --git a/cartridge/lua-api/edit-topology.lua b/cartridge/lua-api/edit-topology.lua index 659d48dc9..130f920e8 100644 --- a/cartridge/lua-api/edit-topology.lua +++ b/cartridge/lua-api/edit-topology.lua @@ -32,6 +32,7 @@ local function __join_server(topology_cfg, params) uuid = 'string', zone = '?string', labels = '?table', + rebalancer = '?boolean', replicaset_uuid = 'string', }) @@ -59,6 +60,7 @@ local function __join_server(topology_cfg, params) labels = params.labels, disabled = false, electable = true, + rebalancer = params.rebalancer, replicaset_uuid = params.replicaset_uuid, } @@ -74,6 +76,7 @@ local function __edit_server(topology_cfg, params) labels = '?table', disabled = '?boolean', electable = '?boolean', + rebalancer = '?boolean', expelled = '?boolean', }) @@ -122,6 +125,7 @@ local function __edit_replicaset(topology_cfg, params) all_rw = '?boolean', roles = '?table', weight = '?number', + rebalancer = '?boolean', failover_priority = '?table', vshard_group = '?string', join_servers = '?table', @@ -254,6 +258,7 @@ end -- @tfield ?{string,...} roles -- @tfield ?boolean all_rw -- @tfield ?number weight +-- @tfield ?boolean rebalancer -- @tfield ?{string,...} failover_priority -- array of uuids specifying servers failover priority -- @tfield ?string vshard_group @@ -276,6 +281,7 @@ end -- @tfield ?table labels -- @tfield ?boolean disabled -- @tfield ?boolean electable +-- @tfield ?boolean rebalancer -- @tfield ?boolean expelled -- Expelling an instance is permanent and can't be undone. -- It's suitable for situations when the hardware is destroyed, diff --git a/cartridge/lua-api/get-topology.lua b/cartridge/lua-api/get-topology.lua index 691502fc3..7c12ee644 100644 --- a/cartridge/lua-api/get-topology.lua +++ b/cartridge/lua-api/get-topology.lua @@ -28,6 +28,9 @@ local lua_api_proxy = require('cartridge.lua-api.proxy') -- @tfield number weight -- Vshard replicaset weight. -- Matters only if vshard-storage role is enabled. +-- @tfield boolean rebalancer +-- Is rebalancer enabled on this replicaset. +-- Matters only if vshard-storage role is enabled. -- @tfield string vshard_group -- Name of vshard group the replicaset belongs to. -- @tfield boolean all_rw @@ -45,6 +48,7 @@ local lua_api_proxy = require('cartridge.lua-api.proxy') -- @tfield string uuid -- @tfield boolean disabled -- @tfield boolean electable +-- @tfield boolean rebalancer -- @tfield string status -- Instance health. -- @tfield string message @@ -152,6 +156,7 @@ local function get_topology() disabled = not topology.not_disabled(instance_uuid, server), electable = topology.electable(instance_uuid, server), zone = server.zone, + rebalancer = server.rebalancer, alias = nil, status = nil, message = nil, diff --git a/cartridge/roles/vshard-storage.lua b/cartridge/roles/vshard-storage.lua index 432a98feb..1e244720b 100644 --- a/cartridge/roles/vshard-storage.lua +++ b/cartridge/roles/vshard-storage.lua @@ -51,6 +51,7 @@ local function apply_config(conf, opts) end log.info('Reconfiguring vshard.storage...') + log.info(vshard_cfg) vshard.storage.cfg(vshard_cfg, vars.instance_uuid) vars.vshard_cfg = vshard_cfg end diff --git a/cartridge/test-helpers/cluster.lua b/cartridge/test-helpers/cluster.lua index 38bf5140f..f43018490 100644 --- a/cartridge/test-helpers/cluster.lua +++ b/cartridge/test-helpers/cluster.lua @@ -66,6 +66,7 @@ function Cluster:new(object) -- @tparam {string} roles List of roles for servers in the replicaset. -- @tparam ?string vshard_group Name of vshard group. -- @tparam ?number weight Vshard group weight. + -- @tparam ?boolean rebalancer Is rebalancer enabled. -- @tparam ?boolean all_rw Make all replicas writable. -- @tparam table|number servers List of objects to build `Server`s with or -- number of servers in replicaset. @@ -76,6 +77,7 @@ function Cluster:new(object) roles = 'table', vshard_group = '?string', weight = '?number', + rebalancer = '?boolean', servers = 'table|number', all_rw = '?boolean', }) end)(replicaset) @@ -205,6 +207,7 @@ function Cluster:apply_topology() uuid = server.instance_uuid, labels = server.labels, zone = server.zone, + rebalancer = server.rebalancer, }) end diff --git a/cartridge/test-helpers/server.lua b/cartridge/test-helpers/server.lua index 3214a69eb..b20b6e06c 100644 --- a/cartridge/test-helpers/server.lua +++ b/cartridge/test-helpers/server.lua @@ -26,6 +26,7 @@ local checks = require('checks') -- @string[opt] object.instance_uuid Server identifier. -- @string[opt] object.replicaset_uuid Replicaset identifier. -- @string[opt] object.zone Vshard zone. +-- @bool[opt] object.rebalancer Is vshard rebalancer enabled. -- @number[opt] object.swim_period SWIM protocol period in seconds. -- @return input object local Server = luatest.Server:inherit({}) @@ -42,6 +43,7 @@ Server.constructor_checks = fun.chain(Server.constructor_checks, { labels = '?table', zone = '?string', swim_period = '?number', + rebalancer = '?boolean', transport = '?string', ssl_ciphers = '?string', @@ -294,6 +296,7 @@ function Server:setup_replicaset(config) $roles: [String!], $master: [String!], $weight: Float, + $rebalancer: Boolean, $vshard_group: String ) { edit_replicaset( @@ -302,6 +305,7 @@ function Server:setup_replicaset(config) roles: $roles, master: $master, weight: $weight, + rebalancer: $rebalancer, vshard_group: $vshard_group ) } @@ -312,6 +316,7 @@ function Server:setup_replicaset(config) roles = config.roles, master = config.master, weight = config.weight, + rebalancer = config.rebalancer, vshard_group = config.vshard_group, } }) diff --git a/cartridge/topology.lua b/cartridge/topology.lua index 1bf92c1f6..47ab866eb 100755 --- a/cartridge/topology.lua +++ b/cartridge/topology.lua @@ -48,6 +48,7 @@ local e_config = errors.new_class('Invalid cluster topology config') -- electable = true, -- replicaset_uuid = 'replicaset-uuid-2', -- zone = nil | string, + -- rebalancer = nil | true, -- }, }, replicasets = { @@ -60,6 +61,7 @@ local e_config = errors.new_class('Invalid cluster topology config') -- master = {'instance-uuid-2', 'instance-uuid-1'} -- new format -- weight = 1.0, -- vshard_group = 'group_name', + -- rebalancer = nil | true, -- } }, }]] @@ -258,6 +260,12 @@ local function validate_schema(field, topology) '%s.zone must be a string, got %s', field, type(server.zone) ) + e_config:assert( + server.rebalancer == nil or + type(server.rebalancer) == 'boolean', + '%s.rebalancer must be a string, got %s', field, type(server.rebalancer) + ) + if rawget(_G, '__cartridge_log_invalid_labels') == true then local ok, err = label_utils.validate_labels(field, server) if not ok then @@ -270,6 +278,7 @@ local function validate_schema(field, topology) ['disabled'] = true, ['electable'] = true, ['replicaset_uuid'] = true, + ['rebalancer'] = true, ['labels'] = true, ['zone'] = true, } @@ -345,6 +354,11 @@ local function validate_schema(field, topology) '%s.vshard_group must be a string, got %s', field, type(replicaset.vshard_group) ) + e_config:assert( + (replicaset.rebalancer == nil) or (type(replicaset.rebalancer) == 'boolean'), + '%s.rebalancer must be a boolean, got %s', field, type(replicaset.rebalancer) + ) + for k, v in pairs(replicaset.roles) do e_config:assert( type(k) == 'string', @@ -368,6 +382,7 @@ local function validate_schema(field, topology) ['roles'] = true, ['master'] = true, ['weight'] = true, + ['rebalancer'] = true, ['vshard_group'] = true, ['all_rw'] = true, ['alias'] = true, diff --git a/cartridge/vshard-utils.lua b/cartridge/vshard-utils.lua index 157b61169..40ca4b5ac 100644 --- a/cartridge/vshard-utils.lua +++ b/cartridge/vshard-utils.lua @@ -30,7 +30,6 @@ vars:new('known_groups', nil -- rebalancer_disbalance_threshold = number, -- sched_ref_quota = number, -- sched_move_quota = number, - -- rebalancer = boolean, -- rebalancer_mode = string, -- } --} @@ -231,13 +230,6 @@ local function validate_vshard_group(field, vsgroup_new, vsgroup_old) ) end - if vsgroup_new.rebalancer ~= nil then - ValidateConfigError:assert( - type(vsgroup_new.rebalancer) == 'boolean', - '%s.rebalancer must be a boolean', field - ) - end - if vsgroup_new.rebalancer_mode ~= nil then ValidateConfigError:assert( type(vsgroup_new.rebalancer_mode) == 'string' @@ -289,7 +281,6 @@ local function validate_vshard_group(field, vsgroup_new, vsgroup_old) ['sync_timeout'] = true, ['collect_bucket_garbage_interval'] = true, ['rebalancer_disbalance_threshold'] = true, - ['rebalancer'] = true, ['rebalancer_mode'] = true, ['sched_ref_quota'] = true, ['sched_move_quota'] = true, @@ -462,11 +453,6 @@ local function get_known_groups() g.rebalancer_disbalance_threshold = vshard_consts.DEFAULT_REBALANCER_DISBALANCE_THRESHOLD end - -- just for the code consistancy - if g.rebalancer == nil then - g.rebalancer = nil - end - if g.rebalancer_mode == nil then g.rebalancer_mode = 'auto' end @@ -507,6 +493,7 @@ local function get_vshard_config(group_name, conf) sharding[replicaset_uuid] = { replicas = {}, weight = replicaset.weight or 0.0, + rebalancer = replicaset.rebalancer ~= nil and replicaset.rebalancer or nil, } end @@ -544,6 +531,7 @@ local function get_vshard_config(group_name, conf) name = server.uri, zone = server.zone, listen = listen_uri, + rebalancer = server.rebalancer ~= nil and server.rebalancer or nil, uri = client_uri, master = (active_leaders[replicaset_uuid] == instance_uuid), } @@ -588,7 +576,6 @@ local function get_vshard_config(group_name, conf) sync_timeout = vshard_groups[group_name].sync_timeout, collect_bucket_garbage_interval = vshard_groups[group_name].collect_bucket_garbage_interval, rebalancer_disbalance_threshold = vshard_groups[group_name].rebalancer_disbalance_threshold, - rebalancer = vshard_groups[group_name].rebalancer, rebalancer_mode = vshard_groups[group_name].rebalancer_mode, sharding = sharding, read_only = not failover.is_rw(), @@ -649,7 +636,6 @@ local function edit_vshard_options(group_name, vshard_options) sync_timeout = '?number', collect_bucket_garbage_interval = '?number', rebalancer_disbalance_threshold = '?number', - rebalancer = '?boolean', rebalancer_mode = '?string', sched_ref_quota = '?number', sched_move_quota = '?number', diff --git a/cartridge/webui/api-topology.lua b/cartridge/webui/api-topology.lua index fab340fa9..40d9df4b4 100755 --- a/cartridge/webui/api-topology.lua +++ b/cartridge/webui/api-topology.lua @@ -34,6 +34,10 @@ local gql_type_replicaset = gql_types.object { description = 'Vshard replica set weight.' .. ' Null for replica sets with vshard-storage role disabled.' }, + rebalancer = { + kind = gql_types.boolean, + description = 'Is the rebalancer enabled for the replica set.', + }, vshard_group = { kind = gql_types.string, description = 'Vshard storage group name.' .. @@ -91,6 +95,10 @@ local gql_type_server = gql_types.object { kind = gql_types.boolean, description = 'Is allowed to elect this instance as leader', }, + rebalancer = { + kind = gql_types.boolean, + description = 'Is rebalancer enabled for this instance', + }, disabled = gql_types.boolean, priority = { kind = gql_types.int, @@ -120,6 +128,7 @@ local gql_type_edit_server_input = gql_types.inputObject { zone = gql_types.string, labels = gql_types.list(gql_type_label_input), electable = gql_types.boolean, + rebalancer = gql_types.boolean, disabled = gql_types.boolean, expelled = gql_types.boolean, } @@ -140,6 +149,7 @@ local gql_type_edit_replicaset_input = gql_types.inputObject { uri = gql_types.string.nonNull, uuid = gql_types.string, zone = gql_types.string, + rebalancer = gql_types.boolean, labels = gql_types.list(gql_type_label_input), } }) @@ -147,6 +157,7 @@ local gql_type_edit_replicaset_input = gql_types.inputObject { failover_priority = gql_types.list(gql_types.string.nonNull), all_rw = gql_types.boolean, weight = gql_types.float, + rebalancer = gql_types.boolean, vshard_group = gql_types.string, } } diff --git a/cartridge/webui/api-vshard.lua b/cartridge/webui/api-vshard.lua index 9f78fb515..671d0b3d9 100644 --- a/cartridge/webui/api-vshard.lua +++ b/cartridge/webui/api-vshard.lua @@ -49,10 +49,6 @@ local gql_type_vsgroup = gql_types.object({ kind = gql_types.float.nonNull, description = 'A maximum bucket disbalance threshold, in percent' }, - rebalancer = { - kind = gql_types.boolean, - description = 'Run rebalancer on a specific replicaset' - }, rebalancer_mode = { kind = gql_types.string.nonNull, description = 'Rebalancer mode' @@ -179,7 +175,6 @@ local function init(graphql) sync_timeout = gql_types.float, collect_bucket_garbage_interval = gql_types.float, rebalancer_disbalance_threshold = gql_types.float, - rebalancer = gql_types.boolean, rebalancer_mode = gql_types.string, sched_ref_quota = gql_types.long, sched_move_quota = gql_types.long, diff --git a/cartridge/webui/gql-boxinfo.lua b/cartridge/webui/gql-boxinfo.lua index ad870aab5..5b2ca3907 100644 --- a/cartridge/webui/gql-boxinfo.lua +++ b/cartridge/webui/gql-boxinfo.lua @@ -428,6 +428,10 @@ local boxinfo_schema = { kind = gql_types.int, description = 'The number of buckets that are sending at this time', }, + rebalancer_enabled = { + kind = gql_types.boolean, + description = 'Whether the rebalancer is enabled', + } } }), } diff --git a/test/integration/vshard_rebalancer_api_test.lua b/test/integration/vshard_rebalancer_api_test.lua new file mode 100644 index 000000000..7e55c68b4 --- /dev/null +++ b/test/integration/vshard_rebalancer_api_test.lua @@ -0,0 +1,202 @@ +local fio = require('fio') +local t = require('luatest') +local g = t.group() + +local helpers = require('test.helper') + +g.before_all = function() + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_basic'), + cookie = 'secret', -- helpers.random_cookie(), + use_vshard = true, + replicasets = { + { + alias = 'A', + roles = {'vshard-router', 'vshard-storage'}, + servers = 2 + }, + }, + }) + g.cluster:start() +end + +g.after_all = function() + g.cluster:stop() + fio.rmtree(g.cluster.datadir) +end + +local function get_replicaset() + local res = g.cluster.main_server:graphql({ + query = [[{ + replicasets(uuid: "aaaaaaaa-0000-0000-0000-000000000000") { + weight + vshard_group + rebalancer + } + }]] + }) + return res['data']['replicasets'][1] +end + +g.test_rebalancer_replicaset_level = function() + local server = g.cluster:server('A-1') + local resp = g.cluster.main_server:graphql({ + query = [[ + mutation($replicasets: [EditReplicasetInput]) { + cluster { + edit_topology(replicasets: $replicasets) { + replicasets { + uuid + } + } + } + } + ]], + variables = { + replicasets = {{ + uuid = server.replicaset_uuid, + rebalancer = true, + }} + } + }) + + resp = g.cluster.main_server:graphql({ + query = [[{ + servers { + uri + boxinfo { + vshard_storage { + rebalancer_enabled + } + } + } + } + ]]}) + + table.sort(resp['data']['servers'], function(a, b) return a.uri < b.uri end) + t.assert_items_equals(resp['data']['servers'], { + { + uri = 'localhost:13301', + boxinfo = { + vshard_storage = { + rebalancer_enabled = true + } + } + }, + { + uri = 'localhost:13302', + boxinfo = { + vshard_storage = { + rebalancer_enabled = false + } + } + } + }) +end + +g.after_test('test_rebalancer_replicaset_level', function() + local server = g.cluster:server('A-1') + local resp = g.cluster.main_server:graphql({ + query = [[ + mutation($replicasets: [EditReplicasetInput]) { + cluster { + edit_topology(replicasets: $replicasets) { + replicasets { + uuid + rebalancer + } + } + } + } + ]], + variables = { + replicasets = {{ + uuid = server.replicaset_uuid, + rebalancer = box.NULL, + }} + } + }) + t.assert_equals(resp['data']['cluster']['edit_topology']['replicasets'], { + { + uuid = server.replicaset_uuid, + rebalancer = box.NULL, + } + }) +end) + +g.test_rebalancer_server_level = function() + local server = g.cluster:server('A-2') + local resp = g.cluster.main_server:graphql({query = [[ + mutation($servers: [EditServerInput]) { + cluster { + edit_topology(servers: $servers){ + servers {uuid} + } + } + } + ]], variables = { + servers = { + {uuid = server.instance_uuid, rebalancer = true}, + } + }}) + + resp = g.cluster.main_server:graphql({ + query = [[{ + servers { + uri + boxinfo { + vshard_storage { + rebalancer_enabled + } + } + } + } + ]]}) + + table.sort(resp['data']['servers'], function(a, b) return a.uri < b.uri end) + t.assert_items_equals(resp['data']['servers'], { + { + uri = 'localhost:13301', + boxinfo = { + vshard_storage = { + rebalancer_enabled = true + } + } + }, + { + uri = 'localhost:13302', + boxinfo = { + vshard_storage = { + rebalancer_enabled = false + } + } + } + }) +end + +g.after_test('test_rebalancer_server_level', function() + local server = g.cluster:server('A-1') + local resp = g.cluster.main_server:graphql({query = [[ + mutation($servers: [EditServerInput]) { + cluster { + edit_topology(servers: $servers){ + servers { + uuid + rebalancer + } + } + } + } + ]], variables = { + servers = { + {uuid = server.instance_uuid, rebalancer = box.NULL}, + } + }}) + t.assert_equals(resp['data']['cluster']['edit_topology']['servers'], { + { + uuid = server.instance_uuid, + rebalancer = box.NULL, + } + }) +end)