From db65182c289b3fd4a1dbac66059ec3c4772912d8 Mon Sep 17 00:00:00 2001 From: RayHuangCN Date: Tue, 18 May 2021 15:33:21 +0800 Subject: [PATCH] fix: unexpect scaling up when some targets is too big during shard alleviation --- pkg/coordinator/rebalance.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/coordinator/rebalance.go b/pkg/coordinator/rebalance.go index 580c2d6..c3f77ea 100644 --- a/pkg/coordinator/rebalance.go +++ b/pkg/coordinator/rebalance.go @@ -50,12 +50,12 @@ func newShardInfo(sd *shard.Shard) *shardInfo { func (s *shardInfo) totalTargetsSeries() int64 { ret := int64(0) - for _, s := range s.scraping { - if s.TargetState != target.StateNormal || s.Health != scrape.HealthGood || s.ScrapeTimes < minWaitScrapeTimes { + for _, tar := range s.scraping { + if tar.TargetState != target.StateNormal || tar.Health != scrape.HealthGood || tar.ScrapeTimes < minWaitScrapeTimes { continue } - ret += s.Series + ret += tar.Series } return ret } @@ -238,6 +238,11 @@ func (c *Coordinator) alleviateShard(s *shardInfo, changeAbleShards []*shardInfo continue } + if tar.Series > c.maxSeries { + c.log.Warnf("too big series [%d] series is [%d], skip alleviate", hash, tar.Series) + return 0 + } + // try transfer target to other shard for _, os := range changeAbleShards { if os == s {