bugfix: add restart empty sd protection

datasource/etcd/ops.go

@@ -98,9 +98,6 @@ func (ds *MetadataManager) CountEnvironment(ctx context.Context, request *ev.Get
 	if err != nil {
 		return nil, err
 	}
-	if err != nil {
-		return nil, err
-	}
 	return &ev.GetEnvironmentCountResponse{
 		Count: all - preEnvNum,
 	}, nil

go.sum

@@ -99,7 +99,7 @@ github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
-github.com/armon/go-radix v1.0.0/go.mod h1:TsTFsXBVHVK4HQ+UrFSsQEhBXZGCDqoY+cr+sUq5ZmA=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk=

pkg/protect/protect.go

@@ -0,0 +1,73 @@
+package protect
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/apache/servicecomb-service-center/server/config"
+
+	"github.com/apache/servicecomb-service-center/pkg/log"
+)
+
+/**
+for restart service center, set a restartProtectInterval time window to return RestartProtectHttpCode on discovery apis,
+indicating that sdk not need to clear cache
+*/
+
+var (
+	isWithinProtection        bool
+	startupTimestamp          int64
+	enableInstanceNullProtect bool
+	restartProtectInterval    time.Duration
+	RestartProtectHttpCode    int
+	validProtectCode          = map[int]struct{}{http.StatusNotModified: {}, http.StatusUnprocessableEntity: {}, http.StatusInternalServerError: {}}
+)
+
+const (
+	maxInterval                   = 120 * time.Second
+	minInterval                   = 0 * time.Second
+	defaultRestartProtectInterval = 120 * time.Second
+)
+
+func Init() {
+	enableInstanceNullProtect = config.GetBool("instance_null_protect.enable", false)
+	if !enableInstanceNullProtect {
+		return
+	}
+	restartProtectInterval = time.Duration(config.GetInt("instance_null_protect.restart_protect_interval", 120)) * time.Second
+	if restartProtectInterval > maxInterval || restartProtectInterval < minInterval {
+		log.Warn(fmt.Sprintf("invalid instance_null_protect.restart_protect_interval: %d,"+
+			" must between %d-%ds inclusively", restartProtectInterval, minInterval, maxInterval))
+		restartProtectInterval = defaultRestartProtectInterval
+	}
+	RestartProtectHttpCode = config.GetInt("instance_null_protect.http_status", http.StatusNotModified)
+	if _, ok := validProtectCode[RestartProtectHttpCode]; !ok {
+		log.Warn(fmt.Sprintf("invalid instance_null_protect.http_status: %d, must be %v", RestartProtectHttpCode, validProtectCode))
+		RestartProtectHttpCode = http.StatusNotModified
+	}
+
+	log.Info(fmt.Sprintf("instance_null_protect.enable: %t", enableInstanceNullProtect))
+	log.Info(fmt.Sprintf("instance_null_protect.restart_protect_interval: %d", restartProtectInterval))
+	log.Info(fmt.Sprintf("instance_null_protect.http_status: %d", RestartProtectHttpCode))
+	startupTimestamp = time.Now().UnixNano()
+	isWithinProtection = true
+}
+
+func IsWithinRestartProtection() bool {
+	if !enableInstanceNullProtect {
+		return false
+	}
+
+	if !isWithinProtection {
+		return false
+	}
+
+	if time.Now().Add(-restartProtectInterval).UnixNano() > startupTimestamp {
+		log.Info("restart protection stop")
+		isWithinProtection = false
+		return false
+	}
+	log.Info("within restart protection")
+	return true
+}

pkg/protect/protect_test.go

@@ -0,0 +1,30 @@
+package protect
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsWithinRestartProtection(t *testing.T) {
+	restartProtectInterval = 2 * time.Minute
+
+	// protection switch off
+	enableInstanceNullProtect = false
+	assert.False(t, IsWithinRestartProtection())
+	// within protection
+	enableInstanceNullProtect = true
+	isWithinProtection = true
+	startupTimestamp = time.Now().Add(-1 * time.Minute).UnixNano()
+	assert.True(t, IsWithinRestartProtection())
+
+	// protection delay exceed
+	enableInstanceNullProtect = true
+	isWithinProtection = true
+	startupTimestamp = time.Now().Add(-2 * time.Minute).Unix()
+	assert.False(t, IsWithinRestartProtection())
+
+	// always false after exceed
+	assert.False(t, IsWithinRestartProtection())
+}

pkg/rpc/client.go

@@ -20,8 +20,10 @@ package rpc
 import (
 	"crypto/tls"
 	"errors"
+	"time"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/resolver"
@@ -50,9 +52,12 @@ func GetPickFirstLbConn(config *Config) (*grpc.ClientConn, error) {
 }
 
 func GetRoundRobinLbConn(config *Config) (*grpc.ClientConn, error) {
+	connBackoff := backoff.DefaultConfig
+	connBackoff.MaxDelay = 30 * time.Second
 	return getLbConn(config, func() []grpc.DialOption {
 		return []grpc.DialOption{
 			grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`),
+			grpc.WithConnectParams(grpc.ConnectParams{Backoff: connBackoff}),
 		}
 	})
 }

server/resource/disco/instance_resource.go

@@ -19,18 +19,20 @@ package disco
 
 import (
 	"fmt"
+	"github.com/apache/servicecomb-service-center/pkg/protect"
 	"io"
 	"net/http"
 	"strings"
 
 	"github.com/go-chassis/go-chassis/v2/pkg/codec"
 
+	pb "github.com/go-chassis/cari/discovery"
+
 	"github.com/apache/servicecomb-service-center/datasource"
 	"github.com/apache/servicecomb-service-center/pkg/log"
 	"github.com/apache/servicecomb-service-center/pkg/rest"
 	"github.com/apache/servicecomb-service-center/pkg/util"
 	discosvc "github.com/apache/servicecomb-service-center/server/service/disco"
-	pb "github.com/go-chassis/cari/discovery"
 )
 
 type InstanceResource struct {
@@ -167,6 +169,10 @@ func (s *InstanceResource) FindInstances(w http.ResponseWriter, r *http.Request)
 		w.WriteHeader(http.StatusNotModified)
 		return
 	}
+	if len(resp.Instances) == 0 && protect.IsWithinRestartProtection() {
+		w.WriteHeader(protect.RestartProtectHttpCode)
+		return
+	}
 	rest.WriteResponse(w, r, nil, resp)
 }
 
@@ -266,6 +272,10 @@ func (s *InstanceResource) ListInstance(w http.ResponseWriter, r *http.Request)
 		w.WriteHeader(http.StatusNotModified)
 		return
 	}
+	if len(resp.Instances) == 0 && protect.IsWithinRestartProtection() {
+		w.WriteHeader(protect.RestartProtectHttpCode)
+		return
+	}
 	rest.WriteResponse(w, r, nil, resp)
 }
 

server/server.go

@@ -20,15 +20,19 @@ package server
 import (
 	"context"
 	"crypto/tls"
+	"github.com/apache/servicecomb-service-center/pkg/protect"
 	"os"
 
+	"github.com/gofiber/fiber/v2"
+
 	"github.com/apache/servicecomb-service-center/server/middleware"
 	"github.com/apache/servicecomb-service-center/server/resource/disco"
-	"github.com/gofiber/fiber/v2"
 
 	"github.com/go-chassis/go-chassis/v2"
 	chassisServer "github.com/go-chassis/go-chassis/v2/core/server"
 
+	"github.com/go-chassis/foundation/gopool"
+
 	"github.com/apache/servicecomb-service-center/datasource"
 	nf "github.com/apache/servicecomb-service-center/pkg/event"
 	"github.com/apache/servicecomb-service-center/pkg/log"
@@ -41,7 +45,6 @@ import (
 	"github.com/apache/servicecomb-service-center/server/plugin/security/tlsconf"
 	"github.com/apache/servicecomb-service-center/server/service/grc"
 	"github.com/apache/servicecomb-service-center/server/service/rbac"
-	"github.com/go-chassis/foundation/gopool"
 )
 
 var sc ServiceCenterServer
@@ -211,6 +214,7 @@ func (s *ServiceCenterServer) startServices() {
 
 func (s *ServiceCenterServer) startAPIService() {
 	s.APIServer.SetHostPort(s.Endpoint.Host, s.Endpoint.Port)
+	protect.Init()
 	s.APIServer.Start()
 }
 

server/service/disco/instance.go

@@ -25,6 +25,9 @@ import (
 	"sync"
 	"time"
 
+	pb "github.com/go-chassis/cari/discovery"
+	"github.com/go-chassis/cari/pkg/errsvc"
+
 	"github.com/apache/servicecomb-service-center/datasource"
 	"github.com/apache/servicecomb-service-center/pkg/log"
 	"github.com/apache/servicecomb-service-center/pkg/util"
@@ -33,8 +36,6 @@ import (
 	"github.com/apache/servicecomb-service-center/server/health"
 	quotasvc "github.com/apache/servicecomb-service-center/server/service/quota"
 	"github.com/apache/servicecomb-service-center/server/service/validator"
-	pb "github.com/go-chassis/cari/discovery"
-	"github.com/go-chassis/cari/pkg/errsvc"
 )
 
 const (