Merge pull request #136 from swapnilgm/cherry-pick-fixes

Cherry pick fixes from master
gardener · Mar 13, 2019 · 89fbe7f · 89fbe7f
2 parents 6663f2f + 167a406
commit 89fbe7f
Show file tree

Hide file tree

Showing 14 changed files with 188 additions and 139 deletions.
diff --git a/chart/templates/etcd-statefulset.yaml b/chart/templates/etcd-statefulset.yaml
@@ -91,6 +91,9 @@ spec:
         - --data-dir=/var/etcd/data/new.etcd
         - --storage-provider={{ .Values.backup.storageProvider }}
         - --store-prefix=etcd-{{ .Values.role }}
+{{- if .Values.backup.embeddedEtcdQuotaBytes }}
+        - --embeddedEtcdQuotaBytes={{ .Values.backup.embeddedEtcdQuotaBytes }}
+{{- end }}
 {{- if .Values.tls }}
         - --cert=/var/etcd/ssl/client/tls.crt
         - --key=/var/etcd/ssl/client/tls.key

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -29,6 +29,8 @@ backup:
 
   storageContainer: ""
 
+  embeddedEtcdQuotaBytes: 8589934592
+
   # env should include the right list of values for the
   # storageProvider you use, follow the comments below
   env: []           # Follow comments below

diff --git a/cmd/initializer.go b/cmd/initializer.go
@@ -49,13 +49,14 @@ func NewInitializeCommand(stopCh <-chan struct{}) *cobra.Command {
 			}
 
 			options := &restorer.RestoreOptions{
-				RestoreDataDir: path.Clean(restoreDataDir),
-				Name:           restoreName,
-				ClusterURLs:    clusterUrlsMap,
-				PeerURLs:       peerUrls,
-				ClusterToken:   restoreClusterToken,
-				SkipHashCheck:  skipHashCheck,
-				MaxFetchers:    restoreMaxFetchers,
+				RestoreDataDir:         path.Clean(restoreDataDir),
+				Name:                   restoreName,
+				ClusterURLs:            clusterUrlsMap,
+				PeerURLs:               peerUrls,
+				ClusterToken:           restoreClusterToken,
+				SkipHashCheck:          skipHashCheck,
+				MaxFetchers:            restoreMaxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
 			}
 
 			var snapstoreConfig *snapstore.Config

diff --git a/cmd/restore.go b/cmd/restore.go
@@ -73,15 +73,16 @@ func NewRestoreCommand(stopCh <-chan struct{}) *cobra.Command {
 			rs := restorer.NewRestorer(store, logger)
 
 			options := &restorer.RestoreOptions{
-				RestoreDataDir: path.Clean(restoreDataDir),
-				Name:           restoreName,
-				BaseSnapshot:   *baseSnap,
-				DeltaSnapList:  deltaSnapList,
-				ClusterURLs:    clusterUrlsMap,
-				PeerURLs:       peerUrls,
-				ClusterToken:   restoreClusterToken,
-				SkipHashCheck:  skipHashCheck,
-				MaxFetchers:    restoreMaxFetchers,
+				RestoreDataDir:         path.Clean(restoreDataDir),
+				Name:                   restoreName,
+				BaseSnapshot:           *baseSnap,
+				DeltaSnapList:          deltaSnapList,
+				ClusterURLs:            clusterUrlsMap,
+				PeerURLs:               peerUrls,
+				ClusterToken:           restoreClusterToken,
+				SkipHashCheck:          skipHashCheck,
+				MaxFetchers:            restoreMaxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
 			}
 
 			err = rs.Restore(*options)
@@ -107,6 +108,7 @@ func initializeEtcdFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&restoreName, "name", defaultName, "human-readable name for this member")
 	cmd.Flags().BoolVar(&skipHashCheck, "skip-hash-check", false, "ignore snapshot integrity hash value (required if copied from data directory)")
 	cmd.Flags().IntVar(&restoreMaxFetchers, "max-fetchers", 6, "maximum number of threads that will fetch delta snapshots in parallel")
+	cmd.Flags().Int64Var(&embeddedEtcdQuotaBytes, "embedded-etcd-quota-bytes", int64(8*1024*1024*1024), "maximum backend quota for the embedded etcd used for applying delta snapshots")
 }
 
 func initialClusterFromName(name string) string {

diff --git a/cmd/server.go b/cmd/server.go
@@ -57,13 +57,14 @@ func NewServerCommand(stopCh <-chan struct{}) *cobra.Command {
 			}
 
 			options := &restorer.RestoreOptions{
-				RestoreDataDir: path.Clean(restoreDataDir),
-				Name:           restoreName,
-				ClusterURLs:    clusterUrlsMap,
-				PeerURLs:       peerUrls,
-				ClusterToken:   restoreClusterToken,
-				SkipHashCheck:  skipHashCheck,
-				MaxFetchers:    restoreMaxFetchers,
+				RestoreDataDir:         path.Clean(restoreDataDir),
+				Name:                   restoreName,
+				ClusterURLs:            clusterUrlsMap,
+				PeerURLs:               peerUrls,
+				ClusterToken:           restoreClusterToken,
+				SkipHashCheck:          skipHashCheck,
+				MaxFetchers:            restoreMaxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
 			}
 
 			if storageProvider != "" {

diff --git a/cmd/types.go b/cmd/types.go
@@ -49,13 +49,14 @@ var (
 	enableProfiling bool
 
 	//restore flags
-	restoreCluster      string
-	restoreClusterToken string
-	restoreDataDir      string
-	restorePeerURLs     []string
-	restoreName         string
-	skipHashCheck       bool
-	restoreMaxFetchers  int
+	restoreCluster         string
+	restoreClusterToken    string
+	restoreDataDir         string
+	restorePeerURLs        []string
+	restoreName            string
+	skipHashCheck          bool
+	restoreMaxFetchers     int
+	embeddedEtcdQuotaBytes int64
 
 	//snapstore flags
 	storageProvider         string

diff --git a/pkg/snapshot/restorer/restorer.go b/pkg/snapshot/restorer/restorer.go
@@ -63,6 +63,10 @@ func (r *Restorer) Restore(ro RestoreOptions) error {
 	if ro.MaxFetchers < 1 {
 		return fmt.Errorf("Maximum number of fetchers should be greater than zero. Input MaxFetchers: %d", ro.MaxFetchers)
 	}
+	if ro.EmbeddedEtcdQuotaBytes <= 0 {
+		r.logger.Infof("Quota size for etcd must be greater than 0. Input EmbeddedEtcdQuotaBytes: %d. Defaulting to 8GB.", ro.EmbeddedEtcdQuotaBytes)
+		ro.EmbeddedEtcdQuotaBytes = int64(8 * 1024 * 1024 * 1024)
+	}
 	if err := r.restoreFromBaseSnapshot(ro); err != nil {
 		return fmt.Errorf("failed to restore from the base snapshot :%v", err)
 	}
@@ -308,6 +312,7 @@ func makeWALAndSnap(waldir, snapdir string, cl *membership.RaftCluster, restoreN
 func startEmbeddedEtcd(ro RestoreOptions) (*embed.Etcd, error) {
 	cfg := embed.NewConfig()
 	cfg.Dir = filepath.Join(ro.RestoreDataDir)
+	cfg.QuotaBackendBytes = ro.EmbeddedEtcdQuotaBytes
 	e, err := embed.StartEtcd(cfg)
 	if err != nil {
 		return nil, err

diff --git a/pkg/snapshot/restorer/restorer_test.go b/pkg/snapshot/restorer/restorer_test.go
@@ -39,13 +39,14 @@ var _ = Describe("Running Restorer", func() {
 		store snapstore.SnapStore
 		rstr  *Restorer
 
-		restoreCluster      string
-		restoreClusterToken string
-		restoreDataDir      string
-		restorePeerURLs     []string
-		restoreName         string
-		skipHashCheck       bool
-		maxFetchers         int
+		restoreCluster         string
+		restoreClusterToken    string
+		restoreDataDir         string
+		restorePeerURLs        []string
+		restoreName            string
+		skipHashCheck          bool
+		maxFetchers            int
+		embeddedEtcdQuotaBytes int64
 
 		clusterUrlsMap types.URLsMap
 		peerUrls       types.URLs
@@ -68,6 +69,7 @@ var _ = Describe("Running Restorer", func() {
 		Expect(err).ShouldNot(HaveOccurred())
 		skipHashCheck = false
 		maxFetchers = 6
+		embeddedEtcdQuotaBytes = 8 * 1024 * 1024 * 1024
 
 		err = corruptEtcdDir()
 		Expect(err).ShouldNot(HaveOccurred())
@@ -87,15 +89,16 @@ var _ = Describe("Running Restorer", func() {
 			maxFetchers = 0
 
 			restoreOptions := RestoreOptions{
-				ClusterURLs:    clusterUrlsMap,
-				ClusterToken:   restoreClusterToken,
-				RestoreDataDir: restoreDataDir,
-				PeerURLs:       peerUrls,
-				SkipHashCheck:  skipHashCheck,
-				Name:           restoreName,
-				MaxFetchers:    maxFetchers,
-				BaseSnapshot:   *baseSnapshot,
-				DeltaSnapList:  deltaSnapList,
+				ClusterURLs:            clusterUrlsMap,
+				ClusterToken:           restoreClusterToken,
+				RestoreDataDir:         restoreDataDir,
+				PeerURLs:               peerUrls,
+				SkipHashCheck:          skipHashCheck,
+				Name:                   restoreName,
+				MaxFetchers:            maxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
+				BaseSnapshot:           *baseSnapshot,
+				DeltaSnapList:          deltaSnapList,
 			}
 			err = rstr.Restore(restoreOptions)
 			Expect(err).Should(HaveOccurred())
@@ -108,15 +111,16 @@ var _ = Describe("Running Restorer", func() {
 			maxFetchers = 1
 
 			restoreOptions := RestoreOptions{
-				ClusterURLs:    clusterUrlsMap,
-				ClusterToken:   restoreClusterToken,
-				RestoreDataDir: restoreDataDir,
-				PeerURLs:       peerUrls,
-				SkipHashCheck:  skipHashCheck,
-				Name:           restoreName,
-				MaxFetchers:    maxFetchers,
-				BaseSnapshot:   *baseSnapshot,
-				DeltaSnapList:  deltaSnapList,
+				ClusterURLs:            clusterUrlsMap,
+				ClusterToken:           restoreClusterToken,
+				RestoreDataDir:         restoreDataDir,
+				PeerURLs:               peerUrls,
+				SkipHashCheck:          skipHashCheck,
+				Name:                   restoreName,
+				MaxFetchers:            maxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
+				BaseSnapshot:           *baseSnapshot,
+				DeltaSnapList:          deltaSnapList,
 			}
 			err = rstr.Restore(restoreOptions)
 			Expect(err).ShouldNot(HaveOccurred())
@@ -132,15 +136,16 @@ var _ = Describe("Running Restorer", func() {
 			maxFetchers = 4
 
 			restoreOptions := RestoreOptions{
-				ClusterURLs:    clusterUrlsMap,
-				ClusterToken:   restoreClusterToken,
-				RestoreDataDir: restoreDataDir,
-				PeerURLs:       peerUrls,
-				SkipHashCheck:  skipHashCheck,
-				Name:           restoreName,
-				MaxFetchers:    maxFetchers,
-				BaseSnapshot:   *baseSnapshot,
-				DeltaSnapList:  deltaSnapList,
+				ClusterURLs:            clusterUrlsMap,
+				ClusterToken:           restoreClusterToken,
+				RestoreDataDir:         restoreDataDir,
+				PeerURLs:               peerUrls,
+				SkipHashCheck:          skipHashCheck,
+				Name:                   restoreName,
+				MaxFetchers:            maxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
+				BaseSnapshot:           *baseSnapshot,
+				DeltaSnapList:          deltaSnapList,
 			}
 			err = rstr.Restore(restoreOptions)
 			Expect(err).ShouldNot(HaveOccurred())
@@ -156,15 +161,16 @@ var _ = Describe("Running Restorer", func() {
 			maxFetchers = 100
 
 			restoreOptions := RestoreOptions{
-				ClusterURLs:    clusterUrlsMap,
-				ClusterToken:   restoreClusterToken,
-				RestoreDataDir: restoreDataDir,
-				PeerURLs:       peerUrls,
-				SkipHashCheck:  skipHashCheck,
-				Name:           restoreName,
-				MaxFetchers:    maxFetchers,
-				BaseSnapshot:   *baseSnapshot,
-				DeltaSnapList:  deltaSnapList,
+				ClusterURLs:            clusterUrlsMap,
+				ClusterToken:           restoreClusterToken,
+				RestoreDataDir:         restoreDataDir,
+				PeerURLs:               peerUrls,
+				SkipHashCheck:          skipHashCheck,
+				Name:                   restoreName,
+				MaxFetchers:            maxFetchers,
+				EmbeddedEtcdQuotaBytes: embeddedEtcdQuotaBytes,
+				BaseSnapshot:           *baseSnapshot,
+				DeltaSnapList:          deltaSnapList,
 			}
 			err = rstr.Restore(restoreOptions)
 			Expect(err).ShouldNot(HaveOccurred())

diff --git a/pkg/snapshot/restorer/types.go b/pkg/snapshot/restorer/types.go
@@ -36,13 +36,14 @@ type Restorer struct {
 
 // RestoreOptions hold all snapshot restore related fields
 type RestoreOptions struct {
-	ClusterURLs    types.URLsMap
-	ClusterToken   string
-	RestoreDataDir string
-	PeerURLs       types.URLs
-	SkipHashCheck  bool
-	Name           string
-	MaxFetchers    int
+	ClusterURLs            types.URLsMap
+	ClusterToken           string
+	RestoreDataDir         string
+	PeerURLs               types.URLs
+	SkipHashCheck          bool
+	Name                   string
+	MaxFetchers            int
+	EmbeddedEtcdQuotaBytes int64
 	// Base full snapshot + delta snapshots to restore from
 	BaseSnapshot  snapstore.Snapshot
 	DeltaSnapList snapstore.SnapList