diff --git a/go.mod b/go.mod index eaad9f94c..41de99982 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/frankban/quicktest v1.9.0 // indirect github.com/gardener/etcd-druid v0.3.0 - github.com/gardener/gardener v1.5.0 + github.com/gardener/gardener v1.5.1 github.com/gardener/gardener-extension-networking-calico v1.7.1-0.20200522070525-f9aa28d3c83a github.com/gardener/machine-controller-manager v0.27.0 github.com/go-logr/logr v0.1.0 diff --git a/go.sum b/go.sum index 983454408..4ba76ea55 100644 --- a/go.sum +++ b/go.sum @@ -152,8 +152,8 @@ github.com/gardener/external-dns-management v0.7.7/go.mod h1:egCe/FPOsUbXA4WV0ne github.com/gardener/gardener v1.1.2/go.mod h1:CP9I0tCDVXTLPkJv/jUtXVUh948kSNKEGUg0haLz9gk= github.com/gardener/gardener v1.3.1/go.mod h1:936P5tQbg6ViiW8BVC9ELM95sFrk4DgobKrxMNtn/LU= github.com/gardener/gardener v1.4.1-0.20200519155656-a8ccc6cc779a/go.mod h1:t9oESM37bAMIuezi9I0H0I8+++8jy8BUPitcf4ERRXY= -github.com/gardener/gardener v1.5.0 h1:rFnoBqnZqL/POYn6r7dDu7Bpxe9jF4Wj6/9LtsXPWrw= -github.com/gardener/gardener v1.5.0/go.mod h1:V+RVjUftDzhmb2ztBpf0uzyo1xoBjoDn0KKe0z+8IE0= +github.com/gardener/gardener v1.5.1 h1:sI/DY72/N1d+t8TmdgmkrpsdDhnkWcnjBW1MbkGvr40= +github.com/gardener/gardener v1.5.1/go.mod h1:V+RVjUftDzhmb2ztBpf0uzyo1xoBjoDn0KKe0z+8IE0= github.com/gardener/gardener-extension-networking-calico v1.7.1-0.20200522070525-f9aa28d3c83a h1:jBvyEhkRzW11Nz2y9IIQAo9HUaCvCqxEko5Nf9NRYUI= github.com/gardener/gardener-extension-networking-calico v1.7.1-0.20200522070525-f9aa28d3c83a/go.mod h1:bmD89OLvEBbXLlznsHe90ZlgTU+OrKErwHb6NWlSTvY= github.com/gardener/gardener-resource-manager v0.10.0/go.mod h1:0pKTHOhvU91eQB0EYr/6Ymd7lXc/5Hi8P8tF/gpV0VQ= diff --git a/vendor/github.com/gardener/gardener/extensions/pkg/controller/healthcheck/reconciler.go b/vendor/github.com/gardener/gardener/extensions/pkg/controller/healthcheck/reconciler.go index c4028ffa7..9885f6abb 100644 --- a/vendor/github.com/gardener/gardener/extensions/pkg/controller/healthcheck/reconciler.go +++ b/vendor/github.com/gardener/gardener/extensions/pkg/controller/healthcheck/reconciler.go @@ -210,7 +210,7 @@ func (r *reconciler) updateExtensionConditionToConditionCheckError(ctx context.C func (r *reconciler) updateExtensionConditionToUnsuccessful(ctx context.Context, conditionBuilder gardencorev1beta1helper.ConditionBuilder, healthConditionType string, extension extensionsv1alpha1.Object, healthCheckResult Result) error { var ( numberOfChecks = healthCheckResult.UnsuccessfulChecks + healthCheckResult.ProgressingChecks + healthCheckResult.SuccessfulChecks - detail = fmt.Sprintf("Health check summary: %d/%d unsuccessful, %d/%d progressing, %d/%d successful. ", healthCheckResult.UnsuccessfulChecks, numberOfChecks, healthCheckResult.ProgressingChecks, numberOfChecks, healthCheckResult.SuccessfulChecks, numberOfChecks) + detail = fmt.Sprintf("Health check summary: %d/%d unsuccessful, %d/%d progressing, %d/%d successful. %v", healthCheckResult.UnsuccessfulChecks, numberOfChecks, healthCheckResult.ProgressingChecks, numberOfChecks, healthCheckResult.SuccessfulChecks, numberOfChecks, healthCheckResult.GetDetails()) status = gardencorev1beta1.ConditionFalse reason = ReasonUnsuccessful ) diff --git a/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go b/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go index 0cbafebc5..f34562fb4 100644 --- a/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go +++ b/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go @@ -46,19 +46,19 @@ func (a *genericActuator) Migrate(ctx context.Context, worker *extensionsv1alpha } if err := a.waitUntilMachineControllerManagerIsDeleted(ctx, worker.Namespace); err != nil { - return errors.Wrap(err, "failed deleting machine-controller-manager manager") + return errors.Wrap(err, "failed deleting machine-controller-manager") } if err := a.shallowDeleteAllObjects(ctx, worker.Namespace, &machinev1alpha1.MachineList{}); err != nil { - return errors.Wrap(err, "shallow Deletion of all machine failed") + return errors.Wrap(err, "shallow deletion of all machine failed") } if err := a.shallowDeleteAllObjects(ctx, worker.Namespace, &machinev1alpha1.MachineSetList{}); err != nil { - return errors.Wrap(err, "shallow Deletion of all machineSets failed") + return errors.Wrap(err, "shallow deletion of all machineSets failed") } if err := a.shallowDeleteAllObjects(ctx, worker.Namespace, &machinev1alpha1.MachineDeploymentList{}); err != nil { - return errors.Wrap(err, "shallow Deletion of all machineDeployments failed") + return errors.Wrap(err, "shallow deletion of all machineDeployments failed") } if err := a.shallowDeleteAllObjects(ctx, worker.Namespace, workerDelegate.MachineClassList()); err != nil { diff --git a/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go b/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go index 97775fbc6..c0793c8cc 100644 --- a/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go +++ b/vendor/github.com/gardener/gardener/extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go @@ -311,7 +311,7 @@ func (a *genericActuator) deployMachineDeployments(ctx context.Context, cluster // available by the machine-controller-manager. It polls the status every 5 seconds. func (a *genericActuator) waitUntilWantedMachineDeploymentsAvailable(ctx context.Context, cluster *controller.Cluster, worker *extensionsv1alpha1.Worker, wantedMachineDeployments worker.MachineDeployments) error { return retryutils.UntilTimeout(ctx, 5*time.Second, 5*time.Minute, func(ctx context.Context) (bool, error) { - var numHealthyDeployments, numUpdated, numDesired, numberOfAwakeMachines int32 + var numHealthyDeployments, numUpdated, numAvailable, numDesired, numberOfAwakeMachines int32 // Get the list of all existing machine deployments existingMachineDeployments := &machinev1alpha1.MachineDeploymentList{} @@ -319,7 +319,7 @@ func (a *genericActuator) waitUntilWantedMachineDeploymentsAvailable(ctx context return retryutils.SevereError(err) } - // Collect the numbers of ready and desired replicas. + // Collect the numbers of available and desired replicas. for _, existingMachineDeployment := range existingMachineDeployments.Items { // Filter out all machine deployments that are not desired (any more). if !wantedMachineDeployments.HasDeployment(existingMachineDeployment.Name) { @@ -334,19 +334,20 @@ func (a *genericActuator) waitUntilWantedMachineDeploymentsAvailable(ctx context } // If the Shoot is not hibernated we want to wait until all wanted machine deployments have as many - // ready replicas as desired (specified in the .spec.replicas). However, if we see any error in the + // available replicas as desired (specified in the .spec.replicas). However, if we see any error in the // status of the deployment then we return it. for _, failedMachine := range existingMachineDeployment.Status.FailedMachines { return retryutils.SevereError(fmt.Errorf("machine %s failed: %s", failedMachine.Name, failedMachine.LastOperation.Description)) } // If the Shoot is not hibernated we want to wait until all wanted machine deployments have as many - // ready replicas as desired (specified in the .spec.replicas). + // available replicas as desired (specified in the .spec.replicas). if workerhealthcheck.CheckMachineDeployment(&existingMachineDeployment) == nil { numHealthyDeployments++ } numDesired += existingMachineDeployment.Spec.Replicas numUpdated += existingMachineDeployment.Status.UpdatedReplicas + numAvailable += existingMachineDeployment.Status.AvailableReplicas } var msg string @@ -356,7 +357,7 @@ func (a *genericActuator) waitUntilWantedMachineDeploymentsAvailable(ctx context if numUpdated >= numDesired && int(numHealthyDeployments) == len(wantedMachineDeployments) { return retryutils.Ok() } - msg = fmt.Sprintf("Waiting until all desired machines are ready (%d/%d machine objects up-to-date, %d/%d machinedeployments available)...", numUpdated, numDesired, numHealthyDeployments, len(wantedMachineDeployments)) + msg = fmt.Sprintf("Waiting until all desired machines are available (%d/%d machine objects available, %d/%d machinedeployments available)...", numAvailable, numDesired, numHealthyDeployments, len(wantedMachineDeployments)) default: if numberOfAwakeMachines == 0 { return retryutils.Ok() diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/errors.go b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/errors.go index b2b056a20..a1e260e22 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/errors.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/errors.go @@ -28,23 +28,24 @@ import ( "k8s.io/apimachinery/pkg/util/sets" ) -type errorWithCodes struct { +// ErrorWithCodes contains error codes and an error message. +type ErrorWithCodes struct { message string codes []gardencorev1beta1.ErrorCode } // NewErrorWithCodes creates a new error that additionally exposes the given codes via the Coder interface. func NewErrorWithCodes(message string, codes ...gardencorev1beta1.ErrorCode) error { - return &errorWithCodes{message, codes} + return &ErrorWithCodes{message, codes} } // Codes returns all error codes. -func (e *errorWithCodes) Codes() []gardencorev1beta1.ErrorCode { +func (e *ErrorWithCodes) Codes() []gardencorev1beta1.ErrorCode { return e.codes } // Error returns the error message. -func (e *errorWithCodes) Error() string { +func (e *ErrorWithCodes) Error() string { return e.message } @@ -72,7 +73,7 @@ func DetermineError(err error, message string) error { if codes == nil { return errors.New(errMsg) } - return &errorWithCodes{errMsg, codes} + return &ErrorWithCodes{errMsg, codes} } // DetermineErrorCodes determines error codes based on the given error. diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go index 3819f0aa6..e58b1ace3 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go @@ -878,3 +878,38 @@ func GetResourceByName(resources []gardencorev1beta1.NamedResourceReference, nam } return nil } + +// UpsertLastError adds a 'last error' to the given list of existing 'last errors' if it does not exist yet. Otherwise, +// it updates it. +func UpsertLastError(lastErrors []gardencorev1beta1.LastError, lastError gardencorev1beta1.LastError) []gardencorev1beta1.LastError { + var ( + out []gardencorev1beta1.LastError + found bool + ) + + for _, lastErr := range lastErrors { + if lastErr.TaskID != nil && lastError.TaskID != nil && *lastErr.TaskID == *lastError.TaskID { + out = append(out, lastError) + found = true + } else { + out = append(out, lastErr) + } + } + + if !found { + out = append(out, lastError) + } + + return out +} + +// DeleteLastErrorByTaskID removes the 'last error' with the given task ID from the given 'last error' list. +func DeleteLastErrorByTaskID(lastErrors []gardencorev1beta1.LastError, taskID string) []gardencorev1beta1.LastError { + var out []gardencorev1beta1.LastError + for _, lastErr := range lastErrors { + if lastErr.TaskID == nil || taskID != *lastErr.TaskID { + out = append(out, lastErr) + } + } + return out +} diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go index 5206d18e0..cf8ecf995 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go @@ -19,6 +19,7 @@ import ( "fmt" "sort" "strings" + "time" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" @@ -33,6 +34,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + // DefaultInterval is the default interval for retry operations. + DefaultInterval = 5 * time.Second + // DefaultSevereThreshold is the default threshold until an error reported by another component is treated as 'severe'. + DefaultSevereThreshold = 30 * time.Second +) + // New takes an operation object and creates a new Botanist object. It checks whether the given Shoot DNS // domain is covered by a default domain, and if so, it sets the attribute on the Botanist // object. diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/cleanup.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/cleanup.go index 44a69309c..39e29616a 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/cleanup.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/cleanup.go @@ -16,7 +16,6 @@ package botanist import ( "context" - "time" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -46,9 +45,6 @@ import ( ) const ( - // DefaultInterval is the default interval for retry operations. - DefaultInterval = 5 * time.Second - // Provider is the kubernetes provider label. Provider = "provider" // KubernetesProvider is the 'kubernetes' value of the Provider label. diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/containerruntime.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/containerruntime.go index 76fe0db9a..9bfc86ac6 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/containerruntime.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/containerruntime.go @@ -100,6 +100,7 @@ func (b *Botanist) WaitUntilContainerRuntimeResourcesReady(ctx context.Context) b.Shoot.SeedNamespace, getContainerRuntimeKey(containerRuntime.Type, worker.Name), DefaultInterval, + DefaultSevereThreshold, shoot.ExtensionDefaultTimeout, nil, ) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go index c55236c85..a42f8d045 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go @@ -498,6 +498,7 @@ func (b *Botanist) waitUntilControlPlaneReady(ctx context.Context, name string) b.Shoot.SeedNamespace, name, DefaultInterval, + DefaultSevereThreshold, ControlPlaneDefaultTimeout, func(o runtime.Object) error { obj, ok := o.(extensionsv1alpha1.Object) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsentry.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsentry.go index 037287747..0087faca3 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsentry.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsentry.go @@ -90,9 +90,16 @@ func (d *dnsEntry) Wait(ctx context.Context) error { var ( status string message string + + retryCountUntilSevere int + interval = 5 * time.Second + severeThreshold = 15 * time.Second + timeout = 2 * time.Minute ) - if err := d.waiter.UntilTimeout(ctx, 5*time.Second, 2*time.Minute, func(ctx context.Context) (done bool, err error) { + if err := d.waiter.UntilTimeout(ctx, interval, timeout, func(ctx context.Context) (done bool, err error) { + retryCountUntilSevere++ + entry := &dnsv1alpha1.DNSEntry{} if err := d.client.Get( ctx, @@ -114,8 +121,9 @@ func (d *dnsEntry) Wait(ctx context.Context) error { d.logger.Infof("Waiting for %q DNS record to be ready... (status=%s, message=%s)", d.values.Name, status, message) if status == dnsv1alpha1.STATE_ERROR || status == dnsv1alpha1.STATE_INVALID { - return retry.SevereError(entryErr) + return retry.MinorOrSevereError(retryCountUntilSevere, int(severeThreshold.Nanoseconds()/interval.Nanoseconds()), entryErr) } + return retry.MinorError(entryErr) }); err != nil { return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Failed to create %q DNS record: %q (status=%s, message=%s)", d.values.Name, err.Error(), status, message)) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsprovider.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsprovider.go index e8e07e832..656614c9d 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsprovider.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/dns/dnsprovider.go @@ -97,9 +97,14 @@ func (d *dnsProvider) Wait(ctx context.Context) error { var ( status string message string + + retryCountUntilSevere int + interval = 5 * time.Second + severeThreshold = 15 * time.Second + timeout = 2 * time.Minute ) - if err := d.waiter.UntilTimeout(ctx, 5*time.Second, 2*time.Minute, func(ctx context.Context) (done bool, err error) { + if err := d.waiter.UntilTimeout(ctx, interval, timeout, func(ctx context.Context) (done bool, err error) { provider := &dnsv1alpha1.DNSProvider{} if err := d.client.Get( ctx, @@ -121,8 +126,9 @@ func (d *dnsProvider) Wait(ctx context.Context) error { d.logger.Infof("Waiting for %q DNS provider to be ready... (status=%s, message=%s)", d.values.Name, status, message) if status == dnsv1alpha1.STATE_ERROR || status == dnsv1alpha1.STATE_INVALID { - return retry.SevereError(providerErr) + return retry.MinorOrSevereError(retryCountUntilSevere, int(severeThreshold.Nanoseconds()/interval.Nanoseconds()), providerErr) } + return retry.MinorError(providerErr) }); err != nil { return gardencorev1beta1helper.DetermineError(err, fmt.Sprintf("Failed to create DNS provider for %q DNS record: %q (status=%s, message=%s)", d.values.Name, err.Error(), status, message)) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/extension.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/extension.go index 74045b863..c4186a60b 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/extension.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/extension.go @@ -95,6 +95,7 @@ func (b *Botanist) WaitUntilExtensionResourcesReady(ctx context.Context) error { extension.Namespace, extension.Name, DefaultInterval, + DefaultSevereThreshold, extension.Timeout, nil, ) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/infrastructure.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/infrastructure.go index f9d4d8c5f..c0dec06d8 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/infrastructure.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/infrastructure.go @@ -120,6 +120,7 @@ func (b *Botanist) WaitUntilInfrastructureReady(ctx context.Context) error { b.Shoot.SeedNamespace, b.Shoot.Info.Name, DefaultInterval, + DefaultSevereThreshold, InfrastructureDefaultTimeout, func(obj runtime.Object) error { infrastructure, ok := obj.(*extensionsv1alpha1.Infrastructure) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/network.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/network.go index 4e7c03bec..b438c0f1b 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/network.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/network.go @@ -101,6 +101,7 @@ func (b *Botanist) WaitUntilNetworkIsReady(ctx context.Context) error { b.Shoot.SeedNamespace, b.Shoot.Info.Name, DefaultInterval, + DefaultSevereThreshold, NetworkDefaultTimeout, nil, ) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/operatingsystemconfig.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/operatingsystemconfig.go index b3b0161ea..d53288d63 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/operatingsystemconfig.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/operatingsystemconfig.go @@ -375,6 +375,7 @@ func (b *Botanist) applyAndWaitForShootOperatingSystemConfig(ctx context.Context b.Shoot.SeedNamespace, name, DefaultInterval, + 15*time.Second, 30*time.Second, func(obj runtime.Object) error { o, ok := obj.(*extensionsv1alpha1.OperatingSystemConfig) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/waiter.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/waiter.go index dc72c9f9e..db15df773 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/waiter.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/waiter.go @@ -78,7 +78,16 @@ func (b *Botanist) WaitUntilNginxIngressServiceIsReady(ctx context.Context) erro // WaitUntilEtcdReady waits until the etcd statefulsets indicate readiness in their statuses. func (b *Botanist) WaitUntilEtcdReady(ctx context.Context) error { - return retry.UntilTimeout(ctx, 5*time.Second, 300*time.Second, func(ctx context.Context) (done bool, err error) { + var ( + retryCountUntilSevere int + interval = 5 * time.Second + severeThreshold = 30 * time.Second + timeout = 5 * time.Minute + ) + + return retry.UntilTimeout(ctx, interval, timeout, func(ctx context.Context) (done bool, err error) { + retryCountUntilSevere++ + etcdList := &druidv1alpha1.EtcdList{} if err := b.K8sSeedClient.Client().List(ctx, etcdList, client.InNamespace(b.Shoot.SeedNamespace), @@ -97,7 +106,7 @@ func (b *Botanist) WaitUntilEtcdReady(ctx context.Context) error { for _, etcd := range etcdList.Items { switch { case etcd.Status.LastError != nil: - return retry.SevereError(fmt.Errorf("%s reconciliation errored: %s", etcd.Name, *etcd.Status.LastError)) + return retry.MinorOrSevereError(retryCountUntilSevere, int(severeThreshold.Nanoseconds()/interval.Nanoseconds()), fmt.Errorf("%s reconciliation errored: %s", etcd.Name, *etcd.Status.LastError)) case etcd.DeletionTimestamp != nil: lastErrors = multierror.Append(lastErrors, fmt.Errorf("%s unexpectedly has a deletion timestamp", etcd.Name)) case etcd.Status.ObservedGeneration == nil || etcd.Generation != *etcd.Status.ObservedGeneration: diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/worker.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/worker.go index 026eb18b3..79a4064ae 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/worker.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/worker.go @@ -194,6 +194,7 @@ func (b *Botanist) WaitUntilWorkerReady(ctx context.Context) error { b.Shoot.SeedNamespace, b.Shoot.Info.Name, DefaultInterval, + DefaultSevereThreshold, WorkerDefaultTimeout, func(obj runtime.Object) error { worker, ok := obj.(*extensionsv1alpha1.Worker) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/common/extensions.go b/vendor/github.com/gardener/gardener/pkg/operation/common/extensions.go index 7ad89a9ba..ee3d28b8c 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/common/extensions.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/common/extensions.go @@ -44,6 +44,7 @@ func WaitUntilExtensionCRReady( namespace string, name string, interval time.Duration, + severeThreshold time.Duration, timeout time.Duration, postReadyFunc func(runtime.Object) error, ) error { @@ -57,6 +58,7 @@ func WaitUntilExtensionCRReady( namespace, name, interval, + severeThreshold, timeout, postReadyFunc, ) @@ -68,26 +70,37 @@ func WaitUntilObjectReadyWithHealthFunction( ctx context.Context, c client.Client, logger *logrus.Entry, - healthFunc func(obj runtime.Object) (bool, error), + healthFunc func(obj runtime.Object) error, newObjFunc func() runtime.Object, kind string, namespace string, name string, interval time.Duration, + severeThreshold time.Duration, timeout time.Duration, postReadyFunc func(runtime.Object) error, ) error { - var lastObservedError error + var ( + errorWithCode *gardencorev1beta1helper.ErrorWithCodes + lastObservedError error + retryCountUntilSevere int + ) if err := retry.UntilTimeout(ctx, interval, timeout, func(ctx context.Context) (bool, error) { + retryCountUntilSevere++ + obj := newObjFunc() if err := c.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, obj); err != nil { return retry.SevereError(err) } - if retry, err := healthFunc(obj); err != nil { + if err := healthFunc(obj); err != nil { + lastObservedError = err logger.WithError(err).Errorf("%s did not get ready yet", extensionKey(kind, namespace, name)) - return retry, err + if errors.As(err, &errorWithCode) { + return retry.MinorOrSevereError(retryCountUntilSevere, int(severeThreshold.Nanoseconds()/interval.Nanoseconds()), err) + } + return retry.MinorError(err) } if postReadyFunc != nil { diff --git a/vendor/github.com/gardener/gardener/pkg/operation/operation.go b/vendor/github.com/gardener/gardener/pkg/operation/operation.go index f3ff7801d..91bff266c 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/operation.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/operation.go @@ -447,18 +447,12 @@ func (o *Operation) ReportShootProgress(ctx context.Context, stats *flow.Stats) // CleanShootTaskError removes the error with taskID from the Shoot's status.LastErrors array. // If the status.LastErrors array is empty then status.LastError is also removed. func (o *Operation) CleanShootTaskError(_ context.Context, taskID string) { - var remainingErrors []gardencorev1beta1.LastError - for _, lastErr := range o.Shoot.Info.Status.LastErrors { - if lastErr.TaskID == nil || taskID != *lastErr.TaskID { - remainingErrors = append(remainingErrors, lastErr) - } - } - newShoot, err := kutil.TryUpdateShootStatus(o.K8sGardenClient.GardenCore(), retry.DefaultRetry, o.Shoot.Info.ObjectMeta, func(shoot *gardencorev1beta1.Shoot) (*gardencorev1beta1.Shoot, error) { - shoot.Status.LastErrors = remainingErrors + shoot.Status.LastErrors = gardencorev1beta1helper.DeleteLastErrorByTaskID(o.Shoot.Info.Status.LastErrors, taskID) return shoot, nil - }) + }, + ) if err != nil { o.Logger.Errorf("Could not report shoot progress: %v", err) return diff --git a/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/health/health.go b/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/health/health.go index d877b068b..0db150ce2 100644 --- a/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/health/health.go +++ b/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/health/health.go @@ -24,7 +24,6 @@ import ( gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/utils" - "github.com/gardener/gardener/pkg/utils/retry" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" @@ -265,10 +264,10 @@ func CheckSeed(seed *gardencorev1beta1.Seed, identity *gardencorev1beta1.Gardene } // CheckExtensionObject checks if an extension Object is healthy or not. -func CheckExtensionObject(o runtime.Object) (bool, error) { +func CheckExtensionObject(o runtime.Object) error { obj, ok := o.(extensionsv1alpha1.Object) if !ok { - return retry.SevereError(fmt.Errorf("expected extensionsv1alpha1.Object but got %T", o)) + return fmt.Errorf("expected extensionsv1alpha1.Object but got %T", o) } status := obj.GetExtensionStatus() @@ -276,10 +275,10 @@ func CheckExtensionObject(o runtime.Object) (bool, error) { } // CheckBackupBucket checks if an backup bucket Object is healthy or not. -func CheckBackupBucket(bb runtime.Object) (bool, error) { +func CheckBackupBucket(bb runtime.Object) error { obj, ok := bb.(*gardencorev1beta1.BackupBucket) if !ok { - return retry.SevereError(fmt.Errorf("expected gardencorev1beta1.BackupBucket but got %T", bb)) + return fmt.Errorf("expected gardencorev1beta1.BackupBucket but got %T", bb) } return checkExtensionObject(obj.Generation, obj.Status.ObservedGeneration, obj.Annotations, obj.Status.LastError, obj.Status.LastOperation) } @@ -290,28 +289,28 @@ func CheckBackupBucket(bb runtime.Object) (bool, error) { // * No gardener.cloud/operation is set // * No lastError is in the status // * A last operation is state succeeded is present -func checkExtensionObject(generation int64, observedGeneration int64, annotations map[string]string, lastError *gardencorev1beta1.LastError, lastOperation *gardencorev1beta1.LastOperation) (bool, error) { +func checkExtensionObject(generation int64, observedGeneration int64, annotations map[string]string, lastError *gardencorev1beta1.LastError, lastOperation *gardencorev1beta1.LastOperation) error { if lastError != nil { - return retry.SevereError(gardencorev1beta1helper.NewErrorWithCodes(fmt.Sprintf("extension encountered error during reconciliation: %s", lastError.Description), lastError.Codes...)) + return gardencorev1beta1helper.NewErrorWithCodes(fmt.Sprintf("extension encountered error during reconciliation: %s", lastError.Description), lastError.Codes...) } if observedGeneration != generation { - return retry.MinorError(fmt.Errorf("observed generation outdated (%d/%d)", observedGeneration, generation)) + return fmt.Errorf("observed generation outdated (%d/%d)", observedGeneration, generation) } if op, ok := annotations[v1beta1constants.GardenerOperation]; ok { - return retry.MinorError(fmt.Errorf("gardener operation %q is not yet picked up by controller", op)) + return fmt.Errorf("gardener operation %q is not yet picked up by controller", op) } if lastOperation == nil { - return retry.MinorError(fmt.Errorf("extension did not record a last operation yet")) + return fmt.Errorf("extension did not record a last operation yet") } if lastOperation.State != gardencorev1beta1.LastOperationStateSucceeded { - return retry.MinorError(fmt.Errorf("extension state is not succeeded but %v", lastOperation.State)) + return fmt.Errorf("extension state is not succeeded but %v", lastOperation.State) } - return retry.Ok() + return nil } // Now determines the current time. diff --git a/vendor/github.com/gardener/gardener/pkg/utils/retry/retry.go b/vendor/github.com/gardener/gardener/pkg/utils/retry/retry.go index ceb50e895..b0ef8fe6b 100644 --- a/vendor/github.com/gardener/gardener/pkg/utils/retry/retry.go +++ b/vendor/github.com/gardener/gardener/pkg/utils/retry/retry.go @@ -78,26 +78,35 @@ func DefaultIntervalFactory() IntervalFactory { } // SevereError indicates an operation was not successful due to the given error and cannot be retried. -func SevereError(severeErr error) (done bool, err error) { +func SevereError(severeErr error) (bool, error) { return true, severeErr } // MinorError indicates an operation was not successful due to the given error but can be retried. -func MinorError(minorErr error) (done bool, err error) { +func MinorError(minorErr error) (bool, error) { return false, minorErr } // Ok indicates that an operation was successful and does not need to be retried. -func Ok() (done bool, err error) { +func Ok() (bool, error) { return true, nil } // NotOk indicates that an operation was not successful but can be retried. // It does not indicate an error. For better error reporting, consider MinorError. -func NotOk() (done bool, err error) { +func NotOk() (bool, error) { return false, nil } +// MinorOrSevereError returns a "severe" error in case the retry count exceeds the threshold. Otherwise, it returns +// a "minor" error. +func MinorOrSevereError(retryCountUntilSevere, threshold int, err error) (bool, error) { + if retryCountUntilSevere > threshold { + return SevereError(err) + } + return MinorError(err) +} + type retryError struct { ctxError error err error diff --git a/vendor/github.com/gardener/gardener/test/framework/dump.go b/vendor/github.com/gardener/gardener/test/framework/dump.go index d820f9478..f5eb78770 100644 --- a/vendor/github.com/gardener/gardener/test/framework/dump.go +++ b/vendor/github.com/gardener/gardener/test/framework/dump.go @@ -198,7 +198,7 @@ func (f *GardenerFramework) dumpGardenerExtensionsInNamespace(ctx context.Contex // dumpGardenerExtensions prints all gardener extension crds in the shoot namespace func (f *GardenerFramework) dumpGardenerExtension(extension v1alpha1.Object) { - if _, err := health.CheckExtensionObject(extension); err != nil { + if err := health.CheckExtensionObject(extension); err != nil { f.Logger.Printf("%s of type %s is %s - Error: %s", extension.GetName(), extension.GetExtensionSpec().GetExtensionType(), unhealthy, err.Error()) } else { f.Logger.Printf("%s of type %s is %s", extension.GetName(), extension.GetExtensionSpec().GetExtensionType(), healthy) diff --git a/vendor/modules.txt b/vendor/modules.txt index 4adf2e921..641f19a5a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -91,7 +91,7 @@ github.com/gardener/etcd-druid/api/v1alpha1 github.com/gardener/external-dns-management/pkg/apis/dns github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1 github.com/gardener/external-dns-management/pkg/client/dns/clientset/versioned/scheme -# github.com/gardener/gardener v1.5.0 +# github.com/gardener/gardener v1.5.1 ## explicit github.com/gardener/gardener/.github github.com/gardener/gardener/.github/ISSUE_TEMPLATE