diff --git a/apis/config/v1beta1/configuration_types.go b/apis/config/v1beta1/configuration_types.go index 2b75e2d1d4..dffc6bdd0c 100644 --- a/apis/config/v1beta1/configuration_types.go +++ b/apis/config/v1beta1/configuration_types.go @@ -211,6 +211,16 @@ type WaitForPodsReady struct { // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` + // RecoveryTimeout defines an optional timeout, measured since the + // last transition to the PodsReady=false condition after a Workload is Admitted and running. + // Such a transition may happen when a Pod failed and the replacement Pod + // is awaited to be scheduled. + // After exceeding the timeout the corresponding job gets suspended again + // and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. + // Defaults to 3 mins. + // +optional + RecoveryTimeout *metav1.Duration `json:"recoveryTimeout,omitempty"` + // BlockAdmission when true, cluster queue will block admissions for all // subsequent jobs until the jobs reach the PodsReady=true condition. // This setting is only honored when `Enable` is set to true. diff --git a/apis/config/v1beta1/defaults.go b/apis/config/v1beta1/defaults.go index 26fb9ce360..97e43ca063 100644 --- a/apis/config/v1beta1/defaults.go +++ b/apis/config/v1beta1/defaults.go @@ -41,6 +41,7 @@ const ( DefaultClientConnectionQPS float32 = 20.0 DefaultClientConnectionBurst int32 = 30 defaultPodsReadyTimeout = 5 * time.Minute + defaultPodsRecoveryTimeout = 3 * time.Minute DefaultQueueVisibilityUpdateIntervalSeconds int32 = 5 DefaultClusterQueuesMaxCount int32 = 10 defaultJobFrameworkName = "batch/job" @@ -119,6 +120,9 @@ func SetDefaults_Configuration(cfg *Configuration) { if cfg.WaitForPodsReady.Timeout == nil { cfg.WaitForPodsReady.Timeout = &metav1.Duration{Duration: defaultPodsReadyTimeout} } + if cfg.WaitForPodsReady.RecoveryTimeout == nil { + cfg.WaitForPodsReady.RecoveryTimeout = &metav1.Duration{Duration: defaultPodsRecoveryTimeout} + } if cfg.WaitForPodsReady.BlockAdmission == nil { defaultBlockAdmission := true if !cfg.WaitForPodsReady.Enable { diff --git a/apis/config/v1beta1/defaults_test.go b/apis/config/v1beta1/defaults_test.go index a0da4076ba..2de09a17eb 100644 --- a/apis/config/v1beta1/defaults_test.go +++ b/apis/config/v1beta1/defaults_test.go @@ -122,7 +122,8 @@ func TestSetDefaults_Configuration(t *testing.T) { WorkerLostTimeout: &metav1.Duration{Duration: DefaultMultiKueueWorkerLostTimeout}, } - podsReadyTimeoutTimeout := metav1.Duration{Duration: defaultPodsReadyTimeout} + podsReadyTimeout := metav1.Duration{Duration: defaultPodsReadyTimeout} + podsReadyRecoveryTimeout := metav1.Duration{Duration: defaultPodsRecoveryTimeout} podsReadyTimeoutOverwrite := metav1.Duration{Duration: time.Minute} testCases := map[string]struct { @@ -388,9 +389,10 @@ func TestSetDefaults_Configuration(t *testing.T) { }, want: &Configuration{ WaitForPodsReady: &WaitForPodsReady{ - Enable: true, - BlockAdmission: ptr.To(true), - Timeout: &podsReadyTimeoutTimeout, + Enable: true, + BlockAdmission: ptr.To(true), + Timeout: &podsReadyTimeout, + RecoveryTimeout: &podsReadyRecoveryTimeout, RequeuingStrategy: &RequeuingStrategy{ Timestamp: ptr.To(EvictionTimestamp), BackoffBaseSeconds: ptr.To[int32](DefaultRequeuingBackoffBaseSeconds), @@ -420,9 +422,10 @@ func TestSetDefaults_Configuration(t *testing.T) { }, want: &Configuration{ WaitForPodsReady: &WaitForPodsReady{ - Enable: false, - BlockAdmission: ptr.To(false), - Timeout: &podsReadyTimeoutTimeout, + Enable: false, + BlockAdmission: ptr.To(false), + Timeout: &podsReadyTimeout, + RecoveryTimeout: &podsReadyRecoveryTimeout, RequeuingStrategy: &RequeuingStrategy{ Timestamp: ptr.To(EvictionTimestamp), BackoffBaseSeconds: ptr.To[int32](DefaultRequeuingBackoffBaseSeconds), @@ -458,9 +461,10 @@ func TestSetDefaults_Configuration(t *testing.T) { }, want: &Configuration{ WaitForPodsReady: &WaitForPodsReady{ - Enable: true, - BlockAdmission: ptr.To(true), - Timeout: &podsReadyTimeoutOverwrite, + Enable: true, + BlockAdmission: ptr.To(true), + Timeout: &podsReadyTimeoutOverwrite, + RecoveryTimeout: &podsReadyRecoveryTimeout, RequeuingStrategy: &RequeuingStrategy{ Timestamp: ptr.To(CreationTimestamp), BackoffBaseSeconds: ptr.To[int32](63), diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go index 3cd2aa1e5d..1c788bea7e 100644 --- a/apis/config/v1beta1/zz_generated.deepcopy.go +++ b/apis/config/v1beta1/zz_generated.deepcopy.go @@ -512,6 +512,11 @@ func (in *WaitForPodsReady) DeepCopyInto(out *WaitForPodsReady) { *out = new(v1.Duration) **out = **in } + if in.RecoveryTimeout != nil { + in, out := &in.RecoveryTimeout, &out.RecoveryTimeout + *out = new(v1.Duration) + **out = **in + } if in.BlockAdmission != nil { in, out := &in.BlockAdmission, &out.BlockAdmission *out = new(bool) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0905837006..e40a17c540 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -629,6 +629,7 @@ webhook: Enable: true, BlockAdmission: ptr.To(false), Timeout: &metav1.Duration{Duration: 50 * time.Second}, + RecoveryTimeout: &metav1.Duration{Duration: 3 * time.Minute}, RequeuingStrategy: &configapi.RequeuingStrategy{ Timestamp: ptr.To(configapi.CreationTimestamp), BackoffLimitCount: ptr.To[int32](10), diff --git a/pkg/config/validation.go b/pkg/config/validation.go index b65f282eb8..84e74da1a1 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -123,6 +123,10 @@ func validateWaitForPodsReady(c *configapi.Configuration) field.ErrorList { allErrs = append(allErrs, field.Invalid(waitForPodsReadyPath.Child("timeout"), c.WaitForPodsReady.Timeout, apimachineryvalidation.IsNegativeErrorMsg)) } + if c.WaitForPodsReady.RecoveryTimeout != nil && c.WaitForPodsReady.RecoveryTimeout.Duration < 0 { + allErrs = append(allErrs, field.Invalid(waitForPodsReadyPath.Child("recoveryTimeout"), + c.WaitForPodsReady.RecoveryTimeout, apimachineryvalidation.IsNegativeErrorMsg)) + } if strategy := c.WaitForPodsReady.RequeuingStrategy; strategy != nil { if strategy.Timestamp != nil && *strategy.Timestamp != configapi.CreationTimestamp && *strategy.Timestamp != configapi.EvictionTimestamp { diff --git a/pkg/config/validation_test.go b/pkg/config/validation_test.go index ac503bda20..e6adfdf187 100644 --- a/pkg/config/validation_test.go +++ b/pkg/config/validation_test.go @@ -429,6 +429,23 @@ func TestValidate(t *testing.T) { }, }, }, + "negative waitForPodsReady.recoveryTimeout": { + cfg: &configapi.Configuration{ + Integrations: defaultIntegrations, + WaitForPodsReady: &configapi.WaitForPodsReady{ + Enable: true, + RecoveryTimeout: &metav1.Duration{ + Duration: -1, + }, + }, + }, + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "waitForPodsReady.recoveryTimeout", + }, + }, + }, "valid waitForPodsReady": { cfg: &configapi.Configuration{ Integrations: defaultIntegrations, @@ -437,6 +454,9 @@ func TestValidate(t *testing.T) { Timeout: &metav1.Duration{ Duration: 50, }, + RecoveryTimeout: &metav1.Duration{ + Duration: 5, + }, BlockAdmission: ptr.To(false), RequeuingStrategy: &configapi.RequeuingStrategy{ Timestamp: ptr.To(configapi.CreationTimestamp), diff --git a/pkg/controller/core/core.go b/pkg/controller/core/core.go index 3c6862f87b..b18ba91a8a 100644 --- a/pkg/controller/core/core.go +++ b/pkg/controller/core/core.go @@ -92,7 +92,8 @@ func waitForPodsReady(cfg *configapi.WaitForPodsReady) *waitForPodsReadyConfig { return nil } result := waitForPodsReadyConfig{ - timeout: cfg.Timeout.Duration, + timeout: cfg.Timeout.Duration, + recoveryTimeout: cfg.RecoveryTimeout.Duration, } if cfg.RequeuingStrategy != nil { result.requeuingBackoffBaseSeconds = *cfg.RequeuingStrategy.BackoffBaseSeconds diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 8d502d2927..1765e13fa0 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -62,6 +62,7 @@ var ( type waitForPodsReadyConfig struct { timeout time.Duration + recoveryTimeout time.Duration requeuingBackoffLimitCount *int32 requeuingBackoffBaseSeconds int32 requeuingBackoffMaxDuration time.Duration diff --git a/site/content/en/docs/reference/kueue-config.v1beta1.md b/site/content/en/docs/reference/kueue-config.v1beta1.md index a2e17e6043..49fdcd5dd5 100644 --- a/site/content/en/docs/reference/kueue-config.v1beta1.md +++ b/site/content/en/docs/reference/kueue-config.v1beta1.md @@ -902,6 +902,19 @@ evicted and requeued in the same cluster queue. Defaults to 5min.

+recoveryTimeout
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration + + +

RecoveryTimeout defines an optional timeout, measured since the +last transition to the PodsReady=false condition after a Workload is Admitted and running. +Such a transition may happen when a Pod failed and the replacement Pod +is awaited to be scheduled. +After exceeding the timeout the corresponding job gets suspended again +and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. +Defaults to 3 mins.

+ + blockAdmission [Required]
bool