diff --git a/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/internal/life_cycle.go index 2975fb3654..6f5a428616 100644 --- a/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/internal/life_cycle.go @@ -106,23 +106,40 @@ func (h LifeCycleHandler) Handle(ctx context.Context, req reconcile.Request, dep return reconcile.Result{}, fmt.Errorf("list resources in provisioning: %w", err) } remainInProvisioning := len(resourcesInProvisioning) - if remainInProvisioning > 0 { + if remainInProvisioning == 0 { + // All provisioners are finished, switch DVCR to garbage collection. + err = h.dvcrService.SwitchToGarbageCollectionMode(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("switch to garbage collection mode: %w", err) + } dvcrcondition.UpdateGarbageCollectionCondition(deploy, dvcrdeploymentcondition.InProgress, - "Wait for cvi/vi/vd finish provisioning: %d resources remain.", remainInProvisioning, + "Wait for garbage collection to finish.", ) - return reconcile.Result{RequeueAfter: time.Second * 20}, nil + return reconcile.Result{}, nil } - // All provisioners are finished, switch to garbage collection. - err = h.dvcrService.SwitchToGarbageCollectionMode(ctx) - if err != nil { - return reconcile.Result{}, fmt.Errorf("switch to garbage collection mode: %w", err) + + // Cancel garbage collection if wait for provisioners for too long. + hasCreationTimestamp := !secret.GetCreationTimestamp().Time.IsZero() + waitDuration := time.Since(secret.GetCreationTimestamp().Time) + if hasCreationTimestamp && waitDuration > dvcrtypes.WaitProvisionersTimeout { + // Wait for provisioners timed out: report error and stop garbage collection. + dvcrcondition.UpdateGarbageCollectionCondition(deploy, + dvcrdeploymentcondition.Error, + "Wait for resources provisioners more than %s timeout: %s elapsed, garbage collection canceled", + dvcrtypes.WaitProvisionersTimeout.String(), + waitDuration.String(), + ) + annotations.AddAnnotation(deploy, annotations.AnnDVCRGarbageCollectionResult, "") + return reconcile.Result{}, h.dvcrService.DeleteGarbageCollectionSecret(ctx) } + + // Use requeue to wait for provisioners to finish. dvcrcondition.UpdateGarbageCollectionCondition(deploy, dvcrdeploymentcondition.InProgress, - "Wait for garbage collection to finish.", + "Wait for cvi/vi/vd finish provisioning: %d resources remain.", remainInProvisioning, ) - return reconcile.Result{}, nil + return reconcile.Result{RequeueAfter: time.Second * 20}, nil } return reconcile.Result{}, nil diff --git a/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/types/const.go b/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/types/const.go index 4b1e916515..d45c213da9 100644 --- a/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/types/const.go +++ b/images/virtualization-artifact/pkg/controller/dvcr-garbage-collection/types/const.go @@ -16,7 +16,11 @@ limitations under the License. package types -import "k8s.io/apimachinery/pkg/types" +import ( + "time" + + "k8s.io/apimachinery/pkg/types" +) const ( ModuleNamespace = "d8-virtualization" @@ -25,6 +29,8 @@ const ( DVCRGarbageCollectionSecretName = "dvcr-garbage-collection" CronSourceNamespace = "__cron_source__" CronSourceRunGC = "run-gc" + + WaitProvisionersTimeout = time.Hour * 2 ) func DVCRDeploymentKey() types.NamespacedName {