diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go
index b429aca2cb9..d295b313900 100644
--- a/pkg/upgrade/upgrade.go
+++ b/pkg/upgrade/upgrade.go
@@ -48,6 +48,8 @@ type JobSpec struct {
 	Path []string
 	// set of values for the field to match object, any one matches
 	Values []string
+	// hook to call before action (delete)
+	Fixup func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error
 }
 
 // TODO: move to common place.
@@ -63,6 +65,7 @@ var (
 	gvkServiceMonitor     = schema.GroupVersionKind{Group: "monitoring.coreos.com", Version: "v1", Kind: "ServiceMonitor"}
 	gvkOdhApplication     = schema.GroupVersionKind{Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication"}
 	gvkOdhDashboardConfig = schema.GroupVersionKind{Group: "opendatahub.io", Version: "v1alpha", Kind: "OdhDashboardConfig"}
+	gvkKfDef              = schema.GroupVersionKind{Group: "kfdef.apps.kubeflow.org", Version: "v1", Kind: "KfDef"}
 )
 
 // CreateDefaultDSC creates a default instance of DSC.
@@ -388,62 +391,78 @@ func deleteOneJob(ctx context.Context, c client.Client, job JobSpec) error {
 
 	for _, item := range list.Items {
 		item := item
-		v, ok, err := unstructured.NestedString(item.Object, job.Path...)
+		isMatched := func(item *unstructured.Unstructured) (bool, error) {
+			if len(job.Path) == 0 || len(job.Values) == 0 {
+				return true, nil
+			}
+
+			v, ok, err := unstructured.NestedString(item.Object, job.Path...)
+			if err != nil {
+				return false, fmt.Errorf("failed to get field %v for %s %s/%s: %w", job.Path, job.Gvk.Kind, job.Namespace, item.GetName(), err)
+			}
+
+			if !ok {
+				return false, fmt.Errorf("unexisting path to delete: %v", job.Path)
+			}
+
+			for _, toDelete := range job.Values {
+				if v == toDelete {
+					return true, nil
+				}
+			}
+
+			return false, nil
+		}
+
+		matched, err := isMatched(&item)
 		if err != nil {
-			return fmt.Errorf("failed to get field %v for %s %s/%s: %w", job.Path, job.Gvk.Kind, job.Namespace, item.GetName(), err)
+			return err
 		}
 
-		if !ok {
-			return fmt.Errorf("unexisting path to delete: %v", job.Path)
+		if !matched {
+			continue
 		}
 
-		for _, toDelete := range job.Values {
-			if v == toDelete {
-				err = c.Delete(ctx, &item)
-				if err != nil {
-					return fmt.Errorf("failed to delete %s %s/%s: %w", job.Gvk.Kind, job.Namespace, item.GetName(), err)
-				}
-				fmt.Println("Deleted object", item.GetName(), job.Gvk, "in namespace", job.Namespace)
+		if job.Fixup != nil {
+			err = job.Fixup(ctx, c, &item)
+			if err != nil {
+				return fmt.Errorf("failed to fix %s %s/%s: %w", job.Gvk.Kind, job.Namespace, item.GetName(), err)
 			}
 		}
+
+		err = c.Delete(ctx, &item)
+		if err != nil {
+			return fmt.Errorf("failed to delete %s %s/%s: %w", job.Gvk.Kind, job.Namespace, item.GetName(), err)
+		}
+		fmt.Println("Deleted object", item.GetName(), job.Gvk, "in namespace", job.Namespace)
 	}
 
 	return nil
 }
 
 func RemoveKfDefInstances(ctx context.Context, cli client.Client) error {
-	// Check if kfdef are deployed
-	kfdefCrd := &apiextv1.CustomResourceDefinition{}
+	del := []JobSpec{
+		{
+			Gvk: gvkKfDef,
+			Fixup: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error {
+				// Delete the finalizers
+				meta, ok := obj.Object["metadata"].(map[string]interface{})
+				if !ok {
+					return fmt.Errorf("cannot find metadata in the object %v", obj.GetName())
+				}
 
-	err := cli.Get(ctx, client.ObjectKey{Name: "kfdefs.kfdef.apps.kubeflow.org"}, kfdefCrd)
-	if err != nil {
-		if apierrs.IsNotFound(err) {
-			// If no Crd found, return, since its a new Installation
-			return nil
-		}
-		return fmt.Errorf("error retrieving kfdef CRD : %w", err)
-	}
-	expectedKfDefList := &kfdefv1.KfDefList{}
-	err = cli.List(ctx, expectedKfDefList)
-	if err != nil {
-		return fmt.Errorf("error getting list of kfdefs: %w", err)
-	}
-	// Delete kfdefs
-	for _, kfdef := range expectedKfDefList.Items {
-		kfdef := kfdef
-		// Remove finalizer
-		updatedKfDef := &kfdef
-		updatedKfDef.Finalizers = []string{}
-		err = cli.Update(ctx, updatedKfDef)
-		if err != nil {
-			return fmt.Errorf("error removing finalizers from kfdef %v : %w", kfdef.Name, err)
-		}
-		err = cli.Delete(ctx, updatedKfDef)
-		if err != nil {
-			return fmt.Errorf("error deleting kfdef %v : %w", kfdef.Name, err)
-		}
+				delete(meta, "finalizers")
+
+				err := c.Update(ctx, obj)
+				if err != nil {
+					return fmt.Errorf("error removing finalizers from %v : %w", obj.GetName(), err)
+				}
+
+				return nil
+			},
+		},
 	}
-	return nil
+	return deleteResources(ctx, cli, &del)
 }
 
 func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error {