diff --git a/pkg/action/action.go b/pkg/action/action.go new file mode 100644 index 00000000000..ff3a54396ac --- /dev/null +++ b/pkg/action/action.go @@ -0,0 +1,307 @@ +package action + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/hashicorp/go-multierror" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ResourceSpec struct { + Gvk schema.GroupVersionKind + Namespace string + // path to the field, like "metadata", "name" + Path []string + // set of values for the field to match object, any one matches + Values []string +} + +type MatcherFunc func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) +type Func func(ctx context.Context, c client.Client, r ResourceSpec, obj *unstructured.Unstructured) error +type RetryCheckFunc func(ctx context.Context, c client.Client, resources ...ResourceSpec) (bool, error) + +type Action struct { + client client.Client + + matcher MatcherFunc + actions []Func +} + +// shouldn't just return false on error? +func DefaultMatcher(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + if len(r.Path) == 0 || len(r.Values) == 0 { + return true, nil + } + + v, ok, err := unstructured.NestedString(obj.Object, r.Path...) + if err != nil { + return false, fmt.Errorf("failed to get field %v for %s %s/%s: %w", r.Path, r.Gvk.Kind, r.Namespace, obj.GetName(), err) + } + + if !ok { + return false, fmt.Errorf("unexisting path to handle: %v", r.Path) + } + + for _, toDelete := range r.Values { + if v == toDelete { + return true, nil + } + } + + return false, nil +} + +func New(c client.Client) *Action { + return &Action{ + client: c, + matcher: DefaultMatcher, + } +} + +func Not(m MatcherFunc) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + matched, err := m(r, obj) + return !matched, err + } +} + +func Any(matchers ...MatcherFunc) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + for _, m := range matchers { + matched, err := m(r, obj) + if err != nil { + return false, err + } + if matched { + return true, err + } + } + return false, nil + } +} + +func All(matchers ...MatcherFunc) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + for _, m := range matchers { + matched, err := m(r, obj) + if err != nil { + return false, err + } + if !matched { + return false, err + } + } + return true, nil + } +} + +func (o *Action) ForMatched(m MatcherFunc) *Action { + o.matcher = m + return o +} + +func (o *Action) Do(a Func) *Action { + o.actions = append(o.actions, a) + return o +} + +func (o *Action) execOneResource(ctx context.Context, r ResourceSpec, objs []*unstructured.Unstructured) error { + for _, item := range objs { + for _, a := range o.actions { + err := a(ctx, o.client, r, item) + if err != nil { + return err + } + } + } + + return nil +} + +func ListMatched(ctx context.Context, c client.Client, matcher MatcherFunc, resources ...ResourceSpec) (map[*ResourceSpec][]*unstructured.Unstructured, error) { + ret := make(map[*ResourceSpec][]*unstructured.Unstructured) + + for _, r := range resources { + r := r + var items []*unstructured.Unstructured + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(r.Gvk) + + err := c.List(ctx, list, client.InNamespace(r.Namespace)) + if err != nil { + if errors.Is(err, &meta.NoKindMatchError{}) { + fmt.Printf("Could not list %v: CRD not found\n", r.Gvk) + continue + } + return ret, fmt.Errorf("failed to list %s: %w", r.Gvk.Kind, err) + } + + for _, item := range list.Items { + item := item + + matched, err := matcher(r, &item) + if err != nil { + return ret, err + } + + if !matched { + continue + } + + items = append(items, &item) + } + + if len(items) > 0 { + ret[&r] = items + } + } + + return ret, nil +} + +func (o *Action) Exec(ctx context.Context, resources ...ResourceSpec) error { + var errors *multierror.Error + + matched, err := ListMatched(ctx, o.client, o.matcher, resources...) + if err != nil { + return err + } + + for r, objs := range matched { + err := o.execOneResource(ctx, *r, objs) + errors = multierror.Append(errors, err) + } + + return errors.ErrorOrNil() +} + +func (o *Action) ExecWithRetry(ctx context.Context, shouldRetry RetryCheckFunc, resources ...ResourceSpec) error { + return wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + // 5, 10, ,20, 40 then timeout + Duration: 5 * time.Second, + Factor: 2.0, + Jitter: 0.1, + Steps: 4, + Cap: 1 * time.Minute, + }, func(ctx context.Context) (bool, error) { + err := o.Exec(ctx, resources...) + if err != nil { + return false, err + } + return shouldRetry(ctx, o.client, resources...) + }) +} + +func (o *Action) DryRun(_ context.Context, _ ...ResourceSpec) error { + return nil +} + +func Delete(ctx context.Context, c client.Client, _ ResourceSpec, obj *unstructured.Unstructured) error { + return client.IgnoreNotFound(c.Delete(ctx, obj)) +} + +func IfAnyLeft(matcher MatcherFunc) RetryCheckFunc { + return func(ctx context.Context, c client.Client, resources ...ResourceSpec) (bool, error) { + matched, err := ListMatched(ctx, c, matcher, resources...) + if err != nil { + return false, err + } + + return len(matched) == 0, nil + } +} + +func deleteField(obj map[string]any, path ...string) error { + if len(path) < 1 { + return fmt.Errorf("path is empty") + } + + parent := path[:len(path)-1] + field := path[len(path)-1] + + v, ok, err := unstructured.NestedFieldNoCopy(obj, parent...) + if err != nil || !ok { + return fmt.Errorf("Not found or error") + } + + m, ok := v.(map[string]any) + if !ok { + return fmt.Errorf("field is not map") + } + delete(m, field) + return nil +} + +func DeleteField(path ...string) Func { + return func(ctx context.Context, c client.Client, r ResourceSpec, obj *unstructured.Unstructured) error { + err := deleteField(obj.Object, path...) + if err != nil { + return fmt.Errorf("could not delete field %v in object %s : %w", path, obj.GetName(), err) + } + + err = c.Update(ctx, obj) + if err != nil { + return fmt.Errorf("error updating object while removing %v from %v : %w", path, obj.GetName(), err) + } + + return nil + } +} + +func MatchMap(key, value string, keyMatch func(value, pattern string) bool, path ...string) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + m, ok, err := unstructured.NestedStringMap(obj.Object, path...) + if err != nil || !ok { + return false, err + } + + for k, v := range m { + if !keyMatch(k, key) { + continue + } + + if value == "" || v == value { + return true, nil + } + } + + return false, nil + } +} + +func MatchMapKeyContains(key string, path ...string) MatcherFunc { + return MatchMap(key, "", strings.Contains, path...) +} + +func NewDelete(c client.Client) *Action { + return New(c).Do(Delete) +} + +func NewDeleteMatched(c client.Client, m MatcherFunc) *Action { + return New(c).Do(Delete).ForMatched(m) +} + +func NewDeleteWithFinalizer(c client.Client) *Action { + return New(c). + Do(DeleteField("metadata", "finalizer")). + Do(Delete) +} + +func NewDeleteOwnersReferences(c client.Client) *Action { + return New(c). + Do(DeleteField("metadata", "ownerReferences")) +} + +func NewDeleteLabel(c client.Client, label string) *Action { + return New(c). + Do(DeleteField("metadata", "labels", label)) +} diff --git a/pkg/cluster/gvk/gvk.go b/pkg/cluster/gvk/gvk.go index 3a4bad2c363..fea2aa3234c 100644 --- a/pkg/cluster/gvk/gvk.go +++ b/pkg/cluster/gvk/gvk.go @@ -3,29 +3,44 @@ package gvk import "k8s.io/apimachinery/pkg/runtime/schema" var ( + ClusterRole = schema.GroupVersionKind{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + } + ClusterRoleBinding = schema.GroupVersionKind{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRoleBinding", + } + Deployment = schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + } + KfDef = schema.GroupVersionKind{ + Group: "kfdef.apps.kubeflow.org", + Version: "v1", Kind: "KfDef", + } KnativeServing = schema.GroupVersionKind{ Group: "operator.knative.dev", Version: "v1beta1", Kind: "KnativeServing", } - - OpenshiftIngress = schema.GroupVersionKind{ - Group: "config.openshift.io", - Version: "v1", - Kind: "Ingress", - } - - ServiceMeshControlPlane = schema.GroupVersionKind{ - Group: "maistra.io", - Version: "v2", - Kind: "ServiceMeshControlPlane", + Namespace = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "Namespace", } - OdhApplication = schema.GroupVersionKind{ Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication", } + OdhDashboardConfig = schema.GroupVersionKind{ + Group: "opendatahub.io", + Version: "v1alpha", + Kind: "OdhDashboardConfig", + } OdhDocument = schema.GroupVersionKind{ Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhDocument", @@ -34,4 +49,37 @@ var ( Group: "console.openshift.io", Version: "v1", Kind: "OdhQuickStart", } + OpenshiftIngress = schema.GroupVersionKind{ + Group: "config.openshift.io", + Version: "v1", + Kind: "Ingress", + } + Route = schema.GroupVersionKind{ + Group: "route.openshift.io", + Version: "v1", Kind: "Route", + } + Secret = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "Secret", + } + Service = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "Service", + } + ServiceAccount = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "ServiceAccount", + } + ServiceMeshControlPlane = schema.GroupVersionKind{ + Group: "maistra.io", + Version: "v2", Kind: "ServiceMeshControlPlane", + } + ServiceMonitor = schema.GroupVersionKind{ + Group: "monitoring.coreos.com", + Version: "v1", Kind: "ServiceMonitor", + } + StatefulSet = schema.GroupVersionKind{ + Group: "apps", + Version: "v1", Kind: "StatefulSet", + } ) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index dbe211b301c..2da53c237d6 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -4,26 +4,14 @@ package upgrade import ( "context" - "errors" "fmt" - "reflect" - "strings" - "time" "github.com/hashicorp/go-multierror" operatorv1 "github.com/openshift/api/operator/v1" - routev1 "github.com/openshift/api/route/v1" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - authv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" kfdefv1 "github.com/opendatahub-io/opendatahub-operator/apis/kfdef.apps.kubeflow.org/v1" @@ -42,20 +30,12 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components/trainingoperator" "github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/action" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" ) -type ResourceSpec struct { - Gvk schema.GroupVersionKind - Namespace string - // path to the field, like "metadata", "name" - Path []string - // set of values for the field to match object, any one matches - Values []string -} - // CreateDefaultDSC creates a default instance of DSC. // Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource. func CreateDefaultDSC(ctx context.Context, cli client.Client) error { @@ -267,12 +247,12 @@ func UpdateFromLegacyVersion(cli client.Client, platform cluster.Platform, appNS return nil } -func getDashboardWatsonResources(ns string) []ResourceSpec { +func getDashboardWatsonResources(ns string) []action.ResourceSpec { metadataName := []string{"metadata", "name"} specAppName := []string{"spec", "appName"} appName := []string{"watson-studio"} - return []ResourceSpec{ + return []action.ResourceSpec{ { Gvk: gvk.OdhQuickStart, Namespace: ns, @@ -294,249 +274,119 @@ func getDashboardWatsonResources(ns string) []ResourceSpec { } } +func getModelMeshResources(ns string, platform cluster.Platform) []action.ResourceSpec { + metadataName := []string{"metadata", "name"} + var toDelete []action.ResourceSpec + + if platform == cluster.ManagedRhods { + del := []action.ResourceSpec{ + { + Gvk: gvk.Deployment, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-prometheus-operator"}, + }, + { + Gvk: gvk.StatefulSet, + Namespace: ns, + Path: metadataName, + Values: []string{"prometheus-rhods-model-monitoring"}, + }, + { + Gvk: gvk.Service, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-model-monitoring"}, + }, + { + Gvk: gvk.Route, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-model-monitoring"}, + }, + { + Gvk: gvk.Secret, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-monitoring-oauth-config"}, + }, + { + Gvk: gvk.ClusterRole, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"}, + }, + { + Gvk: gvk.ClusterRoleBinding, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"}, + }, + { + Gvk: gvk.ServiceAccount, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-prometheus-operator"}, + }, + { + Gvk: gvk.ServiceMonitor, + Namespace: ns, + Path: metadataName, + Values: []string{"modelmesh-federated-metrics"}, + }, + } + toDelete = append(toDelete, del...) + } + // common logic for both self-managed and managed + del := action.ResourceSpec{ + Gvk: gvk.ServiceMonitor, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-monitor-federation2"}, + } + + toDelete = append(toDelete, del) + return toDelete +} + // TODO: remove function once we have a generic solution across all components. func CleanupExistingResource(ctx context.Context, cli client.Client, platform cluster.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error { var multiErr *multierror.Error // Special Handling of cleanup of deprecated model monitoring stack - if platform == cluster.ManagedRhods { - deprecatedDeployments := []string{"rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedDeployments, &appsv1.DeploymentList{})) - - deprecatedStatefulsets := []string{"prometheus-rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedStatefulsets, &appsv1.StatefulSetList{})) - - deprecatedServices := []string{"rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServices, &corev1.ServiceList{})) - - deprecatedRoutes := []string{"rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedRoutes, &routev1.RouteList{})) - - deprecatedSecrets := []string{"rhods-monitoring-oauth-config"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedSecrets, &corev1.SecretList{})) - - deprecatedClusterroles := []string{"rhods-namespace-read", "rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterroles, &authv1.ClusterRoleList{})) - - deprecatedClusterrolebindings := []string{"rhods-namespace-read", "rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterrolebindings, &authv1.ClusterRoleBindingList{})) - - deprecatedServiceAccounts := []string{"rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServiceAccounts, &corev1.ServiceAccountList{})) - - deprecatedServicemonitors := []string{"modelmesh-federated-metrics"} - multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedServicemonitors)) - } - // common logic for both self-managed and managed - deprecatedOperatorSM := []string{"rhods-monitor-federation2"} - multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedOperatorSM)) // Remove deprecated opendatahub namespace(owned by kuberay) multiErr = multierror.Append(multiErr, deleteDeprecatedNamespace(ctx, cli, "opendatahub")) + toDelete := getModelMeshResources(dscMonitoringNamespace, platform) + toDelete = append(toDelete, getDashboardWatsonResources(dscApplicationsNamespace)...) // Handling for dashboard Jupyterhub CR, see jira #443 - JupyterhubApp := schema.GroupVersionKind{ - Group: "dashboard.opendatahub.io", - Version: "v1", - Kind: "OdhApplication", - } - multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, JupyterhubApp, "jupyterhub", dscApplicationsNamespace)) + toDelete = append(toDelete, action.ResourceSpec{ + Gvk: gvk.OdhApplication, + Namespace: dscApplicationsNamespace, + Path: []string{"metadata", "name"}, + Values: []string{"jupyterhub"}, + }) - // to take a reference - toDelete := getDashboardWatsonResources(dscApplicationsNamespace) - multiErr = multierror.Append(multiErr, deleteResources(ctx, cli, &toDelete)) + multiErr = multierror.Append(multiErr, action.NewDelete(cli).Exec(ctx, toDelete...)) return multiErr.ErrorOrNil() } -func deleteResources(ctx context.Context, c client.Client, resources *[]ResourceSpec) error { - var errors *multierror.Error - - for _, res := range *resources { - err := deleteOneResource(ctx, c, res) - errors = multierror.Append(errors, err) - } - - return errors.ErrorOrNil() -} - -func deleteOneResource(ctx context.Context, c client.Client, res ResourceSpec) error { - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(res.Gvk) - - err := c.List(ctx, list, client.InNamespace(res.Namespace)) - if err != nil { - if errors.Is(err, &meta.NoKindMatchError{}) { - fmt.Printf("Could not delete %v: CRD not found\n", res.Gvk) - return nil - } - return fmt.Errorf("failed to list %s: %w", res.Gvk.Kind, err) - } - - for _, item := range list.Items { - item := item - v, ok, err := unstructured.NestedString(item.Object, res.Path...) - if err != nil { - return fmt.Errorf("failed to get field %v for %s %s/%s: %w", res.Path, res.Gvk.Kind, res.Namespace, item.GetName(), err) - } - - if !ok { - return fmt.Errorf("unexisting path to delete: %v", res.Path) - } - - for _, toDelete := range res.Values { - if v == toDelete { - err = c.Delete(ctx, &item) - if err != nil { - return fmt.Errorf("failed to delete %s %s/%s: %w", res.Gvk.Kind, res.Namespace, item.GetName(), err) - } - fmt.Println("Deleted object", item.GetName(), res.Gvk, "in namespace", res.Namespace) - } - } - } - - return nil -} - func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { - // Check if kfdef are deployed - kfdefCrd := &apiextv1.CustomResourceDefinition{} - - err := cli.Get(ctx, client.ObjectKey{Name: "kfdefs.kfdef.apps.kubeflow.org"}, kfdefCrd) - if err != nil { - if apierrs.IsNotFound(err) { - // If no Crd found, return, since its a new Installation - return nil - } - return fmt.Errorf("error retrieving kfdef CRD : %w", err) - } - expectedKfDefList := &kfdefv1.KfDefList{} - err = cli.List(ctx, expectedKfDefList) - if err != nil { - return fmt.Errorf("error getting list of kfdefs: %w", err) - } - // Delete kfdefs - for _, kfdef := range expectedKfDefList.Items { - kfdef := kfdef - // Remove finalizer - updatedKfDef := &kfdef - updatedKfDef.Finalizers = []string{} - err = cli.Update(ctx, updatedKfDef) - if err != nil { - return fmt.Errorf("error removing finalizers from kfdef %v : %w", kfdef.Name, err) - } - err = cli.Delete(ctx, updatedKfDef) - if err != nil { - return fmt.Errorf("error deleting kfdef %v : %w", kfdef.Name, err) - } - } - return nil -} - -func deleteDeprecatedResources(ctx context.Context, cli client.Client, namespace string, resourceList []string, resourceType client.ObjectList) error { - var multiErr *multierror.Error - listOpts := &client.ListOptions{Namespace: namespace} - if err := cli.List(ctx, resourceType, listOpts); err != nil { - multiErr = multierror.Append(multiErr, err) - } - items := reflect.ValueOf(resourceType).Elem().FieldByName("Items") - for i := 0; i < items.Len(); i++ { - item := items.Index(i).Addr().Interface().(client.Object) //nolint:errcheck,forcetypeassert - for _, name := range resourceList { - if name == item.GetName() { - fmt.Printf("Attempting to delete %s in namespace %s\n", item.GetName(), namespace) - err := cli.Delete(ctx, item) - if err != nil { - if apierrs.IsNotFound(err) { - fmt.Printf("Could not find %s in namespace %s\n", item.GetName(), namespace) - } else { - multiErr = multierror.Append(multiErr, err) - } - } - fmt.Printf("Successfully deleted %s\n", item.GetName()) - } - } - } - return multiErr.ErrorOrNil() -} - -// Need to handle ServiceMonitor deletion separately as the generic function does not work for ServiceMonitors because of how the package is built. -func deleteDeprecatedServiceMonitors(ctx context.Context, cli client.Client, namespace string, resourceList []string) error { - var multiErr *multierror.Error - listOpts := &client.ListOptions{Namespace: namespace} - servicemonitors := &monitoringv1.ServiceMonitorList{} - if err := cli.List(ctx, servicemonitors, listOpts); err != nil { - multiErr = multierror.Append(multiErr, err) - } - - for _, servicemonitor := range servicemonitors.Items { - servicemonitor := servicemonitor - for _, name := range resourceList { - if name == servicemonitor.Name { - fmt.Printf("Attempting to delete %s in namespace %s\n", servicemonitor.Name, namespace) - err := cli.Delete(ctx, servicemonitor) - if err != nil { - if apierrs.IsNotFound(err) { - fmt.Printf("Could not find %s in namespace %s\n", servicemonitor.Name, namespace) - } else { - multiErr = multierror.Append(multiErr, err) - } - } - fmt.Printf("Successfully deleted %s\n", servicemonitor.Name) - } - } - } - return multiErr.ErrorOrNil() -} - -func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind, instanceName string, applicationNS string) error { - // first check if CRD in cluster - crd := &apiextv1.CustomResourceDefinition{} - if err := cli.Get(ctx, client.ObjectKey{Name: "odhapplications.dashboard.opendatahub.io"}, crd); err != nil { - return client.IgnoreNotFound(err) - } - - // then check if CR in cluster to delete - odhObject := &unstructured.Unstructured{} - odhObject.SetGroupVersionKind(gvk) - if err := cli.Get(ctx, client.ObjectKey{ - Namespace: applicationNS, - Name: instanceName, - }, odhObject); err != nil { - return client.IgnoreNotFound(err) - } - if err := cli.Delete(ctx, odhObject); err != nil { - return fmt.Errorf("error deleting CR %s : %w", instanceName, err) - } - - return nil + return action.NewDeleteWithFinalizer(cli). + Exec(ctx, action.ResourceSpec{ + Gvk: gvk.KfDef, + }) } func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error { - OdhDashboardConfig := schema.GroupVersionKind{ - Group: "opendatahub.io", - Version: "v1alpha", - Kind: "OdhDashboardConfig", - } - crd := &apiextv1.CustomResourceDefinition{} - if err := cli.Get(context.TODO(), client.ObjectKey{Name: "odhdashboardconfigs.opendatahub.io"}, crd); err != nil { - return client.IgnoreNotFound(err) - } - odhObject := &unstructured.Unstructured{} - odhObject.SetGroupVersionKind(OdhDashboardConfig) - if err := cli.Get(context.TODO(), client.ObjectKey{ - Namespace: applicationNS, - Name: instanceName, - }, odhObject); err != nil { - return client.IgnoreNotFound(err) - } - if odhObject.GetOwnerReferences() != nil { - // set to nil as updates - odhObject.SetOwnerReferences(nil) - if err := cli.Update(context.TODO(), odhObject); err != nil { - return fmt.Errorf("error unset ownerreference for CR %s : %w", instanceName, err) - } - } - return nil + return action.NewDeleteOwnersReferences(cli). + Exec(context.TODO(), action.ResourceSpec{ + Gvk: gvk.OdhDashboardConfig, + Namespace: applicationNS, + Path: []string{"metadata", "name"}, + Values: []string{instanceName}, + }) } func deleteResource(cli client.Client, namespace string, resourceType string) error { @@ -545,152 +395,31 @@ func deleteResource(cli client.Client, namespace string, resourceType string) er // deployments and recreated them. // because we can't proceed if a deployment is not deleted, we use exponential backoff // to retry the deletion until it succeeds - var err error - switch resourceType { - case "deployment": - err = wait.ExponentialBackoffWithContext(context.TODO(), wait.Backoff{ - // 5, 10, ,20, 40 then timeout - Duration: 5 * time.Second, - Factor: 2.0, - Jitter: 0.1, - Steps: 4, - Cap: 1 * time.Minute, - }, func(ctx context.Context) (bool, error) { - done, err := deleteDeploymentsAndCheck(ctx, cli, namespace) - return done, err - }) - case "statefulset": - err = wait.ExponentialBackoffWithContext(context.TODO(), wait.Backoff{ - // 10, 20 then timeout - Duration: 10 * time.Second, - Factor: 2.0, - Jitter: 0.1, - Steps: 2, - Cap: 1 * time.Minute, - }, func(ctx context.Context) (bool, error) { - done, err := deleteStatefulsetsAndCheck(ctx, cli, namespace) - return done, err - }) - } - return err -} -func deleteDeploymentsAndCheck(ctx context.Context, cli client.Client, namespace string) (bool, error) { - // Delete Deployment objects - var multiErr *multierror.Error - deployments := &appsv1.DeploymentList{} - listOpts := &client.ListOptions{ + toDelete := action.ResourceSpec{ Namespace: namespace, } - if err := cli.List(ctx, deployments, listOpts); err != nil { - return false, nil //nolint:nilerr - } - // filter deployment which has the new label to limit that we do not overkill other deployment - // this logic can be used even when upgrade from v2.4 to v2.5 without remove it - markedForDeletion := []appsv1.Deployment{} - for _, deployment := range deployments.Items { - deployment := deployment - v2 := false - selectorLabels := deployment.Spec.Selector.MatchLabels - for label := range selectorLabels { - if strings.Contains(label, labels.ODHAppPrefix) { - // this deployment has the new label, this is a v2 to v2 upgrade - // there is no need to recreate it, as labels are matching - v2 = true - continue - } - } - if !v2 { - markedForDeletion = append(markedForDeletion, deployment) - multiErr = multierror.Append(multiErr, cli.Delete(ctx, &deployment)) - } - } - - for _, deployment := range markedForDeletion { - deployment := deployment - if e := cli.Get(ctx, client.ObjectKey{ - Namespace: namespace, - Name: deployment.Name, - }, &deployment); e != nil { - if apierrs.IsNotFound(e) { - // resource has been successfully deleted - continue - } - // unexpected error, report it - multiErr = multierror.Append(multiErr, e) //nolint:staticcheck,wastedassign - } - // resource still exists, wait for it to be deleted - return false, nil - } - - return true, multiErr.ErrorOrNil() -} - -func deleteStatefulsetsAndCheck(ctx context.Context, cli client.Client, namespace string) (bool, error) { - // Delete statefulset objects - var multiErr *multierror.Error - statefulsets := &appsv1.StatefulSetList{} - listOpts := &client.ListOptions{ - Namespace: namespace, - } - - if err := cli.List(ctx, statefulsets, listOpts); err != nil { - return false, nil //nolint:nilerr - } - - // even only we have one item to delete to avoid nil point still use range - markedForDeletion := []appsv1.StatefulSet{} - for _, statefulset := range statefulsets.Items { - v2 := false - statefulset := statefulset - selectorLabels := statefulset.Spec.Selector.MatchLabels - for label := range selectorLabels { - if strings.Contains(label, labels.ODHAppPrefix) { - v2 = true - continue - } - } - if !v2 { - markedForDeletion = append(markedForDeletion, statefulset) - multiErr = multierror.Append(multiErr, cli.Delete(ctx, &statefulset)) - } + switch resourceType { + case "deployment": + toDelete.Gvk = gvk.Deployment + case "statefulset": + toDelete.Gvk = gvk.StatefulSet } - for _, statefulset := range markedForDeletion { - statefulset := statefulset - if e := cli.Get(ctx, client.ObjectKey{ - Namespace: namespace, - Name: statefulset.Name, - }, &statefulset); e != nil { - if apierrs.IsNotFound(e) { - // resource has been successfully deleted - continue - } - // unexpected error, report it - multiErr = multierror.Append(multiErr, e) - } else { - // resource still exists, wait for it to be deleted - return false, nil - } - } + matcher := action.Not(action.MatchMapKeyContains(labels.ODHAppPrefix, "spec", "selector", "matchLabels")) + act := action.NewDeleteMatched(cli, matcher) - return true, multiErr.ErrorOrNil() + return act.ExecWithRetry(context.TODO(), action.IfAnyLeft(matcher), toDelete) } func RemoveLabel(cli client.Client, objectName string, labelKey string) error { - foundNamespace := &corev1.Namespace{} - if err := cli.Get(context.TODO(), client.ObjectKey{Name: objectName}, foundNamespace); err != nil { - if apierrs.IsNotFound(err) { - return nil - } - return fmt.Errorf("could not get %s namespace: %w", objectName, err) - } - delete(foundNamespace.Labels, labelKey) - if err := cli.Update(context.TODO(), foundNamespace); err != nil { - return fmt.Errorf("error removing %s from %s : %w", labelKey, objectName, err) - } - return nil + return action.NewDeleteLabel(cli, labelKey). + Exec(context.TODO(), action.ResourceSpec{ + Gvk: gvk.Namespace, + Path: []string{"metadata", "name"}, + Values: []string{objectName}, + }) } func deleteDeprecatedNamespace(ctx context.Context, cli client.Client, namespace string) error {