From f7bac10c1fcdde735c1d50955ce3c9458cfd90f6 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Thu, 18 Apr 2024 15:52:28 +0300 Subject: [PATCH 1/9] pkg/action: add module Add abstraction to perfrom action on a set of resources defined as type ResourceSpec struct { Gvk schema.GroupVersionKind Namespace string // path to the field, like "metadata", "name" Path []string // set of values for the field to match object, any one matches Values []string } Provide high level wrappers and lowlevel builder interface. It uses Action directly since there is no any validation or generation logic at the moment. Generated Action does not contain resource list and can be applied to a set of resources with Exec() or ExecWithRetry() methods. Provide also predicates to cover matching for current usecases. User can define its own actions, matchers and retry checkers with signatures: type MatcherFunc func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) type ActionFunc func(ctx context.Context, c client.Client, r ResourceSpec, obj *unstructured.Unstructured) error type RetryCheckFunc func(ctx context.Context, c client.Client, resources ...ResourceSpec) (bool, error) The latter one is taken by ExecWithRetry to check is the job is done. Examples: err = action.NewDeleteMatched(c, action.Not( action.MatchMapKeyContains(ODHAppPrefix, "spec", "selector", "matchLabels"))). Exec(ctx, d...) err = action.NewDelete(c).Exec(ctx, d...) With the builder: err = action.New(c). Do(action.Delete). ExecWithRetry(ctx, action.IfAnyLeft(action.DefaultMatcher), d...) Signed-off-by: Yauheni Kaliuta --- pkg/action/action.go | 307 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 pkg/action/action.go diff --git a/pkg/action/action.go b/pkg/action/action.go new file mode 100644 index 00000000000..ff3a54396ac --- /dev/null +++ b/pkg/action/action.go @@ -0,0 +1,307 @@ +package action + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/hashicorp/go-multierror" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ResourceSpec struct { + Gvk schema.GroupVersionKind + Namespace string + // path to the field, like "metadata", "name" + Path []string + // set of values for the field to match object, any one matches + Values []string +} + +type MatcherFunc func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) +type Func func(ctx context.Context, c client.Client, r ResourceSpec, obj *unstructured.Unstructured) error +type RetryCheckFunc func(ctx context.Context, c client.Client, resources ...ResourceSpec) (bool, error) + +type Action struct { + client client.Client + + matcher MatcherFunc + actions []Func +} + +// shouldn't just return false on error? +func DefaultMatcher(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + if len(r.Path) == 0 || len(r.Values) == 0 { + return true, nil + } + + v, ok, err := unstructured.NestedString(obj.Object, r.Path...) + if err != nil { + return false, fmt.Errorf("failed to get field %v for %s %s/%s: %w", r.Path, r.Gvk.Kind, r.Namespace, obj.GetName(), err) + } + + if !ok { + return false, fmt.Errorf("unexisting path to handle: %v", r.Path) + } + + for _, toDelete := range r.Values { + if v == toDelete { + return true, nil + } + } + + return false, nil +} + +func New(c client.Client) *Action { + return &Action{ + client: c, + matcher: DefaultMatcher, + } +} + +func Not(m MatcherFunc) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + matched, err := m(r, obj) + return !matched, err + } +} + +func Any(matchers ...MatcherFunc) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + for _, m := range matchers { + matched, err := m(r, obj) + if err != nil { + return false, err + } + if matched { + return true, err + } + } + return false, nil + } +} + +func All(matchers ...MatcherFunc) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + for _, m := range matchers { + matched, err := m(r, obj) + if err != nil { + return false, err + } + if !matched { + return false, err + } + } + return true, nil + } +} + +func (o *Action) ForMatched(m MatcherFunc) *Action { + o.matcher = m + return o +} + +func (o *Action) Do(a Func) *Action { + o.actions = append(o.actions, a) + return o +} + +func (o *Action) execOneResource(ctx context.Context, r ResourceSpec, objs []*unstructured.Unstructured) error { + for _, item := range objs { + for _, a := range o.actions { + err := a(ctx, o.client, r, item) + if err != nil { + return err + } + } + } + + return nil +} + +func ListMatched(ctx context.Context, c client.Client, matcher MatcherFunc, resources ...ResourceSpec) (map[*ResourceSpec][]*unstructured.Unstructured, error) { + ret := make(map[*ResourceSpec][]*unstructured.Unstructured) + + for _, r := range resources { + r := r + var items []*unstructured.Unstructured + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(r.Gvk) + + err := c.List(ctx, list, client.InNamespace(r.Namespace)) + if err != nil { + if errors.Is(err, &meta.NoKindMatchError{}) { + fmt.Printf("Could not list %v: CRD not found\n", r.Gvk) + continue + } + return ret, fmt.Errorf("failed to list %s: %w", r.Gvk.Kind, err) + } + + for _, item := range list.Items { + item := item + + matched, err := matcher(r, &item) + if err != nil { + return ret, err + } + + if !matched { + continue + } + + items = append(items, &item) + } + + if len(items) > 0 { + ret[&r] = items + } + } + + return ret, nil +} + +func (o *Action) Exec(ctx context.Context, resources ...ResourceSpec) error { + var errors *multierror.Error + + matched, err := ListMatched(ctx, o.client, o.matcher, resources...) + if err != nil { + return err + } + + for r, objs := range matched { + err := o.execOneResource(ctx, *r, objs) + errors = multierror.Append(errors, err) + } + + return errors.ErrorOrNil() +} + +func (o *Action) ExecWithRetry(ctx context.Context, shouldRetry RetryCheckFunc, resources ...ResourceSpec) error { + return wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + // 5, 10, ,20, 40 then timeout + Duration: 5 * time.Second, + Factor: 2.0, + Jitter: 0.1, + Steps: 4, + Cap: 1 * time.Minute, + }, func(ctx context.Context) (bool, error) { + err := o.Exec(ctx, resources...) + if err != nil { + return false, err + } + return shouldRetry(ctx, o.client, resources...) + }) +} + +func (o *Action) DryRun(_ context.Context, _ ...ResourceSpec) error { + return nil +} + +func Delete(ctx context.Context, c client.Client, _ ResourceSpec, obj *unstructured.Unstructured) error { + return client.IgnoreNotFound(c.Delete(ctx, obj)) +} + +func IfAnyLeft(matcher MatcherFunc) RetryCheckFunc { + return func(ctx context.Context, c client.Client, resources ...ResourceSpec) (bool, error) { + matched, err := ListMatched(ctx, c, matcher, resources...) + if err != nil { + return false, err + } + + return len(matched) == 0, nil + } +} + +func deleteField(obj map[string]any, path ...string) error { + if len(path) < 1 { + return fmt.Errorf("path is empty") + } + + parent := path[:len(path)-1] + field := path[len(path)-1] + + v, ok, err := unstructured.NestedFieldNoCopy(obj, parent...) + if err != nil || !ok { + return fmt.Errorf("Not found or error") + } + + m, ok := v.(map[string]any) + if !ok { + return fmt.Errorf("field is not map") + } + delete(m, field) + return nil +} + +func DeleteField(path ...string) Func { + return func(ctx context.Context, c client.Client, r ResourceSpec, obj *unstructured.Unstructured) error { + err := deleteField(obj.Object, path...) + if err != nil { + return fmt.Errorf("could not delete field %v in object %s : %w", path, obj.GetName(), err) + } + + err = c.Update(ctx, obj) + if err != nil { + return fmt.Errorf("error updating object while removing %v from %v : %w", path, obj.GetName(), err) + } + + return nil + } +} + +func MatchMap(key, value string, keyMatch func(value, pattern string) bool, path ...string) MatcherFunc { + return func(r ResourceSpec, obj *unstructured.Unstructured) (bool, error) { + m, ok, err := unstructured.NestedStringMap(obj.Object, path...) + if err != nil || !ok { + return false, err + } + + for k, v := range m { + if !keyMatch(k, key) { + continue + } + + if value == "" || v == value { + return true, nil + } + } + + return false, nil + } +} + +func MatchMapKeyContains(key string, path ...string) MatcherFunc { + return MatchMap(key, "", strings.Contains, path...) +} + +func NewDelete(c client.Client) *Action { + return New(c).Do(Delete) +} + +func NewDeleteMatched(c client.Client, m MatcherFunc) *Action { + return New(c).Do(Delete).ForMatched(m) +} + +func NewDeleteWithFinalizer(c client.Client) *Action { + return New(c). + Do(DeleteField("metadata", "finalizer")). + Do(Delete) +} + +func NewDeleteOwnersReferences(c client.Client) *Action { + return New(c). + Do(DeleteField("metadata", "ownerReferences")) +} + +func NewDeleteLabel(c client.Client, label string) *Action { + return New(c). + Do(DeleteField("metadata", "labels", label)) +} From b621616aa4b4a511ba5693e0aa95d5f6ed4533e3 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 14:04:45 +0300 Subject: [PATCH 2/9] update gvk --- pkg/cluster/gvk/gvk.go | 72 +++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/pkg/cluster/gvk/gvk.go b/pkg/cluster/gvk/gvk.go index 3a4bad2c363..fea2aa3234c 100644 --- a/pkg/cluster/gvk/gvk.go +++ b/pkg/cluster/gvk/gvk.go @@ -3,29 +3,44 @@ package gvk import "k8s.io/apimachinery/pkg/runtime/schema" var ( + ClusterRole = schema.GroupVersionKind{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + } + ClusterRoleBinding = schema.GroupVersionKind{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRoleBinding", + } + Deployment = schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + } + KfDef = schema.GroupVersionKind{ + Group: "kfdef.apps.kubeflow.org", + Version: "v1", Kind: "KfDef", + } KnativeServing = schema.GroupVersionKind{ Group: "operator.knative.dev", Version: "v1beta1", Kind: "KnativeServing", } - - OpenshiftIngress = schema.GroupVersionKind{ - Group: "config.openshift.io", - Version: "v1", - Kind: "Ingress", - } - - ServiceMeshControlPlane = schema.GroupVersionKind{ - Group: "maistra.io", - Version: "v2", - Kind: "ServiceMeshControlPlane", + Namespace = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "Namespace", } - OdhApplication = schema.GroupVersionKind{ Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication", } + OdhDashboardConfig = schema.GroupVersionKind{ + Group: "opendatahub.io", + Version: "v1alpha", + Kind: "OdhDashboardConfig", + } OdhDocument = schema.GroupVersionKind{ Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhDocument", @@ -34,4 +49,37 @@ var ( Group: "console.openshift.io", Version: "v1", Kind: "OdhQuickStart", } + OpenshiftIngress = schema.GroupVersionKind{ + Group: "config.openshift.io", + Version: "v1", + Kind: "Ingress", + } + Route = schema.GroupVersionKind{ + Group: "route.openshift.io", + Version: "v1", Kind: "Route", + } + Secret = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "Secret", + } + Service = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "Service", + } + ServiceAccount = schema.GroupVersionKind{ + Group: "", + Version: "v1", Kind: "ServiceAccount", + } + ServiceMeshControlPlane = schema.GroupVersionKind{ + Group: "maistra.io", + Version: "v2", Kind: "ServiceMeshControlPlane", + } + ServiceMonitor = schema.GroupVersionKind{ + Group: "monitoring.coreos.com", + Version: "v1", Kind: "ServiceMonitor", + } + StatefulSet = schema.GroupVersionKind{ + Group: "apps", + Version: "v1", Kind: "StatefulSet", + } ) From d26f5531e5a84b19abde4cecd3f3c0130a848d48 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 13:18:50 +0300 Subject: [PATCH 3/9] pkg/upgrade: convert to action watson deletion Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 67 +++--------------------------------------- 1 file changed, 4 insertions(+), 63 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index dbe211b301c..fffb3a33c8d 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -4,7 +4,6 @@ package upgrade import ( "context" - "errors" "fmt" "reflect" "strings" @@ -19,7 +18,6 @@ import ( authv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -42,20 +40,12 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components/trainingoperator" "github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/action" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" ) -type ResourceSpec struct { - Gvk schema.GroupVersionKind - Namespace string - // path to the field, like "metadata", "name" - Path []string - // set of values for the field to match object, any one matches - Values []string -} - // CreateDefaultDSC creates a default instance of DSC. // Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource. func CreateDefaultDSC(ctx context.Context, cli client.Client) error { @@ -267,12 +257,12 @@ func UpdateFromLegacyVersion(cli client.Client, platform cluster.Platform, appNS return nil } -func getDashboardWatsonResources(ns string) []ResourceSpec { +func getDashboardWatsonResources(ns string) []action.ResourceSpec { metadataName := []string{"metadata", "name"} specAppName := []string{"spec", "appName"} appName := []string{"watson-studio"} - return []ResourceSpec{ + return []action.ResourceSpec{ { Gvk: gvk.OdhQuickStart, Namespace: ns, @@ -343,60 +333,11 @@ func CleanupExistingResource(ctx context.Context, cli client.Client, platform cl // to take a reference toDelete := getDashboardWatsonResources(dscApplicationsNamespace) - multiErr = multierror.Append(multiErr, deleteResources(ctx, cli, &toDelete)) + multiErr = multierror.Append(multiErr, action.NewDelete(cli).Exec(ctx, toDelete...)) return multiErr.ErrorOrNil() } -func deleteResources(ctx context.Context, c client.Client, resources *[]ResourceSpec) error { - var errors *multierror.Error - - for _, res := range *resources { - err := deleteOneResource(ctx, c, res) - errors = multierror.Append(errors, err) - } - - return errors.ErrorOrNil() -} - -func deleteOneResource(ctx context.Context, c client.Client, res ResourceSpec) error { - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(res.Gvk) - - err := c.List(ctx, list, client.InNamespace(res.Namespace)) - if err != nil { - if errors.Is(err, &meta.NoKindMatchError{}) { - fmt.Printf("Could not delete %v: CRD not found\n", res.Gvk) - return nil - } - return fmt.Errorf("failed to list %s: %w", res.Gvk.Kind, err) - } - - for _, item := range list.Items { - item := item - v, ok, err := unstructured.NestedString(item.Object, res.Path...) - if err != nil { - return fmt.Errorf("failed to get field %v for %s %s/%s: %w", res.Path, res.Gvk.Kind, res.Namespace, item.GetName(), err) - } - - if !ok { - return fmt.Errorf("unexisting path to delete: %v", res.Path) - } - - for _, toDelete := range res.Values { - if v == toDelete { - err = c.Delete(ctx, &item) - if err != nil { - return fmt.Errorf("failed to delete %s %s/%s: %w", res.Gvk.Kind, res.Namespace, item.GetName(), err) - } - fmt.Println("Deleted object", item.GetName(), res.Gvk, "in namespace", res.Namespace) - } - } - } - - return nil -} - func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { // Check if kfdef are deployed kfdefCrd := &apiextv1.CustomResourceDefinition{} From 0674fd2cf0fb7239ef31cc8a1c009748348043ab Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 13:38:27 +0300 Subject: [PATCH 4/9] pkg/upgrade: convert ModelMesh resource removing to action Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 178 ++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 99 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index fffb3a33c8d..735b9c8d8de 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -5,17 +5,13 @@ package upgrade import ( "context" "fmt" - "reflect" "strings" "time" "github.com/hashicorp/go-multierror" operatorv1 "github.com/openshift/api/operator/v1" - routev1 "github.com/openshift/api/route/v1" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - authv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -284,55 +280,95 @@ func getDashboardWatsonResources(ns string) []action.ResourceSpec { } } +func getModelMeshResources(ns string, platform cluster.Platform) []action.ResourceSpec { + metadataName := []string{"metadata", "name"} + var toDelete []action.ResourceSpec + + if platform == cluster.ManagedRhods { + del := []action.ResourceSpec{ + { + Gvk: gvk.Deployment, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-prometheus-operator"}, + }, + { + Gvk: gvk.StatefulSet, + Namespace: ns, + Path: metadataName, + Values: []string{"prometheus-rhods-model-monitoring"}, + }, + { + Gvk: gvk.Service, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-model-monitoring"}, + }, + { + Gvk: gvk.Route, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-model-monitoring"}, + }, + { + Gvk: gvk.Secret, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-monitoring-oauth-config"}, + }, + { + Gvk: gvk.ClusterRole, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"}, + }, + { + Gvk: gvk.ClusterRoleBinding, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"}, + }, + { + Gvk: gvk.ServiceAccount, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-prometheus-operator"}, + }, + { + Gvk: gvk.ServiceMonitor, + Namespace: ns, + Path: metadataName, + Values: []string{"modelmesh-federated-metrics"}, + }, + } + toDelete = append(toDelete, del...) + } + // common logic for both self-managed and managed + del := action.ResourceSpec{ + Gvk: gvk.ServiceMonitor, + Namespace: ns, + Path: metadataName, + Values: []string{"rhods-monitor-federation2"}, + } + + toDelete = append(toDelete, del) + return toDelete +} + // TODO: remove function once we have a generic solution across all components. func CleanupExistingResource(ctx context.Context, cli client.Client, platform cluster.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error { var multiErr *multierror.Error // Special Handling of cleanup of deprecated model monitoring stack - if platform == cluster.ManagedRhods { - deprecatedDeployments := []string{"rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedDeployments, &appsv1.DeploymentList{})) - - deprecatedStatefulsets := []string{"prometheus-rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedStatefulsets, &appsv1.StatefulSetList{})) - - deprecatedServices := []string{"rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServices, &corev1.ServiceList{})) - - deprecatedRoutes := []string{"rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedRoutes, &routev1.RouteList{})) - - deprecatedSecrets := []string{"rhods-monitoring-oauth-config"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedSecrets, &corev1.SecretList{})) - - deprecatedClusterroles := []string{"rhods-namespace-read", "rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterroles, &authv1.ClusterRoleList{})) - - deprecatedClusterrolebindings := []string{"rhods-namespace-read", "rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterrolebindings, &authv1.ClusterRoleBindingList{})) - - deprecatedServiceAccounts := []string{"rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServiceAccounts, &corev1.ServiceAccountList{})) - - deprecatedServicemonitors := []string{"modelmesh-federated-metrics"} - multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedServicemonitors)) - } - // common logic for both self-managed and managed - deprecatedOperatorSM := []string{"rhods-monitor-federation2"} - multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedOperatorSM)) // Remove deprecated opendatahub namespace(owned by kuberay) multiErr = multierror.Append(multiErr, deleteDeprecatedNamespace(ctx, cli, "opendatahub")) // Handling for dashboard Jupyterhub CR, see jira #443 - JupyterhubApp := schema.GroupVersionKind{ - Group: "dashboard.opendatahub.io", - Version: "v1", - Kind: "OdhApplication", - } - multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, JupyterhubApp, "jupyterhub", dscApplicationsNamespace)) + multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, gvk.OdhApplication, "jupyterhub", dscApplicationsNamespace)) + + toDelete := getModelMeshResources(dscMonitoringNamespace, platform) + toDelete = append(toDelete, getDashboardWatsonResources(dscApplicationsNamespace)...) - // to take a reference - toDelete := getDashboardWatsonResources(dscApplicationsNamespace) multiErr = multierror.Append(multiErr, action.NewDelete(cli).Exec(ctx, toDelete...)) return multiErr.ErrorOrNil() @@ -373,62 +409,6 @@ func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { return nil } -func deleteDeprecatedResources(ctx context.Context, cli client.Client, namespace string, resourceList []string, resourceType client.ObjectList) error { - var multiErr *multierror.Error - listOpts := &client.ListOptions{Namespace: namespace} - if err := cli.List(ctx, resourceType, listOpts); err != nil { - multiErr = multierror.Append(multiErr, err) - } - items := reflect.ValueOf(resourceType).Elem().FieldByName("Items") - for i := 0; i < items.Len(); i++ { - item := items.Index(i).Addr().Interface().(client.Object) //nolint:errcheck,forcetypeassert - for _, name := range resourceList { - if name == item.GetName() { - fmt.Printf("Attempting to delete %s in namespace %s\n", item.GetName(), namespace) - err := cli.Delete(ctx, item) - if err != nil { - if apierrs.IsNotFound(err) { - fmt.Printf("Could not find %s in namespace %s\n", item.GetName(), namespace) - } else { - multiErr = multierror.Append(multiErr, err) - } - } - fmt.Printf("Successfully deleted %s\n", item.GetName()) - } - } - } - return multiErr.ErrorOrNil() -} - -// Need to handle ServiceMonitor deletion separately as the generic function does not work for ServiceMonitors because of how the package is built. -func deleteDeprecatedServiceMonitors(ctx context.Context, cli client.Client, namespace string, resourceList []string) error { - var multiErr *multierror.Error - listOpts := &client.ListOptions{Namespace: namespace} - servicemonitors := &monitoringv1.ServiceMonitorList{} - if err := cli.List(ctx, servicemonitors, listOpts); err != nil { - multiErr = multierror.Append(multiErr, err) - } - - for _, servicemonitor := range servicemonitors.Items { - servicemonitor := servicemonitor - for _, name := range resourceList { - if name == servicemonitor.Name { - fmt.Printf("Attempting to delete %s in namespace %s\n", servicemonitor.Name, namespace) - err := cli.Delete(ctx, servicemonitor) - if err != nil { - if apierrs.IsNotFound(err) { - fmt.Printf("Could not find %s in namespace %s\n", servicemonitor.Name, namespace) - } else { - multiErr = multierror.Append(multiErr, err) - } - } - fmt.Printf("Successfully deleted %s\n", servicemonitor.Name) - } - } - } - return multiErr.ErrorOrNil() -} - func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind, instanceName string, applicationNS string) error { // first check if CRD in cluster crd := &apiextv1.CustomResourceDefinition{} From 19ce6e784940c0893a5864ed0b2a2ca1c5024e99 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 14:12:50 +0300 Subject: [PATCH 5/9] pkt/upgrade: convert removOdhApplicationsCR to Action Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 735b9c8d8de..515c5710866 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -363,11 +363,15 @@ func CleanupExistingResource(ctx context.Context, cli client.Client, platform cl // Remove deprecated opendatahub namespace(owned by kuberay) multiErr = multierror.Append(multiErr, deleteDeprecatedNamespace(ctx, cli, "opendatahub")) - // Handling for dashboard Jupyterhub CR, see jira #443 - multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, gvk.OdhApplication, "jupyterhub", dscApplicationsNamespace)) - toDelete := getModelMeshResources(dscMonitoringNamespace, platform) toDelete = append(toDelete, getDashboardWatsonResources(dscApplicationsNamespace)...) + // Handling for dashboard Jupyterhub CR, see jira #443 + toDelete = append(toDelete, action.ResourceSpec{ + Gvk: gvk.OdhApplication, + Namespace: dscApplicationsNamespace, + Path: []string{"metadata", "name"}, + Values: []string{"jupyterhub"}, + }) multiErr = multierror.Append(multiErr, action.NewDelete(cli).Exec(ctx, toDelete...)) @@ -409,29 +413,6 @@ func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { return nil } -func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind, instanceName string, applicationNS string) error { - // first check if CRD in cluster - crd := &apiextv1.CustomResourceDefinition{} - if err := cli.Get(ctx, client.ObjectKey{Name: "odhapplications.dashboard.opendatahub.io"}, crd); err != nil { - return client.IgnoreNotFound(err) - } - - // then check if CR in cluster to delete - odhObject := &unstructured.Unstructured{} - odhObject.SetGroupVersionKind(gvk) - if err := cli.Get(ctx, client.ObjectKey{ - Namespace: applicationNS, - Name: instanceName, - }, odhObject); err != nil { - return client.IgnoreNotFound(err) - } - if err := cli.Delete(ctx, odhObject); err != nil { - return fmt.Errorf("error deleting CR %s : %w", instanceName, err) - } - - return nil -} - func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error { OdhDashboardConfig := schema.GroupVersionKind{ Group: "opendatahub.io", From 16aa6679e63b272973ef99ee8b086b9db6aa8b19 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 14:21:53 +0300 Subject: [PATCH 6/9] pkg/upgrade: convert RemoveKfDefInstances to Action Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 515c5710866..0f7b6633b27 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -379,38 +379,10 @@ func CleanupExistingResource(ctx context.Context, cli client.Client, platform cl } func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { - // Check if kfdef are deployed - kfdefCrd := &apiextv1.CustomResourceDefinition{} - - err := cli.Get(ctx, client.ObjectKey{Name: "kfdefs.kfdef.apps.kubeflow.org"}, kfdefCrd) - if err != nil { - if apierrs.IsNotFound(err) { - // If no Crd found, return, since its a new Installation - return nil - } - return fmt.Errorf("error retrieving kfdef CRD : %w", err) - } - expectedKfDefList := &kfdefv1.KfDefList{} - err = cli.List(ctx, expectedKfDefList) - if err != nil { - return fmt.Errorf("error getting list of kfdefs: %w", err) - } - // Delete kfdefs - for _, kfdef := range expectedKfDefList.Items { - kfdef := kfdef - // Remove finalizer - updatedKfDef := &kfdef - updatedKfDef.Finalizers = []string{} - err = cli.Update(ctx, updatedKfDef) - if err != nil { - return fmt.Errorf("error removing finalizers from kfdef %v : %w", kfdef.Name, err) - } - err = cli.Delete(ctx, updatedKfDef) - if err != nil { - return fmt.Errorf("error deleting kfdef %v : %w", kfdef.Name, err) - } - } - return nil + return action.NewDeleteWithFinalizer(cli). + Exec(ctx, action.ResourceSpec{ + Gvk: gvk.KfDef, + }) } func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error { From d1f65c82b8ce1a26547604c7d4ee5e8389e4dc05 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 14:40:23 +0300 Subject: [PATCH 7/9] pkg/upgrade: convert deleteResource to Action Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 137 +++-------------------------------------- 1 file changed, 9 insertions(+), 128 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 0f7b6633b27..72761e60d6d 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -5,19 +5,15 @@ package upgrade import ( "context" "fmt" - "strings" - "time" "github.com/hashicorp/go-multierror" operatorv1 "github.com/openshift/api/operator/v1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" kfdefv1 "github.com/opendatahub-io/opendatahub-operator/apis/kfdef.apps.kubeflow.org/v1" @@ -419,137 +415,22 @@ func deleteResource(cli client.Client, namespace string, resourceType string) er // deployments and recreated them. // because we can't proceed if a deployment is not deleted, we use exponential backoff // to retry the deletion until it succeeds - var err error - switch resourceType { - case "deployment": - err = wait.ExponentialBackoffWithContext(context.TODO(), wait.Backoff{ - // 5, 10, ,20, 40 then timeout - Duration: 5 * time.Second, - Factor: 2.0, - Jitter: 0.1, - Steps: 4, - Cap: 1 * time.Minute, - }, func(ctx context.Context) (bool, error) { - done, err := deleteDeploymentsAndCheck(ctx, cli, namespace) - return done, err - }) - case "statefulset": - err = wait.ExponentialBackoffWithContext(context.TODO(), wait.Backoff{ - // 10, 20 then timeout - Duration: 10 * time.Second, - Factor: 2.0, - Jitter: 0.1, - Steps: 2, - Cap: 1 * time.Minute, - }, func(ctx context.Context) (bool, error) { - done, err := deleteStatefulsetsAndCheck(ctx, cli, namespace) - return done, err - }) - } - return err -} -func deleteDeploymentsAndCheck(ctx context.Context, cli client.Client, namespace string) (bool, error) { - // Delete Deployment objects - var multiErr *multierror.Error - deployments := &appsv1.DeploymentList{} - listOpts := &client.ListOptions{ + toDelete := action.ResourceSpec{ Namespace: namespace, } - if err := cli.List(ctx, deployments, listOpts); err != nil { - return false, nil //nolint:nilerr - } - // filter deployment which has the new label to limit that we do not overkill other deployment - // this logic can be used even when upgrade from v2.4 to v2.5 without remove it - markedForDeletion := []appsv1.Deployment{} - for _, deployment := range deployments.Items { - deployment := deployment - v2 := false - selectorLabels := deployment.Spec.Selector.MatchLabels - for label := range selectorLabels { - if strings.Contains(label, labels.ODHAppPrefix) { - // this deployment has the new label, this is a v2 to v2 upgrade - // there is no need to recreate it, as labels are matching - v2 = true - continue - } - } - if !v2 { - markedForDeletion = append(markedForDeletion, deployment) - multiErr = multierror.Append(multiErr, cli.Delete(ctx, &deployment)) - } - } - - for _, deployment := range markedForDeletion { - deployment := deployment - if e := cli.Get(ctx, client.ObjectKey{ - Namespace: namespace, - Name: deployment.Name, - }, &deployment); e != nil { - if apierrs.IsNotFound(e) { - // resource has been successfully deleted - continue - } - // unexpected error, report it - multiErr = multierror.Append(multiErr, e) //nolint:staticcheck,wastedassign - } - // resource still exists, wait for it to be deleted - return false, nil - } - - return true, multiErr.ErrorOrNil() -} - -func deleteStatefulsetsAndCheck(ctx context.Context, cli client.Client, namespace string) (bool, error) { - // Delete statefulset objects - var multiErr *multierror.Error - statefulsets := &appsv1.StatefulSetList{} - listOpts := &client.ListOptions{ - Namespace: namespace, - } - - if err := cli.List(ctx, statefulsets, listOpts); err != nil { - return false, nil //nolint:nilerr - } - - // even only we have one item to delete to avoid nil point still use range - markedForDeletion := []appsv1.StatefulSet{} - for _, statefulset := range statefulsets.Items { - v2 := false - statefulset := statefulset - selectorLabels := statefulset.Spec.Selector.MatchLabels - for label := range selectorLabels { - if strings.Contains(label, labels.ODHAppPrefix) { - v2 = true - continue - } - } - if !v2 { - markedForDeletion = append(markedForDeletion, statefulset) - multiErr = multierror.Append(multiErr, cli.Delete(ctx, &statefulset)) - } + switch resourceType { + case "deployment": + toDelete.Gvk = gvk.Deployment + case "statefulset": + toDelete.Gvk = gvk.StatefulSet } - for _, statefulset := range markedForDeletion { - statefulset := statefulset - if e := cli.Get(ctx, client.ObjectKey{ - Namespace: namespace, - Name: statefulset.Name, - }, &statefulset); e != nil { - if apierrs.IsNotFound(e) { - // resource has been successfully deleted - continue - } - // unexpected error, report it - multiErr = multierror.Append(multiErr, e) - } else { - // resource still exists, wait for it to be deleted - return false, nil - } - } + matcher := action.Not(action.MatchMapKeyContains(labels.ODHAppPrefix, "spec", "selector", "matchLabels")) + act := action.NewDeleteMatched(cli, matcher) - return true, multiErr.ErrorOrNil() + return act.ExecWithRetry(context.TODO(), action.IfAnyLeft(matcher), toDelete) } func RemoveLabel(cli client.Client, objectName string, labelKey string) error { From 3e2f2053bac874891ed0ded8012190e6bc9c7245 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 14:46:44 +0300 Subject: [PATCH 8/9] pkg/upgrade: convert unsetOwnerReference to Action Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 72761e60d6d..547e0bdd5ea 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -12,8 +12,6 @@ import ( apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" kfdefv1 "github.com/opendatahub-io/opendatahub-operator/apis/kfdef.apps.kubeflow.org/v1" @@ -382,31 +380,13 @@ func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { } func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error { - OdhDashboardConfig := schema.GroupVersionKind{ - Group: "opendatahub.io", - Version: "v1alpha", - Kind: "OdhDashboardConfig", - } - crd := &apiextv1.CustomResourceDefinition{} - if err := cli.Get(context.TODO(), client.ObjectKey{Name: "odhdashboardconfigs.opendatahub.io"}, crd); err != nil { - return client.IgnoreNotFound(err) - } - odhObject := &unstructured.Unstructured{} - odhObject.SetGroupVersionKind(OdhDashboardConfig) - if err := cli.Get(context.TODO(), client.ObjectKey{ - Namespace: applicationNS, - Name: instanceName, - }, odhObject); err != nil { - return client.IgnoreNotFound(err) - } - if odhObject.GetOwnerReferences() != nil { - // set to nil as updates - odhObject.SetOwnerReferences(nil) - if err := cli.Update(context.TODO(), odhObject); err != nil { - return fmt.Errorf("error unset ownerreference for CR %s : %w", instanceName, err) - } - } - return nil + return action.NewDeleteOwnersReferences(cli). + Exec(context.TODO(), action.ResourceSpec{ + Gvk: gvk.OdhDashboardConfig, + Namespace: applicationNS, + Path: []string{"metadata", "name"}, + Values: []string{instanceName}, + }) } func deleteResource(cli client.Client, namespace string, resourceType string) error { From fe4c4ac55d2b5e67fd22d459a091a5f5529d5a34 Mon Sep 17 00:00:00 2001 From: Yauheni Kaliuta Date: Fri, 26 Apr 2024 14:52:43 +0300 Subject: [PATCH 9/9] pkg/upgrade: convert RemoveLabel to Action Signed-off-by: Yauheni Kaliuta --- pkg/upgrade/upgrade.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 547e0bdd5ea..2da53c237d6 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -414,18 +414,12 @@ func deleteResource(cli client.Client, namespace string, resourceType string) er } func RemoveLabel(cli client.Client, objectName string, labelKey string) error { - foundNamespace := &corev1.Namespace{} - if err := cli.Get(context.TODO(), client.ObjectKey{Name: objectName}, foundNamespace); err != nil { - if apierrs.IsNotFound(err) { - return nil - } - return fmt.Errorf("could not get %s namespace: %w", objectName, err) - } - delete(foundNamespace.Labels, labelKey) - if err := cli.Update(context.TODO(), foundNamespace); err != nil { - return fmt.Errorf("error removing %s from %s : %w", labelKey, objectName, err) - } - return nil + return action.NewDeleteLabel(cli, labelKey). + Exec(context.TODO(), action.ResourceSpec{ + Gvk: gvk.Namespace, + Path: []string{"metadata", "name"}, + Values: []string{objectName}, + }) } func deleteDeprecatedNamespace(ctx context.Context, cli client.Client, namespace string) error {