diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 5e70a5bf89e..3ea6dbd09ed 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -3,17 +3,13 @@ package upgrade import ( "context" "fmt" - "reflect" "strings" "time" "github.com/hashicorp/go-multierror" operatorv1 "github.com/openshift/api/operator/v1" - routev1 "github.com/openshift/api/route/v1" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - authv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,6 +39,30 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels" ) +type JobSpec struct { + Gvk schema.GroupVersionKind + Namespace string + // path to the field, like "metadata", "name" + Path []string + // set of values for the field to match object, any one matches + Values []string +} + +// TODO: move to common place. +var ( + gvkDeployment = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} + gvkStatefulSet = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"} + gvkService = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"} + gvkRoute = schema.GroupVersionKind{Group: "route.openshift.io", Version: "v1", Kind: "Route"} + gvkSecret = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"} + gvkClusterRole = schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"} + gvkClusterRoleBinding = schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"} + gvkServiceAccount = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ServiceAccount"} + gvkServiceMonitor = schema.GroupVersionKind{Group: "monitoring.coreos.com", Version: "v1", Kind: "ServiceMonitor"} + gvkOdhApplication = schema.GroupVersionKind{Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication"} + gvkOdhDashboardConfig = schema.GroupVersionKind{Group: "opendatahub.io", Version: "v1alpha", Kind: "OdhDashboardConfig"} +) + // CreateDefaultDSC creates a default instance of DSC. // Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource. func CreateDefaultDSC(ctx context.Context, cli client.Client) error { @@ -257,47 +277,129 @@ func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform, appNS // TODO: remove function once we have a generic solution across all components. func CleanupExistingResource(ctx context.Context, cli client.Client, platform deploy.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error { var multiErr *multierror.Error + var toDelete []JobSpec + pathName := []string{"metadata", "name"} // Special Handling of cleanup of deprecated model monitoring stack if platform == deploy.ManagedRhods { - deprecatedDeployments := []string{"rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedDeployments, &appsv1.DeploymentList{})) + del := []JobSpec{ + { + Gvk: gvkDeployment, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-prometheus-operator"}, + }, + { + Gvk: gvkStatefulSet, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"prometheus-rhods-model-monitoring"}, + }, + { + Gvk: gvkService, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-model-monitoring"}, + }, + { + Gvk: gvkRoute, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-model-monitoring"}, + }, + { + Gvk: gvkSecret, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-monitoring-oauth-config"}, + }, + { + Gvk: gvkClusterRole, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"}, + }, + { + Gvk: gvkClusterRoleBinding, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"}, + }, + { + Gvk: gvkServiceAccount, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-prometheus-operator"}, + }, + { + Gvk: gvkServiceMonitor, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"modelmesh-federated-metrics"}, + }, + } + toDelete = append(toDelete, del...) + } - deprecatedStatefulsets := []string{"prometheus-rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedStatefulsets, &appsv1.StatefulSetList{})) + // common logic for both self-managed and managed + del := JobSpec{ + Gvk: gvkServiceMonitor, + Namespace: dscMonitoringNamespace, + Path: pathName, + Values: []string{"rhods-monitor-federation2"}, + } - deprecatedServices := []string{"rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServices, &corev1.ServiceList{})) + toDelete = append(toDelete, del) - deprecatedRoutes := []string{"rhods-model-monitoring"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedRoutes, &routev1.RouteList{})) + multiErr = multierror.Append(multiErr, deleteResources(ctx, cli, &toDelete)) - deprecatedSecrets := []string{"rhods-monitoring-oauth-config"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedSecrets, &corev1.SecretList{})) + // Handling for dashboard Jupyterhub CR, see jira #443 + multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, gvkOdhApplication, "jupyterhub", dscApplicationsNamespace)) + return multiErr.ErrorOrNil() +} + +func deleteResources(ctx context.Context, c client.Client, jobs *[]JobSpec) error { + var errors *multierror.Error - deprecatedClusterroles := []string{"rhods-namespace-read", "rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterroles, &authv1.ClusterRoleList{})) + for _, job := range *jobs { + err := deleteOneJob(ctx, c, job) + errors = multierror.Append(errors, err) + } - deprecatedClusterrolebindings := []string{"rhods-namespace-read", "rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterrolebindings, &authv1.ClusterRoleBindingList{})) + return errors.ErrorOrNil() +} - deprecatedServiceAccounts := []string{"rhods-prometheus-operator"} - multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServiceAccounts, &corev1.ServiceAccountList{})) +func deleteOneJob(ctx context.Context, c client.Client, job JobSpec) error { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(job.Gvk) - deprecatedServicemonitors := []string{"modelmesh-federated-metrics"} - multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedServicemonitors)) + err := c.List(ctx, list, client.InNamespace(job.Namespace)) + if err != nil { + return fmt.Errorf("failed to list %s: %w", job.Gvk.Kind, err) } - // common logic for both self-managed and managed - deprecatedOperatorSM := []string{"rhods-monitor-federation2"} - multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedOperatorSM)) - // Handling for dashboard Jupyterhub CR, see jira #443 - JupyterhubApp := schema.GroupVersionKind{ - Group: "dashboard.opendatahub.io", - Version: "v1", - Kind: "OdhApplication", + for _, item := range list.Items { + item := item + v, ok, err := unstructured.NestedString(item.Object, job.Path...) + if err != nil { + return fmt.Errorf("failed to get field %v for %s %s/%s: %w", job.Path, job.Gvk.Kind, job.Namespace, item.GetName(), err) + } + + if !ok { + return fmt.Errorf("unexisting path to delete: %v", job.Path) + } + + for _, toDelete := range job.Values { + if v == toDelete { + err = c.Delete(ctx, &item) + if err != nil { + return fmt.Errorf("failed to delete %s %s/%s: %w", job.Gvk.Kind, job.Namespace, item.GetName(), err) + } + fmt.Println("Deleted object", item.GetName(), job.Gvk, "in namespace", job.Namespace) + } + } } - multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, JupyterhubApp, "jupyterhub", dscApplicationsNamespace)) - return multiErr.ErrorOrNil() + + return nil } func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { @@ -335,62 +437,6 @@ func RemoveKfDefInstances(ctx context.Context, cli client.Client) error { return nil } -func deleteDeprecatedResources(ctx context.Context, cli client.Client, namespace string, resourceList []string, resourceType client.ObjectList) error { - var multiErr *multierror.Error - listOpts := &client.ListOptions{Namespace: namespace} - if err := cli.List(ctx, resourceType, listOpts); err != nil { - multiErr = multierror.Append(multiErr, err) - } - items := reflect.ValueOf(resourceType).Elem().FieldByName("Items") - for i := 0; i < items.Len(); i++ { - item := items.Index(i).Addr().Interface().(client.Object) //nolint:errcheck,forcetypeassert - for _, name := range resourceList { - if name == item.GetName() { - fmt.Printf("Attempting to delete %s in namespace %s\n", item.GetName(), namespace) - err := cli.Delete(ctx, item) - if err != nil { - if apierrs.IsNotFound(err) { - fmt.Printf("Could not find %s in namespace %s\n", item.GetName(), namespace) - } else { - multiErr = multierror.Append(multiErr, err) - } - } - fmt.Printf("Successfully deleted %s\n", item.GetName()) - } - } - } - return multiErr.ErrorOrNil() -} - -// Need to handle ServiceMonitor deletion separately as the generic function does not work for ServiceMonitors because of how the package is built. -func deleteDeprecatedServiceMonitors(ctx context.Context, cli client.Client, namespace string, resourceList []string) error { - var multiErr *multierror.Error - listOpts := &client.ListOptions{Namespace: namespace} - servicemonitors := &monitoringv1.ServiceMonitorList{} - if err := cli.List(ctx, servicemonitors, listOpts); err != nil { - multiErr = multierror.Append(multiErr, err) - } - - for _, servicemonitor := range servicemonitors.Items { - servicemonitor := servicemonitor - for _, name := range resourceList { - if name == servicemonitor.Name { - fmt.Printf("Attempting to delete %s in namespace %s\n", servicemonitor.Name, namespace) - err := cli.Delete(ctx, servicemonitor) - if err != nil { - if apierrs.IsNotFound(err) { - fmt.Printf("Could not find %s in namespace %s\n", servicemonitor.Name, namespace) - } else { - multiErr = multierror.Append(multiErr, err) - } - } - fmt.Printf("Successfully deleted %s\n", servicemonitor.Name) - } - } - } - return multiErr.ErrorOrNil() -} - func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind, instanceName string, applicationNS string) error { // first check if CRD in cluster crd := &apiextv1.CustomResourceDefinition{} @@ -415,17 +461,12 @@ func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.G } func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error { - OdhDashboardConfig := schema.GroupVersionKind{ - Group: "opendatahub.io", - Version: "v1alpha", - Kind: "OdhDashboardConfig", - } crd := &apiextv1.CustomResourceDefinition{} if err := cli.Get(context.TODO(), client.ObjectKey{Name: "odhdashboardconfigs.opendatahub.io"}, crd); err != nil { return client.IgnoreNotFound(err) } odhObject := &unstructured.Unstructured{} - odhObject.SetGroupVersionKind(OdhDashboardConfig) + odhObject.SetGroupVersionKind(gvkOdhDashboardConfig) if err := cli.Get(context.TODO(), client.ObjectKey{ Namespace: applicationNS, Name: instanceName,