Skip to content

Commit

Permalink
pkg/upgrade: abstract resource deletion with Unstructured
Browse files Browse the repository at this point in the history 
For the coming requirement to remove resources not just by their
names but a field inside of spec, abstract the existing
deleteDeprecatedResources/deleteDeprecatedServiceMonitors to take a
resource to delete description as

```
type JobSpec struct {
	Gvk       schema.GroupVersionKind
	Namespace string
	Path      []string
	Values    []string
}
```

and convert existing calls to used it.

The function fetches UnstructuredList by gvk/namespace and uses
public unstructured.NestedString() to access the field by path.
It should work for ServiceMonitor, where ServiceMonitorList contains
list of the pointers, as well.

Ignore NoKindMatchError. CRD may not exist on fresh installation for
example.

Make a wrapper which takes an array of them for convenience and
avoid one indentation level and simplifies multierror wrapping a
bit.

It requires to have a library of GroupVersionKinds which is a future
work to make common for the whole project.

Move OdhApplication and OdhDashboardConfig to the gvk list, it's
proper place now.

Signed-off-by: Yauheni Kaliuta <[email protected]>
  • Loading branch information
@ykaliuta
ykaliuta committed Apr 8, 2024
1 parent 562ae3d commit a206bbe
Showing 1 changed file with 141 additions and 94 deletions.
235 changes: 141 additions & 94 deletions pkg/upgrade/upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,18 @@ package upgrade

import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"time"

"github.com/hashicorp/go-multierror"
operatorv1 "github.com/openshift/api/operator/v1"
routev1 "github.com/openshift/api/route/v1"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
authv1 "k8s.io/api/rbac/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand Down Expand Up @@ -43,6 +41,30 @@ import (
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
)

type JobSpec struct {
Gvk schema.GroupVersionKind
Namespace string
// path to the field, like "metadata", "name"
Path []string
// set of values for the field to match object, any one matches
Values []string
}

// TODO: move to common place.
var (
gvkDeployment = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}
gvkStatefulSet = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}
gvkService = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"}
gvkRoute = schema.GroupVersionKind{Group: "route.openshift.io", Version: "v1", Kind: "Route"}
gvkSecret = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}
gvkClusterRole = schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}
gvkClusterRoleBinding = schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}
gvkServiceAccount = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ServiceAccount"}
gvkServiceMonitor = schema.GroupVersionKind{Group: "monitoring.coreos.com", Version: "v1", Kind: "ServiceMonitor"}
gvkOdhApplication = schema.GroupVersionKind{Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication"}
gvkOdhDashboardConfig = schema.GroupVersionKind{Group: "opendatahub.io", Version: "v1alpha", Kind: "OdhDashboardConfig"}
)

// CreateDefaultDSC creates a default instance of DSC.
// Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource.
func CreateDefaultDSC(ctx context.Context, cli client.Client) error {
Expand Down Expand Up @@ -257,47 +279,133 @@ func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform, appNS
// TODO: remove function once we have a generic solution across all components.
func CleanupExistingResource(ctx context.Context, cli client.Client, platform deploy.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error {
var multiErr *multierror.Error
var toDelete []JobSpec
pathName := []string{"metadata", "name"}
// Special Handling of cleanup of deprecated model monitoring stack
if platform == deploy.ManagedRhods {
deprecatedDeployments := []string{"rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedDeployments, &appsv1.DeploymentList{}))
del := []JobSpec{
{
Gvk: gvkDeployment,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-prometheus-operator"},
},
{
Gvk: gvkStatefulSet,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"prometheus-rhods-model-monitoring"},
},
{
Gvk: gvkService,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-model-monitoring"},
},
{
Gvk: gvkRoute,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-model-monitoring"},
},
{
Gvk: gvkSecret,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-monitoring-oauth-config"},
},
{
Gvk: gvkClusterRole,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"},
},
{
Gvk: gvkClusterRoleBinding,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"},
},
{
Gvk: gvkServiceAccount,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-prometheus-operator"},
},
{
Gvk: gvkServiceMonitor,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"modelmesh-federated-metrics"},
},
}
toDelete = append(toDelete, del...)
}

deprecatedStatefulsets := []string{"prometheus-rhods-model-monitoring"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedStatefulsets, &appsv1.StatefulSetList{}))
// common logic for both self-managed and managed
del := JobSpec{
Gvk: gvkServiceMonitor,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-monitor-federation2"},
}

deprecatedServices := []string{"rhods-model-monitoring"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServices, &corev1.ServiceList{}))
toDelete = append(toDelete, del)

deprecatedRoutes := []string{"rhods-model-monitoring"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedRoutes, &routev1.RouteList{}))
multiErr = multierror.Append(multiErr, deleteResources(ctx, cli, &toDelete))

// Handling for dashboard Jupyterhub CR, see jira #443
multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, gvkOdhApplication, "jupyterhub", dscApplicationsNamespace))
return multiErr.ErrorOrNil()
}

deprecatedSecrets := []string{"rhods-monitoring-oauth-config"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedSecrets, &corev1.SecretList{}))
func deleteResources(ctx context.Context, c client.Client, jobs *[]JobSpec) error {
var errors *multierror.Error

deprecatedClusterroles := []string{"rhods-namespace-read", "rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterroles, &authv1.ClusterRoleList{}))
for _, job := range *jobs {
err := deleteOneJob(ctx, c, job)
errors = multierror.Append(errors, err)
}

deprecatedClusterrolebindings := []string{"rhods-namespace-read", "rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterrolebindings, &authv1.ClusterRoleBindingList{}))
return errors.ErrorOrNil()
}

deprecatedServiceAccounts := []string{"rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServiceAccounts, &corev1.ServiceAccountList{}))
func deleteOneJob(ctx context.Context, c client.Client, job JobSpec) error {
list := &unstructured.UnstructuredList{}
list.SetGroupVersionKind(job.Gvk)

deprecatedServicemonitors := []string{"modelmesh-federated-metrics"}
multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedServicemonitors))
err := c.List(ctx, list, client.InNamespace(job.Namespace))
if err != nil {
if errors.Is(err, &meta.NoKindMatchError{}) {
fmt.Printf("Could not delete %v: CRD not found\n", job.Gvk)
} else {
return fmt.Errorf("failed to list %s: %w", job.Gvk.Kind, err)
}
}
// common logic for both self-managed and managed
deprecatedOperatorSM := []string{"rhods-monitor-federation2"}
multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedOperatorSM))

// Handling for dashboard Jupyterhub CR, see jira #443
JupyterhubApp := schema.GroupVersionKind{
Group: "dashboard.opendatahub.io",
Version: "v1",
Kind: "OdhApplication",
for _, item := range list.Items {
item := item
v, ok, err := unstructured.NestedString(item.Object, job.Path...)
if err != nil {
return fmt.Errorf("failed to get field %v for %s %s/%s: %w", job.Path, job.Gvk.Kind, job.Namespace, item.GetName(), err)
}

if !ok {
return fmt.Errorf("unexisting path to delete: %v", job.Path)
}

for _, toDelete := range job.Values {
if v == toDelete {
err = c.Delete(ctx, &item)
if err != nil {
return fmt.Errorf("failed to delete %s %s/%s: %w", job.Gvk.Kind, job.Namespace, item.GetName(), err)
}
fmt.Println("Deleted object", item.GetName(), job.Gvk, "in namespace", job.Namespace)
}
}
}
multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, JupyterhubApp, "jupyterhub", dscApplicationsNamespace))
return multiErr.ErrorOrNil()

return nil
}

func RemoveKfDefInstances(ctx context.Context, cli client.Client) error {
Expand Down Expand Up @@ -335,62 +443,6 @@ func RemoveKfDefInstances(ctx context.Context, cli client.Client) error {
return nil
}

func deleteDeprecatedResources(ctx context.Context, cli client.Client, namespace string, resourceList []string, resourceType client.ObjectList) error {
var multiErr *multierror.Error
listOpts := &client.ListOptions{Namespace: namespace}
if err := cli.List(ctx, resourceType, listOpts); err != nil {
multiErr = multierror.Append(multiErr, err)
}
items := reflect.ValueOf(resourceType).Elem().FieldByName("Items")
for i := 0; i < items.Len(); i++ {
item := items.Index(i).Addr().Interface().(client.Object) //nolint:errcheck,forcetypeassert
for _, name := range resourceList {
if name == item.GetName() {
fmt.Printf("Attempting to delete %s in namespace %s\n", item.GetName(), namespace)
err := cli.Delete(ctx, item)
if err != nil {
if apierrs.IsNotFound(err) {
fmt.Printf("Could not find %s in namespace %s\n", item.GetName(), namespace)
} else {
multiErr = multierror.Append(multiErr, err)
}
}
fmt.Printf("Successfully deleted %s\n", item.GetName())
}
}
}
return multiErr.ErrorOrNil()
}

// Need to handle ServiceMonitor deletion separately as the generic function does not work for ServiceMonitors because of how the package is built.
func deleteDeprecatedServiceMonitors(ctx context.Context, cli client.Client, namespace string, resourceList []string) error {
var multiErr *multierror.Error
listOpts := &client.ListOptions{Namespace: namespace}
servicemonitors := &monitoringv1.ServiceMonitorList{}
if err := cli.List(ctx, servicemonitors, listOpts); err != nil {
multiErr = multierror.Append(multiErr, err)
}

for _, servicemonitor := range servicemonitors.Items {
servicemonitor := servicemonitor
for _, name := range resourceList {
if name == servicemonitor.Name {
fmt.Printf("Attempting to delete %s in namespace %s\n", servicemonitor.Name, namespace)
err := cli.Delete(ctx, servicemonitor)
if err != nil {
if apierrs.IsNotFound(err) {
fmt.Printf("Could not find %s in namespace %s\n", servicemonitor.Name, namespace)
} else {
multiErr = multierror.Append(multiErr, err)
}
}
fmt.Printf("Successfully deleted %s\n", servicemonitor.Name)
}
}
}
return multiErr.ErrorOrNil()
}

func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind, instanceName string, applicationNS string) error {
// first check if CRD in cluster
crd := &apiextv1.CustomResourceDefinition{}
Expand All @@ -415,17 +467,12 @@ func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.G
}

func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error {
OdhDashboardConfig := schema.GroupVersionKind{
Group: "opendatahub.io",
Version: "v1alpha",
Kind: "OdhDashboardConfig",
}
crd := &apiextv1.CustomResourceDefinition{}
if err := cli.Get(context.TODO(), client.ObjectKey{Name: "odhdashboardconfigs.opendatahub.io"}, crd); err != nil {
return client.IgnoreNotFound(err)
}
odhObject := &unstructured.Unstructured{}
odhObject.SetGroupVersionKind(OdhDashboardConfig)
odhObject.SetGroupVersionKind(gvkOdhDashboardConfig)
if err := cli.Get(context.TODO(), client.ObjectKey{
Namespace: applicationNS,
Name: instanceName,
Expand Down

0 comments on commit a206bbe

Please sign in to comment.