Skip to content

Commit

Permalink
pkg/upgrade: abstract resource deletion with Unstructured
Browse files Browse the repository at this point in the history 
For the coming requirement to remove resources not just by their
names but a field inside of spec, abstract the existing
deleteDeprecatedResources/deleteDeprecatedServiceMonitors to take a
resource to delete description as

```
type JobSpec struct {
	Gvk       schema.GroupVersionKind
	Namespace string
	Path      []string
	Values    []string
}
```

and convert existing calls to used it.

The function fetches UnstructuredList by gvk/namespace and uses
public unstructured.NestedString() to access the field by path.
It should work for ServiceMonitor, where ServiceMonitorList contains
list of the pointers, as well.

Make a wrapper which takes an array of them for convenience and
avoid one indentation level and simplifies multierror wrapping a
bit.

It requires to have a library of GroupVersionKinds which is a future
work to make common for the whole project.

Move OdhApplication and OdhDashboardConfig to the gvk list, it's
proper place now.

Signed-off-by: Yauheni Kaliuta <[email protected]>
  • Loading branch information
@ykaliuta
ykaliuta committed Apr 8, 2024
1 parent 562ae3d commit 0f73430
Showing 1 changed file with 135 additions and 94 deletions.
229 changes: 135 additions & 94 deletions pkg/upgrade/upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,13 @@ package upgrade
import (
"context"
"fmt"
"reflect"
"strings"
"time"

"github.com/hashicorp/go-multierror"
operatorv1 "github.com/openshift/api/operator/v1"
routev1 "github.com/openshift/api/route/v1"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
authv1 "k8s.io/api/rbac/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -43,6 +39,30 @@ import (
"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
)

type JobSpec struct {
Gvk schema.GroupVersionKind
Namespace string
// path to the field, like "metadata", "name"
Path []string
// set of values for the field to match object, any one matches
Values []string
}

// TODO: move to common place.
var (
gvkDeployment = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}
gvkStatefulSet = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}
gvkService = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"}
gvkRoute = schema.GroupVersionKind{Group: "route.openshift.io", Version: "v1", Kind: "Route"}
gvkSecret = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}
gvkClusterRole = schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}
gvkClusterRoleBinding = schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}
gvkServiceAccount = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ServiceAccount"}
gvkServiceMonitor = schema.GroupVersionKind{Group: "monitoring.coreos.com", Version: "v1", Kind: "ServiceMonitor"}
gvkOdhApplication = schema.GroupVersionKind{Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication"}
gvkOdhDashboardConfig = schema.GroupVersionKind{Group: "opendatahub.io", Version: "v1alpha", Kind: "OdhDashboardConfig"}
)

// CreateDefaultDSC creates a default instance of DSC.
// Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource.
func CreateDefaultDSC(ctx context.Context, cli client.Client) error {
Expand Down Expand Up @@ -257,47 +277,129 @@ func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform, appNS
// TODO: remove function once we have a generic solution across all components.
func CleanupExistingResource(ctx context.Context, cli client.Client, platform deploy.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error {
var multiErr *multierror.Error
var toDelete []JobSpec
pathName := []string{"metadata", "name"}
// Special Handling of cleanup of deprecated model monitoring stack
if platform == deploy.ManagedRhods {
deprecatedDeployments := []string{"rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedDeployments, &appsv1.DeploymentList{}))
del := []JobSpec{
{
Gvk: gvkDeployment,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-prometheus-operator"},
},
{
Gvk: gvkStatefulSet,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"prometheus-rhods-model-monitoring"},
},
{
Gvk: gvkService,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-model-monitoring"},
},
{
Gvk: gvkRoute,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-model-monitoring"},
},
{
Gvk: gvkSecret,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-monitoring-oauth-config"},
},
{
Gvk: gvkClusterRole,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"},
},
{
Gvk: gvkClusterRoleBinding,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-namespace-read", "rhods-prometheus-operator"},
},
{
Gvk: gvkServiceAccount,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-prometheus-operator"},
},
{
Gvk: gvkServiceMonitor,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"modelmesh-federated-metrics"},
},
}
toDelete = append(toDelete, del...)
}

deprecatedStatefulsets := []string{"prometheus-rhods-model-monitoring"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedStatefulsets, &appsv1.StatefulSetList{}))
// common logic for both self-managed and managed
del := JobSpec{
Gvk: gvkServiceMonitor,
Namespace: dscMonitoringNamespace,
Path: pathName,
Values: []string{"rhods-monitor-federation2"},
}

deprecatedServices := []string{"rhods-model-monitoring"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServices, &corev1.ServiceList{}))
toDelete = append(toDelete, del)

deprecatedRoutes := []string{"rhods-model-monitoring"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedRoutes, &routev1.RouteList{}))
multiErr = multierror.Append(multiErr, deleteResources(ctx, cli, &toDelete))

deprecatedSecrets := []string{"rhods-monitoring-oauth-config"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedSecrets, &corev1.SecretList{}))
// Handling for dashboard Jupyterhub CR, see jira #443
multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, gvkOdhApplication, "jupyterhub", dscApplicationsNamespace))
return multiErr.ErrorOrNil()
}

func deleteResources(ctx context.Context, c client.Client, jobs *[]JobSpec) error {
var errors *multierror.Error

deprecatedClusterroles := []string{"rhods-namespace-read", "rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterroles, &authv1.ClusterRoleList{}))
for _, job := range *jobs {
err := deleteOneJob(ctx, c, job)
errors = multierror.Append(errors, err)
}

deprecatedClusterrolebindings := []string{"rhods-namespace-read", "rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedClusterrolebindings, &authv1.ClusterRoleBindingList{}))
return errors.ErrorOrNil()
}

deprecatedServiceAccounts := []string{"rhods-prometheus-operator"}
multiErr = multierror.Append(multiErr, deleteDeprecatedResources(ctx, cli, dscMonitoringNamespace, deprecatedServiceAccounts, &corev1.ServiceAccountList{}))
func deleteOneJob(ctx context.Context, c client.Client, job JobSpec) error {
list := &unstructured.UnstructuredList{}
list.SetGroupVersionKind(job.Gvk)

deprecatedServicemonitors := []string{"modelmesh-federated-metrics"}
multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedServicemonitors))
err := c.List(ctx, list, client.InNamespace(job.Namespace))
if err != nil {
return fmt.Errorf("failed to list %s: %w", job.Gvk.Kind, err)
}
// common logic for both self-managed and managed
deprecatedOperatorSM := []string{"rhods-monitor-federation2"}
multiErr = multierror.Append(multiErr, deleteDeprecatedServiceMonitors(ctx, cli, dscMonitoringNamespace, deprecatedOperatorSM))

// Handling for dashboard Jupyterhub CR, see jira #443
JupyterhubApp := schema.GroupVersionKind{
Group: "dashboard.opendatahub.io",
Version: "v1",
Kind: "OdhApplication",
for _, item := range list.Items {
item := item
v, ok, err := unstructured.NestedString(item.Object, job.Path...)
if err != nil {
return fmt.Errorf("failed to get field %v for %s %s/%s: %w", job.Path, job.Gvk.Kind, job.Namespace, item.GetName(), err)
}

if !ok {
return fmt.Errorf("unexisting path to delete: %v", job.Path)
}

for _, toDelete := range job.Values {
if v == toDelete {
err = c.Delete(ctx, &item)
if err != nil {
return fmt.Errorf("failed to delete %s %s/%s: %w", job.Gvk.Kind, job.Namespace, item.GetName(), err)
}
fmt.Println("Deleted object", item.GetName(), job.Gvk, "in namespace", job.Namespace)
}
}
}
multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, JupyterhubApp, "jupyterhub", dscApplicationsNamespace))
return multiErr.ErrorOrNil()

return nil
}

func RemoveKfDefInstances(ctx context.Context, cli client.Client) error {
Expand Down Expand Up @@ -335,62 +437,6 @@ func RemoveKfDefInstances(ctx context.Context, cli client.Client) error {
return nil
}

func deleteDeprecatedResources(ctx context.Context, cli client.Client, namespace string, resourceList []string, resourceType client.ObjectList) error {
var multiErr *multierror.Error
listOpts := &client.ListOptions{Namespace: namespace}
if err := cli.List(ctx, resourceType, listOpts); err != nil {
multiErr = multierror.Append(multiErr, err)
}
items := reflect.ValueOf(resourceType).Elem().FieldByName("Items")
for i := 0; i < items.Len(); i++ {
item := items.Index(i).Addr().Interface().(client.Object) //nolint:errcheck,forcetypeassert
for _, name := range resourceList {
if name == item.GetName() {
fmt.Printf("Attempting to delete %s in namespace %s\n", item.GetName(), namespace)
err := cli.Delete(ctx, item)
if err != nil {
if apierrs.IsNotFound(err) {
fmt.Printf("Could not find %s in namespace %s\n", item.GetName(), namespace)
} else {
multiErr = multierror.Append(multiErr, err)
}
}
fmt.Printf("Successfully deleted %s\n", item.GetName())
}
}
}
return multiErr.ErrorOrNil()
}

// Need to handle ServiceMonitor deletion separately as the generic function does not work for ServiceMonitors because of how the package is built.
func deleteDeprecatedServiceMonitors(ctx context.Context, cli client.Client, namespace string, resourceList []string) error {
var multiErr *multierror.Error
listOpts := &client.ListOptions{Namespace: namespace}
servicemonitors := &monitoringv1.ServiceMonitorList{}
if err := cli.List(ctx, servicemonitors, listOpts); err != nil {
multiErr = multierror.Append(multiErr, err)
}

for _, servicemonitor := range servicemonitors.Items {
servicemonitor := servicemonitor
for _, name := range resourceList {
if name == servicemonitor.Name {
fmt.Printf("Attempting to delete %s in namespace %s\n", servicemonitor.Name, namespace)
err := cli.Delete(ctx, servicemonitor)
if err != nil {
if apierrs.IsNotFound(err) {
fmt.Printf("Could not find %s in namespace %s\n", servicemonitor.Name, namespace)
} else {
multiErr = multierror.Append(multiErr, err)
}
}
fmt.Printf("Successfully deleted %s\n", servicemonitor.Name)
}
}
}
return multiErr.ErrorOrNil()
}

func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.GroupVersionKind, instanceName string, applicationNS string) error {
// first check if CRD in cluster
crd := &apiextv1.CustomResourceDefinition{}
Expand All @@ -415,17 +461,12 @@ func removOdhApplicationsCR(ctx context.Context, cli client.Client, gvk schema.G
}

func unsetOwnerReference(cli client.Client, instanceName string, applicationNS string) error {
OdhDashboardConfig := schema.GroupVersionKind{
Group: "opendatahub.io",
Version: "v1alpha",
Kind: "OdhDashboardConfig",
}
crd := &apiextv1.CustomResourceDefinition{}
if err := cli.Get(context.TODO(), client.ObjectKey{Name: "odhdashboardconfigs.opendatahub.io"}, crd); err != nil {
return client.IgnoreNotFound(err)
}
odhObject := &unstructured.Unstructured{}
odhObject.SetGroupVersionKind(OdhDashboardConfig)
odhObject.SetGroupVersionKind(gvkOdhDashboardConfig)
if err := cli.Get(context.TODO(), client.ObjectKey{
Namespace: applicationNS,
Name: instanceName,
Expand Down

0 comments on commit 0f73430

Please sign in to comment.