pkg/upgrade: remove watson-studio dashboard application (#959)

* pkg/upgrade: abstract resource deletion with Unstructured

For the coming requirement to remove resources not just by their
names but a field inside of spec, abstract the existing
deleteDeprecatedResources/deleteDeprecatedServiceMonitors to take a
resource to delete description as

```
type ResourceSpec struct {
	Gvk       schema.GroupVersionKind
	Namespace string
	Path      []string
	Values    []string
}
```

TODO: This patch just adds new functionality without changing the
existing calls of deleteDeprecatedResources and
deleteDeprecatedServiceMonitors yet.

The function fetches UnstructuredList by gvk/namespace and uses
public unstructured.NestedString() to access the field by path.
It should work for ServiceMonitor, where ServiceMonitorList contains
list of the pointers, as well.

Ignore NoKindMatchError. CRD may not exist on fresh installation for
example.

Make a wrapper which takes an array of them for convenience and
avoid one indentation level and simplifies multierror wrapping a
bit.

It requires to have a library of GroupVersionKinds which is a future
work to make common for the whole project.

Signed-off-by: Yauheni Kaliuta <[email protected]>

* pkg/gvk: add package to store GroupVersionKind objects

schema.GroupVersionKind objects are used around the code, introduce
a common global place for it.

Add Odh GVKs to be used in the next patch.

Signed-off-by: Yauheni Kaliuta <[email protected]>

* pkg/upgrade: remove watson-studio dashboard application

Jira: https://issues.redhat.com/browse/RHOAIENG-4764

Remove deprecated dashboard resources.
Reuse existing infra just make the array of resources in a separate
function to avoid growing of CleanupExistingResource().

Signed-off-by: Yauheni Kaliuta <[email protected]>

---------

Signed-off-by: Yauheni Kaliuta <[email protected]>

pkg/gvk/gvk.go

@@ -0,0 +1,9 @@
+package gvk
+
+import "k8s.io/apimachinery/pkg/runtime/schema"
+
+var (
+	OdhApplication = schema.GroupVersionKind{Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhApplication"}
+	OdhDocument    = schema.GroupVersionKind{Group: "dashboard.opendatahub.io", Version: "v1", Kind: "OdhDocument"}
+	OdhQuickStart  = schema.GroupVersionKind{Group: "console.openshift.io", Version: "v1", Kind: "OdhQuickStart"}
+)

pkg/upgrade/upgrade.go

@@ -2,6 +2,7 @@ package upgrade
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
 	"strings"
@@ -16,6 +17,7 @@ import (
 	authv1 "k8s.io/api/rbac/v1"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -40,9 +42,19 @@ import (
 	"github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvk"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/labels"
 )
 
+type ResourceSpec struct {
+	Gvk       schema.GroupVersionKind
+	Namespace string
+	// path to the field, like "metadata", "name"
+	Path []string
+	// set of values for the field to match object, any one matches
+	Values []string
+}
+
 // CreateDefaultDSC creates a default instance of DSC.
 // Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource.
 func CreateDefaultDSC(ctx context.Context, cli client.Client) error {
@@ -254,6 +266,33 @@ func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform, appNS
 	return nil
 }
 
+func getDashboardWatsonResources(ns string) []ResourceSpec {
+	metadataName := []string{"metadata", "name"}
+	specAppName := []string{"spec", "appName"}
+	appName := []string{"watson-studio"}
+
+	return []ResourceSpec{
+		{
+			Gvk:       gvk.OdhQuickStart,
+			Namespace: ns,
+			Path:      specAppName,
+			Values:    appName,
+		},
+		{
+			Gvk:       gvk.OdhDocument,
+			Namespace: ns,
+			Path:      specAppName,
+			Values:    appName,
+		},
+		{
+			Gvk:       gvk.OdhApplication,
+			Namespace: ns,
+			Path:      metadataName,
+			Values:    appName,
+		},
+	}
+}
+
 // TODO: remove function once we have a generic solution across all components.
 func CleanupExistingResource(ctx context.Context, cli client.Client, platform deploy.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error {
 	var multiErr *multierror.Error
@@ -297,9 +336,63 @@ func CleanupExistingResource(ctx context.Context, cli client.Client, platform de
 		Kind:    "OdhApplication",
 	}
 	multiErr = multierror.Append(multiErr, removOdhApplicationsCR(ctx, cli, JupyterhubApp, "jupyterhub", dscApplicationsNamespace))
+
+	// to take a reference
+	toDelete := getDashboardWatsonResources(dscApplicationsNamespace)
+	multiErr = multierror.Append(multiErr, deleteResources(ctx, cli, &toDelete))
+
 	return multiErr.ErrorOrNil()
 }
 
+func deleteResources(ctx context.Context, c client.Client, resources *[]ResourceSpec) error {
+	var errors *multierror.Error
+
+	for _, res := range *resources {
+		err := deleteOneResource(ctx, c, res)
+		errors = multierror.Append(errors, err)
+	}
+
+	return errors.ErrorOrNil()
+}
+
+func deleteOneResource(ctx context.Context, c client.Client, res ResourceSpec) error {
+	list := &unstructured.UnstructuredList{}
+	list.SetGroupVersionKind(res.Gvk)
+
+	err := c.List(ctx, list, client.InNamespace(res.Namespace))
+	if err != nil {
+		if errors.Is(err, &meta.NoKindMatchError{}) {
+			fmt.Printf("Could not delete %v: CRD not found\n", res.Gvk)
+			return nil
+		}
+		return fmt.Errorf("failed to list %s: %w", res.Gvk.Kind, err)
+	}
+
+	for _, item := range list.Items {
+		item := item
+		v, ok, err := unstructured.NestedString(item.Object, res.Path...)
+		if err != nil {
+			return fmt.Errorf("failed to get field %v for %s %s/%s: %w", res.Path, res.Gvk.Kind, res.Namespace, item.GetName(), err)
+		}
+
+		if !ok {
+			return fmt.Errorf("unexisting path to delete: %v", res.Path)
+		}
+
+		for _, toDelete := range res.Values {
+			if v == toDelete {
+				err = c.Delete(ctx, &item)
+				if err != nil {
+					return fmt.Errorf("failed to delete %s %s/%s: %w", res.Gvk.Kind, res.Namespace, item.GetName(), err)
+				}
+				fmt.Println("Deleted object", item.GetName(), res.Gvk, "in namespace", res.Namespace)
+			}
+		}
+	}
+
+	return nil
+}
+
 func RemoveKfDefInstances(ctx context.Context, cli client.Client) error {
 	// Check if kfdef are deployed
 	kfdefCrd := &apiextv1.CustomResourceDefinition{}