partial revert of temporary grpc-only EnvoyFilter

This partially reverts commit 48e0b8b
as well as including a mechanism to deleted the then-introduced feature
by removing the corresponding FeatureTracker.

Related: RHOAIENG-165, #888

components/kserve/servicemesh_setup.go

@@ -44,18 +44,6 @@ func (k *Kserve) defineServiceMeshFeatures() feature.FeaturesProvider {
 			return kserveExtAuthzErr
 		}
 
-		temporaryFixesErr := feature.CreateFeature("kserve-temporary-fixes").
-			For(handler).
-			Manifests(
-				path.Join(feature.KServeDir, "grpc-envoyfilter-temp-fix.tmpl"),
-			).
-			WithData(servicemesh.ClusterDetails).
-			Load()
-
-		if temporaryFixesErr != nil {
-			return temporaryFixesErr
-		}
-
 		return nil
 	}
 }

main.go

@@ -237,6 +237,17 @@ func main() { //nolint:funlen
 		setupLog.Error(err, "error remove deprecated resources from previous version")
 	}
 
+	var cleanDeprecatedFeatures manager.RunnableFunc = func(ctx context.Context) error {
+		if err = upgrade.CleanupDeprecatedFeatures(ctx, setupClient, platform); err != nil {
+			setupLog.Error(err, "unable to clean deprecated features")
+		}
+		return err
+	}
+	err = mgr.Add(cleanDeprecatedFeatures)
+	if err != nil {
+		setupLog.Error(err, "unable to cleanup deprecated features")
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

pkg/upgrade/upgrade.go

@@ -351,6 +351,46 @@ func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform, appNS
 	return nil
 }
 
+// CleanupDeprecatedFeatures deletes deprecated features by removing the corresponding FeatureTrackers
+func CleanupDeprecatedFeatures(ctx context.Context, cli client.Client, platform deploy.Platform) error {
+	if platform != deploy.ManagedRhods {
+		return nil
+	}
+
+	crd := &apiextv1.CustomResourceDefinition{}
+	if err := cli.Get(ctx, client.ObjectKey{Name: "featuretrackers.features.opendatahub.io"}, crd); err != nil {
+		return err
+	}
+
+	type ResourceToDelete struct {
+		InstanceName     string
+		GroupVersionKind schema.GroupVersionKind
+	}
+
+	toDelete := []string{
+		"opendatahub-kserve-temporary-fixes", // Workaround for RHOAIENG-165
+	}
+
+	var multiErr *multierror.Error
+	for _, name := range toDelete {
+		obj := &unstructured.Unstructured{}
+		obj.SetGroupVersionKind(schema.GroupVersionKind{Kind: "FeatureTracker", Group: "features.opendatahub.io", Version: "v1"})
+
+		if err := cli.Get(ctx, client.ObjectKey{Name: name}, obj); err != nil {
+			multiErr = multierror.Append(multiErr, client.IgnoreNotFound(err))
+			continue
+		}
+
+		if err := cli.Delete(ctx, obj); err != nil {
+			msg := fmt.Sprintf("error deleting FeatureTracker instance %s: %s", name, err)
+			fmt.Printf(msg)
+			multiErr = multierror.Append(multiErr, fmt.Errorf(msg))
+		}
+	}
+
+	return multiErr.ErrorOrNil()
+}
+
 // TODO: remove function once we have a generic solution across all components.
 func CleanupExistingResource(ctx context.Context, cli client.Client, platform deploy.Platform, dscApplicationsNamespace, dscMonitoringNamespace string) error {
 	var multiErr *multierror.Error