Deprecate source_type config arg and add source_types config arg closes #166 (#167)

Co-authored-by: Graza <code@graza.io>

Author: cbruno10

.gitignore

@@ -15,3 +15,4 @@
 # vendor/
 
 .vscode/*
+.idea

README.md

@@ -54,13 +54,12 @@ connection "kubernetes" {
   # If no kubeconfig file can be found, the plugin will attempt to use the service account Kubernetes gives to pods.
   # This authentication method is intended for clients that expect to be running inside a pod running on Kubernetes.
 
-  # Specify the source of the resource. Possible values: `deployed`, `helm`, `manifest`, and `all`.
-  # Default set to `all`. Set the argument to override the default value.
-  # If the value is set to `deployed`, tables will show all the deployed resources.
-  # If set to `helm`, tables will only show resources from the configured helm charts.
-  # If set to `manifest`, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set.
-  # If `all`, tables will show all the deployed and manifest resources.
-  # source_type = "all"
+  # Specify the source(s) of the resource(s). Possible values: `deployed`, `helm` and `manifest`.
+  # Defaults to all possible values. Set the argument to override the default value.
+  # If `deployed` is contained in the value, tables will show all the deployed resources.
+  # If `helm` is contained in the value, tables will show resources from the configured helm charts.
+  # If `manifest` is contained in the value, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set.
+  # source_types = ["deployed", "helm", "manifest"]
 
   # Manifest File Configuration
 

config/kubernetes.spc

@@ -28,13 +28,12 @@ connection "kubernetes" {
   # If no kubeconfig file can be found, the plugin will attempt to use the service account Kubernetes gives to pods.
   # This authentication method is intended for clients that expect to be running inside a pod running on Kubernetes.
 
-  # Specify the source of the resource. Possible values: `deployed`, `helm`, `manifest`, and `all`.
-  # Default set to `all`. Set the argument to override the default value.
-  # If the value is set to `deployed`, tables will show all the deployed resources.
-  # If set to `helm`, tables will only show resources from the configured helm charts.
-  # If set to `manifest`, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set.
-  # If `all`, tables will show all the deployed and manifest resources.
-  # source_type = "all"
+  # Specify the source(s) of the resource(s). Possible values: `deployed`, `helm` and `manifest`.
+  # Defaults to all possible values. Set the argument to override the default value.
+  # If `deployed` is contained in the value, tables will show all the deployed resources.
+  # If `helm` is contained in the value, tables will show resources from the configured helm charts.
+  # If `manifest` is contained in the value, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set.
+  # source_types = ["deployed", "helm", "manifest"]
 
   # Manifest File Configuration
 

docs/index.md

@@ -43,6 +43,7 @@ connection "kubernetes" {
   plugin         = "kubernetes"
   config_path    = "~/.kube/config"
   config_context = "myCluster"
+  source_types   = ["deployed"]
 }
 ```
 
@@ -113,12 +114,14 @@ connection "kubernetes_cluster_aks" {
   plugin         = "kubernetes"
   config_path    = "~/.kube/config"
   config_context = "myAKSCluster"
+  source_types   = ["deployed"]
 }
 
 connection "kubernetes_cluster_eks" {
   plugin         = "kubernetes"
   config_path    = "~/.kube/config"
   config_context = "arn:aws:eks:us-east-1:123456789012:cluster/myEKSCluster"
+  source_types   = ["deployed"]
 }
 ```
 
@@ -174,20 +177,22 @@ connection "kubernetes" {
     "github.com/GoogleCloudPlatform/microservices-demo//release//kubernetes-manifests.yaml",
     "s3::https://bucket.s3.us-east-1.amazonaws.com/test_folder//*.yml"
   ]
+  
+  source_types = ["manifest"]
 }
 ```
 
 **Note**: If any path matches on `*` without `.yml` or `.yaml` or `.json`, all files (including non-Kubernetes manifest files) in the directory will be matched, which may cause errors if incompatible file types exist.
 
-By default the plugin always lists the resources deployed in the current Kubernetes cluster context. If you want to restrict this behavior to read resource configurations from the configured manifest files only, add the `source_type` argument to the config and set the value to `manifest`. For example:
+By default the plugin always lists the resources deployed in the current Kubernetes cluster context. If you want to restrict this behavior to read resource configurations from the configured manifest files only, add the `source_types` argument to the config and set the value to `manifest`. For example:
 
 ```hcl
 connection "kubernetes" {
   plugin = "kubernetes"
 
   manifest_file_paths = [ ... ]
 
-  source_type = "manifest"
+  source_types = ["manifest"]
 }
 ```
 
@@ -209,6 +214,8 @@ connection "kubernetes" {
   plugin = "kubernetes"
 
   manifest_file_paths = [ "*.yml", "*.yaml", "*.json", "/path/to/dir/main.yml" ]
+  
+  source_types = ["manifest"]
 }
 ```
 
@@ -230,6 +237,8 @@ connection "kubernetes" {
   plugin = "kubernetes"
 
   manifest_file_paths = [ "bitbucket.org/atlassian/kubectl-run//test/kustomization//deploy.yml" ]
+  
+  source_types = ["manifest"]
 }
 ```
 
@@ -255,6 +264,8 @@ connection "kubernetes" {
     "s3::https://bucket-2.s3.us-east-1.amazonaws.com//*.yml?aws_profile=<AWS_PROFILE>",
     "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//*.yaml?aws_profile=<AWS_PROFILE>"
   ]
+  
+  source_types = ["manifest"]
 }
 ```
 
@@ -299,6 +310,8 @@ connection "kubernetes" {
     "s3::https://bucket-1.s3.us-east-1.amazonaws.com/test_folder//*.yml",
     "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//**/*.yaml"
   ]
+  
+  source_types = ["manifest"]
 }
 ```
 
@@ -332,6 +345,8 @@ connection "kubernetes" {
       values_file_paths = [] # works with values from chart's default values.yaml file
     }
   }
+  
+  source_types = ["helm"]
 }
 ```
 

kubernetes/connection_config.go

@@ -11,6 +11,7 @@ type kubernetesConfig struct {
 	CustomResourceTables []string               `hcl:"custom_resource_tables,optional"`
 	ManifestFilePaths    []string               `hcl:"manifest_file_paths,optional" steampipe:"watch"`
 	SourceType           *string                `hcl:"source_type,optional"`
+	SourceTypes          []string               `hcl:"source_types,optional"`
 	HelmRenderedCharts   map[string]chartConfig `hcl:"helm_rendered_charts,optional"`
 }
 

kubernetes/helm_template_render.go

@@ -77,7 +77,7 @@ func getHelmRenderedTemplatesUncached(ctx context.Context, d *plugin.QueryData,
 	if err != nil {
 		return nil, err
 	}
-	helmConfig := GetConfig(d.Connection)
+	kubernetesConfig := GetConfig(d.Connection)
 
 	var renderedTemplates []HelmRenderedTemplate
 	for _, chart := range charts {
@@ -89,7 +89,7 @@ func getHelmRenderedTemplatesUncached(ctx context.Context, d *plugin.QueryData,
 		}
 
 		var processedHelmConfigs []string
-		for name, c := range helmConfig.HelmRenderedCharts {
+		for name, c := range kubernetesConfig.HelmRenderedCharts {
 			if c.ChartPath == chart.Path && !helpers.StringSliceContains(processedHelmConfigs, name) {
 
 				// Add the processed Helm render configs into processedHelmConfigs to avoid duplicate entries

kubernetes/helm_utils.go

@@ -44,23 +44,32 @@ var parsedHelmChartCached = plugin.HydrateFunc(parsedHelmChartUncached).Memoize(
 // getParsedHelmChart instead.
 func parsedHelmChartUncached(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (any, error) {
 	// Read the config
-	helmConfig := GetConfig(d.Connection)
+	kubernetesConfig := GetConfig(d.Connection)
 
-	// Return nil, if the source_type is set to other than "all" or "helm"
-	if helmConfig.SourceType != nil &&
-		!helpers.StringSliceContains([]string{"all", "helm"}, *helmConfig.SourceType) {
+	// Check for the sourceTypes argument in the config.
+	// Default set to include values.
+	var sources = All.ToSourceTypes()
+	if kubernetesConfig.SourceTypes != nil {
+		sources = kubernetesConfig.SourceTypes
+	}
+	// TODO: Remove once `SourceType` is obsolete
+	if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil {
+		if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default
+			sources = []string{*kubernetesConfig.SourceType}
+		}
+	}
+
+	if !helpers.StringSliceContains(sources, "helm") {
 		return nil, nil
 	}
 
 	var charts []*parsedHelmChart
 
-	for _, v := range helmConfig.HelmRenderedCharts {
-		// Return error if source_tpe arg is explicitly set to "helm" in the config, but
+	for _, v := range kubernetesConfig.HelmRenderedCharts {
+		// Return error if source_types arg includes "helm" in the config, but
 		// helm_chart_dir arg is not set.
-		if helmConfig.SourceType != nil &&
-			*helmConfig.SourceType == "helm" &&
-			v.ChartPath == "" {
-			return nil, errors.New("helm_chart_dir must be set in the config while the source_type is 'helm'")
+		if v.ChartPath == "" {
+			return nil, errors.New("helm_chart_dir must be set in the config while source_types includes 'helm'")
 		}
 
 		// Return empty parsedHelmChart object if no Helm chart directory path provided in the config
@@ -124,7 +133,7 @@ func getUniqueValueFilesFromConfig(ctx context.Context, d *plugin.QueryData) []s
 
 // getHelmClient creates the client for Helm
 func getHelmClient(ctx context.Context, namespace string) (helmClient.Client, error) {
-	// Return nil, if no namespace provided
+	// Return nil if no namespace provided
 	if namespace == "" {
 		return nil, nil
 	}

kubernetes/utils.go

@@ -58,6 +58,15 @@ func (sourceType SourceType) String() string {
 	return string(sourceType)
 }
 
+// ToSourceTypes is used to convert SourceType to []string
+func (sourceType SourceType) ToSourceTypes() []string {
+	if sourceType == All {
+		return []string{Deployed.String(), Helm.String(), Manifest.String()}
+	} else {
+		return []string{sourceType.String()}
+	}
+}
+
 // GetNewClientset :: gets client for querying k8s apis for the provided context
 func GetNewClientset(ctx context.Context, d *plugin.QueryData) (*kubernetes.Clientset, error) {
 	logger := plugin.Logger(ctx)
@@ -75,7 +84,7 @@ func GetNewClientset(ctx context.Context, d *plugin.QueryData) (*kubernetes.Clie
 		return nil, err
 	}
 
-	// Return nil, if the config is set only to list the manifest resources.
+	// Return nil if deployed resources should not be included
 	if kubeconfig == nil {
 		return nil, nil
 	}
@@ -142,7 +151,7 @@ func GetNewClientCRD(ctx context.Context, d *plugin.QueryData) (*apiextension.Cl
 		return nil, err
 	}
 
-	// Return nil, if the config is set to only list the manifest resources.
+	// Return nil if deployed resources should not be included
 	if kubeconfig == nil {
 		return nil, nil
 	}
@@ -196,7 +205,7 @@ func GetNewClientCRDRaw(ctx context.Context, cc *connection.ConnectionCache, c *
 		return nil, err
 	}
 
-	// Return nil, if the config is set to only list the manifest resources.
+	// Return nil if deployed resources should not be included
 	if kubeconfig == nil {
 		return nil, nil
 	}
@@ -336,24 +345,21 @@ func getK8Config(ctx context.Context, d *plugin.QueryData) (clientcmd.ClientConf
 	// get kubernetes config info
 	kubernetesConfig := GetConfig(d.Connection)
 
-	// Check for the sourceType argument in the config. Valid values are: "deployed", "manifest" and "all".
-	// Default set to "all".
-	var source SourceType = "all"
-	if kubernetesConfig.SourceType != nil {
-		source = SourceType(*kubernetesConfig.SourceType)
-		if err := source.IsValid(); err != nil {
-			plugin.Logger(ctx).Debug("getK8Config", "invalid_source_type_error", "connection", d.Connection.Name, "error", err)
-			return nil, err
+	// Check for the sourceTypes argument in the config
+	// Default set to include values
+	var sources = All.ToSourceTypes()
+	if kubernetesConfig.SourceTypes != nil {
+		sources = kubernetesConfig.SourceTypes
+	}
+	// TODO: Remove once `SourceType` is obsolete
+	if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil {
+		if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default
+			sources = []string{*kubernetesConfig.SourceType}
 		}
 	}
 
-	// By default source type is set to "all", which indicates querying the table will return both deployed, helm and manifest resources.
-	// If the source type is explicitly set, other plugin will list resources based on that. For example:
-	// If set to "manifest", the table will only return the manifest resources.
-	// If set to "helm", the table will return the resources after rendering the templates defined in the configured chart.
-	// Similarly, setting the value as "deployed" will return all the deployed resources.
-	if source.String() == "manifest" || source.String() == "helm" {
-		plugin.Logger(ctx).Debug("getK8Config", "Returning nil for API server client.", "Source type", source.String(), "connection", d.Connection.Name)
+	if !helpers.StringSliceContains(sources, "deployed") {
+		plugin.Logger(ctx).Debug("getK8Config", "Returning nil for API server client.", "source_types", sources, "connection", d.Connection.Name)
 		return nil, nil
 	}
 
@@ -427,24 +433,21 @@ func getK8ConfigRaw(ctx context.Context, cc *connection.ConnectionCache, c *plug
 	// get kubernetes config info
 	kubernetesConfig := GetConfig(c)
 
-	// Check for the sourceType argument in the config. Valid values are: "deployed", "manifest" and "all".
-	// Default set to "all".
-	var source SourceType = "all"
-	if kubernetesConfig.SourceType != nil {
-		source = SourceType(*kubernetesConfig.SourceType)
-		if err := source.IsValid(); err != nil {
-			plugin.Logger(ctx).Debug("getK8ConfigRaw", "invalid_source_type_error", "connection", c.Name, "error", err)
-			return nil, err
+	// Check for the sourceTypes argument in the config.
+	// Default set to include values.
+	var sources = All.ToSourceTypes()
+	if kubernetesConfig.SourceTypes != nil {
+		sources = kubernetesConfig.SourceTypes
+	}
+	// TODO: Remove once `SourceType` is obsolete
+	if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil {
+		if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default
+			sources = []string{*kubernetesConfig.SourceType}
 		}
 	}
 
-	// By default source type is set to "all", which indicates querying the table will return all the deployed, helm and manifest resources.
-	// If the source type is explicitly set, other plugin will list resources based on that. For example:
-	// If set to "manifest", the table will only return the manifest resources.
-	// If set to "helm", the table will return the resources after rendering the templates defined in the configured chart.
-	// Similarly, setting the value as "deployed" will return all the deployed resources.
-	if source.String() == "manifest" || source.String() == "helm" {
-		plugin.Logger(ctx).Debug("getK8ConfigRaw", "Returning nil for API server client.", "Source type", source.String(), "connection", c.Name)
+	if !helpers.StringSliceContains(sources, "deployed") {
+		plugin.Logger(ctx).Debug("getK8ConfigRaw", "Returning nil for API server client.", "source_types", sources, "connection", c.Name)
 		return nil, nil
 	}
 
@@ -794,25 +797,34 @@ func parsedManifestFileContentUncached(ctx context.Context, d *plugin.QueryData,
 // Returns the list of file paths/glob patterns after resolving all the given manifest file paths.
 func resolveManifestFilePaths(ctx context.Context, d *plugin.QueryData) ([]string, error) {
 	// Read the config
-	k8sConfig := GetConfig(d.Connection)
+	kubernetesConfig := GetConfig(d.Connection)
+
+	// Check for the sourceTypes argument in the config. Valid values are: "deployed", "manifest" and "helm".
+	// Default set to include values.
+	var sources = All.ToSourceTypes()
+	if kubernetesConfig.SourceTypes != nil {
+		sources = kubernetesConfig.SourceTypes
+	}
+	// TODO: Remove once `SourceType` is obsolete
+	if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil {
+		if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default
+			sources = []string{*kubernetesConfig.SourceType}
+		}
+	}
 
-	// Return nil, if the source_type is set other than "manifest", or "all"
-	if k8sConfig.SourceType != nil &&
-		!helpers.StringSliceContains([]string{"all", "manifest"}, *k8sConfig.SourceType) {
+	if !helpers.StringSliceContains(sources, "manifest") {
 		return nil, nil
 	}
 
-	// Return error if source_tpe arg is explicitly set to "manifest" in the config, but
+	// Return error if source_types arg includes "manifest" in the config, but
 	// manifest_file_paths arg is not set.
-	if k8sConfig.SourceType != nil &&
-		*k8sConfig.SourceType == "manifest" &&
-		k8sConfig.ManifestFilePaths == nil {
-		return nil, errors.New("manifest_file_paths must be set in the config while the source_type is 'manifest'")
+	if kubernetesConfig.ManifestFilePaths == nil {
+		return nil, errors.New("manifest_file_paths must be set in the config while the source_types includes 'manifest'")
 	}
 
 	// Gather file path matches for the glob
 	var matches, resolvedPaths []string
-	paths := k8sConfig.ManifestFilePaths
+	paths := kubernetesConfig.ManifestFilePaths
 	for _, i := range paths {
 
 		// List the files in the given source directory