Merge pull request #785 from wzshiming/feat/flag-enable-crds

[kwokctl] Add --enable-crds

kustomize/kwok/deployment.yaml

@@ -21,15 +21,16 @@ spec:
             - --node-port=10247
             - --cidr=10.0.0.1/24
             - --node-lease-duration-seconds=40
-            - --enable-crd=Stage
-            - --enable-crd=Attach
-            - --enable-crd=ClusterAttach
-            - --enable-crd=Exec
-            - --enable-crd=ClusterExec
-            - --enable-crd=Logs
-            - --enable-crd=ClusterLogs
-            - --enable-crd=PortForward
-            - --enable-crd=ClusterPortForward
+            - --enable-crds=Stage
+            - --enable-crds=Metric
+            - --enable-crds=Attach
+            - --enable-crds=ClusterAttach
+            - --enable-crds=Exec
+            - --enable-crds=ClusterExec
+            - --enable-crds=Logs
+            - --enable-crds=ClusterLogs
+            - --enable-crds=PortForward
+            - --enable-crds=ClusterPortForward
           env:
             - name: POD_IP
               valueFrom:

pkg/apis/config/v1alpha1/kwokctl_configuration_types.go

@@ -76,6 +76,10 @@ type ComponentPatches struct {
 
 // KwokctlConfigurationOptions holds information about the options.
 type KwokctlConfigurationOptions struct {
+	// EnableCRDs is a list of CRDs to enable.
+	// Once listed in this field, it will no longer be supported by the --config flag.
+	EnableCRDs []string `json:"enableCRDs,omitempty"`
+
 	// KubeApiserverPort is the port to expose apiserver.
 	// is the default value for flag --kube-apiserver-port and env KWOK_KUBE_APISERVER_PORT
 	KubeApiserverPort uint32 `json:"kubeApiserverPort,omitempty"`

pkg/apis/internalversion/kwok_configuration_types.go

@@ -32,7 +32,6 @@ type KwokConfiguration struct {
 // KwokConfigurationOptions holds information about the options.
 type KwokConfigurationOptions struct {
 	// EnableCRDs is a list of CRDs to enable.
-	// Once listed in this field, it will no longer be supported by the --config flag.
 	EnableCRDs []string
 
 	// The default IP assigned to the Pod on maintained Nodes.

pkg/apis/internalversion/kwokctl_configuration_types.go

@@ -63,6 +63,9 @@ type ComponentPatches struct {
 
 // KwokctlConfigurationOptions holds information about the options.
 type KwokctlConfigurationOptions struct {
+	// EnableCRDs is a list of CRDs to enable.
+	EnableCRDs []string
+
 	// KubeApiserverPort is the port to expose apiserver.
 	KubeApiserverPort uint32
 

pkg/kwok/cmd/root.go

@@ -90,7 +90,7 @@ func NewCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().StringVar(&flags.Master, "master", flags.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	cmd.Flags().StringVar(&flags.Options.ServerAddress, "server-address", flags.Options.ServerAddress, "Address to expose the server on")
 	cmd.Flags().UintVar(&flags.Options.NodeLeaseDurationSeconds, "node-lease-duration-seconds", flags.Options.NodeLeaseDurationSeconds, "Duration of node lease seconds")
-	cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crd", flags.Options.EnableCRDs, "List of CRDs to enable")
+	cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable")
 
 	cmd.Flags().BoolVar(&flags.Options.EnableCNI, "experimental-enable-cni", flags.Options.EnableCNI, "Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux")
 	if config.GOOS != "linux" {

pkg/kwokctl/cmd/create/cluster/cluster.go

@@ -133,6 +133,8 @@ func NewCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().DurationVar(&flags.Wait, "wait", 0, "Wait for the cluster to be ready")
 	cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context")
 	cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components")
+	cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable")
+
 	return cmd
 }
 

pkg/kwokctl/components/kwok_controller.go

@@ -17,6 +17,8 @@ limitations under the License.
 package components
 
 import (
+	"strings"
+
 	"sigs.k8s.io/kwok/pkg/apis/internalversion"
 	"sigs.k8s.io/kwok/pkg/consts"
 	"sigs.k8s.io/kwok/pkg/log"
@@ -40,6 +42,7 @@ type BuildKwokControllerComponentConfig struct {
 	NodeName                 string
 	Verbosity                log.Level
 	NodeLeaseDurationSeconds uint
+	EnableCRDs               []string
 	ExtraArgs                []internalversion.ExtraArgs
 	ExtraVolumes             []internalversion.Volume
 	ExtraEnvs                []internalversion.Env
@@ -121,6 +124,10 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp
 		kwokControllerArgs = append(kwokControllerArgs, "--v="+format.String(conf.Verbosity))
 	}
 
+	if len(conf.EnableCRDs) != 0 {
+		kwokControllerArgs = append(kwokControllerArgs, "--enable-crds="+strings.Join(conf.EnableCRDs, ","))
+	}
+
 	envs := []internalversion.Env{}
 	envs = append(envs, conf.ExtraEnvs...)
 

pkg/kwokctl/runtime/binary/cluster.go

@@ -592,6 +592,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) {
 		NodeName:                 "localhost",
 		Verbosity:                env.verbosity,
 		NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds,
+		EnableCRDs:               conf.EnableCRDs,
 		ExtraArgs:                kwokControllerComponentPatches.ExtraArgs,
 		ExtraEnvs:                kwokControllerComponentPatches.ExtraEnvs,
 	})

pkg/kwokctl/runtime/cluster.go

@@ -173,33 +173,97 @@ func (c *Cluster) Save(ctx context.Context) error {
 		return nil
 	}
 
+	var objs []config.InternalObject
 	conf := c.conf.DeepCopy()
-	objs := []config.InternalObject{
-		conf,
-	}
-
 	if conf.Status.Version == "" {
 		conf.Status.Version = consts.Version
 	}
-
-	others := config.FilterWithoutTypeFromContext[*internalversion.KwokctlConfiguration](ctx)
-	objs = append(objs, others...)
+	objs = append(objs, conf)
 
 	kwokConfigs := config.FilterWithTypeFromContext[*internalversion.KwokConfiguration](ctx)
-	if (len(kwokConfigs) == 0 || !slices.Contains(kwokConfigs[0].Options.EnableCRDs, v1alpha1.StageKind)) &&
+	objs = appendIntoInternalObjects(objs, kwokConfigs...)
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.StageKind) &&
 		conf.Options.Runtime != consts.RuntimeTypeKind &&
 		conf.Options.Runtime != consts.RuntimeTypeKindPodman &&
 		len(config.FilterWithTypeFromContext[*internalversion.Stage](ctx)) == 0 {
 		defaultStages, err := c.getDefaultStages(conf.Options.NodeStatusUpdateFrequencyMilliseconds, conf.Options.NodeLeaseDurationSeconds != 0)
 		if err != nil {
 			return err
 		}
-		objs = append(objs, defaultStages...)
+		objs = appendIntoInternalObjects(objs, defaultStages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.StageKind) {
+		if conf.Options.Runtime != consts.RuntimeTypeKind &&
+			conf.Options.Runtime != consts.RuntimeTypeKindPodman &&
+			len(config.FilterWithTypeFromContext[*internalversion.Stage](ctx)) == 0 {
+			defaultStages, err := c.getDefaultStages(conf.Options.NodeStatusUpdateFrequencyMilliseconds, conf.Options.NodeLeaseDurationSeconds != 0)
+			if err != nil {
+				return err
+			}
+			objs = appendIntoInternalObjects(objs, defaultStages...)
+		} else {
+			stages := config.FilterWithTypeFromContext[*internalversion.Stage](ctx)
+			objs = appendIntoInternalObjects(objs, stages...)
+		}
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.MetricKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.Metric](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.AttachKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.Attach](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterAttachKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.ClusterAttach](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ExecKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.Exec](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterExecKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.ClusterExec](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.LogsKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.Logs](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterLogsKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.ClusterLogs](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.PortForwardKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.PortForward](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
+	}
+
+	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterPortForwardKind) {
+		stages := config.FilterWithTypeFromContext[*internalversion.ClusterPortForward](ctx)
+		objs = appendIntoInternalObjects(objs, stages...)
 	}
 
 	return config.Save(ctx, c.GetWorkdirPath(ConfigName), objs)
 }
 
+func appendIntoInternalObjects[T config.InternalObject](objs []config.InternalObject, a ...T) []config.InternalObject {
+	for _, o := range a {
+		objs = append(objs, o)
+	}
+	return objs
+}
+
 func (c *Cluster) getDefaultStages(updateFrequency int64, lease bool) ([]config.InternalObject, error) {
 	objs := []config.InternalObject{}
 
@@ -468,11 +532,13 @@ func (c *Cluster) IsDryRun() bool {
 
 // InitCRDs initializes the CRDs.
 func (c *Cluster) InitCRDs(ctx context.Context) error {
-	kwokConfigs := config.FilterWithTypeFromContext[*internalversion.KwokConfiguration](ctx)
-	if len(kwokConfigs) == 0 {
-		return nil
+	config, err := c.Config(ctx)
+	if err != nil {
+		return err
 	}
-	crds := kwokConfigs[0].Options.EnableCRDs
+	conf := &config.Options
+
+	crds := conf.EnableCRDs
 	if len(crds) == 0 {
 		return nil
 	}

pkg/kwokctl/runtime/compose/cluster.go

@@ -610,6 +610,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) {
 		NodeName:                 c.Name() + "-kwok-controller",
 		Verbosity:                env.verbosity,
 		NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds,
+		EnableCRDs:               conf.EnableCRDs,
 		ExtraArgs:                kwokControllerComponentPatches.ExtraArgs,
 		ExtraVolumes:             kwokControllerExtraVolumes,
 		ExtraEnvs:                kwokControllerComponentPatches.ExtraEnvs,

pkg/kwokctl/runtime/kind/cluster.go

@@ -272,6 +272,7 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) {
 		Name:                     c.Name(),
 		Verbosity:                env.verbosity,
 		NodeLeaseDurationSeconds: 40,
+		EnableCRDs:               conf.EnableCRDs,
 		ExtraArgs:                kwokControllerComponentPatches.ExtraArgs,
 		ExtraVolumes:             kwokControllerExtraVolumes,
 		ExtraEnvs:                kwokControllerComponentPatches.ExtraEnvs,

pkg/kwokctl/runtime/kind/kwok_controller_pod.go

@@ -58,4 +58,5 @@ type BuildKwokControllerPodConfig struct {
 	ExtraArgs                []internalversion.ExtraArgs
 	ExtraVolumes             []internalversion.Volume
 	ExtraEnvs                []internalversion.Env
+	EnableCRDs               []string
 }

pkg/kwokctl/runtime/kind/kwok_controller_pod.yaml.tpl

@@ -22,6 +22,9 @@ spec:
     - --node-name=kwok-controller.kube-system.svc
     - --node-port=10247
     - --node-lease-duration-seconds={{ .NodeLeaseDurationSeconds }}
+    {{ range .EnableCRDs }}
+    - --enable-crds={{ . }}
+    {{ end }}
     {{ range .ExtraArgs }}
     - --{{ .Key }}={{ .Value }}
     {{ end }}

site/content/en/docs/generated/apis.md

@@ -2067,6 +2067,18 @@ KwokctlConfigurationOptions
 <tbody>
 <tr>
 <td>
+<code>enableCRDs</code>
+<em>
+[]string
+</em>
+</td>
+<td>
+<p>EnableCRDs is a list of CRDs to enable.
+Once listed in this field, it will no longer be supported by the &ndash;config flag.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>kubeApiserverPort</code>
 <em>
 uint32

site/content/en/docs/generated/kwok.md

@@ -13,7 +13,7 @@ kwok [flags]
   -c, --config strings                                     config path (default [~/.kwok/kwok.yaml])
       --disregard-status-with-annotation-selector string   All node/pod status excluding the ones that match the annotation selector will be watched and managed.
       --disregard-status-with-label-selector string        All node/pod status excluding the ones that match the label selector will be watched and managed.
-      --enable-crd strings                                 List of CRDs to enable
+      --enable-crds strings                                List of CRDs to enable
       --experimental-enable-cni                            Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux
   -h, --help                                               help for kwok
       --kubeconfig string                                  Path to the kubeconfig file to use (default "~/.kube/config")

site/content/en/docs/generated/kwokctl_create_cluster.md

@@ -17,6 +17,7 @@ kwokctl create cluster [flags]
       --disable-kube-controller-manager         Disable the kube-controller-manager
       --disable-kube-scheduler                  Disable the kube-scheduler
       --disable-qps-limits                      Disable QPS limits for components
+      --enable-crds strings                     List of CRDs to enable
       --etcd-binary string                      Binary of etcd, only for binary runtime
       --etcd-binary-tar string                  Tar of etcd, if --etcd-binary is set, this is ignored, only for binary runtime
                                                  (default "https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9-linux-amd64.tar.gz")

test/kwokctl/kwokctl_exec_test.sh

@@ -95,14 +95,7 @@ function main() {
     else
       yaml="${DIR}/exec.yaml"
     fi
-    create_cluster "${name}" "${release}" --config - <<EOF
-apiVersion: config.kwok.x-k8s.io/v1alpha1
-kind: KwokConfiguration
-options:
-  enableCRDs:
-  - ClusterExec
-  - Exec
-EOF
+    create_cluster "${name}" "${release}" --enable-crds=Exec,ClusterExec
     if [[ "${KWOK_RUNTIME}" != "binary" && "${KWOK_RUNTIME}" != "kind" && "${KWOK_RUNTIME}" != "kind-podman" ]]; then
       create_user "${KWOK_RUNTIME}" "${name}" "kwok-controller" 1001 "test" 1002 "test" "/home/test" "/bin/sh"
     fi
@@ -117,14 +110,7 @@ EOF
     delete_cluster "${name}"
 
     name="crd-exec-cluster-${KWOK_RUNTIME}-${release//./-}"
-    create_cluster "${name}" "${release}" --config - <<EOF
-apiVersion: config.kwok.x-k8s.io/v1alpha1
-kind: KwokConfiguration
-options:
-  enableCRDs:
-  - ClusterExec
-  - Exec
-EOF
+    create_cluster "${name}" "${release}" --enable-crds=Exec,ClusterExec
     test_apply_node_and_pod "${name}" || failed+=("apply_node_and_pod")
     kwokctl --name "${name}" kubectl apply -f "${DIR}/exec.yaml"
     test_exec "${name}" other pod/fake-pod "pwd" "/tmp" || failed+=("${name}_target_exec")

test/kwokctl/kwokctl_port_forward_test.sh

@@ -128,14 +128,7 @@ function main() {
     delete_cluster "${name}"
 
     name="crd-port-forward-cluster-${KWOK_RUNTIME}-${release//./-}"
-    create_cluster "${name}" "${release}" --config - <<EOF
-apiVersion: config.kwok.x-k8s.io/v1alpha1
-kind: KwokConfiguration
-options:
-  enableCRDs:
-  - ClusterPortForward
-  - PortForward
-EOF
+    create_cluster "${name}" "${release}" --enable-crds=PortForward,ClusterPortForward
     test_apply_node_and_pod "${name}" || failed+=("apply_node_and_pod")
     kwokctl --name "${name}" kubectl apply -f "${DIR}/port-forward.yaml"
     sleep 1