diff --git a/kustomize/kwok/deployment.yaml b/kustomize/kwok/deployment.yaml index ab7193ebb..4a6563e18 100644 --- a/kustomize/kwok/deployment.yaml +++ b/kustomize/kwok/deployment.yaml @@ -21,15 +21,16 @@ spec: - --node-port=10247 - --cidr=10.0.0.1/24 - --node-lease-duration-seconds=40 - - --enable-crd=Stage - - --enable-crd=Attach - - --enable-crd=ClusterAttach - - --enable-crd=Exec - - --enable-crd=ClusterExec - - --enable-crd=Logs - - --enable-crd=ClusterLogs - - --enable-crd=PortForward - - --enable-crd=ClusterPortForward + - --enable-crds=Stage + - --enable-crds=Metric + - --enable-crds=Attach + - --enable-crds=ClusterAttach + - --enable-crds=Exec + - --enable-crds=ClusterExec + - --enable-crds=Logs + - --enable-crds=ClusterLogs + - --enable-crds=PortForward + - --enable-crds=ClusterPortForward env: - name: POD_IP valueFrom: diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index b67279da7..e155baace 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -76,6 +76,10 @@ type ComponentPatches struct { // KwokctlConfigurationOptions holds information about the options. type KwokctlConfigurationOptions struct { + // EnableCRDs is a list of CRDs to enable. + // Once listed in this field, it will no longer be supported by the --config flag. + EnableCRDs []string `json:"enableCRDs,omitempty"` + // KubeApiserverPort is the port to expose apiserver. // is the default value for flag --kube-apiserver-port and env KWOK_KUBE_APISERVER_PORT KubeApiserverPort uint32 `json:"kubeApiserverPort,omitempty"` diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 8cf676259..443fc8af2 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -258,6 +258,11 @@ func (in *KwokctlConfiguration) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokctlConfigurationOptions) DeepCopyInto(out *KwokctlConfigurationOptions) { *out = *in + if in.EnableCRDs != nil { + in, out := &in.EnableCRDs, &out.EnableCRDs + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Runtimes != nil { in, out := &in.Runtimes, &out.Runtimes *out = make([]string, len(*in)) diff --git a/pkg/apis/internalversion/kwok_configuration_types.go b/pkg/apis/internalversion/kwok_configuration_types.go index 57b8729f9..cdfcdb470 100644 --- a/pkg/apis/internalversion/kwok_configuration_types.go +++ b/pkg/apis/internalversion/kwok_configuration_types.go @@ -32,7 +32,6 @@ type KwokConfiguration struct { // KwokConfigurationOptions holds information about the options. type KwokConfigurationOptions struct { // EnableCRDs is a list of CRDs to enable. - // Once listed in this field, it will no longer be supported by the --config flag. EnableCRDs []string // The default IP assigned to the Pod on maintained Nodes. diff --git a/pkg/apis/internalversion/kwokctl_configuration_types.go b/pkg/apis/internalversion/kwokctl_configuration_types.go index 9a4e21b33..1fb2e9247 100644 --- a/pkg/apis/internalversion/kwokctl_configuration_types.go +++ b/pkg/apis/internalversion/kwokctl_configuration_types.go @@ -63,6 +63,9 @@ type ComponentPatches struct { // KwokctlConfigurationOptions holds information about the options. type KwokctlConfigurationOptions struct { + // EnableCRDs is a list of CRDs to enable. + EnableCRDs []string + // KubeApiserverPort is the port to expose apiserver. KubeApiserverPort uint32 diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index fdf7af486..0503f15fb 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -1447,6 +1447,7 @@ func Convert_v1alpha1_KwokctlConfiguration_To_internalversion_KwokctlConfigurati } func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_KwokctlConfigurationOptions(in *KwokctlConfigurationOptions, out *configv1alpha1.KwokctlConfigurationOptions, s conversion.Scope) error { + out.EnableCRDs = *(*[]string)(unsafe.Pointer(&in.EnableCRDs)) out.KubeApiserverPort = in.KubeApiserverPort out.Runtime = in.Runtime out.Runtimes = *(*[]string)(unsafe.Pointer(&in.Runtimes)) @@ -1532,6 +1533,7 @@ func Convert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_KwokctlConf } func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_KwokctlConfigurationOptions(in *configv1alpha1.KwokctlConfigurationOptions, out *KwokctlConfigurationOptions, s conversion.Scope) error { + out.EnableCRDs = *(*[]string)(unsafe.Pointer(&in.EnableCRDs)) out.KubeApiserverPort = in.KubeApiserverPort out.Runtime = in.Runtime out.Runtimes = *(*[]string)(unsafe.Pointer(&in.Runtimes)) diff --git a/pkg/apis/internalversion/zz_generated.deepcopy.go b/pkg/apis/internalversion/zz_generated.deepcopy.go index 0ee0a78d6..2c03bf13f 100644 --- a/pkg/apis/internalversion/zz_generated.deepcopy.go +++ b/pkg/apis/internalversion/zz_generated.deepcopy.go @@ -643,6 +643,11 @@ func (in *KwokctlConfiguration) DeepCopy() *KwokctlConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokctlConfigurationOptions) DeepCopyInto(out *KwokctlConfigurationOptions) { *out = *in + if in.EnableCRDs != nil { + in, out := &in.EnableCRDs, &out.EnableCRDs + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Runtimes != nil { in, out := &in.Runtimes, &out.Runtimes *out = make([]string, len(*in)) diff --git a/pkg/kwok/cmd/root.go b/pkg/kwok/cmd/root.go index 9877650f8..a4d039672 100644 --- a/pkg/kwok/cmd/root.go +++ b/pkg/kwok/cmd/root.go @@ -90,7 +90,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Master, "master", flags.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") cmd.Flags().StringVar(&flags.Options.ServerAddress, "server-address", flags.Options.ServerAddress, "Address to expose the server on") cmd.Flags().UintVar(&flags.Options.NodeLeaseDurationSeconds, "node-lease-duration-seconds", flags.Options.NodeLeaseDurationSeconds, "Duration of node lease seconds") - cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crd", flags.Options.EnableCRDs, "List of CRDs to enable") + cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable") cmd.Flags().BoolVar(&flags.Options.EnableCNI, "experimental-enable-cni", flags.Options.EnableCNI, "Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux") if config.GOOS != "linux" { diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index 90dbd6c5f..b9c548f1c 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -133,6 +133,8 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().DurationVar(&flags.Wait, "wait", 0, "Wait for the cluster to be ready") cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context") cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components") + cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable") + return cmd } diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go index 3466d6dc5..65c93ab59 100644 --- a/pkg/kwokctl/components/kwok_controller.go +++ b/pkg/kwokctl/components/kwok_controller.go @@ -17,6 +17,8 @@ limitations under the License. package components import ( + "strings" + "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" @@ -40,6 +42,7 @@ type BuildKwokControllerComponentConfig struct { NodeName string Verbosity log.Level NodeLeaseDurationSeconds uint + EnableCRDs []string ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume ExtraEnvs []internalversion.Env @@ -121,6 +124,10 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp kwokControllerArgs = append(kwokControllerArgs, "--v="+format.String(conf.Verbosity)) } + if len(conf.EnableCRDs) != 0 { + kwokControllerArgs = append(kwokControllerArgs, "--enable-crds="+strings.Join(conf.EnableCRDs, ",")) + } + envs := []internalversion.Env{} envs = append(envs, conf.ExtraEnvs...) diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index e4b3be7b6..d6363a53b 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -592,6 +592,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { NodeName: "localhost", Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, + EnableCRDs: conf.EnableCRDs, ExtraArgs: kwokControllerComponentPatches.ExtraArgs, ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, }) diff --git a/pkg/kwokctl/runtime/cluster.go b/pkg/kwokctl/runtime/cluster.go index f8e908b4c..22b434bb4 100644 --- a/pkg/kwokctl/runtime/cluster.go +++ b/pkg/kwokctl/runtime/cluster.go @@ -173,20 +173,17 @@ func (c *Cluster) Save(ctx context.Context) error { return nil } + var objs []config.InternalObject conf := c.conf.DeepCopy() - objs := []config.InternalObject{ - conf, - } - if conf.Status.Version == "" { conf.Status.Version = consts.Version } - - others := config.FilterWithoutTypeFromContext[*internalversion.KwokctlConfiguration](ctx) - objs = append(objs, others...) + objs = append(objs, conf) kwokConfigs := config.FilterWithTypeFromContext[*internalversion.KwokConfiguration](ctx) - if (len(kwokConfigs) == 0 || !slices.Contains(kwokConfigs[0].Options.EnableCRDs, v1alpha1.StageKind)) && + objs = appendIntoInternalObjects(objs, kwokConfigs...) + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.StageKind) && conf.Options.Runtime != consts.RuntimeTypeKind && conf.Options.Runtime != consts.RuntimeTypeKindPodman && len(config.FilterWithTypeFromContext[*internalversion.Stage](ctx)) == 0 { @@ -194,12 +191,79 @@ func (c *Cluster) Save(ctx context.Context) error { if err != nil { return err } - objs = append(objs, defaultStages...) + objs = appendIntoInternalObjects(objs, defaultStages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.StageKind) { + if conf.Options.Runtime != consts.RuntimeTypeKind && + conf.Options.Runtime != consts.RuntimeTypeKindPodman && + len(config.FilterWithTypeFromContext[*internalversion.Stage](ctx)) == 0 { + defaultStages, err := c.getDefaultStages(conf.Options.NodeStatusUpdateFrequencyMilliseconds, conf.Options.NodeLeaseDurationSeconds != 0) + if err != nil { + return err + } + objs = appendIntoInternalObjects(objs, defaultStages...) + } else { + stages := config.FilterWithTypeFromContext[*internalversion.Stage](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.MetricKind) { + stages := config.FilterWithTypeFromContext[*internalversion.Metric](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.AttachKind) { + stages := config.FilterWithTypeFromContext[*internalversion.Attach](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterAttachKind) { + stages := config.FilterWithTypeFromContext[*internalversion.ClusterAttach](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ExecKind) { + stages := config.FilterWithTypeFromContext[*internalversion.Exec](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterExecKind) { + stages := config.FilterWithTypeFromContext[*internalversion.ClusterExec](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.LogsKind) { + stages := config.FilterWithTypeFromContext[*internalversion.Logs](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterLogsKind) { + stages := config.FilterWithTypeFromContext[*internalversion.ClusterLogs](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.PortForwardKind) { + stages := config.FilterWithTypeFromContext[*internalversion.PortForward](ctx) + objs = appendIntoInternalObjects(objs, stages...) + } + + if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.ClusterPortForwardKind) { + stages := config.FilterWithTypeFromContext[*internalversion.ClusterPortForward](ctx) + objs = appendIntoInternalObjects(objs, stages...) } return config.Save(ctx, c.GetWorkdirPath(ConfigName), objs) } +func appendIntoInternalObjects[T config.InternalObject](objs []config.InternalObject, a ...T) []config.InternalObject { + for _, o := range a { + objs = append(objs, o) + } + return objs +} + func (c *Cluster) getDefaultStages(updateFrequency int64, lease bool) ([]config.InternalObject, error) { objs := []config.InternalObject{} @@ -468,11 +532,13 @@ func (c *Cluster) IsDryRun() bool { // InitCRDs initializes the CRDs. func (c *Cluster) InitCRDs(ctx context.Context) error { - kwokConfigs := config.FilterWithTypeFromContext[*internalversion.KwokConfiguration](ctx) - if len(kwokConfigs) == 0 { - return nil + config, err := c.Config(ctx) + if err != nil { + return err } - crds := kwokConfigs[0].Options.EnableCRDs + conf := &config.Options + + crds := conf.EnableCRDs if len(crds) == 0 { return nil } diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 1c8bb9809..32a3630ef 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -610,6 +610,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { NodeName: c.Name() + "-kwok-controller", Verbosity: env.verbosity, NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, + EnableCRDs: conf.EnableCRDs, ExtraArgs: kwokControllerComponentPatches.ExtraArgs, ExtraVolumes: kwokControllerExtraVolumes, ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 2475b2127..ff10fc51d 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -272,6 +272,7 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { Name: c.Name(), Verbosity: env.verbosity, NodeLeaseDurationSeconds: 40, + EnableCRDs: conf.EnableCRDs, ExtraArgs: kwokControllerComponentPatches.ExtraArgs, ExtraVolumes: kwokControllerExtraVolumes, ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, diff --git a/pkg/kwokctl/runtime/kind/kwok_controller_pod.go b/pkg/kwokctl/runtime/kind/kwok_controller_pod.go index 90f65c477..5d5ebf8fe 100644 --- a/pkg/kwokctl/runtime/kind/kwok_controller_pod.go +++ b/pkg/kwokctl/runtime/kind/kwok_controller_pod.go @@ -58,4 +58,5 @@ type BuildKwokControllerPodConfig struct { ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume ExtraEnvs []internalversion.Env + EnableCRDs []string } diff --git a/pkg/kwokctl/runtime/kind/kwok_controller_pod.yaml.tpl b/pkg/kwokctl/runtime/kind/kwok_controller_pod.yaml.tpl index 192f5062f..6e509c88a 100644 --- a/pkg/kwokctl/runtime/kind/kwok_controller_pod.yaml.tpl +++ b/pkg/kwokctl/runtime/kind/kwok_controller_pod.yaml.tpl @@ -22,6 +22,9 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --node-lease-duration-seconds={{ .NodeLeaseDurationSeconds }} + {{ range .EnableCRDs }} + - --enable-crds={{ . }} + {{ end }} {{ range .ExtraArgs }} - --{{ .Key }}={{ .Value }} {{ end }} diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index 43520013d..c4eb9fa80 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -2067,6 +2067,18 @@ KwokctlConfigurationOptions +enableCRDs + +[]string + + + +

EnableCRDs is a list of CRDs to enable. +Once listed in this field, it will no longer be supported by the –config flag.

+ + + + kubeApiserverPort uint32 diff --git a/site/content/en/docs/generated/kwok.md b/site/content/en/docs/generated/kwok.md index 7f90908a6..a411a8edc 100644 --- a/site/content/en/docs/generated/kwok.md +++ b/site/content/en/docs/generated/kwok.md @@ -13,7 +13,7 @@ kwok [flags] -c, --config strings config path (default [~/.kwok/kwok.yaml]) --disregard-status-with-annotation-selector string All node/pod status excluding the ones that match the annotation selector will be watched and managed. --disregard-status-with-label-selector string All node/pod status excluding the ones that match the label selector will be watched and managed. - --enable-crd strings List of CRDs to enable + --enable-crds strings List of CRDs to enable --experimental-enable-cni Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux -h, --help help for kwok --kubeconfig string Path to the kubeconfig file to use (default "~/.kube/config") diff --git a/site/content/en/docs/generated/kwokctl_create_cluster.md b/site/content/en/docs/generated/kwokctl_create_cluster.md index 6a08747fb..eeada03f2 100644 --- a/site/content/en/docs/generated/kwokctl_create_cluster.md +++ b/site/content/en/docs/generated/kwokctl_create_cluster.md @@ -17,6 +17,7 @@ kwokctl create cluster [flags] --disable-kube-controller-manager Disable the kube-controller-manager --disable-kube-scheduler Disable the kube-scheduler --disable-qps-limits Disable QPS limits for components + --enable-crds strings List of CRDs to enable --etcd-binary string Binary of etcd, only for binary runtime --etcd-binary-tar string Tar of etcd, if --etcd-binary is set, this is ignored, only for binary runtime (default "https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9-linux-amd64.tar.gz") diff --git a/test/kwokctl/kwokctl_exec_test.sh b/test/kwokctl/kwokctl_exec_test.sh index 87291d9f4..f0aa1cbf7 100755 --- a/test/kwokctl/kwokctl_exec_test.sh +++ b/test/kwokctl/kwokctl_exec_test.sh @@ -95,14 +95,7 @@ function main() { else yaml="${DIR}/exec.yaml" fi - create_cluster "${name}" "${release}" --config - <