Skip to content

Commit 1f3cf52

Browse files
morvencaomorvencao
committed
add cloudevents driver feature gate.
Signed-off-by: morvencao <[email protected]>
1 parent 6b86d67 commit 1f3cf52

16 files changed

+250
-21
lines changed

‎deploy/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

+45
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,51 @@ spec:
181181
description: RegistrationConfiguration contains the configuration
182182
of registration
183183
properties:
184+
bootstrapKubeConfigs:
185+
description: "BootstrapKubeConfigs defines the ordered list of
186+
bootstrap kubeconfigs. The order decides which bootstrap kubeconfig
187+
to use first when rebootstrap. \n When the agent loses the connection
188+
to the current hub over HubConnectionTimeoutSeconds, or the
189+
managedcluster CR is set `hubAcceptsClient=false` on the hub,
190+
the controller marks the related bootstrap kubeconfig as \"failed\".
191+
\n A failed bootstrapkubeconfig won't be used for the duration
192+
specified by SkipFailedBootstrapKubeConfigSeconds. But if the
193+
user updates the content of a failed bootstrapkubeconfig, the
194+
\"failed\" mark will be cleared."
195+
properties:
196+
localSecretsConfig:
197+
description: LocalSecretsConfig include a list of secrets
198+
that contains the kubeconfigs for ordered bootstrap kubeconifigs.
199+
The secrets must be in the same namespace where the agent
200+
controller runs.
201+
properties:
202+
hubConnectionTimeoutSeconds:
203+
default: 600
204+
description: HubConnectionTimeoutSeconds is used to set
205+
the timeout of connecting to the hub cluster. When agent
206+
loses the connection to the hub over the timeout seconds,
207+
the agent do a rebootstrap. By default is 10 mins.
208+
format: int32
209+
minimum: 180
210+
type: integer
211+
secretNames:
212+
description: SecretNames is a list of secret names. The
213+
secrets are in the same namespace where the agent controller
214+
runs.
215+
items:
216+
type: string
217+
type: array
218+
type: object
219+
type:
220+
default: None
221+
description: Type specifies the type of priority bootstrap
222+
kubeconfigs. By default, it is set to None, representing
223+
no priority bootstrap kubeconfigs are set.
224+
enum:
225+
- None
226+
- LocalSecrets
227+
type: string
228+
type: object
184229
clientCertExpirationSeconds:
185230
description: clientCertExpirationSeconds represents the seconds
186231
of a client certificate to expire. If it is not set or 0, the

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ require (
3535
k8s.io/kube-aggregator v0.29.3
3636
k8s.io/utils v0.0.0-20240310230437-4693a0247e57
3737
open-cluster-management.io/addon-framework v0.9.1-0.20240416063208-ecb7f349df05
38-
open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c
38+
open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad
3939
open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379
4040
sigs.k8s.io/controller-runtime v0.17.3
4141
sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -425,8 +425,8 @@ k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0g
425425
k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
426426
open-cluster-management.io/addon-framework v0.9.1-0.20240416063208-ecb7f349df05 h1:oj4IEyxmQFMxdpnOPE/Y7Xkyfkz4hGVkcYZ4dn8WtKU=
427427
open-cluster-management.io/addon-framework v0.9.1-0.20240416063208-ecb7f349df05/go.mod h1:K+/TwCUz2PZ7L+svaodw6029S/iPJ7eVX21F7NMw+Kw=
428-
open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c h1:/iUoY6/PqBmcBq3v0+UBFvIcI39k/QPRGqpOv9XtDIc=
429-
open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
428+
open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad h1:DB3GpK5vzbGu9ss13bfodi8pGTkPcpdcLvOPEPMptTk=
429+
open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad/go.mod h1:yrNuMMpciXjXPnj2yznb6LTyrGliiTrFZAJDp/Ck3c4=
430430
open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379 h1:8jXVHfgy+wgXq1mrWC1mTieoP77WsAAHNpzILMIzWB0=
431431
open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379/go.mod h1:w2OaxtCyegxeyFLU42UQ3oxUz01QdsBQkcHI17T/l48=
432432
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=

‎manifests/cluster-manager/management/cluster-manager-manifestworkreplicaset-deployment.yaml

+9-10
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,16 @@ spec:
4646
args:
4747
- "/work"
4848
- "manager"
49+
{{ if .CloudEventsDriverEnabled }}
4950
- "--work-driver={{ .WorkDriver }}"
50-
{{ if eq .WorkDriver "kube" }}
51+
{{ if ne .WorkDriver "kube" }}
52+
- "--cloudevents-client-id=work-controller-$(POD_NAME)"
53+
- "--work-driver-config=/var/run/secrets/work/config.yaml"
54+
{{ end }}
55+
{{ end }}
5156
{{ if .HostedMode }}
5257
- "--kubeconfig=/var/run/secrets/hub/kubeconfig"
5358
{{ end }}
54-
{{ else }}
55-
- "--cloudevents-client-id=work-controller-$(POD_NAME)"
56-
- "--work-driver-config=/var/run/secrets/hub/config.yaml"
57-
{{ end }}
5859
env:
5960
- name: POD_NAME
6061
valueFrom:
@@ -97,27 +98,25 @@ spec:
9798
volumeMounts:
9899
- name: tmpdir
99100
mountPath: /tmp
100-
{{ if eq .WorkDriver "kube" }}
101101
{{ if .HostedMode }}
102102
- mountPath: /var/run/secrets/hub
103103
name: kubeconfig
104104
readOnly: true
105105
{{ end }}
106-
{{ else }}
107-
- mountPath: /var/run/secrets/hub
106+
{{ if and .CloudEventsDriverEnabled (ne .WorkDriver "kube") }}
107+
- mountPath: /var/run/secrets/work
108108
name: workdriverconfig
109109
readOnly: true
110110
{{ end }}
111111
volumes:
112112
- name: tmpdir
113113
emptyDir: { }
114-
{{ if eq .WorkDriver "kube" }}
115114
{{ if .HostedMode }}
116115
- name: kubeconfig
117116
secret:
118117
secretName: work-controller-sa-kubeconfig
119118
{{ end }}
120-
{{ else }}
119+
{{ if and .CloudEventsDriverEnabled (ne .WorkDriver "kube") }}
121120
- name: workdriverconfig
122121
secret:
123122
secretName: work-driver-config

‎manifests/config.go

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type HubConfig struct {
1919
AddOnManagerImage string
2020
AddOnManagerEnabled bool
2121
MWReplicaSetEnabled bool
22+
CloudEventsDriverEnabled bool
2223
WorkDriver string
2324
AutoApproveUsers string
2425
// ResourceRequirementResourceType is the resource requirement resource type for the cluster manager managed containers.

‎pkg/operator/helpers/queuekey.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,12 @@ const (
3636
// ExternalManagedKubeConfigAgent is the secret name of kubeconfig secret to connecting to the managed cluster
3737
// Only applicable to SingletonHosted mode, agent uses it to connect to the managed cluster.
3838
ExternalManagedKubeConfigAgent = "external-managed-kubeconfig-agent"
39-
// WorkDriverConfig is the secret name of work driver config to connect to work driver.
40-
// Only applicable to non-kube work driver, hub controllers connect to work driver using this config.
41-
WorkDriverConfig = "work-driver-config"
4239

4340
RegistrationWebhookSecret = "registration-webhook-serving-cert"
4441
RegistrationWebhookService = "cluster-manager-registration-webhook"
4542
WorkWebhookSecret = "work-webhook-serving-cert" // #nosec G101
4643
WorkWebhookService = "cluster-manager-work-webhook"
44+
WorkDriverConfig = "work-driver-config"
4745

4846
SignerSecret = "signer-secret"
4947
CaBundleConfigmap = "ca-bundle-configmap"

‎pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
138138

139139
// default driver is kube
140140
workDriver := operatorapiv1.WorkDriverTypeKube
141-
if clusterManager.Spec.WorkConfiguration.WorkDriver != "" {
141+
if clusterManager.Spec.WorkConfiguration != nil && clusterManager.Spec.WorkConfiguration.WorkDriver != "" {
142142
workDriver = clusterManager.Spec.WorkConfiguration.WorkDriver
143143
}
144144

@@ -180,6 +180,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
180180
}
181181
config.WorkFeatureGates, workFeatureMsgs = helpers.ConvertToFeatureGateFlags("Work", workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates)
182182
config.MWReplicaSetEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.ManifestWorkReplicaSet)
183+
config.CloudEventsDriverEnabled = helpers.FeatureGateEnabled(workFeatureGates, ocmfeature.DefaultHubWorkFeatureGates, ocmfeature.CloudEventsDrivers)
183184

184185
var addonFeatureGates []operatorapiv1.FeatureGate
185186
if clusterManager.Spec.AddOnManagerConfiguration != nil {

‎pkg/work/hub/options.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ type WorkHubManagerOptions struct {
1313
}
1414

1515
func NewWorkHubManagerOptions() *WorkHubManagerOptions {
16-
return &WorkHubManagerOptions{}
16+
return &WorkHubManagerOptions{
17+
WorkDriver: "kube",
18+
}
1719
}
1820

1921
// AddFlags register and binds the default flags

‎test/integration/operator/clustermanager_hosted_test.go

+22-1
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,28 @@ var _ = ginkgo.Describe("ClusterManager Hosted Mode", func() {
497497
if err != nil {
498498
return err
499499
}
500-
clusterManager.Spec.WorkConfiguration.WorkDriver = work.ConfigTypeGRPC
500+
featureGates := []operatorapiv1.FeatureGate{
501+
{
502+
Feature: string(feature.ManifestWorkReplicaSet),
503+
Mode: operatorapiv1.FeatureGateModeTypeEnable,
504+
},
505+
{
506+
Feature: string(feature.CloudEventsDrivers),
507+
Mode: operatorapiv1.FeatureGateModeTypeEnable,
508+
},
509+
}
510+
if clusterManager.Spec.WorkConfiguration != nil {
511+
for _, fg := range clusterManager.Spec.WorkConfiguration.FeatureGates {
512+
if fg.Feature != string(feature.ManifestWorkReplicaSet) &&
513+
fg.Feature != string(feature.CloudEventsDrivers) {
514+
featureGates = append(featureGates, fg)
515+
}
516+
}
517+
}
518+
clusterManager.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
519+
FeatureGates: featureGates,
520+
WorkDriver: work.ConfigTypeGRPC,
521+
}
501522
_, err = hostedOperatorClient.OperatorV1().ClusterManagers().Update(
502523
context.Background(), clusterManager, metav1.UpdateOptions{})
503524
return err

‎test/integration/operator/clustermanager_test.go

+22-1
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,28 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
472472
if err != nil {
473473
return err
474474
}
475-
clusterManager.Spec.WorkConfiguration.WorkDriver = work.ConfigTypeGRPC
475+
featureGates := []operatorapiv1.FeatureGate{
476+
{
477+
Feature: string(feature.ManifestWorkReplicaSet),
478+
Mode: operatorapiv1.FeatureGateModeTypeEnable,
479+
},
480+
{
481+
Feature: string(feature.CloudEventsDrivers),
482+
Mode: operatorapiv1.FeatureGateModeTypeEnable,
483+
},
484+
}
485+
if clusterManager.Spec.WorkConfiguration != nil {
486+
for _, fg := range clusterManager.Spec.WorkConfiguration.FeatureGates {
487+
if fg.Feature != string(feature.ManifestWorkReplicaSet) &&
488+
fg.Feature != string(feature.CloudEventsDrivers) {
489+
featureGates = append(featureGates, fg)
490+
}
491+
}
492+
}
493+
clusterManager.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
494+
FeatureGates: featureGates,
495+
WorkDriver: work.ConfigTypeGRPC,
496+
}
476497
_, err = operatorClient.OperatorV1().ClusterManagers().Update(
477498
context.Background(), clusterManager, metav1.UpdateOptions{})
478499
return err

‎vendor/modules.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1507,7 +1507,7 @@ open-cluster-management.io/addon-framework/pkg/basecontroller/events
15071507
open-cluster-management.io/addon-framework/pkg/basecontroller/factory
15081508
open-cluster-management.io/addon-framework/pkg/index
15091509
open-cluster-management.io/addon-framework/pkg/utils
1510-
# open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c
1510+
# open-cluster-management.io/api v0.13.1-0.20240419062633-aacb530ea4ad
15111511
## explicit; go 1.21
15121512
open-cluster-management.io/api/addon/v1alpha1
15131513
open-cluster-management.io/api/client/addon/clientset/versioned

‎vendor/open-cluster-management.io/api/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml

+26
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/open-cluster-management.io/api/feature/feature.go

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/open-cluster-management.io/api/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

+45
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)