Skip to content

Commit e2b01df

Browse files
arshadd-barshadd-b
committed
debug Objects with Changed Resource Versions
1 parent ffa63d2 commit e2b01df

File tree

3 files changed

+188
-131
lines changed

3 files changed

+188
-131
lines changed

‎test/e2e/quick_start.go‎

Lines changed: 158 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ import (
2727
. "github.com/onsi/ginkgo/v2"
2828
. "github.com/onsi/gomega"
2929
corev1 "k8s.io/api/core/v1"
30+
"k8s.io/apimachinery/pkg/api/errors"
31+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3032
"k8s.io/utils/ptr"
3133

34+
"sigs.k8s.io/cluster-api/test/e2e/internal/log"
3235
"sigs.k8s.io/cluster-api/test/framework"
3336
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
3437
"sigs.k8s.io/cluster-api/util"
@@ -111,156 +114,182 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput)
111114
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
112115
)
113116

114-
BeforeEach(func() {
115-
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
116-
input = inputGetter()
117-
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
118-
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
119-
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
120-
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
117+
for i := 0; i < 20; i++ {
121118

122-
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
119+
BeforeEach(func() {
120+
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
121+
input = inputGetter()
122+
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
123+
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
124+
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
125+
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
123126

124-
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
125-
if input.ExtensionConfigName == "" {
126-
input.ExtensionConfigName = specName
127+
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
128+
129+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
130+
if input.ExtensionConfigName == "" {
131+
input.ExtensionConfigName = specName
132+
}
127133
}
128-
}
129134

130-
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
131-
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
135+
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
136+
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
132137

133-
if input.DeployClusterClassInSeparateNamespace {
134-
clusterClassNamespace = framework.CreateNamespace(ctx, framework.CreateNamespaceInput{Creator: input.BootstrapClusterProxy.GetClient(), Name: fmt.Sprintf("%s-clusterclass", namespace.Name)}, "40s", "10s")
135-
Expect(clusterClassNamespace).ToNot(BeNil(), "Failed to create namespace")
136-
}
138+
if input.DeployClusterClassInSeparateNamespace {
139+
clusterClassNamespace = framework.CreateNamespace(ctx, framework.CreateNamespaceInput{Creator: input.BootstrapClusterProxy.GetClient(), Name: fmt.Sprintf("%s-clusterclass", namespace.Name)}, "40s", "10s")
140+
Expect(clusterClassNamespace).ToNot(BeNil(), "Failed to create namespace")
141+
}
137142

138-
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
139-
})
143+
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
144+
})
140145

141-
It("Should create a workload cluster", func() {
142-
By("Creating a workload cluster")
146+
It("Should create a workload cluster", func() {
147+
By("Creating a workload cluster")
143148

144-
infrastructureProvider := clusterctl.DefaultInfrastructureProvider
145-
if input.InfrastructureProvider != nil {
146-
infrastructureProvider = *input.InfrastructureProvider
147-
}
149+
infrastructureProvider := clusterctl.DefaultInfrastructureProvider
150+
if input.InfrastructureProvider != nil {
151+
infrastructureProvider = *input.InfrastructureProvider
152+
}
148153

149-
flavor := clusterctl.DefaultFlavor
150-
if input.Flavor != nil {
151-
flavor = *input.Flavor
152-
}
154+
flavor := clusterctl.DefaultFlavor
155+
if input.Flavor != nil {
156+
flavor = *input.Flavor
157+
}
153158

154-
controlPlaneMachineCount := ptr.To[int64](1)
155-
if input.ControlPlaneMachineCount != nil {
156-
controlPlaneMachineCount = input.ControlPlaneMachineCount
157-
}
159+
controlPlaneMachineCount := ptr.To[int64](1)
160+
if input.ControlPlaneMachineCount != nil {
161+
controlPlaneMachineCount = input.ControlPlaneMachineCount
162+
}
158163

159-
workerMachineCount := ptr.To[int64](1)
160-
if input.WorkerMachineCount != nil {
161-
workerMachineCount = input.WorkerMachineCount
162-
}
164+
workerMachineCount := ptr.To[int64](1)
165+
if input.WorkerMachineCount != nil {
166+
workerMachineCount = input.WorkerMachineCount
167+
}
163168

164-
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
165-
if input.ClusterName != nil {
166-
clusterName = *input.ClusterName
167-
}
169+
clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6))
170+
if input.ClusterName != nil {
171+
clusterName = *input.ClusterName
172+
}
168173

169-
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
170-
// NOTE: test extension is already deployed in the management cluster. If for any reason in future we want
171-
// to make this test more self-contained this test should be modified in order to create an additional
172-
// management cluster; also the E2E test configuration should be modified introducing something like
173-
// optional:true allowing to define which providers should not be installed by default in
174-
// a management cluster.
175-
By("Deploy Test Extension ExtensionConfig")
176-
177-
// In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the
178-
// cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension
179-
// automatically creates the ConfigMap with non-blocking preloaded responses.
180-
defaultAllHandlersToBlocking := false
181-
// select on the current namespace
182-
// This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks
183-
// in other test namespaces.
184-
namespaces := []string{namespace.Name}
185-
if input.DeployClusterClassInSeparateNamespace {
186-
// Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace.
187-
namespaces = append(namespaces, clusterClassNamespace.Name)
174+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
175+
// NOTE: test extension is already deployed in the management cluster. If for any reason in future we want
176+
// to make this test more self-contained this test should be modified in order to create an additional
177+
// management cluster; also the E2E test configuration should be modified introducing something like
178+
// optional:true allowing to define which providers should not be installed by default in
179+
// a management cluster.
180+
By("Deploy Test Extension ExtensionConfig")
181+
182+
// In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the
183+
// cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension
184+
// automatically creates the ConfigMap with non-blocking preloaded responses.
185+
defaultAllHandlersToBlocking := false
186+
// select on the current namespace
187+
// This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks
188+
// in other test namespaces.
189+
namespaces := []string{namespace.Name}
190+
if input.DeployClusterClassInSeparateNamespace {
191+
// Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace.
192+
namespaces = append(namespaces, clusterClassNamespace.Name)
193+
}
194+
extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...)
195+
Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
196+
extensionConfig)).
197+
To(Succeed(), "Failed to create the ExtensionConfig")
188198
}
189-
extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...)
190-
Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
191-
extensionConfig)).
192-
To(Succeed(), "Failed to create the ExtensionConfig")
193-
}
194199

195-
variables := map[string]string{
196-
// This is used to template the name of the ExtensionConfig into the ClusterClass.
197-
"EXTENSION_CONFIG_NAME": input.ExtensionConfigName,
198-
}
199-
maps.Copy(variables, input.ClusterctlVariables)
200+
variables := map[string]string{
201+
// This is used to template the name of the ExtensionConfig into the ClusterClass.
202+
"EXTENSION_CONFIG_NAME": input.ExtensionConfigName,
203+
}
204+
maps.Copy(variables, input.ClusterctlVariables)
200205

201-
if input.DeployClusterClassInSeparateNamespace {
202-
variables["CLUSTER_CLASS_NAMESPACE"] = clusterClassNamespace.Name
203-
By("Creating a cluster referencing a ClusterClass from another namespace")
204-
}
206+
if input.DeployClusterClassInSeparateNamespace {
207+
variables["CLUSTER_CLASS_NAMESPACE"] = clusterClassNamespace.Name
208+
By("Creating a cluster referencing a ClusterClass from another namespace")
209+
}
210+
211+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
212+
ClusterProxy: input.BootstrapClusterProxy,
213+
ConfigCluster: clusterctl.ConfigClusterInput{
214+
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
215+
ClusterctlConfigPath: input.ClusterctlConfigPath,
216+
ClusterctlVariables: variables,
217+
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
218+
InfrastructureProvider: infrastructureProvider,
219+
Flavor: flavor,
220+
Namespace: namespace.Name,
221+
ClusterName: clusterName,
222+
KubernetesVersion: input.E2EConfig.MustGetVariable(KubernetesVersion),
223+
ControlPlaneMachineCount: controlPlaneMachineCount,
224+
WorkerMachineCount: workerMachineCount,
225+
},
226+
ControlPlaneWaiters: input.ControlPlaneWaiters,
227+
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
228+
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
229+
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
230+
PostMachinesProvisioned: func() {
231+
if input.PostMachinesProvisioned != nil {
232+
input.PostMachinesProvisioned(input.BootstrapClusterProxy, namespace.Name, clusterName)
233+
}
234+
},
235+
}, clusterResources)
236+
237+
Byf("Verify Cluster Available condition is true")
238+
framework.VerifyClusterAvailable(ctx, framework.VerifyClusterAvailableInput{
239+
Getter: input.BootstrapClusterProxy.GetClient(),
240+
Name: clusterResources.Cluster.Name,
241+
Namespace: clusterResources.Cluster.Namespace,
242+
})
243+
244+
Byf("Verify Machines Ready condition is true")
245+
framework.VerifyMachinesReady(ctx, framework.VerifyMachinesReadyInput{
246+
Lister: input.BootstrapClusterProxy.GetClient(),
247+
Name: clusterResources.Cluster.Name,
248+
Namespace: clusterResources.Cluster.Namespace,
249+
})
250+
251+
By("PASSED!")
205252

206-
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
207-
ClusterProxy: input.BootstrapClusterProxy,
208-
ConfigCluster: clusterctl.ConfigClusterInput{
209-
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
210-
ClusterctlConfigPath: input.ClusterctlConfigPath,
211-
ClusterctlVariables: variables,
212-
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
213-
InfrastructureProvider: infrastructureProvider,
214-
Flavor: flavor,
215-
Namespace: namespace.Name,
216-
ClusterName: clusterName,
217-
KubernetesVersion: input.E2EConfig.MustGetVariable(KubernetesVersion),
218-
ControlPlaneMachineCount: controlPlaneMachineCount,
219-
WorkerMachineCount: workerMachineCount,
220-
},
221-
ControlPlaneWaiters: input.ControlPlaneWaiters,
222-
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
223-
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
224-
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
225-
PostMachinesProvisioned: func() {
226-
if input.PostMachinesProvisioned != nil {
227-
input.PostMachinesProvisioned(input.BootstrapClusterProxy, namespace.Name, clusterName)
228-
}
229-
},
230-
}, clusterResources)
231-
232-
Byf("Verify Cluster Available condition is true")
233-
framework.VerifyClusterAvailable(ctx, framework.VerifyClusterAvailableInput{
234-
Getter: input.BootstrapClusterProxy.GetClient(),
235-
Name: clusterResources.Cluster.Name,
236-
Namespace: clusterResources.Cluster.Namespace,
237253
})
238254

239-
Byf("Verify Machines Ready condition is true")
240-
framework.VerifyMachinesReady(ctx, framework.VerifyMachinesReadyInput{
241-
Lister: input.BootstrapClusterProxy.GetClient(),
242-
Name: clusterResources.Cluster.Name,
243-
Namespace: clusterResources.Cluster.Namespace,
255+
AfterEach(func() {
256+
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
257+
framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
258+
if !input.SkipCleanup {
259+
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
260+
Eventually(func() error {
261+
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true))
262+
}, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed")
263+
}
264+
if input.DeployClusterClassInSeparateNamespace {
265+
DeleteNamespace11(ctx, framework.DeleteNamespaceInput{
266+
Deleter: input.BootstrapClusterProxy.GetClient(),
267+
Name: clusterClassNamespace.Name,
268+
})
269+
}
270+
}
244271
})
272+
}
273+
}
245274

246-
By("PASSED!")
247-
})
275+
// DeleteNamespace is used to delete namespace object.
276+
func DeleteNamespace11(ctx context.Context, input framework.DeleteNamespaceInput, intervals ...interface{}) {
277+
Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteNamespace")
278+
Expect(input.Deleter).NotTo(BeNil(), "input.Deleter is required for DeleteNamespace")
279+
Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for DeleteNamespace")
280+
ns := &corev1.Namespace{
281+
ObjectMeta: metav1.ObjectMeta{
282+
Name: input.Name,
283+
},
284+
}
285+
log.Logf("Deleting namespace %s", input.Name)
286+
Eventually(func() error {
287+
err := input.Deleter.Delete(ctx, ns)
288+
if err != nil && !errors.IsNotFound(err) {
289+
290+
return err
248291

249-
AfterEach(func() {
250-
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
251-
framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
252-
if !input.SkipCleanup {
253-
if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" {
254-
Eventually(func() error {
255-
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true))
256-
}, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed")
257-
}
258-
if input.DeployClusterClassInSeparateNamespace {
259-
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
260-
Deleter: input.BootstrapClusterProxy.GetClient(),
261-
Name: clusterClassNamespace.Name,
262-
})
263-
}
264292
}
265-
})
293+
return nil
294+
}, intervals...).Should(Succeed(), "Failed to delete namespace %s", input.Name)
266295
}

‎test/e2e/quick_start_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
"sigs.k8s.io/cluster-api/test/framework/kubetest"
3030
)
3131

32-
var _ = Describe("When following the Cluster API quick-start", func() {
32+
var _ = Describe("When following the Cluster API quick-start11 [PR-Blocking] [ClusterClass]", Label("PR-Blocking", "ClusterClass"), func() {
3333
QuickStartSpec(ctx, func() QuickStartSpecInput {
3434
return QuickStartSpecInput{
3535
E2EConfig: e2eConfig,
@@ -75,7 +75,7 @@ var _ = Describe("When following the Cluster API quick-start", func() {
7575
})
7676
})
7777

78-
var _ = Describe("When following the Cluster API quick-start with ClusterClass [PR-Blocking] [ClusterClass]", Label("PR-Blocking", "ClusterClass"), func() {
78+
var _ = Describe("When following the Cluster API quick-start with ClusterClass ", func() {
7979
QuickStartSpec(ctx, func() QuickStartSpecInput {
8080
return QuickStartSpecInput{
8181
E2EConfig: e2eConfig,

‎test/framework/resourceversion_helpers.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,12 @@ func printObjectDiff(previousObjects, newObjects map[string]client.Object) func(
8787

8888
if len(preservedObjects) > 0 {
8989
output.WriteString("\nDetected objects with changed resourceVersion\n")
90+
91+
newresourceYAML, _ := yaml.Marshal(newObjects)
92+
output.WriteString(fmt.Sprintf("\nComplete New object:\n%s\n", newresourceYAML))
93+
preresourceYAML, _ := yaml.Marshal(previousObjects)
94+
output.WriteString(fmt.Sprintf("\nComplete previous object:\n%s\n", preresourceYAML))
95+
9096
for objID := range preservedObjects {
9197
previousObj := previousObjects[objID]
9298
newObj := newObjects[objID]
@@ -159,6 +165,28 @@ func getObjectsWithResourceVersion(ctx context.Context, proxy ClusterProxy, name
159165
}
160166
}
161167

168+
// Drop any object that has a status condition with type: EtcdMemberHealthy.
169+
for key, obj := range objects {
170+
u, ok := obj.(*unstructured.Unstructured)
171+
if !ok {
172+
continue
173+
}
174+
conds, found, err := unstructured.NestedSlice(u.Object, "status", "conditions")
175+
if err != nil || !found || len(conds) == 0 {
176+
continue
177+
}
178+
for _, c := range conds {
179+
cm, ok := c.(map[string]interface{})
180+
if !ok {
181+
continue
182+
}
183+
if t, ok := cm["type"].(string); ok && t == "EtcdMemberHealthy" {
184+
keysToDelete.Insert(key)
185+
break
186+
}
187+
}
188+
}
189+
162190
for _, key := range keysToDelete.UnsortedList() {
163191
delete(objectsWithResourceVersion, key)
164192
delete(objects, key)

0 commit comments

Comments
 (0)