From 32e2c5fd904e61815a3292b5288d89516f9e6092 Mon Sep 17 00:00:00 2001 From: Rafal Lal Date: Wed, 4 Dec 2024 14:12:49 +0000 Subject: [PATCH] Add CPUScalingConfiguration part of CPUScalingProfile Controller Signed-off-by: Rafal Lal --- .../cpuscalingprofile_controller.go | 185 ++++- .../cpuscalingprofile_controller_test.go | 736 +++++++++++++++++- 2 files changed, 895 insertions(+), 26 deletions(-) diff --git a/internal/controller/cpuscalingprofile_controller.go b/internal/controller/cpuscalingprofile_controller.go index 30b8e23..166977d 100644 --- a/internal/controller/cpuscalingprofile_controller.go +++ b/internal/controller/cpuscalingprofile_controller.go @@ -19,6 +19,9 @@ package controller import ( "context" "fmt" + "reflect" + "slices" + "strings" powerv1 "github.com/intel/kubernetes-power-manager/api/v1" @@ -27,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -68,6 +72,34 @@ func (r *CPUScalingProfileReconciler) Reconcile(ctx context.Context, req ctrl.Re } logger.V(5).Info("cpuscalingprofile profile not found") + + // Owned PowerProfile will be automatically deleted by K8s GC + + // Search for CPUScalingProfile in CPUScalingConfigurations to delete associated entries + cpuScalingConfList := &powerv1.CPUScalingConfigurationList{} + err = r.Client.List(context.TODO(), cpuScalingConfList) + logger.V(5).Info("retrieving the cpuscalingconfiguration list") + if err != nil { + logger.Error(err, "error retrieving cpuscalingconfiguration list") + return ctrl.Result{}, err + } + + // Set Name and Namespace so helper func controllerutil.RemoveOwnerReference can be used + scalingProfile = &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: req.Namespace, + }, + } + for _, cpuScalingConf := range cpuScalingConfList.Items { + if err = r.updateOrDeleteCPUScalingConfiguration( + &cpuScalingConf, []powerv1.ConfigItem{}, scalingProfile, logger); err != nil { + logger.Error(err, "error while cleaning after deleted cpuscalingprofile") + return ctrl.Result{}, err + } + } + logger.V(5).Info("succesfully cleaned after deleted cpuscalingprofile") + return ctrl.Result{}, nil } @@ -84,7 +116,58 @@ func (r *CPUScalingProfileReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, err } - // TODO: CPUScalingConfiguration related code + // Search for CPUs associated with CPUScalingProfile in PowerWorkloads + powerWorkloadList := &powerv1.PowerWorkloadList{} + err = r.Client.List(context.TODO(), powerWorkloadList) + logger.V(5).Info("retrieving the powerworkload list") + if err != nil { + logger.Error(err, "error retrieving powerworkload list") + return ctrl.Result{}, err + } + nodesItems := r.createConfigItems(powerWorkloadList, scalingProfile) + + // Create, delete or update CPUScalingConfiguration on nodes on which CPUScalingProfile is requested + for nodeName, items := range nodesItems { + confKey := client.ObjectKey{Name: nodeName, Namespace: IntelPowerNamespace} + cpuScalingConf := &powerv1.CPUScalingConfiguration{} + err := r.Client.Get(context.TODO(), confKey, cpuScalingConf) + // CPUScalingConfiguration could not be retrieved + if err != nil { + if !errors.IsNotFound(err) { + logger.Error(err, "error retrieving cpuscalingconfiguration") + return ctrl.Result{}, err + } + + cpuScalingConf = &powerv1.CPUScalingConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Namespace: IntelPowerNamespace, + }, + } + if err = r.setCPUScalingConfiguration(cpuScalingConf, items, scalingProfile); err != nil { + return ctrl.Result{}, err + } + + if len(cpuScalingConf.Spec.Items) == 0 { + logger.V(5).Info("not creating cpuscalingconfiguration, spec would be empty") + continue + } + + if err := r.Client.Create(context.TODO(), cpuScalingConf); err != nil { + err = fmt.Errorf("error creating cpuscalingconfiguration: %w", err) + logger.Error(err, "") + return ctrl.Result{}, err + } + + logger.V(5).Info("cpuscalingconfiguration successfully created") + continue + } + + if err = r.updateOrDeleteCPUScalingConfiguration(cpuScalingConf, items, scalingProfile, logger); err != nil { + logger.Error(err, "error while reconciling after cluster state change") + return ctrl.Result{}, err + } + } return ctrl.Result{}, nil } @@ -159,10 +242,110 @@ func (r *CPUScalingProfileReconciler) verifyCPUScalingProfileParams(scalingSpec return nil } +// updateOrDeleteCPUScalingConfiguration sets existing CPUScalingConfiguration with newItems and then decide if +// CPUScalingConfiguration should be updated or deleted in apiserver. +func (r *CPUScalingProfileReconciler) updateOrDeleteCPUScalingConfiguration( + scalingConfig *powerv1.CPUScalingConfiguration, newItems []powerv1.ConfigItem, scalingProfile metav1.Object, + logger logr.Logger) error { + logger = logger.WithValues("cpuscalingconfiguration", scalingConfig.Name) + + origScalingConfig := scalingConfig.DeepCopy() + if err := r.setCPUScalingConfiguration(scalingConfig, newItems, scalingProfile); err != nil { + err = fmt.Errorf("error modifying cpuscalingconfiguration object: %w", err) + logger.Error(err, "") + return err + } + + if len(scalingConfig.Spec.Items) == 0 { + if err := r.Client.Delete(context.TODO(), scalingConfig); err != nil { + err = fmt.Errorf("error deleting cpuscalingconfiguration from the cluster: %w", err) + logger.Error(err, "") + return err + } + logger.V(5).Info( + "succesfully deleted empty cpuscalingconfiguration", + ) + return nil + } + + // Guard to not call unnecessary Update + if reflect.DeepEqual(origScalingConfig, scalingConfig) { + logger.V(5).Info("cpuscalingconfiguration is already in desired state") + return nil + } + + if err := r.Client.Update(context.TODO(), scalingConfig); err != nil { + err = fmt.Errorf("error updating cpuscalingconfiguration: %w", err) + logger.Error(err, "") + return err + } + logger.V(5).Info("cpuscalingconfiguration successfully updated") + + return nil +} + +// createConfigItems creates slices of powerv1.ConfigItem mapped to node names. If no Containers are requesting +// passed CPUScalingProfile on iterated node, empty powerv1.ConfigItem slice is mapped to node name. +func (r *CPUScalingProfileReconciler) createConfigItems(powerWorkloadList *powerv1.PowerWorkloadList, + scalingProfile *powerv1.CPUScalingProfile) map[string][]powerv1.ConfigItem { + nodesItems := make(map[string][]powerv1.ConfigItem) + for _, powerWorkload := range powerWorkloadList.Items { + if powerWorkload.Spec.PowerProfile == scalingProfile.Name && !powerWorkload.Spec.AllCores { + // ConfigItem is per Container + for _, container := range powerWorkload.Spec.Node.Containers { + nodesItems[powerWorkload.Spec.Node.Name] = append(nodesItems[powerWorkload.Spec.Node.Name], + powerv1.ConfigItem{ + PowerProfile: scalingProfile.Name, + CpuIDs: container.ExclusiveCPUs, + SamplePeriod: scalingProfile.Spec.SamplePeriod, + PodUID: container.PodUID, + }, + ) + } + // No containers are using PowerProfile owned by this CPUScalingProfile + if _, found := nodesItems[powerWorkload.Spec.Node.Name]; !found { + nodesItems[powerWorkload.Spec.Node.Name] = []powerv1.ConfigItem{} + } + } + } + + return nodesItems +} + +// setCPUScalingConfiguration sets .Spec.Items to newItems and refreshes Ownership by CPUScalingProfile accordingly. +func (r *CPUScalingProfileReconciler) setCPUScalingConfiguration(scalingConfig *powerv1.CPUScalingConfiguration, + newItems []powerv1.ConfigItem, scalingProfile metav1.Object) error { + // Handle CPUScalingConfiguration.Spec.Items + scalingConfig.Spec.Items = slices.DeleteFunc(scalingConfig.Spec.Items, + func(item powerv1.ConfigItem) bool { + return item.PowerProfile == scalingProfile.GetName() + }, + ) + scalingConfig.Spec.Items = append(scalingConfig.Spec.Items, newItems...) + // Sort is crucial to update decision, we don't want to update resource in apiserver just because order has changed + slices.SortFunc(scalingConfig.Spec.Items, func(a, b powerv1.ConfigItem) int { + return strings.Compare(string(a.PodUID), string(b.PodUID)) + }) + + // Handle Ownership + if len(newItems) == 0 { + if err := controllerutil.RemoveOwnerReference(scalingProfile, scalingConfig, r.Scheme); err != nil { + return fmt.Errorf("error deleting ownership of updated cpuscalingconfiguration: %w", err) + } + } else { + if err := controllerutil.SetOwnerReference(scalingProfile, scalingConfig, r.Scheme); err != nil { + return fmt.Errorf("error setting up ownership of updated cpuscalingconfiguration: %w", err) + } + } + + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *CPUScalingProfileReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&powerv1.CPUScalingProfile{}). Owns(&powerv1.PowerProfile{}). + Owns(&powerv1.CPUScalingConfiguration{}, builder.MatchEveryOwner). Complete(r) } diff --git a/internal/controller/cpuscalingprofile_controller_test.go b/internal/controller/cpuscalingprofile_controller_test.go index 4334e85..da35227 100644 --- a/internal/controller/cpuscalingprofile_controller_test.go +++ b/internal/controller/cpuscalingprofile_controller_test.go @@ -41,7 +41,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -func createCPUScalingProfileReconcilerObject(objs []client.Object) (*CPUScalingProfileReconciler, error) { +func createCPUScalingProfileReconcilerObject(objs []client.Object, objsLists []client.ObjectList, +) (*CPUScalingProfileReconciler, error) { log.SetLogger(zap.New(zap.UseDevMode(true), func(opts *zap.Options) { opts.TimeEncoder = zapcore.ISO8601TimeEncoder })) @@ -53,7 +54,12 @@ func createCPUScalingProfileReconcilerObject(objs []client.Object) (*CPUScalingP } // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithObjects(objs...).WithStatusSubresource(objs...).WithScheme(s).Build() + cl := fake.NewClientBuilder(). + WithObjects(objs...). + WithStatusSubresource(objs...). + WithLists(objsLists...). + WithScheme(s). + Build() // Create a reconciler object with the scheme and fake client. r := &CPUScalingProfileReconciler{ @@ -85,7 +91,7 @@ func TestCPUScalingProfile_SetupWithManager(t *testing.T) { } func TestCPUScalingProfile_WrongNamespace(t *testing.T) { - r, err := createCPUScalingProfileReconcilerObject([]client.Object{}) + r, err := createCPUScalingProfileReconcilerObject([]client.Object{}, []client.ObjectList{}) if !assert.NoError(t, err) { return } @@ -194,7 +200,7 @@ func TestCPUScalingProfile_Reconcile_Validate(t *testing.T) { for _, tc := range tCases { t.Log(tc.testCase) - r, err := createCPUScalingProfileReconcilerObject(tc.clientObjs) + r, err := createCPUScalingProfileReconcilerObject(tc.clientObjs, []client.ObjectList{}) if !assert.NoError(t, err) { return } @@ -212,13 +218,14 @@ func TestCPUScalingProfile_Reconcile_Validate(t *testing.T) { } } -func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { +func TestCPUScalingProfile_Reconcile(t *testing.T) { tCases := []struct { testCase string cpuScalingProfileName string validateReconcileAndStatus func(error, client.Client) validateObjects func(client.Client) clientObjs []client.Object + objsLists []client.ObjectList }{ { testCase: "Test Case 1 - CPUScalingProfile is added", @@ -255,7 +262,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, Governor: userspaceGovernor, - Epp: "power", + Epp: "balance_performance", Shared: false, }, pp.Spec) }, @@ -270,7 +277,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, }, @@ -288,7 +295,6 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { return } assert.Empty(t, csc.Status.Errors) - }, validateObjects: func(c client.Client) { pp := &powerv1.PowerProfile{} @@ -311,7 +317,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, Governor: userspaceGovernor, - Epp: "power", + Epp: "balance_performance", Shared: false, }, pp.Spec) }, @@ -326,7 +332,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, &powerv1.PowerProfile{ @@ -351,7 +357,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 1999, Max: 2001, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: "powersave", }, }, @@ -366,6 +372,11 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { validateObjects: func(c client.Client) { // We leave deleting of PowerProfile to K8s GC, fakeclient does not // have GC capabilities, hence cannot assert PowerProfile deletion here + + // CPUScalingConfiguration should be deleted + assert.True(t, errors.IsNotFound(c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, + &powerv1.CPUScalingConfiguration{}))) }, clientObjs: []client.Object{ &powerv1.PowerProfile{ @@ -387,6 +398,36 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { }, }, }, + objsLists: []client.ObjectList{ + &powerv1.CPUScalingConfigurationList{ + Items: []powerv1.CPUScalingConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Namespace: IntelPowerNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, + }, + Spec: powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{5, 6, 7, 8}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, + }, + }, + }, + }, }, { testCase: "Test Case 4 - CPUScalingProfile is added, but PowerProfile with the same name" + @@ -430,7 +471,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, &powerv1.PowerProfile{ @@ -493,7 +534,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: userspaceGovernor, }, pp.Spec) }, @@ -508,7 +549,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, &powerv1.PowerProfile{ @@ -539,7 +580,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 1000, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: "powersave", }, }, @@ -580,7 +621,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: userspaceGovernor, }, pp.Spec) }, @@ -595,7 +636,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 3000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, &powerv1.PowerProfile{ @@ -620,14 +661,591 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 2000, Max: 2500, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: "powersave", }, }, }, }, { - testCase: "Test Case 7 - Empty reconciliation", + testCase: "Test Case 7 - Valid CPUScalingProfile is added, containers are requesting it", + cpuScalingProfileName: "cpuscalingprofile1", + validateReconcileAndStatus: func(err error, c client.Client) { + if !assert.NoError(t, err) { + return + } + csc := &powerv1.CPUScalingProfile{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "cpuscalingprofile1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.Empty(t, csc.Status.Errors) + }, + validateObjects: func(c client.Client) { + csc := &powerv1.CPUScalingConfiguration{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.ElementsMatch(t, []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, csc.ObjectMeta.OwnerReferences) + assert.Equal(t, powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{5, 6, 7, 8}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, csc.Spec) + }, + clientObjs: []client.Object{ + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1", + Namespace: IntelPowerNamespace, + UID: "lkj", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 2000, + Max: 3000, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + Epp: "balance_performance", + }, + }, + }, + objsLists: []client.ObjectList{ + &powerv1.PowerWorkloadList{ + Items: []powerv1.PowerWorkload{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile1", + Node: powerv1.WorkloadNode{ + Name: "worker1", + Containers: []powerv1.Container{ + { + Name: "container1", + Namespace: IntelPowerNamespace, + Pod: "pod1", + PodUID: "abcde", + ExclusiveCPUs: []uint{5, 6, 7, 8}, + PowerProfile: "cpuscalingprofile1", + }, + }, + CpuIds: []uint{5, 6, 7, 8}, + }, + }, + }, + }, + }, + }, + }, + { + testCase: "Test Case 8 - One CPUScalingProfile is already used by containers, " + + "another is requested.", + cpuScalingProfileName: "cpuscalingprofile2", + validateReconcileAndStatus: func(err error, c client.Client) { + if !assert.NoError(t, err) { + return + } + csc := &powerv1.CPUScalingProfile{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "cpuscalingprofile1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.Empty(t, csc.Status.Errors) + }, + validateObjects: func(c client.Client) { + csc := &powerv1.CPUScalingConfiguration{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.ElementsMatch(t, []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + { + Name: "cpuscalingprofile2", + UID: "hgf", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, csc.ObjectMeta.OwnerReferences) + assert.Equal(t, powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{3, 4}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + { + PowerProfile: "cpuscalingprofile2", + CpuIDs: []uint{1, 2}, + SamplePeriod: metav1.Duration{Duration: 89 * time.Millisecond}, + PodUID: "fghij", + }, + }, + }, csc.Spec) + }, + clientObjs: []client.Object{ + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1", + Namespace: IntelPowerNamespace, + UID: "lkj", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 2000, + Max: 3000, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + Epp: "balance_performance", + }, + }, + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile2", + Namespace: IntelPowerNamespace, + UID: "hgf", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 1000, + Max: 2000, + SamplePeriod: metav1.Duration{Duration: 89 * time.Millisecond}, + Epp: "balance_power", + }, + }, + &powerv1.CPUScalingConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Namespace: IntelPowerNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, + }, + Spec: powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{3, 4}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, + }, + }, + objsLists: []client.ObjectList{ + &powerv1.PowerWorkloadList{ + Items: []powerv1.PowerWorkload{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile1", + Node: powerv1.WorkloadNode{ + Name: "worker1", + Containers: []powerv1.Container{ + { + Name: "container1", + Namespace: IntelPowerNamespace, + Pod: "pod1", + PodUID: "abcde", + ExclusiveCPUs: []uint{3, 4}, + PowerProfile: "cpuscalingprofile2", + }, + }, + CpuIds: []uint{3, 4}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile2-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile2", + Node: powerv1.WorkloadNode{ + Name: "worker1", + Containers: []powerv1.Container{ + { + Name: "container1", + Namespace: IntelPowerNamespace, + Pod: "pod2", + PodUID: "fghij", + ExclusiveCPUs: []uint{1, 2}, + PowerProfile: "cpuscalingprofile2", + }, + }, + CpuIds: []uint{1, 2}, + }, + }, + }, + }, + }, + }, + }, + { + testCase: "Test Case 9 - Containers are no longer requesting one CPUScalingProfile, " + + "but other CPUScalingProfiles are still in use", + cpuScalingProfileName: "cpuscalingprofile1", + validateReconcileAndStatus: func(err error, c client.Client) { + if !assert.NoError(t, err) { + return + } + csc := &powerv1.CPUScalingProfile{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "cpuscalingprofile1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.Empty(t, csc.Status.Errors) + }, + validateObjects: func(c client.Client) { + csc := &powerv1.CPUScalingConfiguration{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.ElementsMatch(t, []metav1.OwnerReference{ + { + Name: "cpuscalingprofile2", + UID: "hgf", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, csc.ObjectMeta.OwnerReferences) + assert.Equal(t, powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile2", + CpuIDs: []uint{1, 2}, + SamplePeriod: metav1.Duration{Duration: 89 * time.Millisecond}, + PodUID: "fghij", + }, + }, + }, csc.Spec) + }, + clientObjs: []client.Object{ + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1", + Namespace: IntelPowerNamespace, + UID: "lkj", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 2000, + Max: 3000, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + Epp: "balance_performance", + }, + }, + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile2", + Namespace: IntelPowerNamespace, + UID: "hgf", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 1000, + Max: 2000, + SamplePeriod: metav1.Duration{Duration: 89 * time.Millisecond}, + Epp: "balance_power", + }, + }, + &powerv1.CPUScalingConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Namespace: IntelPowerNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + { + Name: "cpuscalingprofile2", + UID: "hgf", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, + }, + Spec: powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{5, 6, 7, 8}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + { + PowerProfile: "cpuscalingprofile2", + CpuIDs: []uint{1, 2}, + SamplePeriod: metav1.Duration{Duration: 89 * time.Millisecond}, + PodUID: "fghij", + }, + }, + }, + }, + }, + objsLists: []client.ObjectList{ + &powerv1.PowerWorkloadList{ + Items: []powerv1.PowerWorkload{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile2-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile2", + Node: powerv1.WorkloadNode{ + Name: "worker1", + Containers: []powerv1.Container{ + { + Name: "container1", + Namespace: IntelPowerNamespace, + Pod: "pod2", + PodUID: "fghij", + ExclusiveCPUs: []uint{1, 2}, + PowerProfile: "cpuscalingprofile2", + }, + }, + CpuIds: []uint{1, 2}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile1", + Node: powerv1.WorkloadNode{ + Name: "worker1", + }, + }, + }, + }, + }, + }, + }, + { + testCase: "Test Case 10 - CPUScalingProfile is present, containers are no longer " + + "requesting it", + cpuScalingProfileName: "cpuscalingprofile1", + validateReconcileAndStatus: func(err error, c client.Client) { + if !assert.NoError(t, err) { + return + } + csc := &powerv1.CPUScalingProfile{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "cpuscalingprofile1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.Empty(t, csc.Status.Errors) + }, + validateObjects: func(c client.Client) { + // Empty CPUScalingConfiguration should be deleted + assert.True(t, errors.IsNotFound(c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, + &powerv1.CPUScalingConfiguration{}), + )) + }, + clientObjs: []client.Object{ + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1", + Namespace: IntelPowerNamespace, + UID: "lkj", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 2000, + Max: 3000, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + Epp: "balance_performance", + }, + }, + &powerv1.CPUScalingConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Namespace: IntelPowerNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, + }, + Spec: powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{5, 6, 7, 8}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, + }, + }, + objsLists: []client.ObjectList{ + &powerv1.PowerWorkloadList{ + Items: []powerv1.PowerWorkload{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile1", + Node: powerv1.WorkloadNode{ + Name: "worker1", + }, + }, + }, + }, + }, + }, + }, + { + testCase: "Test Case 11 - Owned CPUScalingConfiguration got modified", + cpuScalingProfileName: "cpuscalingprofile1", + validateReconcileAndStatus: func(err error, c client.Client) { + if !assert.NoError(t, err) { + return + } + csc := &powerv1.CPUScalingProfile{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "cpuscalingprofile1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.Empty(t, csc.Status.Errors) + }, + validateObjects: func(c client.Client) { + csc := &powerv1.CPUScalingConfiguration{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.ElementsMatch(t, []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, csc.ObjectMeta.OwnerReferences) + assert.Equal(t, powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{5, 6, 7, 8}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, csc.Spec) + }, + clientObjs: []client.Object{ + &powerv1.CPUScalingProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1", + Namespace: IntelPowerNamespace, + UID: "lkj", + }, + Spec: powerv1.CPUScalingProfileSpec{ + Min: 2000, + Max: 3000, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + Epp: "balance_performance", + }, + }, + &powerv1.CPUScalingConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Namespace: IntelPowerNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + Controller: ptr.To(false), + BlockOwnerDeletion: ptr.To(false), + }, + }, + }, + Spec: powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{99, 999}, + SamplePeriod: metav1.Duration{Duration: 99 * time.Millisecond}, + PodUID: "mnbvc", + }, + }, + }, + }, + }, + objsLists: []client.ObjectList{ + &powerv1.PowerWorkloadList{ + Items: []powerv1.PowerWorkload{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile1", + Node: powerv1.WorkloadNode{ + Name: "worker1", + Containers: []powerv1.Container{ + { + Name: "container1", + Namespace: IntelPowerNamespace, + Pod: "pod1", + PodUID: "abcde", + ExclusiveCPUs: []uint{5, 6, 7, 8}, + PowerProfile: "cpuscalingprofile1", + }, + }, + CpuIds: []uint{5, 6, 7, 8}, + }, + }, + }, + }, + }, + }, + }, + { + testCase: "Test Case 12 - Empty reconciliation", cpuScalingProfileName: "cpuscalingprofile1", validateReconcileAndStatus: func(err error, c client.Client) { if !assert.NoError(t, err) { @@ -661,9 +1279,32 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 1000, Max: 2000, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: userspaceGovernor, }, pp.Spec) + csc := &powerv1.CPUScalingConfiguration{} + if !assert.NoError(t, c.Get(context.TODO(), + client.ObjectKey{Name: "worker1", Namespace: IntelPowerNamespace}, csc)) { + return + } + assert.ElementsMatch(t, []metav1.OwnerReference{ + { + Name: "cpuscalingprofile1", + UID: "lkj", + Kind: "CPUScalingProfile", + APIVersion: "power.intel.com/v1", + }, + }, csc.ObjectMeta.OwnerReferences) + assert.Equal(t, powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{1, 2}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, csc.Spec) }, clientObjs: []client.Object{ &powerv1.CPUScalingProfile{ @@ -676,7 +1317,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 1000, Max: 2000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, &powerv1.PowerProfile{ @@ -701,14 +1342,59 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 1000, Max: 2000, Shared: false, - Epp: "power", + Epp: "balance_performance", Governor: userspaceGovernor, }, }, + &powerv1.CPUScalingConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.CPUScalingConfigurationSpec{ + Items: []powerv1.ConfigItem{ + { + PowerProfile: "cpuscalingprofile1", + CpuIDs: []uint{1, 2}, + SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, + PodUID: "abcde", + }, + }, + }, + }, + }, + objsLists: []client.ObjectList{ + &powerv1.PowerWorkloadList{ + Items: []powerv1.PowerWorkload{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cpuscalingprofile1-worker1", + Namespace: IntelPowerNamespace, + }, + Spec: powerv1.PowerWorkloadSpec{ + PowerProfile: "cpuscalingprofile1", + Node: powerv1.WorkloadNode{ + Name: "worker1", + Containers: []powerv1.Container{ + { + Name: "container1", + Namespace: IntelPowerNamespace, + Pod: "pod1", + PodUID: "abcde", + ExclusiveCPUs: []uint{1, 2}, + PowerProfile: "cpuscalingprofile1", + }, + }, + CpuIds: []uint{1, 2}, + }, + }, + }, + }, + }, }, }, { - testCase: "Test Case 8 - Status cleared from previous reconciliation error", + testCase: "Test Case 13 - Status cleared from previous reconciliation error", cpuScalingProfileName: "cpuscalingprofile1", validateReconcileAndStatus: func(err error, c client.Client) { if !assert.NoError(t, err) { @@ -739,7 +1425,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { Min: 1000, Max: 2000, SamplePeriod: metav1.Duration{Duration: 15 * time.Millisecond}, - Epp: "power", + Epp: "balance_performance", }, }, }, @@ -748,7 +1434,7 @@ func TestCPUScalingProfile_Reconcile_PowerProfile(t *testing.T) { for _, tc := range tCases { t.Log(tc.testCase) - r, err := createCPUScalingProfileReconcilerObject(tc.clientObjs) + r, err := createCPUScalingProfileReconcilerObject(tc.clientObjs, tc.objsLists) if !assert.NoError(t, err) { return }