From ddb76fcda30bed3ae32ad4eac45058981bbe1ff2 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Mon, 25 Aug 2025 20:24:28 +0800 Subject: [PATCH 01/17] add xset label manager --- xset/api/xset_controller_types.go | 1 + xset/api/xset_types.go | 16 ++++ xset/synccontrols/inexclude.go | 11 +-- xset/synccontrols/inexclude_test.go | 4 +- xset/synccontrols/label_manager.go | 44 ++++++++++ xset/synccontrols/sync_control.go | 127 +++++++++++++++------------- xset/synccontrols/x_replace.go | 16 ++-- xset/synccontrols/x_scale.go | 2 +- xset/synccontrols/x_update.go | 33 ++++---- xset/synccontrols/x_utils.go | 26 ++++-- xset/xset_controller.go | 15 ++++ 11 files changed, 202 insertions(+), 93 deletions(-) create mode 100644 xset/synccontrols/label_manager.go diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index 983bfd04..30147510 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -40,6 +40,7 @@ type XSetController interface { SetXSetStatus(object XSetObject, status *XSetStatus) GetLifeCycleLabelManager() LifeCycleLabelManager + GetXSetControllerLabelManager() XSetControllerLabelManager GetScaleInOpsLifecycleAdapter() LifecycleAdapter GetUpdateOpsLifecycleAdapter() LifecycleAdapter GetResourceContextAdapter() ResourceContextAdapter diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index ff8056d8..2ab3869e 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -196,3 +196,19 @@ type XSetStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` } + +type XSetControllerLabelEnum int + +const ( + EnumXSetControlledLabel XSetControllerLabelEnum = iota + + EnumXSetReplaceIndicationLabelKey + + EnumXSetReplacePairNewId + + EnumXSetReplacePairOriginName +) + +type XSetControllerLabelManager interface { + Get(labelType XSetControllerLabelEnum) string +} diff --git a/xset/synccontrols/inexclude.go b/xset/synccontrols/inexclude.go index 1f1aa4fc..bb07a9ec 100644 --- a/xset/synccontrols/inexclude.go +++ b/xset/synccontrols/inexclude.go @@ -20,16 +20,17 @@ import ( "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + + "kusionstack.io/kube-utils/xset/api" ) // AllowResourceExclude checks if pod or pvc is allowed to exclude -func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { +func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetControllerLabelManager) (bool, string) { labels := obj.GetLabels() // not controlled by ks manager if labels == nil { return false, "object's label is empty" - } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { + } else if val, exist := labels[manager.Get(api.EnumXSetControlledLabel)]; !exist || val != "true" { return false, "object is not controlled by kusionstack system" } @@ -41,14 +42,14 @@ func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string) (bool, } // AllowResourceInclude checks if pod or pvc is allowed to include -func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { +func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetControllerLabelManager) (bool, string) { labels := obj.GetLabels() ownerRefs := obj.GetOwnerReferences() // not controlled by ks manager if labels == nil { return false, "object's label is empty" - } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { + } else if val, exist := labels[manager.Get(api.EnumXSetControlledLabel)]; !exist || val != "true" { return false, "object is not controlled by kusionstack system" } diff --git a/xset/synccontrols/inexclude_test.go b/xset/synccontrols/inexclude_test.go index 7241b326..903690df 100644 --- a/xset/synccontrols/inexclude_test.go +++ b/xset/synccontrols/inexclude_test.go @@ -128,7 +128,7 @@ func TestAllowResourceExclude(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind) + got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind, NewXSetControllerLabelManager()) if got != tt.allow { t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) } @@ -288,7 +288,7 @@ func TestAllowResourceInclude(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind) + got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind, NewXSetControllerLabelManager()) if got != tt.allow { t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) } diff --git a/xset/synccontrols/label_manager.go b/xset/synccontrols/label_manager.go new file mode 100644 index 00000000..c847e7cf --- /dev/null +++ b/xset/synccontrols/label_manager.go @@ -0,0 +1,44 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +import ( + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + + "kusionstack.io/kube-utils/xset/api" +) + +var defaultXSetControllerLabelManager = map[api.XSetControllerLabelEnum]string{ + api.EnumXSetControlledLabel: appsv1alpha1.ControlledByKusionStackLabelKey, + api.EnumXSetReplaceIndicationLabelKey: appsv1alpha1.PodReplaceIndicationLabelKey, + api.EnumXSetReplacePairNewId: appsv1alpha1.PodReplacePairNewId, + api.EnumXSetReplacePairOriginName: appsv1alpha1.PodReplacePairOriginName, +} + +func NewXSetControllerLabelManager() api.XSetControllerLabelManager { + return &xSetControllerLabelManager{ + labelManager: defaultXSetControllerLabelManager, + } +} + +type xSetControllerLabelManager struct { + labelManager map[api.XSetControllerLabelEnum]string +} + +func (m *xSetControllerLabelManager) Get(key api.XSetControllerLabelEnum) string { + return m.labelManager[key] +} diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index ce4b4f16..1087284c 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -67,6 +67,10 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, if lifeCycleLabelManager == nil { lifeCycleLabelManager = opslifecycle.NewLabelManager(nil) } + xSetControllerLabelManager := xsetController.GetXSetControllerLabelManager() + if xSetControllerLabelManager == nil { + xSetControllerLabelManager = NewXSetControllerLabelManager() + } scaleInOpsLifecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() if scaleInOpsLifecycleAdapter == nil { scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} @@ -82,23 +86,25 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xsetGVK := xsetMeta.GroupVersionKind() updateConfig := &UpdateConfig{ - xsetController: xsetController, - client: reconcileMixIn.Client, - targetControl: xControl, - resourContextControl: resourceContexts, - recorder: reconcileMixIn.Recorder, + xsetController: xsetController, + client: reconcileMixIn.Client, + targetControl: xControl, + resourceContextControl: resourceContexts, + recorder: reconcileMixIn.Recorder, - opsLifecycleMgr: lifeCycleLabelManager, + opsLifecycleLabelMgr: lifeCycleLabelManager, scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, + xSetControllerLabelMgr: xSetControllerLabelManager, updateLifecycleAdapter: updateLifecycleAdapter, cacheExpectations: cacheExpectations, targetGVK: targetGVK, } return &RealSyncControl{ - ReconcilerMixin: *reconcileMixIn, - xsetController: xsetController, - resourContextControl: resourceContexts, - xControl: xControl, + ReconcilerMixin: *reconcileMixIn, + xsetController: xsetController, + xsetControllerLabelMgr: xSetControllerLabelManager, + resourceContextControl: resourceContexts, + xControl: xControl, updateConfig: updateConfig, cacheExpectations: cacheExpectations, @@ -114,9 +120,10 @@ var _ SyncControl = &RealSyncControl{} type RealSyncControl struct { mixin.ReconcilerMixin - xControl xcontrol.TargetControl - xsetController api.XSetController - resourContextControl resourcecontexts.ResourceContextControl + xControl xcontrol.TargetControl + xsetController api.XSetController + xsetControllerLabelMgr api.XSetControllerLabelManager + resourceContextControl resourcecontexts.ResourceContextControl updateConfig *UpdateConfig scaleInLifecycleAdapter api.LifecycleAdapter @@ -142,6 +149,8 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje return false, fmt.Errorf("fail to get filtered Targets: %w", err) } + // TODO list, adopt and retain pvcs pvcs (for pods) + toExcludeTargetNames, toIncludeTargetNames, err := r.dealIncludeExcludeTargets(ctx, instance, syncContext.FilteredTarget) if err != nil { return false, fmt.Errorf("fail to deal with include exclude targets: %s", err.Error()) @@ -150,7 +159,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje // get owned IDs var ownedIDs map[int]*api.ContextDetail if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - ownedIDs, err = r.resourContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(RealValue(xspec.Replicas))) + ownedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(RealValue(xspec.Replicas))) syncContext.OwnedIds = ownedIDs return err }); err != nil { @@ -175,7 +184,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje toDeleteTargetNames.Delete(xName) } if toExclude { - if targetDuringReplace(target) || toDelete { + if targetDuringReplace(r.xsetControllerLabelMgr, target) || toDelete { // skip exclude until replace and toDelete done toExcludeTargetNames.Delete(xName) } else { @@ -186,17 +195,19 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje if target.GetDeletionTimestamp() != nil { // 1. Reclaim ID from Target which is scaling in and terminating. - if contextDetail, exist := ownedIDs[id]; exist && r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { + if contextDetail, exist := ownedIDs[id]; exist && r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { idToReclaim.Insert(id) } - _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] + _, replaceIndicate := target.GetLabels()[r.xsetControllerLabelMgr.Get(api.EnumXSetReplaceIndicationLabelKey)] // 2. filter out Targets which are terminating and not replace indicate if !replaceIndicate { continue } } + // TODO delete unused pvcs (for pods) + targetWrappers = append(targetWrappers, targetWrapper{ Object: target, ID: id, @@ -206,8 +217,8 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje ToDelete: toDelete, ToExclude: toExclude, - IsDuringScaleInOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, target), - IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, target), + IsDuringScaleInOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, target), + IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, target), }) if id >= 0 { @@ -293,9 +304,8 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "DupExIncludedTarget", "duplicated targets %s in both excluding and including sets", strings.Join(intersection.List(), ", ")) } - // seem no need to check allow ResourceExclude, since filterTargets already filter only owned targets - toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude) - toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude) + toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude, r.xsetControllerLabelMgr) + toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude, r.xsetControllerLabelMgr) if notAllowedExcludeTargets.Len() > 0 { r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ExcludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to exclude, please find out the reason from target's event", notAllowedExcludeTargets.List())) } @@ -306,10 +316,10 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj } // checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude -type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string) (bool, string) +type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string, labelMgr api.XSetControllerLabelManager) (bool, string) // allowIncludeExcludeTargets try to classify targetNames to allowedTargets and notAllowedTargets, using checkAllowFunc func -func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc) (allowTargets, notAllowTargets sets.String, err error) { +func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc, labelMgr api.XSetControllerLabelManager) (allowTargets, notAllowTargets sets.String, err error) { allowTargets = sets.String{} notAllowTargets = sets.String{} for i := range targetNames { @@ -325,7 +335,7 @@ func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset a } // check allowance for target - if allowed, reason := fn(target, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind); !allowed { + if allowed, reason := fn(target, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind, labelMgr); !allowed { r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("target is not allowed to exclude/include from/to %s %s/%s: %s", r.xsetGVK.Kind, xset.GetNamespace(), xset.GetName(), reason)) notAllowTargets.Insert(targetName) @@ -341,8 +351,9 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject var err error var needUpdateContext bool var idToReclaim sets.Int + logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xsetObject)) - needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(syncContext.FilteredTarget) + needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(syncContext.FilteredTarget, logger) // delete origin targets for replace err = BatchDeleteTargetByLabel(ctx, r.xControl, needDeleteTargets) @@ -424,7 +435,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // find IDs and their contexts which have not been used by owned Targets - availableContext := r.resourContextControl.ExtractAvailableContexts(diff, syncContext.OwnedIds, targetInstanceIDSet) + availableContext := r.resourceContextControl.ExtractAvailableContexts(diff, syncContext.OwnedIds, targetInstanceIDSet) needUpdateContext := atomic.Bool{} succCount, err := controllerutils.SlowStartBatch(len(availableContext), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { availableIDContext := availableContext[i] @@ -435,7 +446,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, }() // use revision recorded in Context revision := syncContext.UpdatedRevision - if revisionName, exist := r.resourContextControl.Get(availableIDContext, api.EnumRevisionContextDataKey); exist && revisionName != "" { + if revisionName, exist := r.resourceContextControl.Get(availableIDContext, api.EnumRevisionContextDataKey); exist && revisionName != "" { for i := range syncContext.Revisions { if syncContext.Revisions[i].GetName() == revisionName { revision = syncContext.Revisions[i] @@ -461,7 +472,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, if needUpdateContext.Load() { logger.V(1).Info("try to update ResourceContext for XSet after scaling out") if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) + return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); updateContextErr != nil { err = errors.Join(updateContextErr, err) } @@ -497,7 +508,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // trigger TargetOpsLifecycle with scaleIn OperationType logger.V(1).Info("try to begin TargetOpsLifecycle for scaling in Target in XSet", "wrapper", ObjectKeyString(object)) // todo switch to x opslifecycle - if updated, err := opslifecycle.Begin(r.updateConfig.opsLifecycleMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { + if updated, err := opslifecycle.Begin(r.updateConfig.opsLifecycleLabelMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { return fmt.Errorf("fail to begin TargetOpsLifecycle for Scaling in Target %s/%s: %w", object.GetNamespace(), object.GetName(), err) } else if updated { r.Recorder.Eventf(object, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin TargetOpsLifecycle for scaling in") @@ -520,7 +531,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, needUpdateContext := false for i, targetWrapper := range targetsToScaleIn { - requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) + requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) if !allowed && targetWrapper.Object.GetDeletionTimestamp() == nil { r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "Target is not allowed to scale in") continue @@ -536,9 +547,9 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // if Target is allowed to operate or Target has already been deleted, promte to delete Target - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && !r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && !r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { needUpdateContext = true - r.resourContextControl.Put(contextDetail, api.EnumScaleInContextDataKey, "true") + r.resourceContextControl.Put(contextDetail, api.EnumScaleInContextDataKey, "true") } if targetWrapper.GetDeletionTimestamp() != nil { @@ -552,7 +563,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, if needUpdateContext { logger.V(1).Info("try to update ResourceContext for XSet when scaling in Target") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) + return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); err != nil { AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) return scaling, recordedRequeueAfter, err @@ -590,17 +601,17 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // reset ContextDetail.ScalingIn, if there are Targets had its TargetOpsLifecycle reverted needUpdateTargetContext := false for _, targetWrapper := range syncContext.activeTargets { - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") && - !opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, targetWrapper) { + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") && + !opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, targetWrapper) { needUpdateTargetContext = true - r.resourContextControl.Remove(contextDetail, api.EnumScaleInContextDataKey) + r.resourceContextControl.Remove(contextDetail, api.EnumScaleInContextDataKey) } } if needUpdateTargetContext { logger.V(1).Info("try to update ResourceContext for XSet after scaling") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) + return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); err != nil { return scaling, recordedRequeueAfter, fmt.Errorf("fail to reset ResourceContext: %w", err) } @@ -641,7 +652,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, continue } - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, targetInfo) { + if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, targetInfo) { continue } @@ -761,8 +772,8 @@ func (r *RealSyncControl) CalculateStatus(_ context.Context, instance api.XSetOb updatedReplicas++ } - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, targetWrapper) || - opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, targetWrapper) { + if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, targetWrapper) || + opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, targetWrapper) { operatingReplicas++ } @@ -813,7 +824,7 @@ func (r *RealSyncControl) getAvailableTargetIDs( ownedIDs := syncContext.OwnedIds currentIDs := syncContext.CurrentIDs - availableContexts := r.resourContextControl.ExtractAvailableContexts(want, ownedIDs, currentIDs) + availableContexts := r.resourceContextControl.ExtractAvailableContexts(want, ownedIDs, currentIDs) if len(availableContexts) >= want { return availableContexts, ownedIDs, nil } @@ -823,13 +834,13 @@ func (r *RealSyncControl) getAvailableTargetIDs( var newOwnedIDs map[int]*api.ContextDetail var err error if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - newOwnedIDs, err = r.resourContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), len(ownedIDs)+diff) + newOwnedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), len(ownedIDs)+diff) return err }); err != nil { return nil, ownedIDs, fmt.Errorf("fail to allocate IDs using context when include Targets: %w", err) } - return r.resourContextControl.ExtractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil + return r.resourceContextControl.ExtractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil } // reclaimOwnedIDs delete and reclaim unused IDs @@ -848,7 +859,7 @@ func (r *RealSyncControl) reclaimOwnedIDs( if _, exist := currentIDs[id]; exist { continue } - if r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { + if r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { idToReclaim.Insert(id) } } @@ -866,7 +877,7 @@ func (r *RealSyncControl) reclaimOwnedIDs( logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xset)) logger.V(1).Info("try to update ResourceContext for XSet when sync") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xset, ownedIDs) + return r.resourceContextControl.UpdateToTargetContext(ctx, xset, ownedIDs) }); err != nil { return fmt.Errorf("fail to update ResourceContextControl when reclaiming IDs: %w", err) } @@ -886,14 +897,14 @@ func FilterOutActiveTargetWrappers(targets []targetWrapper) []*targetWrapper { return filteredTargetWrappers } -func targetDuringReplace(target client.Object) bool { +func targetDuringReplace(labelMgr api.XSetControllerLabelManager, target client.Object) bool { labels := target.GetLabels() if labels == nil { return false } - _, replaceIndicate := labels[TargetReplaceIndicationLabelKey] - _, replaceOriginTarget := labels[TargetReplacePairNewId] - _, replaceNewTarget := labels[TargetReplacePairOriginName] + _, replaceIndicate := labels[labelMgr.Get(api.EnumXSetReplaceIndicationLabelKey)] + _, replaceOriginTarget := labels[labelMgr.Get(api.EnumXSetReplacePairOriginName)] + _, replaceNewTarget := labels[labelMgr.Get(api.EnumXSetReplacePairNewId)] return replaceIndicate || replaceOriginTarget || replaceNewTarget } @@ -916,21 +927,21 @@ func BatchDeleteTargetByLabel(ctx context.Context, targetControl xcontrol.Target func (r *RealSyncControl) decideContextRevision(contextDetail *api.ContextDetail, updatedRevision *appsv1.ControllerRevision, createSucceeded bool) bool { needUpdateContext := false if !createSucceeded { - if r.resourContextControl.Contains(contextDetail, api.EnumJustCreateContextDataKey, "true") { + if r.resourceContextControl.Contains(contextDetail, api.EnumJustCreateContextDataKey, "true") { // TODO choose just create targets' revision according to scaleStrategy - r.resourContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) - r.resourContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) + r.resourceContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) + r.resourceContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) needUpdateContext = true - } else if r.resourContextControl.Contains(contextDetail, api.EnumRecreateUpdateContextDataKey, "true") { - r.resourContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) - r.resourContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) + } else if r.resourceContextControl.Contains(contextDetail, api.EnumRecreateUpdateContextDataKey, "true") { + r.resourceContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) + r.resourceContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) needUpdateContext = true } // if target is delete and recreate, never change revisionKey } else { // TODO delete ID if create succeeded - r.resourContextControl.Remove(contextDetail, api.EnumJustCreateContextDataKey) - r.resourContextControl.Remove(contextDetail, api.EnumRecreateUpdateContextDataKey) + r.resourceContextControl.Remove(contextDetail, api.EnumJustCreateContextDataKey) + r.resourceContextControl.Remove(contextDetail, api.EnumRecreateUpdateContextDataKey) needUpdateContext = true } return needUpdateContext diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index b8c16481..66fb572d 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -139,10 +140,10 @@ func (r *RealSyncControl) replaceOriginTargets( newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId ownedIDs[originTargetId].Put(ReplaceNewTargetIDContextDataKey, instanceId) ownedIDs[newTargetContext.ID].Put(ReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) - r.resourContextControl.Remove(ownedIDs[newTargetContext.ID], api.EnumJustCreateContextDataKey) + r.resourceContextControl.Remove(ownedIDs[newTargetContext.ID], api.EnumJustCreateContextDataKey) } newTarget.GetLabels()[TargetReplacePairOriginName] = originTarget.GetName() - r.resourContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) + r.resourceContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) if newCreatedTarget, err := r.xControl.CreateTarget(ctx, newTarget); err == nil { r.Recorder.Eventf(originTarget, @@ -174,7 +175,9 @@ func (r *RealSyncControl) replaceOriginTargets( return successCount, err } -func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needReplaceTargets, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object) { +func (r *RealSyncControl) dealReplaceTargets(targets []client.Object, logger logr.Logger) ( + needReplaceTargets, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object, +) { targetInstanceIdMap := make(map[string]client.Object) targetNameMap := make(map[string]client.Object) @@ -197,7 +200,8 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needRepla } // origin target is about to scaleIn, skip replace - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, target) { + if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, target) { + logger.Info("dealReplaceTargets", "target is during scaleIn ops lifecycle, skip replacing", target.GetName()) continue } @@ -230,12 +234,12 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needRepla needCleanLabels = append(needCleanLabels, TargetReplacePairOriginName) } else if originTarget.GetLabels()[TargetReplaceIndicationLabelKey] == "" { // replace canceled, delete replace new target if new target is not service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); !serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); !serviceAvailable { needDeleteTargets = append(needDeleteTargets, target) } } else if !replaceByUpdate { // not replace update, delete origin target when new created target is service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); serviceAvailable { needDeleteTargets = append(needDeleteTargets, originTarget) } } diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 34819506..6dd854d0 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -61,7 +61,7 @@ func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, r continue } // when scaleIn origin Target, newTarget should be deleted if not service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); !serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); !serviceAvailable { needDeleteTargets = append(needDeleteTargets, replacePairTarget) } } diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 253d7383..c68515dd 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -91,7 +91,7 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync spec := r.xsetController.GetXSetSpec(xsetObject) // decide whether the TargetOpsLifecycle is during ops or not updateInfo.IsDuringOps = target.IsDuringUpdateOps - updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) + updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) targetUpdateInfoList[i] = updateInfo } @@ -133,7 +133,7 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync InPlaceUpdateSupport: true, UpdateRevision: syncContext.UpdatedRevision, } - if revision, exist := r.resourContextControl.Get(target.ContextDetail, api.EnumRevisionContextDataKey); exist && + if revision, exist := r.resourceContextControl.Get(target.ContextDetail, api.EnumRevisionContextDataKey); exist && revision == syncContext.UpdatedRevision.GetName() { updateInfo.IsUpdatedRevision = true } @@ -276,15 +276,16 @@ func (o *orderByDefault) Less(i, j int) bool { } type UpdateConfig struct { - xsetController api.XSetController - client client.Client - targetControl xcontrol.TargetControl - resourContextControl resourcecontexts.ResourceContextControl - recorder record.EventRecorder + xsetController api.XSetController + client client.Client + targetControl xcontrol.TargetControl + resourceContextControl resourcecontexts.ResourceContextControl + recorder record.EventRecorder - opsLifecycleMgr api.LifeCycleLabelManager + opsLifecycleLabelMgr api.LifeCycleLabelManager scaleInLifecycleAdapter api.LifecycleAdapter updateLifecycleAdapter api.LifecycleAdapter + xSetControllerLabelMgr api.XSetControllerLabelManager cacheExpectations expectations.CacheExpectationsInterface targetGVK schema.GroupVersionKind @@ -316,9 +317,9 @@ func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext targetInfo := <-targetCh u.recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "TargetUpdateLifecycle", "try to begin TargetOpsLifecycle for updating Target of XSet") - if updated, err := opslifecycle.BeginWithCleaningOld(u.opsLifecycleMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { + if updated, err := opslifecycle.BeginWithCleaningOld(u.opsLifecycleLabelMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { if !targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport { - return opslifecycle.WhenBeginDelete(u.opsLifecycleMgr, obj) + return opslifecycle.WhenBeginDelete(u.opsLifecycleLabelMgr, obj) } return false, nil }); err != nil { @@ -373,9 +374,9 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid continue } - if !u.resourContextControl.Contains(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) { + if !u.resourceContextControl.Contains(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) { needUpdateContext = true - u.resourContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) + u.resourceContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) } spec := u.xsetController.GetXSetSpec(u.OwnerObject) @@ -383,7 +384,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid // mark targetContext "TargetRecreateUpgrade" if upgrade by recreate isRecreateUpdatePolicy := spec.UpdateStrategy.UpdatePolicy == api.XSetRecreateTargetUpdateStrategyType if (!targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport) || isRecreateUpdatePolicy { - u.resourContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRecreateUpdateContextDataKey, "true") + u.resourceContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRecreateUpdateContextDataKey, "true") } if targetInfo.PlaceHolder { @@ -397,7 +398,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid if needUpdateContext { u.recorder.Eventf(u.OwnerObject, corev1.EventTypeNormal, "UpdateToTargetContext", "try to update ResourceContext for XSet") err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return u.resourContextControl.UpdateToTargetContext(ctx, u.OwnerObject, ownedIDs) + return u.resourceContextControl.UpdateToTargetContext(ctx, u.OwnerObject, ownedIDs) }) return recordedRequeueAfter, err } @@ -405,7 +406,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid } func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo) error { - if updated, err := opslifecycle.Finish(u.opsLifecycleMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { + if updated, err := opslifecycle.Finish(u.opsLifecycleLabelMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { return fmt.Errorf("failed to finish TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) } else if updated { // add an expectation for this target update, before next reconciling @@ -671,7 +672,7 @@ func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *targe return false, "replace origin target", nil } - if serviceAvailable := opslifecycle.IsServiceAvailable(u.opsLifecycleMgr, targetInfo.Object); serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(u.opsLifecycleLabelMgr, targetInfo.Object); serviceAvailable { return true, "", nil } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index 1cad1f63..bfec8f7c 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -24,7 +24,6 @@ import ( appsv1 "k8s.io/api/apps/v1" apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "kusionstack.io/kube-api/apps/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-utils/xset/api" @@ -64,7 +63,7 @@ func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revis labels := targetObj.GetLabels() labels[TargetInstanceIDLabelKey] = fmt.Sprintf("%d", id) labels[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() - controlByKusionStack(targetObj) + controlByXSet(setController, targetObj) for _, fn := range updateFuncs { if err := fn(targetObj); err != nil { @@ -171,12 +170,29 @@ func filterOutCondition(conditions []metav1.Condition, condType string) []metav1 return newConditions } -func controlByKusionStack(obj client.Object) { +func controlByXSet(setController api.XSetController, obj client.Object) { if obj.GetLabels() == nil { obj.SetLabels(map[string]string{}) } + controlLabel := setController.GetXSetControllerLabelManager() + if controlLabel == nil { + controlLabel = NewXSetControllerLabelManager() + } + + if v, ok := obj.GetLabels()[controlLabel.Get(api.EnumXSetControlledLabel)]; !ok || v != "true" { + obj.GetLabels()[controlLabel.Get(api.EnumXSetControlledLabel)] = "true" + } +} +func IsControlledByXSet(setController api.XSetController, obj client.Object) bool { + if obj.GetLabels() == nil { + return false + } - if v, ok := obj.GetLabels()[v1alpha1.ControlledByKusionStackLabelKey]; !ok || v != "true" { - obj.GetLabels()[v1alpha1.ControlledByKusionStackLabelKey] = "true" + controlLabel := setController.GetXSetControllerLabelManager() + if controlLabel == nil { + controlLabel = NewXSetControllerLabelManager() } + + v, ok := obj.GetLabels()[controlLabel.Get(api.EnumXSetControlledLabel)] + return ok && v == "true" } diff --git a/xset/xset_controller.go b/xset/xset_controller.go index 7653dd08..258b0319 100644 --- a/xset/xset_controller.go +++ b/xset/xset_controller.go @@ -29,7 +29,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -114,6 +116,19 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error if err := c.Watch(&source.Kind{Type: xsetController.NewXObject()}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: xsetController.NewXSetObject(), + }, predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return synccontrols.IsControlledByXSet(xsetController, event.Object) + }, + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return synccontrols.IsControlledByXSet(xsetController, updateEvent.ObjectNew) || synccontrols.IsControlledByXSet(xsetController, updateEvent.ObjectOld) + }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { + return synccontrols.IsControlledByXSet(xsetController, deleteEvent.Object) + }, + GenericFunc: func(genericEvent event.GenericEvent) bool { + return synccontrols.IsControlledByXSet(xsetController, genericEvent.Object) + }, }); err != nil { return fmt.Errorf("failed to watch %s: %s", targetMeta.Kind, err.Error()) } From 87c3fa9ae34dc55c0f5bcb861737a6ee859807d4 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 26 Aug 2025 17:41:52 +0800 Subject: [PATCH 02/17] sync kuperator code --- xset/api/resourcecontext_types.go | 2 + xset/api/xset_controller_types.go | 2 +- xset/api/xset_types.go | 24 +++- xset/resourcecontexts/default_adapters.go | 14 +- xset/synccontrols/inexclude.go | 8 +- xset/synccontrols/label_manager.go | 35 ++++- xset/synccontrols/sync_control.go | 168 ++++++++++++---------- xset/synccontrols/types.go | 9 +- xset/synccontrols/x_replace.go | 127 +++++++++------- xset/synccontrols/x_scale.go | 16 ++- xset/synccontrols/x_update.go | 99 ++++++------- xset/synccontrols/x_utils.go | 35 ++--- xset/xset_controller.go | 22 ++- 13 files changed, 324 insertions(+), 237 deletions(-) diff --git a/xset/api/resourcecontext_types.go b/xset/api/resourcecontext_types.go index 67f3b5e9..7ec50b03 100644 --- a/xset/api/resourcecontext_types.go +++ b/xset/api/resourcecontext_types.go @@ -45,6 +45,8 @@ const ( EnumJustCreateContextDataKey EnumRecreateUpdateContextDataKey EnumScaleInContextDataKey + EnumReplaceNewTargetIDContextDataKey + EnumReplaceOriginTargetIDContextDataKey ) // ResourceContextSpec defines the desired state of ResourceContext diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index 30147510..6010b07c 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -40,7 +40,7 @@ type XSetController interface { SetXSetStatus(object XSetObject, status *XSetStatus) GetLifeCycleLabelManager() LifeCycleLabelManager - GetXSetControllerLabelManager() XSetControllerLabelManager + GetXSetControllerLabelManager() XSetLabelManager GetScaleInOpsLifecycleAdapter() LifecycleAdapter GetUpdateOpsLifecycleAdapter() LifecycleAdapter GetResourceContextAdapter() ResourceContextAdapter diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index 2ab3869e..28da50ee 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -202,13 +202,27 @@ type XSetControllerLabelEnum int const ( EnumXSetControlledLabel XSetControllerLabelEnum = iota - EnumXSetReplaceIndicationLabelKey + EnumXSetInstanceIdLabel - EnumXSetReplacePairNewId + EnumXSetUpdateIndicationLabel - EnumXSetReplacePairOriginName + EnumXSetDeletionIndicationLabel + + EnumXSetReplaceIndicationLabel + + EnumXSetReplacePairNewIdLabel + + EnumXSetReplacePairOriginNameLabel + + EnumXSetReplaceByReplaceUpdateLabel + + EnumXSetOrphanedLabel + + EnumXSetTargetCreatingLabel ) -type XSetControllerLabelManager interface { - Get(labelType XSetControllerLabelEnum) string +type XSetLabelManager interface { + Get(labels map[string]string, labelType XSetControllerLabelEnum) (string, bool) + Set(labels map[string]string, labelType XSetControllerLabelEnum, value string) + Label(labelType XSetControllerLabelEnum) string } diff --git a/xset/resourcecontexts/default_adapters.go b/xset/resourcecontexts/default_adapters.go index ad852de4..d4894ab5 100644 --- a/xset/resourcecontexts/default_adapters.go +++ b/xset/resourcecontexts/default_adapters.go @@ -26,12 +26,14 @@ import ( var _ api.ResourceContextAdapter = &DefaultResourceContextAdapter{} var defaultResourceContextKeys = map[api.ResourceContextKeyEnum]string{ - api.EnumOwnerContextKey: "Owner", - api.EnumRevisionContextDataKey: "Revision", - api.EnumTargetDecorationRevisionKey: "TargetDecorationRevisions", - api.EnumJustCreateContextDataKey: "TargetJustCreate", - api.EnumRecreateUpdateContextDataKey: "TargetRecreateUpdate", - api.EnumScaleInContextDataKey: "ScaleIn", + api.EnumOwnerContextKey: "Owner", + api.EnumRevisionContextDataKey: "Revision", + api.EnumTargetDecorationRevisionKey: "TargetDecorationRevisions", + api.EnumJustCreateContextDataKey: "TargetJustCreate", + api.EnumRecreateUpdateContextDataKey: "TargetRecreateUpdate", + api.EnumScaleInContextDataKey: "ScaleIn", + api.EnumReplaceNewTargetIDContextDataKey: "ReplaceNewTargetID", + api.EnumReplaceOriginTargetIDContextDataKey: "ReplaceOriginTargetID", } // DefaultResourceContextAdapter is the adapter to api apps.kusionstack.io.resourcecontexts diff --git a/xset/synccontrols/inexclude.go b/xset/synccontrols/inexclude.go index bb07a9ec..ce1e294e 100644 --- a/xset/synccontrols/inexclude.go +++ b/xset/synccontrols/inexclude.go @@ -25,12 +25,12 @@ import ( ) // AllowResourceExclude checks if pod or pvc is allowed to exclude -func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetControllerLabelManager) (bool, string) { +func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetLabelManager) (bool, string) { labels := obj.GetLabels() // not controlled by ks manager if labels == nil { return false, "object's label is empty" - } else if val, exist := labels[manager.Get(api.EnumXSetControlledLabel)]; !exist || val != "true" { + } else if val, exist := manager.Get(labels, api.EnumXSetControlledLabel); !exist || val != "true" { return false, "object is not controlled by kusionstack system" } @@ -42,14 +42,14 @@ func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string, manage } // AllowResourceInclude checks if pod or pvc is allowed to include -func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetControllerLabelManager) (bool, string) { +func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetLabelManager) (bool, string) { labels := obj.GetLabels() ownerRefs := obj.GetOwnerReferences() // not controlled by ks manager if labels == nil { return false, "object's label is empty" - } else if val, exist := labels[manager.Get(api.EnumXSetControlledLabel)]; !exist || val != "true" { + } else if val, exist := manager.Get(labels, api.EnumXSetControlledLabel); !exist || val != "true" { return false, "object is not controlled by kusionstack system" } diff --git a/xset/synccontrols/label_manager.go b/xset/synccontrols/label_manager.go index c847e7cf..9b41c711 100644 --- a/xset/synccontrols/label_manager.go +++ b/xset/synccontrols/label_manager.go @@ -23,13 +23,19 @@ import ( ) var defaultXSetControllerLabelManager = map[api.XSetControllerLabelEnum]string{ - api.EnumXSetControlledLabel: appsv1alpha1.ControlledByKusionStackLabelKey, - api.EnumXSetReplaceIndicationLabelKey: appsv1alpha1.PodReplaceIndicationLabelKey, - api.EnumXSetReplacePairNewId: appsv1alpha1.PodReplacePairNewId, - api.EnumXSetReplacePairOriginName: appsv1alpha1.PodReplacePairOriginName, + api.EnumXSetControlledLabel: appsv1alpha1.ControlledByKusionStackLabelKey, + api.EnumXSetInstanceIdLabel: appsv1alpha1.PodInstanceIDLabelKey, + api.EnumXSetUpdateIndicationLabel: appsv1alpha1.CollaSetUpdateIndicateLabelKey, + api.EnumXSetDeletionIndicationLabel: appsv1alpha1.PodDeletionIndicationLabelKey, + api.EnumXSetReplaceIndicationLabel: appsv1alpha1.PodReplaceIndicationLabelKey, + api.EnumXSetReplacePairNewIdLabel: appsv1alpha1.PodReplacePairNewId, + api.EnumXSetReplacePairOriginNameLabel: appsv1alpha1.PodReplacePairOriginName, + api.EnumXSetReplaceByReplaceUpdateLabel: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, + api.EnumXSetOrphanedLabel: appsv1alpha1.PodOrphanedIndicateLabelKey, + api.EnumXSetTargetCreatingLabel: appsv1alpha1.PodCreatingLabel, } -func NewXSetControllerLabelManager() api.XSetControllerLabelManager { +func NewXSetControllerLabelManager() api.XSetLabelManager { return &xSetControllerLabelManager{ labelManager: defaultXSetControllerLabelManager, } @@ -39,6 +45,23 @@ type xSetControllerLabelManager struct { labelManager map[api.XSetControllerLabelEnum]string } -func (m *xSetControllerLabelManager) Get(key api.XSetControllerLabelEnum) string { +func (m *xSetControllerLabelManager) Get(labels map[string]string, key api.XSetControllerLabelEnum) (string, bool) { + if labels == nil { + return "", false + } + labelKey := m.labelManager[key] + val, exist := labels[labelKey] + return val, exist +} + +func (m *xSetControllerLabelManager) Set(labels map[string]string, key api.XSetControllerLabelEnum, val string) { + if labels == nil { + labels = make(map[string]string) + } + labelKey := m.labelManager[key] + labels[labelKey] = val +} + +func (m *xSetControllerLabelManager) Label(key api.XSetControllerLabelEnum) string { return m.labelManager[key] } diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 1087284c..4a655518 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -24,6 +24,7 @@ import ( "sync/atomic" "time" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" clientutil "kusionstack.io/kube-utils/client" @@ -55,11 +56,14 @@ type SyncControl interface { Update(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) CalculateStatus(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus + + BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error } func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xsetController api.XSetController, xControl xcontrol.TargetControl, + xsetLabelManager api.XSetLabelManager, resourceContexts resourcecontexts.ResourceContextControl, cacheExpectations expectations.CacheExpectationsInterface, ) SyncControl { @@ -67,10 +71,6 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, if lifeCycleLabelManager == nil { lifeCycleLabelManager = opslifecycle.NewLabelManager(nil) } - xSetControllerLabelManager := xsetController.GetXSetControllerLabelManager() - if xSetControllerLabelManager == nil { - xSetControllerLabelManager = NewXSetControllerLabelManager() - } scaleInOpsLifecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() if scaleInOpsLifecycleAdapter == nil { scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} @@ -87,6 +87,7 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, updateConfig := &UpdateConfig{ xsetController: xsetController, + xsetLabelMgr: xsetLabelManager, client: reconcileMixIn.Client, targetControl: xControl, resourceContextControl: resourceContexts, @@ -94,7 +95,6 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, opsLifecycleLabelMgr: lifeCycleLabelManager, scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, - xSetControllerLabelMgr: xSetControllerLabelManager, updateLifecycleAdapter: updateLifecycleAdapter, cacheExpectations: cacheExpectations, targetGVK: targetGVK, @@ -102,7 +102,7 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, return &RealSyncControl{ ReconcilerMixin: *reconcileMixIn, xsetController: xsetController, - xsetControllerLabelMgr: xSetControllerLabelManager, + xsetLabelMgr: xsetLabelManager, resourceContextControl: resourceContexts, xControl: xControl, @@ -122,7 +122,7 @@ type RealSyncControl struct { mixin.ReconcilerMixin xControl xcontrol.TargetControl xsetController api.XSetController - xsetControllerLabelMgr api.XSetControllerLabelManager + xsetLabelMgr api.XSetLabelManager resourceContextControl resourcecontexts.ResourceContextControl updateConfig *UpdateConfig @@ -159,7 +159,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje // get owned IDs var ownedIDs map[int]*api.ContextDetail if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - ownedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(RealValue(xspec.Replicas))) + ownedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(ptr.Deref(xspec.Replicas, 0))) syncContext.OwnedIds = ownedIDs return err }); err != nil { @@ -184,7 +184,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje toDeleteTargetNames.Delete(xName) } if toExclude { - if targetDuringReplace(r.xsetControllerLabelMgr, target) || toDelete { + if targetDuringReplace(r.xsetLabelMgr, target) || toDelete { // skip exclude until replace and toDelete done toExcludeTargetNames.Delete(xName) } else { @@ -199,7 +199,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje idToReclaim.Insert(id) } - _, replaceIndicate := target.GetLabels()[r.xsetControllerLabelMgr.Get(api.EnumXSetReplaceIndicationLabelKey)] + _, replaceIndicate := r.xsetLabelMgr.Get(target.GetLabels(), api.EnumXSetReplaceIndicationLabel) // 2. filter out Targets which are terminating and not replace indicate if !replaceIndicate { continue @@ -259,9 +259,6 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje syncContext.TargetWrappers = targetWrappers syncContext.OwnedIds = ownedIDs - syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) - syncContext.replacingMap = classifyTargetReplacingMapping(syncContext.activeTargets) - return inExSucceed, nil } @@ -304,8 +301,8 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "DupExIncludedTarget", "duplicated targets %s in both excluding and including sets", strings.Join(intersection.List(), ", ")) } - toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude, r.xsetControllerLabelMgr) - toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude, r.xsetControllerLabelMgr) + toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude, r.xsetLabelMgr) + toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude, r.xsetLabelMgr) if notAllowedExcludeTargets.Len() > 0 { r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ExcludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to exclude, please find out the reason from target's event", notAllowedExcludeTargets.List())) } @@ -316,10 +313,10 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj } // checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude -type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string, labelMgr api.XSetControllerLabelManager) (bool, string) +type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string, labelMgr api.XSetLabelManager) (bool, string) // allowIncludeExcludeTargets try to classify targetNames to allowedTargets and notAllowedTargets, using checkAllowFunc func -func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc, labelMgr api.XSetControllerLabelManager) (allowTargets, notAllowTargets sets.String, err error) { +func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc, labelMgr api.XSetLabelManager) (allowTargets, notAllowTargets sets.String, err error) { allowTargets = sets.String{} notAllowTargets = sets.String{} for i := range targetNames { @@ -351,12 +348,16 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject var err error var needUpdateContext bool var idToReclaim sets.Int - logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xsetObject)) - needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(syncContext.FilteredTarget, logger) + defer func() { + syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) + syncContext.replacingMap = classifyTargetReplacingMapping(syncContext.activeTargets) + }() + + needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(ctx, syncContext.FilteredTarget) // delete origin targets for replace - err = BatchDeleteTargetByLabel(ctx, r.xControl, needDeleteTargets) + err = r.BatchDeleteTargetsByLabel(ctx, r.xControl, needDeleteTargets) if err != nil { r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", "delete targets by label with error: %s", err.Error()) return err @@ -409,17 +410,17 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { spec := r.xsetController.GetXSetSpec(xsetObject) - logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xsetObject)) + logger := logr.FromContext(ctx) var recordedRequeueAfter *time.Duration - diff := int(RealValue(spec.Replicas)) - len(syncContext.replacingMap) + diff := int(ptr.Deref(spec.Replicas, 0)) - len(syncContext.replacingMap) scaling := false if diff >= 0 { // trigger delete targets indicated in ScaleStrategy.TargetToDelete by label for _, targetWrapper := range syncContext.activeTargets { if targetWrapper.ToDelete { - err := BatchDeleteTargetByLabel(ctx, r.xControl, []client.Object{targetWrapper.Object}) + err := r.BatchDeleteTargetsByLabel(ctx, r.xControl, []client.Object{targetWrapper.Object}) if err != nil { return false, recordedRequeueAfter, err } @@ -435,10 +436,16 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // find IDs and their contexts which have not been used by owned Targets - availableContext := r.resourceContextControl.ExtractAvailableContexts(diff, syncContext.OwnedIds, targetInstanceIDSet) + var availableContexts []*api.ContextDetail + var getErr error + availableContexts, syncContext.OwnedIds, getErr = r.getAvailableTargetIDs(ctx, diff, xsetObject, syncContext) + if getErr != nil { + return false, recordedRequeueAfter, getErr + } + needUpdateContext := atomic.Bool{} - succCount, err := controllerutils.SlowStartBatch(len(availableContext), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { - availableIDContext := availableContext[i] + succCount, err := controllerutils.SlowStartBatch(len(availableContexts), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { + availableIDContext := availableContexts[i] defer func() { if r.decideContextRevision(availableIDContext, syncContext.UpdatedRevision, err == nil) { needUpdateContext.Store(true) @@ -456,13 +463,15 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // scale out new Targets with updatedRevision // TODO use cache - target, err := NewTargetFrom(r.xsetController, xsetObject, revision, availableIDContext.ID) + // TODO decoration for target template + target, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, xsetObject, revision, availableIDContext.ID) if err != nil { return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) } + // TODO create pvcs for targets (pod) newTarget := target.DeepCopyObject().(client.Object) - logger.V(1).Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) + logger.Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) if target, err = r.xControl.CreateTarget(ctx, newTarget); err != nil { return err } @@ -470,7 +479,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, return r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName()) }) if needUpdateContext.Load() { - logger.V(1).Info("try to update ResourceContext for XSet after scaling out") + logger.Info("try to update ResourceContext for XSet after scaling out", "Context", syncContext.OwnedIds) if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); updateContextErr != nil { @@ -484,13 +493,12 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleOut", "scale out %d Target(s)", succCount) AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") return succCount > 0, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") - return false, nil, nil } - } else if diff < 0 { + } + + if diff <= 0 { // chose the targets to scale in - targetsToScaleIn := r.getTargetsToDelete(syncContext.activeTargets, syncContext.replacingMap, diff*-1) + targetsToScaleIn := r.getTargetsToDelete(xsetObject, syncContext.activeTargets, syncContext.replacingMap, diff*-1) // filter out Targets need to trigger TargetOpsLifecycle wrapperCh := make(chan *targetWrapper, len(targetsToScaleIn)) for i := range targetsToScaleIn { @@ -507,10 +515,10 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // trigger TargetOpsLifecycle with scaleIn OperationType logger.V(1).Info("try to begin TargetOpsLifecycle for scaling in Target in XSet", "wrapper", ObjectKeyString(object)) - // todo switch to x opslifecycle if updated, err := opslifecycle.Begin(r.updateConfig.opsLifecycleLabelMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { return fmt.Errorf("fail to begin TargetOpsLifecycle for Scaling in Target %s/%s: %w", object.GetNamespace(), object.GetName(), err) } else if updated { + wrapper.IsDuringScaleInOps = true r.Recorder.Eventf(object, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin TargetOpsLifecycle for scaling in") // add an expectation for this wrapper creation, before next reconciling if err := r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, object.GetNamespace(), object.GetName(), object.GetResourceVersion()); err != nil { @@ -520,7 +528,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, return nil }) - scaling = succCount != 0 + scaling = succCount > 0 if err != nil { AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", err.Error()) @@ -531,7 +539,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, needUpdateContext := false for i, targetWrapper := range targetsToScaleIn { - requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) + requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, ptr.Deref(spec.ScaleStrategy.OperationDelaySeconds, 0), targetWrapper.Object) if !allowed && targetWrapper.Object.GetDeletionTimestamp() == nil { r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "Target is not allowed to scale in") continue @@ -561,7 +569,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // mark these Targets to scalingIn if needUpdateContext { - logger.V(1).Info("try to update ResourceContext for XSet when scaling in Target") + logger.Info("try to update ResourceContext for XSet when scaling in Target") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); err != nil { @@ -575,12 +583,13 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // do delete Target resource succCount, err = controllerutils.SlowStartBatch(len(wrapperCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { target := <-wrapperCh - logger.V(1).Info("try to scale in Target", "target", ObjectKeyString(target)) + logger.Info("try to scale in Target", "target", ObjectKeyString(target)) if err := r.xControl.DeleteTarget(ctx, target.Object); err != nil { return fmt.Errorf("fail to delete Target %s/%s when scaling in: %w", target.GetNamespace(), target.GetName(), err) } r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "TargetDeleted", "succeed to scale in Target %s/%s", target.GetNamespace(), target.GetName()) + // TODO delete pvcs if target is in update replace, or retention policy is "Deleted" return r.cacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName()) }) scaling = scaling || succCount > 0 @@ -594,15 +603,13 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } else { AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") } - - return scaling, recordedRequeueAfter, err } // reset ContextDetail.ScalingIn, if there are Targets had its TargetOpsLifecycle reverted needUpdateTargetContext := false for _, targetWrapper := range syncContext.activeTargets { - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") && - !opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, targetWrapper) { + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && + r.resourceContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") && !targetWrapper.IsDuringScaleInOps { needUpdateTargetContext = true r.resourceContextControl.Remove(contextDetail, api.EnumScaleInContextDataKey) } @@ -617,18 +624,22 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } } + if !scaling { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "Scaled", "") + } + return scaling, recordedRequeueAfter, nil } func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { - logger := r.Logger.WithValues("xset", ObjectKeyString(xsetObject)) + logger := logr.FromContext(ctx) var err error var recordedRequeueAfter *time.Duration // 1. scan and analysis targets update info for active targets and PlaceHolder targets targetUpdateInfos := r.attachTargetUpdateInfo(xsetObject, syncContext) // 2. decide Target update candidates - candidates := decideTargetToUpdate(r.xsetController, xsetObject, targetUpdateInfos) + candidates := r.decideTargetToUpdate(r.xsetController, xsetObject, targetUpdateInfos) targetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) targetCh := make(chan *targetUpdateInfo, len(targetToUpdate)) updater := r.newTargetUpdater(xsetObject) @@ -636,6 +647,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, // 3. filter already updated revision, for i, targetInfo := range targetToUpdate { + // TODO check decoration and pvc template changed if targetInfo.IsUpdatedRevision { continue } @@ -652,7 +664,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, continue } - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, targetInfo) { + if targetInfo.IsDuringScaleInOps || targetInfo.IsDuringUpdateOps { continue } @@ -680,7 +692,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, // 6. update Target succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { targetInfo := <-targetCh - logger.V(1).Info("before target update operation", + logger.Info("before target update operation", "target", ObjectKeyString(targetInfo.Object), "revision.from", targetInfo.CurrentRevision.GetName(), "revision.to", syncContext.UpdatedRevision.GetName(), @@ -689,10 +701,9 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, ) spec := r.xsetController.GetXSetSpec(xsetObject) - isReplaceUpdate := spec.UpdateStrategy.UpdatePolicy == api.XSetReplaceTargetUpdateStrategyType - if targetInfo.IsInReplacing && !isReplaceUpdate { + if targetInfo.IsInReplace && spec.UpdateStrategy.UpdatePolicy != api.XSetReplaceTargetUpdateStrategyType { // a replacing target should be replaced by an updated revision target when encountering upgrade - if err := updateReplaceOriginTarget(ctx, r.Client, r.Recorder, targetInfo, targetInfo.ReplacePairNewTargetInfo); err != nil { + if err := updateReplaceOriginTarget(ctx, r.Client, r.Recorder, r.xsetLabelMgr, targetInfo, targetInfo.ReplacePairNewTargetInfo); err != nil { return err } } else { @@ -720,31 +731,44 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, succCount, err = controllerutils.SlowStartBatch(len(targetUpdateInfos), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { targetInfo := targetUpdateInfos[i] - if !targetInfo.IsDuringOps || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { + if !(targetInfo.IsDuringUpdateOps || targetInfo.PlaceHolder) || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { return nil } - // check Target is during updating, and it is finished or not - finished, msg, err := updater.GetTargetUpdateFinishStatus(ctx, targetInfo) - if err != nil { - return fmt.Errorf("failed to get target %s/%s update finished: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) + var finishByCancelUpdate bool + var updateFinished bool + var msg string + var err error + + if !targetToUpdateSet.Has(targetInfo.GetName()) { + // target is out of scope (partition or by label) and not start update yet, finish update by cancel + finishByCancelUpdate = !targetInfo.IsAllowUpdateOps + logger.Info("out of update scope", "target", ObjectKeyString(targetInfo.Object), "finishByCancelUpdate", finishByCancelUpdate) + } else if !targetInfo.IsAllowUpdateOps { + // target is in update scope, but is not start update yet, if pod is updatedRevision, just finish update by cancel + finishByCancelUpdate = targetInfo.IsUpdatedRevision + } else { + // target is in update scope and allowed to update, check and finish update gracefully + if updateFinished, msg, err = updater.GetTargetUpdateFinishStatus(ctx, targetInfo); err != nil { + return fmt.Errorf("failed to get target %s/%s update finished: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) + } else if !updateFinished { + r.Recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "WaitingUpdateReady", + "waiting for target %s/%s to update finished: %s", + targetInfo.GetNamespace(), targetInfo.GetName(), msg) + } } - if finished { + if updateFinished || finishByCancelUpdate { if err := updater.FinishUpdateTarget(ctx, targetInfo); err != nil { - return err + return fmt.Errorf("failed to finish target %s/%s update: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) } r.Recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "UpdateTargetFinished", "target %s/%s is finished for upgrade to revision %s", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.UpdateRevision.GetName()) - } else { - r.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "WaitingUpdateReady", - "waiting for target %s/%s to update finished: %s", - targetInfo.GetNamespace(), targetInfo.GetName(), msg) } return nil @@ -897,23 +921,23 @@ func FilterOutActiveTargetWrappers(targets []targetWrapper) []*targetWrapper { return filteredTargetWrappers } -func targetDuringReplace(labelMgr api.XSetControllerLabelManager, target client.Object) bool { +func targetDuringReplace(labelMgr api.XSetLabelManager, target client.Object) bool { labels := target.GetLabels() if labels == nil { return false } - _, replaceIndicate := labels[labelMgr.Get(api.EnumXSetReplaceIndicationLabelKey)] - _, replaceOriginTarget := labels[labelMgr.Get(api.EnumXSetReplacePairOriginName)] - _, replaceNewTarget := labels[labelMgr.Get(api.EnumXSetReplacePairNewId)] + _, replaceIndicate := labelMgr.Get(labels, api.EnumXSetReplaceIndicationLabel) + _, replaceOriginTarget := labelMgr.Get(labels, api.EnumXSetReplacePairOriginNameLabel) + _, replaceNewTarget := labelMgr.Get(labels, api.EnumXSetReplacePairNewIdLabel) return replaceIndicate || replaceOriginTarget || replaceNewTarget } -// BatchDeleteTargetByLabel try to trigger target deletion by to-delete label -func BatchDeleteTargetByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { +// BatchDeleteTargetsByLabel try to trigger target deletion by to-delete label +func (r *RealSyncControl) BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { _, err := controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { target := needDeleteTargets[i] - if _, exist := target.GetLabels()[appsv1alpha1.PodDeletionIndicationLabelKey]; !exist { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%q":"%d"}}}`, appsv1alpha1.PodDeletionIndicationLabelKey, time.Now().UnixNano()))) + if _, exist := r.xsetLabelMgr.Get(target.GetLabels(), api.EnumXSetDeletionIndicationLabel); !exist { + patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%q":"%d"}}}`, r.xsetLabelMgr.Label(api.EnumXSetDeletionIndicationLabel), time.Now().UnixNano()))) if err := targetControl.PatchTarget(ctx, target, patch); err != nil { return fmt.Errorf("failed to delete target when syncTargets %s/%s/%w", target.GetNamespace(), target.GetName(), err) } diff --git a/xset/synccontrols/types.go b/xset/synccontrols/types.go index db07c851..f90a68bc 100644 --- a/xset/synccontrols/types.go +++ b/xset/synccontrols/types.go @@ -72,16 +72,17 @@ type targetUpdateInfo struct { // carry the desired update revision UpdateRevision *appsv1.ControllerRevision - // indicates the TargetOpsLifecycle is started. - IsDuringOps bool + // TODO decoration revisions + // indicates operate is allowed for TargetOpsLifecycle. - IsAllowOps bool + IsAllowUpdateOps bool // requeue after for operationDelaySeconds RequeueForOperationDelay *time.Duration // for replace update // judge target in replace updating - IsInReplacing bool + IsInReplace bool + IsInReplaceUpdate bool // replace new created target ReplacePairNewTargetInfo *targetUpdateInfo diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index 66fb572d..fa0d246b 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -38,17 +38,24 @@ import ( "kusionstack.io/kube-utils/xset/opslifecycle" ) -const ( - ReplaceNewTargetIDContextDataKey = "ReplaceNewTargetID" - ReplaceOriginTargetIDContextDataKey = "ReplaceOriginTargetID" -) - -func (r *RealSyncControl) cleanReplaceTargetLabels(ctx context.Context, needCleanLabelTargets []client.Object, targetsNeedCleanLabels [][]string, ownedIDs map[int]*api.ContextDetail, currentIDs sets.Int) (bool, sets.Int, error) { +func (r *RealSyncControl) cleanReplaceTargetLabels( + ctx context.Context, + needCleanLabelTargets []client.Object, + targetsNeedCleanLabels [][]string, + ownedIDs map[int]*api.ContextDetail, + currentIDs sets.Int, +) (bool, sets.Int, error) { + logger := logr.FromContext(ctx) needUpdateContext := false needDeleteTargetsIDs := sets.Int{} - mapOriginToNewTargetContext := mapReplaceOriginToNewTargetContext(ownedIDs) - mapNewToOriginTargetContext := mapReplaceNewToOriginTargetContext(ownedIDs) - _, err := controllerutils.SlowStartBatch(len(needCleanLabelTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + mapOriginToNewTargetContext := r.mapReplaceOriginToNewTargetContext(ownedIDs) + mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) + _, err := controllerutils.SlowStartBatch(len(needCleanLabelTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { + defer func() { + if err == nil { + logger.Info("cleanReplaceTargetLabels clean replace labels success", "kind", needCleanLabelTargets[i].GetObjectKind(), "target", needCleanLabelTargets[i].GetName(), "labels", targetsNeedCleanLabels[i]) + } + }() target := needCleanLabelTargets[i] needCleanLabels := targetsNeedCleanLabels[i] var deletePatch []map[string]string @@ -59,32 +66,32 @@ func (r *RealSyncControl) cleanReplaceTargetLabels(ctx context.Context, needClea } deletePatch = append(deletePatch, patchOperation) // replace finished, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete origin Target's ID - if labelKey == TargetReplacePairOriginName { + if labelKey == r.xsetLabelMgr.Label(api.EnumXSetReplacePairOriginNameLabel) { needUpdateContext = true newTargetId, _ := GetInstanceID(target) if originTargetContext, exist := mapOriginToNewTargetContext[newTargetId]; exist && originTargetContext != nil { - originTargetContext.Remove(ReplaceNewTargetIDContextDataKey) + r.resourceContextControl.Remove(originTargetContext, api.EnumReplaceNewTargetIDContextDataKey) if _, exist := currentIDs[originTargetContext.ID]; !exist { needDeleteTargetsIDs.Insert(originTargetContext.ID) } } if contextDetail, exist := ownedIDs[newTargetId]; exist { - contextDetail.Remove(ReplaceOriginTargetIDContextDataKey) + r.resourceContextControl.Remove(contextDetail, api.EnumReplaceOriginTargetIDContextDataKey) } } // replace canceled, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete new Target's ID _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] - if !replaceIndicate && labelKey == TargetReplacePairNewId { + if !replaceIndicate && labelKey == r.xsetLabelMgr.Label(api.EnumXSetReplacePairNewIdLabel) { needUpdateContext = true originTargetId, _ := GetInstanceID(target) if newTargetContext, exist := mapNewToOriginTargetContext[originTargetId]; exist && newTargetContext != nil { - newTargetContext.Remove(ReplaceOriginTargetIDContextDataKey) + r.resourceContextControl.Remove(newTargetContext, api.EnumReplaceOriginTargetIDContextDataKey) if _, exist := currentIDs[newTargetContext.ID]; !exist { needDeleteTargetsIDs.Insert(newTargetContext.ID) } } if contextDetail, exist := ownedIDs[originTargetId]; exist { - contextDetail.Remove(ReplaceNewTargetIDContextDataKey) + r.resourceContextControl.Remove(contextDetail, api.EnumReplaceNewTargetIDContextDataKey) } } } @@ -110,41 +117,52 @@ func (r *RealSyncControl) replaceOriginTargets( ownedIDs map[int]*api.ContextDetail, availableContexts []*api.ContextDetail, ) (int, error) { - mapNewToOriginTargetContext := mapReplaceNewToOriginTargetContext(ownedIDs) + logger := logr.FromContext(ctx) + mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) successCount, err := controllerutils.SlowStartBatch(len(needReplaceOriginTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { originTarget := needReplaceOriginTargets[i] originTargetId, _ := GetInstanceID(originTarget) - replaceRevision := getReplaceRevision(originTarget, syncContext) + if ownedIDs[originTargetId] == nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "OriginTargetContext", "cannot found resource context id %d of origin target %s/%s", originTargetId, originTarget.GetNamespace(), originTarget.GetName()) + return fmt.Errorf("cannot found context for replace origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) + } + + replaceRevision := r.getReplaceRevision(originTarget, syncContext) // create target using update revision if replaced by update, otherwise using current revision - newTarget, err := NewTargetFrom(r.xsetController, instance, replaceRevision, originTargetId) + newTarget, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, instance, replaceRevision, originTargetId) if err != nil { return err } // add instance id and replace pair label - var instanceId string + var newInstanceId string var newTargetContext *api.ContextDetail if contextDetail, exist := mapNewToOriginTargetContext[originTargetId]; exist && contextDetail != nil { newTargetContext = contextDetail // reuse targetContext ID if pair-relation exists - instanceId = fmt.Sprintf("%d", newTargetContext.ID) - newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId + newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) + newTarget.GetLabels()[TargetInstanceIDLabelKey] = newInstanceId + logger.Info("replaceOriginTargets", "try to reuse new pod resourceContext id", newInstanceId) } else { if availableContexts[i] == nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "AvailableContext", "cannot found available context for replace origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) return fmt.Errorf("cannot found available context for replace new target when replacing origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) } newTargetContext = availableContexts[i] // add replace pair-relation to targetContexts for originTarget and newTarget - instanceId = fmt.Sprintf("%d", newTargetContext.ID) - newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId - ownedIDs[originTargetId].Put(ReplaceNewTargetIDContextDataKey, instanceId) - ownedIDs[newTargetContext.ID].Put(ReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) + newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) + r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetInstanceIdLabel, newInstanceId) + r.resourceContextControl.Put(ownedIDs[originTargetId], api.EnumReplaceNewTargetIDContextDataKey, newInstanceId) + r.resourceContextControl.Put(ownedIDs[newTargetContext.ID], api.EnumReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) r.resourceContextControl.Remove(ownedIDs[newTargetContext.ID], api.EnumJustCreateContextDataKey) } - newTarget.GetLabels()[TargetReplacePairOriginName] = originTarget.GetName() + r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetReplacePairOriginNameLabel, originTarget.GetName()) + r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetTargetCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) r.resourceContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) + // TODO create pvcs for new target (pod) + if newCreatedTarget, err := r.xControl.CreateTarget(ctx, newTarget); err == nil { r.Recorder.Eventf(originTarget, corev1.EventTypeNormal, @@ -154,11 +172,16 @@ func (r *RealSyncControl) replaceOriginTargets( originTarget.GetName(), replaceRevision.GetName()) - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, TargetReplacePairNewId, instanceId))) + if err := r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(instance), r.targetGVK, newTarget.GetNamespace(), newTarget.GetName()); err != nil { + return err + } + + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, TargetReplacePairNewId, newInstanceId))) if err = r.xControl.PatchTarget(ctx, originTarget, patch); err != nil { return fmt.Errorf("fail to update origin target %s/%s pair label %s when updating by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), newCreatedTarget.GetName(), err.Error()) } - return r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(instance), r.targetGVK, newTarget.GetNamespace(), newTarget.GetName()) + logger.Info("replaceOriginTargets", "replacing originTarget", originTarget.GetName(), "originTargetId", originTargetId, "newTargetContextID", newInstanceId) + return nil } else { r.Recorder.Eventf(originTarget, corev1.EventTypeNormal, @@ -175,9 +198,10 @@ func (r *RealSyncControl) replaceOriginTargets( return successCount, err } -func (r *RealSyncControl) dealReplaceTargets(targets []client.Object, logger logr.Logger) ( +func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []client.Object) ( needReplaceTargets, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object, ) { + logger := logr.FromContext(ctx) targetInstanceIdMap := make(map[string]client.Object) targetNameMap := make(map[string]client.Object) @@ -224,15 +248,15 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object, logger log for _, target := range targets { targetLabels := target.GetLabels() - _, replaceByUpdate := targetLabels[TargetReplaceByReplaceUpdateLabelKey] + _, replaceByUpdate := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetReplaceByReplaceUpdateLabel) var needCleanLabels []string // target is replace new created target, skip replace - if originTargetName, exist := targetLabels[TargetReplacePairOriginName]; exist { + if originTargetName, exist := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetReplacePairOriginNameLabel); exist { // replace pair origin target is not exist, clean label. if originTarget, exist := targetNameMap[originTargetName]; !exist { - needCleanLabels = append(needCleanLabels, TargetReplacePairOriginName) - } else if originTarget.GetLabels()[TargetReplaceIndicationLabelKey] == "" { + needCleanLabels = append(needCleanLabels, r.xsetLabelMgr.Label(api.EnumXSetReplacePairOriginNameLabel)) + } else if _, exist := r.xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplaceIndicationLabel); !exist { // replace canceled, delete replace new target if new target is not service available if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); !serviceAvailable { needDeleteTargets = append(needDeleteTargets, target) @@ -245,9 +269,9 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object, logger log } } - if newPairTargetId, exist := targetLabels[TargetReplacePairNewId]; exist { + if newPairTargetId, exist := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetReplacePairNewIdLabel); exist { if _, exist := targetInstanceIdMap[newPairTargetId]; !exist { - needCleanLabels = append(needCleanLabels, TargetReplacePairNewId) + needCleanLabels = append(needCleanLabels, r.xsetLabelMgr.Label(api.EnumXSetReplacePairNewIdLabel)) } } @@ -263,6 +287,7 @@ func updateReplaceOriginTarget( ctx context.Context, c client.Client, recorder record.EventRecorder, + xsetLabelMgr api.XSetLabelManager, originTargetUpdateInfo, newTargetUpdateInfo *targetUpdateInfo, ) error { originTarget := originTargetUpdateInfo.Object @@ -270,10 +295,10 @@ func updateReplaceOriginTarget( // 1. delete the new target if not updated if newTargetUpdateInfo != nil { newTarget := newTargetUpdateInfo.Object - _, deletionIndicate := newTarget.GetLabels()[TargetDeletionIndicationLabelKey] + _, deletionIndicate := xsetLabelMgr.Get(newTarget.GetLabels(), api.EnumXSetDeletionIndicationLabel) currentRevision, exist := newTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] if exist && currentRevision != originTargetUpdateInfo.UpdateRevision.GetName() && !deletionIndicate { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, xsetLabelMgr.Label(api.EnumXSetDeletionIndicationLabel), time.Now().UnixNano()))) if patchErr := c.Patch(ctx, newTarget, patch); patchErr != nil { err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", newTarget.GetNamespace(), newTarget.GetName(), patchErr.Error()) @@ -290,11 +315,11 @@ func updateReplaceOriginTarget( } // 2. replace the origin target with updated target - _, replaceIndicate := originTarget.GetLabels()[TargetReplaceIndicationLabelKey] - _, replaceByUpdate := originTarget.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] - if !replaceIndicate || !replaceByUpdate { + _, replaceIndicate := xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplaceIndicationLabel) + replaceRevision, replaceByUpdate := xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplaceByReplaceUpdateLabel) + if !replaceIndicate || !replaceByUpdate || replaceRevision != originTargetUpdateInfo.UpdateRevision.Name { now := time.Now().UnixNano() - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, TargetReplaceIndicationLabelKey, now, TargetReplaceByReplaceUpdateLabelKey, originTargetUpdateInfo.UpdateRevision.GetName()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, xsetLabelMgr.Label(api.EnumXSetReplaceIndicationLabel), now, xsetLabelMgr.Label(api.EnumXSetReplaceByReplaceUpdateLabel), originTargetUpdateInfo.UpdateRevision.Name))) if err := c.Patch(ctx, originTarget, patch); err != nil { return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), err.Error()) } @@ -311,11 +336,11 @@ func updateReplaceOriginTarget( } // getReplaceRevision finds replaceNewTarget's revision from originTarget -func getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *appsv1.ControllerRevision { +func (r *RealSyncControl) getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *appsv1.ControllerRevision { // replace update, first find revision from label, if revision not found, just replace with updated revision - if updateRevisionName, exist := originTarget.GetLabels()[TargetReplaceByReplaceUpdateLabelKey]; exist { + if updateRevisionName, exist := r.xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplaceByReplaceUpdateLabel); exist { for _, rv := range syncContext.Revisions { - if updateRevisionName == rv.GetName() { + if updateRevisionName == rv.Name { return rv } } @@ -374,13 +399,14 @@ func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string] return replaceTargetMapping } -func mapReplaceNewToOriginTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { +func (r *RealSyncControl) mapReplaceNewToOriginTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { mapNewToOriginTargetContext := make(map[int]*api.ContextDetail) for id, contextDetail := range ownedIDs { - if val, exist := contextDetail.Data[ReplaceNewTargetIDContextDataKey]; exist { + if val, exist := r.resourceContextControl.Get(contextDetail, api.EnumReplaceNewTargetIDContextDataKey); exist { newTargetId, _ := strconv.ParseInt(val, 10, 32) newTargetContextDetail, exist := ownedIDs[int(newTargetId)] - if exist && newTargetContextDetail.Data[ReplaceOriginTargetIDContextDataKey] == strconv.Itoa(id) { + originTargetId, _ := r.resourceContextControl.Get(newTargetContextDetail, api.EnumReplaceOriginTargetIDContextDataKey) + if exist && originTargetId == strconv.Itoa(id) { mapNewToOriginTargetContext[id] = newTargetContextDetail } else { mapNewToOriginTargetContext[id] = nil @@ -390,13 +416,14 @@ func mapReplaceNewToOriginTargetContext(ownedIDs map[int]*api.ContextDetail) map return mapNewToOriginTargetContext } -func mapReplaceOriginToNewTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { +func (r *RealSyncControl) mapReplaceOriginToNewTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { mapOriginToNewTargetContext := make(map[int]*api.ContextDetail) for id, contextDetail := range ownedIDs { - if val, exist := contextDetail.Data[ReplaceOriginTargetIDContextDataKey]; exist { + if val, exist := r.resourceContextControl.Get(contextDetail, api.EnumReplaceOriginTargetIDContextDataKey); exist { originTargetId, _ := strconv.ParseInt(val, 10, 32) originTargetContextDetail, exist := ownedIDs[int(originTargetId)] - if exist && originTargetContextDetail.Data[ReplaceNewTargetIDContextDataKey] == strconv.Itoa(id) { + newTargetId, _ := r.resourceContextControl.Get(originTargetContextDetail, api.EnumReplaceNewTargetIDContextDataKey) + if exist && newTargetId == strconv.Itoa(id) { mapOriginToNewTargetContext[id] = originTargetContextDetail } else { mapOriginToNewTargetContext[id] = nil diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 6dd854d0..f890bcab 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" clientutil "kusionstack.io/kube-utils/client" controllerutils "kusionstack.io/kube-utils/controller/utils" @@ -32,8 +33,10 @@ import ( "kusionstack.io/kube-utils/xset/opslifecycle" ) -// getTargetsToDelete finds number of diff targets from filteredTargets to do scaleIn -func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, replaceMapping map[string]*targetWrapper, diff int) []*targetWrapper { +// getTargetsToDelete +// 1. finds number of diff targets from filteredPods to do scaleIn +// 2. finds targets allowed to scale in out of diff +func (r *RealSyncControl) getTargetsToDelete(xsetObject api.XSetObject, filteredTargets []*targetWrapper, replaceMapping map[string]*targetWrapper, diff int) []*targetWrapper { var countedTargets []*targetWrapper for _, target := range filteredTargets { if _, exist := replaceMapping[target.GetName()]; exist { @@ -49,7 +52,14 @@ func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, r // 2. select targets to delete in second round according to replace, delete, exclude var needDeleteTargets []*targetWrapper - for _, target := range countedTargets[:diff] { + for i, target := range countedTargets { + // find pods to be scaleIn out of diff, is allowed to ops + spec := r.xsetController.GetXSetSpec(xsetObject) + _, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.scaleInLifecycleAdapter, ptr.Deref(spec.ScaleStrategy.OperationDelaySeconds, 0), target) + if i >= diff && !allowed { + continue + } + // don't scaleIn exclude target and its newTarget (if exist) if target.ToExclude { continue diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index c68515dd..b4f95600 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -52,10 +52,10 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync for i, target := range activeTargets { updateInfo := &targetUpdateInfo{ - targetWrapper: &syncContext.TargetWrappers[i], - InPlaceUpdateSupport: true, + targetWrapper: &syncContext.TargetWrappers[i], } + // TODO decoration for target template updateInfo.UpdateRevision = syncContext.UpdatedRevision // decide this target current revision, or nil if not indicated if target.GetLabels() != nil { @@ -90,9 +90,8 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync spec := r.xsetController.GetXSetSpec(xsetObject) // decide whether the TargetOpsLifecycle is during ops or not - updateInfo.IsDuringOps = target.IsDuringUpdateOps - updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) - + updateInfo.RequeueForOperationDelay, updateInfo.IsAllowUpdateOps = opslifecycle.AllowOps(r.updateConfig.opsLifecycleLabelMgr, r.updateLifecycleAdapter, ptr.Deref(spec.UpdateStrategy.OperationDelaySeconds, 0), target) + // TODO check pvc template changed targetUpdateInfoList[i] = updateInfo } @@ -101,25 +100,30 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync for _, targetUpdateInfo := range targetUpdateInfoList { targetUpdateInfoMap[targetUpdateInfo.GetName()] = targetUpdateInfo } + // originTarget's isAllowUpdateOps depends on these 2 cases: + // (1) target is during replacing but not during replaceUpdate, keep it legacy value + // (2) target is during replaceUpdate, set to "true" if newTarget is service available for originTargetName, replacePairNewTarget := range syncContext.replacingMap { originTargetInfo := targetUpdateInfoMap[originTargetName] + _, replaceIndicated := r.xsetLabelMgr.Get(originTargetInfo.GetLabels(), api.EnumXSetReplaceIndicationLabel) + _, replaceByReplaceUpdate := r.xsetLabelMgr.Get(originTargetInfo.GetLabels(), api.EnumXSetReplaceByReplaceUpdateLabel) + isReplaceUpdating := replaceIndicated && replaceByReplaceUpdate + + originTargetInfo.IsInReplace = replaceIndicated + originTargetInfo.IsInReplaceUpdate = isReplaceUpdating + if replacePairNewTarget != nil { - originTargetInfo.IsInReplacing = true - // replace origin target not go through lifecycle, mark during ops manual - originTargetInfo.IsDuringOps = true - originTargetInfo.IsAllowOps = true + // origin target is allowed to ops if new pod is serviceAvailable + _, newTargetSa := replacePairNewTarget.GetLabels()[r.updateConfig.opsLifecycleLabelMgr.Get(api.ServiceAvailableLabel)] + originTargetInfo.IsAllowUpdateOps = originTargetInfo.IsAllowUpdateOps || newTargetSa + // attach replace new target updateInfo ReplacePairNewTargetInfo := targetUpdateInfoMap[replacePairNewTarget.GetName()] - ReplacePairNewTargetInfo.IsInReplacing = true + ReplacePairNewTargetInfo.IsInReplace = true + // in case of to-replace label is removed from origin target, new target is still in replaceUpdate + ReplacePairNewTargetInfo.IsInReplaceUpdate = replaceByReplaceUpdate + ReplacePairNewTargetInfo.ReplacePairOriginTargetName = originTargetName originTargetInfo.ReplacePairNewTargetInfo = ReplacePairNewTargetInfo - } else { - _, replaceIndicated := originTargetInfo.GetLabels()[TargetReplaceIndicationLabelKey] - _, replaceByReplaceUpdate := originTargetInfo.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] - if replaceIndicated && replaceByReplaceUpdate { - originTargetInfo.IsInReplacing = true - originTargetInfo.IsDuringOps = true - originTargetInfo.IsAllowOps = true - } } } @@ -129,9 +133,8 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync continue } updateInfo := &targetUpdateInfo{ - targetWrapper: &target, - InPlaceUpdateSupport: true, - UpdateRevision: syncContext.UpdatedRevision, + targetWrapper: &target, + UpdateRevision: syncContext.UpdatedRevision, } if revision, exist := r.resourceContextControl.Get(target.ContextDetail, api.EnumRevisionContextDataKey); exist && revision == syncContext.UpdatedRevision.GetName() { @@ -154,46 +157,40 @@ func filterOutPlaceHolderUpdateInfos(targets []*targetUpdateInfo) []*targetUpdat return filteredTargetUpdateInfos } -func decideTargetToUpdate(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { +func (r *RealSyncControl) decideTargetToUpdate(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { spec := xsetController.GetXSetSpec(xset) + filteredPodInfos := r.getTargetsUpdateTargets(targetInfos) + if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByLabel != nil { - activeTargetInfos := filterOutPlaceHolderUpdateInfos(targetInfos) - return decideTargetToUpdateByLabel(xset, activeTargetInfos) + activeTargetInfos := filterOutPlaceHolderUpdateInfos(filteredPodInfos) + return r.decideTargetToUpdateByLabel(activeTargetInfos) } - return decideTargetToUpdateByPartition(xsetController, xset, targetInfos) + return r.decideTargetToUpdateByPartition(xsetController, xset, targetInfos) } -func decideTargetToUpdateByLabel(_ api.XSetObject, targetInfos []*targetUpdateInfo) (targetToUpdate []*targetUpdateInfo) { +func (r *RealSyncControl) decideTargetToUpdateByLabel(targetInfos []*targetUpdateInfo) (targetToUpdate []*targetUpdateInfo) { for i := range targetInfos { - if _, exist := targetInfos[i].GetLabels()[XSetUpdateIndicateLabelKey]; exist { - // filter target which is in replace update and is the new created target - if targetInfos[i].IsInReplacing && targetInfos[i].ReplacePairOriginTargetName != "" { - continue - } + if _, exist := r.xsetLabelMgr.Get(targetInfos[i].GetLabels(), api.EnumXSetUpdateIndicationLabel); exist { targetToUpdate = append(targetToUpdate, targetInfos[i]) continue } - // already in replace update. - if targetInfos[i].IsInReplacing && targetInfos[i].ReplacePairNewTargetInfo != nil { - targetToUpdate = append(targetToUpdate, targetInfos[i]) - continue - } + // TODO separate decoration and xset update progress } return targetToUpdate } -func decideTargetToUpdateByPartition(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { +func (r *RealSyncControl) decideTargetToUpdateByPartition(xsetController api.XSetController, xset api.XSetObject, filteredTargetInfos []*targetUpdateInfo) []*targetUpdateInfo { spec := xsetController.GetXSetSpec(xset) replicas := ptr.Deref(spec.Replicas, 0) + partition := int32(0) if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByPartition != nil { partition = ptr.Deref(spec.UpdateStrategy.RollingUpdate.ByPartition.Partition, 0) } - filteredTargetInfos := getTargetsUpdateTargets(targetInfos) // update all or not update any replicas if partition == 0 { return filteredTargetInfos @@ -210,20 +207,16 @@ func decideTargetToUpdateByPartition(xsetController api.XSetController, xset api } // when sort targets to choose update, only sort (1) replace origin targets, (2) non-exclude targets -func getTargetsUpdateTargets(targetInfos []*targetUpdateInfo) (filteredTargetInfos []*targetUpdateInfo) { +func (r *RealSyncControl) getTargetsUpdateTargets(targetInfos []*targetUpdateInfo) (filteredTargetInfos []*targetUpdateInfo) { for _, targetInfo := range targetInfos { - if targetInfo.IsInReplacing && targetInfo.ReplacePairOriginTargetName != "" { + if targetInfo.ReplacePairOriginTargetName != "" { continue } - if targetInfo.PlaceHolder { - _, isReplaceNewTarget := targetInfo.ContextDetail.Data[ReplaceOriginTargetIDContextDataKey] - _, isReplaceOriginTarget := targetInfo.ContextDetail.Data[ReplaceNewTargetIDContextDataKey] - if isReplaceNewTarget || isReplaceOriginTarget { + if _, isReplaceNewTarget := r.resourceContextControl.Get(targetInfo.ContextDetail, api.EnumReplaceOriginTargetIDContextDataKey); isReplaceNewTarget { continue } } - filteredTargetInfos = append(filteredTargetInfos, targetInfo) } return filteredTargetInfos @@ -261,8 +254,8 @@ func (o *orderByDefault) Less(i, j int) bool { return true } - if l.IsDuringOps != r.IsDuringOps { - return l.IsDuringOps + if l.IsDuringUpdateOps != r.IsDuringUpdateOps { + return l.IsDuringUpdateOps } lReady, rReady := o.checkReadyFunc(l.Object), o.checkReadyFunc(r.Object) @@ -277,6 +270,7 @@ func (o *orderByDefault) Less(i, j int) bool { type UpdateConfig struct { xsetController api.XSetController + xsetLabelMgr api.XSetLabelManager client client.Client targetControl xcontrol.TargetControl resourceContextControl resourcecontexts.ResourceContextControl @@ -285,7 +279,6 @@ type UpdateConfig struct { opsLifecycleLabelMgr api.LifeCycleLabelManager scaleInLifecycleAdapter api.LifecycleAdapter updateLifecycleAdapter api.LifecycleAdapter - xSetControllerLabelMgr api.XSetControllerLabelManager cacheExpectations expectations.CacheExpectationsInterface targetGVK schema.GroupVersionKind @@ -351,7 +344,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid targetInfo := candidates[i] if !targetInfo.PlaceHolder { - if !targetInfo.IsAllowOps { + if !targetInfo.IsAllowUpdateOps { continue } if targetInfo.RequeueForOperationDelay != nil { @@ -363,7 +356,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid } } - targetInfo.IsAllowOps = true + targetInfo.IsAllowUpdateOps = true if targetInfo.IsUpdatedRevision { continue @@ -466,14 +459,14 @@ type inPlaceIfPossibleUpdater struct { func (u *inPlaceIfPossibleUpdater) FulfillTargetUpdatedInfo(_ context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *targetUpdateInfo) error { // 1. build target from current and updated revision // TODO: use cache - currentTarget, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetUpdateInfo.CurrentRevision, targetUpdateInfo.ID) + currentTarget, err := NewTargetFrom(u.xsetController, u.xsetLabelMgr, u.OwnerObject, targetUpdateInfo.CurrentRevision, targetUpdateInfo.ID) if err != nil { return fmt.Errorf("fail to build Target from current revision %s: %v", targetUpdateInfo.CurrentRevision.GetName(), err.Error()) } // TODO: use cache - UpdatedTarget, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetUpdateInfo.UpdateRevision, targetUpdateInfo.ID) + UpdatedTarget, err := NewTargetFrom(u.xsetController, u.xsetLabelMgr, u.OwnerObject, targetUpdateInfo.UpdateRevision, targetUpdateInfo.ID) if err != nil { return fmt.Errorf("fail to build Target from updated revision %s: %v", targetUpdateInfo.UpdateRevision.GetName(), err.Error()) } @@ -668,7 +661,7 @@ func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *targe if targetInfo.GetLabels() == nil { return false, "no labels on target", nil } - if targetInfo.IsInReplacing && targetInfo.ReplacePairNewTargetInfo != nil { + if targetInfo.IsInReplace && targetInfo.ReplacePairNewTargetInfo != nil { return false, "replace origin target", nil } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index bfec8f7c..19b2bcdf 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -48,7 +48,7 @@ func GetInstanceID(target client.Object) (int, error) { return int(id), nil } -func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { +func NewTargetFrom(setController api.XSetController, xsetLabelMgr api.XSetLabelManager, owner api.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { targetObj, err := setController.GetXObjectFromRevision(revision) if err != nil { return nil, err @@ -61,9 +61,9 @@ func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revis targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) labels := targetObj.GetLabels() - labels[TargetInstanceIDLabelKey] = fmt.Sprintf("%d", id) + xsetLabelMgr.Set(labels, api.EnumXSetInstanceIdLabel, fmt.Sprintf("%d", id)) labels[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() - controlByXSet(setController, targetObj) + controlByXSet(xsetLabelMgr, targetObj) for _, fn := range updateFuncs { if err := fn(targetObj); err != nil { @@ -74,14 +74,6 @@ func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revis return targetObj, nil } -func RealValue(val *int32) int32 { - if val == nil { - return 0 - } - - return *val -} - const ConditionUpdatePeriodBackOff = 30 * time.Second func AddOrUpdateCondition(status *api.XSetStatus, conditionType api.XSetConditionType, err error, reason, message string) { @@ -170,29 +162,20 @@ func filterOutCondition(conditions []metav1.Condition, condType string) []metav1 return newConditions } -func controlByXSet(setController api.XSetController, obj client.Object) { +func controlByXSet(xsetLabelMgr api.XSetLabelManager, obj client.Object) { if obj.GetLabels() == nil { obj.SetLabels(map[string]string{}) } - controlLabel := setController.GetXSetControllerLabelManager() - if controlLabel == nil { - controlLabel = NewXSetControllerLabelManager() - } - - if v, ok := obj.GetLabels()[controlLabel.Get(api.EnumXSetControlledLabel)]; !ok || v != "true" { - obj.GetLabels()[controlLabel.Get(api.EnumXSetControlledLabel)] = "true" + if v, ok := xsetLabelMgr.Get(obj.GetLabels(), api.EnumXSetControlledLabel); !ok || v != "true" { + xsetLabelMgr.Set(obj.GetLabels(), api.EnumXSetControlledLabel, "true") } } -func IsControlledByXSet(setController api.XSetController, obj client.Object) bool { + +func IsControlledByXSet(xsetLabelManager api.XSetLabelManager, obj client.Object) bool { if obj.GetLabels() == nil { return false } - controlLabel := setController.GetXSetControllerLabelManager() - if controlLabel == nil { - controlLabel = NewXSetControllerLabelManager() - } - - v, ok := obj.GetLabels()[controlLabel.Get(api.EnumXSetControlledLabel)] + v, ok := xsetLabelManager.Get(obj.GetLabels(), api.EnumXSetControlledLabel) return ok && v == "true" } diff --git a/xset/xset_controller.go b/xset/xset_controller.go index 258b0319..99a2f313 100644 --- a/xset/xset_controller.go +++ b/xset/xset_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + "github.com/go-logr/logr" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -70,6 +71,11 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error return err } + xsetLabelManager := xsetController.GetXSetControllerLabelManager() + if xsetLabelManager == nil { + xsetLabelManager = synccontrols.NewXSetControllerLabelManager() + } + reconcilerMixin := mixin.NewReconcilerMixin(xsetController.ControllerName(), mgr) xsetMeta := xsetController.XSetMeta() xsetGVK := xsetMeta.GroupVersionKind() @@ -83,7 +89,7 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error } cacheExpectations := expectations.NewxCacheExpectations(reconcilerMixin.Client, reconcilerMixin.Scheme, clock.RealClock{}) resourceContextControl := resourcecontexts.NewRealResourceContextControl(reconcilerMixin.Client, xsetController, resourceContextAdapter, resourceContextGVK, cacheExpectations) - syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, resourceContextControl, cacheExpectations) + syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, xsetLabelManager, resourceContextControl, cacheExpectations) revisionControl := history.NewRevisionControl(reconcilerMixin.Client, reconcilerMixin.Client) revisionOwner := revisionowner.NewRevisionOwner(xsetController, targetControl) revisionManager := history.NewHistoryManager(revisionControl, revisionOwner) @@ -118,16 +124,17 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error OwnerType: xsetController.NewXSetObject(), }, predicate.Funcs{ CreateFunc: func(event event.CreateEvent) bool { - return synccontrols.IsControlledByXSet(xsetController, event.Object) + return synccontrols.IsControlledByXSet(xsetLabelManager, event.Object) }, UpdateFunc: func(updateEvent event.UpdateEvent) bool { - return synccontrols.IsControlledByXSet(xsetController, updateEvent.ObjectNew) || synccontrols.IsControlledByXSet(xsetController, updateEvent.ObjectOld) + return synccontrols.IsControlledByXSet(xsetLabelManager, updateEvent.ObjectNew) || + synccontrols.IsControlledByXSet(xsetLabelManager, updateEvent.ObjectOld) }, DeleteFunc: func(deleteEvent event.DeleteEvent) bool { - return synccontrols.IsControlledByXSet(xsetController, deleteEvent.Object) + return synccontrols.IsControlledByXSet(xsetLabelManager, deleteEvent.Object) }, GenericFunc: func(genericEvent event.GenericEvent) bool { - return synccontrols.IsControlledByXSet(xsetController, genericEvent.Object) + return synccontrols.IsControlledByXSet(xsetLabelManager, genericEvent.Object) }, }); err != nil { return fmt.Errorf("failed to watch %s: %s", targetMeta.Kind, err.Error()) @@ -150,7 +157,8 @@ func getAndValidateResourceContextAdapter(xsetController api.XSetController) (ap func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { kind := r.meta.Kind key := req.String() - logger := r.Logger.WithValues(kind, key) + ctx = logr.NewContext(ctx, r.Logger.WithValues(kind, key)) + logger := logr.FromContext(ctx) instance := r.XSetController.NewXSetObject() if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { if !apierrors.IsNotFound(err) { @@ -248,7 +256,7 @@ func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, if err != nil { return fmt.Errorf("fail to get filtered Targets: %s", err.Error()) } - return synccontrols.BatchDeleteTargetByLabel(ctx, r.targetControl, targets) + return r.syncControl.BatchDeleteTargetsByLabel(ctx, r.targetControl, targets) } func (r *xSetCommonReconciler) updateStatus(ctx context.Context, instance api.XSetObject, status *api.XSetStatus) error { From cdfb4a9eae591a0edd06350c313e3da9ca2c923a Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 26 Aug 2025 19:44:37 +0800 Subject: [PATCH 03/17] sync kuperator code2 --- xset/api/xset_types.go | 5 ++++ xset/opslifecycle/utils.go | 15 ++++++++++ xset/synccontrols/const.go | 48 ------------------------------ xset/synccontrols/label_manager.go | 30 ++++++++++++------- xset/synccontrols/sync_control.go | 8 ++--- xset/synccontrols/x_replace.go | 29 +++++++++--------- xset/synccontrols/x_scale.go | 6 ++-- xset/synccontrols/x_update.go | 32 ++++++++++++-------- xset/synccontrols/x_utils.go | 7 +++-- 9 files changed, 85 insertions(+), 95 deletions(-) delete mode 100644 xset/synccontrols/const.go diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index 28da50ee..b90267f7 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -219,10 +219,15 @@ const ( EnumXSetOrphanedLabel EnumXSetTargetCreatingLabel + + EnumXSetTargetExcludeIndicationLabel + + EnumXSetLastTargetStatusAnnotationKey ) type XSetLabelManager interface { Get(labels map[string]string, labelType XSetControllerLabelEnum) (string, bool) Set(labels map[string]string, labelType XSetControllerLabelEnum, value string) + Delete(labels map[string]string, labelType XSetControllerLabelEnum) Label(labelType XSetControllerLabelEnum) string } diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go index 4f56a03a..ec929b28 100644 --- a/xset/opslifecycle/utils.go +++ b/xset/opslifecycle/utils.go @@ -310,6 +310,21 @@ func IsLifecycleOnTarget(m api.LifeCycleLabelManager, operatingID string, target return false, nil } +func CancelOpsLifecycle(m api.LifeCycleLabelManager, client client.Client, adapter api.LifecycleAdapter, target client.Object) error { + if target == nil { + return nil + } + + // only cancel when lifecycle exist on pod + if exist, err := IsLifecycleOnTarget(m, adapter.GetID(), target); err != nil { + return fmt.Errorf("fail to check %s PodOpsLifecycle on Pod %s/%s: %w", adapter.GetID(), target.GetNamespace(), target.GetName(), err) + } else if !exist { + return nil + } + + return Undo(m, client, adapter, target) +} + func DefaultUpdateAll(target client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { for _, updateFunc := range updateFuncs { ok, updateErr := updateFunc(target) diff --git a/xset/synccontrols/const.go b/xset/synccontrols/const.go deleted file mode 100644 index a3fcf8b9..00000000 --- a/xset/synccontrols/const.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package synccontrols - -// XSet labels -var ( - // TargetInstanceIDLabelKey is used to attach target instance ID on target - TargetInstanceIDLabelKey = "xset.kusionstack.io/instance-id" - // XSetUpdateIndicateLabelKey is used to indicate a target should be updated by label - XSetUpdateIndicateLabelKey = "xset.kusionstack.io/update-included" - - // TargetDeletionIndicationLabelKey indicates a target will be deleted by xset - TargetDeletionIndicationLabelKey = "xset.kusionstack.io/to-delete" - // TargetReplaceIndicationLabelKey indicates a target will be replaced by xset - TargetReplaceIndicationLabelKey = "xset.kusionstack.io/to-replace" - // TargetReplaceByReplaceUpdateLabelKey indicates a target is replaced by update by xset - TargetReplaceByReplaceUpdateLabelKey = "xset.kusionstack.io/replaced-by-replace-update" - // TargetExcludeIndicationLabelKey indicates a target will be excluded by xset - TargetExcludeIndicationLabelKey = "xset.kusionstack.io/to-exclude" - - // TargetReplacePairOriginName is used to indicate replace origin target name on the new created target - TargetReplacePairOriginName = "xset.kusionstack.io/replace-pair-origin-name" - // TargetReplacePairNewId is used to indicate the new created target instance on replace origin target - TargetReplacePairNewId = "xset.kusionstack.io/replace-pair-new-id" - - // TargetOrphanedIndicateLabelKey indicates target is orphaned - TargetOrphanedIndicateLabelKey = "xset.kusionstack.io/orphaned" -) - -// XSet annotations -var ( - // LastTargetStatusAnnotationKey is used to attach last target status on target - LastTargetStatusAnnotationKey = "xset.kusionstack.io/last-target-status" -) diff --git a/xset/synccontrols/label_manager.go b/xset/synccontrols/label_manager.go index 9b41c711..a8c0809c 100644 --- a/xset/synccontrols/label_manager.go +++ b/xset/synccontrols/label_manager.go @@ -23,16 +23,18 @@ import ( ) var defaultXSetControllerLabelManager = map[api.XSetControllerLabelEnum]string{ - api.EnumXSetControlledLabel: appsv1alpha1.ControlledByKusionStackLabelKey, - api.EnumXSetInstanceIdLabel: appsv1alpha1.PodInstanceIDLabelKey, - api.EnumXSetUpdateIndicationLabel: appsv1alpha1.CollaSetUpdateIndicateLabelKey, - api.EnumXSetDeletionIndicationLabel: appsv1alpha1.PodDeletionIndicationLabelKey, - api.EnumXSetReplaceIndicationLabel: appsv1alpha1.PodReplaceIndicationLabelKey, - api.EnumXSetReplacePairNewIdLabel: appsv1alpha1.PodReplacePairNewId, - api.EnumXSetReplacePairOriginNameLabel: appsv1alpha1.PodReplacePairOriginName, - api.EnumXSetReplaceByReplaceUpdateLabel: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, - api.EnumXSetOrphanedLabel: appsv1alpha1.PodOrphanedIndicateLabelKey, - api.EnumXSetTargetCreatingLabel: appsv1alpha1.PodCreatingLabel, + api.EnumXSetControlledLabel: appsv1alpha1.ControlledByKusionStackLabelKey, + api.EnumXSetInstanceIdLabel: appsv1alpha1.PodInstanceIDLabelKey, + api.EnumXSetUpdateIndicationLabel: appsv1alpha1.CollaSetUpdateIndicateLabelKey, + api.EnumXSetDeletionIndicationLabel: appsv1alpha1.PodDeletionIndicationLabelKey, + api.EnumXSetReplaceIndicationLabel: appsv1alpha1.PodReplaceIndicationLabelKey, + api.EnumXSetReplacePairNewIdLabel: appsv1alpha1.PodReplacePairNewId, + api.EnumXSetReplacePairOriginNameLabel: appsv1alpha1.PodReplacePairOriginName, + api.EnumXSetReplaceByReplaceUpdateLabel: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, + api.EnumXSetOrphanedLabel: appsv1alpha1.PodOrphanedIndicateLabelKey, + api.EnumXSetTargetCreatingLabel: appsv1alpha1.PodCreatingLabel, + api.EnumXSetTargetExcludeIndicationLabel: appsv1alpha1.PodExcludeIndicationLabelKey, + api.EnumXSetLastTargetStatusAnnotationKey: appsv1alpha1.LastPodStatusAnnotationKey, } func NewXSetControllerLabelManager() api.XSetLabelManager { @@ -62,6 +64,14 @@ func (m *xSetControllerLabelManager) Set(labels map[string]string, key api.XSetC labels[labelKey] = val } +func (m *xSetControllerLabelManager) Delete(labels map[string]string, key api.XSetControllerLabelEnum) { + if labels == nil { + return + } + labelKey := m.labelManager[key] + delete(labels, labelKey) +} + func (m *xSetControllerLabelManager) Label(key api.XSetControllerLabelEnum) string { return m.labelManager[key] } diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 4a655518..5287b805 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -175,7 +175,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje for i := range syncContext.FilteredTarget { target := syncContext.FilteredTarget[i] xName := target.GetName() - id, _ := GetInstanceID(target) + id, _ := GetInstanceID(r.xsetLabelMgr, target) toDelete := toDeleteTargetNames.Has(xName) toExclude := toExcludeTargetNames.Has(xName) @@ -271,7 +271,7 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj for _, target := range targets { ownedTargets.Insert(target.GetName()) - if _, exist := target.GetLabels()[TargetExcludeIndicationLabelKey]; exist { + if _, exist := r.xsetLabelMgr.Get(target.GetLabels(), api.EnumXSetTargetExcludeIndicationLabel); exist { excludeTargetNames.Insert(target.GetName()) } } @@ -351,7 +351,7 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject defer func() { syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) - syncContext.replacingMap = classifyTargetReplacingMapping(syncContext.activeTargets) + syncContext.replacingMap = classifyTargetReplacingMapping(r.xsetLabelMgr, syncContext.activeTargets) }() needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(ctx, syncContext.FilteredTarget) @@ -761,7 +761,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, } if updateFinished || finishByCancelUpdate { - if err := updater.FinishUpdateTarget(ctx, targetInfo); err != nil { + if err := updater.FinishUpdateTarget(ctx, targetInfo, finishByCancelUpdate); err != nil { return fmt.Errorf("failed to finish target %s/%s update: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) } r.Recorder.Eventf(targetInfo.Object, diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index fa0d246b..fd0a46d5 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -68,7 +68,7 @@ func (r *RealSyncControl) cleanReplaceTargetLabels( // replace finished, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete origin Target's ID if labelKey == r.xsetLabelMgr.Label(api.EnumXSetReplacePairOriginNameLabel) { needUpdateContext = true - newTargetId, _ := GetInstanceID(target) + newTargetId, _ := GetInstanceID(r.xsetLabelMgr, target) if originTargetContext, exist := mapOriginToNewTargetContext[newTargetId]; exist && originTargetContext != nil { r.resourceContextControl.Remove(originTargetContext, api.EnumReplaceNewTargetIDContextDataKey) if _, exist := currentIDs[originTargetContext.ID]; !exist { @@ -80,10 +80,10 @@ func (r *RealSyncControl) cleanReplaceTargetLabels( } } // replace canceled, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete new Target's ID - _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] + _, replaceIndicate := r.xsetLabelMgr.Get(target.GetLabels(), api.EnumXSetReplaceIndicationLabel) if !replaceIndicate && labelKey == r.xsetLabelMgr.Label(api.EnumXSetReplacePairNewIdLabel) { needUpdateContext = true - originTargetId, _ := GetInstanceID(target) + originTargetId, _ := GetInstanceID(r.xsetLabelMgr, target) if newTargetContext, exist := mapNewToOriginTargetContext[originTargetId]; exist && newTargetContext != nil { r.resourceContextControl.Remove(newTargetContext, api.EnumReplaceOriginTargetIDContextDataKey) if _, exist := currentIDs[newTargetContext.ID]; !exist { @@ -121,7 +121,7 @@ func (r *RealSyncControl) replaceOriginTargets( mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) successCount, err := controllerutils.SlowStartBatch(len(needReplaceOriginTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { originTarget := needReplaceOriginTargets[i] - originTargetId, _ := GetInstanceID(originTarget) + originTargetId, _ := GetInstanceID(r.xsetLabelMgr, originTarget) if ownedIDs[originTargetId] == nil { r.Recorder.Eventf(instance, corev1.EventTypeWarning, "OriginTargetContext", "cannot found resource context id %d of origin target %s/%s", originTargetId, originTarget.GetNamespace(), originTarget.GetName()) @@ -142,7 +142,7 @@ func (r *RealSyncControl) replaceOriginTargets( newTargetContext = contextDetail // reuse targetContext ID if pair-relation exists newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) - newTarget.GetLabels()[TargetInstanceIDLabelKey] = newInstanceId + r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetInstanceIdLabel, newInstanceId) logger.Info("replaceOriginTargets", "try to reuse new pod resourceContext id", newInstanceId) } else { if availableContexts[i] == nil { @@ -176,7 +176,7 @@ func (r *RealSyncControl) replaceOriginTargets( return err } - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, TargetReplacePairNewId, newInstanceId))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, r.xsetLabelMgr.Label(api.EnumXSetReplacePairNewIdLabel), newInstanceId))) if err = r.xControl.PatchTarget(ctx, originTarget, patch); err != nil { return fmt.Errorf("fail to update origin target %s/%s pair label %s when updating by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), newCreatedTarget.GetName(), err.Error()) } @@ -208,7 +208,7 @@ func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []clie for _, target := range targets { targetLabels := target.GetLabels() - if instanceId, ok := targetLabels[TargetInstanceIDLabelKey]; ok { + if instanceId, ok := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetInstanceIdLabel); ok { targetInstanceIdMap[instanceId] = target } targetNameMap[target.GetName()] = target @@ -219,7 +219,7 @@ func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []clie targetLabels := target.GetLabels() // no replace indication label - if _, exist := targetLabels[TargetReplaceIndicationLabelKey]; !exist { + if _, exist := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetReplaceIndicationLabel); !exist { continue } @@ -230,14 +230,14 @@ func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []clie } // target is replace new created target, skip replace - if originTargetName, exist := targetLabels[TargetReplacePairOriginName]; exist { + if originTargetName, exist := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetReplacePairOriginNameLabel); exist { if _, exist := targetNameMap[originTargetName]; exist { continue } } // target already has a new created target for replacement - if newPairTargetId, exist := targetLabels[TargetReplacePairNewId]; exist { + if newPairTargetId, exist := r.xsetLabelMgr.Get(targetLabels, api.EnumXSetReplacePairNewIdLabel); exist { if _, exist := targetInstanceIdMap[newPairTargetId]; exist { continue } @@ -363,7 +363,7 @@ func (r *RealSyncControl) getReplaceRevision(originTarget client.Object, syncCon } // classify the pair relationship for Target replacement. -func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string]*targetWrapper { +func classifyTargetReplacingMapping(xsetLabelMgr api.XSetLabelManager, targetWrappers []*targetWrapper) map[string]*targetWrapper { targetNameMap := make(map[string]*targetWrapper) targetIdMap := make(map[string]*targetWrapper) for _, targetWrapper := range targetWrappers { @@ -378,16 +378,17 @@ func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string] continue } name := targetWrapper.GetName() - if replacePairNewIdStr, exist := targetWrapper.GetLabels()[TargetReplacePairNewId]; exist { + if replacePairNewIdStr, exist := xsetLabelMgr.Get(targetWrapper.GetLabels(), api.EnumXSetReplacePairNewIdLabel); exist { if pairNewTarget, exist := targetIdMap[replacePairNewIdStr]; exist { replaceTargetMapping[name] = pairNewTarget // if one of pair targets is to Exclude, both targets should not scaleIn targetWrapper.ToExclude = targetWrapper.ToExclude || pairNewTarget.ToExclude continue } - } else if replaceOriginStr, exist := targetWrapper.GetLabels()[TargetReplacePairOriginName]; exist { + } else if replaceOriginStr, exist := xsetLabelMgr.Get(targetWrapper.GetLabels(), api.EnumXSetReplacePairOriginNameLabel); exist { if originTarget, exist := targetNameMap[replaceOriginStr]; exist { - if originTarget.GetLabels()[TargetReplacePairNewId] == strconv.Itoa(targetWrapper.ID) { + id, exist := xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplacePairNewIdLabel) + if exist && id == strconv.Itoa(targetWrapper.ID) { continue } } diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index f890bcab..c002c069 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -141,7 +141,7 @@ func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject api.XSet return err } - target.GetLabels()[TargetOrphanedIndicateLabelKey] = "true" + r.xsetLabelMgr.Set(target.GetLabels(), api.EnumXSetOrphanedLabel, "true") return r.xControl.OrphanTarget(xsetObject, target) } @@ -152,8 +152,8 @@ func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject api.XSet return err } - target.GetLabels()[TargetInstanceIDLabelKey] = instanceId - delete(target.GetLabels(), TargetOrphanedIndicateLabelKey) + r.xsetLabelMgr.Set(target.GetLabels(), api.EnumXSetInstanceIdLabel, instanceId) + r.xsetLabelMgr.Delete(target.GetLabels(), api.EnumXSetOrphanedLabel) return r.xControl.AdoptTarget(xsetObject, target) } diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index b4f95600..e1c4f520 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -291,7 +291,7 @@ type TargetUpdater interface { FilterAllowOpsTargets(ctx context.Context, targetToUpdate []*targetUpdateInfo, ownedIDs map[int]*api.ContextDetail, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error GetTargetUpdateFinishStatus(ctx context.Context, targetUpdateInfo *targetUpdateInfo) (bool, string, error) - FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error + FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error } type GenericTargetUpdater struct { @@ -323,7 +323,6 @@ func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext return err } } - return nil }) @@ -363,7 +362,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid } if _, exist := ownedIDs[targetInfo.ID]; !exist { - u.recorder.Eventf(u.OwnerObject, corev1.EventTypeWarning, "TargetBeforeUpdate", "target %s/%s is not allowed to update because cannot find context id %s in resourceContext", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetLabels()[TargetInstanceIDLabelKey]) + u.recorder.Eventf(u.OwnerObject, corev1.EventTypeWarning, "TargetBeforeUpdate", "target %s/%s is not allowed to update because cannot find context id %s in resourceContext", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetLabels()[u.xsetLabelMgr.Label(api.EnumXSetInstanceIdLabel)]) continue } @@ -398,7 +397,13 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid return recordedRequeueAfter, nil } -func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo) error { +func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error { + if finishByCancelUpdate { + // cancel update lifecycle + return opslifecycle.CancelOpsLifecycle(u.opsLifecycleLabelMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object) + } + + // target is ops finished, finish the lifecycle gracefully if updated, err := opslifecycle.Finish(u.opsLifecycleLabelMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { return fmt.Errorf("failed to finish TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) } else if updated { @@ -524,7 +529,7 @@ func (u *inPlaceIfPossibleUpdater) GetTargetUpdateFinishStatus(_ context.Context } targetLastState := &TargetStatus{} - if lastStateJson, exist := targetUpdateInfo.GetAnnotations()[LastTargetStatusAnnotationKey]; !exist { + if lastStateJson, exist := u.xsetLabelMgr.Get(targetUpdateInfo.GetAnnotations(), api.EnumXSetLastTargetStatusAnnotationKey); !exist { return false, "no target last state annotation", nil } else if err := json.Unmarshal([]byte(lastStateJson), targetLastState); err != nil { msg := fmt.Sprintf("malformat target last state annotation [%s]: %s", lastStateJson, err.Error()) @@ -572,7 +577,7 @@ func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, sync if exist && newTargetRevision == targetInfo.UpdateRevision.GetName() { return nil } - if _, exist := replacePairNewTarget.GetLabels()[TargetDeletionIndicationLabelKey]; exist { + if _, exist := u.xsetLabelMgr.Get(replacePairNewTarget.GetLabels(), api.EnumXSetDeletionIndicationLabel); exist { return nil } @@ -584,7 +589,7 @@ func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, sync replacePairNewTarget.GetName(), newTargetRevision, syncContext.UpdatedRevision.GetName()) - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, u.xsetLabelMgr.Label(api.EnumXSetDeletionIndicationLabel), time.Now().UnixNano()))) if patchErr := u.client.Patch(ctx, targetInfo.ReplacePairNewTargetInfo.Object, patch); patchErr != nil { err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", targetInfo.ReplacePairNewTargetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), patchErr.Error()) @@ -615,12 +620,12 @@ func (u *replaceUpdateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, func (u *replaceUpdateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { // add replace labels and wait to replace when syncTargets - _, replaceIndicate := targetInfo.Object.GetLabels()[TargetReplaceIndicationLabelKey] - _, replaceByUpdate := targetInfo.Object.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] + _, replaceIndicate := u.xsetLabelMgr.Get(targetInfo.Object.GetLabels(), api.EnumXSetReplaceIndicationLabel) + _, replaceByUpdate := u.xsetLabelMgr.Get(targetInfo.Object.GetLabels(), api.EnumXSetReplaceByReplaceUpdateLabel) if !replaceIndicate || !replaceByUpdate { // need replace update target, label target with replace-indicate and replace-update now := time.Now().UnixNano() - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, TargetReplaceIndicationLabelKey, now, TargetReplaceByReplaceUpdateLabelKey, targetInfo.UpdateRevision.GetName()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, u.xsetLabelMgr.Label(api.EnumXSetReplaceIndicationLabel), now, u.xsetLabelMgr.Label(api.EnumXSetReplaceByReplaceUpdateLabel), targetInfo.UpdateRevision.GetName()))) if err := u.client.Patch(ctx, targetInfo.Object, patch); err != nil { return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) } @@ -644,11 +649,11 @@ func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Conte return u.isTargetUpdatedServiceAvailable(replaceNewTargetInfo) } -func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { +func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error { ReplacePairNewTargetInfo := targetInfo.ReplacePairNewTargetInfo if ReplacePairNewTargetInfo != nil { - if _, exist := targetInfo.GetLabels()[TargetDeletionIndicationLabelKey]; !exist { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + if _, exist := u.xsetLabelMgr.Get(targetInfo.GetLabels(), api.EnumXSetDeletionIndicationLabel); !exist { + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, u.xsetLabelMgr.Label(api.EnumXSetDeletionIndicationLabel), time.Now().UnixNano()))) if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { return fmt.Errorf("failed to delete replace pair origin target %s/%s %s", targetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), err.Error()) } @@ -658,6 +663,7 @@ func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, tar } func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *targetUpdateInfo) (finished bool, msg string, err error) { + // TODO check decoration changed if targetInfo.GetLabels() == nil { return false, "no labels on target", nil } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index 19b2bcdf..c11217ae 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -29,14 +29,15 @@ import ( "kusionstack.io/kube-utils/xset/api" ) -func GetInstanceID(target client.Object) (int, error) { +func GetInstanceID(xsetLabelMgr api.XSetLabelManager, target client.Object) (int, error) { if target.GetLabels() == nil { return -1, fmt.Errorf("no labels found for instance ID") } - val, exist := target.GetLabels()[TargetInstanceIDLabelKey] + instanceIdLabelKey := xsetLabelMgr.Label(api.EnumXSetInstanceIdLabel) + val, exist := target.GetLabels()[instanceIdLabelKey] if !exist { - return -1, fmt.Errorf("failed to find instance ID label %s", TargetInstanceIDLabelKey) + return -1, fmt.Errorf("failed to find instance ID label %s", instanceIdLabelKey) } id, err := strconv.ParseInt(val, 10, 32) From 69aa976aa239607f283d07b85286c455cae83f44 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 26 Aug 2025 21:06:37 +0800 Subject: [PATCH 04/17] fix1 --- xset/api/xset_types.go | 2 ++ xset/synccontrols/label_manager.go | 1 + xset/synccontrols/sync_control.go | 13 +++++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index b90267f7..8d1ba71a 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -220,6 +220,8 @@ const ( EnumXSetTargetCreatingLabel + EnumXSetTargetCompletingLabel + EnumXSetTargetExcludeIndicationLabel EnumXSetLastTargetStatusAnnotationKey diff --git a/xset/synccontrols/label_manager.go b/xset/synccontrols/label_manager.go index a8c0809c..85a3bdfe 100644 --- a/xset/synccontrols/label_manager.go +++ b/xset/synccontrols/label_manager.go @@ -33,6 +33,7 @@ var defaultXSetControllerLabelManager = map[api.XSetControllerLabelEnum]string{ api.EnumXSetReplaceByReplaceUpdateLabel: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, api.EnumXSetOrphanedLabel: appsv1alpha1.PodOrphanedIndicateLabelKey, api.EnumXSetTargetCreatingLabel: appsv1alpha1.PodCreatingLabel, + api.EnumXSetTargetCompletingLabel: appsv1alpha1.PodCompletingLabel, api.EnumXSetTargetExcludeIndicationLabel: appsv1alpha1.PodExcludeIndicationLabelKey, api.EnumXSetLastTargetStatusAnnotationKey: appsv1alpha1.LastPodStatusAnnotationKey, } diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 5287b805..21c1872f 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "strconv" "strings" "sync/atomic" "time" @@ -464,7 +465,15 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // scale out new Targets with updatedRevision // TODO use cache // TODO decoration for target template - target, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, xsetObject, revision, availableIDContext.ID) + target, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, xsetObject, revision, availableIDContext.ID, + func(object client.Object) error { + if _, exist := r.resourceContextControl.Get(availableIDContext, api.EnumJustCreateContextDataKey); exist { + r.xsetLabelMgr.Set(object.GetLabels(), api.EnumXSetTargetCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + } else { + r.xsetLabelMgr.Set(object.GetLabels(), api.EnumXSetTargetCompletingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + } + return nil + }) if err != nil { return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) } @@ -937,7 +946,7 @@ func (r *RealSyncControl) BatchDeleteTargetsByLabel(ctx context.Context, targetC _, err := controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { target := needDeleteTargets[i] if _, exist := r.xsetLabelMgr.Get(target.GetLabels(), api.EnumXSetDeletionIndicationLabel); !exist { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%q":"%d"}}}`, r.xsetLabelMgr.Label(api.EnumXSetDeletionIndicationLabel), time.Now().UnixNano()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, r.xsetLabelMgr.Label(api.EnumXSetDeletionIndicationLabel), time.Now().UnixNano()))) // nolint if err := targetControl.PatchTarget(ctx, target, patch); err != nil { return fmt.Errorf("failed to delete target when syncTargets %s/%s/%w", target.GetNamespace(), target.GetName(), err) } From ae0950ad080c01e2af451b8483bc645d9e7113bf Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 26 Aug 2025 23:19:48 +0800 Subject: [PATCH 05/17] refactor default lifrcycle id --- xset/opslifecycle/default_adapters.go | 17 ++++++++++------- xset/synccontrols/sync_control.go | 4 ++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/xset/opslifecycle/default_adapters.go b/xset/opslifecycle/default_adapters.go index 323b6466..d8fe68fd 100644 --- a/xset/opslifecycle/default_adapters.go +++ b/xset/opslifecycle/default_adapters.go @@ -17,6 +17,9 @@ package opslifecycle import ( + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-utils/xset/api" @@ -26,10 +29,11 @@ var _ api.LifecycleAdapter = &DefaultUpdateLifecycleAdapter{} type DefaultUpdateLifecycleAdapter struct { LabelManager api.LifeCycleLabelManager + XSetType metav1.TypeMeta } func (d *DefaultUpdateLifecycleAdapter) GetID() string { - return "xset" + return strings.ToLower(d.XSetType.Kind) } func (d *DefaultUpdateLifecycleAdapter) GetType() api.OperationType { @@ -40,8 +44,7 @@ func (d *DefaultUpdateLifecycleAdapter) AllowMultiType() bool { return true } -func (d *DefaultUpdateLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - setOperate(d.LabelManager, d, target) +func (d *DefaultUpdateLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) { return true, nil } @@ -53,10 +56,11 @@ var _ api.LifecycleAdapter = &DefaultScaleInLifecycleAdapter{} type DefaultScaleInLifecycleAdapter struct { LabelManager api.LifeCycleLabelManager + XSetType metav1.TypeMeta } func (d *DefaultScaleInLifecycleAdapter) GetID() string { - return "xset" + return strings.ToLower(d.XSetType.Kind) } func (d *DefaultScaleInLifecycleAdapter) GetType() api.OperationType { @@ -68,10 +72,9 @@ func (d *DefaultScaleInLifecycleAdapter) AllowMultiType() bool { } func (d *DefaultScaleInLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - setOperate(d.LabelManager, d, target) - return true, nil + return WhenBeginDelete(d.LabelManager, target) } -func (d *DefaultScaleInLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { +func (d *DefaultScaleInLifecycleAdapter) WhenFinish(_ client.Object) (bool, error) { return false, nil } diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 21c1872f..ef951b82 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -74,11 +74,11 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, } scaleInOpsLifecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() if scaleInOpsLifecycleAdapter == nil { - scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} + scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager, XSetType: xsetController.XSetMeta()} } updateLifecycleAdapter := xsetController.GetUpdateOpsLifecycleAdapter() if updateLifecycleAdapter == nil { - updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{LabelManager: lifeCycleLabelManager} + updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{LabelManager: lifeCycleLabelManager, XSetType: xsetController.XSetMeta()} } xMeta := xsetController.XMeta() From 9fc8ff1cc83dbf917777cb7d9be383192e300c67 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 26 Aug 2025 23:23:01 +0800 Subject: [PATCH 06/17] add todo --- xset/opslifecycle/default_adapters.go | 1 + 1 file changed, 1 insertion(+) diff --git a/xset/opslifecycle/default_adapters.go b/xset/opslifecycle/default_adapters.go index d8fe68fd..6a6ce9a7 100644 --- a/xset/opslifecycle/default_adapters.go +++ b/xset/opslifecycle/default_adapters.go @@ -49,6 +49,7 @@ func (d *DefaultUpdateLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) } func (d *DefaultUpdateLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { + // TODO inplace update post actions return false, nil } From d96009919e5e44d7f079aaef735aa50f2ff45223 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 26 Aug 2025 23:35:36 +0800 Subject: [PATCH 07/17] refactor scale and update order --- xset/synccontrols/x_scale.go | 4 +++- xset/synccontrols/x_update.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index c002c069..441cd43c 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -115,9 +115,11 @@ func (s *ActiveTargetsForDeletion) Less(i, j int) bool { // TODO consider service available timestamps + // TODO abstract interface + lCreationTime := l.Object.GetCreationTimestamp().Time rCreationTime := r.Object.GetCreationTimestamp().Time - return lCreationTime.Before(rCreationTime) + return lCreationTime.After(rCreationTime) } // doIncludeExcludeTargets do real include and exclude for targets which are allowed to in/exclude diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index e1c4f520..81029160 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -263,9 +263,11 @@ func (o *orderByDefault) Less(i, j int) bool { return lReady } + // TODO abstract interface + lCreationTime := l.Object.GetCreationTimestamp().Time rCreationTime := r.Object.GetCreationTimestamp().Time - return lCreationTime.Before(rCreationTime) + return lCreationTime.After(rCreationTime) } type UpdateConfig struct { From d652245cd5460b5b3f1f056c03e9a08166e1d841 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Wed, 27 Aug 2025 16:16:19 +0800 Subject: [PATCH 08/17] fix decideTargetToUpdate --- xset/synccontrols/x_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 81029160..6d2faa2a 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -166,7 +166,7 @@ func (r *RealSyncControl) decideTargetToUpdate(xsetController api.XSetController return r.decideTargetToUpdateByLabel(activeTargetInfos) } - return r.decideTargetToUpdateByPartition(xsetController, xset, targetInfos) + return r.decideTargetToUpdateByPartition(xsetController, xset, filteredPodInfos) } func (r *RealSyncControl) decideTargetToUpdateByLabel(targetInfos []*targetUpdateInfo) (targetToUpdate []*targetUpdateInfo) { From a6bc913767bfac7a5e6d0e2bf1bd45c7c48d087a Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Wed, 27 Aug 2025 17:07:47 +0800 Subject: [PATCH 09/17] fix replace UpgradeTarget --- xset/synccontrols/x_update.go | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 6d2faa2a..fb915b71 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -621,25 +621,7 @@ func (u *replaceUpdateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, } func (u *replaceUpdateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { - // add replace labels and wait to replace when syncTargets - _, replaceIndicate := u.xsetLabelMgr.Get(targetInfo.Object.GetLabels(), api.EnumXSetReplaceIndicationLabel) - _, replaceByUpdate := u.xsetLabelMgr.Get(targetInfo.Object.GetLabels(), api.EnumXSetReplaceByReplaceUpdateLabel) - if !replaceIndicate || !replaceByUpdate { - // need replace update target, label target with replace-indicate and replace-update - now := time.Now().UnixNano() - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, u.xsetLabelMgr.Label(api.EnumXSetReplaceIndicationLabel), now, u.xsetLabelMgr.Label(api.EnumXSetReplaceByReplaceUpdateLabel), targetInfo.UpdateRevision.GetName()))) - if err := u.client.Patch(ctx, targetInfo.Object, patch); err != nil { - return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) - } - u.recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "UpdateTarget", - "succeed to update Target %s/%s by label to-replace", - targetInfo.GetNamespace(), - targetInfo.GetName(), - ) - } - return nil + return updateReplaceOriginTarget(ctx, u.client, u.recorder, u.xsetLabelMgr, targetInfo, targetInfo.ReplacePairNewTargetInfo) } func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetUpdateInfo *targetUpdateInfo) (finished bool, msg string, err error) { From 0c5656881050e9a883545d1b72fcbceb6cd73662 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Wed, 27 Aug 2025 17:25:48 +0800 Subject: [PATCH 10/17] fix FinishUpdateTarget --- xset/synccontrols/sync_control.go | 2 +- xset/synccontrols/x_update.go | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index ef951b82..5a19e9ba 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -740,7 +740,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, succCount, err = controllerutils.SlowStartBatch(len(targetUpdateInfos), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { targetInfo := targetUpdateInfos[i] - if !(targetInfo.IsDuringUpdateOps || targetInfo.PlaceHolder) || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { + if !(targetInfo.IsDuringUpdateOps || targetInfo.IsInReplaceUpdate) || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { return nil } diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index fb915b71..6bcf507d 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -634,6 +634,17 @@ func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Conte } func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error { + if finishByCancelUpdate { + // cancel replace update by removing to-replace and replace-by-update label from origin model + if targetInfo.IsInReplace { + patch := client.RawPatch(types.MergePatchType, fmt.Appendf(nil, `{"metadata":{"labels":{"%s":null, "%s":null}}}`, u.xsetLabelMgr.Label(api.EnumXSetReplaceIndicationLabel), u.xsetLabelMgr.Label(api.EnumXSetReplaceByReplaceUpdateLabel))) + if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { + return fmt.Errorf("failed to patch replace pair origin model %s/%s %w when cancel replace update", targetInfo.GetNamespace(), targetInfo.GetName(), err) + } + return u.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(u.OwnerObject), u.targetGVK, targetInfo.Object.GetNamespace(), targetInfo.Object.GetName(), targetInfo.Object.GetResourceVersion()) + } + } + ReplacePairNewTargetInfo := targetInfo.ReplacePairNewTargetInfo if ReplacePairNewTargetInfo != nil { if _, exist := u.xsetLabelMgr.Get(targetInfo.GetLabels(), api.EnumXSetDeletionIndicationLabel); !exist { From 9c2e4020abe5eb4305e6f7dcb9ca9e0d47ca25a5 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Wed, 27 Aug 2025 18:15:32 +0800 Subject: [PATCH 11/17] fix FinishUpdateTarget --- xset/synccontrols/x_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 6bcf507d..103aec05 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -641,8 +641,8 @@ func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, tar if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { return fmt.Errorf("failed to patch replace pair origin model %s/%s %w when cancel replace update", targetInfo.GetNamespace(), targetInfo.GetName(), err) } - return u.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(u.OwnerObject), u.targetGVK, targetInfo.Object.GetNamespace(), targetInfo.Object.GetName(), targetInfo.Object.GetResourceVersion()) } + return nil } ReplacePairNewTargetInfo := targetInfo.ReplacePairNewTargetInfo From 260ec2639b162633aeb2758ba8cadbb8a7f1bceb Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Thu, 28 Aug 2025 15:50:27 +0800 Subject: [PATCH 12/17] add template patcher --- xset/api/xset_controller_types.go | 1 + xset/synccontrols/sync_control.go | 4 +++- xset/synccontrols/x_replace.go | 3 ++- xset/synccontrols/x_utils.go | 14 ++++++++++++++ xset/xset_controller.go | 3 ++- 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index 6010b07c..fe3977d8 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -35,6 +35,7 @@ type XSetController interface { GetXSetSpec(object XSetObject) *XSetSpec GetXSetPatch(object metav1.Object) ([]byte, error) + GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error UpdateScaleStrategy(object XSetObject, scaleStrategy *ScaleStrategy) (err error) GetXSetStatus(object XSetObject) *XSetStatus SetXSetStatus(object XSetObject, status *XSetStatus) diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 5a19e9ba..3be1d254 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -473,7 +473,9 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, r.xsetLabelMgr.Set(object.GetLabels(), api.EnumXSetTargetCompletingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) } return nil - }) + }, + r.xsetController.GetXSetTemplatePatcher(xsetObject), + ) if err != nil { return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) } diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index fd0a46d5..5693d479 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -131,7 +131,8 @@ func (r *RealSyncControl) replaceOriginTargets( replaceRevision := r.getReplaceRevision(originTarget, syncContext) // create target using update revision if replaced by update, otherwise using current revision - newTarget, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, instance, replaceRevision, originTargetId) + newTarget, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, instance, replaceRevision, originTargetId, + r.xsetController.GetXSetTemplatePatcher(instance)) if err != nil { return err } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index c11217ae..84a3fc65 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -17,6 +17,7 @@ package synccontrols import ( + "context" "fmt" "strconv" "time" @@ -26,6 +27,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + clientutils "kusionstack.io/kube-utils/client" + controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" ) @@ -180,3 +183,14 @@ func IsControlledByXSet(xsetLabelManager api.XSetLabelManager, obj client.Object v, ok := xsetLabelManager.Get(obj.GetLabels(), api.EnumXSetControlledLabel) return ok && v == "true" } + +func ApplyTemplatePatcher(ctx context.Context, xsetController api.XSetController, c client.Client, xset api.XSetObject, targets []targetWrapper) error { + _, patchErr := controllerutils.SlowStartBatch(len(targets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + if targets[i].Object == nil || targets[i].PlaceHolder { + return nil + } + _, err := clientutils.UpdateOnConflict(ctx, c, c, targets[i].Object, xsetController.GetXSetTemplatePatcher(xset)) + return err + }) + return patchErr +} diff --git a/xset/xset_controller.go b/xset/xset_controller.go index 99a2f313..f8322c62 100644 --- a/xset/xset_controller.go +++ b/xset/xset_controller.go @@ -242,8 +242,9 @@ func (r *xSetCommonReconciler) doSync(ctx context.Context, instance api.XSetObje _, scaleRequeueAfter, scaleErr := r.syncControl.Scale(ctx, instance, syncContext) _, updateRequeueAfter, updateErr := r.syncControl.Update(ctx, instance, syncContext) + patcherErr := synccontrols.ApplyTemplatePatcher(ctx, r.XSetController, r.Client, instance, syncContext.TargetWrappers) - err = errors.Join(scaleErr, updateErr) + err = errors.Join(scaleErr, updateErr, patcherErr) if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { return updateRequeueAfter, err } From e5e18065b0745c450d3bb392f9a8c9cb18765389 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Thu, 28 Aug 2025 20:54:26 +0800 Subject: [PATCH 13/17] add ops priority --- xset/api/xset_controller_types.go | 6 +++++- xset/api/xset_types.go | 8 +++++++ xset/opslifecycle/utils.go | 5 ----- xset/synccontrols/sync_control.go | 36 ++++++++++++++++++++++++++----- xset/synccontrols/types.go | 4 +++- xset/synccontrols/x_replace.go | 4 ++-- xset/synccontrols/x_scale.go | 4 ++-- xset/synccontrols/x_update.go | 8 +++---- xset/synccontrols/x_utils.go | 2 +- 9 files changed, 56 insertions(+), 21 deletions(-) diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index fe3977d8..8e0edf10 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -17,6 +17,8 @@ package api import ( + "context" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,7 +38,6 @@ type XSetController interface { GetXSetSpec(object XSetObject) *XSetSpec GetXSetPatch(object metav1.Object) ([]byte, error) GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error - UpdateScaleStrategy(object XSetObject, scaleStrategy *ScaleStrategy) (err error) GetXSetStatus(object XSetObject) *XSetStatus SetXSetStatus(object XSetObject, status *XSetStatus) @@ -49,6 +50,9 @@ type XSetController interface { CheckScheduled(object client.Object) bool CheckReady(object client.Object) bool CheckAvailable(object client.Object) bool + + UpdateScaleStrategy(ctx context.Context, c client.Client, object XSetObject, scaleStrategy *ScaleStrategy) error + GetXOpsPriority(ctx context.Context, c client.Client, object client.Object) (*OpsPriority, error) } type XSetObject client.Object diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index 8d1ba71a..2b304cad 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -197,6 +197,14 @@ type XSetStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } +// OpsPriority is used to store the ops priority of a target +type OpsPriority struct { + // PriorityClass is the priority class of the target + PriorityClass int32 + // DeletionCost is the deletion cost of the target + DeletionCost int32 +} + type XSetControllerLabelEnum int const ( diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go index ec929b28..2000061a 100644 --- a/xset/opslifecycle/utils.go +++ b/xset/opslifecycle/utils.go @@ -31,11 +31,6 @@ import ( type UpdateFunc func(object client.Object) (bool, error) -func IsServiceAvailable(m api.LifeCycleLabelManager, target client.Object) bool { - _, exists := target.GetLabels()[m.Get(api.ServiceAvailableLabel)] - return exists -} - // IDToLabelsMap returns a map of pod id to labels map and a map of operation type to number of pods. func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[string]string, map[string]int, error) { idToLabelsMap := map[string]map[string]string{} diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 3be1d254..052bfc3f 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -168,7 +168,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje } // stateless case - var targetWrappers []targetWrapper + var targetWrappers []*targetWrapper syncContext.CurrentIDs = sets.Int{} idToReclaim := sets.Int{} toDeleteTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) @@ -209,7 +209,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje // TODO delete unused pvcs (for pods) - targetWrappers = append(targetWrappers, targetWrapper{ + targetWrappers = append(targetWrappers, &targetWrapper{ Object: target, ID: id, ContextDetail: ownedIDs[id], @@ -398,7 +398,7 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject if _, inUsed := syncContext.CurrentIDs[id]; inUsed { continue } - syncContext.TargetWrappers = append(syncContext.TargetWrappers, targetWrapper{ + syncContext.TargetWrappers = append(syncContext.TargetWrappers, &targetWrapper{ ID: id, Object: nil, ContextDetail: contextDetail, @@ -508,6 +508,10 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } if diff <= 0 { + // get targets ops priority + if err := r.getTargetsOpsPriority(ctx, r.Client, syncContext.activeTargets); err != nil { + return false, recordedRequeueAfter, err + } // chose the targets to scale in targetsToScaleIn := r.getTargetsToDelete(xsetObject, syncContext.activeTargets, syncContext.replacingMap, diff*-1) // filter out Targets need to trigger TargetOpsLifecycle @@ -646,6 +650,12 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, logger := logr.FromContext(ctx) var err error var recordedRequeueAfter *time.Duration + + // 0. get targets ops priority + if err := r.getTargetsOpsPriority(ctx, r.Client, syncContext.TargetWrappers); err != nil { + return false, recordedRequeueAfter, err + } + // 1. scan and analysis targets update info for active targets and PlaceHolder targets targetUpdateInfos := r.attachTargetUpdateInfo(xsetObject, syncContext) @@ -920,14 +930,30 @@ func (r *RealSyncControl) reclaimOwnedIDs( return nil } +// getTargetsOpsPriority try to set targets' ops priority +func (r *RealSyncControl) getTargetsOpsPriority(ctx context.Context, c client.Client, targets []*targetWrapper) error { + _, err := controllerutils.SlowStartBatch(len(targets), controllerutils.SlowStartInitialBatchSize, true, func(i int, _ error) error { + if targets[i].PlaceHolder || targets[i].Object == nil || targets[i].OpsPriority != nil { + return nil + } + var iErr error + targets[i].OpsPriority, iErr = r.xsetController.GetXOpsPriority(ctx, c, targets[i].Object) + if iErr != nil { + return fmt.Errorf("failed to get target %s/%s ops priority: %w", targets[i].Object.GetNamespace(), targets[i].Object.GetName(), iErr) + } + return nil + }) + return err +} + // FilterOutActiveTargetWrappers filter out non placeholder targets -func FilterOutActiveTargetWrappers(targets []targetWrapper) []*targetWrapper { +func FilterOutActiveTargetWrappers(targets []*targetWrapper) []*targetWrapper { var filteredTargetWrappers []*targetWrapper for i, target := range targets { if target.PlaceHolder { continue } - filteredTargetWrappers = append(filteredTargetWrappers, &targets[i]) + filteredTargetWrappers = append(filteredTargetWrappers, targets[i]) } return filteredTargetWrappers } diff --git a/xset/synccontrols/types.go b/xset/synccontrols/types.go index f90a68bc..e84969f1 100644 --- a/xset/synccontrols/types.go +++ b/xset/synccontrols/types.go @@ -33,7 +33,7 @@ type SyncContext struct { ExistingSubResource []client.Object FilteredTarget []client.Object - TargetWrappers []targetWrapper + TargetWrappers []*targetWrapper activeTargets []*targetWrapper replacingMap map[string]*targetWrapper @@ -55,6 +55,8 @@ type targetWrapper struct { IsDuringScaleInOps bool IsDuringUpdateOps bool + + OpsPriority *api.OpsPriority } type targetUpdateInfo struct { diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index 5693d479..44edce40 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -259,12 +259,12 @@ func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []clie needCleanLabels = append(needCleanLabels, r.xsetLabelMgr.Label(api.EnumXSetReplacePairOriginNameLabel)) } else if _, exist := r.xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplaceIndicationLabel); !exist { // replace canceled, delete replace new target if new target is not service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); !serviceAvailable { + if r.xsetController.CheckAvailable(target) { needDeleteTargets = append(needDeleteTargets, target) } } else if !replaceByUpdate { // not replace update, delete origin target when new created target is service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); serviceAvailable { + if r.xsetController.CheckAvailable(target) { needDeleteTargets = append(needDeleteTargets, originTarget) } } diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 441cd43c..3dd747eb 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -71,7 +71,7 @@ func (r *RealSyncControl) getTargetsToDelete(xsetObject api.XSetObject, filtered continue } // when scaleIn origin Target, newTarget should be deleted if not service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleLabelMgr, target); !serviceAvailable { + if r.xsetController.CheckAvailable(target) { needDeleteTargets = append(needDeleteTargets, replacePairTarget) } } @@ -174,7 +174,7 @@ func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedTarge toIncludeTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToInclude...) notIncludeTargets := toIncludeTargetNames.Delete(includedTargets.List()...) xspec.ScaleStrategy.TargetToInclude = notIncludeTargets.List() - if err := r.xsetController.UpdateScaleStrategy(xsetObject, &xspec.ScaleStrategy); err != nil { + if err := r.xsetController.UpdateScaleStrategy(ctx, r.Client, xsetObject, &xspec.ScaleStrategy); err != nil { return err } // update xsetObject.spec.scaleStrategy diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 103aec05..4c0e7d29 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -52,7 +52,7 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync for i, target := range activeTargets { updateInfo := &targetUpdateInfo{ - targetWrapper: &syncContext.TargetWrappers[i], + targetWrapper: syncContext.TargetWrappers[i], } // TODO decoration for target template @@ -114,7 +114,7 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync if replacePairNewTarget != nil { // origin target is allowed to ops if new pod is serviceAvailable - _, newTargetSa := replacePairNewTarget.GetLabels()[r.updateConfig.opsLifecycleLabelMgr.Get(api.ServiceAvailableLabel)] + newTargetSa := r.xsetController.CheckAvailable(replacePairNewTarget.Object) originTargetInfo.IsAllowUpdateOps = originTargetInfo.IsAllowUpdateOps || newTargetSa // attach replace new target updateInfo ReplacePairNewTargetInfo := targetUpdateInfoMap[replacePairNewTarget.GetName()] @@ -133,7 +133,7 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync continue } updateInfo := &targetUpdateInfo{ - targetWrapper: &target, + targetWrapper: target, UpdateRevision: syncContext.UpdatedRevision, } if revision, exist := r.resourceContextControl.Get(target.ContextDetail, api.EnumRevisionContextDataKey); exist && @@ -666,7 +666,7 @@ func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *targe return false, "replace origin target", nil } - if serviceAvailable := opslifecycle.IsServiceAvailable(u.opsLifecycleLabelMgr, targetInfo.Object); serviceAvailable { + if u.xsetController.CheckAvailable(targetInfo.Object) { return true, "", nil } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index 84a3fc65..1839539d 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -184,7 +184,7 @@ func IsControlledByXSet(xsetLabelManager api.XSetLabelManager, obj client.Object return ok && v == "true" } -func ApplyTemplatePatcher(ctx context.Context, xsetController api.XSetController, c client.Client, xset api.XSetObject, targets []targetWrapper) error { +func ApplyTemplatePatcher(ctx context.Context, xsetController api.XSetController, c client.Client, xset api.XSetObject, targets []*targetWrapper) error { _, patchErr := controllerutils.SlowStartBatch(len(targets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { if targets[i].Object == nil || targets[i].PlaceHolder { return nil From fe2ae4d21a0bd0e40225864d3760670862a3af82 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Thu, 28 Aug 2025 21:26:42 +0800 Subject: [PATCH 14/17] fix service available --- xset/synccontrols/x_replace.go | 2 +- xset/synccontrols/x_scale.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index 44edce40..eaa8b5cd 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -259,7 +259,7 @@ func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []clie needCleanLabels = append(needCleanLabels, r.xsetLabelMgr.Label(api.EnumXSetReplacePairOriginNameLabel)) } else if _, exist := r.xsetLabelMgr.Get(originTarget.GetLabels(), api.EnumXSetReplaceIndicationLabel); !exist { // replace canceled, delete replace new target if new target is not service available - if r.xsetController.CheckAvailable(target) { + if !r.xsetController.CheckAvailable(target) { needDeleteTargets = append(needDeleteTargets, target) } } else if !replaceByUpdate { diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 3dd747eb..732f8cbd 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -71,7 +71,7 @@ func (r *RealSyncControl) getTargetsToDelete(xsetObject api.XSetObject, filtered continue } // when scaleIn origin Target, newTarget should be deleted if not service available - if r.xsetController.CheckAvailable(target) { + if !r.xsetController.CheckAvailable(target.Object) { needDeleteTargets = append(needDeleteTargets, replacePairTarget) } } From 73322f782b07b289a385876804bf29aa441af7d3 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Fri, 29 Aug 2025 10:28:23 +0800 Subject: [PATCH 15/17] refactor order of update and scale --- xset/api/xset_controller_types.go | 1 + xset/api/xset_types.go | 2 +- xset/synccontrols/x_scale.go | 37 +++++++++++++++++++++++-------- xset/synccontrols/x_update.go | 35 +++++++++++++++++++---------- xset/synccontrols/x_utils.go | 23 +++++++++++++++++++ 5 files changed, 76 insertions(+), 22 deletions(-) diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index 8e0edf10..22b04428 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -40,6 +40,7 @@ type XSetController interface { GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error GetXSetStatus(object XSetObject) *XSetStatus SetXSetStatus(object XSetObject, status *XSetStatus) + GetReadyTime(object client.Object) *metav1.Time GetLifeCycleLabelManager() LifeCycleLabelManager GetXSetControllerLabelManager() XSetLabelManager diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index 2b304cad..1f42e30d 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -188,7 +188,7 @@ type XSetStatus struct { AvailableReplicas int32 `json:"availableReplicas,omitempty"` // UpdatedAvailableReplicas indicates the number of available updated revision replicas for this replicas set. - // A model is updated available means the model is ready for updated revision and accessible + // A target is updated available means the target is ready for updated revision and accessible // +optional UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"` diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 732f8cbd..3da81ab5 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -22,10 +22,12 @@ import ( "sort" "strconv" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" clientutil "kusionstack.io/kube-utils/client" controllerutils "kusionstack.io/kube-utils/controller/utils" @@ -45,7 +47,7 @@ func (r *RealSyncControl) getTargetsToDelete(xsetObject api.XSetObject, filtered } // 1. select targets to delete in first round according to diff - sort.Sort(newActiveTargetsForDeletion(countedTargets)) + sort.Sort(newActiveTargetsForDeletion(countedTargets, r.xsetController.CheckReady, r.xsetController.GetReadyTime)) if diff > len(countedTargets) { diff = len(countedTargets) } @@ -82,12 +84,20 @@ func (r *RealSyncControl) getTargetsToDelete(xsetObject api.XSetObject, filtered } type ActiveTargetsForDeletion struct { - targets []*targetWrapper + targets []*targetWrapper + checkReadyFunc func(object client.Object) bool + getReadyTimeFunc func(object client.Object) *metav1.Time } -func newActiveTargetsForDeletion(targets []*targetWrapper) *ActiveTargetsForDeletion { +func newActiveTargetsForDeletion( + targets []*targetWrapper, + checkReadyFunc func(object client.Object) bool, + getReadyTimeFunc func(object client.Object) *metav1.Time, +) *ActiveTargetsForDeletion { return &ActiveTargetsForDeletion{ - targets: targets, + targets: targets, + checkReadyFunc: checkReadyFunc, + getReadyTimeFunc: getReadyTimeFunc, } } @@ -113,13 +123,22 @@ func (s *ActiveTargetsForDeletion) Less(i, j int) bool { return l.IsDuringScaleInOps } - // TODO consider service available timestamps + lReady, rReady := s.checkReadyFunc(l.Object), s.checkReadyFunc(r.Object) + if lReady != rReady { + return lReady + } - // TODO abstract interface + if l.OpsPriority != nil && r.OpsPriority != nil { + if l.OpsPriority.PriorityClass != r.OpsPriority.PriorityClass { + return l.OpsPriority.PriorityClass < r.OpsPriority.PriorityClass + } + if l.OpsPriority.DeletionCost != r.OpsPriority.DeletionCost { + return l.OpsPriority.DeletionCost < r.OpsPriority.DeletionCost + } + } - lCreationTime := l.Object.GetCreationTimestamp().Time - rCreationTime := r.Object.GetCreationTimestamp().Time - return lCreationTime.After(rCreationTime) + // TODO consider service available timestamps + return CompareTarget(l, r, s.checkReadyFunc, s.getReadyTimeFunc) } // doIncludeExcludeTargets do real include and exclude for targets which are allowed to in/exclude diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 4c0e7d29..041bd315 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -200,7 +200,7 @@ func (r *RealSyncControl) decideTargetToUpdateByPartition(xsetController api.XSe } // partial update replicas - ordered := newOrderedTargetUpdateInfos(filteredTargetInfos, xsetController.CheckReady) + ordered := newOrderedTargetUpdateInfos(filteredTargetInfos, xsetController.CheckReady, xsetController.GetReadyTime) sort.Sort(ordered) targetToUpdate := ordered.targets[:replicas-partition] return targetToUpdate @@ -222,16 +222,22 @@ func (r *RealSyncControl) getTargetsUpdateTargets(targetInfos []*targetUpdateInf return filteredTargetInfos } -func newOrderedTargetUpdateInfos(targetInfos []*targetUpdateInfo, checkReadyFunc func(object client.Object) bool) *orderByDefault { +func newOrderedTargetUpdateInfos( + targetInfos []*targetUpdateInfo, + checkReadyFunc func(object client.Object) bool, + getReadyTimeFunc func(object client.Object) *metav1.Time, +) *orderByDefault { return &orderByDefault{ - targets: targetInfos, - checkReadyFunc: checkReadyFunc, + targets: targetInfos, + checkReadyFunc: checkReadyFunc, + getReadyTimeFunc: getReadyTimeFunc, } } type orderByDefault struct { - targets []*targetUpdateInfo - checkReadyFunc func(object client.Object) bool + targets []*targetUpdateInfo + checkReadyFunc func(object client.Object) bool + getReadyTimeFunc func(object client.Object) *metav1.Time } func (o *orderByDefault) Len() int { @@ -263,11 +269,16 @@ func (o *orderByDefault) Less(i, j int) bool { return lReady } - // TODO abstract interface + if l.OpsPriority != nil && r.OpsPriority != nil { + if l.OpsPriority.PriorityClass != r.OpsPriority.PriorityClass { + return l.OpsPriority.PriorityClass < r.OpsPriority.PriorityClass + } + if l.OpsPriority.DeletionCost != r.OpsPriority.DeletionCost { + return l.OpsPriority.DeletionCost < r.OpsPriority.DeletionCost + } + } - lCreationTime := l.Object.GetCreationTimestamp().Time - rCreationTime := r.Object.GetCreationTimestamp().Time - return lCreationTime.After(rCreationTime) + return CompareTarget(l, r, o.checkReadyFunc, o.getReadyTimeFunc) } type UpdateConfig struct { @@ -635,11 +646,11 @@ func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Conte func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error { if finishByCancelUpdate { - // cancel replace update by removing to-replace and replace-by-update label from origin model + // cancel replace update by removing to-replace and replace-by-update label from origin target if targetInfo.IsInReplace { patch := client.RawPatch(types.MergePatchType, fmt.Appendf(nil, `{"metadata":{"labels":{"%s":null, "%s":null}}}`, u.xsetLabelMgr.Label(api.EnumXSetReplaceIndicationLabel), u.xsetLabelMgr.Label(api.EnumXSetReplaceByReplaceUpdateLabel))) if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { - return fmt.Errorf("failed to patch replace pair origin model %s/%s %w when cancel replace update", targetInfo.GetNamespace(), targetInfo.GetName(), err) + return fmt.Errorf("failed to patch replace pair target %s/%s %w when cancel replace update", targetInfo.GetNamespace(), targetInfo.GetName(), err) } } return nil diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index 1839539d..67a11fc9 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -194,3 +194,26 @@ func ApplyTemplatePatcher(ctx context.Context, xsetController api.XSetController }) return patchErr } + +func CompareTarget(l, r client.Object, + checkReadyFunc func(object client.Object) bool, + getReadyTimeFunc func(object client.Object) *metav1.Time, +) bool { + // If both targets are ready, the latest ready one is smaller + if checkReadyFunc(l) && checkReadyFunc(r) && !getReadyTimeFunc(l).Equal(getReadyTimeFunc(r)) { + return afterOrZero(getReadyTimeFunc(l), getReadyTimeFunc(r)) + } + // Empty creation time targets < newer targets < older targets + lCreationTime, rCreationTime := l.GetCreationTimestamp(), r.GetCreationTimestamp() + if !(&lCreationTime).Equal(&rCreationTime) { + return afterOrZero(&lCreationTime, &rCreationTime) + } + return false +} + +func afterOrZero(t1, t2 *metav1.Time) bool { + if t1.Time.IsZero() || t2.Time.IsZero() { + return t1.Time.IsZero() + } + return t1.After(t2.Time) +} From 50647a2a33357b749be24bcad22c26837738ba30 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Fri, 29 Aug 2025 11:12:03 +0800 Subject: [PATCH 16/17] fix compare target --- xset/synccontrols/x_scale.go | 2 +- xset/synccontrols/x_update.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 3da81ab5..ce668d5f 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -138,7 +138,7 @@ func (s *ActiveTargetsForDeletion) Less(i, j int) bool { } // TODO consider service available timestamps - return CompareTarget(l, r, s.checkReadyFunc, s.getReadyTimeFunc) + return CompareTarget(l.Object, r.Object, s.checkReadyFunc, s.getReadyTimeFunc) } // doIncludeExcludeTargets do real include and exclude for targets which are allowed to in/exclude diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 041bd315..e60aaf11 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -278,7 +278,7 @@ func (o *orderByDefault) Less(i, j int) bool { } } - return CompareTarget(l, r, o.checkReadyFunc, o.getReadyTimeFunc) + return CompareTarget(l.Object, r.Object, o.checkReadyFunc, o.getReadyTimeFunc) } type UpdateConfig struct { From 78cb736929d1a2044a2e65bb8a5045856123c929 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Wed, 3 Sep 2025 16:49:47 +0800 Subject: [PATCH 17/17] fix label set --- xset/api/xset_types.go | 3 ++- xset/opslifecycle/utils.go | 8 ++++---- xset/synccontrols/label_manager.go | 9 +++++---- xset/synccontrols/sync_control.go | 4 ++-- xset/synccontrols/x_replace.go | 8 ++++---- xset/synccontrols/x_scale.go | 4 ++-- xset/synccontrols/x_utils.go | 7 +++---- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index 1f42e30d..823c1cd1 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -20,6 +20,7 @@ package api import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) type XSetConditionType string @@ -237,7 +238,7 @@ const ( type XSetLabelManager interface { Get(labels map[string]string, labelType XSetControllerLabelEnum) (string, bool) - Set(labels map[string]string, labelType XSetControllerLabelEnum, value string) + Set(obj client.Object, labelType XSetControllerLabelEnum, value string) Delete(labels map[string]string, labelType XSetControllerLabelEnum) Label(labelType XSetControllerLabelEnum) string } diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go index 2000061a..560dee1f 100644 --- a/xset/opslifecycle/utils.go +++ b/xset/opslifecycle/utils.go @@ -31,7 +31,7 @@ import ( type UpdateFunc func(object client.Object) (bool, error) -// IDToLabelsMap returns a map of pod id to labels map and a map of operation type to number of pods. +// IDToLabelsMap returns a map of target id to labels map and a map of operation type to number of targets. func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[string]string, map[string]int, error) { idToLabelsMap := map[string]map[string]string{} typeToNumsMap := map[string]int{} @@ -72,7 +72,7 @@ func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[st return idToLabelsMap, typeToNumsMap, nil } -// NumOfLifecycleOnTarget returns the nums of lifecycles on pod +// NumOfLifecycleOnTarget returns the nums of lifecycles on target func NumOfLifecycleOnTarget(m *LabelManagerImpl, target client.Object) (int, error) { if target == nil { return 0, nil @@ -310,9 +310,9 @@ func CancelOpsLifecycle(m api.LifeCycleLabelManager, client client.Client, adapt return nil } - // only cancel when lifecycle exist on pod + // only cancel when lifecycle exist on target if exist, err := IsLifecycleOnTarget(m, adapter.GetID(), target); err != nil { - return fmt.Errorf("fail to check %s PodOpsLifecycle on Pod %s/%s: %w", adapter.GetID(), target.GetNamespace(), target.GetName(), err) + return fmt.Errorf("fail to check %s TargetOpsLifecycle on Target %s/%s: %w", adapter.GetID(), target.GetNamespace(), target.GetName(), err) } else if !exist { return nil } diff --git a/xset/synccontrols/label_manager.go b/xset/synccontrols/label_manager.go index 85a3bdfe..4f1f3a18 100644 --- a/xset/synccontrols/label_manager.go +++ b/xset/synccontrols/label_manager.go @@ -18,6 +18,7 @@ package synccontrols import ( appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-utils/xset/api" ) @@ -57,12 +58,12 @@ func (m *xSetControllerLabelManager) Get(labels map[string]string, key api.XSetC return val, exist } -func (m *xSetControllerLabelManager) Set(labels map[string]string, key api.XSetControllerLabelEnum, val string) { - if labels == nil { - labels = make(map[string]string) +func (m *xSetControllerLabelManager) Set(obj client.Object, key api.XSetControllerLabelEnum, val string) { + if obj.GetLabels() == nil { + obj.SetLabels(map[string]string{}) } labelKey := m.labelManager[key] - labels[labelKey] = val + obj.GetLabels()[labelKey] = val } func (m *xSetControllerLabelManager) Delete(labels map[string]string, key api.XSetControllerLabelEnum) { diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 052bfc3f..b4412604 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -468,9 +468,9 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, target, err := NewTargetFrom(r.xsetController, r.xsetLabelMgr, xsetObject, revision, availableIDContext.ID, func(object client.Object) error { if _, exist := r.resourceContextControl.Get(availableIDContext, api.EnumJustCreateContextDataKey); exist { - r.xsetLabelMgr.Set(object.GetLabels(), api.EnumXSetTargetCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + r.xsetLabelMgr.Set(object, api.EnumXSetTargetCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) } else { - r.xsetLabelMgr.Set(object.GetLabels(), api.EnumXSetTargetCompletingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + r.xsetLabelMgr.Set(object, api.EnumXSetTargetCompletingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) } return nil }, diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index eaa8b5cd..b82b7c2b 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -143,7 +143,7 @@ func (r *RealSyncControl) replaceOriginTargets( newTargetContext = contextDetail // reuse targetContext ID if pair-relation exists newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) - r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetInstanceIdLabel, newInstanceId) + r.xsetLabelMgr.Set(newTarget, api.EnumXSetInstanceIdLabel, newInstanceId) logger.Info("replaceOriginTargets", "try to reuse new pod resourceContext id", newInstanceId) } else { if availableContexts[i] == nil { @@ -153,13 +153,13 @@ func (r *RealSyncControl) replaceOriginTargets( newTargetContext = availableContexts[i] // add replace pair-relation to targetContexts for originTarget and newTarget newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) - r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetInstanceIdLabel, newInstanceId) + r.xsetLabelMgr.Set(newTarget, api.EnumXSetInstanceIdLabel, newInstanceId) r.resourceContextControl.Put(ownedIDs[originTargetId], api.EnumReplaceNewTargetIDContextDataKey, newInstanceId) r.resourceContextControl.Put(ownedIDs[newTargetContext.ID], api.EnumReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) r.resourceContextControl.Remove(ownedIDs[newTargetContext.ID], api.EnumJustCreateContextDataKey) } - r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetReplacePairOriginNameLabel, originTarget.GetName()) - r.xsetLabelMgr.Set(newTarget.GetLabels(), api.EnumXSetTargetCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + r.xsetLabelMgr.Set(newTarget, api.EnumXSetReplacePairOriginNameLabel, originTarget.GetName()) + r.xsetLabelMgr.Set(newTarget, api.EnumXSetTargetCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) r.resourceContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) // TODO create pvcs for new target (pod) diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index ce668d5f..5e8a1ed4 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -162,7 +162,7 @@ func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject api.XSet return err } - r.xsetLabelMgr.Set(target.GetLabels(), api.EnumXSetOrphanedLabel, "true") + r.xsetLabelMgr.Set(target, api.EnumXSetOrphanedLabel, "true") return r.xControl.OrphanTarget(xsetObject, target) } @@ -173,7 +173,7 @@ func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject api.XSet return err } - r.xsetLabelMgr.Set(target.GetLabels(), api.EnumXSetInstanceIdLabel, instanceId) + r.xsetLabelMgr.Set(target, api.EnumXSetInstanceIdLabel, instanceId) r.xsetLabelMgr.Delete(target.GetLabels(), api.EnumXSetOrphanedLabel) return r.xControl.AdoptTarget(xsetObject, target) } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index 67a11fc9..e78d336e 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -64,9 +64,8 @@ func NewTargetFrom(setController api.XSetController, xsetLabelMgr api.XSetLabelM targetObj.SetNamespace(owner.GetNamespace()) targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) - labels := targetObj.GetLabels() - xsetLabelMgr.Set(labels, api.EnumXSetInstanceIdLabel, fmt.Sprintf("%d", id)) - labels[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() + xsetLabelMgr.Set(targetObj, api.EnumXSetInstanceIdLabel, fmt.Sprintf("%d", id)) + targetObj.GetLabels()[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() controlByXSet(xsetLabelMgr, targetObj) for _, fn := range updateFuncs { @@ -171,7 +170,7 @@ func controlByXSet(xsetLabelMgr api.XSetLabelManager, obj client.Object) { obj.SetLabels(map[string]string{}) } if v, ok := xsetLabelMgr.Get(obj.GetLabels(), api.EnumXSetControlledLabel); !ok || v != "true" { - xsetLabelMgr.Set(obj.GetLabels(), api.EnumXSetControlledLabel, "true") + xsetLabelMgr.Set(obj, api.EnumXSetControlledLabel, "true") } }