Skip to content

Commit 97981b0

Browse files
isubasingheJoibel
authored andcommitted
fix(semaphore): ensure holderKeys carry all information needed. Fixes argoproj#8684 (argoproj#13553)
Signed-off-by: isubasinghe <isitha@pipekit.io>
1 parent d1f569b commit 97981b0

5 files changed

Lines changed: 499 additions & 60 deletions

File tree

pkg/apis/workflow/v1alpha1/workflow_types.go

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,13 @@ const (
133133
VolumeClaimGCOnSuccess VolumeClaimGCStrategy = "OnWorkflowSuccess"
134134
)
135135

136+
type HoldingNameVersion int
137+
138+
const (
139+
HoldingNameV1 HoldingNameVersion = 1
140+
HoldingNameV2 HoldingNameVersion = 2
141+
)
142+
136143
// Workflow is the definition of a workflow resource
137144
// +genclient
138145
// +genclient:noStatus
@@ -3771,11 +3778,7 @@ func (ss *SemaphoreStatus) LockWaiting(holderKey, lockKey string, currentHolders
37713778

37723779
func (ss *SemaphoreStatus) LockAcquired(holderKey, lockKey string, currentHolders []string) bool {
37733780
i, semaphoreHolding := ss.GetHolding(lockKey)
3774-
items := strings.Split(holderKey, "/")
3775-
if len(items) == 0 {
3776-
return false
3777-
}
3778-
holdingName := items[len(items)-1]
3781+
holdingName := holderKey
37793782
if i < 0 {
37803783
ss.Holding = append(ss.Holding, SemaphoreHolding{Semaphore: lockKey, Holders: []string{holdingName}})
37813784
return true
@@ -3789,11 +3792,8 @@ func (ss *SemaphoreStatus) LockAcquired(holderKey, lockKey string, currentHolder
37893792

37903793
func (ss *SemaphoreStatus) LockReleased(holderKey, lockKey string) bool {
37913794
i, semaphoreHolding := ss.GetHolding(lockKey)
3792-
items := strings.Split(holderKey, "/")
3793-
if len(items) == 0 {
3794-
return false
3795-
}
3796-
holdingName := items[len(items)-1]
3795+
holdingName := holderKey
3796+
37973797
if i >= 0 {
37983798
semaphoreHolding.Holders = slice.RemoveString(semaphoreHolding.Holders, holdingName)
37993799
ss.Holding[i] = semaphoreHolding
@@ -3864,13 +3864,17 @@ func (ms *MutexStatus) LockWaiting(holderKey, lockKey string, currentHolders []s
38643864
return false
38653865
}
38663866

3867-
func (ms *MutexStatus) LockAcquired(holderKey, lockKey string, currentHolders []string) bool {
3868-
i, mutexHolding := ms.GetHolding(lockKey)
3867+
func CheckHolderKeyVersion(holderKey string) HoldingNameVersion {
38693868
items := strings.Split(holderKey, "/")
3870-
if len(items) == 0 {
3871-
return false
3869+
if len(items) == 2 || len(items) == 3 {
3870+
return HoldingNameV2
38723871
}
3873-
holdingName := items[len(items)-1]
3872+
return HoldingNameV1
3873+
}
3874+
3875+
func (ms *MutexStatus) LockAcquired(holderKey, lockKey string, currentHolders []string) bool {
3876+
i, mutexHolding := ms.GetHolding(lockKey)
3877+
holdingName := holderKey
38743878
if i < 0 {
38753879
ms.Holding = append(ms.Holding, MutexHolding{Mutex: lockKey, Holder: holdingName})
38763880
return true
@@ -3884,11 +3888,7 @@ func (ms *MutexStatus) LockAcquired(holderKey, lockKey string, currentHolders []
38843888

38853889
func (ms *MutexStatus) LockReleased(holderKey, lockKey string) bool {
38863890
i, holder := ms.GetHolding(lockKey)
3887-
items := strings.Split(holderKey, "/")
3888-
if len(items) == 0 {
3889-
return false
3890-
}
3891-
holdingName := items[len(items)-1]
3891+
holdingName := holderKey
38923892
if i >= 0 && holder.Holder == holdingName {
38933893
ms.Holding = append(ms.Holding[:i], ms.Holding[i+1:]...)
38943894
return true

workflow/controller/operator_concurrency_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,17 +1091,17 @@ spec:
10911091
configMap:
10921092
name: cache-example-steps-simple
10931093
`)
1094+
wf.Name = "example-steps-simple-gas12"
10941095
cancel, controller := newController(wf)
10951096
defer cancel()
10961097

10971098
ctx := context.Background()
1098-
10991099
woc := newWorkflowOperationCtx(wf, controller)
11001100
woc.operate(ctx)
11011101

11021102
holdingJobs := make(map[string]string)
11031103
for _, node := range woc.wf.Status.Nodes {
1104-
holdingJobs[node.ID] = node.DisplayName
1104+
holdingJobs[fmt.Sprintf("%s/%s/%s", wf.Namespace, wf.Name, node.ID)] = node.DisplayName
11051105
}
11061106

11071107
// Check initial status: job-1 acquired the lock

workflow/sync/mutex_test.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ status:
106106
mutex:
107107
holding:
108108
- holder: synchronization-wf-level-xxs94
109-
mutex: default/mutex/test
109+
mutex: default/Mutex/test
110110
`
111111

112112
func TestMutexLock(t *testing.T) {
@@ -141,7 +141,7 @@ func TestMutexLock(t *testing.T) {
141141
assert.NotNil(t, wf.Status.Synchronization)
142142
assert.NotNil(t, wf.Status.Synchronization.Mutex)
143143
assert.NotNil(t, wf.Status.Synchronization.Mutex.Holding)
144-
assert.Equal(t, wf.Name, wf.Status.Synchronization.Mutex.Holding[0].Holder)
144+
assert.Equal(t, getHolderKey(wf, ""), wf.Status.Synchronization.Mutex.Holding[0].Holder)
145145

146146
// Try to acquire again
147147
status, wfUpdate, msg, err = concurrenyMgr.TryAcquire(wf, "", wf.Spec.Synchronization)
@@ -193,7 +193,7 @@ func TestMutexLock(t *testing.T) {
193193
assert.True(t, wfUpdate)
194194
assert.NotNil(t, wf2.Status.Synchronization)
195195
assert.NotNil(t, wf2.Status.Synchronization.Mutex)
196-
assert.Equal(t, wf2.Name, wf2.Status.Synchronization.Mutex.Holding[0].Holder)
196+
assert.Equal(t, getHolderKey(wf2, ""), wf2.Status.Synchronization.Mutex.Holding[0].Holder)
197197
concurrenyMgr.ReleaseAll(wf2)
198198
assert.Nil(t, wf2.Status.Synchronization)
199199
})
@@ -215,7 +215,8 @@ func TestMutexLock(t *testing.T) {
215215
assert.NotNil(t, wf.Status.Synchronization)
216216
assert.NotNil(t, wf.Status.Synchronization.Mutex)
217217
assert.NotNil(t, wf.Status.Synchronization.Mutex.Holding)
218-
assert.Equal(t, wf.Name, wf.Status.Synchronization.Mutex.Holding[0].Holder)
218+
expected := getHolderKey(wf, "")
219+
assert.Equal(t, expected, wf.Status.Synchronization.Mutex.Holding[0].Holder)
219220

220221
// Try to acquire again
221222
status, wfUpdate, msg, err = concurrenyMgr.TryAcquire(wf, "", wf.Spec.Synchronization)
@@ -270,7 +271,8 @@ func TestMutexLock(t *testing.T) {
270271
assert.True(t, wfUpdate)
271272
assert.NotNil(t, wf2.Status.Synchronization)
272273
assert.NotNil(t, wf2.Status.Synchronization.Mutex)
273-
assert.Equal(t, wf2.Name, wf2.Status.Synchronization.Mutex.Holding[0].Holder)
274+
expected = getHolderKey(wf2, "")
275+
assert.Equal(t, expected, wf2.Status.Synchronization.Mutex.Holding[0].Holder)
274276
concurrenyMgr.ReleaseAll(wf2)
275277
assert.Nil(t, wf2.Status.Synchronization)
276278
})
@@ -394,7 +396,8 @@ func TestMutexTmplLevel(t *testing.T) {
394396
assert.True(t, wfUpdate)
395397
assert.NotNil(t, wf.Status.Synchronization)
396398
assert.NotNil(t, wf.Status.Synchronization.Mutex)
397-
assert.Equal(t, "synchronization-tmpl-level-mutex-vjcdk-3941195474", wf.Status.Synchronization.Mutex.Holding[0].Holder)
399+
expected := getHolderKey(wf, "synchronization-tmpl-level-mutex-vjcdk-3941195474")
400+
assert.Equal(t, expected, wf.Status.Synchronization.Mutex.Holding[0].Holder)
398401

399402
// Try to acquire again
400403
status, wfUpdate, msg, err = concurrenyMgr.TryAcquire(wf, "synchronization-tmpl-level-mutex-vjcdk-2216915482", tmpl.Synchronization)
@@ -409,7 +412,8 @@ func TestMutexTmplLevel(t *testing.T) {
409412
assert.False(t, wfUpdate)
410413
assert.False(t, status)
411414

412-
assert.Equal(t, "synchronization-tmpl-level-mutex-vjcdk-3941195474", wf.Status.Synchronization.Mutex.Holding[0].Holder)
415+
expected = getHolderKey(wf, "synchronization-tmpl-level-mutex-vjcdk-3941195474")
416+
assert.Equal(t, expected, wf.Status.Synchronization.Mutex.Holding[0].Holder)
413417
concurrenyMgr.Release(wf, "synchronization-tmpl-level-mutex-vjcdk-3941195474", tmpl.Synchronization)
414418
assert.NotNil(t, wf.Status.Synchronization)
415419
assert.NotNil(t, wf.Status.Synchronization.Mutex)
@@ -422,7 +426,8 @@ func TestMutexTmplLevel(t *testing.T) {
422426
assert.True(t, wfUpdate)
423427
assert.NotNil(t, wf.Status.Synchronization)
424428
assert.NotNil(t, wf.Status.Synchronization.Mutex)
425-
assert.Equal(t, "synchronization-tmpl-level-mutex-vjcdk-2216915482", wf.Status.Synchronization.Mutex.Holding[0].Holder)
429+
expected = getHolderKey(wf, "synchronization-tmpl-level-mutex-vjcdk-2216915482")
430+
assert.Equal(t, expected, wf.Status.Synchronization.Mutex.Holding[0].Holder)
426431

427432
assert.NotEqual(t, "synchronization-tmpl-level-mutex-vjcdk-3941195474", wf.Status.Synchronization.Mutex.Holding[0].Holder)
428433
concurrenyMgr.Release(wf, "synchronization-tmpl-level-mutex-vjcdk-3941195474", tmpl.Synchronization)

workflow/sync/sync_manager.go

Lines changed: 95 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,77 @@ func (cm *Manager) CheckWorkflowExistence() {
6666
}
6767
}
6868

69+
func getUpgradedKey(wf *wfv1.Workflow, key string, level SyncLevelType) string {
70+
if wfv1.CheckHolderKeyVersion(key) == wfv1.HoldingNameV1 {
71+
if level == WorkflowLevel {
72+
return getHolderKey(wf, "")
73+
}
74+
return getHolderKey(wf, key)
75+
}
76+
return key
77+
}
78+
79+
type SyncLevelType int
80+
81+
const (
82+
WorkflowLevel SyncLevelType = 1
83+
TemplateLevel SyncLevelType = 2
84+
ErrorLevel SyncLevelType = 3
85+
)
86+
87+
// HoldingNameV1 keys can be of the form
88+
// x where x is a workflow name
89+
// unfortunately this doesn't differentiate between workflow level keys
90+
// and template level keys. So upgrading is a bit tricky here.
91+
92+
// given a legacy holding name x, namespace y and workflow name z.
93+
// in the case of a workflow level
94+
// if x != z
95+
// upgradedKey := y/z
96+
// elseif x == z
97+
// upgradedKey := y/z
98+
// in the case of a template level
99+
// if x != z
100+
// upgradedKey := y/z/x
101+
// elif x == z
102+
// upgradedKey := y/z/x
103+
104+
// there is a possibility that
105+
// a synchronization exists both at the template level
106+
// and at the workflow level -> impossible to upgrade correctly
107+
// due to ambiguity. Currently we just assume workflow level.
108+
func getWorkflowSyncLevelByName(wf *wfv1.Workflow, lockName string) (SyncLevelType, error) {
109+
if wf.Spec.Synchronization != nil {
110+
syncLockName, err := GetLockName(wf.Spec.Synchronization, wf.Namespace)
111+
if err != nil {
112+
return ErrorLevel, err
113+
}
114+
checkName := syncLockName.EncodeName()
115+
if lockName == checkName {
116+
return WorkflowLevel, nil
117+
}
118+
}
119+
120+
var lastErr error
121+
for _, template := range wf.Spec.Templates {
122+
if template.Synchronization != nil {
123+
syncLockName, err := GetLockName(template.Synchronization, wf.Namespace)
124+
if err != nil {
125+
lastErr = err
126+
continue
127+
}
128+
checkName := syncLockName.EncodeName()
129+
if lockName == checkName {
130+
return TemplateLevel, nil
131+
}
132+
}
133+
}
134+
if lastErr == nil {
135+
lastErr = fmt.Errorf("was unable to determine level for %s", lockName)
136+
}
137+
return ErrorLevel, lastErr
138+
}
139+
69140
func (cm *Manager) Initialize(wfs []wfv1.Workflow) {
70141
for _, wf := range wfs {
71142
if wf.Status.Synchronization == nil {
@@ -86,11 +157,17 @@ func (cm *Manager) Initialize(wfs []wfv1.Workflow) {
86157
}
87158

88159
for _, holders := range holding.Holders {
89-
resourceKey := getResourceKey(wf.Namespace, wf.Name, holders)
90-
if semaphore != nil && semaphore.acquire(resourceKey) {
91-
log.Infof("Lock acquired by %s from %s", resourceKey, holding.Semaphore)
160+
level, err := getWorkflowSyncLevelByName(&wf, holding.Semaphore)
161+
if err != nil {
162+
log.Warnf("cannot obtain lock level for '%s' : %v", holding.Semaphore, err)
163+
continue
164+
}
165+
key := getUpgradedKey(&wf, holders, level)
166+
if semaphore != nil && semaphore.acquire(key) {
167+
log.Infof("Lock acquired by %s from %s", key, holding.Semaphore)
92168
}
93169
}
170+
94171
}
95172
}
96173

@@ -101,8 +178,13 @@ func (cm *Manager) Initialize(wfs []wfv1.Workflow) {
101178
if mutex == nil {
102179
mutex := cm.initializeMutex(holding.Mutex)
103180
if holding.Holder != "" {
104-
resourceKey := getResourceKey(wf.Namespace, wf.Name, holding.Holder)
105-
mutex.acquire(resourceKey)
181+
level, err := getWorkflowSyncLevelByName(&wf, holding.Mutex)
182+
if err != nil {
183+
log.Warnf("cannot obtain lock level for '%s' : %v", holding.Mutex, err)
184+
continue
185+
}
186+
key := getUpgradedKey(&wf, holding.Holder, level)
187+
mutex.acquire(key)
106188
}
107189
cm.syncLockMap[holding.Mutex] = mutex
108190
}
@@ -214,10 +296,9 @@ func (cm *Manager) ReleaseAll(wf *wfv1.Workflow) bool {
214296
}
215297

216298
for _, holderKey := range holding.Holders {
217-
resourceKey := getResourceKey(wf.Namespace, wf.Name, holderKey)
218-
syncLockHolder.release(resourceKey)
299+
syncLockHolder.release(holderKey)
219300
wf.Status.Synchronization.Semaphore.LockReleased(holderKey, holding.Semaphore)
220-
log.Infof("%s released a lock from %s", resourceKey, holding.Semaphore)
301+
log.Infof("%s released a lock from %s", holderKey, holding.Semaphore)
221302
}
222303
}
223304

@@ -227,8 +308,8 @@ func (cm *Manager) ReleaseAll(wf *wfv1.Workflow) bool {
227308
if syncLockHolder == nil {
228309
continue
229310
}
230-
resourceKey := getResourceKey(wf.Namespace, wf.Name, wf.Name)
231-
syncLockHolder.removeFromQueue(resourceKey)
311+
key := getHolderKey(wf, "")
312+
syncLockHolder.removeFromQueue(key)
232313
}
233314
wf.Status.Synchronization.Semaphore = nil
234315
}
@@ -240,10 +321,9 @@ func (cm *Manager) ReleaseAll(wf *wfv1.Workflow) bool {
240321
continue
241322
}
242323

243-
resourceKey := getResourceKey(wf.Namespace, wf.Name, holding.Holder)
244-
syncLockHolder.release(resourceKey)
324+
syncLockHolder.release(holding.Holder)
245325
wf.Status.Synchronization.Mutex.LockReleased(holding.Holder, holding.Mutex)
246-
log.Infof("%s released a lock from %s", resourceKey, holding.Mutex)
326+
log.Infof("%s released a lock from %s", holding.Holder, holding.Mutex)
247327
}
248328

249329
// Remove the pending Workflow level mutex keys
@@ -252,8 +332,8 @@ func (cm *Manager) ReleaseAll(wf *wfv1.Workflow) bool {
252332
if syncLockHolder == nil {
253333
continue
254334
}
255-
resourceKey := getResourceKey(wf.Namespace, wf.Name, wf.Name)
256-
syncLockHolder.removeFromQueue(resourceKey)
335+
key := getHolderKey(wf, "")
336+
syncLockHolder.removeFromQueue(key)
257337
}
258338
wf.Status.Synchronization.Mutex = nil
259339
}
@@ -296,14 +376,6 @@ func getHolderKey(wf *wfv1.Workflow, nodeName string) string {
296376
return key
297377
}
298378

299-
func getResourceKey(namespace, wfName, resourceName string) string {
300-
resourceKey := fmt.Sprintf("%s/%s", namespace, wfName)
301-
if resourceName != wfName {
302-
resourceKey = fmt.Sprintf("%s/%s", resourceKey, resourceName)
303-
}
304-
return resourceKey
305-
}
306-
307379
func (cm *Manager) getCurrentLockHolders(lockName string) []string {
308380
if concurrency, ok := cm.syncLockMap[lockName]; ok {
309381
return concurrency.getCurrentHolders()

0 commit comments

Comments
 (0)