Skip to content

Commit ae32d2c

Browse files
weicaoweicao
committed
fix(qdrant): preserve shard replicas on member leave
1 parent 955bc9b commit ae32d2c

2 files changed

Lines changed: 116 additions & 31 deletions

File tree

‎addons/qdrant/scripts-ut-spec/qdrant_member_leave_spec.sh‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,27 +33,50 @@ Describe "Qdrant Member Leave Bash Script Tests"
3333
End
3434
End
3535

36-
Describe "qdrant_select_target_peer_id()"
36+
Describe "qdrant_select_target_peer_id_for_shard()"
3737
cluster_info='{"result":{"peers":{"1":{"uri":"http://qdrant-0:6335/"},"2":{"uri":"http://qdrant-1:6335/"},"3":{"uri":"http://qdrant-2:6335/"}}}}'
3838

39-
It "keeps the raft leader as the shard target when the leader is not leaving"
40-
When call qdrant_select_target_peer_id "$cluster_info" "1" "2"
39+
It "uses the raft leader when it does not already own the shard"
40+
collection_info='{"result":{"peer_id":3,"local_shards":[{"shard_id":8}],"remote_shards":[{"shard_id":7,"peer_id":1}],"shard_transfers":[]}}'
41+
When call qdrant_select_target_peer_id_for_shard "$cluster_info" "$collection_info" "1" "2" "7"
4142
The status should be success
4243
The output should eq "2"
4344
End
4445

45-
It "selects a surviving peer when the leaving peer is the raft leader"
46-
When call qdrant_select_target_peer_id "$cluster_info" "1" "1"
46+
It "skips a leader that owns the shard as a remote replica"
47+
collection_info='{"result":{"peer_id":3,"local_shards":[{"shard_id":8}],"remote_shards":[{"shard_id":7,"peer_id":1},{"shard_id":7,"peer_id":2}],"shard_transfers":[]}}'
48+
When call qdrant_select_target_peer_id_for_shard "$cluster_info" "$collection_info" "1" "2" "7"
4749
The status should be success
48-
The output should eq "2"
50+
The output should eq "3"
51+
End
52+
53+
It "skips a leader that owns the shard locally"
54+
collection_info='{"result":{"peer_id":2,"local_shards":[{"shard_id":7}],"remote_shards":[{"shard_id":7,"peer_id":1}],"shard_transfers":[]}}'
55+
When call qdrant_select_target_peer_id_for_shard "$cluster_info" "$collection_info" "1" "2" "7"
56+
The status should be success
57+
The output should eq "3"
58+
End
59+
60+
It "skips an in-flight transfer destination for the shard"
61+
collection_info='{"result":{"peer_id":3,"local_shards":[{"shard_id":8}],"remote_shards":[{"shard_id":7,"peer_id":1}],"shard_transfers":[{"shard_id":7,"from":3,"to":2}]}}'
62+
When call qdrant_select_target_peer_id_for_shard "$cluster_info" "$collection_info" "1" "2" "7"
63+
The status should be success
64+
The output should eq "3"
4965
End
5066

51-
It "returns empty when there is no surviving peer"
52-
single_peer_cluster='{"result":{"peers":{"1":{"uri":"http://qdrant-0:6335/"}}}}'
53-
When call qdrant_select_target_peer_id "$single_peer_cluster" "1" "1"
67+
It "returns empty when every surviving peer already owns the shard"
68+
collection_info='{"result":{"peer_id":2,"local_shards":[{"shard_id":7}],"remote_shards":[{"shard_id":7,"peer_id":1},{"shard_id":7,"peer_id":3}],"shard_transfers":[]}}'
69+
When call qdrant_select_target_peer_id_for_shard "$cluster_info" "$collection_info" "1" "2" "7"
5470
The status should be success
5571
The output should eq ""
5672
End
73+
74+
It "fails closed when a local shard has no serving peer identity"
75+
collection_info='{"result":{"local_shards":[{"shard_id":7}],"remote_shards":[{"shard_id":7,"peer_id":1}],"shard_transfers":[]}}'
76+
When call qdrant_select_target_peer_id_for_shard "$cluster_info" "$collection_info" "1" "2" "7"
77+
The status should be failure
78+
The stderr should include "local shard without peer_id"
79+
End
5780
End
5881

5982
Describe "memberLeave action-wide deadline"
@@ -138,7 +161,8 @@ Describe "Qdrant Member Leave Bash Script Tests"
138161
control_uri="http://control"
139162
leave_peer_uri="http://leaving"
140163
leave_peer_id="1"
141-
target_peer_id="2"
164+
leader_peer_id="2"
165+
cluster_info='{"result":{"peers":{"1":{"uri":"http://qdrant-0:6335/"},"2":{"uri":"http://qdrant-1:6335/"},"3":{"uri":"http://qdrant-2:6335/"}}}}'
142166
qdrant_member_leave_deadline=$((SECONDS + 50))
143167
}
144168
Before "setup_replay_state"
@@ -177,6 +201,31 @@ Describe "Qdrant Member Leave Bash Script Tests"
177201
The output should include "already moving after failed submit"
178202
End
179203

204+
It "submits the move to an unused peer instead of an existing replica"
205+
replicated_info='{"result":{"peer_id":3,"local_shards":[{"shard_id":8}],"remote_shards":[{"shard_id":7,"peer_id":1},{"shard_id":7,"peer_id":2}],"shard_transfers":[]}}'
206+
qdrant_curl() {
207+
printf '%s\n' "$*"
208+
}
209+
210+
When call qdrant_submit_shard_move_if_needed "demo" "7" "$replicated_info"
211+
The status should be success
212+
The output should include '"to_peer_id":3'
213+
The output should not include '"to_peer_id":2'
214+
End
215+
216+
It "does not submit a move when every surviving peer already owns the shard"
217+
replicated_info='{"result":{"peer_id":2,"local_shards":[{"shard_id":7}],"remote_shards":[{"shard_id":7,"peer_id":1},{"shard_id":7,"peer_id":3}],"shard_transfers":[]}}'
218+
qdrant_curl() {
219+
echo "unexpected qdrant_curl call" >&2
220+
return 99
221+
}
222+
223+
When call qdrant_submit_shard_move_if_needed "demo" "7" "$replicated_info"
224+
The status should be failure
225+
The stderr should include "refusing to reduce replica count"
226+
The stderr should not include "unexpected qdrant_curl call"
227+
End
228+
180229
It "discovers remaining shards from the control endpoint when the leaving endpoint is unavailable"
181230
qdrant_curl() {
182231
last_arg="${*: -1}"

‎addons/qdrant/scripts/qdrant-member-leave.sh‎

Lines changed: 57 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,54 @@
11
#!/usr/bin/env bash
22

3-
qdrant_select_target_peer_id() {
3+
qdrant_select_target_peer_id_for_shard() {
44
local cluster_info="$1"
5-
local leave_peer_id="$2"
6-
local leader_peer_id="$3"
5+
local collection_cluster_info="$2"
6+
local leave_peer_id="$3"
7+
local leader_peer_id="$4"
8+
local shard_id="$5"
79
local jq_bin="${JQ:-jq}"
810

9-
if [ -n "$leader_peer_id" ] && [ "$leader_peer_id" != "null" ] && [ "$leader_peer_id" != "$leave_peer_id" ]; then
10-
echo "$leader_peer_id"
11-
return 0
12-
fi
13-
1411
echo "$cluster_info" | "$jq_bin" -r \
12+
--argjson collection "$collection_cluster_info" \
1513
--arg leave "$leave_peer_id" \
16-
'.result.peers
17-
| to_entries
18-
| map(select(.key != $leave))
19-
| .[0].key // ""'
14+
--arg leader "$leader_peer_id" \
15+
--argjson shard "$shard_id" \
16+
'.result.peers as $peers
17+
| ($collection.result // {}) as $distribution
18+
| if (((($distribution.local_shards // [])
19+
| map(select(.shard_id == $shard))
20+
| length) > 0)
21+
and (($distribution.peer_id // null) == null))
22+
then error("collection cluster response has a local shard without peer_id")
23+
else .
24+
end
25+
| ([
26+
(($distribution.local_shards // [])[]?
27+
| select(.shard_id == $shard)
28+
| $distribution.peer_id),
29+
(($distribution.remote_shards // [])[]?
30+
| select(.shard_id == $shard)
31+
| .peer_id),
32+
(($distribution.shard_transfers // [])[]?
33+
| select(.shard_id == $shard)
34+
| .to)
35+
]
36+
| map(select(. != null) | tostring)
37+
| unique) as $occupied
38+
| def eligible($peer):
39+
($peer != ""
40+
and $peer != "null"
41+
and $peer != $leave
42+
and ($peers | has($peer))
43+
and (($occupied | index($peer)) == null));
44+
if eligible($leader)
45+
then $leader
46+
else ($peers
47+
| to_entries
48+
| map(.key)
49+
| map(select(eligible(.)))
50+
| .[0] // "")
51+
end'
2052
}
2153

2254
qdrant_peer_id_for_pod() {
@@ -339,6 +371,7 @@ qdrant_submit_shard_move_if_needed() {
339371
local col_name="$1"
340372
local shard_id="$2"
341373
local col_cluster_info="$3"
374+
local move_target_peer_id
342375
local move_payload
343376

344377
if qdrant_shard_transfer_exists "$col_cluster_info" "$shard_id" "$leave_peer_id"; then
@@ -351,9 +384,19 @@ qdrant_submit_shard_move_if_needed() {
351384
return 0
352385
fi
353386

354-
echo "INFO: move shard ${shard_id} in ${col_name} from peer ${leave_peer_id} to peer ${target_peer_id}"
387+
move_target_peer_id="$(qdrant_select_target_peer_id_for_shard \
388+
"$cluster_info" "$col_cluster_info" "$leave_peer_id" "$leader_peer_id" "$shard_id")" || {
389+
echo "ERROR: failed to select a safe target for shard ${shard_id} in ${col_name}" >&2
390+
return 1
391+
}
392+
if [ -z "$move_target_peer_id" ]; then
393+
echo "ERROR: no surviving peer without shard ${shard_id} in ${col_name}; refusing to reduce replica count" >&2
394+
return 1
395+
fi
396+
397+
echo "INFO: move shard ${shard_id} in ${col_name} from peer ${leave_peer_id} to peer ${move_target_peer_id}"
355398
move_payload="$(printf '{"move_shard":{"shard_id":%s,"to_peer_id":%s,"from_peer_id":%s}}' \
356-
"$shard_id" "$target_peer_id" "$leave_peer_id")"
399+
"$shard_id" "$move_target_peer_id" "$leave_peer_id")"
357400

358401
if qdrant_member_leave_curl -X POST -H "Content-Type: application/json" \
359402
-d "$move_payload" \
@@ -561,16 +604,9 @@ qdrant_load_cluster_state() {
561604
fi
562605

563606
leader_peer_id="$(echo "$cluster_info" | "$JQ" -r .result.raft_info.leader)"
564-
target_peer_id="$(qdrant_select_target_peer_id "$cluster_info" "$leave_peer_id" "$leader_peer_id")"
565-
if [ -z "$target_peer_id" ]; then
566-
echo "ERROR: no surviving qdrant peer is available as shard move target for leaving peer ${leave_peer_id}"
567-
return 1
568-
fi
569-
570607
leave_peer_uri="${SCHEME}://${KB_LEAVE_MEMBER_POD_FQDN}:6333"
571608
echo "INFO: leaving peer=${KB_LEAVE_MEMBER_POD_NAME}, peer_id=${leave_peer_id}, uri=${leave_peer_uri}"
572609
echo "INFO: leader peer_id=${leader_peer_id}"
573-
echo "INFO: target peer_id=${target_peer_id}"
574610
echo "INFO: control endpoint=${control_uri}"
575611
echo "INFO: control endpoints:"
576612
printf "%s\n" "$control_uris" | sed 's/^/INFO: - /'

0 commit comments

Comments
 (0)