Skip to content

Commit 23376f5

Browse files
weicaowei
weicao
and
wei
authored
fix(falkordb): use redis dts type for rebalance (#2948)
Co-authored-by: wei <wei@xiaobuou.local>
1 parent 513066b commit 23376f5

4 files changed

Lines changed: 255 additions & 9 deletions

File tree

‎addons/falkordb/falkordb-cluster-scripts/falkordb-cluster-manage.sh‎

Lines changed: 102 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,102 @@ init_environment(){
9494
export KB_CLUSTER_COMPONENT_DELETING_LIST
9595
}
9696

97+
component_list_contains_exact() {
98+
local component_list="$1"
99+
local component="$2"
100+
101+
if [[ -z "$component_list" || -z "$component" ]]; then
102+
return 1
103+
fi
104+
105+
IFS=',' read -ra components <<< "$component_list"
106+
for item in "${components[@]}"; do
107+
if [[ "$item" == "$component" ]]; then
108+
return 0
109+
fi
110+
done
111+
return 1
112+
}
113+
114+
component_matches_remove_shard_name() {
115+
local remove_shard_name="$1"
116+
local component_name="$2"
117+
local component_short_name="$3"
118+
119+
if [[ -z "$remove_shard_name" || -z "$component_name" ]]; then
120+
return 1
121+
fi
122+
if [[ "$remove_shard_name" == "$component_name" ]]; then
123+
return 0
124+
fi
125+
if [[ -n "$component_short_name" && "$remove_shard_name" == "$component_short_name" ]]; then
126+
return 0
127+
fi
128+
if [[ "$component_name" == *-"$remove_shard_name" ]]; then
129+
return 0
130+
fi
131+
return 1
132+
}
133+
134+
component_list_without_exact() {
135+
local component_list="$1"
136+
local component="$2"
137+
local result=""
138+
139+
if [[ -z "$component_list" || -z "$component" ]]; then
140+
return 0
141+
fi
142+
143+
IFS=',' read -ra components <<< "$component_list"
144+
for item in "${components[@]}"; do
145+
if [[ -z "$item" || "$item" == "$component" ]]; then
146+
continue
147+
fi
148+
if [[ -z "$result" ]]; then
149+
result="$item"
150+
else
151+
result="$result,$item"
152+
fi
153+
done
154+
echo "$result"
155+
}
156+
157+
detect_scale_in_context() {
158+
if ! is_empty "$KB_CLUSTER_COMPONENT_IS_SCALING_IN"; then
159+
return 0
160+
fi
161+
162+
if component_matches_remove_shard_name "$KB_REMOVE_SHARD_NAME" "$CURRENT_SHARD_COMPONENT_NAME" "$CURRENT_SHARD_COMPONENT_SHORT_NAME"; then
163+
KB_CLUSTER_COMPONENT_IS_SCALING_IN="true"
164+
export KB_CLUSTER_COMPONENT_IS_SCALING_IN
165+
if is_empty "$KB_CLUSTER_COMPONENT_DELETING_LIST"; then
166+
KB_CLUSTER_COMPONENT_DELETING_LIST="$CURRENT_SHARD_COMPONENT_NAME"
167+
export KB_CLUSTER_COMPONENT_DELETING_LIST
168+
fi
169+
if is_empty "$KB_CLUSTER_COMPONENT_UNDELETED_LIST"; then
170+
KB_CLUSTER_COMPONENT_UNDELETED_LIST=$(component_list_without_exact "$KB_CLUSTER_COMPONENT_LIST" "$CURRENT_SHARD_COMPONENT_NAME")
171+
export KB_CLUSTER_COMPONENT_UNDELETED_LIST
172+
fi
173+
echo "Detected scale-in context from KB_REMOVE_SHARD_NAME"
174+
return 0
175+
fi
176+
177+
if ! component_list_contains_exact "$KB_CLUSTER_COMPONENT_DELETING_LIST" "$CURRENT_SHARD_COMPONENT_NAME"; then
178+
return 1
179+
fi
180+
if component_list_contains_exact "$KB_CLUSTER_COMPONENT_UNDELETED_LIST" "$CURRENT_SHARD_COMPONENT_NAME"; then
181+
return 1
182+
fi
183+
if is_empty "$KB_CLUSTER_COMPONENT_UNDELETED_LIST"; then
184+
return 1
185+
fi
186+
187+
KB_CLUSTER_COMPONENT_IS_SCALING_IN="true"
188+
export KB_CLUSTER_COMPONENT_IS_SCALING_IN
189+
echo "Detected scale-in context from KB_CLUSTER_COMPONENT_DELETING_LIST/UNDELETED_LIST"
190+
return 0
191+
}
192+
97193
load_redis_cluster_common_utils() {
98194
# the common.sh and falkordb-cluster-common.sh scripts are defined in the falkordb-cluster-scripts-template configmap
99195
# and are mounted to the same path which defined in the cmpd.spec.scripts
@@ -1005,9 +1101,8 @@ sync_acl_for_redis_cluster_shard() {
10051101
}
10061102

10071103
scale_in_redis_cluster_shard() {
1008-
# check KB_CLUSTER_COMPONENT_IS_SCALING_IN env
1009-
if is_empty "$KB_CLUSTER_COMPONENT_IS_SCALING_IN"; then
1010-
echo "The KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set, skip scaling in"
1104+
if ! detect_scale_in_context; then
1105+
echo "The KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set and current component is not exclusively in KB_CLUSTER_COMPONENT_DELETING_LIST, skip scaling in"
10111106
return 0
10121107
fi
10131108

@@ -1019,6 +1114,10 @@ scale_in_redis_cluster_shard() {
10191114
# init information for the other components and pods
10201115
init_other_components_and_pods_info "$CURRENT_SHARD_COMPONENT_SHORT_NAME" "$KB_CLUSTER_POD_IP_LIST" "$KB_CLUSTER_POD_NAME_LIST" "$KB_CLUSTER_COMPONENT_LIST" "$KB_CLUSTER_COMPONENT_DELETING_LIST" "$KB_CLUSTER_COMPONENT_UNDELETED_LIST"
10211116
available_node=$(find_exist_available_node)
1117+
if is_empty "$available_node"; then
1118+
echo "No stable available FalkorDB Cluster node found before scaling in shard" >&2
1119+
return 1
1120+
fi
10221121
available_node_fqdn=$(echo "$available_node" | awk -F ':' '{print $1}')
10231122
available_node_port=$(echo "$available_node" | awk -F ':' '{print $2}')
10241123
get_current_comp_nodes_for_scale_in "$available_node_fqdn" "$available_node_port"

‎addons/falkordb/scripts-ut-spec/falkordb_cluster_manage_spec.sh‎

Lines changed: 140 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1509,21 +1509,157 @@ d-98x-redis-advertised-1:31318.shard-7hy@falkordb-shard-7hy-redis-advertised-0:3
15091509
End
15101510

15111511
Describe "scale_in_redis_cluster_shard()"
1512-
Context "when KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set"
1512+
Context "when no scale-in signal is set"
15131513
setup() {
1514-
export KB_CLUSTER_COMPONENT_IS_SCALING_IN=""
1514+
unset KB_CLUSTER_COMPONENT_IS_SCALING_IN
1515+
export CURRENT_SHARD_COMPONENT_NAME="falkordb-shard-98x"
1516+
export KB_CLUSTER_COMPONENT_DELETING_LIST=""
1517+
export KB_CLUSTER_COMPONENT_UNDELETED_LIST="falkordb-shard-98x,falkordb-shard-7hy"
1518+
}
1519+
Before "setup"
1520+
1521+
un_setup() {
1522+
unset KB_CLUSTER_COMPONENT_IS_SCALING_IN
1523+
unset CURRENT_SHARD_COMPONENT_NAME
1524+
unset KB_CLUSTER_COMPONENT_DELETING_LIST
1525+
unset KB_CLUSTER_COMPONENT_UNDELETED_LIST
1526+
}
1527+
After "un_setup"
1528+
1529+
It "returns 0 and skips scale-in cleanup"
1530+
When call scale_in_redis_cluster_shard
1531+
The status should be success
1532+
The output should include "The KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set and current component is not exclusively in KB_CLUSTER_COMPONENT_DELETING_LIST, skip scaling in"
1533+
End
1534+
End
1535+
1536+
Context "when legacy KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set but deleting lists identify scale-in target"
1537+
find_exist_available_node() {
1538+
echo "falkordb-shard-7hy-0.namespace.svc.cluster.local:6379"
1539+
}
1540+
1541+
get_current_comp_nodes_for_scale_in() {
1542+
current_comp_primary_node=("falkordb-shard-98x-0.namespace.svc.cluster.local:6379")
1543+
current_comp_other_nodes=("falkordb-shard-98x-1.namespace.svc.cluster.local:6379")
1544+
}
1545+
1546+
get_cluster_id() {
1547+
echo "cluster_id_123"
1548+
}
1549+
1550+
scale_in_shard_rebalance_to_zero() {
1551+
return 0
1552+
}
1553+
1554+
scale_in_shard_del_node() {
1555+
return 0
1556+
}
1557+
1558+
setup() {
1559+
unset KB_CLUSTER_COMPONENT_IS_SCALING_IN
1560+
export CURRENT_SHARD_COMPONENT_SHORT_NAME="shard-98x"
1561+
export CURRENT_SHARD_COMPONENT_NAME="falkordb-shard-98x"
1562+
export KB_CLUSTER_POD_NAME_LIST="falkordb-shard-98x-0,falkordb-shard-98x-1,falkordb-shard-7hy-0,falkordb-shard-7hy-1,falkordb-shard-jwl-0,falkordb-shard-jwl-1,falkordb-shard-kpl-0,falkordb-shard-kpl-1"
1563+
export KB_CLUSTER_POD_HOST_IP_LIST="10.42.0.1,10.42.0.2,10.42.0.3,10.42.0.4,10.42.0.5,10.42.0.6,10.42.0.7,10.42.0.8"
1564+
export KB_CLUSTER_COMPONENT_POD_NAME_LIST="falkordb-shard-98x-0,falkordb-shard-98x-1"
1565+
export KB_CLUSTER_COMPONENT_POD_HOST_IP_LIST="10.42.0.1,10.42.0.2"
1566+
export KB_CLUSTER_POD_IP_LIST="172.42.0.1,172.42.0.2,172.42.0.3,172.42.0.4,172.42.0.5,172.42.0.6,172.42.0.7,172.42.0.8"
1567+
export KB_CLUSTER_COMPONENT_LIST="falkordb-shard-98x,falkordb-shard-7hy,falkordb-shard-jwl,falkordb-shard-kpl"
1568+
export KB_CLUSTER_COMPONENT_DELETING_LIST="falkordb-shard-98x"
1569+
export KB_CLUSTER_COMPONENT_UNDELETED_LIST="falkordb-shard-7hy,falkordb-shard-jwl,falkordb-shard-kpl"
1570+
export SERVICE_PORT="6379"
1571+
}
1572+
Before "setup"
1573+
1574+
un_setup() {
1575+
unset KB_CLUSTER_COMPONENT_IS_SCALING_IN
1576+
unset CURRENT_SHARD_COMPONENT_SHORT_NAME
1577+
unset CURRENT_SHARD_COMPONENT_NAME
1578+
unset KB_CLUSTER_POD_NAME_LIST
1579+
unset KB_CLUSTER_POD_HOST_IP_LIST
1580+
unset KB_CLUSTER_COMPONENT_POD_NAME_LIST
1581+
unset KB_CLUSTER_COMPONENT_POD_HOST_IP_LIST
1582+
unset KB_CLUSTER_POD_IP_LIST
1583+
unset KB_CLUSTER_COMPONENT_LIST
1584+
unset KB_CLUSTER_COMPONENT_DELETING_LIST
1585+
unset KB_CLUSTER_COMPONENT_UNDELETED_LIST
1586+
unset SERVICE_PORT
1587+
}
1588+
After "un_setup"
1589+
1590+
It "runs scale-in cleanup without the legacy scaling-in flag"
1591+
When call scale_in_redis_cluster_shard
1592+
The status should be success
1593+
The output should include "Detected scale-in context from KB_CLUSTER_COMPONENT_DELETING_LIST/UNDELETED_LIST"
1594+
The output should include "FalkorDB cluster scale in shard rebalance to zero successfully"
1595+
The output should include "FalkorDB cluster scale in shard delete node falkordb-shard-98x-0.namespace.svc.cluster.local:6379 successfully"
1596+
The output should include "FalkorDB cluster scale in shard delete node falkordb-shard-98x-1.namespace.svc.cluster.local:6379 successfully"
1597+
End
1598+
End
1599+
1600+
Context "when shardRemove action identifies the removed shard"
1601+
find_exist_available_node() {
1602+
echo "falkordb-shard-7hy-0.namespace.svc.cluster.local:6379"
1603+
}
1604+
1605+
get_current_comp_nodes_for_scale_in() {
1606+
current_comp_primary_node=("falkordb-shard-98x-0.namespace.svc.cluster.local:6379")
1607+
current_comp_other_nodes=("falkordb-shard-98x-1.namespace.svc.cluster.local:6379")
1608+
}
1609+
1610+
get_cluster_id() {
1611+
echo "cluster_id_123"
1612+
}
1613+
1614+
scale_in_shard_rebalance_to_zero() {
1615+
return 0
1616+
}
1617+
1618+
scale_in_shard_del_node() {
1619+
return 0
1620+
}
1621+
1622+
setup() {
1623+
unset KB_CLUSTER_COMPONENT_IS_SCALING_IN
1624+
export KB_REMOVE_SHARD_NAME="98x"
1625+
export CURRENT_SHARD_COMPONENT_SHORT_NAME="shard-98x"
1626+
export CURRENT_SHARD_COMPONENT_NAME="falkordb-shard-98x"
1627+
export KB_CLUSTER_POD_NAME_LIST="falkordb-shard-98x-0,falkordb-shard-98x-1,falkordb-shard-7hy-0,falkordb-shard-7hy-1,falkordb-shard-jwl-0,falkordb-shard-jwl-1,falkordb-shard-kpl-0,falkordb-shard-kpl-1"
1628+
export KB_CLUSTER_POD_HOST_IP_LIST="10.42.0.1,10.42.0.2,10.42.0.3,10.42.0.4,10.42.0.5,10.42.0.6,10.42.0.7,10.42.0.8"
1629+
export KB_CLUSTER_COMPONENT_POD_NAME_LIST="falkordb-shard-98x-0,falkordb-shard-98x-1"
1630+
export KB_CLUSTER_COMPONENT_POD_HOST_IP_LIST="10.42.0.1,10.42.0.2"
1631+
export KB_CLUSTER_POD_IP_LIST="172.42.0.1,172.42.0.2,172.42.0.3,172.42.0.4,172.42.0.5,172.42.0.6,172.42.0.7,172.42.0.8"
1632+
export KB_CLUSTER_COMPONENT_LIST="falkordb-shard-98x,falkordb-shard-7hy,falkordb-shard-jwl,falkordb-shard-kpl"
1633+
export KB_CLUSTER_COMPONENT_DELETING_LIST=""
1634+
export KB_CLUSTER_COMPONENT_UNDELETED_LIST=""
1635+
export SERVICE_PORT="6379"
15151636
}
15161637
Before "setup"
15171638

15181639
un_setup() {
15191640
unset KB_CLUSTER_COMPONENT_IS_SCALING_IN
1641+
unset KB_REMOVE_SHARD_NAME
1642+
unset CURRENT_SHARD_COMPONENT_SHORT_NAME
1643+
unset CURRENT_SHARD_COMPONENT_NAME
1644+
unset KB_CLUSTER_POD_NAME_LIST
1645+
unset KB_CLUSTER_POD_HOST_IP_LIST
1646+
unset KB_CLUSTER_COMPONENT_POD_NAME_LIST
1647+
unset KB_CLUSTER_COMPONENT_POD_HOST_IP_LIST
1648+
unset KB_CLUSTER_POD_IP_LIST
1649+
unset KB_CLUSTER_COMPONENT_LIST
1650+
unset KB_CLUSTER_COMPONENT_DELETING_LIST
1651+
unset KB_CLUSTER_COMPONENT_UNDELETED_LIST
1652+
unset SERVICE_PORT
15201653
}
15211654
After "un_setup"
15221655

1523-
It "returns 0 when KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set"
1656+
It "runs scale-in cleanup and derives deleting lists from KB_REMOVE_SHARD_NAME"
15241657
When call scale_in_redis_cluster_shard
15251658
The status should be success
1526-
The output should include "The KB_CLUSTER_COMPONENT_IS_SCALING_IN env is not set, skip scaling in"
1659+
The output should include "Detected scale-in context from KB_REMOVE_SHARD_NAME"
1660+
The output should include "FalkorDB cluster scale in shard rebalance to zero successfully"
1661+
The output should include "FalkorDB cluster scale in shard delete node falkordb-shard-98x-0.namespace.svc.cluster.local:6379 successfully"
1662+
The output should include "FalkorDB cluster scale in shard delete node falkordb-shard-98x-1.namespace.svc.cluster.local:6379 successfully"
15271663
End
15281664
End
15291665

‎addons/falkordb/templates/opsdefinition-rebalance.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ spec:
4545
args:
4646
- |
4747
POD_FQDN=$CURRENT_POD_NAME.$CURRENT_SHARD_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.$CLUSTER_DOMAIN
48-
REDIS_URL="falkordb://${REDIS_DEFAULT_USER}:${REDIS_DEFAULT_PASSWORD}@${POD_FQDN}"
48+
REDIS_URL="redis://${REDIS_DEFAULT_USER}:${REDIS_DEFAULT_PASSWORD}@${POD_FQDN}"
4949
echo ${REDIS_URL}
5050
cat <<EOF > /tmp/reshard.ini
5151
[extractor]
52-
db_type=falkordb
52+
db_type=redis
5353
extract_type=reshard
5454
url=${REDIS_URL}
5555
[sinker]

‎addons/falkordb/templates/shardingdefinition.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,14 @@ spec:
1414
maxShards: 64
1515
provisionStrategy: Parallel
1616
updateStrategy: Parallel
17+
lifecycleActions:
18+
shardRemove:
19+
timeoutSeconds: 600
20+
exec:
21+
container: falkordb-cluster
22+
command:
23+
- /bin/bash
24+
- -c
25+
- /scripts/falkordb-cluster-manage.sh --pre-terminate > /tmp/pre-terminate.log 2>&1
26+
retryPolicy:
27+
maxRetries: 10

0 commit comments

Comments
 (0)