Skip to content

Commit f4645e0

Browse files
ikreymerikreymer
committed
Merge branch 'kvrocks' into temp-feature-all-1.22
2 parents b59d21d + e5c94b0 commit f4645e0

File tree

3 files changed

+284
-16
lines changed

3 files changed

+284
-16
lines changed

‎backend/btrixcloud/operator/collindexes.py‎

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,20 @@ def __init__(self, *args):
6666
self.shared_params["cpu"] = self.shared_params["redis_cpu"]
6767
self.shared_params["obj_type"] = "coll"
6868

69-
self.shared_params["local_file"] = "dump.rdb"
69+
self.is_kvrocks = True
70+
71+
if self.is_kvrocks:
72+
self.shared_params["local_file_src"] = "backup"
73+
self.shared_params["local_file_dest"] = "db"
74+
else:
75+
self.shared_params["local_file_src"] = "dump.rdb"
76+
self.shared_params["local_file_dest"] = "dump.rdb"
77+
78+
if self.is_kvrocks:
79+
self.shared_params["redis_image"] = "apache/kvrocks:latest"
80+
self.pod_yaml = "kvrocks.yaml"
81+
else:
82+
self.pod_yaml = "redis.yaml"
7083

7184
self.fast_retry = int(os.environ.get("FAST_RETRY_SECS") or 0)
7285

@@ -141,8 +154,7 @@ async def sync_index(self, data: MCSyncData):
141154
# Saving process
142155
# 1. run bgsave while redis is active
143156
if not skip_redis:
144-
if status.state != "saving":
145-
await self.do_redis_save(spec.id, status)
157+
await self.do_redis_save(spec.id, status)
146158

147159
# 2. once redis has shutdown, check if fully finished
148160
else:
@@ -257,16 +269,37 @@ async def do_delete(self, index_id: UUID):
257269
async def do_redis_save(self, coll_id: UUID, status: CollIndexStatus):
258270
"""shutdown save redis"""
259271
try:
272+
if status.state != "saving":
273+
if not self.is_kvrocks:
274+
return
275+
260276
redis = await self.k8s.get_redis_connected("coll-" + str(coll_id))
261-
if redis:
277+
if not redis:
278+
return
279+
280+
if status.state != "saving":
262281
await self.set_state("saving", status, coll_id)
263-
self.run_task(self.do_redis_shutdown(redis, coll_id, status))
282+
283+
if self.is_kvrocks:
284+
await redis.bgsave(False)
285+
else:
286+
self.run_task(self.do_redis_shutdown(redis, coll_id, status))
287+
288+
if self.is_kvrocks:
289+
if await self.is_bgsave_done(redis):
290+
await redis.shutdown()
264291

265292
# pylint: disable=broad-exception-caught
266-
except Exception as e:
267-
print(e)
293+
except Exception:
294+
await self.set_state("ready", status, coll_id)
268295
traceback.print_exc()
269296

297+
async def is_bgsave_done(self, redis: Redis) -> bool:
298+
"""return true if bgsave has successfully finished"""
299+
info = await redis.execute_command("INFO persistence")
300+
301+
return "bgsave_in_progress:0" in info and "last_bgsave_status:ok" in info
302+
270303
async def do_redis_shutdown(
271304
self, redis: Redis, coll_id: UUID, status: CollIndexStatus
272305
):
@@ -369,14 +402,12 @@ async def load_redis(
369402
spec.id, org
370403
)
371404

372-
return self.load_from_yaml("redis.yaml", params)
405+
return self.load_from_yaml(self.pod_yaml, params)
373406

374407
def get_index_storage_filename(self, coll_id: UUID, org: Organization):
375408
"""get index filename for storage"""
376409
storage_path = org.storage.get_storage_extra_path(str(org.id))
377-
filename = storage_path + f"dedupe-index/{coll_id}.rdb"
378-
379-
return filename
410+
return storage_path + f"dedupe-index/{coll_id}"
380411

381412
async def check_redis_saved(
382413
self,
@@ -420,14 +451,19 @@ async def update_saved_dedupe_index_state_in_db(
420451
hash_ = ""
421452
size = -1
422453
logs = await self.k8s.get_pod_logs(
423-
pod_name, container=self.rclone_save, lines=10
454+
pod_name, container=self.rclone_save, lines=100
424455
)
425-
m = re.search(r"md5 = ([^\s]+) OK", logs)
426-
if m:
427-
hash_ = "md5:" + m.group(1)
428-
m = re.search(r"size = ([\d]+) OK", logs)
456+
# m = re.search(r"md5 = ([^\s]+) OK", logs)
457+
# if m:
458+
# hash_ = "md5:" + m.group(1)
459+
# m = re.search(r"size = ([\d]+) OK", logs)
460+
# if m:
461+
# size = int(m.group(1))
462+
463+
m = re.search(r"([\d]+),([^,]+)," + str(coll_id), logs)
429464
if m:
430465
size = int(m.group(1))
466+
hash_ = "md5:" + m.group(2)
431467

432468
print("UPLOAD LOGS")
433469
print("-----------")

‎chart/app-templates/kvrocks.yaml‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
{% if not no_pvc %}
2+
# -------
3+
# PVC
4+
# -------
5+
6+
apiVersion: v1
7+
kind: PersistentVolumeClaim
8+
metadata:
9+
name: {{ name }}
10+
namespace: {{ namespace }}
11+
labels:
12+
{{ obj_type }}: {{ id }}
13+
role: redis
14+
15+
spec:
16+
accessModes:
17+
- ReadWriteOnce
18+
19+
resources:
20+
requests:
21+
storage: {{ redis_storage }}
22+
23+
{% if volume_storage_class %}
24+
storageClassName: {{ volume_storage_class }}
25+
{% endif %}
26+
{% endif %}
27+
28+
# --------
29+
# REDIS
30+
# --------
31+
{% if init_redis %}
32+
---
33+
apiVersion: v1
34+
kind: Pod
35+
metadata:
36+
name: {{ name }}
37+
namespace: {{ namespace }}
38+
labels:
39+
{{ obj_type }}: {{ id }}
40+
role: redis
41+
42+
spec:
43+
hostname: {{ name }}
44+
subdomain: redis
45+
securityContext:
46+
runAsNonRoot: true
47+
runAsUser: 999
48+
runAsGroup: 999
49+
fsGroup: 999
50+
51+
terminationGracePeriodSeconds: 300
52+
restartPolicy: {{ "OnFailure" if save_dump else "Always" }}
53+
54+
volumes:
55+
- name: shared-redis-conf
56+
configMap:
57+
name: shared-redis-conf
58+
items:
59+
- key: redis.conf
60+
path: redis.conf
61+
62+
- name: redis-data
63+
{% if not no_pvc %}
64+
persistentVolumeClaim:
65+
claimName: {{ name }}
66+
{% else %}
67+
emptyDir: {}
68+
{% endif %}
69+
70+
affinity:
71+
nodeAffinity:
72+
preferredDuringSchedulingIgnoredDuringExecution:
73+
- weight: 1
74+
preference:
75+
matchExpressions:
76+
- key: nodeType
77+
operator: In
78+
values:
79+
- "{{ redis_node_type }}"
80+
81+
podAffinity:
82+
preferredDuringSchedulingIgnoredDuringExecution:
83+
- weight: 2
84+
podAffinityTerm:
85+
topologyKey: "failure-domain.beta.kubernetes.io/zone"
86+
labelSelector:
87+
matchLabels:
88+
{{ obj_type }}: {{ id }}
89+
90+
tolerations:
91+
- key: nodeType
92+
operator: Equal
93+
value: crawling
94+
effect: NoSchedule
95+
- key: node.kubernetes.io/not-ready
96+
operator: Exists
97+
tolerationSeconds: 300
98+
effect: NoExecute
99+
- key: node.kubernetes.io/unreachable
100+
operator: Exists
101+
effect: NoExecute
102+
tolerationSeconds: 300
103+
104+
containers:
105+
- name: redis
106+
image: {{ redis_image }}
107+
imagePullPolicy: {{ redis_image_pull_policy }}
108+
109+
args: ["--port", "6379", "--dir", "/data", "--appendonly", "{{ 'no' if load_dump else 'yes' }}"]
110+
volumeMounts:
111+
- name: redis-data
112+
mountPath: /data
113+
114+
- name: shared-redis-conf
115+
mountPath: /redis-conf
116+
117+
resources:
118+
limits:
119+
memory: {{ memory }}
120+
121+
requests:
122+
cpu: {{ cpu }}
123+
memory: {{ memory }}
124+
125+
readinessProbe:
126+
initialDelaySeconds: 10
127+
timeoutSeconds: 5
128+
exec:
129+
command:
130+
- bash
131+
- -c
132+
- "res=$(redis-cli ping); [[ $res = 'PONG' ]]"
133+
134+
livenessProbe:
135+
initialDelaySeconds: 10
136+
timeoutSeconds: 5
137+
exec:
138+
command:
139+
- bash
140+
- -c
141+
- "res=$(redis-cli ping); [[ $res = 'PONG' ]]"
142+
143+
initContainers:
144+
{% if save_dump %}
145+
- name: rclone-save
146+
image: rclone/rclone:latest
147+
148+
command: ["sh", "-c"]
149+
args:
150+
- |
151+
echo "Waiting until redis is done";
152+
while [ ! -f /tmp/done ]; do
153+
sleep 1;
154+
done;
155+
156+
cd /data/{{ local_file_src }}
157+
158+
tar cvfz - . | rclone --config "" rcat remote:{{ remote_file_path }}
159+
160+
echo "STATS (size,hash,file)"
161+
echo "----------------------"
162+
rclone --config "" lsf --separator "," --format "shp" remote:{{ remote_file_path }}
163+
164+
restartPolicy: Always
165+
lifecycle:
166+
preStop:
167+
exec:
168+
command: ["touch", "/tmp/done"]
169+
170+
volumeMounts: &rclone_volumes
171+
- name: redis-data
172+
mountPath: /data
173+
174+
env: &rclone_env
175+
- name: RCLONE_CONFIG_REMOTE_TYPE
176+
value: "s3"
177+
178+
- name: RCLONE_CONFIG_REMOTE_ACCESS_KEY_ID
179+
valueFrom:
180+
secretKeyRef:
181+
name: "{{ storage_secret_name }}"
182+
key: STORE_ACCESS_KEY
183+
184+
- name: RCLONE_CONFIG_REMOTE_SECRET_ACCESS_KEY
185+
valueFrom:
186+
secretKeyRef:
187+
name: "{{ storage_secret_name }}"
188+
key: STORE_SECRET_KEY
189+
190+
- name: RCLONE_CONFIG_REMOTE_REGION
191+
valueFrom:
192+
secretKeyRef:
193+
name: "{{ storage_secret_name }}"
194+
key: STORE_REGION
195+
196+
- name: RCLONE_CONFIG_REMOTE_PROVIDER
197+
valueFrom:
198+
secretKeyRef:
199+
name: "{{ storage_secret_name }}"
200+
key: STORE_S3_PROVIDER
201+
202+
- name: RCLONE_CONFIG_REMOTE_ENDPOINT
203+
value: "{{ storage_endpoint }}"
204+
205+
resources: &rclone_resources
206+
limits:
207+
memory: "200Mi"
208+
209+
requests:
210+
memory: "200Mi"
211+
cpu: "50m"
212+
213+
{% if load_dump %}
214+
- name: rclone-load
215+
image: rclone/rclone:latest
216+
217+
command: ["sh", "-c"]
218+
args:
219+
- |
220+
mkdir -p /data/{{ local_file_dest }}
221+
cd /data/{{ local_file_dest }}
222+
223+
rclone --config "" cat remote:{{ remote_file_path }} | tar xzf -
224+
225+
volumeMounts: *rclone_volumes
226+
env: *rclone_env
227+
resources: *rclone_resources
228+
{% endif %}
229+
{% endif %}
230+
231+
{% endif %}

‎chart/templates/configmap.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ metadata:
213213
data:
214214
redis.conf: |
215215
dir /data
216+
port 6379
216217
217218
---
218219
apiVersion: v1

0 commit comments

Comments
 (0)