Merge branch 'feature-dedupe-serialize-index' into temp-feature-all-1.22

Author: ikreymer

backend/btrixcloud/colls.py

@@ -149,7 +149,7 @@ async def add_collection(self, org: Organization, coll_in: CollIn):
 
         slug = coll_in.slug or slug_from_name(coll_in.name)
 
-        dedupeIndex = DedupeIndexStats() if coll_in.hasDedupeIndex else None
+        indexStats = DedupeIndexStats() if coll_in.hasDedupeIndex else None
 
         coll = Collection(
             id=coll_id,
@@ -163,14 +163,11 @@ async def add_collection(self, org: Organization, coll_in: CollIn):
             access=coll_in.access,
             defaultThumbnailName=coll_in.defaultThumbnailName,
             allowPublicDownload=coll_in.allowPublicDownload,
-            dedupeIndex=dedupeIndex,
+            indexStats=indexStats,
         )
         try:
             await self.collections.insert_one(coll.to_dict())
             await self.clear_org_previous_slugs_matching_slug(slug, org)
-            # create collection index
-            # if coll.dedupeIndex:
-            #    await self.crawl_manager.create_coll_index(coll)
 
             if crawl_ids:
                 await self.crawl_ops.add_to_collection(crawl_ids, coll_id, org)
@@ -214,11 +211,11 @@ async def update_collection(
 
         query["modified"] = dt_now()
 
-        if update.hasDedupeIndex and not coll.dedupeIndex:
-            query["dedupeIndex"] = DedupeIndexStats().dict()
+        if update.hasDedupeIndex is True and not coll.indexStats:
+            query["indexStats"] = DedupeIndexStats().dict()
             await self.update_coll_index(coll, org.id)
 
-        elif not update.hasDedupeIndex and coll.dedupeIndex:
+        elif update.hasDedupeIndex is False and coll.indexStats:
             await self.delete_coll_index(coll, org)
 
         db_update = {"$set": query}
@@ -541,7 +538,7 @@ async def list_collections(
             match_query["name"] = {"$regex": regex_pattern, "$options": "i"}
 
         if has_dedupe_index is not None:
-            match_query["dedupeIndex"] = {"$ne" if has_dedupe_index else "$eq": None}
+            match_query["indexStats"] = {"$ne" if has_dedupe_index else "$eq": None}
 
         if public_colls_out:
             match_query["access"] = CollAccessType.PUBLIC
@@ -688,6 +685,11 @@ async def get_collection_crawl_ids(
 
     async def update_coll_index(self, coll: Collection, oid: UUID, is_purge=False):
         """create index import job"""
+
+        # don't update if no crawls if not purging removed crawls
+        if not is_purge and not coll.crawlCount:
+            return
+
         crawler_image = self.crawl_ops.crawl_configs.get_channel_crawler_image(
             self.dedupe_importer_channel
         )
@@ -710,7 +712,7 @@ async def delete_coll_index(self, coll: Collection, org: Organization):
         """delete coll dedupe index, if possible"""
 
         # if index is not idle, can't delete it yet
-        if coll.dedupeIndex and coll.indexState != "idle":
+        if coll.indexStats and coll.indexState != "idle":
             raise HTTPException(status_code=400, detail="dedupe_index_is_in_use")
 
         if coll.indexFile:
@@ -768,7 +770,7 @@ async def update_dedupe_index_stats(
         """update dedupe index stats for specified collection"""
         self.collections.find_one_and_update(
             {"_id": coll_id},
-            {"$set": {"dedupeIndex": stats.dict() if stats else None}},
+            {"$set": {"indexStats": stats.dict() if stats else None}},
         )
 
     async def update_dedupe_index_info(
@@ -785,7 +787,7 @@ async def update_dedupe_index_info(
             query["indexFile"] = index_file.model_dump()
 
         res = self.collections.find_one_and_update(
-            {"_id": coll_id, "dedupeIndex": {"$ne": None}},
+            {"_id": coll_id, "indexStats": {"$ne": None}},
             {"$set": query},
         )
         return res is not None
@@ -883,7 +885,7 @@ async def update_collection_dates(
         latest_ts = None
 
         # update_index is set, update dedupe index if it exists
-        if update_index and coll.dedupeIndex:
+        if update_index and coll.indexStats:
             await self.update_coll_index(coll, oid)
 
         match_query = {
@@ -952,7 +954,7 @@ async def add_successful_crawl_to_collections(
     async def purge_dedupe_index(self, coll_id: UUID, org: Organization):
         """purge dedupe index on collection, raise exception if no index or not ready"""
         coll = await self.get_collection(coll_id, org.id)
-        if not coll.dedupeIndex:
+        if not coll.indexStats:
             raise HTTPException(status_code=400, detail="no_dedupe_index_on_collection")
 
         if coll.indexState not in ("ready", "idle"):

backend/btrixcloud/models.py

@@ -1668,7 +1668,7 @@ class Collection(BaseMongoModel):
     indexFile: Optional[DedupeIndexFile] = None
     indexState: Optional[TYPE_DEDUPE_INDEX_STATES] = None
 
-    dedupeIndex: Optional[DedupeIndexStats] = None
+    indexStats: Optional[DedupeIndexStats] = None
 
 
 # ============================================================================
@@ -1736,7 +1736,7 @@ class CollOut(BaseMongoModel):
     indexLastSavedAt: Optional[datetime] = None
     indexState: Optional[TYPE_DEDUPE_INDEX_STATES] = None
 
-    dedupeIndex: Optional[DedupeIndexStats] = None
+    indexStats: Optional[DedupeIndexStats] = None
 
 
 # ============================================================================

backend/btrixcloud/operator/collindexes.py

@@ -391,7 +391,7 @@ async def check_redis_saved(
             finished_at = None
             finished_at_str = ""
             try:
-                finished_at_str = redis_pod["status"]["initContainerStatuses"][1][
+                finished_at_str = redis_pod["status"]["initContainerStatuses"][0][
                     "state"
                 ]["terminated"]["finishedAt"]
             # pylint: disable=bare-except

chart/app-templates/redis.yaml

@@ -141,56 +141,6 @@ spec:
             - "res=$(redis-cli ping); [[ $res = 'PONG' ]]"
 
   initContainers:
-{% if load_dump %}
-    - name: rclone-load
-      image: rclone/rclone:latest
-
-      args: ["-vv", "copyto", "--checksum", "remote:{{ remote_file_path }}", "/data/{{ local_file }}"]
-
-      volumeMounts:
-        - name: redis-data
-          mountPath: /data
-
-      env:
-      - name: RCLONE_CONFIG_REMOTE_TYPE
-        value: "s3"
-
-      - name: RCLONE_CONFIG_REMOTE_ACCESS_KEY_ID
-        valueFrom:
-          secretKeyRef:
-            name: "{{ storage_secret_name }}"
-            key: STORE_ACCESS_KEY
-
-      - name: RCLONE_CONFIG_REMOTE_SECRET_ACCESS_KEY
-        valueFrom:
-          secretKeyRef:
-            name: "{{ storage_secret_name }}"
-            key: STORE_SECRET_KEY
-
-      - name: RCLONE_CONFIG_REMOTE_REGION
-        valueFrom:
-          secretKeyRef:
-            name: "{{ storage_secret_name }}"
-            key: STORE_REGION
-
-      - name: RCLONE_CONFIG_REMOTE_PROVIDER
-        valueFrom:
-          secretKeyRef:
-            name: "{{ storage_secret_name }}"
-            key: STORE_S3_PROVIDER
-
-      - name: RCLONE_CONFIG_REMOTE_ENDPOINT
-        value: "{{ storage_endpoint }}"
-
-      resources:
-        limits:
-          memory: "200Mi"
-
-        requests:
-          memory: "200Mi"
-          cpu: "50m"
-{% endif %}
-
 {% if save_dump %}
     - name: rclone-save
       image: rclone/rclone:latest
@@ -210,11 +160,11 @@ spec:
           exec:
             command: ["touch", "/tmp/done"]
 
-      volumeMounts:
+      volumeMounts: &rclone_volumes
         - name: redis-data
           mountPath: /data
 
-      env:
+      env: &rclone_env
       - name: RCLONE_CONFIG_REMOTE_TYPE
         value: "s3"
 
@@ -245,14 +195,24 @@ spec:
       - name: RCLONE_CONFIG_REMOTE_ENDPOINT
         value: "{{ storage_endpoint }}"
 
-      resources:
+      resources: &rclone_resources
         limits:
           memory: "200Mi"
 
         requests:
           memory: "200Mi"
           cpu: "50m"
 
+{% if load_dump %}
+    - name: rclone-load
+      image: rclone/rclone:latest
+
+      args: ["-vv", "copyto", "--checksum", "remote:{{ remote_file_path }}", "/data/{{ local_file }}"]
+
+      volumeMounts: *rclone_volumes
+      env: *rclone_env
+      resources: *rclone_resources
+{% endif %}
 {% endif %}
 
 {% endif %}