Updates/migration - Re-run tag update, re-save to cleanup changedetection.json, code refactor (#3898)

Author: dgtlmoon

changedetectionio/api/Tags.py

@@ -97,17 +97,6 @@ def delete(self, uuid):
         # Delete the tag, and any tag reference
         del self.datastore.data['settings']['application']['tags'][uuid]
 
-        # Delete tag.json file if it exists
-        import os
-        tag_dir = os.path.join(self.datastore.datastore_path, uuid)
-        tag_json = os.path.join(tag_dir, "tag.json")
-        if os.path.exists(tag_json):
-            try:
-                os.unlink(tag_json)
-                logger.info(f"Deleted tag.json for tag {uuid}")
-            except Exception as e:
-                logger.error(f"Failed to delete tag.json for tag {uuid}: {e}")
-
         # Remove tag from all watches
         for watch_uuid, watch in self.datastore.data['watching'].items():
             if watch.get('tags') and uuid in watch['tags']:

changedetectionio/blueprint/tags/__init__.py

@@ -70,17 +70,6 @@ def delete(uuid):
         if datastore.data['settings']['application']['tags'].get(uuid):
             del datastore.data['settings']['application']['tags'][uuid]
 
-        # Delete tag.json file if it exists
-        import os
-        tag_dir = os.path.join(datastore.datastore_path, uuid)
-        tag_json = os.path.join(tag_dir, "tag.json")
-        if os.path.exists(tag_json):
-            try:
-                os.unlink(tag_json)
-                logger.info(f"Deleted tag.json for tag {uuid}")
-            except Exception as e:
-                logger.error(f"Failed to delete tag.json for tag {uuid}: {e}")
-
         # Remove tag from all watches in background thread to avoid blocking
         def remove_tag_background(tag_uuid):
             """Background thread to remove tag from watches - discarded after completion."""
@@ -127,19 +116,11 @@ def unlink_tag_background(tag_uuid):
     @tags_blueprint.route("/delete_all", methods=['GET'])
     @login_optionally_required
     def delete_all():
-        # Delete all tag.json files
-        import os
+
         for tag_uuid in list(datastore.data['settings']['application']['tags'].keys()):
-            tag_dir = os.path.join(datastore.datastore_path, tag_uuid)
-            tag_json = os.path.join(tag_dir, "tag.json")
-            if os.path.exists(tag_json):
-                try:
-                    os.unlink(tag_json)
-                except Exception as e:
-                    logger.error(f"Failed to delete tag.json for tag {tag_uuid}: {e}")
+# TagsDict 'del' handler will remove the dir
+            del datastore.data['settings']['application']['tags'][tag_uuid]
 
-        # Clear all tags from settings immediately
-        datastore.data['settings']['application']['tags'] = {}
 
         # Clear tags from all watches in background thread to avoid blocking
         def clear_all_tags_background():
@@ -255,7 +236,4 @@ def form_tag_edit_submit(uuid):
         return redirect(url_for('tags.tags_overview_page'))
 
 
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    def form_tag_delete(uuid):
-        return redirect(url_for('tags.tags_overview_page'))
     return tags_blueprint

changedetectionio/model/App.py

@@ -2,6 +2,7 @@
 from copy import deepcopy
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_CONTENT_FORMAT_DEFAULT
+from changedetectionio.model.Tags import TagsDict
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -68,7 +69,7 @@ class model(dict):
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'strip_ignored_lines': False,
-                    'tags': {}, #@todo use Tag.model initialisers
+                    'tags': None,  # Initialized in __init__ with real datastore_path
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'ui': {
                         'use_page_title_in_list': True,
@@ -80,10 +81,16 @@ class model(dict):
             }
         }
 
-    def __init__(self, *arg, **kw):
+    def __init__(self, *arg, datastore_path=None, **kw):
         super(model, self).__init__(*arg, **kw)
+        # Capture any tags data passed in before base_config overwrites the structure
+        existing_tags = self.get('settings', {}).get('application', {}).get('tags') or {}
         # CRITICAL: deepcopy to avoid sharing mutable objects between instances
         self.update(deepcopy(self.base_config))
+        # TagsDict requires the real datastore_path at runtime (cannot be set at class-definition time)
+        if datastore_path is None:
+            raise ValueError("App.model() requires 'datastore_path' keyword argument")
+        self['settings']['application']['tags'] = TagsDict(existing_tags, datastore_path=datastore_path)
 
 
 def parse_headers_from_text_file(filepath):

changedetectionio/model/Tags.py

@@ -0,0 +1,39 @@
+import os
+import shutil
+from pathlib import Path
+from loguru import logger
+
+_SENTINEL = object()
+
+
+class TagsDict(dict):
+    """Dict subclass that removes the corresponding tag.json file when a tag is deleted."""
+
+    def __init__(self, *args, datastore_path: str | os.PathLike, **kwargs) -> None:
+        self._datastore_path = Path(datastore_path)
+        super().__init__(*args, **kwargs)
+
+    def __delitem__(self, key: str) -> None:
+        super().__delitem__(key)
+        tag_dir = self._datastore_path / key
+        tag_json_file = tag_dir / "tag.json"
+        if not os.path.exists(tag_json_file):
+            logger.critical(f"Aborting deletion of directory '{tag_dir}' because '{tag_json_file}' does not exist.")
+            return
+        try:
+            shutil.rmtree(tag_dir)
+            logger.info(f"Deleted tag directory for tag {key!r}")
+        except FileNotFoundError:
+            pass
+        except OSError as e:
+            logger.error(f"Failed to delete tag directory for tag {key!r}: {e}")
+
+    def pop(self, key: str, default=_SENTINEL):
+        """Remove and return tag, deleting its tag.json file. Raises KeyError if missing and no default given."""
+        if key in self:
+            value = self[key]
+            del self[key]
+            return value
+        if default is _SENTINEL:
+            raise KeyError(key)
+        return default

changedetectionio/store/__init__.py

@@ -22,6 +22,8 @@
 from loguru import logger
 from blinker import signal
 
+from ..model.Tags import TagsDict
+
 # Try to import orjson for faster JSON serialization
 try:
     import orjson
@@ -121,6 +123,11 @@ def _apply_settings(self, settings_data):
             if 'application' in settings_data['settings']:
                 self.__data['settings']['application'].update(settings_data['settings']['application'])
 
+                # Use our Tags dict with cleanup helpers etc
+                # @todo Same for Watches
+                existing_tags = settings_data.get('settings', {}).get('application', {}).get('tags') or {}
+                self.__data['settings']['application']['tags'] = TagsDict(existing_tags, datastore_path=self.datastore_path)
+
         # More or less for the old format which had this data in the one url-watches.json
         # cant hurt to leave it here,
         if 'watching' in settings_data:
@@ -196,7 +203,7 @@ def reload_state(self, datastore_path, include_default_watches, version_tag):
         self.datastore_path = datastore_path
 
         # Initialize data structure
-        self.__data = App.model()
+        self.__data = App.model(datastore_path=datastore_path)
         self.json_store_path = os.path.join(self.datastore_path, "changedetection.json")
 
         # Base definition for all watchers (deepcopy part of #569)
@@ -355,6 +362,9 @@ def _build_settings_data(self):
         # Deep copy settings to avoid modifying the original
         settings_copy = copy.deepcopy(self.__data['settings'])
 
+        # Is saved as {uuid}/tag.json
+        settings_copy['application']['tags'] = {}
+
         return {
             'note': 'Settings file - watches are in {uuid}/watch.json, tags are in {uuid}/tag.json',
             'app_guid': self.__data.get('app_guid'),

changedetectionio/store/updates.py

@@ -669,7 +669,9 @@ def migrate_legacy_db_format(self):
     def update_26(self):
         self.migrate_legacy_db_format()
 
-    def update_28(self):
+    # Re-run tag to JSON migration
+    def update_29(self):
+
         """
         Migrate tags to individual tag.json files.
 
@@ -682,8 +684,6 @@ def update_28(self):
         - Enables independent tag versioning/backup
         - Maintains backwards compatibility (tags stay in settings too)
         """
-        # Force save as tag.json (not watch.json) even if object is corrupted
-
         logger.critical("=" * 80)
         logger.critical("Running migration: Individual tag persistence (update_28)")
         logger.critical("Creating individual tag.json files")
@@ -702,6 +702,9 @@ def update_28(self):
         failed_count = 0
 
         for uuid, tag_data in tags.items():
+            if os.path.isfile(os.path.join(self.datastore_path, uuid, "tag.json")):
+                logger.debug(f"Tag {uuid} tag.json exists, skipping")
+                continue
             try:
                 tag_data.commit()
                 saved_count += 1
@@ -723,3 +726,7 @@ def update_28(self):
         logger.info("Future tag edits will update both locations (dual storage)")
         logger.critical("=" * 80)
 
+        # write it to disk, it will be saved without ['tags'] in the JSON db because we find it from disk glob
+        # (left this out by accident in previous update, added tags={} in the changedetection.json save_to_disk)
+        self._save_settings()
+