[DPE-2290] [DPE-2681] Upgrade from 14/stable and add integration tests (#235)

* Added initial upgrade implementation

* Updated the code with the new library

* Improved code and added unit tests

* Added one more check in unit test

* Removed upgrade integration tests

* Revert "Removed upgrade integration tests"

This reverts commit 339830b.

* Added replication health check and snap dependency

* Added replication health check and snap dependency

* Remove dependencies version hashes

* Add logic to update dependencies file

* Add extra tests

* Implement upgrade from stable logic

* Fix single unit cluster upgrade

* Remove comment

* Remove integration tests

* Fix unit tests

* Minor fixes

* Add database setup call

* Initial code for test about upgrade from stable

* Add integration tests

* Test fixes

* Fix upgrade test

* Fix stable deployment

* Add configuration update on upgrade

* Fix path on CI

* Fix charm channel check

Author: marceloneppel

.github/workflows/ci.yaml

@@ -82,6 +82,8 @@ jobs:
           - password-rotation-integration
           - plugins-integration
           - tls-integration
+          - upgrade-integration
+          - upgrade-from-stable-integration
         agent-versions:
           - "2.9.45"  # renovate: latest juju 2
           - "3.1.5"  # renovate: latest juju 3

src/charm.py

@@ -148,6 +148,14 @@ def __init__(self, *args):
             log_slots=[f"{POSTGRESQL_SNAP_NAME}:logs"],
         )
 
+    @property
+    def app_units(self) -> set[Unit]:
+        """The peer-related units in the application."""
+        if not self._peers:
+            return set()
+
+        return {self.unit, *self._peers.units}
+
     @property
     def app_peer_data(self) -> Dict:
         """Application peer relation data object."""
@@ -946,6 +954,12 @@ def _can_start(self, event: StartEvent) -> bool:
             self._reboot_on_detached_storage(event)
             return False
 
+        # Safeguard against starting while upgrading.
+        if not self.upgrade.idle:
+            logger.debug("Defer on_start: Cluster is upgrading")
+            event.defer()
+            return False
+
         # Doesn't try to bootstrap the cluster if it's in a blocked state
         # caused, for example, because a failed installation of packages.
         if self.is_blocked:
@@ -992,6 +1006,17 @@ def _setup_exporter(self) -> None:
         cache = snap.SnapCache()
         postgres_snap = cache[POSTGRESQL_SNAP_NAME]
 
+        if (
+            postgres_snap.revision
+            != list(
+                filter(lambda snap_package: snap_package[0] == POSTGRESQL_SNAP_NAME, SNAP_PACKAGES)
+            )[0][1]["revision"]
+        ):
+            logger.debug(
+                "Early exit _setup_exporter: snap was not refreshed to the right version yet"
+            )
+            return
+
         postgres_snap.set(
             {
                 "exporter.user": MONITORING_USER,
@@ -1140,11 +1165,7 @@ def _on_set_password(self, event: ActionEvent) -> None:
 
     def _on_update_status(self, _) -> None:
         """Update the unit status message and users list in the database."""
-        if "cluster_initialised" not in self._peers.data[self.app]:
-            return
-
-        if self.is_blocked:
-            logger.debug("on_update_status early exit: Unit is in Blocked status")
+        if not self._can_run_on_update_status():
             return
 
         if "restoring-backup" in self.app_peer_data:
@@ -1182,6 +1203,20 @@ def _on_update_status(self, _) -> None:
         # Restart topology observer if it is gone
         self._observer.start_observer()
 
+    def _can_run_on_update_status(self) -> bool:
+        if "cluster_initialised" not in self._peers.data[self.app]:
+            return False
+
+        if not self.upgrade.idle:
+            logger.debug("Early exit on_update_status: upgrade in progress")
+            return False
+
+        if self.is_blocked:
+            logger.debug("on_update_status early exit: Unit is in Blocked status")
+            return False
+
+        return True
+
     def _handle_processes_failures(self) -> bool:
         """Handle Patroni and PostgreSQL OS processes failures.
 

src/upgrade.py

@@ -12,12 +12,13 @@
     DependencyModel,
     UpgradeGrantedEvent,
 )
-from ops.model import ActiveStatus, MaintenanceStatus, WaitingStatus
+from ops.model import MaintenanceStatus, RelationDataContent, WaitingStatus
 from pydantic import BaseModel
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed
 from typing_extensions import override
 
-from constants import SNAP_PACKAGES
+from constants import APP_SCOPE, MONITORING_PASSWORD_KEY, MONITORING_USER, SNAP_PACKAGES
+from utils import new_password
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +27,7 @@ class PostgreSQLDependencyModel(BaseModel):
     """PostgreSQL dependencies model."""
 
     charm: DependencyModel
+    snap: DependencyModel
 
 
 def get_postgresql_dependencies_model() -> PostgreSQLDependencyModel:
@@ -42,6 +44,7 @@ def __init__(self, charm, model: BaseModel, **kwargs) -> None:
         """Initialize the class."""
         super().__init__(charm, model, **kwargs)
         self.charm = charm
+        self._on_upgrade_charm_check_legacy()
 
     @override
     def build_upgrade_stack(self) -> List[int]:
@@ -77,9 +80,56 @@ def log_rollback_instructions(self) -> None:
             "Run `juju refresh --revision <previous-revision> postgresql` to initiate the rollback"
         )
 
+    def _on_upgrade_charm_check_legacy(self) -> None:
+        if not self.peer_relation or len(self.app_units) < len(self.charm.app_units):
+            logger.debug("Wait all units join the upgrade relation")
+            return
+
+        if self.state:
+            # If state set, upgrade is supported. Just set the snap information
+            # in the dependencies, as it's missing in the first revisions that
+            # support upgrades.
+            dependencies = self.peer_relation.data[self.charm.app].get("dependencies")
+            if (
+                self.charm.unit.is_leader()
+                and dependencies is not None
+                and "snap" not in json.loads(dependencies)
+            ):
+                fixed_dependencies = json.loads(dependencies)
+                fixed_dependencies["snap"] = {
+                    "dependencies": {},
+                    "name": "charmed-postgresql",
+                    "upgrade_supported": "^14",
+                    "version": "14.9",
+                }
+                self.peer_relation.data[self.charm.app].update(
+                    {"dependencies": json.dumps(fixed_dependencies)}
+                )
+            return
+
+        if not self.charm.unit.is_leader():
+            # set ready state on non-leader units
+            self.unit_upgrade_data.update({"state": "ready"})
+            return
+
+        peers_state = list(filter(lambda state: state != "", self.unit_states))
+
+        if len(peers_state) == len(self.peer_relation.units) and (
+            set(peers_state) == {"ready"} or len(peers_state) == 0
+        ):
+            if self.charm._patroni.member_started:
+                # All peers have set the state to ready
+                self.unit_upgrade_data.update({"state": "ready"})
+                self._prepare_upgrade_from_legacy()
+            getattr(self.on, "upgrade_charm").emit()
+
     @override
     def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:
         # Refresh the charmed PostgreSQL snap and restart the database.
+        # Update the configuration.
+        self.charm.unit.status = MaintenanceStatus("updating configuration")
+        self.charm.update_config()
+
         self.charm.unit.status = MaintenanceStatus("refreshing the snap")
         self.charm._install_snap_packages(packages=SNAP_PACKAGES, refresh=True)
 
@@ -91,6 +141,14 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:
         self.charm._setup_exporter()
         self.charm.backup.start_stop_pgbackrest_service()
 
+        try:
+            self.charm.unit.set_workload_version(
+                self.charm._patroni.get_postgresql_version() or "unset"
+            )
+        except TypeError:
+            # Don't fail on this, just log it.
+            logger.warning("Failed to get PostgreSQL version")
+
         # Wait until the database initialise.
         self.charm.unit.status = WaitingStatus("waiting for database initialisation")
         try:
@@ -110,7 +168,6 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:
                         raise Exception()
 
                     self.set_unit_completed()
-                    self.charm.unit.status = ActiveStatus()
 
                     # Ensures leader gets its own relation-changed when it upgrades
                     if self.charm.unit.is_leader():
@@ -144,3 +201,36 @@ def pre_upgrade_check(self) -> None:
                 "a backup is being created",
                 "wait for the backup creation to finish before starting the upgrade",
             )
+
+    def _prepare_upgrade_from_legacy(self) -> None:
+        """Prepare upgrade from legacy charm without upgrade support.
+
+        Assumes run on leader unit only.
+        """
+        logger.warning("Upgrading from unsupported version")
+
+        # Populate app upgrade databag to allow upgrade procedure
+        logger.debug("Building upgrade stack")
+        upgrade_stack = self.build_upgrade_stack()
+        logger.debug(f"Upgrade stack: {upgrade_stack}")
+        self.upgrade_stack = upgrade_stack
+        logger.debug("Persisting dependencies to upgrade relation data...")
+        self.peer_relation.data[self.charm.app].update(
+            {"dependencies": json.dumps(self.dependency_model.dict())}
+        )
+        if self.charm.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY) is None:
+            self.charm.set_secret(APP_SCOPE, MONITORING_PASSWORD_KEY, new_password())
+        users = self.charm.postgresql.list_users()
+        if MONITORING_USER not in users:
+            # Create the monitoring user.
+            self.charm.postgresql.create_user(
+                MONITORING_USER,
+                self.charm.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY),
+                extra_user_roles="pg_monitor",
+            )
+        self.charm.postgresql.set_up_database()
+
+    @property
+    def unit_upgrade_data(self) -> RelationDataContent:
+        """Return the application upgrade data."""
+        return self.peer_relation.data[self.charm.unit]