Robustify pool setup and teardown in storage tests, first step

* New Pool.exec_on_hosts_on_error_rollback method: attempts
  to run a function for every host, stops at the first error,
  tries to run a rollback function to undo what may have been
  partially done, then raises with the collected exception(s).
  Mainly meant to be used in fixtures setups that alter all hosts in a
  pool.
* New Pool.exec_on_hosts_on_error_continue method: attempts to run a
  function for every host. Collects errors but continues with the next
  hosts anyway. Raises at the end with the collected exception(s).
* New host_with_saved_yum_state fixture to handle state saving and
  rollback in a shared package-scope fixture.
* New pool_with_saved_yum_state, to handle the same on multiple hosts.
* Updated MooseFS fixtures using the above methods and fixtures.

Signed-off-by: Samuel Verschelde <[email protected]>

Author: stormi

lib/pool.py

@@ -1,4 +1,5 @@
 import logging
+import traceback
 
 import lib.commands as commands
 
@@ -19,6 +20,58 @@ def __init__(self, master_hostname_or_ip):
         self.uuid = self.master.xe('pool-list', minimal=True)
         self.saved_uefi_certs = None
 
+    def exec_on_hosts_on_error_rollback(self, func, rollback_func, host_list=[]):
+        """
+        Execute a function on all hosts of the pool.
+
+        If something fails: execute rollback_func on each host on which the command was already executed.
+        Then stop.
+        Designed mainly for use in fixture setups that alter the hosts state.
+        """
+        hosts_done = []
+        hosts = host_list if host_list else self.hosts
+        for h in hosts:
+            try:
+                func(h)
+                hosts_done.append(h)
+            except Exception as e:
+                if rollback_func:
+                    logging.warning(
+                        f"An error occurred in `exec_on_hosts_on_error_rollback` for host {h}\n"
+                        f"Backtrace:\n{traceback.format_exc()}"
+                    )
+                    rollback_hosts = hosts_done + [h]
+
+                    logging.info("Attempting to run the rollback function on host(s) "
+                                 f"{', '.join([str(h) for h in rollback_hosts])}...")
+                    try:
+                        self.exec_on_hosts_on_error_continue(rollback_func, rollback_hosts)
+                    except Exception:
+                        pass
+                raise e
+
+    def exec_on_hosts_on_error_continue(self, func, host_list=[]):
+        """
+        Execute a function on all hosts of the pool.
+
+        If something fails: store the exception but still attempt the function on the next hosts.
+        Designed mainly for use in fixture teardowns.
+        """
+        errors = {}
+        hosts = host_list if host_list else self.hosts
+        for h in hosts:
+            try:
+                func(h)
+            except Exception as e:
+                logging.warning(
+                    f"An error occurred in `exec_on_hosts_on_error_continue` for host {h}\n"
+                    f"Backtrace:\n{traceback.format_exc()}"
+                )
+                logging.info("Attempting to run the function on the next hosts of the pool if there are any left...")
+                errors[h.hostname_or_ip] = e
+        if errors:
+            raise Exception(f"One or more exceptions were raised in `exec_on_hosts_on_error_continue`: {errors}")
+
     def hosts_uuids(self):
         return safe_split(self.master.xe('host-list', {}, minimal=True))
 

pkgfixtures.py

@@ -23,3 +23,16 @@ def formatted_and_mounted_ext4_disk(host, sr_disk):
     setup_formatted_and_mounted_disk(host, sr_disk, 'ext4', mountpoint)
     yield mountpoint
     teardown_formatted_and_mounted_disk(host, mountpoint)
+
+@pytest.fixture(scope='package')
+def host_with_saved_yum_state(host):
+    host.yum_save_state()
+    yield host
+    host.yum_restore_saved_state()
+
+@pytest.fixture(scope='package')
+def pool_with_saved_yum_state(host):
+    for h in host.pool.hosts:
+        h.yum_save_state()
+    yield host.pool
+    host.pool.exec_on_hosts_on_error_continue(lambda h: h.yum_restore_saved_state())

tests/storage/moosefs/conftest.py

@@ -1,35 +1,38 @@
 import logging
 import pytest
 
-from lib.pool import Pool
+# explicit import for package-scope fixtures
+from pkgfixtures import pool_with_saved_yum_state
 
-def enable_moosefs(host):
+def install_moosefs(host):
     assert not host.file_exists('/usr/sbin/mount.moosefs'), \
         "MooseFS client should not be installed on the host before all tests"
     host.ssh(['curl https://ppa.moosefs.com/RPM-GPG-KEY-MooseFS > /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS'])
     host.ssh(['curl http://ppa.moosefs.com/MooseFS-3-el7.repo > /etc/yum.repos.d/MooseFS.repo'])
-    host.yum_save_state()
     host.yum_install(['fuse', 'moosefs-client'])
 
+def uninstall_moosefs_repo(host):
+    host.ssh(['rm -f /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS/etc/yum.repos.d/MooseFS.repo'])
+
+def enable_moosefs(host):
     host.activate_smapi_driver('moosefs')
 
 def disable_moosefs(host):
     host.deactivate_smapi_driver('moosefs')
 
-    host.yum_restore_saved_state()
-    host.ssh(['rm -f /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS'])
-    host.ssh(['rm -f /etc/yum.repos.d/MooseFS.repo'])
-
 @pytest.fixture(scope='package')
-def pool_with_moosefs(hostA1):
-    pool = Pool(hostA1.hostname_or_ip)
-    for host in pool.hosts:
-        enable_moosefs(host)
-
+def pool_with_moosefs_installed(pool_with_saved_yum_state):
+    pool = pool_with_saved_yum_state
+    pool.exec_on_hosts_on_error_rollback(install_moosefs, uninstall_moosefs_repo)
     yield pool
+    pool.exec_on_hosts_on_error_continue(uninstall_moosefs_repo)
 
-    for host in pool.hosts:
-        disable_moosefs(host)
+@pytest.fixture(scope='package')
+def pool_with_moosefs_enabled(pool_with_moosefs_installed):
+    pool = pool_with_moosefs_installed
+    pool.exec_on_hosts_on_error_rollback(enable_moosefs, disable_moosefs)
+    yield pool
+    pool.exec_on_hosts_on_error_continue(disable_moosefs)
 
 @pytest.fixture(scope='package')
 def moosefs_device_config(sr_device_config):
@@ -49,9 +52,9 @@ def moosefs_device_config(sr_device_config):
     return config
 
 @pytest.fixture(scope='package')
-def moosefs_sr(moosefs_device_config, pool_with_moosefs):
+def moosefs_sr(moosefs_device_config, pool_with_moosefs_enabled):
     """ MooseFS SR on a specific host. """
-    sr = pool_with_moosefs.master.sr_create('moosefs', "MooseFS-SR-test", moosefs_device_config, shared=True)
+    sr = pool_with_moosefs_enabled.master.sr_create('moosefs', "MooseFS-SR-test", moosefs_device_config, shared=True)
     yield sr
     # teardown
     sr.destroy()

tests/storage/moosefs/test_moosefs_sr.py

@@ -30,9 +30,9 @@ def test_create_moosefs_sr_without_mfsmount(self, host, moosefs_device_config):
             sr.destroy()
             assert False, "MooseFS SR creation should failed!"
 
-    def test_create_and_destroy_sr(self, moosefs_device_config, pool_with_moosefs):
+    def test_create_and_destroy_sr(self, moosefs_device_config, pool_with_moosefs_enabled):
         # Create and destroy tested in the same test to leave the host as unchanged as possible
-        master = pool_with_moosefs.master
+        master = pool_with_moosefs_enabled.master
         sr = master.sr_create('moosefs', "MooseFS-SR-test2", moosefs_device_config, shared=True, verify=True)
         # import a VM in order to detect vm import issues here rather than in the vm_on_moosefs_sr used in
         # the next tests, because errors in fixtures break teardown