Make module for building images standalone

Previously inherited from BaseProductTest and depended
on testing functionality to operate properly.

Author: zachyee

.travis.yml

@@ -20,13 +20,13 @@ env:
     - PRODUCT_TEST_GROUP_SA_PA_PRESTO="tests/product/test_control.py"
     - PRODUCT_TEST_GROUP_SA_PA_PRESTO="tests/product/test_server_uninstall.py"
     - PRODUCT_TEST_GROUP_YS_PA="tests/product/yarn_slider/test_slider_installation.py"
-    #-QUARANTINED="tests/product/yarn_slider/test_server_install.py tests/product/test_create_images.py"
-install: 
+    #-QUARANTINED="tests/product/yarn_slider/test_server_install.py"
+install:
   - pip install --upgrade pip==6.1.1
   - pip install -r requirements.txt
 before_script:
   - make docker-images
-script: 
+script:
   - |
     if [ -v PRODUCT_TEST_GROUP_SA_BARE ]; then
       export IMAGE_NAMES="--standalone_bare"

Makefile

@@ -89,7 +89,7 @@ test-all: clean-test docker-images
 IMAGE_NAMES?="--all"
 
 test-images: docker-images
-	python tests/product/test_create_images.py ${IMAGE_NAMES}
+	python tests/product/image_builder.py ${IMAGE_NAMES}
 
 docker-images:
 	docker pull teradatalabs/centos6-ssh-oj8

tests/product/image_builder.py

@@ -0,0 +1,203 @@
+from subprocess import check_output
+from sys import argv, exit
+from tests.configurable_cluster import ConfigurableCluster
+from tests.docker_cluster import DockerCluster, DockerClusterException
+from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider
+
+from tests.product.mode_installers import StandaloneModeInstaller, \
+    YarnSliderModeInstaller
+from tests.product.prestoadmin_installer import PrestoadminInstaller
+from tests.product.standalone.presto_installer import StandalonePrestoInstaller
+from tests.product.topology_installer import TopologyInstaller
+from tests.product.yarn_slider.slider_installer import SliderInstaller
+
+import argparse
+
+
+class ProductTestImageBuilder:
+    STANDALONE_BARE_CLUSTER = 'bare'
+    BARE_CLUSTER = 'bare'
+    PA_ONLY_CLUSTER = 'pa_only_standalone'
+    STANDALONE_PRESTO_CLUSTER = 'presto'
+
+    PA_ONLY_YS_CLUSTER = 'pa_only_ys'
+    PA_SLIDER_CLUSTER = 'pa_slider'
+
+    _cluster_types = {
+        BARE_CLUSTER: [],
+        PA_ONLY_CLUSTER: [PrestoadminInstaller,
+                          StandaloneModeInstaller],
+        STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller,
+                                    StandaloneModeInstaller,
+                                    TopologyInstaller,
+                                    StandalonePrestoInstaller],
+        PA_ONLY_YS_CLUSTER: [PrestoadminInstaller,
+                             YarnSliderModeInstaller],
+        PA_SLIDER_CLUSTER: [PrestoadminInstaller,
+                            YarnSliderModeInstaller,
+                            SliderInstaller]
+    }
+
+    def __init__(self):
+        self.maxDiff = None
+        self.cluster = None
+        self.default_keywords = {}
+
+    def _apply_post_install_hooks(self, installers):
+        for installer in installers:
+            self.cluster.postinstall(installer)
+
+    def _update_replacement_keywords(self, installers):
+        for installer in installers:
+            installer_instance = installer(self)
+            self.default_keywords.update(installer_instance.get_keywords())
+
+    def _run_installers(self, installers):
+        cluster = self.cluster
+        for installer in installers:
+            dependencies = installer.get_dependencies()
+
+            for dependency in dependencies:
+                dependency.assert_installed(self)
+
+            installer_instance = installer(self)
+            installer_instance.install()
+
+            self.default_keywords.update(installer_instance.get_keywords())
+            cluster.postinstall(installer)
+
+    def _setup_cluster(self, bare_image_provider, cluster_type):
+        installers = None
+        try:
+            installers = self._cluster_types[cluster_type]
+        except KeyError:
+            print '%s is not a valid cluster type. Valid cluster types are %s' % \
+                  (cluster_type, ', '.join(self._cluster_types.keys()))
+            exit(1)
+
+        config_filename = ConfigurableCluster.check_for_cluster_config()
+
+        if config_filename:
+            self.cluster = ConfigurableCluster.start_bare_cluster(
+                config_filename, self,
+                StandalonePrestoInstaller.assert_installed)
+        else:
+            try:
+                self.cluster, bare_cluster = DockerCluster.start_cluster(
+                    bare_image_provider, cluster_type)
+
+                # If we've found images and started a non-bare cluster, the
+                # containers have already had the installers applied to them.
+                # We do need to get the test environment in sync with the
+                # containers by calling the following two functions.
+                #
+                # Once that's done, the cluster and test environment is in the
+                # same state it would be as if we'd called _run_installers on
+                # a bare cluster, and we can return.
+                #
+                # We do this to save the cost of running the installers on the
+                # docker containers every time we run a test. In practice,
+                # that turns out to be a fairly expensive thing to do.
+                if not bare_cluster:
+                    self._apply_post_install_hooks(installers)
+                    self._update_replacement_keywords(installers)
+                    return
+            except DockerClusterException as e:
+                print e.msg
+                exit(1)
+
+        # If we got a bare cluster back, we need to run the installers on it.
+        # applying the post-install hooks and updating the replacement
+        # keywords is handled internally in _run_installers.
+        self._run_installers(installers)
+
+        if isinstance(self.cluster, DockerCluster):
+            self.cluster.commit_images(bare_image_provider, cluster_type)
+
+    def _get_expected_image_names(self, cluster_type):
+        if cluster_type == self.BARE_CLUSTER:
+            slave_image_name = 'teradatalabs/pa_test/nohadoop_bare_slave'
+            master_image_name = 'teradatalabs/pa_test/nohadoop_bare_master'
+            return [slave_image_name, master_image_name]
+        elif cluster_type == self.PA_ONLY_CLUSTER:
+            slave_image_name = 'teradatalabs/pa_test/nohadoop_pa_only_standalone_slave'
+            master_image_name = 'teradatalabs/pa_test/nohadoop_pa_only_standalone_master'
+            return [slave_image_name, master_image_name]
+        elif cluster_type == self.STANDALONE_PRESTO_CLUSTER:
+            slave_image_name = 'teradatalabs/pa_test/nohadoop_presto_slave'
+            master_image_name = 'teradatalabs/pa_test/nohadoop_presto_master'
+            return [slave_image_name, master_image_name]
+        elif cluster_type == self.PA_ONLY_YS_CLUSTER:
+            slave_image_name = 'teradatalabs/pa_test/nohadoop_pa_only_ys_slave'
+            master_image_name = 'teradatalabs/pa_test/nohadoop_pa_only_ys_master'
+            return [slave_image_name, master_image_name]
+        else:
+            return []
+
+    def _assert_expected_images_created(self, cluster_type):
+        docker_images = check_output(['docker', 'images'])
+        expected_images = self._get_expected_image_names(cluster_type)
+        for expected_image in expected_images:
+            assert expected_image in docker_images
+
+    def _setup_cluster_with_no_hadoop_provider_assert_images(self, cluster_type):
+        self._setup_cluster(NoHadoopBareImageProvider(),
+                            cluster_type)
+        self._assert_expected_images_created(cluster_type)
+
+    def test_setup_standalone_presto_images(self):
+        cluster_type = self.STANDALONE_PRESTO_CLUSTER
+        self._setup_cluster_with_no_hadoop_provider_assert_images(cluster_type)
+
+    def test_setup_standalone_pa_only_images(self):
+        cluster_type = self.PA_ONLY_CLUSTER
+        self._setup_cluster_with_no_hadoop_provider_assert_images(cluster_type)
+
+    def test_setup_standalone_bare_images(self):
+        cluster_type = self.BARE_CLUSTER
+        self._setup_cluster_with_no_hadoop_provider_assert_images(cluster_type)
+
+    def test_setup_yarn_slider_pa_only_images(self):
+        cluster_type = self.PA_ONLY_YS_CLUSTER
+        self._setup_cluster_with_no_hadoop_provider_assert_images(cluster_type)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--standalone_presto",
+                        help="Create standalone presto images",
+                        action="store_true")
+    parser.add_argument("--standalone_pa",
+                        help="Create standalone presto-admin only images",
+                        action="store_true")
+    parser.add_argument("--standalone_bare",
+                        help="Create standalone bare images",
+                        action="store_true")
+    parser.add_argument("--yarn_slider_pa",
+                        help="Create yarn-slider presto-admin only images",
+                        action="store_true")
+    parser.add_argument("--all",
+                        help="Create standalone presto, standalone presto-admin only,"
+                             "standalone bare, and yarn-slider presto-admin only images",
+                        action="store_true")
+
+    if len(argv) <= 1:
+        parser.print_help()
+        exit(1)
+
+    image_builder = ProductTestImageBuilder()
+
+    args = parser.parse_args()
+    if args.all:
+        image_builder.test_setup_standalone_pa_only_images()
+        image_builder.test_setup_standalone_pa_only_images()
+        image_builder.test_setup_standalone_bare_images()
+        image_builder.test_setup_yarn_slider_pa_only_images()
+    else:
+        if args.standalone_presto:
+            image_builder.test_setup_standalone_presto_images()
+        if args.standalone_pa:
+            image_builder.test_setup_standalone_pa_only_images()
+        if args.standalone_bare:
+            image_builder.test_setup_standalone_bare_images()
+        if args.yarn_slider_pa:
+            image_builder.test_setup_yarn_slider_pa_only_images()