Fix travis - job timeout and test timeout

.travis.yml

@@ -6,12 +6,50 @@ sudo: required
 services:
   - docker
 env:
-install: 
+  global:
+    - PA_TEST_ONLINE_INSTALLER=true
+    - PYTHONPATH=$(pwd)
+    - LONG_PRODUCT_TESTS="tests/product/test_server_install.py tests/product/test_status.py tests/product/test_collect.py tests/product/test_connectors.py tests/product/test_control.py tests/product/test_server_uninstall.py"
+  matrix:
+    - OTHER_TESTS=true
+    - SHORT_PRODUCT_TEST_GROUP=0
+    - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN="tests/product/test_server_install.py"
+    - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_status.py"
+    - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_collect.py tests/product/test_server_uninstall.py"
+    - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_connectors.py"
+    - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_control.py"
+install:
   - pip install --upgrade pip==6.1.1
   - pip install -r requirements.txt
 before_script:
   - make docker-images
-script: 
-  - make clean lint dist docs 
-  - nosetests --with-timer --timer-ok 60s --timer-warning 300s -s tests.unit
-  - nosetests --with-timer --timer-ok 60s --timer-warning 300s -s tests.integration
+  - make presto-server-rpm.rpm
+script:
+  - |
+    if [ -v SHORT_PRODUCT_TEST_GROUP ]; then
+      ALL_PRODUCT_TESTS=$(find tests/product/ -name 'test_*py' | grep -v __init__ | xargs wc -l | sort -n | head -n -1 | awk '{print $2}' | tr '\n' ' ') &&
+      for LONG_PRODUCT_TEST in ${LONG_PRODUCT_TESTS[@]}; do
+        ALL_PRODUCT_TESTS=${ALL_PRODUCT_TESTS//$LONG_PRODUCT_TEST/};
+        if [ $? -ne 0 ]; then
+          exit 1
+        fi
+      done &&
+      SHORT_PRODUCT_TESTS=$(echo $ALL_PRODUCT_TESTS | tr ' ' '\n' | awk "NR % 1 == $SHORT_PRODUCT_TEST_GROUP" | tr '\n' ' ') &&
+      make test-images &&
+      nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${SHORT_PRODUCT_TESTS};
+    elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN ]; then
+      export IMAGE_NAMES="standalone_presto_admin" &&
+      make test-images &&
+      nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN};
+    elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO ]; then
+      export IMAGE_NAMES="standalone_presto standalone_presto_admin" &&
+      make test-images &&
+      nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO};
+    elif [ -v OTHER_TESTS ]; then
+      make clean lint dist docs test-rpm &&
+      nosetests -s tests.unit &&
+      nosetests -s tests.integration;
+    else
+      echo "Unknown test" &&
+      exit 1;
+    fi

Makefile

@@ -1,6 +1,6 @@
 .PHONY: clean-all clean clean-eggs clean-build clean-pyc clean-test-containers clean-test \
-	clean-docs lint smoke test test-all test-rpm coverage docs open-docs release release-builds \
-	dist dist-online dist-offline wheel install precommit
+	clean-docs lint smoke test test-all test-images test-rpm docker-images coverage docs \
+	open-docs release release-builds dist dist-online dist-offline wheel install precommit
 
 help:
 	@echo "precommit - run \`quick' tests and tasks that should pass or succeed prior to pushing"
@@ -16,7 +16,9 @@ help:
 	@echo "smoke - run tests annotated with attr smoke using nosetests"
 	@echo "test - run tests quickly with Python 2.6 and 2.7"
 	@echo "test-all - run tests on every Python version with tox. Specify TEST_SUITE env variable to run only a given suite."
+	@echo "test-images - create product test image(s). Specify IMAGE_NAMES env variable to create only certain images."
 	@echo "test-rpm - run tests for the RPM package"
+	@echo "docker-images - pull docker image(s). Specify DOCKER_IMAGE_NAME env variable for specific image."
 	@echo "coverage - check code coverage quickly with the default Python"
 	@echo "docs - generate Sphinx HTML documentation, including API docs"
 	@echo "open-docs - open the root document (index.html) using xdg-open"
@@ -70,6 +72,9 @@ clean-docs:
 lint:
 	flake8 prestoadmin packaging tests
 
+presto-server-rpm.rpm:
+	wget 'https://repository.sonatype.org/service/local/artifact/maven/content?r=central-proxy&g=com.facebook.presto&a=presto-server-rpm&e=rpm&v=RELEASE' -O $@
+
 smoke: clean-test docker-images
 	tox -e py26 -- -a smoketest,'!quarantine'
 
@@ -84,11 +89,15 @@ test-all: clean-test docker-images
 	tox -- -s tests.integration
 	tox -e py26 -- -s ${TEST_SUITE} -a '!quarantine'
 
+IMAGE_NAMES?="all"
+
+test-images: docker-images presto-server-rpm.rpm
+	python tests/product/image_builder.py ${IMAGE_NAMES}
 
 docker-images:
 	docker pull teradatalabs/centos6-ssh-oj8
 
-test-rpm: clean-test
+test-rpm: clean-test test-images
 	tox -e py26 -- -s tests.rpm -a '!quarantine'
 
 coverage:

tests/product/base_product_case.py

@@ -26,45 +26,19 @@
 
 from prestoadmin.util import constants
 from tests.base_test_case import BaseTestCase
-from tests.configurable_cluster import ConfigurableCluster
-from tests.docker_cluster import DockerCluster, DockerClusterException
+from tests.docker_cluster import DockerCluster
 
-from tests.product.mode_installers import StandaloneModeInstaller, \
-    YarnSliderModeInstaller
-from tests.product.prestoadmin_installer import PrestoadminInstaller
+from tests.configurable_cluster import ConfigurableCluster
+from tests.product.cluster_types import cluster_types
 from tests.product.standalone.presto_installer import StandalonePrestoInstaller
-from tests.product.topology_installer import TopologyInstaller
-from tests.product.yarn_slider.slider_installer import SliderInstaller
+
 
 PRESTO_VERSION = r'.+'
 RETRY_TIMEOUT = 120
 RETRY_INTERVAL = 5
 
 
 class BaseProductTestCase(BaseTestCase):
-    STANDALONE_BARE_CLUSTER = 'bare'
-    BARE_CLUSTER = 'bare'
-    PA_ONLY_CLUSTER = 'pa_only_standalone'
-    STANDALONE_PRESTO_CLUSTER = 'presto'
-
-    PA_ONLY_YS_CLUSTER = 'pa_only_ys'
-    PA_SLIDER_CLUSTER = 'pa_slider'
-
-    _cluster_types = {
-        BARE_CLUSTER: [],
-        PA_ONLY_CLUSTER: [PrestoadminInstaller,
-                          StandaloneModeInstaller],
-        STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller,
-                                    StandaloneModeInstaller,
-                                    TopologyInstaller,
-                                    StandalonePrestoInstaller],
-        PA_ONLY_YS_CLUSTER: [PrestoadminInstaller,
-                             YarnSliderModeInstaller],
-        PA_SLIDER_CLUSTER: [PrestoadminInstaller,
-                            YarnSliderModeInstaller,
-                            SliderInstaller]
-    }
-
     default_workers_config_ = """coordinator=false
 discovery.uri=http://master:8080
 http-server.http.port=8080
@@ -134,19 +108,11 @@ def setUp(self):
         self.cluster = None
         self.default_keywords = {}
 
-    def _run_installers(self, installers):
-        cluster = self.cluster
-        for installer in installers:
-            dependencies = installer.get_dependencies()
-
-            for dependency in dependencies:
-                dependency.assert_installed(self)
-
-            installer_instance = installer(self)
-            installer_instance.install()
-
-            self.default_keywords.update(installer_instance.get_keywords())
-            cluster.postinstall(installer)
+    def tearDown(self):
+        self.restore_stdout_stderr_keep_open()
+        if self.cluster:
+            self.cluster.tear_down()
+        super(BaseProductTestCase, self).tearDown()
 
     def _apply_post_install_hooks(self, installers):
         for installer in installers:
@@ -158,12 +124,7 @@ def _update_replacement_keywords(self, installers):
             self.default_keywords.update(installer_instance.get_keywords())
 
     def setup_cluster(self, bare_image_provider, cluster_type):
-        try:
-            installers = self._cluster_types[cluster_type]
-        except KeyError:
-            self.fail(
-                '%s is not a valid cluster type. Valid cluster types are %s' %
-                (cluster_type, ', '.join(self._cluster_types.keys())))
+        installers = cluster_types[cluster_type]
 
         config_filename = ConfigurableCluster.check_for_cluster_config()
 
@@ -172,42 +133,22 @@ def setup_cluster(self, bare_image_provider, cluster_type):
                 config_filename, self,
                 StandalonePrestoInstaller.assert_installed)
         else:
-            try:
-                self.cluster, bare_cluster = DockerCluster.start_cluster(
-                    bare_image_provider, cluster_type)
-
-                # If we've found images and started a non-bare cluster, the
-                # containers have already had the installers applied to them.
-                # We do need to get the test environment in sync with the
-                # containers by calling the following two functions.
-                #
-                # Once that's done, the cluster and test environment is in the
-                # same state it would be as if we'd called _run_installers on
-                # a bare cluster, and we can return.
-                #
-                # We do this to save the cost of running the installers on the
-                # docker containers every time we run a test. In practice,
-                # that turns out to be a fairly expensive thing to do.
-                if not bare_cluster:
-                    self._apply_post_install_hooks(installers)
-                    self._update_replacement_keywords(installers)
-                    return
-            except DockerClusterException as e:
-                self.fail(e.msg)
-
-        # If we got a bare cluster back, we need to run the installers on it.
-        # applying the post-install hooks and updating the replacement
-        # keywords is handled internally in _run_installers.
-        self._run_installers(installers)
-
-        if isinstance(self.cluster, DockerCluster):
-            self.cluster.commit_images(bare_image_provider, cluster_type)
-
-    def tearDown(self):
-        self.restore_stdout_stderr_keep_open()
-        if self.cluster:
-            self.cluster.tear_down()
-        super(BaseProductTestCase, self).tearDown()
+            self.cluster, bare_cluster = DockerCluster.start_cluster(
+                bare_image_provider, cluster_type)
+
+            # If we've found images and started a non-bare cluster, the
+            # containers have already had the installers applied to them.
+            # We do need to get the test environment in sync with the
+            # containers by calling the following two functions.
+            #
+            # We do this to save the cost of running the installers on the
+            # docker containers every time we run a test. In practice,
+            # that turns out to be a fairly expensive thing to do.
+            if not bare_cluster:
+                self._apply_post_install_hooks(installers)
+                self._update_replacement_keywords(installers)
+            else:
+                raise RuntimeError("Docker images have not been created")
 
     def dump_and_cp_topology(self, topology, cluster=None):
         if not cluster:

tests/product/cluster_types.py

@@ -0,0 +1,30 @@
+from tests.product.mode_installers import StandaloneModeInstaller, \
+    YarnSliderModeInstaller
+from tests.product.prestoadmin_installer import PrestoadminInstaller
+from tests.product.topology_installer import TopologyInstaller
+from tests.product.yarn_slider.slider_installer import SliderInstaller
+from tests.product.standalone.presto_installer import StandalonePrestoInstaller
+
+
+STANDALONE_BARE_CLUSTER = 'bare'
+BARE_CLUSTER = 'bare'
+STANDALONE_PA_CLUSTER = 'pa_only_standalone'
+STANDALONE_PRESTO_CLUSTER = 'presto'
+
+YARN_SLIDER_PA_CLUSTER = 'pa_only_ys'
+YARN_SLIDER_PA_AND_SLIDER_CLUSTER = 'pa_slider'
+
+cluster_types = {
+    BARE_CLUSTER: [],
+    STANDALONE_PA_CLUSTER: [PrestoadminInstaller,
+                            StandaloneModeInstaller],
+    STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller,
+                                StandaloneModeInstaller,
+                                TopologyInstaller,
+                                StandalonePrestoInstaller],
+    YARN_SLIDER_PA_CLUSTER: [PrestoadminInstaller,
+                             YarnSliderModeInstaller],
+    YARN_SLIDER_PA_AND_SLIDER_CLUSTER: [PrestoadminInstaller,
+                                        YarnSliderModeInstaller,
+                                        SliderInstaller]
+}

tests/product/image_builder.py

@@ -0,0 +1,113 @@
+from tests.docker_cluster import DockerCluster
+from tests.hdp_bare_image_provider import HdpBareImageProvider
+from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider
+
+from tests.product.base_product_case import BaseProductTestCase
+from tests.product.cluster_types import STANDALONE_BARE_CLUSTER, STANDALONE_PA_CLUSTER, \
+    STANDALONE_PRESTO_CLUSTER, YARN_SLIDER_PA_CLUSTER, cluster_types
+
+import argparse
+
+
+class ImageBuilder:
+    def __init__(self, testcase):
+        self.testcase = testcase
+        self.testcase.default_keywords = {}
+        self.testcase.cluster = None
+
+    def _run_installers(self, installers):
+        cluster = self.testcase.cluster
+        for installer in installers:
+            dependencies = installer.get_dependencies()
+
+            for dependency in dependencies:
+                dependency.assert_installed(self.testcase)
+
+            installer_instance = installer(self.testcase)
+            installer_instance.install()
+
+            self.testcase.default_keywords.update(installer_instance.get_keywords())
+            cluster.postinstall(installer)
+
+    def _setup_image(self, bare_image_provider, cluster_type):
+        installers = cluster_types[cluster_type]
+
+        self.testcase.cluster, bare_cluster = DockerCluster.start_cluster(
+            bare_image_provider, cluster_type)
+
+        # If we got a bare cluster back, we need to run the installers on it.
+        # applying the post-install hooks and updating the replacement
+        # keywords is handled internally in _run_installers.
+        #
+        # If we got a non-bare cluster back, that means the image already exists
+        # and we created the cluster using that image.
+        if bare_cluster:
+            self._run_installers(installers)
+
+            if isinstance(self.testcase.cluster, DockerCluster):
+                self.testcase.cluster.commit_images(bare_image_provider, cluster_type)
+
+        self.testcase.cluster.tear_down()
+
+    def _setup_image_with_no_hadoop_provider(self, cluster_type):
+        self._setup_image(NoHadoopBareImageProvider(),
+                          cluster_type)
+
+    def setup_standalone_presto_images(self):
+        cluster_type = STANDALONE_PRESTO_CLUSTER
+        self._setup_image_with_no_hadoop_provider(cluster_type)
+
+    def setup_standalone_presto_admin_images(self):
+        cluster_type = STANDALONE_PA_CLUSTER
+        self._setup_image_with_no_hadoop_provider(cluster_type)
+
+    def setup_standalone_bare_images(self):
+        cluster_type = STANDALONE_BARE_CLUSTER
+        self._setup_image_with_no_hadoop_provider(cluster_type)
+
+    def setup_yarn_slider_presto_admin_images(self):
+        cluster_type = YARN_SLIDER_PA_CLUSTER
+        self._setup_image_with_no_hadoop_provider(cluster_type)
+
+    def _setup_cluster_with_hdp_image_provider(self, cluster_type):
+        self._setup_image(HdpBareImageProvider(),
+                          cluster_type)
+
+    def setup_yarn_slider_presto_admin_and_slider_images(self):
+        cluster_type = self.testcase.YARN_SLIDER_PA_AND_SLIDER_CLUSTER
+        self._setup_cluster_with_hdp_image_provider(cluster_type)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("image_type", metavar="image_type", type=str, nargs="+",
+                        choices=["standalone_presto", "standalone_presto_admin",
+                                 "standalone_bare", "yarn_slider_presto_admin",
+                                 "all"],
+                        help="Specify the type of image to create. The available choices are: "
+                             "standalone_presto, standalone_presto_admin, standalone_bare, "
+                             "yarn_slider_presto_admin, all")
+
+    # ImageBuilder needs an input testcase with access to unittest assertions
+    # so the installers can check their resulting installations as well as some
+    # product test helper functions.
+    # This supplies a dummy testcase. BaseProductTestCase inherits from
+    # unittest. A unittest instance can be successfully created if the name
+    # of an existing method of the class is passed into the constructor.
+    dummy_testcase = BaseProductTestCase('__init__')
+    image_builder = ImageBuilder(dummy_testcase)
+
+    args = parser.parse_args()
+    if "all" in args.image_type:
+        image_builder.setup_standalone_presto_images()
+        image_builder.setup_standalone_presto_admin_images()
+        image_builder.setup_standalone_bare_images()
+        image_builder.setup_yarn_slider_presto_admin_images()
+    else:
+        if "standalone_presto" in args.image_type:
+            image_builder.setup_standalone_presto_images()
+        if "standalone_presto_admin" in args.image_type:
+            image_builder.setup_standalone_presto_admin_images()
+        if "standalone_bare" in args.image_type:
+            image_builder.setup_standalone_bare_images()
+        if "yarn_slider_presto_admin" in args.image_type:
+            image_builder.setup_yarn_slider_presto_admin_images()