diff --git a/.travis.yml b/.travis.yml index e8dc5457..636212cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,12 +6,50 @@ sudo: required services: - docker env: -install: + global: + - PA_TEST_ONLINE_INSTALLER=true + - PYTHONPATH=$(pwd) + - LONG_PRODUCT_TESTS="tests/product/test_server_install.py tests/product/test_status.py tests/product/test_collect.py tests/product/test_connectors.py tests/product/test_control.py tests/product/test_server_uninstall.py" + matrix: + - OTHER_TESTS=true + - SHORT_PRODUCT_TEST_GROUP=0 + - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN="tests/product/test_server_install.py" + - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_status.py" + - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_collect.py tests/product/test_server_uninstall.py" + - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_connectors.py" + - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_control.py" +install: - pip install --upgrade pip==6.1.1 - pip install -r requirements.txt before_script: - make docker-images -script: - - make clean lint dist docs - - nosetests --with-timer --timer-ok 60s --timer-warning 300s -s tests.unit - - nosetests --with-timer --timer-ok 60s --timer-warning 300s -s tests.integration + - make presto-server-rpm.rpm +script: + - | + if [ -v SHORT_PRODUCT_TEST_GROUP ]; then + ALL_PRODUCT_TESTS=$(find tests/product/ -name 'test_*py' | grep -v __init__ | xargs wc -l | sort -n | head -n -1 | awk '{print $2}' | tr '\n' ' ') && + for LONG_PRODUCT_TEST in ${LONG_PRODUCT_TESTS[@]}; do + ALL_PRODUCT_TESTS=${ALL_PRODUCT_TESTS//$LONG_PRODUCT_TEST/}; + if [ $? -ne 0 ]; then + exit 1 + fi + done && + SHORT_PRODUCT_TESTS=$(echo $ALL_PRODUCT_TESTS | tr ' ' '\n' | awk "NR % 1 == $SHORT_PRODUCT_TEST_GROUP" | tr '\n' ' ') && + make test-images && + nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${SHORT_PRODUCT_TESTS}; + elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN ]; then + export IMAGE_NAMES="standalone_presto_admin" && + make test-images && + nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN}; + elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO ]; then + export IMAGE_NAMES="standalone_presto standalone_presto_admin" && + make test-images && + nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO}; + elif [ -v OTHER_TESTS ]; then + make clean lint dist docs test-rpm && + nosetests -s tests.unit && + nosetests -s tests.integration; + else + echo "Unknown test" && + exit 1; + fi diff --git a/Makefile b/Makefile index 5bf3d30e..a7c7ac28 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean-all clean clean-eggs clean-build clean-pyc clean-test-containers clean-test \ - clean-docs lint smoke test test-all test-rpm coverage docs open-docs release release-builds \ - dist dist-online dist-offline wheel install precommit + clean-docs lint smoke test test-all test-images test-rpm docker-images coverage docs \ + open-docs release release-builds dist dist-online dist-offline wheel install precommit help: @echo "precommit - run \`quick' tests and tasks that should pass or succeed prior to pushing" @@ -16,7 +16,9 @@ help: @echo "smoke - run tests annotated with attr smoke using nosetests" @echo "test - run tests quickly with Python 2.6 and 2.7" @echo "test-all - run tests on every Python version with tox. Specify TEST_SUITE env variable to run only a given suite." + @echo "test-images - create product test image(s). Specify IMAGE_NAMES env variable to create only certain images." @echo "test-rpm - run tests for the RPM package" + @echo "docker-images - pull docker image(s). Specify DOCKER_IMAGE_NAME env variable for specific image." @echo "coverage - check code coverage quickly with the default Python" @echo "docs - generate Sphinx HTML documentation, including API docs" @echo "open-docs - open the root document (index.html) using xdg-open" @@ -70,6 +72,9 @@ clean-docs: lint: flake8 prestoadmin packaging tests +presto-server-rpm.rpm: + wget 'https://repository.sonatype.org/service/local/artifact/maven/content?r=central-proxy&g=com.facebook.presto&a=presto-server-rpm&e=rpm&v=RELEASE' -O $@ + smoke: clean-test docker-images tox -e py26 -- -a smoketest,'!quarantine' @@ -84,11 +89,15 @@ test-all: clean-test docker-images tox -- -s tests.integration tox -e py26 -- -s ${TEST_SUITE} -a '!quarantine' +IMAGE_NAMES?="all" + +test-images: docker-images presto-server-rpm.rpm + python tests/product/image_builder.py ${IMAGE_NAMES} docker-images: docker pull teradatalabs/centos6-ssh-oj8 -test-rpm: clean-test +test-rpm: clean-test test-images tox -e py26 -- -s tests.rpm -a '!quarantine' coverage: diff --git a/tests/product/base_product_case.py b/tests/product/base_product_case.py index 8975432f..c71cfde4 100644 --- a/tests/product/base_product_case.py +++ b/tests/product/base_product_case.py @@ -26,15 +26,12 @@ from prestoadmin.util import constants from tests.base_test_case import BaseTestCase -from tests.configurable_cluster import ConfigurableCluster -from tests.docker_cluster import DockerCluster, DockerClusterException +from tests.docker_cluster import DockerCluster -from tests.product.mode_installers import StandaloneModeInstaller, \ - YarnSliderModeInstaller -from tests.product.prestoadmin_installer import PrestoadminInstaller +from tests.configurable_cluster import ConfigurableCluster +from tests.product.cluster_types import cluster_types from tests.product.standalone.presto_installer import StandalonePrestoInstaller -from tests.product.topology_installer import TopologyInstaller -from tests.product.yarn_slider.slider_installer import SliderInstaller + PRESTO_VERSION = r'.+' RETRY_TIMEOUT = 120 @@ -42,29 +39,6 @@ class BaseProductTestCase(BaseTestCase): - STANDALONE_BARE_CLUSTER = 'bare' - BARE_CLUSTER = 'bare' - PA_ONLY_CLUSTER = 'pa_only_standalone' - STANDALONE_PRESTO_CLUSTER = 'presto' - - PA_ONLY_YS_CLUSTER = 'pa_only_ys' - PA_SLIDER_CLUSTER = 'pa_slider' - - _cluster_types = { - BARE_CLUSTER: [], - PA_ONLY_CLUSTER: [PrestoadminInstaller, - StandaloneModeInstaller], - STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller, - StandaloneModeInstaller, - TopologyInstaller, - StandalonePrestoInstaller], - PA_ONLY_YS_CLUSTER: [PrestoadminInstaller, - YarnSliderModeInstaller], - PA_SLIDER_CLUSTER: [PrestoadminInstaller, - YarnSliderModeInstaller, - SliderInstaller] - } - default_workers_config_ = """coordinator=false discovery.uri=http://master:8080 http-server.http.port=8080 @@ -134,19 +108,11 @@ def setUp(self): self.cluster = None self.default_keywords = {} - def _run_installers(self, installers): - cluster = self.cluster - for installer in installers: - dependencies = installer.get_dependencies() - - for dependency in dependencies: - dependency.assert_installed(self) - - installer_instance = installer(self) - installer_instance.install() - - self.default_keywords.update(installer_instance.get_keywords()) - cluster.postinstall(installer) + def tearDown(self): + self.restore_stdout_stderr_keep_open() + if self.cluster: + self.cluster.tear_down() + super(BaseProductTestCase, self).tearDown() def _apply_post_install_hooks(self, installers): for installer in installers: @@ -158,12 +124,7 @@ def _update_replacement_keywords(self, installers): self.default_keywords.update(installer_instance.get_keywords()) def setup_cluster(self, bare_image_provider, cluster_type): - try: - installers = self._cluster_types[cluster_type] - except KeyError: - self.fail( - '%s is not a valid cluster type. Valid cluster types are %s' % - (cluster_type, ', '.join(self._cluster_types.keys()))) + installers = cluster_types[cluster_type] config_filename = ConfigurableCluster.check_for_cluster_config() @@ -172,42 +133,22 @@ def setup_cluster(self, bare_image_provider, cluster_type): config_filename, self, StandalonePrestoInstaller.assert_installed) else: - try: - self.cluster, bare_cluster = DockerCluster.start_cluster( - bare_image_provider, cluster_type) - - # If we've found images and started a non-bare cluster, the - # containers have already had the installers applied to them. - # We do need to get the test environment in sync with the - # containers by calling the following two functions. - # - # Once that's done, the cluster and test environment is in the - # same state it would be as if we'd called _run_installers on - # a bare cluster, and we can return. - # - # We do this to save the cost of running the installers on the - # docker containers every time we run a test. In practice, - # that turns out to be a fairly expensive thing to do. - if not bare_cluster: - self._apply_post_install_hooks(installers) - self._update_replacement_keywords(installers) - return - except DockerClusterException as e: - self.fail(e.msg) - - # If we got a bare cluster back, we need to run the installers on it. - # applying the post-install hooks and updating the replacement - # keywords is handled internally in _run_installers. - self._run_installers(installers) - - if isinstance(self.cluster, DockerCluster): - self.cluster.commit_images(bare_image_provider, cluster_type) - - def tearDown(self): - self.restore_stdout_stderr_keep_open() - if self.cluster: - self.cluster.tear_down() - super(BaseProductTestCase, self).tearDown() + self.cluster, bare_cluster = DockerCluster.start_cluster( + bare_image_provider, cluster_type) + + # If we've found images and started a non-bare cluster, the + # containers have already had the installers applied to them. + # We do need to get the test environment in sync with the + # containers by calling the following two functions. + # + # We do this to save the cost of running the installers on the + # docker containers every time we run a test. In practice, + # that turns out to be a fairly expensive thing to do. + if not bare_cluster: + self._apply_post_install_hooks(installers) + self._update_replacement_keywords(installers) + else: + raise RuntimeError("Docker images have not been created") def dump_and_cp_topology(self, topology, cluster=None): if not cluster: diff --git a/tests/product/cluster_types.py b/tests/product/cluster_types.py new file mode 100644 index 00000000..2bf2f862 --- /dev/null +++ b/tests/product/cluster_types.py @@ -0,0 +1,30 @@ +from tests.product.mode_installers import StandaloneModeInstaller, \ + YarnSliderModeInstaller +from tests.product.prestoadmin_installer import PrestoadminInstaller +from tests.product.topology_installer import TopologyInstaller +from tests.product.yarn_slider.slider_installer import SliderInstaller +from tests.product.standalone.presto_installer import StandalonePrestoInstaller + + +STANDALONE_BARE_CLUSTER = 'bare' +BARE_CLUSTER = 'bare' +STANDALONE_PA_CLUSTER = 'pa_only_standalone' +STANDALONE_PRESTO_CLUSTER = 'presto' + +YARN_SLIDER_PA_CLUSTER = 'pa_only_ys' +YARN_SLIDER_PA_AND_SLIDER_CLUSTER = 'pa_slider' + +cluster_types = { + BARE_CLUSTER: [], + STANDALONE_PA_CLUSTER: [PrestoadminInstaller, + StandaloneModeInstaller], + STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller, + StandaloneModeInstaller, + TopologyInstaller, + StandalonePrestoInstaller], + YARN_SLIDER_PA_CLUSTER: [PrestoadminInstaller, + YarnSliderModeInstaller], + YARN_SLIDER_PA_AND_SLIDER_CLUSTER: [PrestoadminInstaller, + YarnSliderModeInstaller, + SliderInstaller] +} diff --git a/tests/product/image_builder.py b/tests/product/image_builder.py new file mode 100644 index 00000000..138a6f58 --- /dev/null +++ b/tests/product/image_builder.py @@ -0,0 +1,113 @@ +from tests.docker_cluster import DockerCluster +from tests.hdp_bare_image_provider import HdpBareImageProvider +from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider + +from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_BARE_CLUSTER, STANDALONE_PA_CLUSTER, \ + STANDALONE_PRESTO_CLUSTER, YARN_SLIDER_PA_CLUSTER, cluster_types + +import argparse + + +class ImageBuilder: + def __init__(self, testcase): + self.testcase = testcase + self.testcase.default_keywords = {} + self.testcase.cluster = None + + def _run_installers(self, installers): + cluster = self.testcase.cluster + for installer in installers: + dependencies = installer.get_dependencies() + + for dependency in dependencies: + dependency.assert_installed(self.testcase) + + installer_instance = installer(self.testcase) + installer_instance.install() + + self.testcase.default_keywords.update(installer_instance.get_keywords()) + cluster.postinstall(installer) + + def _setup_image(self, bare_image_provider, cluster_type): + installers = cluster_types[cluster_type] + + self.testcase.cluster, bare_cluster = DockerCluster.start_cluster( + bare_image_provider, cluster_type) + + # If we got a bare cluster back, we need to run the installers on it. + # applying the post-install hooks and updating the replacement + # keywords is handled internally in _run_installers. + # + # If we got a non-bare cluster back, that means the image already exists + # and we created the cluster using that image. + if bare_cluster: + self._run_installers(installers) + + if isinstance(self.testcase.cluster, DockerCluster): + self.testcase.cluster.commit_images(bare_image_provider, cluster_type) + + self.testcase.cluster.tear_down() + + def _setup_image_with_no_hadoop_provider(self, cluster_type): + self._setup_image(NoHadoopBareImageProvider(), + cluster_type) + + def setup_standalone_presto_images(self): + cluster_type = STANDALONE_PRESTO_CLUSTER + self._setup_image_with_no_hadoop_provider(cluster_type) + + def setup_standalone_presto_admin_images(self): + cluster_type = STANDALONE_PA_CLUSTER + self._setup_image_with_no_hadoop_provider(cluster_type) + + def setup_standalone_bare_images(self): + cluster_type = STANDALONE_BARE_CLUSTER + self._setup_image_with_no_hadoop_provider(cluster_type) + + def setup_yarn_slider_presto_admin_images(self): + cluster_type = YARN_SLIDER_PA_CLUSTER + self._setup_image_with_no_hadoop_provider(cluster_type) + + def _setup_cluster_with_hdp_image_provider(self, cluster_type): + self._setup_image(HdpBareImageProvider(), + cluster_type) + + def setup_yarn_slider_presto_admin_and_slider_images(self): + cluster_type = self.testcase.YARN_SLIDER_PA_AND_SLIDER_CLUSTER + self._setup_cluster_with_hdp_image_provider(cluster_type) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("image_type", metavar="image_type", type=str, nargs="+", + choices=["standalone_presto", "standalone_presto_admin", + "standalone_bare", "yarn_slider_presto_admin", + "all"], + help="Specify the type of image to create. The available choices are: " + "standalone_presto, standalone_presto_admin, standalone_bare, " + "yarn_slider_presto_admin, all") + + # ImageBuilder needs an input testcase with access to unittest assertions + # so the installers can check their resulting installations as well as some + # product test helper functions. + # This supplies a dummy testcase. BaseProductTestCase inherits from + # unittest. A unittest instance can be successfully created if the name + # of an existing method of the class is passed into the constructor. + dummy_testcase = BaseProductTestCase('__init__') + image_builder = ImageBuilder(dummy_testcase) + + args = parser.parse_args() + if "all" in args.image_type: + image_builder.setup_standalone_presto_images() + image_builder.setup_standalone_presto_admin_images() + image_builder.setup_standalone_bare_images() + image_builder.setup_yarn_slider_presto_admin_images() + else: + if "standalone_presto" in args.image_type: + image_builder.setup_standalone_presto_images() + if "standalone_presto_admin" in args.image_type: + image_builder.setup_standalone_presto_admin_images() + if "standalone_bare" in args.image_type: + image_builder.setup_standalone_bare_images() + if "yarn_slider_presto_admin" in args.image_type: + image_builder.setup_yarn_slider_presto_admin_images() diff --git a/tests/product/standalone/presto_installer.py b/tests/product/standalone/presto_installer.py index a07f20f2..a30d2000 100644 --- a/tests/product/standalone/presto_installer.py +++ b/tests/product/standalone/presto_installer.py @@ -18,7 +18,6 @@ import fnmatch import os -import urllib import prestoadmin @@ -102,28 +101,10 @@ def copy_presto_rpm_to_master(self, cluster=None): cluster = self.testcase.cluster rpm_path = os.path.join(self.rpm_dir, self.rpm_name) - try: - cluster.copy_to_host(rpm_path, cluster.master) - self._check_if_corrupted_rpm(self.rpm_name, cluster) - except OSError: - print 'Downloading RPM again' - # try to download the RPM again if it's corrupt (but only once) - StandalonePrestoInstaller._download_rpm() - cluster.copy_to_host(rpm_path, cluster.master) - self._check_if_corrupted_rpm(self.rpm_name, cluster) + cluster.copy_to_host(rpm_path, cluster.master) + self._check_if_corrupted_rpm(self.rpm_name, cluster) return self.rpm_name - @staticmethod - def _download_rpm(): - rpm_filename = 'presto-server-rpm.rpm' - rpm_path = os.path.join(prestoadmin.main_dir, - rpm_filename) - urllib.urlretrieve( - 'https://repository.sonatype.org/service/local/artifact/maven' - '/content?r=central-proxy&g=com.facebook.presto' - '&a=presto-server-rpm&e=rpm&v=RELEASE', rpm_path) - return rpm_filename - @staticmethod def _detect_presto_rpm(): """ @@ -137,11 +118,7 @@ def _detect_presto_rpm(): # are multiple RPMs, the last one is probably the latest rpm_name = sorted(rpm_names)[-1] else: - try: - rpm_name = StandalonePrestoInstaller._download_rpm() - except: - # retry once - rpm_name = StandalonePrestoInstaller._download_rpm() + raise OSError(1, 'Presto RPM not detected.') return prestoadmin.main_dir, rpm_name diff --git a/tests/product/standalone/test_installation.py b/tests/product/standalone/test_installation.py index d55d3f69..877e975e 100644 --- a/tests/product/standalone/test_installation.py +++ b/tests/product/standalone/test_installation.py @@ -22,6 +22,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.cluster_types import STANDALONE_BARE_CLUSTER from tests.product.prestoadmin_installer import PrestoadminInstaller from tests.docker_cluster import DockerCluster from tests.product.constants import DEFAULT_DOCKER_MOUNT_POINT, \ @@ -44,7 +45,7 @@ class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) - self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) diff --git a/tests/product/test_authentication.py b/tests/product/test_authentication.py index d652018f..af4537bb 100644 --- a/tests/product/test_authentication.py +++ b/tests/product/test_authentication.py @@ -23,14 +23,14 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from constants import LOCAL_RESOURCES_DIR class TestAuthentication(BaseProductTestCase): def setUp(self): super(TestAuthentication, self).setUp() - self.setup_cluster( - NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) success_output = ( 'Deploying tpch.properties connector configurations on: slave1 \n' diff --git a/tests/product/test_collect.py b/tests/product/test_collect.py index 963d62e5..31c8276d 100644 --- a/tests/product/test_collect.py +++ b/tests/product/test_collect.py @@ -27,6 +27,7 @@ from prestoadmin.server import run_sql from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, PrestoError +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller @@ -36,8 +37,7 @@ def setUp(self): @attr('smoketest') def test_collect_logs_basic(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect logs') @@ -69,8 +69,7 @@ def log_msg(self, actual, expected): @attr('smoketest') def test_collect_system_info_basic(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info') self.test_basic_system_info(actual) @@ -113,7 +112,7 @@ def test_basic_system_info(self, actual, coordinator=None, hosts=None): def test_collect_system_info_dash_h_coord_worker(self): self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info ' '-H %(master)s,%(slave1)s') @@ -124,7 +123,7 @@ def test_collect_system_info_dash_h_coord_worker(self): def test_collect_system_info_dash_x_two_workers(self): self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info ' '-x %(slave2)s,%(slave3)s') @@ -135,7 +134,7 @@ def test_collect_system_info_dash_x_two_workers(self): def test_system_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) @@ -149,8 +148,7 @@ def test_system_info_pa_separate_node(self): @attr('smoketest') def test_collect_query_info(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') sql_to_run = 'SELECT * FROM system.runtime.nodes WHERE 1234 = 1234' query_id = self.retry(lambda: self.get_query_id(sql_to_run)) @@ -169,7 +167,7 @@ def test_collect_query_info(self): def test_query_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) @@ -207,8 +205,7 @@ def get_query_id(self, sql, host=None): return row[0] def test_query_info_invalid_id(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') invalid_id = '1234_invalid' actual = self.run_prestoadmin('collect query_info ' + invalid_id, @@ -220,13 +217,11 @@ def test_query_info_invalid_id(self): self.assertEqual(actual, expected) def test_collect_logs_server_stopped(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self._assert_no_logs_downloaded() def test_collect_system_info_server_stopped(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) actual = self.run_prestoadmin('collect system_info', raise_error=False) message = '\nFatal error: [%s] Unable to access node ' \ 'information. Please check that server is up with ' \ @@ -257,8 +252,7 @@ def _collect_logs_and_unzip(self): 'cp %s .; tar xvf %s' % (OUTPUT_FILENAME_FOR_LOGS, log_filename)) def test_collect_logs_nonstandard_location(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) version = self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm' @@ -295,14 +289,12 @@ def _assert_no_logs_downloaded(self): '/opt/prestoadmin/logs/%s/*' % host) def test_collect_logs_server_not_installed(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self._assert_no_logs_downloaded() def test_collect_logs_multiple_server_logs(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') self.cluster.write_content_to_host('Stuff that I logged!', 'var/log/presto/server.log-2', diff --git a/tests/product/test_configuration.py b/tests/product/test_configuration.py index da9a8e1d..39134b52 100644 --- a/tests/product/test_configuration.py +++ b/tests/product/test_configuration.py @@ -24,6 +24,7 @@ from prestoadmin.util import constants from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from tests.product.constants import LOCAL_RESOURCES_DIR @@ -31,8 +32,7 @@ class TestConfiguration(BaseProductTestCase): def setUp(self): super(TestConfiguration, self).setUp() - self.setup_cluster( - NoHadoopBareImageProvider(), self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.write_test_configs(self.cluster) def deploy_and_assert_default_config(self): diff --git a/tests/product/test_connectors.py b/tests/product/test_connectors.py index 81cd628f..59d1b68d 100644 --- a/tests/product/test_connectors.py +++ b/tests/product/test_connectors.py @@ -25,14 +25,17 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, \ docker_only, PrestoError +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.constants import LOCAL_RESOURCES_DIR from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestConnectors(BaseProductTestCase): + def setUp(self): + super(TestConnectors, self).setUp() + def setup_cluster_assert_connectors(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) @@ -117,8 +120,7 @@ def test_connector_add_remove_coord_worker_using_dash_x(self): @docker_only def test_connector_add_wrong_permissions(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) # test add connector without read permissions on file script = 'chmod 600 /etc/opt/prestoadmin/connectors/tpch.properties;' \ @@ -153,8 +155,7 @@ def test_connector_add_wrong_permissions(self): self.run_script_from_prestoadmin_dir, script) def test_connector_add_missing_connector(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) # test add a connector that does not exist not_found_error = self.fatal_error( @@ -164,8 +165,8 @@ def test_connector_add_missing_connector(self): self.run_prestoadmin, 'connector add tpch') def test_connector_add_no_dir(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) + # test add all connectors when the directory does not exist self.cluster.exec_cmd_on_host( self.cluster.master, @@ -178,8 +179,7 @@ def test_connector_add_no_dir(self): self.run_prestoadmin, 'connector add') def test_connector_add_by_name(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('connector remove tpch') # test add connector by name when it exists @@ -195,8 +195,7 @@ def test_connector_add_by_name(self): self._assert_connectors_loaded([['system'], ['tpch']]) def test_connector_add_empty_dir(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) output = self.run_prestoadmin('connector remove tpch') output = self.run_prestoadmin('connector add') expected = """ @@ -219,8 +218,7 @@ def test_connector_add_empty_dir(self): self.assertEqualIgnoringOrder(expected, output) def test_connector_add_two_connectors(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('connector remove tpch') # test add connectors from directory with more than one connector @@ -257,7 +255,7 @@ def fatal_error(self, error): def test_connector_add_lost_host(self): installer = StandalonePrestoInstaller(self) - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('connector remove tpch') @@ -292,8 +290,7 @@ def test_connector_add_lost_host(self): self._assert_connectors_loaded([['system'], ['tpch']]) def test_connector_remove(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.assert_has_default_connector(host) @@ -357,8 +354,7 @@ def test_connector_remove(self): 'connector remove tpch') def test_connector_name_not_found(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') self.cluster.write_content_to_host( @@ -374,8 +370,7 @@ def test_connector_name_not_found(self): 'connector add example') def test_connector_add_no_presto_user(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( diff --git a/tests/product/test_control.py b/tests/product/test_control.py index 5228a7a9..22dabf33 100644 --- a/tests/product/test_control.py +++ b/tests/product/test_control.py @@ -20,22 +20,23 @@ from prestoadmin.server import RETRY_TIMEOUT from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestControl(BaseProductTestCase): + def setUp(self): + super(TestControl, self).setUp() @attr('smoketest') def test_server_start_stop_simple(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_simple_start_stop(self.expected_start(), self.expected_stop()) @attr('smoketest') def test_server_restart_simple(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) expected_output = self.expected_stop()[:] + self.expected_start()[:] self.assert_simple_server_restart(expected_output) @@ -49,7 +50,7 @@ def test_server_restart_without_presto(self): self.assert_service_fails_without_presto('restart') def assert_service_fails_without_presto(self, service): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() # Start without Presto installed start_output = self.run_prestoadmin('server %s' % service, @@ -59,14 +60,12 @@ def assert_service_fails_without_presto(self, service): '\n'.join(start_output)) def test_server_start_one_host_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_start_with_one_host_started( self.cluster.internal_slaves[0]) def test_server_start_all_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start').splitlines() process_per_host = self.get_process_per_host(start_output) self.assert_started(process_per_host) @@ -79,8 +78,7 @@ def test_server_start_all_started(self): ) def test_server_stop_not_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) stop_output = self.run_prestoadmin('server stop').splitlines() not_started_hosts = self.cluster.all_internal_hosts() @@ -90,18 +88,16 @@ def test_server_stop_not_started(self): ) def test_server_stop_coordinator_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_one_host_stopped(self.cluster.internal_master) def test_server_stop_worker_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_one_host_stopped(self.cluster.internal_slaves[0]) def test_server_restart_nothing_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) + # Restart when the servers aren't started expected_output = self.expected_stop( not_running=self.cluster.all_internal_hosts())[:] +\ @@ -109,8 +105,8 @@ def test_server_restart_nothing_started(self): self.assert_simple_server_restart(expected_output, running_host='') def test_server_restart_coordinator_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) + # Restart when a coordinator is started but workers aren't not_running_hosts = self.cluster.all_internal_hosts()[:] not_running_hosts.remove(self.cluster.internal_master) @@ -120,8 +116,8 @@ def test_server_restart_coordinator_started(self): expected_output, running_host=self.cluster.internal_master) def test_server_restart_worker_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) + # Restart when one worker is started, but nothing else not_running_hosts = self.cluster.all_internal_hosts()[:] not_running_hosts.remove(self.cluster.internal_slaves[0]) @@ -133,7 +129,7 @@ def test_server_restart_worker_started(self): def test_start_stop_restart_coordinator_down(self): installer = StandalonePrestoInstaller(self) - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) @@ -144,7 +140,7 @@ def test_start_stop_restart_coordinator_down(self): def test_start_stop_restart_worker_down(self): installer = StandalonePrestoInstaller(self) - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) @@ -154,8 +150,7 @@ def test_start_stop_restart_worker_down(self): self.cluster.internal_slaves[0]) def test_server_start_twice(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start').splitlines() process_per_host = self.get_process_per_host(start_output) self.assert_started(process_per_host) @@ -236,8 +231,7 @@ def expected_down_node_output_size(self, expected_output): '\n'.join(expected_output).splitlines()) def test_start_restart_config_file_error(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) # Remove a required config file so that the server can't start self.cluster.exec_cmd_on_host( diff --git a/tests/product/test_error_handling.py b/tests/product/test_error_handling.py index 6ef6db41..f0e84c68 100644 --- a/tests/product/test_error_handling.py +++ b/tests/product/test_error_handling.py @@ -15,16 +15,16 @@ """ System tests for error handling in presto-admin """ - from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_PA_CLUSTER class TestErrorHandling(BaseProductTestCase): def setUp(self): super(TestErrorHandling, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() def test_wrong_arguments_parallel(self): diff --git a/tests/product/test_installer.py b/tests/product/test_installer.py index 37255e8d..1576346c 100644 --- a/tests/product/test_installer.py +++ b/tests/product/test_installer.py @@ -25,6 +25,7 @@ from tests.docker_cluster import DockerCluster from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_BARE_CLUSTER from tests.product.prestoadmin_installer import PrestoadminInstaller @@ -32,7 +33,7 @@ class TestInstaller(BaseProductTestCase): def setUp(self): super(TestInstaller, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self) diff --git a/tests/product/test_package_install.py b/tests/product/test_package_install.py index 6adea58b..f8785fe3 100644 --- a/tests/product/test_package_install.py +++ b/tests/product/test_package_install.py @@ -15,14 +15,16 @@ from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider -from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.base_product_case import BaseProductTestCase, \ + docker_only +from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) diff --git a/tests/product/test_plugin.py b/tests/product/test_plugin.py index 6649efc6..2bd8836c 100644 --- a/tests/product/test_plugin.py +++ b/tests/product/test_plugin.py @@ -18,6 +18,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_PA_CLUSTER TMP_JAR_PATH = '/opt/prestoadmin/pretend.jar' STD_REMOTE_PATH = '/usr/lib/presto/lib/plugin/hive-cdh5/pretend.jar' @@ -26,7 +27,7 @@ class TestPlugin(BaseProductTestCase): def setUp(self): super(TestPlugin, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) def deploy_jar_to_master(self): self.cluster.write_content_to_host('A PRETEND JAR', TMP_JAR_PATH, diff --git a/tests/product/test_script.py b/tests/product/test_script.py index 2e359b78..a0a64c10 100644 --- a/tests/product/test_script.py +++ b/tests/product/test_script.py @@ -19,12 +19,13 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_PA_CLUSTER class TestScript(BaseProductTestCase): def setUp(self): super(TestScript, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() @attr('smoketest') diff --git a/tests/product/test_server_install.py b/tests/product/test_server_install.py index f04ff004..fb881fc2 100644 --- a/tests/product/test_server_install.py +++ b/tests/product/test_server_install.py @@ -17,8 +17,10 @@ from nose.plugins.attrib import attr from prestoadmin.util import constants + from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller from tests.product.constants import LOCAL_RESOURCES_DIR, JAVA_VERSION_DIR, \ DEFAULT_JAVA_DIR @@ -146,7 +148,7 @@ class TestServerInstall(BaseProductTestCase): def setUp(self): super(TestServerInstall, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) def assert_common_configs(self, container): installer = StandalonePrestoInstaller(self) diff --git a/tests/product/test_server_uninstall.py b/tests/product/test_server_uninstall.py index 651cba59..d9dfb13c 100644 --- a/tests/product/test_server_uninstall.py +++ b/tests/product/test_server_uninstall.py @@ -18,6 +18,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.constants import LOCAL_RESOURCES_DIR from tests.product.standalone.presto_installer import StandalonePrestoInstaller from tests.product.prestoadmin_installer import PrestoadminInstaller @@ -36,8 +37,7 @@ def setUp(self): @attr('smoketest') def test_uninstall(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) @@ -60,8 +60,7 @@ def assert_uninstalled_dirs_removed(self, container): self.assert_path_removed(container, '/etc/init.d/presto') def test_uninstall_when_server_down(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) @@ -87,7 +86,7 @@ def test_uninstall_twice(self): self.assertEqualIgnoringOrder(expected, output) def test_uninstall_lost_host(self): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) pa_installer = PrestoadminInstaller(self) pa_installer.install() topology = {"coordinator": self.cluster.internal_slaves[0], @@ -120,8 +119,7 @@ def test_uninstall_lost_host(self): self.assert_uninstalled_dirs_removed(container) def test_uninstall_with_dir_readonly(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) @@ -149,7 +147,7 @@ def test_uninstall_with_dir_readonly(self): @docker_only def test_uninstall_as_non_sudo(self): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self.installer.install() diff --git a/tests/product/test_server_upgrade.py b/tests/product/test_server_upgrade.py index 79450377..40a6fec6 100644 --- a/tests/product/test_server_upgrade.py +++ b/tests/product/test_server_upgrade.py @@ -20,6 +20,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller @@ -27,8 +28,7 @@ class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) diff --git a/tests/product/test_status.py b/tests/product/test_status.py index 2b04ab5b..68bda45e 100644 --- a/tests/product/test_status.py +++ b/tests/product/test_status.py @@ -21,6 +21,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, \ PRESTO_VERSION, PrestoError +from tests.product.cluster_types import STANDALONE_PA_CLUSTER, STANDALONE_PRESTO_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller @@ -31,28 +32,25 @@ def setUp(self): self.installer = StandalonePrestoInstaller(self) def test_status_uninstalled(self): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() status_output = self._server_status_with_retries() self.check_status(status_output, self.not_installed_status()) def test_status_not_started(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) status_output = self._server_status_with_retries() self.check_status(status_output, self.not_started_status()) @attr('smoketest') def test_status_happy_path(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') status_output = self._server_status_with_retries(check_connectors=True) self.check_status(status_output, self.base_status()) def test_status_only_coordinator(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start -H master') # don't run with retries because it won't be able to query the @@ -64,8 +62,7 @@ def test_status_only_coordinator(self): ) def test_status_only_worker(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start -H slave1') status_output = self._server_status_with_retries() @@ -81,7 +78,7 @@ def test_status_only_worker(self): self.check_status(status_output, self.not_started_status()) def test_connection_to_coordinator_lost(self): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) @@ -98,7 +95,7 @@ def test_connection_to_coordinator_lost(self): self.check_status(status_output, statuses) def test_connection_to_worker_lost(self): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) @@ -115,7 +112,7 @@ def test_connection_to_worker_lost(self): self.check_status(status_output, statuses) def test_status_port_not_8080(self): - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() port_config = """discovery.uri=http://master:8090 diff --git a/tests/product/test_topology.py b/tests/product/test_topology.py index 5ff12a62..e50f5c56 100644 --- a/tests/product/test_topology.py +++ b/tests/product/test_topology.py @@ -18,6 +18,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.constants import LOCAL_RESOURCES_DIR @@ -48,7 +49,7 @@ class TestTopologyShow(BaseProductTestCase): def setUp(self): super(TestTopologyShow, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) @attr('smoketest') def test_topology_show(self): diff --git a/tests/product/timing_test_decorator.py b/tests/product/timing_test_decorator.py new file mode 100644 index 00000000..1b553ae0 --- /dev/null +++ b/tests/product/timing_test_decorator.py @@ -0,0 +1,44 @@ +import logging +import sys + +from time import time + + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +message_format = '%(levelname)s - %(message)s' +formatter = logging.Formatter(message_format) +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setLevel(logging.INFO) +console_handler.setFormatter(formatter) +logger.addHandler(console_handler) + + +def log_function_time(): + """ + Returns: Prints the execution time of the decorated function to the + console. If the execution time exceeds 10 minutes, it will use 'error' + for the message level. Otherwise, it will use 'info'. + """ + def name_wrapper(function): + def time_wrapper(*args, **kwargs): + global logger + function_name = function.__name__ + + start_time = time() + return_value = function(*args, **kwargs) + elapsed_time = time() - start_time + + travis_output_time_limit = 600 + message_level = logging.ERROR if elapsed_time >= travis_output_time_limit \ + else logging.INFO + logging.disable(logging.NOTSET) + logger.log(message_level, + "%s completed in %s seconds...", + function_name, + str(elapsed_time)) + logging.disable(logging.CRITICAL) + + return return_value + return time_wrapper + return name_wrapper diff --git a/tests/product/yarn_slider/test_server_install.py b/tests/product/yarn_slider/test_server_install.py index 51afbd90..0abd0e5e 100644 --- a/tests/product/yarn_slider/test_server_install.py +++ b/tests/product/yarn_slider/test_server_install.py @@ -24,8 +24,9 @@ from prestoadmin.yarn_slider.config import SLIDER_CONFIG_PATH, PRESTO_PACKAGE -from tests.product.base_product_case import BaseProductTestCase from tests.hdp_bare_image_provider import HdpBareImageProvider +from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import YARN_SLIDER_PA_AND_SLIDER_CLUSTER from tests.product.yarn_slider.slider_presto_installer import \ SliderPrestoInstaller @@ -36,7 +37,7 @@ class TestServerInstall(BaseProductTestCase): def setUp(self): super(TestServerInstall, self).setUp() - self.setup_cluster(HdpBareImageProvider(), self.PA_SLIDER_CLUSTER) + self.setup_cluster(HdpBareImageProvider(), YARN_SLIDER_PA_AND_SLIDER_CLUSTER) self.installer = SliderPrestoInstaller(self) self.await_hdfs() diff --git a/tests/product/yarn_slider/test_slider_installation.py b/tests/product/yarn_slider/test_slider_installation.py index e7baf8f6..75573952 100644 --- a/tests/product/yarn_slider/test_slider_installation.py +++ b/tests/product/yarn_slider/test_slider_installation.py @@ -20,6 +20,7 @@ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase +from tests.product.cluster_types import YARN_SLIDER_PA_CLUSTER from tests.product.yarn_slider.slider_installer import SliderInstaller from tests.product.yarn_slider.pa_slider_config import cluster_config, \ docker_config, get_config, upload_config @@ -31,8 +32,7 @@ class TestSliderInstallation(BaseProductTestCase): def setUp(self): super(TestSliderInstallation, self).setUp() - self.setup_cluster( - NoHadoopBareImageProvider(), self.PA_ONLY_YS_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), YARN_SLIDER_PA_CLUSTER) def _get_interactive_config(self, conf): cluster_conf = cluster_config(conf) diff --git a/tests/rpm/test_rpm.py b/tests/rpm/test_rpm.py index a02332b1..e63a398f 100644 --- a/tests/rpm/test_rpm.py +++ b/tests/rpm/test_rpm.py @@ -19,15 +19,15 @@ import os from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only +from tests.product.cluster_types import STANDALONE_PA_CLUSTER, STANDALONE_PRESTO_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller from tests.product.test_server_install import relocate_default_java class TestRpm(BaseProductTestCase): def setUp(self): - super(TestRpm, self).setUp() - self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) @docker_only def test_install_fails_java8_not_found(self): @@ -71,15 +71,14 @@ def test_server_starts_java8_in_bin_java(self): @docker_only def test_server_starts_no_java8_variable(self): - self.setup_cluster(NoHadoopBareImageProvider(), - self.STANDALONE_PRESTO_CLUSTER) + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_script_from_prestoadmin_dir('rm /etc/presto/env.sh') # tests that no error is encountered self.run_prestoadmin('server start') @docker_only def test_started_with_presto_user(self): - self.setup_cluster(NoHadoopBareImageProvider(), 'presto') + self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start').splitlines() process_per_host = self.get_process_per_host(start_output)