Extract building dist package out of prestoadmin_installer

Build binary files seems to be a better task for the build tool (like make) than
for the product tests framework. Moreover product tests framework was
caching the result in the `tmp/installer` directory so when the changes
were introduced to presto-admin, package was not rebuild. So it was easy
to run product tests with the old version of product-tests.

Instead of that building distribution package was moved to Makefile.
Moreover, disregarding how this file is created it will be stored in
`dist` directory. No more need for having `tmp/installer`.

Fixes prestodb#189

Author: kokosing

Makefile

@@ -1,6 +1,7 @@
 .PHONY: clean-all clean clean-eggs clean-build clean-pyc clean-test-containers clean-test \
 	clean-docs lint smoke test test-all test-rpm coverage docs open-docs release release-builds \
-	dist dist-online dist-offline wheel install precommit
+	dist dist-online dist-offline wheel install precommit \
+	docker-dist docker-dist-online docker-dist-offline
 
 help:
 	@echo "precommit - run \`quick' tests and tasks that should pass or succeed prior to pushing"
@@ -25,6 +26,9 @@ help:
 	@echo "dist - package and build installer that requires an Internet connection"
 	@echo "dist-online - package and build installer that requires an Internet connection"
 	@echo "dist-offline - package and build installer that does not require an Internet connection"
+	@echo "docker-dist - package and build installer (within a docker container) that requires an Internet connection"
+	@echo "docker-dist-online - package and build installer (within a docker container) that requires an Internet connection"
+	@echo "docker-dist-offline - package and build installer (within a docker container) that does not require an Internet connection"
 	@echo "wheel - build wheel only"
 	@echo "install - install the package to the active Python's site-packages"
 
@@ -121,6 +125,14 @@ dist-offline: clean-build clean-pyc
 	python setup.py bdist_prestoadmin
 	ls -l dist
 
+docker-dist: docker-dist-online
+
+docker-dist-online: clean-build clean-pyc
+	docker run --rm -v `pwd`:/workdir -w /workdir -u `id -u`:`id -g` teradatalabs/presto-admin-devenv make dist-online
+
+docker-dist-offline: clean-build clean-pyc
+	docker run --rm -v `pwd`:/workdir -w /workdir -u `id -u`:`id -g` teradatalabs/presto-admin-devenv make dist-offline
+
 wheel: clean
 	python setup.py bdist_wheel
 	ls -l dist

tests/docker_cluster.py

@@ -36,7 +36,7 @@
     DEFAULT_DOCKER_MOUNT_POINT, DEFAULT_LOCAL_MOUNT_POINT, \
     BASE_TD_IMAGE_NAME
 
-DIST_DIR = os.path.join(main_dir, 'tmp/installer')
+DIST_DIR = os.path.join(main_dir, 'dist')
 
 
 class DockerCluster(BaseCluster):

tests/product/prestoadmin_installer.py

@@ -16,18 +16,11 @@
 Module for installing prestoadmin on a cluster.
 """
 
-import errno
 import fnmatch
-import shutil
 import os
 
-import prestoadmin
-
 from tests.base_installer import BaseInstaller
 from tests.product.constants import LOCAL_RESOURCES_DIR
-from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider
-
-from tests.docker_cluster import DockerCluster
 
 
 class PrestoadminInstaller(BaseInstaller):
@@ -39,17 +32,18 @@ def get_dependencies():
         return []
 
     def install(self, cluster=None, dist_dir=None):
-        # Passing in a cluster supports the installation tests. We need to be
-        # able to try an installation against an unsupported OS, and for that
-        # testcase, we create a cluster that is local to the testcase and then
-        # run the install on it. We can't replace self.cluster with the local
-        # cluster in the test, because that would prevent the test's "regular"
-        # cluster from getting torn down.
         if not cluster:
             cluster = self.testcase.cluster
 
         if not dist_dir:
-            dist_dir = self._build_dist_if_necessary(cluster)
+            dist_dir = cluster.get_dist_dir(unique=False)
+
+        if (not os.path.isdir(dist_dir) or
+                    len(fnmatch.filter(os.listdir(dist_dir), 'prestoadmin-*.tar.bz2')) == 0):
+            self.testcase.fail(
+                'Unable to find presto-admin package. '
+                'Have you run one of `make dist*` or `make docker-dist*` command?')
+
         self._copy_dist_to_host(cluster, dist_dir, cluster.master)
         cluster.copy_to_host(
             LOCAL_RESOURCES_DIR + "/install-admin.sh", cluster.master)
@@ -69,80 +63,6 @@ def assert_installed(testcase, msg=None):
     def get_keywords(self):
         return {}
 
-    def _build_dist_if_necessary(self, cluster, unique=False):
-        if (not os.path.isdir(cluster.get_dist_dir(unique)) or
-                not fnmatch.filter(
-                    os.listdir(cluster.get_dist_dir(unique)),
-                    'prestoadmin-*.tar.bz2')):
-            self._build_installer_in_docker(cluster, unique=unique)
-        return cluster.get_dist_dir(unique)
-
-    def _build_installer_in_docker(self, cluster, online_installer=None,
-                                   unique=False):
-        if online_installer is None:
-            paTestOnlineInstaller = os.environ.get('PA_TEST_ONLINE_INSTALLER')
-            online_installer = paTestOnlineInstaller is not None
-
-        container_name = 'installer'
-        cluster_type = 'installer_builder'
-        bare_image_provider = NoHadoopBareImageProvider()
-
-        installer_container, created_bare = DockerCluster.start_cluster(
-            bare_image_provider, cluster_type, 'installer', [])
-
-        if created_bare:
-            installer_container.commit_images(
-                bare_image_provider, cluster_type)
-
-        try:
-            shutil.copytree(
-                prestoadmin.main_dir,
-                os.path.join(
-                    installer_container.get_local_mount_dir(container_name),
-                    'presto-admin'),
-                ignore=shutil.ignore_patterns('tmp', '.git', 'presto*.rpm')
-            )
-
-            # Pin pip to 7.1.2 because 8.0.0 removed support for distutils
-            # installed projects, of which the system setuptools is one on our
-            # Docker image. pip 8.0.1 or 8.0.2 replaced the error with a
-            # deprecation warning, and also warns that Python 2.6 is
-            # deprecated. While we still need to support Python 2.6, we'll pin
-            # pip to a 7.x version, but we should revisit this once we no
-            # longer need to support 2.6:
-            # https://github.com/pypa/pip/issues/3384
-            installer_container.run_script_on_host(
-                'set -e\n'
-                'pip install --upgrade pip==7.1.2\n'
-                'pip install --upgrade wheel==0.23.0\n'
-                'pip install --upgrade setuptools==20.1.1\n'
-                'mv %s/presto-admin ~/\n'
-                'cd ~/presto-admin\n'
-                'make %s\n'
-                'cp dist/prestoadmin-*.tar.bz2 %s'
-                % (installer_container.mount_dir,
-                   'dist' if online_installer else 'dist-offline',
-                   installer_container.mount_dir),
-                container_name)
-
-            try:
-                os.makedirs(cluster.get_dist_dir(unique))
-            except OSError, e:
-                if e.errno != errno.EEXIST:
-                    raise
-            local_container_dist_dir = os.path.join(
-                prestoadmin.main_dir,
-                installer_container.get_local_mount_dir(container_name)
-            )
-            installer_file = fnmatch.filter(
-                os.listdir(local_container_dist_dir),
-                'prestoadmin-*.tar.bz2')[0]
-            shutil.copy(
-                os.path.join(local_container_dist_dir, installer_file),
-                cluster.get_dist_dir(unique))
-        finally:
-            installer_container.tear_down()
-
     @staticmethod
     def _copy_dist_to_host(cluster, local_dist_dir, dest_host):
         for dist_file in os.listdir(local_dist_dir):