Build the Docker images in the Makefile

We had an infrastructure failure where the yum repo wasn't available. This
meant that the docker build process for the base image for every test failed.
Unfortunately, we check for the base image and attempt to build it once per
testcase rather than once per test run, so our tests took forever to run and
had an error for every test. This commit moves building the base image to the
Makefile so that if we hit this issue in the future, we'll fail fast.

Author: Eric Diven

Makefile

@@ -70,7 +70,7 @@ clean-docs:
 lint:
 	flake8 prestoadmin packaging tests
 
-smoke: clean-test
+smoke: clean-test docker-images
 	tox -e py26 -- -a smoketest,'!quarantine'
 
 test: clean-test
@@ -79,11 +79,16 @@ test: clean-test
 
 TEST_SUITE?=tests.product
 
-test-all: clean-test
+test-all: clean-test docker-images
 	tox -- -s tests.unit
 	tox -- -s tests.integration
 	tox -e py26 -- -s ${TEST_SUITE} -a '!quarantine'
 
+
+DOCKERFILE_DIR=tests/product/resources/centos6-ssh-test
+docker-images:
+	docker build --rm=true --tag teradatalabs/centos-ssh-test $(DOCKERFILE_DIR)
+
 test-rpm: clean-test
 	tox -e py26 -- -s tests.rpm -a '!quarantine'
 

tests/bare_image_provider.py

@@ -21,18 +21,49 @@
 
 import abc
 
+from docker import Client
+
 
 class BareImageProvider(object):
     __metaclass__ = abc.ABCMeta
 
+    def __init__(self, tag_decoration):
+        super(BareImageProvider, self).__init__()
+        self.tag_decoration = tag_decoration
+
     @abc.abstractmethod
     def create_bare_images(self, cluster, master_name, slave_name):
         """Create master and slave images to be tagged with master_name and
         slave_name, respectively."""
         pass
 
-    @abc.abstractmethod
     def get_tag_decoration(self):
         """Returns a string that's prepended to docker image tags for images
         based off of the bare image created by the provider."""
-        pass
+        return self.tag_decoration
+
+
+"""
+Provides bare images from existing tags in Docker. For some of the heftier
+images, we don't want to go through a long and drawn-out Docker build on a
+regular basis. For these, we count on having an image in Docker that we can
+tag appropriately into the teradatalabs/pa_tests namespace. Test cleanup can
+continue to obliterate that namespace without disrupting the actual heavyweight
+images.
+
+As an additional benefit, this means we can have tests depend on images that
+the test code doesn't know how to build. That seems like a liability, but it
+that the build process for complex images can be versioned outside of the
+presto-admin codebase.
+"""
+
+class TagBareImageProvider(BareImageProvider):
+    def __init__(self, base_master_name, base_slave_name, tag_decoration):
+        super(TagBareImageProvider, self).__init__(tag_decoration)
+        self.base_master_name = base_master_name
+        self.base_slave_name = base_slave_name
+        self.client = Client()
+
+    def create_bare_images(self, cluster, master_name, slave_name):
+        self.client.tag(self.base_master_name, master_name)
+        self.client.tag(self.base_slave_name, slave_name)

tests/docker_cluster.py

@@ -392,18 +392,21 @@ def start_existing_images(bare_image_provider, cluster_type):
         return centos_cluster
 
     @staticmethod
-    def _check_for_images(master_image_name, slave_image_name):
+    def _check_for_images(master_image_name, slave_image_name, tag='latest'):
+        master_repotag = '%s:%s' % (master_image_name, tag)
+        slave_repotag = '%s:%s' % (slave_image_name, tag)
         with Client(timeout=180) as client:
             images = client.images()
         has_master_image = False
         has_slave_image = False
         for image in images:
-            if master_image_name in image['RepoTags'][0]:
+            if master_repotag in image['RepoTags']:
                 has_master_image = True
-            if slave_image_name in image['RepoTags'][0]:
+            if slave_repotag in image['RepoTags']:
                 has_slave_image = True
         return has_master_image and has_slave_image
 
+
     def commit_images(self, bare_image_provider, cluster_type):
         self.client.commit(self.master,
                            self._get_master_image_name(bare_image_provider,

tests/hdp_bare_image_provider.py

@@ -17,7 +17,7 @@
 that hadoop is actually installed.
 """
 
-from tests.tag_bare_image_provider import TagBareImageProvider
+from tests.bare_image_provider import TagBareImageProvider
 
 
 class HdpBareImageProvider(TagBareImageProvider):

tests/no_hadoop_bare_image_provider.py

@@ -16,30 +16,11 @@
 Provides bare images for standalone clusters.
 """
 
-from tests.bare_image_provider import BareImageProvider
+from tests.bare_image_provider import TagBareImageProvider
 
-from tests.product.constants import \
-    BASE_IMAGE_NAME, BASE_IMAGE_TAG, BASE_TD_DOCKERFILE_DIR
 
-
-class NoHadoopBareImageProvider(BareImageProvider):
+class NoHadoopBareImageProvider(TagBareImageProvider):
     def __init__(self):
-        super(NoHadoopBareImageProvider, self).__init__()
-
-    def create_bare_images(self, cluster, master_name, slave_name):
-        cluster.create_image(
-            BASE_TD_DOCKERFILE_DIR,
-            master_name,
-            BASE_IMAGE_NAME,
-            BASE_IMAGE_TAG
-        )
-
-        cluster.create_image(
-            BASE_TD_DOCKERFILE_DIR,
-            slave_name,
-            BASE_IMAGE_NAME,
-            BASE_IMAGE_TAG
-        )
-
-    def get_tag_decoration(self):
-        return 'nohadoop'
+        super(NoHadoopBareImageProvider, self).__init__(
+            'teradatalabs/centos-ssh-test', 'teradatalabs/centos-ssh-test',
+            'nohadoop')