Skip to content

Commit 9557f8d

Browse files
zachyeezachyee
committed
Refactor product tests to build images before test
To fix travis test timeout issue
1 parent 1d4ee79 commit 9557f8d

25 files changed

+323
-211
lines changed

‎.travis.yml

+43-5
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,50 @@ sudo: required
66
services:
77
- docker
88
env:
9-
install:
9+
global:
10+
- PA_TEST_ONLINE_INSTALLER=true
11+
- PYTHONPATH=$(pwd)
12+
- LONG_PRODUCT_TESTS="tests/product/test_server_install.py tests/product/test_status.py tests/product/test_collect.py tests/product/test_connectors.py tests/product/test_control.py tests/product/test_server_uninstall.py"
13+
matrix:
14+
- OTHER_TESTS=true
15+
- SHORT_PRODUCT_TEST_GROUP=0
16+
- LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN="tests/product/test_server_install.py"
17+
- LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_status.py"
18+
- LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_collect.py tests/product/test_server_uninstall.py"
19+
- LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_connectors.py"
20+
- LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_control.py"
21+
install:
1022
- pip install --upgrade pip==6.1.1
1123
- pip install -r requirements.txt
1224
before_script:
1325
- make docker-images
14-
script:
15-
- make clean lint dist docs
16-
- nosetests --with-timer --timer-ok 60s --timer-warning 300s -s tests.unit
17-
- nosetests --with-timer --timer-ok 60s --timer-warning 300s -s tests.integration
26+
- make presto-server-rpm.rpm
27+
script:
28+
- |
29+
if [ -v SHORT_PRODUCT_TEST_GROUP ]; then
30+
ALL_PRODUCT_TESTS=$(find tests/product/ -name 'test_*py' | grep -v __init__ | xargs wc -l | sort -n | head -n -1 | awk '{print $2}' | tr '\n' ' ') &&
31+
for LONG_PRODUCT_TEST in ${LONG_PRODUCT_TESTS[@]}; do
32+
ALL_PRODUCT_TESTS=${ALL_PRODUCT_TESTS//$LONG_PRODUCT_TEST/};
33+
if [ $? -ne 0 ]; then
34+
exit 1
35+
fi
36+
done &&
37+
SHORT_PRODUCT_TESTS=$(echo $ALL_PRODUCT_TESTS | tr ' ' '\n' | awk "NR % 1 == $SHORT_PRODUCT_TEST_GROUP" | tr '\n' ' ') &&
38+
make test-images &&
39+
nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${SHORT_PRODUCT_TESTS};
40+
elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN ]; then
41+
export IMAGE_NAMES="standalone_presto_admin" &&
42+
make test-images &&
43+
nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN};
44+
elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO ]; then
45+
export IMAGE_NAMES="standalone_presto standalone_presto_admin" &&
46+
make test-images &&
47+
nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine,!offline_installer' ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO};
48+
elif [ -v OTHER_TESTS ]; then
49+
make clean lint dist docs test-rpm &&
50+
nosetests -s tests.unit &&
51+
nosetests -s tests.integration;
52+
else
53+
echo "Unknown test" &&
54+
exit 1;
55+
fi

‎Makefile

+9-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
.PHONY: clean-all clean clean-eggs clean-build clean-pyc clean-test-containers clean-test \
2-
clean-docs lint smoke test test-all test-rpm coverage docs open-docs release release-builds \
3-
dist dist-online dist-offline wheel install precommit
2+
clean-docs lint smoke test test-all test-images test-rpm docker-images coverage docs \
3+
open-docs release release-builds dist dist-online dist-offline wheel install precommit
44

55
help:
66
@echo "precommit - run \`quick' tests and tasks that should pass or succeed prior to pushing"
@@ -16,7 +16,9 @@ help:
1616
@echo "smoke - run tests annotated with attr smoke using nosetests"
1717
@echo "test - run tests quickly with Python 2.6 and 2.7"
1818
@echo "test-all - run tests on every Python version with tox. Specify TEST_SUITE env variable to run only a given suite."
19+
@echo "test-images - create product test image(s). Specify IMAGE_NAMES env variable to create only certain images."
1920
@echo "test-rpm - run tests for the RPM package"
21+
@echo "docker-images - pull docker image(s). Specify DOCKER_IMAGE_NAME env variable for specific image."
2022
@echo "coverage - check code coverage quickly with the default Python"
2123
@echo "docs - generate Sphinx HTML documentation, including API docs"
2224
@echo "open-docs - open the root document (index.html) using xdg-open"
@@ -84,11 +86,15 @@ test-all: clean-test docker-images
8486
tox -- -s tests.integration
8587
tox -e py26 -- -s ${TEST_SUITE} -a '!quarantine'
8688

89+
IMAGE_NAMES?="all"
90+
91+
test-images: docker-images presto-server-rpm.rpm
92+
python tests/product/image_builder.py ${IMAGE_NAMES}
8793

8894
docker-images:
8995
docker pull teradatalabs/centos6-ssh-oj8
9096

91-
test-rpm: clean-test
97+
test-rpm: clean-test test-images
9298
tox -e py26 -- -s tests.rpm -a '!quarantine'
9399

94100
coverage:

‎tests/product/base_product_case.py

+26-85
Original file line numberDiff line numberDiff line change
@@ -26,45 +26,19 @@
2626

2727
from prestoadmin.util import constants
2828
from tests.base_test_case import BaseTestCase
29-
from tests.configurable_cluster import ConfigurableCluster
30-
from tests.docker_cluster import DockerCluster, DockerClusterException
29+
from tests.docker_cluster import DockerCluster
3130

32-
from tests.product.mode_installers import StandaloneModeInstaller, \
33-
YarnSliderModeInstaller
34-
from tests.product.prestoadmin_installer import PrestoadminInstaller
31+
from tests.configurable_cluster import ConfigurableCluster
32+
from tests.product.cluster_types import cluster_types
3533
from tests.product.standalone.presto_installer import StandalonePrestoInstaller
36-
from tests.product.topology_installer import TopologyInstaller
37-
from tests.product.yarn_slider.slider_installer import SliderInstaller
34+
3835

3936
PRESTO_VERSION = r'.+'
4037
RETRY_TIMEOUT = 120
4138
RETRY_INTERVAL = 5
4239

4340

4441
class BaseProductTestCase(BaseTestCase):
45-
STANDALONE_BARE_CLUSTER = 'bare'
46-
BARE_CLUSTER = 'bare'
47-
PA_ONLY_CLUSTER = 'pa_only_standalone'
48-
STANDALONE_PRESTO_CLUSTER = 'presto'
49-
50-
PA_ONLY_YS_CLUSTER = 'pa_only_ys'
51-
PA_SLIDER_CLUSTER = 'pa_slider'
52-
53-
_cluster_types = {
54-
BARE_CLUSTER: [],
55-
PA_ONLY_CLUSTER: [PrestoadminInstaller,
56-
StandaloneModeInstaller],
57-
STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller,
58-
StandaloneModeInstaller,
59-
TopologyInstaller,
60-
StandalonePrestoInstaller],
61-
PA_ONLY_YS_CLUSTER: [PrestoadminInstaller,
62-
YarnSliderModeInstaller],
63-
PA_SLIDER_CLUSTER: [PrestoadminInstaller,
64-
YarnSliderModeInstaller,
65-
SliderInstaller]
66-
}
67-
6842
default_workers_config_ = """coordinator=false
6943
discovery.uri=http://master:8080
7044
http-server.http.port=8080
@@ -134,19 +108,11 @@ def setUp(self):
134108
self.cluster = None
135109
self.default_keywords = {}
136110

137-
def _run_installers(self, installers):
138-
cluster = self.cluster
139-
for installer in installers:
140-
dependencies = installer.get_dependencies()
141-
142-
for dependency in dependencies:
143-
dependency.assert_installed(self)
144-
145-
installer_instance = installer(self)
146-
installer_instance.install()
147-
148-
self.default_keywords.update(installer_instance.get_keywords())
149-
cluster.postinstall(installer)
111+
def tearDown(self):
112+
self.restore_stdout_stderr_keep_open()
113+
if self.cluster:
114+
self.cluster.tear_down()
115+
super(BaseProductTestCase, self).tearDown()
150116

151117
def _apply_post_install_hooks(self, installers):
152118
for installer in installers:
@@ -158,12 +124,7 @@ def _update_replacement_keywords(self, installers):
158124
self.default_keywords.update(installer_instance.get_keywords())
159125

160126
def setup_cluster(self, bare_image_provider, cluster_type):
161-
try:
162-
installers = self._cluster_types[cluster_type]
163-
except KeyError:
164-
self.fail(
165-
'%s is not a valid cluster type. Valid cluster types are %s' %
166-
(cluster_type, ', '.join(self._cluster_types.keys())))
127+
installers = cluster_types[cluster_type]
167128

168129
config_filename = ConfigurableCluster.check_for_cluster_config()
169130

@@ -172,42 +133,22 @@ def setup_cluster(self, bare_image_provider, cluster_type):
172133
config_filename, self,
173134
StandalonePrestoInstaller.assert_installed)
174135
else:
175-
try:
176-
self.cluster, bare_cluster = DockerCluster.start_cluster(
177-
bare_image_provider, cluster_type)
178-
179-
# If we've found images and started a non-bare cluster, the
180-
# containers have already had the installers applied to them.
181-
# We do need to get the test environment in sync with the
182-
# containers by calling the following two functions.
183-
#
184-
# Once that's done, the cluster and test environment is in the
185-
# same state it would be as if we'd called _run_installers on
186-
# a bare cluster, and we can return.
187-
#
188-
# We do this to save the cost of running the installers on the
189-
# docker containers every time we run a test. In practice,
190-
# that turns out to be a fairly expensive thing to do.
191-
if not bare_cluster:
192-
self._apply_post_install_hooks(installers)
193-
self._update_replacement_keywords(installers)
194-
return
195-
except DockerClusterException as e:
196-
self.fail(e.msg)
197-
198-
# If we got a bare cluster back, we need to run the installers on it.
199-
# applying the post-install hooks and updating the replacement
200-
# keywords is handled internally in _run_installers.
201-
self._run_installers(installers)
202-
203-
if isinstance(self.cluster, DockerCluster):
204-
self.cluster.commit_images(bare_image_provider, cluster_type)
205-
206-
def tearDown(self):
207-
self.restore_stdout_stderr_keep_open()
208-
if self.cluster:
209-
self.cluster.tear_down()
210-
super(BaseProductTestCase, self).tearDown()
136+
self.cluster, bare_cluster = DockerCluster.start_cluster(
137+
bare_image_provider, cluster_type)
138+
139+
# If we've found images and started a non-bare cluster, the
140+
# containers have already had the installers applied to them.
141+
# We do need to get the test environment in sync with the
142+
# containers by calling the following two functions.
143+
#
144+
# We do this to save the cost of running the installers on the
145+
# docker containers every time we run a test. In practice,
146+
# that turns out to be a fairly expensive thing to do.
147+
if not bare_cluster:
148+
self._apply_post_install_hooks(installers)
149+
self._update_replacement_keywords(installers)
150+
else:
151+
raise RuntimeError("Docker images have not been created")
211152

212153
def dump_and_cp_topology(self, topology, cluster=None):
213154
if not cluster:

‎tests/product/cluster_types.py

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from tests.product.mode_installers import StandaloneModeInstaller, \
2+
YarnSliderModeInstaller
3+
from tests.product.prestoadmin_installer import PrestoadminInstaller
4+
from tests.product.topology_installer import TopologyInstaller
5+
from tests.product.yarn_slider.slider_installer import SliderInstaller
6+
from tests.product.standalone.presto_installer import StandalonePrestoInstaller
7+
8+
9+
STANDALONE_BARE_CLUSTER = 'bare'
10+
BARE_CLUSTER = 'bare'
11+
STANDALONE_PA_CLUSTER = 'pa_only_standalone'
12+
STANDALONE_PRESTO_CLUSTER = 'presto'
13+
14+
YARN_SLIDER_PA_CLUSTER = 'pa_only_ys'
15+
YARN_SLIDER_PA_AND_SLIDER_CLUSTER = 'pa_slider'
16+
17+
cluster_types = {
18+
BARE_CLUSTER: [],
19+
STANDALONE_PA_CLUSTER: [PrestoadminInstaller,
20+
StandaloneModeInstaller],
21+
STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller,
22+
StandaloneModeInstaller,
23+
TopologyInstaller,
24+
StandalonePrestoInstaller],
25+
YARN_SLIDER_PA_CLUSTER: [PrestoadminInstaller,
26+
YarnSliderModeInstaller],
27+
YARN_SLIDER_PA_AND_SLIDER_CLUSTER: [PrestoadminInstaller,
28+
YarnSliderModeInstaller,
29+
SliderInstaller]
30+
}

‎tests/product/image_builder.py

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
from tests.docker_cluster import DockerCluster
2+
from tests.hdp_bare_image_provider import HdpBareImageProvider
3+
from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider
4+
5+
from tests.product.base_product_case import BaseProductTestCase
6+
from tests.product.cluster_types import STANDALONE_BARE_CLUSTER, STANDALONE_PA_CLUSTER, \
7+
STANDALONE_PRESTO_CLUSTER, YARN_SLIDER_PA_CLUSTER, cluster_types
8+
9+
import argparse
10+
11+
12+
class ImageBuilder:
13+
def __init__(self, testcase):
14+
self.testcase = testcase
15+
self.testcase.default_keywords = {}
16+
self.testcase.cluster = None
17+
18+
def _run_installers(self, installers):
19+
cluster = self.testcase.cluster
20+
for installer in installers:
21+
dependencies = installer.get_dependencies()
22+
23+
for dependency in dependencies:
24+
dependency.assert_installed(self.testcase)
25+
26+
installer_instance = installer(self.testcase)
27+
installer_instance.install()
28+
29+
self.testcase.default_keywords.update(installer_instance.get_keywords())
30+
cluster.postinstall(installer)
31+
32+
def _setup_image(self, bare_image_provider, cluster_type):
33+
installers = cluster_types[cluster_type]
34+
35+
self.testcase.cluster, bare_cluster = DockerCluster.start_cluster(
36+
bare_image_provider, cluster_type)
37+
38+
# If we got a bare cluster back, we need to run the installers on it.
39+
# applying the post-install hooks and updating the replacement
40+
# keywords is handled internally in _run_installers.
41+
#
42+
# If we got a non-bare cluster back, that means the image already exists
43+
# and we created the cluster using that image.
44+
if bare_cluster:
45+
self._run_installers(installers)
46+
47+
if isinstance(self.testcase.cluster, DockerCluster):
48+
self.testcase.cluster.commit_images(bare_image_provider, cluster_type)
49+
50+
self.testcase.cluster.tear_down()
51+
52+
def _setup_image_with_no_hadoop_provider(self, cluster_type):
53+
self._setup_image(NoHadoopBareImageProvider(),
54+
cluster_type)
55+
56+
def setup_standalone_presto_images(self):
57+
cluster_type = STANDALONE_PRESTO_CLUSTER
58+
self._setup_image_with_no_hadoop_provider(cluster_type)
59+
60+
def setup_standalone_presto_admin_images(self):
61+
cluster_type = STANDALONE_PA_CLUSTER
62+
self._setup_image_with_no_hadoop_provider(cluster_type)
63+
64+
def setup_standalone_bare_images(self):
65+
cluster_type = STANDALONE_BARE_CLUSTER
66+
self._setup_image_with_no_hadoop_provider(cluster_type)
67+
68+
def setup_yarn_slider_presto_admin_images(self):
69+
cluster_type = YARN_SLIDER_PA_CLUSTER
70+
self._setup_image_with_no_hadoop_provider(cluster_type)
71+
72+
def _setup_cluster_with_hdp_image_provider(self, cluster_type):
73+
self._setup_image(HdpBareImageProvider(),
74+
cluster_type)
75+
76+
def setup_yarn_slider_presto_admin_and_slider_images(self):
77+
cluster_type = self.testcase.YARN_SLIDER_PA_AND_SLIDER_CLUSTER
78+
self._setup_cluster_with_hdp_image_provider(cluster_type)
79+
80+
if __name__ == "__main__":
81+
parser = argparse.ArgumentParser()
82+
parser.add_argument("image_type", metavar="image_type", type=str, nargs="+",
83+
choices=["standalone_presto", "standalone_presto_admin",
84+
"standalone_bare", "yarn_slider_presto_admin",
85+
"all"],
86+
help="Specify the type of image to create. The available choices are: "
87+
"standalone_presto, standalone_presto_admin, standalone_bare, "
88+
"yarn_slider_presto_admin, all")
89+
90+
# ImageBuilder needs an input testcase with access to unittest assertions
91+
# so the installers can check their resulting installations as well as some
92+
# product test helper functions.
93+
# This supplies a dummy testcase. BaseProductTestCase inherits from
94+
# unittest. A unittest instance can be successfully created if the name
95+
# of an existing method of the class is passed into the constructor.
96+
dummy_testcase = BaseProductTestCase('__init__')
97+
image_builder = ImageBuilder(dummy_testcase)
98+
99+
args = parser.parse_args()
100+
if "all" in args.image_type:
101+
image_builder.setup_standalone_presto_images()
102+
image_builder.setup_standalone_presto_admin_images()
103+
image_builder.setup_standalone_bare_images()
104+
image_builder.setup_yarn_slider_presto_admin_images()
105+
else:
106+
if "standalone_presto" in args.image_type:
107+
image_builder.setup_standalone_presto_images()
108+
if "standalone_presto_admin" in args.image_type:
109+
image_builder.setup_standalone_presto_admin_images()
110+
if "standalone_bare" in args.image_type:
111+
image_builder.setup_standalone_bare_images()
112+
if "yarn_slider_presto_admin" in args.image_type:
113+
image_builder.setup_yarn_slider_presto_admin_images()

‎tests/product/standalone/presto_installer.py

+2-3
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,8 @@ def _download_rpm():
119119
rpm_path = os.path.join(prestoadmin.main_dir,
120120
rpm_filename)
121121
urllib.urlretrieve(
122-
'https://repository.sonatype.org/service/local/artifact/maven'
123-
'/content?r=central-proxy&g=com.facebook.presto'
124-
'&a=presto-server-rpm&e=rpm&v=RELEASE', rpm_path)
122+
'http://search.maven.org/remotecontent?filepath=com/facebook/presto/'
123+
'presto-server-rpm/0.148/presto-server-rpm-0.148.rpm', rpm_path)
125124
return rpm_filename
126125

127126
@staticmethod

0 commit comments

Comments
 (0)