Add presto yarn package install and uninstall

The presto-yarn package is installed and uninstalled by running slider
commands on the slider master. Product tests for the new functionality check
both that:
1) The package ends up in the right place in HDFS
2) Slider recognizes that the package has been installed

Doing 2. requires running slider as the configured user; this required
updating docker-py to support the user= kwarg to exec_create, which in turn
required updating how we pass the mem_limit parameter in docker_cluster.py.

Testing: unit tests, product tests, some manual testing.

Author: Eric Diven

.gitignore

@@ -31,3 +31,6 @@ dist/
 
 #mvn targets
 presto-admin-test/target
+
+# presto yarn package for product tests
+presto-yarn-package.zip

prestoadmin/__init__.py

@@ -42,5 +42,5 @@
 import package
 import script
 import server
-import slider.server as slider
+import yarn_slider.server as slider
 import topology

prestoadmin/main.py

@@ -852,10 +852,29 @@ def _exit_code(results):
     If every entry in the dict has a value of None, the exit code is 0.
     If any entry has a value that is not None, something failed, and we should
     exit with a non-zero exit code.
+
+    That isn't really the whole story: Any task that calls tasks.execute() and
+    returns that as the result will have an item in the dictionary of the form
+    hostname: {hostname: Exception | None}. This means that we need to
+    recursively check any values in the map that are of type dict following the
+    above scheme.
     """
     for v in results.values():
-        if v is not None:
-            return 1
+        # No exception, inspect the next value.
+        if v is None:
+            continue
+
+        # The value is a dict resulting from calling fabric.tasks.execute.
+        # Check the results recursively
+        if type(v) is dict:
+            exit_code = _exit_code(v)
+            if exit_code != 0:
+                return exit_code
+            continue
+
+        # In any case where things were OK above, we've continued the loop. At
+        # this point, we know something failed.
+        return 1
     return 0
 
 

prestoadmin/slider/server.py

@@ -17,6 +17,8 @@
 """
 from fabric.api import env
 
+from overrides import overrides
+
 from prestoadmin import config
 from prestoadmin.util import constants
 from prestoadmin.util.base_config import BaseConfig, SingleConfigItem
@@ -135,7 +137,8 @@ def __init__(self):
         super(StandaloneConfig, self).__init__(constants.TOPOLOGY_CONFIG_PATH,
                                                _TOPOLOGY_CONFIG)
 
-    def load_conf(self):
+    @overrides
+    def read_conf(self):
         conf = self._get_conf_from_file()
         config.fill_defaults(conf, DEFAULT_PROPERTIES)
         validate(conf)
@@ -144,13 +147,16 @@ def load_conf(self):
     def _get_conf_from_file(self):
         return config.get_conf_from_json_file(self.config_path)
 
+    @overrides
     def is_config_loaded(self):
         return STANDALONE_CONFIG_LOADED in env and \
             env[STANDALONE_CONFIG_LOADED]
 
+    @overrides
     def set_config_loaded(self):
         env[STANDALONE_CONFIG_LOADED] = True
 
+    @overrides
     def set_env_from_conf(self, conf):
         env.user = conf['username']
         env.port = conf['port']

prestoadmin/standalone/config.py

@@ -104,10 +104,10 @@ def __init__(self, config_path, config_items):
         self.config_path = config_path
         self.config_items = config_items
 
-    def load_conf(self):
+    def read_conf(self):
         return config.get_conf_from_json_file(self.config_path)
 
-    def store_conf(self, conf):
+    def write_conf(self, conf):
         config.write(config.json_to_string(conf), self.config_path)
         return self.config_path
 
@@ -122,10 +122,10 @@ def get_config(self):
             if not self.is_config_loaded():
                 conf = {}
                 try:
-                    conf = self.load_conf()
+                    conf = self.read_conf()
                 except ConfigFileNotFoundError:
                     conf = self.get_conf_interactive()
-                    self.store_conf(conf)
+                    self.write_conf(conf)
 
                 self.set_env_from_conf(conf)
                 self.set_config_loaded()

prestoadmin/util/base_config.py

@@ -14,11 +14,13 @@
 
 
 """
-Module for setting and validating the presto-admin slider config
+Module for setting and validating the presto-admin Apache Slider config
 """
 
 import os
 
+from overrides import overrides
+
 from fabric.state import env
 
 from prestoadmin.util.base_config import BaseConfig, SingleConfigItem, \
@@ -28,7 +30,8 @@
     validate_username, validate_can_connect, validate_can_sudo
 
 SLIDER_CONFIG_LOADED = 'slider_config_loaded'
-SLIDER_CONFIG_PATH = os.path.join(LOCAL_CONF_DIR, 'slider', 'config.json')
+SLIDER_CONFIG_DIR = os.path.join(LOCAL_CONF_DIR, 'slider')
+SLIDER_CONFIG_PATH = os.path.join(SLIDER_CONFIG_DIR, 'config.json')
 SLIDER_MASTER = 'slider_master'
 
 HOST = 'slider_master'
@@ -42,6 +45,10 @@
 JAVA_HOME = 'JAVA_HOME'
 HADOOP_CONF = 'HADOOP_CONF'
 
+# This key comes from the server install step, NOT a user prompt. Accordingly,
+# there is no SliderConfigItem for it in _SLIDER_CONFIG
+PRESTO_PACKAGE = 'presto_slider_package'
+
 
 _SLIDER_CONFIG = [
     MultiConfigItem([
@@ -57,12 +64,12 @@
                     'Connection failed for %%(%s)s@%%(%s)s:%%(%s)d. ' +
                     'Re-enter connection information.'),
 
-    SingleConfigItem(DIR, 'Enter the directory to install slider into on ' +
+    SingleConfigItem(DIR, 'Enter the directory to install slider into on '
                      'the slider master:', '/opt/slider', None),
 
     MultiConfigItem([
-        SingleConfigItem(SLIDER_USER, 'Enter a user name for conducting ' +
-                         'slider operations on the slider master ', 'yarn',
+        SingleConfigItem(SLIDER_USER, 'Enter a user name for running slider '
+                         'on the slider master ', 'yarn',
                          validate_username)],
                     validate_can_sudo,
                     (SLIDER_USER, ADMIN_USER, HOST, SSH_PORT),
@@ -75,26 +82,54 @@
                      '/usr/lib/jvm/java', None),
     SingleConfigItem(HADOOP_CONF, 'Enter the location of the Hadoop ' +
                      'configuration on the slider master:',
-                     '/etc/hadoop/conf', None)]
+                     '/etc/hadoop/conf', None),
+    SingleConfigItem(APPNAME, 'Enter a name for the presto slider application',
+                     'PRESTO', None)]
 
 
 class SliderConfig(BaseConfig):
+    '''
+    presto-admin needs to update the slider config other than through the
+    interactive config process because it needs to keep track of the name
+    of the presto-yarn-package we install.
+
+    As a result, SliderConfig acts a little funny; it acts like enough of a
+    dict to allow env.conf[NAME] lookups and modifications, and it also exposes
+    the ability to store the config after it's been modified.
+    '''
 
     def __init__(self):
         super(SliderConfig, self).__init__(SLIDER_CONFIG_PATH, _SLIDER_CONFIG)
+        self.config = {}
+
+    def __getitem__(self, key):
+        return self.config[key]
 
+    def __setitem__(self, key, value):
+        self.config[key] = value
+
+    def __delitem__(self, key):
+        del self.config[key]
+
+    @overrides
     def is_config_loaded(self):
         return SLIDER_CONFIG_LOADED in env and env[SLIDER_CONFIG_LOADED]
 
+    @overrides
     def set_config_loaded(self):
         env[SLIDER_CONFIG_LOADED] = True
 
+    @overrides
     def set_env_from_conf(self, conf):
+        self.config.update(conf)
         env.user = conf[ADMIN_USER]
         env.port = conf[SSH_PORT]
         env.roledefs[SLIDER_MASTER] = [conf[HOST]]
         env.roledefs['all'] = env.roledefs[SLIDER_MASTER]
 
-        env.conf = conf
+        env.conf = self
 
         env.hosts = env.roledefs['all'][:]
+
+    def store_conf(self):
+        super(SliderConfig, self).write_conf(self.config)