Skip to content

Commit

Permalink
add interface to specify YARN application name
Browse files Browse the repository at this point in the history
  • Loading branch information
@kmaehashi
kmaehashi committed Oct 1, 2015
1 parent 76b72ca commit e58715a
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 14 deletions.
30 changes: 20 additions & 10 deletions ...n/jubatus-on-yarn-client/src/main/scala/us/jubat/yarn/client/JubatusYarnApplication.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ object JubatusYarnApplication extends HasLogger {

/**
* JubatusYarnApplication を起動します。
* 設定ファイルの内容を文字列で指定します。
* デフォルトのベースパス (hdfs:///jubatus-on-yarn) を使用します。
*
* juba${aLearningMachineType_proxy} がひとつ, juba${aLearningMachineType} が ${aNodeCount} だけ起動します。
* 各 juba${aLearningMachineType} の使用するリソースを ${aResource} に指定してください。
Expand All @@ -82,14 +84,16 @@ object JubatusYarnApplication extends HasLogger {
* @param aConfigString config json string
* @param aResource computer resources in the cluster
* @param aNodeCount number of cluster
* @param aApplicationName yarn-application name
* @return [[JubatusYarnApplication]]
*/
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigString: String, aResource: Resource, aNodeCount: Int): Future[JubatusYarnApplication] = {
start(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigString, aResource, aNodeCount, new Path("hdfs:///jubatus-on-yarn"))
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigString: String, aResource: Resource, aNodeCount: Int, aApplicationName: String): Future[JubatusYarnApplication] = {
start(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigString, aResource, aNodeCount, new Path("hdfs:///jubatus-on-yarn"), aApplicationName)
}

/**
* JubatusYarnApplication を起動します。
* 設定ファイルの内容を文字列で指定します。
*
* juba${aLearningMachineType_proxy} がひとつ, juba${aLearningMachineType} が ${aNodeCount} だけ起動します。
* 各 juba${aLearningMachineType} の使用するリソースを ${aResource} に指定してください。
Expand All @@ -101,9 +105,10 @@ object JubatusYarnApplication extends HasLogger {
* @param aResource computer resources in the cluster
* @param aNodeCount number of cluster
* @param aBasePath base path of jar and sh files
* @param aApplicationName yarn-application name
* @return [[JubatusYarnApplication]]
*/
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigString: String, aResource: Resource, aNodeCount: Int, aBasePath: Path): Future[JubatusYarnApplication] = Future {
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigString: String, aResource: Resource, aNodeCount: Int, aBasePath: Path, aApplicationName: String): Future[JubatusYarnApplication] = Future {
require(aResource.memory > 0, "specify memory than 1MB.")
require(aNodeCount > 0, "specify node count than 1")

Expand All @@ -114,14 +119,16 @@ object JubatusYarnApplication extends HasLogger {
case None => throw new IllegalStateException("Service not running.")
case Some(tYarnClientController) =>
// ApplicationMaster 起動
logger.info(s"startJubatusApplication $aLearningMachineName, $aLearningMachineType, $aZookeepers, $aConfigString, $aResource, $aNodeCount")
val tApplicationMasterProxy = tYarnClientController.startJubatusApplication(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigString, aResource, aNodeCount, aBasePath)
logger.info(s"startJubatusApplication $aLearningMachineName, $aLearningMachineType, $aZookeepers, $aConfigString, $aResource, $aNodeCount, $aApplicationName")
val tApplicationMasterProxy = tYarnClientController.startJubatusApplication(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigString, aResource, aNodeCount, aBasePath, aApplicationName)
waitForStarted(ApplicationContext(tYarnClientController, tApplicationMasterProxy, tService))
}
}

/**
* JubatusYarnApplication を起動します。
* 設定ファイルの場所をパスで指定します。
* デフォルトのベースパス (hdfs:///jubatus-on-yarn) を使用します。
*
* juba${aLearningMachineType_proxy} がひとつ, juba${aLearningMachineType} が ${aNodeCount} だけ起動します。
* 各 juba${aLearningMachineType} の使用するリソースを ${aResource} に指定してください。
Expand All @@ -132,14 +139,16 @@ object JubatusYarnApplication extends HasLogger {
* @param aConfigFile config file
* @param aResource computer resources in the cluster
* @param aNodeCount number of cluster
* @param aApplicationName yarn-application name
* @return [[JubatusYarnApplication]]
*/
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigFile: Path, aResource: Resource, aNodeCount: Int): Future[JubatusYarnApplication] = {
start(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigFile, aResource, aNodeCount, new Path("hdfs:///jubatus-on-yarn"))
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigFile: Path, aResource: Resource, aNodeCount: Int, aApplicationName: String): Future[JubatusYarnApplication] = {
start(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigFile, aResource, aNodeCount, new Path("hdfs:///jubatus-on-yarn"), aApplicationName)
}

/**
* JubatusYarnApplication を起動します。
* 設定ファイルの場所をパスで指定します。
*
* juba${aLearningMachineType_proxy} がひとつ, juba${aLearningMachineType} が ${aNodeCount} だけ起動します。
* 各 juba${aLearningMachineType} の使用するリソースを ${aResource} に指定してください。
Expand All @@ -150,9 +159,10 @@ object JubatusYarnApplication extends HasLogger {
* @param aConfigFile config file
* @param aResource computer resources in the cluster
* @param aNodeCount number of cluster
* @param aApplicationName yarn-application name
* @return [[JubatusYarnApplication]]
*/
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigFile: Path, aResource: Resource, aNodeCount: Int, aBasePath: Path): Future[JubatusYarnApplication] = Future {
def start(aLearningMachineName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigFile: Path, aResource: Resource, aNodeCount: Int, aBasePath: Path, aApplicationName: String): Future[JubatusYarnApplication] = Future {
require(aResource.memory > 0, "specify memory than 1MB.")
require(aNodeCount > 0, "specify node count than 1")

Expand All @@ -163,8 +173,8 @@ object JubatusYarnApplication extends HasLogger {
case None => throw new IllegalStateException("Service not running.")
case Some(tYarnClientController) =>
// ApplicationMaster 起動
logger.info(s"startJubatusApplication $aLearningMachineName, $aLearningMachineType, $aZookeepers, $aConfigFile, $aResource, $aNodeCount")
val tApplicationMasterProxy = tYarnClientController.startJubatusApplication(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigFile, aResource, aNodeCount, aBasePath)
logger.info(s"startJubatusApplication $aLearningMachineName, $aLearningMachineType, $aZookeepers, $aConfigFile, $aResource, $aNodeCount, $aApplicationName")
val tApplicationMasterProxy = tYarnClientController.startJubatusApplication(aLearningMachineName, aLearningMachineType, aZookeepers, aConfigFile, aResource, aNodeCount, aBasePath, aApplicationName)
waitForStarted(ApplicationContext(tYarnClientController, tApplicationMasterProxy, tService))
}
}
Expand Down
32 changes: 28 additions & 4 deletions ...arn/jubatus-on-yarn-client/src/main/scala/us/jubat/yarn/client/YarnClientController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,41 @@ class YarnClientController(location: Location, yarnClient: YarnClient = new Defa
mApplicationMaster.get
}

def startJubatusApplication(aName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigString: String, aResource: Resource, aNodeCount: Int, aBasePath: Path): ApplicationMasterProxy = {
val tFullName = getFullName(aName, aLearningMachineType, aZookeepers)
/**
* Start Jubatus application using specified config string.
*
* @param aName
* @param aLearningMachineType
* @param aZookeepers
* @param aConfigString
* @param aResource
* @param aNodeCount
* @param aBasePath
* @param aApplicationName name of YARN application. When null is given, the default value will be used.
*/
def startJubatusApplication(aName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigString: String, aResource: Resource, aNodeCount: Int, aBasePath: Path, aApplicationName: String): ApplicationMasterProxy = {
val tFullName = Option(aApplicationName).getOrElse(getFullName(aName, aLearningMachineType, aZookeepers))
logger.info(s"starting $tFullName")

val tApplicationId = yarnClient.submitApplicationMaster(tFullName, aName, aLearningMachineType, aZookeepers, aConfigString, aResource, aNodeCount, location, aBasePath)

registerApplication(tFullName, tApplicationId, aNodeCount)
}

def startJubatusApplication(aName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigFile: Path, aResource: Resource, aNodeCount: Int, aBasePath: Path): ApplicationMasterProxy = {
val tFullName = getFullName(aName, aLearningMachineType, aZookeepers)
/**
* Start Jubatus application using specified config path.
*
* @param aName
* @param aLearningMachineType
* @param aZookeepers
* @param aConfigFile
* @param aResource
* @param aNodeCount
* @param aBasePath
* @param aApplicationName name of YARN application. When null is given, the default value will be used.
*/
def startJubatusApplication(aName: String, aLearningMachineType: LearningMachineType, aZookeepers: List[Location], aConfigFile: Path, aResource: Resource, aNodeCount: Int, aBasePath: Path, aApplicationName: String): ApplicationMasterProxy = {
val tFullName = Option(aApplicationName).getOrElse(getFullName(aName, aLearningMachineType, aZookeepers))
logger.info(s"starting $tFullName")

val tApplicationId = yarnClient.submitApplicationMaster(tFullName, aName, aLearningMachineType, aZookeepers, aConfigFile, aResource, aNodeCount, location, aBasePath)
Expand Down

0 comments on commit e58715a

Please sign in to comment.