Revert "TonY release pipeline (#19)" and TonY release pipeline v2 (#26)

* Revert "TonY release pipeline (#19)"

This reverts commit f980fb7.

* TonY release pipeline v2

* Concurrent jobs in same azkaban execution should not overwrite each others tony.xml

* Version bump to 0.1.3

build.gradle

@@ -1,8 +1,11 @@
-group "com.linkedin"
+apply plugin: "java"
+apply plugin: "eclipse"
+apply plugin: "idea"
+
+sourceCompatibility = 1.8
 
 buildscript {
   repositories {
-    jcenter()
     maven {
       url "https://plugins.gradle.org/m2/"
     }
@@ -11,17 +14,9 @@ buildscript {
     classpath "com.github.jengelman.gradle.plugins:shadow:2.0.4"
     classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.1"
     classpath "gradle.plugin.nl.javadude.gradle.plugins:license-gradle-plugin:0.14.0"
-    classpath "org.shipkit:shipkit:2.0.13"
   }
 }
 
-apply plugin: "java"
-apply plugin: "eclipse"
-apply plugin: "idea"
-apply plugin: "org.shipkit.java"
-
-sourceCompatibility = 1.8
-
 def hadoopVersion = "3.1.1"
 
 ext.deps = [
@@ -66,6 +61,17 @@ ext.deps = [
 
 allprojects {
   group = "com.linkedin.tony"
+  project.version = "0.1.3"
+}
+
+task sourcesJar(type: Jar) {
+  from sourceSets.main.allJava
+  classifier = 'sources'
+}
+
+task javadocJar(type: Jar) {
+  from javadoc
+  classifier = 'javadoc'
 }
 
 subprojects {
@@ -74,14 +80,16 @@ subprojects {
   apply plugin: "java"
   apply plugin: "eclipse"
   apply plugin: "idea"
-
+  apply plugin: "maven"
+  apply plugin: "maven-publish"
+  apply plugin: "signing"
   repositories {
     mavenCentral()
   }
-
   plugins.withType(JavaPlugin) {
     sourceCompatibility = 1.8
     dependencies {
+      // dependency defined in product_spec.json
       testCompile deps.external.testng
     }
 
@@ -103,8 +111,63 @@ subprojects {
       exclude group: "org.apache.hadoop"
     }
   }
-}
 
+  publishing {
+    publications {
+      mavenJava(MavenPublication) {
+        from components.java
+        artifactId = "${project.name}"
+        artifact sourcesJar
+        artifact javadocJar
+        pom {
+          name = "${project.name}".toString()
+          description = "${project.name}".toString()
+          url = 'https://github.com/linkedin/TonY'
+          licenses {
+            license {
+              name = "BSD 2-Clause"
+              url = "https://opensource.org/licenses/BSD-2-Clause"
+            }
+          }
+          developers {
+            developer {
+              id = 'erwa'
+              name = 'Anthony Hsu'
+            }
+            developer {
+              id = 'jhung'
+              name = 'Jonathan Hung'
+            }
+            developer {
+              id = 'oliverhu'
+              name = 'Keqiu Hu'
+            }
+          }
+          scm {
+            connection = 'https://github.com/linkedin/TonY.git'
+            developerConnection = '[email protected]:linkedin/TonY.git'
+            url = 'https://github.com/linkedin/TonY'
+          }
+        }
+      }
+    }
+    repositories {
+      maven {
+        def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2"
+        def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots"
+        url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
+        credentials {
+          username findProperty("mavenUser")
+          password findProperty("mavenPassword")
+        }
+      }
+    }
+  }
+  signing {
+    sign publishing.publications.mavenJava
+  }
+}
+
 apply plugin: 'distribution'
 
 // Generates a closure which is used to set up the contents

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-rc-2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip

tony-azkaban/build.gradle

@@ -1,3 +1,5 @@
+apply plugin: 'java'
+
 dependencies {
   compile deps.azkaban.az_core
   compile deps.azkaban.azkaban_common

tony-azkaban/src/main/java/com/linkedin/tony/azkaban/TensorFlowJob.java

@@ -32,9 +32,9 @@ public class TensorFlowJob extends HadoopJavaJob {
   public static final String HADOOP_OPTS = ENV_PREFIX + "HADOOP_OPTS";
   public static final String HADOOP_GLOBAL_OPTS = "hadoop.global.opts";
   public static final String WORKER_ENV_PREFIX = "worker_env.";
-  private static final String TONY_XML = "_tony-conf/tony.xml";
   private static final String TONY_CONF_PREFIX = "tony.";
   private HadoopSecurityManager hadoopSecurityManager;
+  private String tonyXml;
   private File tonyConfFile;
 
   public TensorFlowJob(String jobid, Props sysProps, Props jobProps, Logger log) {
@@ -56,7 +56,8 @@ public TensorFlowJob(String jobid, Props sysProps, Props jobProps, Logger log) {
       }
     }
 
-    tonyConfFile = new File(getWorkingDirectory(), TONY_XML);
+    tonyXml = String.format("_tony-conf-%s/tony.xml", jobid);
+    tonyConfFile = new File(getWorkingDirectory(), tonyXml);
   }
 
   @Override
@@ -84,12 +85,12 @@ public void run() throws Exception {
 
   private void setupHadoopOpts(Props props) {
     if (props.containsKey(HADOOP_GLOBAL_OPTS)) {
-      String hadoopGlobalOps = props.getString(HADOOP_GLOBAL_OPTS);
+      String hadoopGlobalOpts = props.getString(HADOOP_GLOBAL_OPTS);
       if (props.containsKey(HADOOP_OPTS)) {
-        String hadoopOps = props.getString(HADOOP_OPTS);
-        props.put(HADOOP_OPTS, String.format("%s %s", hadoopOps, hadoopGlobalOps));
+        String hadoopOpts = props.getString(HADOOP_OPTS);
+        props.put(HADOOP_OPTS, String.format("%s %s", hadoopOpts, hadoopGlobalOpts));
       } else {
-        props.put(HADOOP_OPTS, hadoopGlobalOps);
+        props.put(HADOOP_OPTS, hadoopGlobalOpts);
       }
     }
   }
@@ -180,7 +181,7 @@ protected String getMainArguments() {
     try (OutputStream os = new FileOutputStream(tonyConfFile)) {
       tfConf.writeXml(os);
     } catch (Exception e) {
-      throw new RuntimeException("Failed to create " + TONY_XML + " conf file. Exiting.", e);
+      throw new RuntimeException("Failed to create " + tonyXml + " conf file. Exiting.", e);
     }
 
     info("Complete main arguments: " + args);

tony-azkaban/src/test/java/com/linkedin/tony/azkaban/TestTensorFlowJob.java

@@ -10,6 +10,7 @@
 
 import azkaban.utils.FileIOUtils;
 import azkaban.utils.Props;
+import java.io.File;
 import java.util.List;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.log4j.Logger;
@@ -56,6 +57,7 @@ public String getWorkingDirectory() {
       }
     };
     String args = tfJob.getMainArguments();
+    Assert.assertTrue(new File(tfJob.getWorkingDirectory(), "_tony-conf-test_tf_job/tony.xml").exists());
     Assert.assertTrue(args.contains(TensorFlowJobArg.HDFS_CLASSPATH.tfParamName + " hdfs://nn:8020"));
     Assert.assertTrue(args.contains(TensorFlowJobArg.SHELL_ENV.tfParamName + " E2=e2"));
     Assert.assertTrue(args.contains(TensorFlowJobArg.SHELL_ENV.tfParamName + " E1=e1"));
@@ -68,7 +70,12 @@ public void testClassPaths() {
     jobProps.put(TensorFlowJob.WORKING_DIR, FilenameUtils.getFullPath(FileIOUtils.getSourcePathFromClass(Props.class)));
     sysProps.put("jobtype.classpath", "123,456,789");
     sysProps.put("plugin.dir", "Plugins");
-    final TensorFlowJob tfJob = new TensorFlowJob("test_tf_job_class_path", sysProps, jobProps, log);
+    final TensorFlowJob tfJob = new TensorFlowJob("test_tf_job_class_path", sysProps, jobProps, log) {
+      @Override
+      public String getWorkingDirectory() {
+        return System.getProperty("java.io.tmpdir");
+      }
+    };
     List<String> paths = tfJob.getClassPaths();
     int counter = 0;
     for (String path : paths) {
@@ -77,6 +84,6 @@ public void testClassPaths() {
       }
     }
     Assert.assertTrue(counter == 3);
+    Assert.assertTrue(paths.contains(new File(tfJob.getWorkingDirectory(), "_tony-conf-test_tf_job_class_path").toString()));
   }
-
 }

tony-cli/build.gradle

@@ -1,3 +1,4 @@
+apply plugin: 'java'
 apply plugin: 'com.github.johnrengelman.shadow'
 
 dependencies {
@@ -12,8 +13,4 @@ shadowJar {
   }
 }
 
-configurations.all {
-  artifacts.removeAll artifacts.findAll { !it.archiveTask.enabled }
-}
-
 build.dependsOn(shadowJar)

tony-core/build.gradle

@@ -1,3 +1,4 @@
+apply plugin: 'java'
 apply plugin: 'com.google.protobuf'
 
 dependencies {

tony-core/src/main/java/com/linkedin/tony/TonyApplicationMaster.java

@@ -294,8 +294,8 @@ private void buildTensorFlowSession() {
   /**
    * Entry point of TonyApplicationMaster
    * The workflow of a training job in AM
-   * 1. prepare 2. start 3a. failed    4a. reset 5a. retry if amRetryCount greater than zero, otherwise fail the job.
-   *                     3b. succeeded 4b. stop  5b. job succeeded
+   * prepare -> start -> failed    -> reset -> retry if amRetryCount > 0 otherwise fail the job.
+   *                  -> succeeded -> stop -> job succeeded
    * @param args the args from user inputs
    */
   public static void main(String[] args) {

tony-mini/build.gradle

@@ -1,3 +1,5 @@
+apply plugin: 'java'
+
 dependencies {
   compile deps.external.avro
   compile deps.external.guava