Fix #279: TonY HistoryFileMover stops after the first job without a h…

…istory file or that is still in progress (#280)

* Catch all exceptions in HistoryFileMover, do null check for jhistFilePath

* Fix #279: TonY HistoryFileMover stops after the first job without a history file or that is still in progress

tony-core/src/main/java/com/linkedin/tony/util/Utils.java

@@ -277,7 +277,9 @@ public static int executeShell(String taskCommand, long timeout, Map<String, Str
     String executablePath = taskCommand.trim().split(" ")[0];
     File executable = new File(executablePath);
     if (!executable.canExecute()) {
-      executable.setExecutable(true);
+      if (!executable.setExecutable(true)) {
+        LOG.error("Failed to make " + executable + " executable");
+      }
     }
 
     // Used for running unit tests in build boxes without Hadoop environment.

tony-portal/app/history/HistoryFileMover.java

@@ -51,17 +51,21 @@ public HistoryFileMover(Config appConf, Requirements requirements, CacheWrapper
         LOG.error("Failed to list files in " + intermediateDir, e);
       }
       if (jobDirs != null) {
-        moveIntermediateToFinished(fs, jobDirs);
+        try {
+          moveIntermediateToFinished(fs, jobDirs);
+        } catch (Exception e) {
+          LOG.error("Encountered exception while moving history directories", e);
+        }
       }
     }, 0, moverIntervalMs, TimeUnit.MILLISECONDS);
   }
 
   private void moveIntermediateToFinished(FileSystem fs, FileStatus[] jobDirs) {
     for (FileStatus jobDir : jobDirs) {
-      String jhistFilePath = ParserUtils.getJhistFilePath(fs, jobDir.getPath());
       cacheWrapper.updateCaches(jobDir.getPath());
-      if (jobInProgress(jhistFilePath)) {
-        return;
+      String jhistFilePath = ParserUtils.getJhistFilePath(fs, jobDir.getPath());
+      if (jhistFilePath == null || jobInProgress(jhistFilePath)) {
+        continue;
       }
 
       Path source = new Path(jhistFilePath).getParent();

tony-portal/conf/application.example.conf

@@ -9,7 +9,7 @@ tony {
     location = "/tmp/tony-history"
     intermediate = "/tmp/tony-history/intermediate"
     finished = "/tmp/tony-history/finished"
-    mover-interval-ms = "5000"
+    mover-interval-ms = "300000"
   }
   portal {
     cache.max-entries = "2000"