[query] Use Log4j 2 Api

Author: ehigham

hail/build.mill

@@ -52,6 +52,14 @@ object Deps {
     def core: Task[Dep] = Task.Anon(mvn"org.apache.spark::spark-core:${Env.sparkVersion()}")
     def mllib: Task[Dep] = Task.Anon(mvn"org.apache.spark::spark-mllib:${Env.sparkVersion()}")
   }
+  
+  object Log4j {
+    // have to forceVersion for reasons unknown?
+    val bom = mvn"org.apache.logging.log4j:log4j-bom:2.25.2".forceVersion()
+    val api = mvn"org.apache.logging.log4j:log4j-api"
+    val core = mvn"org.apache.logging.log4j:log4j-core"
+    val slf4j = mvn"org.apache.logging.log4j:log4j-slf4j2-impl"
+  }
 
   val samtools = mvn"com.github.samtools:htsjdk:3.0.5"
   val jdistlib = mvn"net.sourceforge.jdistlib:jdistlib:0.4.5"
@@ -65,8 +73,6 @@ object Deps {
   val netlib = mvn"com.github.fommil.netlib:all:1.1.2"
   val avro = mvn"org.apache.avro:avro:1.11.2"
   val junixsocket = mvn"com.kohlschutter.junixsocket:junixsocket-core:2.6.1"
-  val log4jApi = mvn"org.apache.logging.log4j:log4j-api:2.20.0"
-  val sfl4jNop = mvn"org.slf4j:slf4j-nop:1.7.36"
   val hadoopClient = mvn"org.apache.hadoop:hadoop-client:3.4.2"
   val jackson = mvn"com.fasterxml.jackson.core:jackson-core:2.15.2"
   val sourcecode = mvn"com.lihaoyi::sourcecode:0.4.2"
@@ -254,6 +260,7 @@ trait RootHailModule extends CrossScalaModule with HailModule { outer =>
 
   override def bomMvnDeps: T[Seq[Dep]] = Seq(
     Deps.Asm.bom,
+    Deps.Log4j.bom,
   )
 
   override def mvnDeps: T[Seq[Dep]] = Seq(
@@ -283,11 +290,12 @@ trait RootHailModule extends CrossScalaModule with HailModule { outer =>
     Deps.netlib,
     Deps.avro.excludeOrg("com.fasterxml.jackson.core"),
     Deps.junixsocket,
-    Deps.sfl4jNop,
+    Deps.Log4j.core,
+    Deps.Log4j.slf4j,
   )
 
   override def compileMvnDeps: T[Seq[Dep]] = Seq(
-    Deps.log4jApi,
+    Deps.Log4j.api,
     Deps.hadoopClient,
     Deps.Spark.core().excludeOrg("org.scalanlp"),  // Hail has an explicit dependency on Breeze 1.1
     Deps.Spark.mllib().excludeOrg("org.scalanlp"),  // Hail has an explicit dependency on Breeze 1.1
@@ -357,8 +365,6 @@ trait RootHailModule extends CrossScalaModule with HailModule { outer =>
   }
 
   object test extends HailTests {
-    override def resources: T[Seq[PathRef]] = outer.resources() ++ super.resources()
-
     override def assemblyRules: Seq[Rule] = outer.assemblyRules ++ Seq(
       Rule.Relocate("org.codehaus.jackson.**", "is.hail.relocated.@0")
     )

hail/hail/resources/log4j.properties

@@ -0,0 +1,7 @@
+appender.0.type           = Console
+appender.0.name           = CONSOLE
+appender.0.layout.type    = PatternLayout
+appender.0.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p %c{1}:%L - %m%n
+
+rootLogger.level             = INFO
+rootLogger.appenderRef.0.ref = CONSOLE

hail/hail/resources/log4j2.properties

@@ -1308,11 +1308,13 @@ object LoadVCF {
   def warnDuplicates(ids: Array[String]): Unit = {
     val duplicates = ids.counter().filter(_._2 > 1)
     if (duplicates.nonEmpty) {
-      warn(
-        s"Found ${duplicates.size} duplicate ${plural(duplicates.size, "sample ID")}:\n  @1",
+      log.warn(
+        "Found {} duplicate {}:\n {}",
+        duplicates.size,
+        plural(duplicates.size, "sample ID"),
         duplicates.toArray.sortBy(-_._2).map { case (id, count) =>
           s"""($count) "$id""""
-        }.truncatable("\n  "),
+        }.mkString("\n  "),
       )
     }
   }

hail/hail/src/is/hail/io/vcf/LoadVCF.scala

@@ -1,58 +1,24 @@
 package is.hail.utils
 
-import org.apache.log4j.{LogManager, Logger}
+import org.apache.logging.log4j.{LogManager, Logger}
 
 object LogHelper {
   // exposed more directly for generated code
   def logInfo(msg: String): Unit = log.info(msg)
-  def warning(msg: String): Unit = consoleLog.warn(msg)
+  def warning(msg: String): Unit = log.warn(msg)
   def consoleInfo(msg: String): Unit = info(msg)
 }
 
 trait Logging {
-  @transient private var logger: Logger = _
-  @transient private var consoleLogger: Logger = _
-
-  def log: Logger = {
-    if (logger == null)
-      logger = LogManager.getRootLogger
-    logger
-  }
-
-  def consoleLog: Logger = {
-    if (consoleLogger == null)
-      consoleLogger = LogManager.getLogger("Hail")
-    consoleLogger
-  }
+  @transient lazy val log: Logger =
+    LogManager.getLogger(getClass.getName.stripSuffix("$"))
 
   def info(msg: String): Unit =
-    consoleLog.info(msg)
-
-  def info(msg: String, t: Truncatable): Unit = {
-    val (screen, logged) = t.strings
-    if (screen == logged)
-      consoleLog.info(format(msg, screen))
-    else {
-      // writes twice to the log file, but this isn't a big problem
-      consoleLog.info(format(msg, screen))
-      log.info(format(msg, logged))
-    }
-  }
+    log.info(msg)
 
   def warn(msg: String): Unit =
-    consoleLog.warn(msg)
-
-  def warn(msg: String, t: Truncatable): Unit = {
-    val (screen, logged) = t.strings
-    if (screen == logged)
-      consoleLog.warn(format(msg, screen))
-    else {
-      // writes twice to the log file, but this isn't a big problem
-      consoleLog.warn(format(msg, screen))
-      log.warn(format(msg, logged))
-    }
-  }
+    log.warn(msg)
 
   def error(msg: String): Unit =
-    consoleLog.error(msg)
+    log.error(msg)
 }

hail/hail/src/is/hail/utils/LoggerOutputStream.scala

@@ -7,9 +7,11 @@ import is.hail.types.virtual.Type
 import scala.jdk.CollectionConverters._
 
 import java.io.{InputStream, OutputStream}
-import java.util.Properties
 
-import org.apache.log4j.{LogManager, PropertyConfigurator}
+import org.apache.logging.log4j.Level
+import org.apache.logging.log4j.core.appender.ConsoleAppender
+import org.apache.logging.log4j.core.config.Configurator
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory
 import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods
 
@@ -137,38 +139,58 @@ trait Py4jUtils extends Logging {
     "%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1}: %p: %m%n"
 
   def configureLogging(logFile: String, quiet: Boolean, append: Boolean): Unit = {
-    org.apache.log4j.helpers.LogLog.setInternalDebugging(true)
-    org.apache.log4j.helpers.LogLog.setQuietMode(false)
-    val logProps = new Properties()
-
-    // uncomment to see log4j LogLog output:
-    // logProps.put("log4j.debug", "true")
-    logProps.put("log4j.rootLogger", "INFO, logfile")
-    logProps.put("log4j.appender.logfile", "org.apache.log4j.FileAppender")
-    logProps.put("log4j.appender.logfile.append", append.toString)
-    logProps.put("log4j.appender.logfile.file", logFile)
-    logProps.put("log4j.appender.logfile.threshold", "INFO")
-    logProps.put("log4j.appender.logfile.layout", "org.apache.log4j.PatternLayout")
-    logProps.put("log4j.appender.logfile.layout.ConversionPattern", LogFormat)
-
-    if (!quiet) {
-      logProps.put("log4j.logger.Hail", "INFO, HailConsoleAppender, HailSocketAppender")
-      logProps.put("log4j.appender.HailConsoleAppender", "org.apache.log4j.ConsoleAppender")
-      logProps.put("log4j.appender.HailConsoleAppender.target", "System.err")
-      logProps.put("log4j.appender.HailConsoleAppender.layout", "org.apache.log4j.PatternLayout")
-    } else
-      logProps.put("log4j.logger.Hail", "INFO, HailSocketAppender")
-
-    logProps.put("log4j.appender.HailSocketAppender", "is.hail.utils.StringSocketAppender")
-    logProps.put("log4j.appender.HailSocketAppender.layout", "org.apache.log4j.PatternLayout")
-
-    LogManager.resetConfiguration()
-    PropertyConfigurator.configure(logProps)
+    val configBuilder =
+      ConfigurationBuilderFactory
+        .newConfigurationBuilder()
+        .setConfigurationName("Hail")
+
+    val layout =
+      configBuilder
+        .newLayout("PatternLayout")
+        .addAttribute("pattern", LogFormat)
+
+    val fileAppender =
+      configBuilder
+        .newAppender("LOGFILE", "File")
+        .addAttribute("append", append.toString)
+        .addAttribute("fileName", logFile)
+        .add(layout)
+
+    val consoleAppender =
+      configBuilder
+        .newAppender("CONSOLE", "Console")
+        .addAttribute("target", ConsoleAppender.Target.SYSTEM_ERR)
+        .add(layout)
+
+    val rootLogger =
+      configBuilder
+        .newRootLogger(Level.INFO)
+        .add(
+          configBuilder
+            .newAppenderRef(fileAppender.getName)
+            .addAttribute("level", Level.INFO)
+        )
+        .add(
+          configBuilder
+            .newAppenderRef(consoleAppender.getName)
+            .addAttribute("level", if (!quiet) Level.WARN else Level.OFF)
+        )
+
+    val sparkLogger =
+      configBuilder
+        .newLogger("org.apache.spark", Level.WARN)
+        .add(configBuilder.newAppenderRef(fileAppender.getName))
+
+    Configurator.reconfigure(
+      configBuilder
+        .add(fileAppender)
+        .add(consoleAppender)
+        .add(rootLogger)
+        .add(sparkLogger)
+        .build(false)
+    )
   }
 
-  def addSocketAppender(hostname: String, port: Int): Unit =
-    StringSocketAppender.get().connect(hostname, port, LogFormat)
-
   def logWarn(msg: String): Unit =
     warn(msg)