Skip to content

Commit ff8b149

Browse files
ehighamehigham
authored
[query] move configureLogging and version out of HailContext (#15098)
`HailContext` jihad: - moves logging configuration from `HailContext` to a dedicated `Logging` object. - removes references to `HailContext.version`​ This change cannot impact the Hail Batch instance as deployed by Broad Institute in GCP
1 parent 79d3b20 commit ff8b149

File tree

8 files changed

+50
-61
lines changed

8 files changed

+50
-61
lines changed

‎hail/hail/src/is/hail/HailContext.scala‎

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,10 @@ import is.hail.expr.ir.functions.IRFunctionRegistry
66
import is.hail.io.fs.FS
77
import is.hail.utils._
88

9-
import java.util.Properties
10-
11-
import org.apache.log4j.{LogManager, PropertyConfigurator}
129
import org.apache.spark._
1310

1411
object HailContext {
1512

16-
val logFormat: String = "%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1}: %p: %m%n"
17-
1813
private var theContext: HailContext = _
1914

2015
def get: HailContext = synchronized {
@@ -23,36 +18,6 @@ object HailContext {
2318
theContext
2419
}
2520

26-
def configureLogging(logFile: String, quiet: Boolean, append: Boolean): Unit = {
27-
org.apache.log4j.helpers.LogLog.setInternalDebugging(true)
28-
org.apache.log4j.helpers.LogLog.setQuietMode(false)
29-
val logProps = new Properties()
30-
31-
// uncomment to see log4j LogLog output:
32-
// logProps.put("log4j.debug", "true")
33-
logProps.put("log4j.rootLogger", "INFO, logfile")
34-
logProps.put("log4j.appender.logfile", "org.apache.log4j.FileAppender")
35-
logProps.put("log4j.appender.logfile.append", append.toString)
36-
logProps.put("log4j.appender.logfile.file", logFile)
37-
logProps.put("log4j.appender.logfile.threshold", "INFO")
38-
logProps.put("log4j.appender.logfile.layout", "org.apache.log4j.PatternLayout")
39-
logProps.put("log4j.appender.logfile.layout.ConversionPattern", HailContext.logFormat)
40-
41-
if (!quiet) {
42-
logProps.put("log4j.logger.Hail", "INFO, HailConsoleAppender, HailSocketAppender")
43-
logProps.put("log4j.appender.HailConsoleAppender", "org.apache.log4j.ConsoleAppender")
44-
logProps.put("log4j.appender.HailConsoleAppender.target", "System.err")
45-
logProps.put("log4j.appender.HailConsoleAppender.layout", "org.apache.log4j.PatternLayout")
46-
} else
47-
logProps.put("log4j.logger.Hail", "INFO, HailSocketAppender")
48-
49-
logProps.put("log4j.appender.HailSocketAppender", "is.hail.utils.StringSocketAppender")
50-
logProps.put("log4j.appender.HailSocketAppender.layout", "org.apache.log4j.PatternLayout")
51-
52-
LogManager.resetConfiguration()
53-
PropertyConfigurator.configure(logProps)
54-
}
55-
5621
def checkJavaVersion(): Unit = {
5722
val javaVersion = raw"(\d+)\.(\d+)\.(\d+).*".r
5823
val versionString = System.getProperty("java.version")
@@ -97,7 +62,7 @@ object HailContext {
9762

9863
theContext = new HailContext(backend)
9964

100-
info(s"Running Hail version ${theContext.version}")
65+
info(s"Running Hail version $HAIL_PRETTY_VERSION")
10166

10267
theContext
10368
}
@@ -115,8 +80,6 @@ class HailContext private (
11580
) {
11681
def stop(): Unit = HailContext.stop()
11782

118-
def version: String = is.hail.HAIL_PRETTY_VERSION
119-
12083
private[this] def fileAndLineCounts(
12184
fs: FS,
12285
regex: String,

‎hail/hail/src/is/hail/backend/local/LocalBackend.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package is.hail.backend.local
22

3-
import is.hail.{CancellingExecutorService, HailContext}
3+
import is.hail.CancellingExecutorService
44
import is.hail.asm4s._
55
import is.hail.backend._
66
import is.hail.expr.Validate
@@ -46,7 +46,7 @@ object LocalBackend extends Backend {
4646
skipLoggingConfiguration: Boolean = false,
4747
): LocalBackend.type =
4848
synchronized {
49-
if (!skipLoggingConfiguration) HailContext.configureLogging(logFile, quiet, append)
49+
if (!skipLoggingConfiguration) Logging.configureLogging(logFile, quiet, append)
5050
this
5151
}
5252

‎hail/hail/src/is/hail/backend/spark/SparkBackend.scala‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package is.hail.backend.spark
22

3-
import is.hail.HailContext
43
import is.hail.annotations._
54
import is.hail.asm4s._
65
import is.hail.backend._
@@ -199,11 +198,6 @@ object SparkBackend {
199198
"org.apache.hadoop.io.compress.GzipCodec",
200199
)
201200

202-
/** If a SparkBackend has already been initialized, this function returns it regardless of the
203-
* parameters with which it was initialized.
204-
*
205-
* Otherwise, it initializes and returns a new HailContext.
206-
*/
207201
def getOrCreate(
208202
sc: SparkContext = null,
209203
appName: String = "Hail",
@@ -260,7 +254,7 @@ object SparkBackend {
260254
require(theSparkBackend == null)
261255

262256
if (!skipLoggingConfiguration)
263-
HailContext.configureLogging(logFile, quiet, append)
257+
Logging.configureLogging(logFile, quiet, append)
264258

265259
var sc1 = sc
266260
if (sc1 == null)

‎hail/hail/src/is/hail/utils/Logging.scala‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package is.hail.utils
22

3-
import org.apache.log4j.{LogManager, Logger}
3+
import java.util.Properties
4+
5+
import org.apache.log4j.{LogManager, Logger, PropertyConfigurator}
46

57
object LogHelper {
68
// exposed more directly for generated code
@@ -56,3 +58,40 @@ trait Logging {
5658
def error(msg: String): Unit =
5759
consoleLog.error(msg)
5860
}
61+
62+
object Logging {
63+
64+
val LogFormat: String =
65+
"%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1}: %p: %m%n"
66+
67+
def configureLogging(logFile: String, quiet: Boolean, append: Boolean): Unit = {
68+
org.apache.log4j.helpers.LogLog.setInternalDebugging(true)
69+
org.apache.log4j.helpers.LogLog.setQuietMode(false)
70+
val logProps = new Properties()
71+
72+
// uncomment to see log4j LogLog output:
73+
// logProps.put("log4j.debug", "true")
74+
logProps.put("log4j.rootLogger", "INFO, logfile")
75+
logProps.put("log4j.appender.logfile", "org.apache.log4j.FileAppender")
76+
logProps.put("log4j.appender.logfile.append", append.toString)
77+
logProps.put("log4j.appender.logfile.file", logFile)
78+
logProps.put("log4j.appender.logfile.threshold", "INFO")
79+
logProps.put("log4j.appender.logfile.layout", "org.apache.log4j.PatternLayout")
80+
logProps.put("log4j.appender.logfile.layout.ConversionPattern", LogFormat)
81+
82+
if (!quiet) {
83+
logProps.put("log4j.logger.Hail", "INFO, HailConsoleAppender, HailSocketAppender")
84+
logProps.put("log4j.appender.HailConsoleAppender", "org.apache.log4j.ConsoleAppender")
85+
logProps.put("log4j.appender.HailConsoleAppender.target", "System.err")
86+
logProps.put("log4j.appender.HailConsoleAppender.layout", "org.apache.log4j.PatternLayout")
87+
} else
88+
logProps.put("log4j.logger.Hail", "INFO, HailSocketAppender")
89+
90+
logProps.put("log4j.appender.HailSocketAppender", "is.hail.utils.StringSocketAppender")
91+
logProps.put("log4j.appender.HailSocketAppender.layout", "org.apache.log4j.PatternLayout")
92+
93+
LogManager.resetConfiguration()
94+
PropertyConfigurator.configure(logProps)
95+
}
96+
97+
}

‎hail/hail/src/is/hail/utils/Py4jUtils.scala‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package is.hail.utils
22

3-
import is.hail.HailContext
43
import is.hail.expr.JSONAnnotationImpex
54
import is.hail.io.fs.{FS, FileListEntry, FileStatus, SeekableDataInputStream}
65
import is.hail.types.virtual.Type
@@ -133,8 +132,7 @@ trait Py4jUtils {
133132
}
134133

135134
def addSocketAppender(hostname: String, port: Int): Unit =
136-
StringSocketAppender.get()
137-
.connect(hostname, port, HailContext.logFormat)
135+
StringSocketAppender.get().connect(hostname, port, Logging.LogFormat)
138136

139137
def logWarn(msg: String): Unit =
140138
warn(msg)

‎hail/hail/src/is/hail/utils/package.scala‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,7 @@ import java.security.SecureRandom
2020
import java.text.SimpleDateFormat
2121
import java.util
2222
import java.util.{Base64, Date}
23-
import java.util.concurrent.{
24-
AbstractExecutorService, Callable, CancellationException, ExecutorCompletionService,
25-
ExecutorService, RunnableFuture, TimeUnit,
26-
}
23+
import java.util.concurrent._
2724
import java.util.concurrent.atomic.AtomicBoolean
2825

2926
import com.google.common.util.concurrent.AbstractFuture
@@ -850,7 +847,7 @@ package object utils
850847
using(new OutputStreamWriter(fs.create(path + "/README.txt"))) { out =>
851848
out.write(
852849
s"""This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
853-
| Written with version ${HailContext.get.version}
850+
| Written with version $HAIL_PRETTY_VERSION
854851
| Created at ${dateFormat.format(new Date())}""".stripMargin
855852
)
856853
}

‎hail/hail/test/src/is/hail/HailSuite.scala‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@ import is.hail.asm4s.HailClassLoader
66
import is.hail.backend.{Backend, ExecuteContext, OwningTempFileManager}
77
import is.hail.backend.spark.SparkBackend
88
import is.hail.expr.ir._
9-
import is.hail.expr.ir.defs.{
10-
BlockMatrixCollect, Cast, MakeTuple, NDArrayRef, Ref, StreamMap, ToArray,
11-
}
9+
import is.hail.expr.ir.defs._
1210
import is.hail.expr.ir.lowering.IrMetadata
1311
import is.hail.io.fs.{FS, HadoopFS}
1412
import is.hail.rvd.RVD
@@ -54,7 +52,7 @@ class HailSuite extends TestNGSuite with TestUtils {
5452

5553
@BeforeSuite
5654
def setupHailContext(): Unit = {
57-
HailContext.configureLogging("/tmp/hail.log", quiet = false, append = false)
55+
Logging.configureLogging("/tmp/hail.log", quiet = false, append = false)
5856
RVD.CheckRvdKeyOrderingForTesting = true
5957
val backend = SparkBackend(
6058
sc = new SparkContext(

‎hail/python/hail/backend/py4j_backend.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ def decode_bytearray(encoded):
210210
# Maybe it does its own patch?
211211
install_exception_handler()
212212

213-
jar_version = self._jhc.version()
213+
jar_version = scala_package_object(self._hail_package).HAIL_PRETTY_VERSION()
214214
if jar_version != __version__:
215215
raise RuntimeError(
216216
f"Hail version mismatch between JAR and Python library\n"

0 commit comments

Comments
 (0)