[query] remove unnecessary references to HailContext

ehigham · ehigham · commit c75ea751d3df · 2025-08-02T00:21:45.000-04:00
diff --git a/hail/hail/src/is/hail/HailContext.scala b/hail/hail/src/is/hail/HailContext.scala
@@ -4,8 +4,6 @@ import is.hail.backend.Backend
 import is.hail.backend.spark.SparkBackend
 import is.hail.expr.ir.functions.IRFunctionRegistry
 import is.hail.io.fs.FS
-import is.hail.io.vcf._
-import is.hail.types.virtual._
 import is.hail.utils._
 
 import scala.reflect.ClassTag
@@ -17,9 +15,6 @@ import org.apache.log4j.{LogManager, PropertyConfigurator}
 import org.apache.spark._
 import org.apache.spark.executor.InputMetrics
 import org.apache.spark.rdd.RDD
-import org.json4s.Extraction
-import org.json4s.jackson.JsonMethods
-import sourcecode.Enclosing
 
 case class FilePartition(index: Int, file: String) extends Partition
 
@@ -42,8 +37,6 @@ object HailContext {
 
   def backend: Backend = get.backend
 
-  def sparkBackend(implicit E: Enclosing): SparkBackend = get.backend.asSpark
-
   def configureLogging(logFile: String, quiet: Boolean, append: Boolean): Unit = {
     org.apache.log4j.helpers.LogLog.setInternalDebugging(true)
     org.apache.log4j.helpers.LogLog.setQuietMode(false)
@@ -194,12 +187,4 @@ class HailContext private (
     : Array[(String, Array[String])] =
     fileAndLineCounts(fs: FS, regex, files, maxLines).mapValues(_.map(_.value)).toArray
 
-  def parseVCFMetadata(fs: FS, file: String): Map[String, Map[String, Map[String, String]]] =
-    LoadVCF.parseHeaderMetadata(fs, Set.empty, TFloat64, file)
-
-  def pyParseVCFMetadataJSON(fs: FS, file: String): String = {
-    val metadata = LoadVCF.parseHeaderMetadata(fs, Set.empty, TFloat64, file)
-    implicit val formats = defaultJSONFormats
-    JsonMethods.compact(Extraction.decompose(metadata))
-  }
 }
diff --git a/hail/hail/src/is/hail/HailFeatureFlags.scala b/hail/hail/src/is/hail/HailFeatureFlags.scala
@@ -2,12 +2,13 @@ package is.hail
 
 import is.hail.backend.ExecutionCache
 import is.hail.backend.spark.SparkBackend
-import is.hail.expr.ir.{Optimize, agg}
+import is.hail.expr.ir.{agg, Optimize}
 import is.hail.io.fs.RequesterPaysConfig
 import is.hail.types.encoded.EType
 import is.hail.utils._
 
 import scala.collection.mutable
+
 import org.json4s.JsonAST.{JArray, JObject, JString}
 
 object HailFeatureFlags {
diff --git a/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala b/hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala
@@ -217,7 +217,8 @@ final class Py4JQueryDriver(backend: Backend) extends Closeable {
   def pyToDF(s: String): DataFrame =
     withExecuteContext(selfContainedExecution = false) { ctx =>
       val tir = IRParser.parse_table_ir(ctx, s)
-      Interpret(tir, ctx).toDF()
+      val tv = Interpret(tir, ctx)
+      tv.toDF(ctx)
     }._1
 
   def pyReadMultipleMatrixTables(jsonQuery: String): util.List[MatrixIR] =
diff --git a/hail/hail/src/is/hail/backend/spark/SparkBackend.scala b/hail/hail/src/is/hail/backend/spark/SparkBackend.scala
@@ -87,7 +87,11 @@ object SparkBackend {
 
   private var theSparkBackend: SparkBackend = _
 
-  def sparkContext(implicit E: Enclosing): SparkContext = HailContext.sparkBackend.sc
+  def sparkContext(implicit E: Enclosing): SparkContext =
+    synchronized {
+      if (theSparkBackend == null) throw new IllegalStateException(E.value)
+      else theSparkBackend.sc
+    }
 
   def checkSparkCompatibility(jarVersion: String, sparkVersion: String): Unit = {
     def majorMinor(version: String): String = version.split("\\.", 3).take(2).mkString(".")
diff --git a/hail/hail/src/is/hail/expr/ir/TableValue.scala b/hail/hail/src/is/hail/expr/ir/TableValue.scala
@@ -1,6 +1,4 @@
 package is.hail.expr.ir
-
-import is.hail.HailContext
 import is.hail.annotations._
 import is.hail.asm4s._
 import is.hail.backend.ExecuteContext
@@ -367,8 +365,8 @@ case class TableValue(ctx: ExecuteContext, typ: TableType, globals: BroadcastRow
     )
   }
 
-  def toDF(): DataFrame =
-    HailContext.sparkBackend.sparkSession.createDataFrame(
+  def toDF(ctx: ExecuteContext): DataFrame =
+    ctx.backend.asSpark.sparkSession.createDataFrame(
       rvd.toRows,
       typ.rowType.schema.asInstanceOf[StructType],
     )
diff --git a/hail/hail/src/is/hail/methods/LinearRegression.scala b/hail/hail/src/is/hail/methods/LinearRegression.scala
@@ -1,6 +1,4 @@
 package is.hail.methods
-
-import is.hail.HailContext
 import is.hail.annotations._
 import is.hail.backend.ExecuteContext
 import is.hail.expr.ir.{IntArrayBuilder, MatrixValue, TableValue}
@@ -67,7 +65,7 @@ case class LinearRegressionRowsSingle(
 
     val Qty = Qt * y
 
-    val backend = HailContext.backend
+    val backend = ctx.backend
     val completeColIdxBc = backend.broadcast(completeColIdx)
     val yBc = backend.broadcast(y)
     val QtBc = backend.broadcast(Qt)
@@ -251,7 +249,7 @@ case class LinearRegressionRowsChained(
       ChainedLinregInput(n, y, completeColIdx, Qt, Qty, yyp, d)
     }
 
-    val bc = HailContext.backend.broadcast(bcData)
+    val bc = ctx.backend.broadcast(bcData)
     val nGroups = bcData.length
 
     val fullRowType = mv.rvd.rowPType
diff --git a/hail/hail/src/is/hail/methods/LogisticRegression.scala b/hail/hail/src/is/hail/methods/LogisticRegression.scala
@@ -1,6 +1,4 @@
 package is.hail.methods
-
-import is.hail.HailContext
 import is.hail.annotations._
 import is.hail.backend.ExecuteContext
 import is.hail.expr.ir.{IntArrayBuilder, MatrixValue, TableValue}
@@ -91,7 +89,7 @@ case class LogisticRegression(
       nullFit
     }
 
-    val backend = HailContext.backend
+    val backend = ctx.backend
     val completeColIdxBc = backend.broadcast(completeColIdx)
 
     val yVecsBc = backend.broadcast(yVecs)
diff --git a/hail/hail/src/is/hail/methods/MatrixExportEntriesByCol.scala b/hail/hail/src/is/hail/methods/MatrixExportEntriesByCol.scala
@@ -1,6 +1,4 @@
 package is.hail.methods
-
-import is.hail.HailContext
 import is.hail.annotations.{UnsafeIndexedSeq, UnsafeRow}
 import is.hail.backend.ExecuteContext
 import is.hail.backend.spark.SparkBackend
@@ -69,7 +67,7 @@ case class MatrixExportEntriesByCol(
       val extension = if (bgzip) ".tsv.bgz" else ".tsv"
       val localHeaderJsonInFile = headerJsonInFile
 
-      val colValuesJSON = HailContext.backend.broadcast(
+      val colValuesJSON = ctx.backend.broadcast(
         (startIdx until endIdx)
           .map(allColValuesJSON)
           .toArray
diff --git a/hail/hail/src/is/hail/methods/PCA.scala b/hail/hail/src/is/hail/methods/PCA.scala
@@ -1,6 +1,4 @@
 package is.hail.methods
-
-import is.hail.HailContext
 import is.hail.annotations._
 import is.hail.backend.ExecuteContext
 import is.hail.expr.ir.{MatrixValue, TableValue}
@@ -63,7 +61,7 @@ case class PCA(entryField: String, k: Int, computeLoadings: Boolean) extends Mat
     ) ++ TStruct("loadings" -> TArray(TFloat64)))
       .setRequired(true)
       .asInstanceOf[PStruct]
-    val rowKeysBc = HailContext.backend.broadcast(collectRowKeys())
+    val rowKeysBc = ctx.backend.broadcast(collectRowKeys())
     val localRowKeySignature = mv.typ.rowKeyStruct.types
 
     val crdd: ContextRDD[Long] = if (computeLoadings) {
diff --git a/hail/hail/src/is/hail/methods/PoissonRegression.scala b/hail/hail/src/is/hail/methods/PoissonRegression.scala
@@ -1,6 +1,4 @@
 package is.hail.methods
-
-import is.hail.HailContext
 import is.hail.annotations._
 import is.hail.backend.ExecuteContext
 import is.hail.expr.ir.{IntArrayBuilder, MatrixValue, TableValue}
@@ -69,7 +67,7 @@ case class PoissonRegression(
           "Newton iteration failed to converge"
       ))
 
-    val backend = HailContext.backend
+    val backend = ctx.backend
     val completeColIdxBc = backend.broadcast(completeColIdx)
 
     val yBc = backend.broadcast(y)
diff --git a/hail/hail/src/is/hail/methods/Skat.scala b/hail/hail/src/is/hail/methods/Skat.scala
@@ -1,6 +1,4 @@
 package is.hail.methods
-
-import is.hail.HailContext
 import is.hail.annotations.{Annotation, Region, UnsafeRow}
 import is.hail.backend.ExecuteContext
 import is.hail.expr.ir.{IntArrayBuilder, MatrixValue, TableValue}
@@ -202,9 +200,9 @@ case class Skat(
     }
 
     val (keyGsWeightRdd, _) =
-      computeKeyGsWeightRdd(mv, xField, completeColIdx, keyField, weightField)
+      computeKeyGsWeightRdd(ctx, mv, xField, completeColIdx, keyField, weightField)
 
-    val backend = HailContext.backend
+    val backend = ctx.backend
 
     def linearSkat(): RDD[Row] = {
       // fit null model
@@ -318,6 +316,7 @@ case class Skat(
   }
 
   def computeKeyGsWeightRdd(
+    ctx: ExecuteContext,
     mv: MatrixValue,
     xField: String,
     completeColIdx: Array[Int],
@@ -345,7 +344,7 @@ case class Skat(
     val fieldIdx = entryType.fieldIdx(xField)
 
     val n = completeColIdx.length
-    val completeColIdxBc = HailContext.backend.broadcast(completeColIdx)
+    val completeColIdxBc = ctx.backend.broadcast(completeColIdx)
 
     /* I believe no `boundary` is needed here because `mapPartitions` calls `run` which calls
      * `cleanupRegions`. */
diff --git a/hail/hail/test/src/is/hail/HailSuite.scala b/hail/hail/test/src/is/hail/HailSuite.scala
@@ -182,7 +182,7 @@ class HailSuite extends TestNGSuite with TestUtils {
     assert(t == TVoid || t.typeCheck(expected), s"$t, $expected")
 
     val filteredExecStrats: Set[ExecStrategy] =
-      if (HailContext.backend.isInstanceOf[SparkBackend])
+      if (backend.isInstanceOf[SparkBackend])
         execStrats
       else {
         info("skipping interpret and non-lowering compile steps on non-spark backend")
@@ -288,7 +288,7 @@ class HailSuite extends TestNGSuite with TestUtils {
   )(implicit execStrats: Set[ExecStrategy]
   ): scalatest.Assertion = {
     val filteredExecStrats: Set[ExecStrategy] =
-      if (HailContext.backend.isInstanceOf[SparkBackend]) execStrats
+      if (backend.isInstanceOf[SparkBackend]) execStrats
       else {
         info("skipping interpret and non-lowering compile steps on non-spark backend")
         execStrats.intersect(ExecStrategy.backendOnly)

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,4 @@`
`1`	`1`	`package is.hail.expr.ir`
`2`		`-`
`3`		`-import is.hail.HailContext`
`4`	`2`	`import is.hail.annotations._`
`5`	`3`	`import is.hail.asm4s._`
`6`	`4`	`import is.hail.backend.ExecuteContext`
`@@ -367,8 +365,8 @@ case class TableValue(ctx: ExecuteContext, typ: TableType, globals: BroadcastRow`
`367`	`365`	`)`
`368`	`366`	`}`
`369`	`367`
`370`		`- def toDF(): DataFrame =`
`371`		`- HailContext.sparkBackend.sparkSession.createDataFrame(`
	`368`	`+ def toDF(ctx: ExecuteContext): DataFrame =`
	`369`	`+ ctx.backend.asSpark.sparkSession.createDataFrame(`
`372`	`370`	`rvd.toRows,`
`373`	`371`	`typ.rowType.schema.asInstanceOf[StructType],`
`374`	`372`	`)`