Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 0 additions & 22 deletions hail/build.mill
Original file line number Diff line number Diff line change
Expand Up @@ -147,27 +147,6 @@ trait HailModule extends ScalaModule with ScalafmtModule with ScalafixModule { o
if (scalaVersion().startsWith("2.13")) {
finalOptions ++
Seq(
// collection types
// type Traversable in package scala is deprecated (since 2.13.0): Use Iterable instead of Traversable
"-Wconf:cat=deprecation&msg=type Traversable in package scala is deprecated:s",
// type TraversableOnce in package scala is deprecated (since 2.13.0): Use IterableOnce instead of TraversableOnce
"-Wconf:cat=deprecation&msg=type TraversableOnce:s",
// type BufferedIterator in package scala is deprecated (since 2.13.0): Use scala.collection.BufferedIterator instead of scala.BufferedIterator
"-Wconf:cat=deprecation&msg=type BufferedIterator in package scala is deprecated:s",
// class Stream in package immutable is deprecated (since 2.13.0): Use LazyList (which is fully lazy) instead of Stream (which has a lazy tail only)
"-Wconf:cat=deprecation&msg=Stream:s",
// type Growable in package generic is deprecated (since 2.13.0): Growable was moved from collection.generic to collection.mutable
"-Wconf:cat=deprecation&msg=type Growable in package generic is deprecated:s",
// type GenTraversableOnce in package collection is deprecated (since 2.13.0): Gen* collection types have been removed
"-Wconf:cat=deprecation&msg=type GenTraversableOnce in package collection is deprecated:s",
// trait MultiMap in package mutable is deprecated (since 2.13.0): Use a scala.collection.mutable.MultiDict in the scala-collection-contrib module
"-Wconf:cat=deprecation&msg=trait MultiMap in package mutable is deprecated:s",
// inheritance from class HashMap in package mutable is deprecated (since 2.13.0): HashMap will be made final; use .withDefault for the common use case of computing a default value
"-Wconf:cat=deprecation&msg=inheritance from class HashMap in package mutable is deprecated:s",
// object JavaConverters in package collection is deprecated (since 2.13.0): Use `scala.jdk.CollectionConverters` instead
"-Wconf:cat=deprecation&msg=object JavaConverters in package collection is deprecated:s",
// object ImplicitConversions in package convert is deprecated (since 2.13.0): Use `scala.jdk.CollectionConverters` instead
"-Wconf:cat=deprecation&msg=object ImplicitConversions in package convert is deprecated:s",
// generic to immutable default
// collection methods
// method copyArrayToImmutableIndexedSeq in class LowPriorityImplicits2 is deprecated (since 2.13.0): implicit conversions from Array to immutable.IndexedSeq are implemented by copying; use `toIndexedSeq` explicitly if you want to copy, or use the more efficient non-copying ArraySeq.unsafeWrapArray
Expand Down Expand Up @@ -317,7 +296,6 @@ trait RootHailModule extends CrossScalaModule with HailModule { outer =>
Rule.Exclude("META-INF/INDEX.LIST"),
Rule.ExcludePattern("scala/(?!collection/compat).*"),
Rule.AppendPattern("META-INF/services/.*", "\n"),
// Rule.Relocate("scala.collection.compat.**", "is.hail.relocated.@0"),
Rule.Relocate("breeze.**", "is.hail.relocated.@0"),
Rule.Relocate("com.google.cloud.**", "is.hail.relocated.@0"),
Rule.Relocate("com.google.common.**", "is.hail.relocated.@0"),
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package is.hail.utils.compat
package is.hail.utils.compat.mutable

import scala.collection.generic.Growable
import scala.collection.generic

trait GrowableCompat[-A] extends Growable[A] {
trait Growable[-A] extends generic.Growable[A] {
override def +=(elem: A): this.type = addOne(elem)
protected def addOne(elem: A): this.type
def knownSize: Int = -1
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package is.hail.utils.compat.mutable

import scala.collection.generic

trait Shrinkable[-A] extends generic.Shrinkable[A] {
override def -=(elem: A): this.type = subtractOne(elem)
protected def subtractOne(elem: A): this.type
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,5 @@ package is.hail.utils.compat

package object immutable {
type ArraySeq[+A] = scala.collection.immutable.ArraySeq[A]

val ArraySeq: scala.collection.compat.immutable.ArraySeq.type =
scala.collection.compat.immutable.ArraySeq
val ArraySeq = scala.collection.immutable.ArraySeq
}
6 changes: 6 additions & 0 deletions hail/hail/src-2.13/is/hail/utils/compat/mutable/package.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package is.hail.utils.compat

package object mutable {
type Growable[-A] = scala.collection.mutable.Growable[A]
type Shrinkable[-A] = scala.collection.mutable.Shrinkable[A]
}
8 changes: 0 additions & 8 deletions hail/hail/src-2.13/is/hail/utils/compat/package.scala

This file was deleted.

2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/annotations/OrderedRVIterator.scala
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import is.hail.backend.HailStateManager
import is.hail.rvd.{RVDContext, RVDType}
import is.hail.types.physical.PInterval
import is.hail.utils._
import is.hail.utils.compat.mutable.Growable

import scala.collection.generic.Growable
import scala.collection.mutable

object OrderedRVIterator {
Expand Down
4 changes: 2 additions & 2 deletions hail/hail/src/is/hail/annotations/WritableRegionValue.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package is.hail.annotations
import is.hail.backend.HailStateManager
import is.hail.rvd.RVDContext
import is.hail.types.physical.{PStruct, PType}
import is.hail.utils.compat.GrowableCompat
import is.hail.utils.compat.mutable.Growable

import scala.collection.mutable.{ArrayBuffer, PriorityQueue}

Expand Down Expand Up @@ -129,7 +129,7 @@ class RegionValuePriorityQueue(
}

class RegionValueArrayBuffer(val t: PType, region: Region, sm: HailStateManager)
extends Iterable[RegionValue] with GrowableCompat[RegionValue] {
extends Iterable[RegionValue] with Growable[RegionValue] {

val value = RegionValue(region, 0)

Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/backend/service/ServiceBackend.scala
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ class ServiceBackend(
val (failures, successes) =
Await.result(
Future
.traverse(failedJobs.map(_.take(1)) append succeededJobs) { jobs =>
.traverse(failedJobs.map(_.take(1)) lazyAppendedAll succeededJobs) { jobs =>
readPartitionOutputs(jobs.map(_.job_id - startJobId))
},
Duration.Inf,
Expand Down
6 changes: 3 additions & 3 deletions hail/hail/src/is/hail/expr/ir/ExtractIntervalFilters.scala
Original file line number Diff line number Diff line change
Expand Up @@ -712,11 +712,11 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) {
private def intervalsFromLiteral(lit: Any, ordering: Ordering[Any], wrapped: Boolean): KeySet =
(lit: @unchecked) match {
case x: Map[_, _] => intervalsFromCollection(x.keys, ordering, wrapped)
case x: Traversable[_] => intervalsFromCollection(x, ordering, wrapped)
case x: Iterable[_] => intervalsFromCollection(x, ordering, wrapped)
}

private def intervalsFromCollection(
lit: Traversable[Any],
lit: Iterable[Any],
ordering: Ordering[Any],
wrapped: Boolean,
): KeySet =
Expand All @@ -733,7 +733,7 @@ class ExtractIntervalFilters(ctx: ExecuteContext, keyType: TStruct) {
TInt32.ordering(null).toOrdering.asInstanceOf[Ordering[Integer]]
)
.flatMap(getIntervalFromContig(_, rg))
case x: Traversable[_] => x.asInstanceOf[Traversable[String]].toArray.toFastSeq
case x: Iterable[_] => x.asInstanceOf[Iterable[String]].toArray.toFastSeq
.sortBy(rg.contigsIndex.get(_))(
TInt32.ordering(null).toOrdering.asInstanceOf[Ordering[Integer]]
)
Expand Down
5 changes: 2 additions & 3 deletions hail/hail/src/is/hail/expr/ir/functions/Functions.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,15 @@ object IRFunctionRegistry {
val irRegistry: mutable.Map[String, mutable.Map[IRFunctionSignature, IRFunctionImplementation]] =
new mutable.HashMap()

val jvmRegistry: mutable.MultiMap[String, JVMFunction] =
new mutable.HashMap[String, mutable.Set[JVMFunction]] with mutable.MultiMap[String, JVMFunction]
val jvmRegistry: mutable.Map[String, mutable.Set[JVMFunction]] = mutable.HashMap.empty

private[this] def requireJavaIdentifier(name: String): Unit =
if (!isJavaIdentifier(name))
throw new IllegalArgumentException(s"Illegal function name, not Java identifier: $name")

def addJVMFunction(f: JVMFunction): Unit = {
requireJavaIdentifier(f.name)
jvmRegistry.addBinding(f.name, f)
jvmRegistry.getOrElseUpdate(f.name, mutable.Set.empty) += f
}

def addIR(
Expand Down
2 changes: 2 additions & 0 deletions hail/hail/src/is/hail/expr/ir/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import is.hail.types.virtual._
import is.hail.types.virtual.TIterable.elementType
import is.hail.utils._

import scala.collection.BufferedIterator

import java.util.UUID

package object ir {
Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/io/ElasticsearchConnector.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package is.hail.io

import scala.collection.JavaConverters._
import scala.collection.Map
import scala.jdk.CollectionConverters._

import org.apache.spark
import org.elasticsearch.spark.sql._
Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/io/avro/AvroPartitionReader.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import is.hail.types.virtual._
import is.hail.utils.compat._
import is.hail.utils.compat.immutable.ArraySeq

import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._

import java.io.InputStream

Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/io/fs/AzureStorageFS.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ import is.hail.shadedazure.com.azure.storage.blob.models.{
import is.hail.shadedazure.com.azure.storage.blob.specialized.BlockBlobClient
import is.hail.utils.FastSeq

import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._

import java.io.{ByteArrayOutputStream, FileNotFoundException, OutputStream}
import java.nio.file.{Path, Paths}
Expand Down
3 changes: 1 addition & 2 deletions hail/hail/src/is/hail/io/fs/FS.scala
Original file line number Diff line number Diff line change
Expand Up @@ -630,8 +630,7 @@ trait FS extends Serializable with Logging {
}
}

def writeTable(filename: String, lines: Traversable[String], header: Option[String] = None)
: Unit = {
def writeTable(filename: String, lines: Iterable[String], header: Option[String] = None): Unit = {
using(new OutputStreamWriter(create(filename))) { fw =>
header.foreach { h =>
fw.write(h)
Expand Down
6 changes: 3 additions & 3 deletions hail/hail/src/is/hail/io/reference/LiftOver.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import is.hail.io.reference.LiftOver.MinMatchDefault
import is.hail.utils.{fatal, using, Interval}
import is.hail.variant.{Locus, ReferenceGenome}

import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `map AsScala`}
import scala.jdk.CollectionConverters._

object LiftOver {
def apply(fs: FS, chainFile: String): LiftOver = {
Expand Down Expand Up @@ -65,9 +65,9 @@ class LiftOver private (chainFile: String) extends Serializable {
}

def checkChainFile(srcRG: ReferenceGenome, destRG: ReferenceGenome): Unit =
asJava.getContigMap.foreach { case (srcContig, destContigs) =>
asJava.getContigMap.asScala.foreach { case (srcContig, destContigs) =>
srcRG.checkContig(srcContig)
destContigs.foreach(destRG.checkContig)
destContigs.asScala.foreach(destRG.checkContig)
}

def restore(fs: FS): Unit = {
Expand Down
3 changes: 2 additions & 1 deletion hail/hail/src/is/hail/io/vcf/LoadVCF.scala
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ import is.hail.variant._

import scala.annotation.meta.param
import scala.annotation.switch
import scala.collection.JavaConverters._
import scala.collection.BufferedIterator
import scala.collection.compat._
import scala.jdk.CollectionConverters._

import htsjdk.variant.vcf._
import org.apache.spark.{Partition, TaskContext}
Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/methods/Nirvana.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import is.hail.types.virtual._
import is.hail.utils._
import is.hail.variant.{Locus, RegionValueVariant}

import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.jdk.CollectionConverters._

import java.io.{FileInputStream, IOException}
import java.util.Properties
Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/methods/VEP.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import is.hail.types.virtual._
import is.hail.utils._
import is.hail.variant.{Locus, RegionValueVariant, VariantMethods}

import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.jdk.CollectionConverters._

import com.fasterxml.jackson.core.JsonParseException
import org.apache.spark.sql.Row
Expand Down
3 changes: 1 addition & 2 deletions hail/hail/src/is/hail/rvd/KeyedRVD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ import is.hail.sparkextras._
import is.hail.types.physical.PStruct
import is.hail.types.virtual.TInterval
import is.hail.utils._

import scala.collection.generic.Growable
import is.hail.utils.compat.mutable.Growable

class KeyedRVD(val rvd: RVD, val key: Int) {
require(key <= rvd.typ.key.length && key >= 0)
Expand Down
8 changes: 4 additions & 4 deletions hail/hail/src/is/hail/services/BatchClient.scala
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import is.hail.services.oauth2.CloudCredentials
import is.hail.services.requests.Requester
import is.hail.utils._

import scala.collection.immutable.Stream.cons
import scala.collection.compat.immutable.LazyList
import scala.util.Random

import java.net.{URL, URLEncoder}
Expand Down Expand Up @@ -252,9 +252,9 @@ case class BatchClient private (req: Requester) extends Logging with AutoCloseab
JobStateDeserializer +
JobListEntryDeserializer

private[this] def paginated[S, A](s0: S)(f: S => (A, S)): Stream[A] = {
private[this] def paginated[S, A](s0: S)(f: S => (A, S)): LazyList[A] = {
val (a, s1) = f(s0)
cons(a, paginated(s1)(f))
LazyList.cons(a, paginated(s1)(f))
}

def newBatch(createRequest: BatchRequest): Int = {
Expand Down Expand Up @@ -287,7 +287,7 @@ case class BatchClient private (req: Requester) extends Logging with AutoCloseab
.extract[JobGroupResponse]

def getJobGroupJobs(batchId: Int, jobGroupId: Int, status: Option[JobState] = None)
: Stream[IndexedSeq[JobListEntry]] = {
: LazyList[IndexedSeq[JobListEntry]] = {
val q = status.map(s => s"state=${s.toString.toLowerCase}").getOrElse("")
paginated(Some(0): Option[Int]) {
case Some(jobId) =>
Expand Down
2 changes: 1 addition & 1 deletion hail/hail/src/is/hail/services/oauth2.scala
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import is.hail.shadedazure.com.azure.identity.{
}
import is.hail.utils.{defaultJSONFormats, using}

import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._

import java.io.Serializable
import java.nio.file.{Files, Path}
Expand Down
5 changes: 3 additions & 2 deletions hail/hail/src/is/hail/sparkextras/ContextRDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import is.hail.backend.spark.{SparkBackend, SparkTaskContext}
import is.hail.rvd.RVDContext
import is.hail.utils._

import scala.collection.compat._
import scala.reflect.ClassTag

import org.apache.spark._
Expand Down Expand Up @@ -182,7 +183,7 @@ class ContextRDD[T: ClassTag](
def filter(f: T => Boolean): ContextRDD[T] =
mapPartitions(_.filter(f), preservesPartitioning = true)

def flatMap[U: ClassTag](f: T => TraversableOnce[U]): ContextRDD[U] =
def flatMap[U: ClassTag](f: T => IterableOnce[U]): ContextRDD[U] =
mapPartitions(_.flatMap(f))

def mapPartitions[U: ClassTag](
Expand All @@ -203,7 +204,7 @@ class ContextRDD[T: ClassTag](
def cfilter(f: (RVDContext, T) => Boolean): ContextRDD[T] =
cmapPartitions((c, it) => it.filter(f(c, _)), true)

def cflatMap[U: ClassTag](f: (RVDContext, T) => TraversableOnce[U]): ContextRDD[U] =
def cflatMap[U: ClassTag](f: (RVDContext, T) => IterableOnce[U]): ContextRDD[U] =
cmapPartitions((c, it) => it.flatMap(f(c, _)))

def cmapPartitions[U: ClassTag](
Expand Down
12 changes: 7 additions & 5 deletions hail/hail/src/is/hail/stats/LeveneHaldane.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package is.hail.stats

import is.hail.utils._

import scala.collection.compat.immutable.LazyList

import org.apache.commons.math3.distribution.AbstractIntegerDistribution
import org.apache.commons.math3.random.RandomGenerator

Expand All @@ -12,8 +14,8 @@ class LeveneHaldane(
val n: Int,
val nA: Int,
val mode: Int,
pRU: Stream[Double],
pLU: Stream[Double],
pRU: LazyList[Double],
pLU: LazyList[Double],
pN: Double,
rng: RandomGenerator,
) extends AbstractIntegerDistribution(rng) {
Expand Down Expand Up @@ -69,7 +71,7 @@ class LeveneHaldane(
0.0
else {
val cutoff = p0U * 0.5e-16
def mpU(s: Stream[Double]): Double = {
def mpU(s: LazyList[Double]): Double = {
val (sEq, sLess) =
s.dropWhile(D_>(_, p0U, tolerance = 1.0e-12)).span(D_==(_, p0U, tolerance = 1.0e-12))
0.5 * sEq.sum + sLess.takeWhile(_ > cutoff).sum
Expand Down Expand Up @@ -100,9 +102,9 @@ object LeveneHaldane {
(nA + 1.0) * (nB + 1) / (2 * n + 3)
).toInt

def pRUfrom(nAB: Int, p: Double): Stream[Double] =
def pRUfrom(nAB: Int, p: Double): LazyList[Double] =
p #:: pRUfrom(nAB + 2, p * (nA - nAB) * (nB - nAB) / ((nAB + 2.0) * (nAB + 1)))
def pLUfrom(nAB: Int, p: Double): Stream[Double] =
def pLUfrom(nAB: Int, p: Double): LazyList[Double] =
p #:: pLUfrom(nAB - 2, p * nAB * (nAB - 1) / ((nA - nAB + 2.0) * (nB - nAB + 2)))

// Unnormalized probability mass function going right, respectively left, from the mode by 2s
Expand Down
Loading