Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SW-2646] Calculate Metrics on Arbitrary Dataset #2745

Open
wants to merge 38 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
753562d
[SW-2646] Calculate Metrics on Arbitrary Dataset
mn-mikke Nov 15, 2021
d8af501
Update api
mn-mikke Mar 18, 2022
5ed9b34
Remove unrelated tests
mn-mikke Mar 21, 2022
c50fffe
Remove unrelated logic
mn-mikke Mar 21, 2022
aeb48eb
Remove unrelated logic
mn-mikke Mar 21, 2022
e25801a
Remove extra dependency
mn-mikke Mar 21, 2022
32db73d
Remove extra dependency and return test back
mn-mikke Mar 21, 2022
a777a7d
Remove extra tests
mn-mikke Mar 21, 2022
66f0451
Update tests
mn-mikke Mar 21, 2022
38e5ab4
Fix tests
mn-mikke Mar 22, 2022
d118f74
Fix tests
mn-mikke Mar 25, 2022
3f4cf56
Python wrappers
mn-mikke Mar 28, 2022
3f2dabd
Fix multinomial tests
mn-mikke Mar 29, 2022
ef0670c
Remove prior distribution
mn-mikke Mar 29, 2022
6a4a15c
remove distribution option
mn-mikke Mar 29, 2022
624a627
spotless Apply
mn-mikke Mar 29, 2022
cefff34
dataType checks
mn-mikke Mar 29, 2022
75d1274
revert test changes in python
mn-mikke Mar 30, 2022
e6887bc
revert change in R
mn-mikke Mar 30, 2022
1708d9b
Add R classes
mn-mikke Mar 30, 2022
5240eba
Remove offset column from multinomial metrics
mn-mikke Mar 30, 2022
0dfc38e
Fix metric factory generatino
mn-mikke Mar 30, 2022
afd7676
Use original builders instead
mn-mikke Apr 4, 2022
756e169
Add python smoke tests
mn-mikke Apr 4, 2022
d1de562
Use master version
mn-mikke Apr 5, 2022
df87eb3
Update python API
mn-mikke Apr 5, 2022
87d434c
fix formatting
mn-mikke Apr 7, 2022
3002e21
fix python test
mn-mikke Apr 7, 2022
93fd203
Add R tests
mn-mikke Apr 8, 2022
c939266
add more conditions to python tests
mn-mikke Apr 8, 2022
da2c335
spotless apply
mn-mikke Apr 8, 2022
da8417c
Update metric calculation to work with just probabilities
mn-mikke Apr 12, 2022
45bc54e
Fix Python test
mn-mikke Apr 12, 2022
1965292
Fix R test
mn-mikke Apr 13, 2022
7e8f954
Address review comments from Bartosz
mn-mikke Apr 14, 2022
2be8b85
spotless apply
mn-mikke Apr 19, 2022
ea696fe
dataframe.isEmpty is not present in spark 2.2
mn-mikke Apr 20, 2022
e5457e8
Merge remote-tracking branch 'origin/master' into mn/SW-2646b
krasinski Jun 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ trait MetricsConfigurations {
"The class makes available all metrics that shared across all algorithms, and ML problems." +
" (classification, regression, dimension reduction)."),
ModelMetricsSubstitutionContext(
"H2OBinomialMetrics",
"H2OBinomialMetricsBase",
classOf[ModelMetricsBinomialV3[_, _]],
Seq("H2OCommonMetrics"),
"The class makes available all metrics that shared across all algorithms supporting binomial classification."),
Expand All @@ -40,7 +40,7 @@ trait MetricsConfigurations {
Seq("H2OBinomialMetrics", "H2OGLMMetrics"),
"The class makes available all binomial metrics supported by GLM algorithm."),
ModelMetricsSubstitutionContext(
"H2ORegressionMetrics",
"H2ORegressionMetricsBase",
classOf[ModelMetricsRegressionV3[_, _]],
Seq("H2OCommonMetrics"),
"The class makes available all metrics that shared across all algorithms supporting regression."),
Expand All @@ -55,7 +55,7 @@ trait MetricsConfigurations {
Seq("H2ORegressionMetrics"),
"The class makes available all regression metrics supported by CoxPH algorithm."),
ModelMetricsSubstitutionContext(
"H2OMultinomialMetrics",
"H2OMultinomialMetricsBase",
classOf[ModelMetricsMultinomialV3[_, _]],
Seq("H2OCommonMetrics"),
"The class makes available all metrics that shared across all algorithms supporting multinomial classification."),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import ai.h2o.sparkling.api.generation.common.{EntitySubstitutionContext, ModelM
object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) => String) with PythonEntityTemplate {

def apply(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
val metricClasses = metricSubstitutionContexts.map(_.entityName)
val metricClasses = getEntityNames(metricSubstitutionContexts)
val imports = Seq("py4j.java_gateway.JavaObject") ++
metricClasses.map(metricClass => s"ai.h2o.sparkling.ml.metrics.$metricClass.$metricClass")

Expand All @@ -46,12 +46,22 @@ object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) =>
}
}

private def generatePatternMatchingCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
private def getEntityNames(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): Seq[String] = {
metricSubstitutionContexts
.map { metricSubstitutionContext =>
val metricsObjectName = metricSubstitutionContext.entityName
s""" elif javaObject.getClass().getSimpleName() == "$metricsObjectName":
| return $metricsObjectName(javaObject)""".stripMargin
if (metricSubstitutionContext.entityName.endsWith("Base")) {
metricSubstitutionContext.entityName.substring(0, metricSubstitutionContext.entityName.length - 4)
} else {
metricSubstitutionContext.entityName
}
}
}

private def generatePatternMatchingCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
getEntityNames(metricSubstitutionContexts)
.map { entityName =>
s""" elif javaObject.getClass().getSimpleName() == "$entityName":
| return $entityName(javaObject)""".stripMargin
}
.mkString("\n")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,13 @@ import ai.h2o.sparkling.api.generation.common.{EntitySubstitutionContext, ModelM
object MetricsInitTemplate extends ((Seq[ModelMetricsSubstitutionContext]) => String) with PythonEntityTemplate {

def apply(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
val metricClasses = metricSubstitutionContexts.map(_.entityName)
val metricClasses = metricSubstitutionContexts.map { metricSubstitutionContext =>
if (metricSubstitutionContext.entityName.endsWith("Base")) {
metricSubstitutionContext.entityName.substring(0, metricSubstitutionContext.entityName.length - 4)
} else {
metricSubstitutionContext.entityName
}
}
val imports = metricClasses.map(metricClass => s"ai.h2o.sparkling.ml.metrics.$metricClass.$metricClass")

val entitySubstitutionContext = EntitySubstitutionContext(null, null, null, imports)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import ai.h2o.sparkling.api.generation.common.ModelMetricsSubstitutionContext
object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) => String) {

def apply(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
val metricClasses = metricSubstitutionContexts.map(_.entityName)
val metricClasses = getEntityNames(metricSubstitutionContexts)
val imports = metricClasses.map(metricClass => s"""source(file.path("R", "${metricClass}.R"))""").mkString("\n")

s"""#
Expand Down Expand Up @@ -55,12 +55,23 @@ object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) =>
|""".stripMargin
}

private def generateCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
private def getEntityNames(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): Seq[String] = {
metricSubstitutionContexts
.map { metricSubstitutionContext =>
val metricsObjectName = metricSubstitutionContext.entityName
s""" } else if (invoke(invoke(javaObject, "getClass"), "getSimpleName") == "$metricsObjectName") {
| rsparkling.$metricsObjectName(javaObject)""".stripMargin
if (metricSubstitutionContext.entityName.endsWith("Base")) {
metricSubstitutionContext.entityName.substring(0, metricSubstitutionContext.entityName.length - 4)
} else {
metricSubstitutionContext.entityName
}
}
}

private def generateCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
val names = getEntityNames(metricSubstitutionContexts)
names
.map { entityName =>
s""" } else if (invoke(invoke(javaObject, "getClass"), "getSimpleName") == "$entityName") {
| rsparkling.$entityName(javaObject)""".stripMargin
}
.mkString("\n")
}
Expand Down
46 changes: 45 additions & 1 deletion core/src/test/scala/ai/h2o/sparkling/TestUtils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import org.apache.spark.mllib
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.functions.{lit, rand}
import org.apache.spark.sql.functions.{lit, rand, col, abs}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
import org.scalatest.Matchers
Expand Down Expand Up @@ -100,6 +100,50 @@ object TestUtils extends Matchers {
""".stripMargin)
}

def assertDataFramesAreEqual(
expected: DataFrame,
produced: DataFrame,
identityColumn: String,
tolerance: Double): Unit = {
val tolerances = expected.schema.fields
.filterNot(_.name == identityColumn)
.filter(_.dataType.isInstanceOf[NumericType])
.map(_.name -> tolerance)
.toMap
assertDataFramesAreEqual(expected, produced, identityColumn, tolerances)
}

def assertDataFramesAreEqual(
expected: DataFrame,
produced: DataFrame,
identityColumn: String,
tolerances: Map[String, Double] = Map.empty): Unit = {
expected.schema shouldEqual produced.schema
val intersection = expected.as("expected").join(produced.as("produced"), identityColumn)
intersection.count() shouldEqual expected.count()
intersection.count() shouldEqual produced.count()
val isEqualExpression = expected.columns.foldLeft(lit(true)) {
case (partialExpression, columnName) =>
val columnComparision = if (tolerances.contains(columnName)) {
val difference = abs(col(s"expected.$columnName") - col(s"produced.$columnName"))
difference <= lit(tolerances(columnName))
} else if (columnName == identityColumn) {
lit(true)
} else {
col(s"expected.$columnName") === col(s"produced.$columnName")
}
partialExpression && columnComparision
}
val withComparisonDF = intersection.withColumn("isEqual", isEqualExpression)
val differentRowsDF = withComparisonDF
.filter(col("isEqual") === lit(false))
.select(col(s"expected.$identityColumn") as "id")
val differentIds = differentRowsDF.collect().map(_.get(0))
assert(
differentIds.length == 0,
s"The rows of ids($identityColumn) [${differentIds.mkString(", ")}] are not equal.")
}

def assertDatasetBasicProperties[T <: Product](
ds: Dataset[T],
df: H2OFrame,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ object Runner {
}
} else {
val metricClasses = getParamClasses("ai.h2o.sparkling.ml.metrics")
.filter(_.getSimpleName.endsWith("Metrics"))
writeResultToFile(MetricsTocTreeTemplate(metricClasses), "metrics", destinationDir)
for (metricClass <- metricClasses) {
val content = MetricsTemplate(metricClass)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hex.MetricsCalculationTypeExtensions
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package hex;

import java.util.Arrays;
import water.TypeMapExtension;
import water.api.schemas3.*;

public class MetricsCalculationTypeExtensions implements TypeMapExtension {
public static final String[] MODEL_BUILDER_CLASSES = {
ModelMetrics.MetricBuilder.class.getName(),
ModelMetricsSupervised.MetricBuilderSupervised.class.getName(),
ModelMetricsBinomial.MetricBuilderBinomial.class.getName(),
AUC2.AUCBuilder.class.getName(),
ModelMetricsRegression.MetricBuilderRegression.class.getName(),
Distribution.class.getName(),
GaussianDistribution.class.getName(),
BernoulliDistribution.class.getName(),
QuasibinomialDistribution.class.getName(),
ModifiedHuberDistribution.class.getName(),
MultinomialDistribution.class.getName(),
PoissonDistribution.class.getName(),
GammaDistribution.class.getName(),
TweedieDistribution.class.getName(),
HuberDistribution.class.getName(),
LaplaceDistribution.class.getName(),
QuantileDistribution.class.getName(),
CustomDistribution.class.getName(),
CustomDistributionWrapper.class.getName(),
LinkFunction.class.getName(),
IdentityFunction.class.getName(),
InverseFunction.class.getName(),
LogFunction.class.getName(),
LogitFunction.class.getName(),
OlogitFunction.class.getName(),
OloglogFunction.class.getName(),
OprobitFunction.class.getName(),
ModelMetricsMultinomial.MetricBuilderMultinomial.class.getName()
};

public static final String[] SCHEMA_CLASSES = {
ModelMetricsBaseV3.class.getName(),
ModelMetricsBinomialV3.class.getName(),
ModelMetricsMultinomialV3.class.getName(),
ModelMetricsRegressionV3.class.getName(),
ConfusionMatrixV3.class.getName(),
TwoDimTableV3.class.getName(),
TwoDimTableV3.ColumnSpecsBase.class.getName()
};

@Override
public String[] getBoostrapClasses() {
String[] result =
Arrays.copyOf(MODEL_BUILDER_CLASSES, MODEL_BUILDER_CLASSES.length + SCHEMA_CLASSES.length);
System.arraycopy(
SCHEMA_CLASSES, 0, result, MODEL_BUILDER_CLASSES.length, SCHEMA_CLASSES.length);
return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -183,57 +183,4 @@ class BinomialPredictionTestSuite extends FunSuite with Matchers with SharedH2OT
assert(schema == expectedSchema)
assert(schema == expectedSchemaByTransform)
}

private def assertMetrics[T](model: H2OMOJOModel): Unit = {
assertMetrics[T](model.getTrainingMetricsObject(), model.getTrainingMetrics())
assertMetrics[T](model.getValidationMetricsObject(), model.getValidationMetrics())
assert(model.getCrossValidationMetricsObject() == null)
assert(model.getCrossValidationMetrics() == Map())
}

private def assertMetrics[T](metricsObject: H2OMetrics, metrics: Map[String, Double]): Unit = {
metricsObject.isInstanceOf[T] should be(true)
MetricsAssertions.assertMetricsObjectAgainstMetricsMap(metricsObject, metrics)
val binomialObject = metricsObject.asInstanceOf[H2OBinomialMetrics]
binomialObject.getConfusionMatrix().count() > 0
binomialObject.getConfusionMatrix().columns.length > 0
binomialObject.getGainsLiftTable().count() > 0
binomialObject.getGainsLiftTable().columns.length > 0
binomialObject.getMaxCriteriaAndMetricScores().count() > 0
binomialObject.getMaxCriteriaAndMetricScores().columns.length > 0
binomialObject.getThresholdsAndMetricScores().count() > 0
binomialObject.getThresholdsAndMetricScores().columns.length > 0
}

test("test binomial metric objects") {
val algo = new H2OGBM()
.setSplitRatio(0.8)
.setSeed(1)
.setFeaturesCols("sepal_len", "sepal_wid")
.setColumnsToCategorical("class")
.setLabelCol("class")

val model = algo.fit(dataset)
assertMetrics[H2OBinomialMetrics](model)

model.write.overwrite().save("ml/build/gbm_binomial_model_metrics")
val loadedModel = H2OGBMMOJOModel.load("ml/build/gbm_binomial_model_metrics")
assertMetrics[H2OBinomialMetrics](loadedModel)
}

test("test binomial glm metric objects") {
val algo = new H2OGLM()
.setSplitRatio(0.8)
.setSeed(1)
.setFeaturesCols("sepal_len", "sepal_wid")
.setColumnsToCategorical("class")
.setLabelCol("class")

val model = algo.fit(dataset)
assertMetrics[H2OBinomialGLMMetrics](model)

model.write.overwrite().save("ml/build/glm_binomial_model_metrics")
val loadedModel = H2OGLMMOJOModel.load("ml/build/glm_binomial_model_metrics")
assertMetrics[H2OBinomialGLMMetrics](loadedModel)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -153,38 +153,4 @@ class RegressionPredictionTestSuite extends FunSuite with Matchers with SharedH2
metricsObject.isInstanceOf[T] should be(true)
MetricsAssertions.assertMetricsObjectAgainstMetricsMap(metricsObject, metrics)
}

test("test regression metric objects") {
val algo = new algos.H2OGBM()
.setSplitRatio(0.8)
.setSeed(1)
.setWithContributions(true)
.setWithLeafNodeAssignments(true)
.setWithStageResults(true)
.setFeaturesCols("CAPSULE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON")
.setLabelCol("AGE")
val model = algo.fit(dataset)
assertMetrics[H2ORegressionMetrics](model)

model.write.overwrite().save("ml/build/gbm_regression_model_metrics")
val loadedModel = H2OGBMMOJOModel.load("ml/build/gbm_regression_model_metrics")
assertMetrics[H2ORegressionMetrics](loadedModel)
}

test("test regression glm metric objects") {
val algo = new algos.H2OGLM()
.setSplitRatio(0.8)
.setSeed(1)
.setWithContributions(true)
.setWithLeafNodeAssignments(true)
.setWithStageResults(true)
.setFeaturesCols("CAPSULE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON")
.setLabelCol("AGE")
val model = algo.fit(dataset)
assertMetrics[H2ORegressionGLMMetrics](model)

model.write.overwrite().save("ml/build/glm_regression_model_metrics")
val loadedModel = H2OGLMMOJOModel.load("ml/build/glm_regression_model_metrics")
assertMetrics[H2ORegressionGLMMetrics](loadedModel)
}
}
Loading