fix: upgrade pyspark (#135)

* fix: upgrade pyspark

* use 2.4.0 instead and fix breaking changes

* remove py2

* remove py27

* upgrade spark version

Co-authored-by: Rui Wang Napieralski <[email protected]>

Author: icywang86rui

buildspec.yml

@@ -43,7 +43,7 @@ phases:
       - tox -e flake8,twine,sphinx
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI=
-        tox -e py27,py36,stats -- tests/
+        tox -e py36,stats -- tests/
 
       # spark integration tests
       - cd $CODEBUILD_SRC_DIR/integration-tests/sagemaker-spark-sdk

sagemaker-pyspark-sdk/setup.py

@@ -101,7 +101,7 @@ def read_version():
         scripts=["bin/sagemakerpyspark-jars", "bin/sagemakerpyspark-emr-jars"],
 
         install_requires=[
-            "pyspark==2.3.4",
+            "pyspark==2.4.0",
             "numpy",
         ],
     )

sagemaker-pyspark-sdk/src/sagemaker_pyspark/algorithms/XGBoostSageMakerEstimator.py

@@ -175,8 +175,8 @@ class XGBoostSageMakerEstimator(SageMakerEstimatorBase):
         "Subsample ratio of columns for each split, in each level. Must be in (0, 1].",
         typeConverter=TypeConverters.toFloat)
 
-    _lambda = Param(
-        Params._dummy(), "lambda",
+    lambda_weights = Param(
+        Params._dummy(), "lambda_weights",
         "L2 regularization term on weights, increase this value"
         " will make model more conservative.",
         typeConverter=TypeConverters.toFloat)
@@ -508,10 +508,10 @@ def setColSampleByLevel(self, value):
         self._set(colsample_bylevel=value)
 
     def getLambda(self):
-        return self.getOrDefault(self._lambda)
+        return self.getOrDefault(self.lambda_weights)
 
     def setLambda(self, value):
-        self._set(_lambda=value)
+        self._set(lambda_weights=value)
 
     def getAlpha(self):
         return self.getOrDefault(self.alpha)

sagemaker-pyspark-sdk/src/sagemaker_pyspark/wrapper.py

@@ -116,6 +116,7 @@ class Option(SageMakerJavaWrapper):
 
     def __init__(self, value):
         self.value = value
+        self._java_obj = None
 
     @classmethod
     def empty(cls):
@@ -147,6 +148,7 @@ class ScalaMap(SageMakerJavaWrapper):
 
     def __init__(self, dictionary):
         self.dictionary = dictionary
+        self._java_obj = None
 
     def _to_java(self):
         map = self._new_java_obj(ScalaMap._wrapped_class)
@@ -169,6 +171,7 @@ class ScalaList(SageMakerJavaWrapper):
 
     def __init__(self, p_list):
         self.p_list = p_list
+        self._java_obj = None
 
     def _to_java(self):
         # Since py4j cannot deal with scala list directly

sagemaker-pyspark-sdk/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,twine,sphinx,py{27,36},stats
+envlist = flake8,twine,sphinx,py36,stats
 skip_missing_interpreters = False
 
 [testenv]

sagemaker-spark-sdk/build.sbt

@@ -18,7 +18,7 @@ scalaVersion := "2.11.7"
 
 // to change the version of spark add -DSPARK_VERSION=2.x.x when running sbt
 // for example: "sbt -DSPARK_VERSION=2.1.1 clean compile test doc package"
-val sparkVersion = System.getProperty("SPARK_VERSION", "2.2.0")
+val sparkVersion = System.getProperty("SPARK_VERSION", "2.4.0")
 
 lazy val SageMakerSpark = (project in file("."))