Change dataset and add train, validation, test

Author: Paolo Di Francesco

scikit_learn_script_mode_local_training_and_serving/code/scikit_learn_iris.py scikit_learn_script_mode_local_training_and_serving/code/scikit_learn_california.py

@@ -19,6 +19,7 @@
 import joblib
 import pandas as pd
 from sklearn import tree
+from sklearn.metrics import mean_squared_error
 
 if __name__ == "__main__":
     print("Training Started")
@@ -31,6 +32,7 @@
     parser.add_argument("--output-data-dir", type=str, default=os.environ["SM_OUTPUT_DATA_DIR"])
     parser.add_argument("--model-dir", type=str, default=os.environ["SM_MODEL_DIR"])
     parser.add_argument("--train", type=str, default=os.environ["SM_CHANNEL_TRAIN"])
+    parser.add_argument("--validation", type=str, default=os.environ["SM_CHANNEL_VALIDATION"])
 
     args = parser.parse_args()
     print("Got Args: {}".format(args))
@@ -57,10 +59,20 @@
     # as your training my require in the ArgumentParser above.
     max_leaf_nodes = args.max_leaf_nodes
 
-    # Now use scikit-learn's decision tree classifier to train the model.
-    clf = tree.DecisionTreeClassifier(max_leaf_nodes=max_leaf_nodes)
+    # Now use scikit-learn's decision tree regression to train the model.
+    clf = tree.DecisionTreeRegressor(max_leaf_nodes=max_leaf_nodes)
     clf = clf.fit(train_X, train_y)
 
+    input_files = [os.path.join(args.validation, file) for file in os.listdir(args.validation)]
+    raw_data = [pd.read_csv(file, header=None, engine="python") for file in input_files]
+    validation_data = pd.concat(raw_data)
+    # labels are in the first column
+    validation_y = validation_data.iloc[:, 0]
+    validation_X = validation_data.iloc[:, 1:]
+    #
+    predictions = clf.predict(validation_X)
+    error = mean_squared_error(predictions, validation_y)
+    print(f"RMSE: {error}")
     # Print the coefficients of the trained classifier, and save the coefficients
     joblib.dump(clf, os.path.join(args.model_dir, "model.joblib"))
 

scikit_learn_script_mode_local_training_and_serving/data/iris.csv

@@ -1,4 +1,5 @@
 numpy
 pandas
+sklearn
 sagemaker>=2.0.0<3.0.0
 sagemaker[local]

scikit_learn_script_mode_local_training_and_serving/requirements.txt

@@ -1,4 +1,4 @@
-# This is a sample Python program that trains a simple scikit-learn model on the Iris dataset.
+# This is a sample Python program that trains a simple scikit-learn model on the California dataset.
 # This implementation will work on your *local computer* or in the *AWS Cloud*.
 #
 # Prerequisites:
@@ -16,40 +16,54 @@
 import os
 
 from sagemaker.sklearn import SKLearn
+import sagemaker
+import boto3
 from sklearn import datasets
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import mean_squared_error
 
-DUMMY_IAM_ROLE = 'arn:aws:iam::111111111111:role/service-role/AmazonSageMaker-ExecutionRole-20200101T000001'
+local_mode = True
+
+if local_mode:
+    instance_type = "local"
+    IAM_ROLE = 'arn:aws:iam::111111111111:role/service-role/AmazonSageMaker-ExecutionRole-20200101T000001'
+else:
+    instance_type = "ml.m5.xlarge"
+    IAM_ROLE = 'arn:aws:iam::<ACCOUNT>:role/service-role/AmazonSageMaker-ExecutionRole-XXX'
+
+sess = sagemaker.Session()
+bucket = sess.default_bucket()                    # Set a default S3 bucket
+prefix = 'DEMO-local-and-managed-infrastructure'
 
 def download_training_and_eval_data():
-    if os.path.isfile('./data/iris.csv'):
-        print('Training and dataset exist. Skipping Download')
-    else:
-        print('Downloading training dataset')
+    print('Downloading training dataset')
 
-        # Load Iris dataset, then join labels and features
-        iris = datasets.load_iris()
-        joined_iris = np.insert(iris.data, 0, iris.target, axis=1)
+    # Load California Housing dataset, then join labels and features
+    california = datasets.fetch_california_housing()
+    dataset = np.insert(california.data, 0, california.target, axis=1)
+    # Create directory and write csv
+    os.makedirs("./data/train", exist_ok=True)
+    os.makedirs("./data/validation", exist_ok=True)
+    os.makedirs("./data/test", exist_ok=True)
 
-        # Create directory and write csv
-        os.makedirs("./data", exist_ok=True)
-        np.savetxt("./data/iris.csv", joined_iris, delimiter=",", fmt="%1.1f, %1.3f, %1.3f, %1.3f, %1.3f")
+    train, other = train_test_split(dataset, test_size=0.3)
+    validation, test = train_test_split(other, test_size=0.5)
 
-        print('Downloading completed')
+    np.savetxt("./data/train/california_train.csv", train, delimiter=",")
+    np.savetxt("./data/validation/california_validation.csv", validation, delimiter=",")
+    np.savetxt("./data/test/california_test.csv", test, delimiter=",")
+
+    print('Downloading completed')
 
 def do_inference_on_local_endpoint(predictor):
     print(f'\nStarting Inference on endpoint (local).')
-    shape = pd.read_csv("data/iris.csv", header=None)
-
-    a = [50 * i for i in range(3)]
-    b = [40 + i for i in range(10)]
-    indices = [i + j for i, j in itertools.product(a, b)]
-
-    test_data = shape.iloc[indices[:-1]]
+    test_data = pd.read_csv("data/test/california_test.csv", header=None)
     test_X = test_data.iloc[:, 1:]
     test_y = test_data.iloc[:, 0]
-    print("Predictions: {}".format(predictor.predict(test_X.values)))
+    predictions = predictor.predict(test_X.values)
+    print("Predictions: {}".format(predictions))
     print("Actual: {}".format(test_y.values))
-
+    print(f"RMSE: {mean_squared_error(predictions, test_y.values)}")
 
 def main():
     download_training_and_eval_data()
@@ -58,21 +72,35 @@ def main():
     print('Note: if launching for the first time in local mode, container image download might take a few minutes to complete.')
 
     sklearn = SKLearn(
-        entry_point="scikit_learn_iris.py",
+        entry_point="scikit_learn_california.py",
         source_dir='code',
         framework_version="1.0-1",
-        instance_type="local",
-        role=DUMMY_IAM_ROLE,
+        instance_type=instance_type,
+        role=IAM_ROLE,
         hyperparameters={"max_leaf_nodes": 30},
     )
 
-    train_input = "file://./data/iris.csv"
+    if local_mode:
+        train_input = "file://./data/train/california_train.csv"
+        validation_input = "file://./data/validation/california_validation.csv"
+    else:
+        # upload data to S3
+        boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'data/train/california_train.csv')).upload_file('data/train/california_train.csv')
+        boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'data/validation/california_validation.csv')).upload_file('data/validation/california_validation.csv')
+        boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'data/test/california_test.csv')).upload_file('data/test/california_test.csv')
 
-    sklearn.fit({"train": train_input})
+        train_input =f"s3://{bucket}/{prefix}/data/train/california_train.csv"
+        validation_input =f"s3://{bucket}/{prefix}/data/validation/california_validation.csv"
+        test_input =f"s3://{bucket}/{prefix}/data/test/california_test.csv"
+
+    sklearn.fit({"train": train_input, "validation": validation_input})
     print('Completed model training')
 
-    print('Deploying endpoint in local mode')
-    predictor = sklearn.deploy(initial_instance_count=1, instance_type='local')
+    if local_mode:
+        print('Deploying endpoint in local mode')
+    else:
+        print(f"deploying on the SageMaker managed infrastructure using a {instance_type} instance type")
+    predictor = sklearn.deploy(initial_instance_count=1, instance_type=instance_type)
 
     do_inference_on_local_endpoint(predictor)