diff --git a/bench.py b/bench.py index e56630ac4..b398bc8fc 100644 --- a/bench.py +++ b/bench.py @@ -389,14 +389,13 @@ def convert_data(data, dtype, data_order, data_format): # Secondly, change format of data if data_format == 'numpy': return data - elif data_format == 'pandas': + if data_format == 'pandas': import pandas as pd if data.ndim == 1: return pd.Series(data) - else: - return pd.DataFrame(data) - elif data_format == 'cudf': + return pd.DataFrame(data) + if data_format == 'cudf': import cudf import pandas as pd @@ -512,36 +511,42 @@ def gen_basic_dict(library, algorithm, stage, params, data, alg_instance=None, def print_output(library, algorithm, stages, params, functions, times, metric_type, metrics, data, alg_instance=None, alg_params=None): - if params.output_format == 'json': - output = [] - for i, stage in enumerate(stages): - result = gen_basic_dict(library, algorithm, stage, params, - data[i], alg_instance, alg_params) - result.update({'time[s]': times[i]}) - if metric_type is not None: - if isinstance(metric_type, str): - result.update({f'{metric_type}': metrics[i]}) - elif isinstance(metric_type, list): - for ind, val in enumerate(metric_type): - if metrics[ind][i] is not None: - result.update({f'{val}': metrics[ind][i]}) - if hasattr(params, 'n_classes'): - result['input_data'].update({'classes': params.n_classes}) - if hasattr(params, 'n_clusters'): - if algorithm == 'kmeans': - result['input_data'].update( - {'n_clusters': params.n_clusters}) - elif algorithm == 'dbscan': - result.update({'n_clusters': params.n_clusters}) - # replace non-string init with string for kmeans benchmarks - if alg_instance is not None: - if 'init' in result['algorithm_parameters'].keys(): - if not isinstance(result['algorithm_parameters']['init'], str): - result['algorithm_parameters']['init'] = 'random' - if 'handle' in result['algorithm_parameters'].keys(): - del result['algorithm_parameters']['handle'] - output.append(result) - print(json.dumps(output, indent=4)) + if params.output_format != 'json': + return + + output = [] + for i, stage in enumerate(stages): + result = gen_basic_dict(library, algorithm, stage, params, + data[i], alg_instance, alg_params) + result.update({'time[s]': times[i]}) + + if metric_type is not None: + if isinstance(metric_type, str): + result.update({f'{metric_type}': metrics[i]}) + elif isinstance(metric_type, list): + for ind, val in enumerate(metric_type): + if metrics[ind][i] is not None: + result.update({f'{val}': metrics[ind][i]}) + + if hasattr(params, 'n_classes'): + result['input_data'].update({'classes': params.n_classes}) + if hasattr(params, 'n_clusters'): + if algorithm == 'kmeans': + result['input_data'].update( + {'n_clusters': params.n_clusters}) + elif algorithm == 'dbscan': + result.update({'n_clusters': params.n_clusters}) + + # replace non-string init with string for kmeans benchmarks + if alg_instance is not None: + if 'init' in result['algorithm_parameters'].keys(): + if not isinstance(result['algorithm_parameters']['init'], str): + result['algorithm_parameters']['init'] = 'random' + if 'handle' in result['algorithm_parameters'].keys(): + del result['algorithm_parameters']['handle'] + output.append(result) + + print(json.dumps(output, indent=4)) def run_with_context(params, function): diff --git a/configs/xgboost/xgb_gpu_additional_config.json b/configs/xgboost/xgb_gpu_additional_config.json index 75036ad4b..b8ca4fda3 100644 --- a/configs/xgboost/xgb_gpu_additional_config.json +++ b/configs/xgboost/xgb_gpu_additional_config.json @@ -1,12 +1,10 @@ { "common": { "lib": "xgboost", - "data-format": "cudf", "data-order": "F", "dtype": "float32", "algorithm": "gbt", "tree-method": "gpu_hist", - "count-dmatrix": "", "max-depth": 8, "learning-rate": 0.1, "reg-lambda": 1, @@ -15,6 +13,7 @@ "cases": [ { "objective": "binary:logistic", + "data-format": "pandas", "scale-pos-weight": 2.1067817411664587, "dataset": [ { @@ -33,6 +32,7 @@ }, { "objective": "binary:logistic", + "data-format": "cudf", "scale-pos-weight": 173.63348001466812, "dataset": [ { @@ -51,6 +51,7 @@ }, { "objective": "multi:softmax", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -68,6 +69,7 @@ }, { "objective": "binary:logistic", + "data-format": "pandas", "scale-pos-weight": 2.0017715678375363, "dataset": [ { @@ -86,6 +88,7 @@ }, { "objective": "binary:logistic", + "data-format": "cudf", "scale-pos-weight": 578.2868020304569, "dataset": [ { @@ -104,6 +107,7 @@ }, { "objective": "binary:logistic", + "data-format": "cudf", "scale-pos-weight": 1.8872389605086624, "dataset": [ { @@ -122,6 +126,7 @@ }, { "objective": "reg:squarederror", + "data-format": "cudf", "dataset": [ { "source": "npy", diff --git a/configs/xgboost/xgb_gpu_main_config.json b/configs/xgboost/xgb_gpu_main_config.json index 11144ca35..3bbe9f1bc 100644 --- a/configs/xgboost/xgb_gpu_main_config.json +++ b/configs/xgboost/xgb_gpu_main_config.json @@ -1,15 +1,15 @@ { "common": { "lib": "xgboost", - "data-format": "cudf", "data-order": "F", "dtype": "float32", "algorithm": "gbt", - "tree-method": "gpu_hist", - "count-dmatrix": "" + "tree-method": "gpu_hist" }, "cases": [ { + "objective": "reg:squarederror", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -26,10 +26,11 @@ ], "learning-rate": 0.03, "max-depth": 6, - "n-estimators": 1000, - "objective": "reg:squarederror" + "n-estimators": 1000 }, { + "objective": "binary:logistic", + "data-format": "pandas", "dataset": [ { "source": "npy", @@ -53,10 +54,11 @@ "min-child-weight": 0, "max-depth": 8, "max-leaves": 256, - "n-estimators": 1000, - "objective": "binary:logistic" + "n-estimators": 1000 }, { + "objective": "binary:logistic", + "data-format": "pandas", "dataset": [ { "source": "npy", @@ -81,10 +83,11 @@ "max-depth": 8, "max-leaves": 256, "n-estimators": 1000, - "objective": "binary:logistic", "inplace-predict": "" }, { + "objective": "multi:softprob", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -101,10 +104,11 @@ ], "learning-rate": 0.03, "max-depth": 6, - "n-estimators": 1000, - "objective": "multi:softprob" + "n-estimators": 1000 }, { + "objective": "multi:softprob", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -122,10 +126,11 @@ "min-child-weight": 1, "min-split-loss": 0.1, "max-depth": 8, - "n-estimators": 200, - "objective": "multi:softprob" + "n-estimators": 200 }, { + "objective": "reg:squarederror", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -137,7 +142,6 @@ } ], "n-estimators": 100, - "objective": "reg:squarederror", "max-depth": 8, "scale-pos-weight": 2, "learning-rate": 0.1, @@ -148,6 +152,8 @@ "max-leaves": 256 }, { + "objective": "multi:softprob", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -163,12 +169,13 @@ } ], "n-estimators": 60, - "objective": "multi:softprob", "max-depth": 7, "subsample": 0.7, "colsample-bytree": 0.7 }, { + "objective": "binary:logistic", + "data-format": "cudf", "dataset": [ { "source": "npy", @@ -184,7 +191,6 @@ } ], "n-estimators": 10000, - "objective": "binary:logistic", "max-depth": 1, "subsample": 0.5, "eta": 0.1, diff --git a/modelbuilders_bench/lgbm_mb.py b/modelbuilders_bench/lgbm_mb.py index d70f46772..e6daab0b2 100644 --- a/modelbuilders_bench/lgbm_mb.py +++ b/modelbuilders_bench/lgbm_mb.py @@ -15,21 +15,20 @@ # =============================================================================== import argparse -import os import bench import daal4py import lightgbm as lgbm import numpy as np -import modelbuilders_bench.mb_utils as utils - parser = argparse.ArgumentParser( description='lightgbm gbt + model transform + daal predict benchmark') parser.add_argument('--colsample-bytree', type=float, default=1, help='Subsample ratio of columns ' 'when constructing each tree') +parser.add_argument('--count-dmatrix', default=False, action='store_true', + help='Count DMatrix creation in time measurements') parser.add_argument('--learning-rate', '--eta', type=float, default=0.3, help='Step size shrinkage used in update ' 'to prevents overfitting') @@ -87,68 +86,80 @@ if params.threads != -1: lgbm_params.update({'nthread': params.threads}) -if 'OMP_NUM_THREADS' in os.environ.keys(): - lgbm_params['nthread'] = int(os.environ['OMP_NUM_THREADS']) - if params.objective.startswith('reg'): task = 'regression' metric_name, metric_func = 'rmse', bench.rmse_score else: task = 'classification' - metric_name, metric_func = 'accuracy[%]', utils.get_accuracy + metric_name, metric_func = 'accuracy', bench.accuracy_score if 'cudf' in str(type(y_train)): params.n_classes = y_train[y_train.columns[0]].nunique() else: params.n_classes = len(np.unique(y_train)) + + # Covtype has one class more than there is in train + if params.dataset_name == 'covtype': + params.n_classes += 1 + if params.n_classes > 2: lgbm_params['num_class'] = params.n_classes -t_creat_train, lgbm_train = bench.measure_function_time(lgbm.Dataset, X_train, - y_train, params=params, - free_raw_data=False) - -t_creat_test, lgbm_test = bench.measure_function_time(lgbm.Dataset, X_test, y_test, - params=params, reference=lgbm_train, - free_raw_data=False) - -t_train, model_lgbm = bench.measure_function_time(lgbm.train, lgbm_params, lgbm_train, - params=params, - num_boost_round=params.n_estimators, - valid_sets=lgbm_train, - verbose_eval=False) -train_metric = None -if not X_train.equals(X_test): - y_train_pred = model_lgbm.predict(X_train) - train_metric = metric_func(y_train, y_train_pred) - -t_lgbm_pred, y_test_pred = bench.measure_function_time(model_lgbm.predict, X_test, - params=params) -test_metric_lgbm = metric_func(y_test, y_test_pred) - -t_trans, model_daal = bench.measure_function_time( +t_creat_train, dtrain = bench.measure_function_time(lgbm.Dataset, X_train, + y_train, params=params, + free_raw_data=False) + +t_creat_test, dtest = bench.measure_function_time(lgbm.Dataset, X_test, y_test, + params=params, reference=dtrain, + free_raw_data=False) + + +def fit(dataset): + if dataset is None: + dataset = lgbm.Dataset(X_train, y_train, free_raw_data=False) + return lgbm.train( + lgbm_params, dataset, num_boost_round=params.n_estimators, valid_sets=dataset, + verbose_eval=False) + + +def predict(dataset): # type: ignore + if dataset is None: + dataset = lgbm.Dataset(X_test, y_test, free_raw_data=False) + return model_lgbm.predict(dataset) + + +fit_time, model_lgbm = bench.measure_function_time( + fit, None if params.count_dmatrix else dtrain, params=params) +train_metric = metric_func(model_lgbm.predict(dtrain), y_train) + +predict_time, y_pred = bench.measure_function_time( + predict, None if params.count_dmatrix else dtest, params=params) +test_metric = metric_func(y_pred, y_test) + +transform_time, model_daal = bench.measure_function_time( daal4py.get_gbt_model_from_lightgbm, model_lgbm, params=params) if hasattr(params, 'n_classes'): predict_algo = daal4py.gbt_classification_prediction( nClasses=params.n_classes, resultsToEvaluate='computeClassLabels', fptype='float') - t_daal_pred, daal_pred = bench.measure_function_time( + predict_time_daal, daal_pred = bench.measure_function_time( predict_algo.compute, X_test, model_daal, params=params) test_metric_daal = metric_func(y_test, daal_pred.prediction) else: predict_algo = daal4py.gbt_regression_prediction() - t_daal_pred, daal_pred = bench.measure_function_time( + predict_time_daal, daal_pred = bench.measure_function_time( predict_algo.compute, X_test, model_daal, params=params) test_metric_daal = metric_func(y_test, daal_pred.prediction) -utils.print_output( - library='modelbuilders', - algorithm=f'lightgbm_{task}_and_modelbuilder', - stages=['lgbm_train', 'lgbm_predict', 'daal4py_predict'], +bench.print_output( + library='modelbuilders', algorithm=f'lightgbm_{task}_and_modelbuilder', + stages=['training_preparation', 'training', 'prediction_preparation', 'prediction', + 'transformation', 'alternative_prediction'], params=params, - functions=['lgbm_dataset', 'lgbm_dataset', 'lgbm_train', - 'lgbm_predict', 'lgbm_to_daal', 'daal_compute'], - times=[t_creat_train, t_train, t_creat_test, t_lgbm_pred, t_trans, t_daal_pred], + functions=['lgbm.Dataset.train', 'lgbm.train', 'lgbm.Dataset.test', 'lgbm.predict', + 'daal4py.get_gbt_model_from_lightgbm', 'daal4py.compute'], + times=[t_creat_train, fit_time, t_creat_test, predict_time, transform_time, + predict_time_daal], metric_type=metric_name, - metrics=[train_metric, test_metric_lgbm, test_metric_daal], - data=[X_train, X_test, X_test], -) + metrics=[None, train_metric, None, test_metric, None, test_metric_daal], + data=[X_train, X_train, X_test, X_test, X_test, X_test], + alg_instance=model_lgbm, alg_params=lgbm_params) diff --git a/modelbuilders_bench/mb_utils.py b/modelbuilders_bench/mb_utils.py deleted file mode 100644 index 7c54efd92..000000000 --- a/modelbuilders_bench/mb_utils.py +++ /dev/null @@ -1,71 +0,0 @@ -# =============================================================================== -# Copyright 2020-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== - -import json - -import numpy as np - - -def get_accuracy(true_labels, prediction): - errors = 0 - for i, true_label in enumerate(true_labels): - pred_label = 0 - if isinstance(prediction[i], (float, np.single, np.float)): - pred_label = prediction[i] > 0.5 - elif prediction[i].shape[0] == 1: - pred_label = prediction[i][0] - else: - pred_label = np.argmax(prediction[i]) - if true_label != pred_label: - errors += 1 - return 100 * (1 - errors / len(true_labels)) - - -def print_output(library, algorithm, stages, params, functions, - times, metric_type, metrics, data): - if params.output_format == 'json': - output = [] - output.append({ - 'library': library, - 'algorithm': algorithm, - 'input_data': { - 'data_format': params.data_format, - 'data_order': params.data_order, - 'data_type': str(params.dtype), - 'dataset_name': params.dataset_name, - 'rows': data[0].shape[0], - 'columns': data[0].shape[1] - } - }) - if hasattr(params, 'n_classes'): - output[-1]['input_data'].update({'classes': params.n_classes}) - for i, stage in enumerate(stages): - result = { - 'stage': stage, - } - if 'daal' in stage: - result.update({'conversion_to_daal4py': times[2 * i], - 'prediction_time': times[2 * i + 1]}) - elif 'train' in stage: - result.update({'matrix_creation_time': times[2 * i], - 'training_time': times[2 * i + 1]}) - else: - result.update({'matrix_creation_time': times[2 * i], - 'prediction_time': times[2 * i + 1]}) - if metrics[i] is not None: - result.update({f'{metric_type}': metrics[i]}) - output.append(result) - print(json.dumps(output, indent=4)) diff --git a/modelbuilders_bench/xgb_mb.py b/modelbuilders_bench/xgb_mb.py index 54bf4bd95..69e1f525a 100644 --- a/modelbuilders_bench/xgb_mb.py +++ b/modelbuilders_bench/xgb_mb.py @@ -26,11 +26,12 @@ def convert_probs_to_classes(y_prob): return np.array([np.argmax(y_prob[i]) for i in range(y_prob.shape[0])]) -def convert_xgb_predictions(y_pred, objective): - if objective == 'multi:softprob': - y_pred = convert_probs_to_classes(y_pred) - elif objective == 'binary:logistic': - y_pred = y_pred.astype(np.int32) +def convert_xgb_predictions(y_pred, objective, metric_name): + if metric_name == "accuracy": + if objective == 'multi:softprob': + y_pred = convert_probs_to_classes(y_pred) + elif objective == 'binary:logistic': + y_pred = y_pred.astype(np.int32) return y_pred @@ -85,6 +86,9 @@ def convert_xgb_predictions(y_pred, objective): help='The tree construction algorithm used in XGBoost') params = bench.parse_args(parser) +# Default seed +if params.seed == 12345: + params.seed = 0 X_train, X_test, y_train, y_test = bench.load_data(params) @@ -123,8 +127,7 @@ def convert_xgb_predictions(y_pred, objective): metric_name, metric_func = 'rmse', bench.rmse_score else: task = 'classification' - metric_name = 'accuracy' - metric_func = bench.accuracy_score + metric_name, metric_func = 'accuracy', bench.accuracy_score if 'cudf' in str(type(y_train)): params.n_classes = y_train[y_train.columns[0]].nunique() else: @@ -165,12 +168,13 @@ def predict(dmatrix): # type: ignore train_metric = metric_func( convert_xgb_predictions( booster.predict(dtrain), - params.objective), + params.objective, metric_name), y_train) predict_time, y_pred = bench.measure_function_time( predict, None if params.inplace_predict or params.count_dmatrix else dtest, params=params) -test_metric = metric_func(convert_xgb_predictions(y_pred, params.objective), y_test) +test_metric = metric_func(convert_xgb_predictions( + y_pred, params.objective, metric_name), y_test) transform_time, model_daal = bench.measure_function_time( daal4py.get_gbt_model_from_xgboost, booster, params=params) @@ -198,4 +202,5 @@ def predict(dmatrix): # type: ignore predict_time_daal], metric_type=metric_name, metrics=[None, train_metric, None, test_metric, None, test_metric_daal], - data=[X_train, X_train, X_test, X_test, X_test, X_test]) + data=[X_train, X_train, X_test, X_test, X_test, X_test], + alg_instance=booster, alg_params=xgb_params) diff --git a/xgboost_bench/gbt.py b/xgboost_bench/gbt.py index 8540f4f5a..fa01c3092 100644 --- a/xgboost_bench/gbt.py +++ b/xgboost_bench/gbt.py @@ -25,11 +25,12 @@ def convert_probs_to_classes(y_prob): return np.array([np.argmax(y_prob[i]) for i in range(y_prob.shape[0])]) -def convert_xgb_predictions(y_pred, objective): - if objective == 'multi:softprob': - y_pred = convert_probs_to_classes(y_pred) - elif objective == 'binary:logistic': - y_pred = y_pred.astype(np.int32) +def convert_xgb_predictions(y_pred, objective, metric_name): + if metric_name == "accuracy": + if objective == 'multi:softprob': + y_pred = convert_probs_to_classes(y_pred) + elif objective == 'binary:logistic': + y_pred = y_pred.astype(np.int32) return y_pred @@ -126,8 +127,7 @@ def convert_xgb_predictions(y_pred, objective): metric_name, metric_func = 'rmse', bench.rmse_score else: task = 'classification' - metric_name = 'accuracy' - metric_func = bench.accuracy_score + metric_name, metric_func = 'accuracy', bench.accuracy_score if 'cudf' in str(type(y_train)): params.n_classes = y_train[y_train.columns[0]].nunique() else: @@ -140,8 +140,10 @@ def convert_xgb_predictions(y_pred, objective): if params.n_classes > 2: xgb_params['num_class'] = params.n_classes -dtrain = xgb.DMatrix(X_train, y_train) -dtest = xgb.DMatrix(X_test, y_test) +t_creat_train, dtrain = bench.measure_function_time(xgb.DMatrix, X_train, + params=params, label=y_train) +t_creat_test, dtest = bench.measure_function_time( + xgb.DMatrix, X_test, params=params, label=y_test) def fit(dmatrix): @@ -166,16 +168,20 @@ def predict(dmatrix): # type: ignore train_metric = metric_func( convert_xgb_predictions( booster.predict(dtrain), - params.objective), + params.objective, metric_name), y_train) predict_time, y_pred = bench.measure_function_time( predict, None if params.inplace_predict or params.count_dmatrix else dtest, params=params) -test_metric = metric_func(convert_xgb_predictions(y_pred, params.objective), y_test) - -bench.print_output(library='xgboost', algorithm=f'gradient_boosted_trees_{task}', - stages=['training', 'prediction'], - params=params, functions=['gbt.fit', 'gbt.predict'], - times=[fit_time, predict_time], metric_type=metric_name, - metrics=[train_metric, test_metric], data=[X_train, X_test], - alg_instance=booster, alg_params=xgb_params) +test_metric = metric_func(convert_xgb_predictions( + y_pred, params.objective, metric_name), y_test) + +bench.print_output( + library='xgboost', algorithm=f'xgboost_{task}', + stages=['training_preparation', 'training', 'prediction_preparation', 'prediction'], + params=params, + functions=['xgb.dmatrix.train', 'xgb.train', 'xgb.dmatrix.test', 'xgb.predict'], + times=[t_creat_train, fit_time, t_creat_test, predict_time], + metric_type=metric_name, metrics=[None, train_metric, None, test_metric], + data=[X_train, X_train, X_test, X_test], + alg_instance=booster, alg_params=xgb_params)