Merge branch 'main' into community_proj

Author: chenyangkang

stemflow/model/AdaSTEM.py

@@ -975,12 +975,14 @@ def find_belonged_points_and_predict(df, st_indexes_df, X_df):
 
                 # if len(res)>0:
                 #     res = res.droplevel(0) # If using as_index=False duing groupby, pandas will automatically generate a group indexing column, so drop the indexing of the new groups
-            
+                if len(res)==0:
+                    continue
+                
                 window_prediction_list.append(res)
 
         if any([i is not None for i in window_prediction_list]):
             ensemble_prediction = pd.concat(window_prediction_list, axis=0)
-            ensemble_prediction = ensemble_prediction.groupby("index").mean().reset_index(drop=False)
+            ensemble_prediction = ensemble_prediction.groupby(level=0, sort=False).mean()
         else:
             ensmeble_index = list(window_single_ensemble_df["ensemble_index"])[0]
             warnings.warn(f"No prediction for this ensemble: {ensmeble_index}")
@@ -1028,7 +1030,7 @@ def mp_predict(ensemble, self=self):
             )
 
         # Prediction
-        pred = [i.set_index("index") for i in output_generator]
+        pred = [i for i in output_generator]
         get_reusable_executor().shutdown(wait=True)
 
         pred = pd.concat(pred, axis=1)

stemflow/model/SphereAdaSTEM.py

@@ -485,7 +485,18 @@ def find_belonged_points(df, st_indexes_df, X_df):
 
                 return X_df.iloc[np.where(intersect)[0], :]
 
-            query_results = (
+            def find_belonged_points_and_predict(df, st_indexes_df, X_df):
+                X = find_belonged_points(df, st_indexes_df, X_df)
+                if len(X)==0:
+                    return None
+                X['ensemble_index'] = df['ensemble_index'].iloc[0]
+                X['unique_stixel_id'] = df['unique_stixel_id'].iloc[0]
+                # X = X.sort_index() # To ensure the input dataframes for the two method (temporal_window_prequery or not) are the same so the tained base models are identical, at least with the same input data
+                pred = self.stixel_predict(X)
+                return pred
+            
+            # predict
+            window_prediction = (
                 window_single_ensemble_df[
                     [
                         "ensemble_index",
@@ -501,32 +512,21 @@ def find_belonged_points(df, st_indexes_df, X_df):
                         "p3z",
                     ]
                 ]
-                .groupby(["ensemble_index", "unique_stixel_id"], as_index=True)
+                .groupby(["ensemble_index", "unique_stixel_id"], as_index=False, group_keys=False)
                 .pipe(lambda x: x[x.obj.columns])
-                .apply(find_belonged_points, st_indexes_df=window_X_df_indexes_only, X_df=window_X_df, include_groups=False)
-                .reset_index(level=["ensemble_index", "unique_stixel_id"])
+                .apply(find_belonged_points_and_predict, st_indexes_df=window_X_df_indexes_only, X_df=window_X_df, include_groups=False)
             )
 
-            if len(query_results) == 0:
+            if len(window_prediction) == 0:
                 """All points fall out of the grids"""
                 continue
 
-            # predict
-            window_prediction = (
-                query_results
-                .dropna(subset="unique_stixel_id")
-                .groupby("unique_stixel_id", as_index=False, group_keys=False)
-                .pipe(lambda x: x[x.obj.columns])
-                .apply(lambda stixel: self.stixel_predict(stixel), include_groups=False)
-            )
             # print('window_prediction:',window_prediction)
             window_prediction_list.append(window_prediction)
 
         if any([i is not None for i in window_prediction_list]):
             ensemble_prediction = pd.concat(window_prediction_list, axis=0)
             ensemble_prediction = ensemble_prediction.groupby(level=0).mean()
-            ensemble_prediction.index.name = "index"
-            ensemble_prediction = ensemble_prediction.reset_index(drop=False)
         else:
             ensmeble_index = list(window_single_ensemble_df["ensemble_index"])[0]
             warnings.warn(f"No prediction for this ensemble: {ensmeble_index}")

tests/test_model.py

@@ -53,6 +53,9 @@ def test_STEMClassifier():
     importances_by_points = model.assign_feature_importances_by_points(verbosity=0, n_jobs=1)
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
+    
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
 
 
 def test_parallel_STEMClassifier():
@@ -85,7 +88,9 @@ def test_parallel_STEMClassifier():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
-
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_STEMRegressor():
     model = make_STEMRegressor()
     y_train.loc[:] = np.where(y_train > 0, 1, 0)
@@ -116,7 +121,9 @@ def test_STEMRegressor():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
-
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_AdaSTEMClassifier():
     model = make_AdaSTEMClassifier()
     y_train.loc[:] = np.where(y_train > 0, 1, 0)
@@ -147,6 +154,9 @@ def test_AdaSTEMClassifier():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 
 def test_AdaSTEMRegressor():
     model = make_AdaSTEMRegressor()
@@ -188,7 +198,9 @@ def test_AdaSTEMRegressor():
     # score
     score_df = model.score(X_test, y_test)
 
-        
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_parallel_AdaSTEMClassifier():
     model = make_parallel_AdaSTEMClassifier()
     y_train.loc[:] = np.where(y_train > 0, 1, 0)
@@ -219,8 +231,53 @@ def test_parallel_AdaSTEMClassifier():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
+
+def test_AdaSTEMRegressor_return_ensemble():
+    model = make_AdaSTEMRegressor()
+    y_train.loc[:] = np.where(y_train > 0, 1, 0)
+    model = model.fit(X_train, y_train)
+
+    pred_mean, pred_std = model.predict(X_test.reset_index(drop=True), return_std=True, verbosity=1, n_jobs=1)
+    assert np.sum(~np.isnan(pred_mean)) > 0
+    assert np.sum(~np.isnan(pred_std)) > 0
+
+    pred_ = model.predict(X_test, aggregation='median')
+    assert len(pred_) == len(X_test)
+    assert np.sum(np.isnan(pred_)) / len(pred_) <= 0.3
+    
+    pred_return_by_separate_ensembles = model.predict(X_test, return_by_separate_ensembles=True)
+    assert pred_return_by_separate_ensembles.shape[1]==model.ensemble_fold
+    
+    pred = model.predict(X_test)
+    assert len(pred) == len(X_test)
+    assert np.sum(np.isnan(pred)) / len(pred) <= 0.3
+
+    pred_df = pd.DataFrame(
+        {"y_true": np.array(y_test).flatten(), "y_pred": np.where(pred < 0, 0, pred).flatten()}
+    ).dropna()
+    assert len(pred_df) > 0
+
+    eval = AdaSTEM.eval_STEM_res("hurdle", pred_df.y_true, pred_df.y_pred)
+    assert eval["AUC"] >= 0.5
+    assert eval["kappa"] >= 0.2
+    assert eval["Spearman_r"] >= 0.2
 
+    model.calculate_feature_importances()
+    assert model.feature_importances_.shape[0] > 0
 
+    importances_by_points = model.assign_feature_importances_by_points(verbosity=0, n_jobs=1)
+    assert importances_by_points.shape[0] > 0
+    assert importances_by_points.shape[1] == len(x_names) + 3
+    
+    # score
+    score_df = model.score(X_test, y_test)
+    
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_SphereAdaClassifier():
     model = make_SphereAdaClassifier()
     y_train.loc[:] = np.where(y_train > 0, 1, 0)
@@ -255,6 +312,9 @@ def test_SphereAdaClassifier():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_parallel_SphereAdaClassifier():
     model = make_parallel_SphereAdaClassifier()
     y_train.loc[:] = np.where(y_train > 0, 1, 0)
@@ -286,7 +346,9 @@ def test_parallel_SphereAdaClassifier():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
-
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_SphereAdaSTEMRegressor():
     model = make_SphereAdaSTEMRegressor()
     y_train.loc[:] = np.where(y_train > 0, 1, 0)
@@ -317,7 +379,9 @@ def test_SphereAdaSTEMRegressor():
     assert importances_by_points.shape[0] > 0
     assert importances_by_points.shape[1] == len(x_names) + 3
 
-
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 def test_AdaSTEMRegressor_Hurdle_for_AdaSTEM():
     model = make_AdaSTEMRegressor_Hurdle_for_AdaSTEM()
     model = model.fit(X_train, y_train)
@@ -339,7 +403,9 @@ def test_AdaSTEMRegressor_Hurdle_for_AdaSTEM():
     assert eval["kappa"] >= 0.1
     assert eval["Spearman_r"] >= 0.1
 
-
+    ## Then predict on limited samples?
+    pred = model.predict(X_test.sample(n=1, replace=False))
+    
 
 # def test_AdaSTEMRegressor_median():
 #     model = make_AdaSTEMRegressor()