Cleanup (uncertainty-toolbox#17)

* Clean Up

* Fixed bug.

Author: IanChar

examples/viz_minimal.py

@@ -8,9 +8,10 @@
 import uncertainty_toolbox.data as udata
 
 import neatplot
+
 neatplot.set_style()
-neatplot.update_rc('figure.dpi', 150)
-neatplot.update_rc('text.usetex', False)
+neatplot.update_rc("figure.dpi", 150)
+neatplot.update_rc("text.usetex", False)
 
 
 # Set random seed
@@ -20,10 +21,10 @@
 (y_pred, y_std, y_true) = udata.synthetic_arange_random()
 
 # Print details about the synthetic results
-print('* y_true: {}'.format(y_true))
-print('* y_pred: {}'.format(y_pred))
-print('* |y_true - y_pred|: {}'.format(np.abs(y_true - y_pred)))
-print('* y_std: {}'.format(y_std))
+print("* y_true: {}".format(y_true))
+print("* y_pred: {}".format(y_pred))
+print("* |y_true - y_pred|: {}".format(np.abs(y_true - y_pred)))
+print("* y_std: {}".format(y_std))
 
 # Plot
 uviz.plot_intervals(y_pred, y_std, y_true, show=True)

examples/viz_recalibrate.py

@@ -12,7 +12,6 @@
 )
 import uncertainty_toolbox.viz as uviz
 from uncertainty_toolbox.recalibration import iso_recal
-from uncertainty_toolbox.viz import plot_calibration
 
 import neatplot
 
@@ -44,27 +43,18 @@ def update_rc_params():
 
 
 # List of predictive means and standard deviations
-pred_mean_list = [
-    f,
-    # f + 0.1,
-    # f - 0.1,
-    # f + 0.25,
-    # f - 0.25,
-]
+pred_mean_list = [f]
 
 pred_std_list = [
     std * 0.5,  # overconfident
     std * 2.0,  # underconfident
-    # std,                # correct
 ]
 
 # Loop through, make plots, and compute metrics
 for i, pred_mean in enumerate(pred_mean_list):
     for j, pred_std in enumerate(pred_std_list):
         # Before recalibration
-        exp_props, obs_props = get_proportion_lists_vectorized(
-            pred_mean, pred_std, y
-        )
+        exp_props, obs_props = get_proportion_lists_vectorized(pred_mean, pred_std, y)
         recal_model = None
         mace = umetrics.mean_absolute_calibration_error(
             pred_mean, pred_std, y, recal_model=recal_model
@@ -76,9 +66,7 @@ def update_rc_params():
             pred_mean, pred_std, y, recal_model=recal_model
         )
         print("Before Recalibration")
-        print(
-            "   MACE: {:.5f}, RMSCE: {:.5f}, MA: {:.5f}".format(mace, rmsce, ma)
-        )
+        print("   MACE: {:.5f}, RMSCE: {:.5f}, MA: {:.5f}".format(mace, rmsce, ma))
 
         uviz.plot_calibration(
             pred_mean,
@@ -104,11 +92,9 @@ def update_rc_params():
             pred_mean, pred_std, y, recal_model=recal_model
         )
         print(" After Recalibration")
-        print(
-            "   MACE: {:.5f}, RMSCE: {:.5f}, MA: {:.5f}".format(mace, rmsce, ma)
-        )
+        print("   MACE: {:.5f}, RMSCE: {:.5f}, MA: {:.5f}".format(mace, rmsce, ma))
 
-        plot_calibration(
+        uviz.plot_calibration(
             pred_mean,
             pred_std,
             y,

examples/viz_synth_sine.py

@@ -4,15 +4,15 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib import rc
-#plt.ion()
 
 import uncertainty_toolbox.data as udata
 import uncertainty_toolbox.metrics as umetrics
 import uncertainty_toolbox.viz as uviz
 
 import neatplot
+
 neatplot.set_style()
-neatplot.update_rc('text.usetex', False) # Set to True for system latex
+neatplot.update_rc("text.usetex", False)  # Set to True for system latex
 
 
 # Set random seed
@@ -26,17 +26,17 @@
 savefig = False
 
 
-def save_figure(name_str, file_type='png'):
+def save_figure(name_str, file_type="png"):
     """Save figure, or do nothing if savefig is False."""
     if savefig:
         neatplot.save_figure(name_str, file_type)
 
 
 def update_rc_params():
     """Update matplotlib rc params."""
-    plt.rcParams.update({'font.size': 14})
-    plt.rcParams.update({'xtick.labelsize': 14})
-    plt.rcParams.update({'ytick.labelsize': 14})
+    plt.rcParams.update({"font.size": 14})
+    plt.rcParams.update({"xtick.labelsize": 14})
+    plt.rcParams.update({"ytick.labelsize": 14})
 
 
 def make_plots(pred_mean, pred_std, idx1, idx2):
@@ -48,38 +48,32 @@ def make_plots(pred_mean, pred_std, idx1, idx2):
 
     # Make xy plot
     uviz.plot_xy(pred_mean, pred_std, y, x, n_subset=300, ylims=ylims, xlims=[0, 15])
-    save_figure(f'xy_{idx1}_{idx2}')
+    save_figure(f"xy_{idx1}_{idx2}")
     plt.show()
 
     # Make intervals plot
     uviz.plot_intervals(pred_mean, pred_std, y, n_subset=n_subset, ylims=ylims)
-    save_figure(f'intervals_{idx1}_{idx2}')
+    save_figure(f"intervals_{idx1}_{idx2}")
     plt.show()
 
     # Make calibration plot
     uviz.plot_calibration(pred_mean, pred_std, y)
-    save_figure(f'calibration_{idx1}_{idx2}')
+    save_figure(f"calibration_{idx1}_{idx2}")
     plt.show()
 
     # Make ordered intervals plot
     uviz.plot_intervals_ordered(pred_mean, pred_std, y, n_subset=n_subset, ylims=ylims)
-    save_figure(f'intervals_ordered_{idx1}_{idx2}')
+    save_figure(f"intervals_ordered_{idx1}_{idx2}")
     plt.show()
 
 
 # List of predictive means and standard deviations
-pred_mean_list = [
-    f,
-    #f + 0.1,
-    #f - 0.1,
-    #f + 0.25,
-    #f - 0.25,
-]
+pred_mean_list = [f]
 
 pred_std_list = [
-    std * 0.5,          # overconfident
-    std * 2.0,          # underconfident
-    std,                # correct
+    std * 0.5,  # overconfident
+    std * 2.0,  # underconfident
+    std,  # correct
 ]
 
 # Loop through, make plots, and compute metrics
@@ -91,4 +85,4 @@ def make_plots(pred_mean, pred_std, idx1, idx2):
 
         make_plots(pred_mean, pred_std, i, j)
 
-        print(f'MACE: {mace}, RMSCE: {rmsce}, MA: {ma}')
+        print(f"MACE: {mace}, RMSCE: {rmsce}, MA: {ma}")

uncertainty_toolbox/__init__.py

@@ -8,7 +8,7 @@
     curvy_cosine,
 )
 
-from .metrics import(
+from .metrics import (
     get_all_metrics,
 )
 

uncertainty_toolbox/data.py

@@ -12,7 +12,7 @@ def synthetic_arange_random(num_points=10):
     """
     y_true = np.arange(num_points)
     y_pred = np.arange(num_points) + np.random.random((num_points,))
-    y_std = np.abs(y_true - y_pred) + .1 * np.random.random((num_points,))
+    y_std = np.abs(y_true - y_pred) + 0.1 * np.random.random((num_points,))
 
     return (y_pred, y_std, y_true)
 

uncertainty_toolbox/metrics.py

@@ -34,12 +34,12 @@
     "crps": "CRPS",
     "check": "Check Score",
     "interval": "Interval Score",
-    "rms_adv_group_cal": "Root-mean-squared Adversarial Group Calibration Error",
+    "rms_adv_group_cal": ("Root-mean-squared Adversarial Group " "Calibration Error"),
     "ma_adv_group_cal": "Mean-absolute Adversarial Group Calibration Error",
 }
 
 
-def get_all_accuracy_metrics(y_pred, y_true, verbose):
+def get_all_accuracy_metrics(y_pred, y_true, verbose=True):
 
     if verbose:
         print(" (1/n) Calculating accuracy metrics")
@@ -48,7 +48,7 @@ def get_all_accuracy_metrics(y_pred, y_true, verbose):
     return acc_metrics
 
 
-def get_all_average_calibration(y_pred, y_std, y_true, num_bins, verbose):
+def get_all_average_calibration(y_pred, y_std, y_true, num_bins, verbose=True):
 
     if verbose:
         print(" (2/n) Calculating average calibration metrics")
@@ -67,7 +67,9 @@ def get_all_average_calibration(y_pred, y_std, y_true, num_bins, verbose):
     return cali_metrics
 
 
-def get_all_adversarial_group_calibration(y_pred, y_std, y_true, num_bins, verbose):
+def get_all_adversarial_group_calibration(
+    y_pred, y_std, y_true, num_bins, verbose=True
+):
 
     adv_group_cali_metrics = {}
     if verbose:
@@ -80,6 +82,7 @@ def get_all_adversarial_group_calibration(y_pred, y_std, y_true, num_bins, verbo
         y_true,
         cali_type="mean_abs",
         num_bins=num_bins,
+        verbose=verbose,
     )
     ma_adv_group_size = ma_adv_group_cali.group_size
     ma_adv_group_cali_score_mean = ma_adv_group_cali.score_mean
@@ -99,6 +102,7 @@ def get_all_adversarial_group_calibration(y_pred, y_std, y_true, num_bins, verbo
         y_true,
         cali_type="root_mean_sq",
         num_bins=num_bins,
+        verbose=verbose,
     )
     rms_adv_group_size = rms_adv_group_cali.group_size
     rms_adv_group_cali_score_mean = rms_adv_group_cali.score_mean
@@ -112,7 +116,7 @@ def get_all_adversarial_group_calibration(y_pred, y_std, y_true, num_bins, verbo
     return adv_group_cali_metrics
 
 
-def get_all_sharpness_metrics(y_std, verbose):
+def get_all_sharpness_metrics(y_std, verbose=True):
 
     if verbose:
         print(" (4/n) Calculating sharpness metrics")
@@ -123,7 +127,9 @@ def get_all_sharpness_metrics(y_std, verbose):
     return sharp_metrics
 
 
-def get_all_scoring_rule_metrics(y_pred, y_std, y_true, resolution, scaled, verbose):
+def get_all_scoring_rule_metrics(
+    y_pred, y_std, y_true, resolution, scaled, verbose=True
+):
 
     if verbose:
         print(" (n/n) Calculating proper scoring rule metrics")
@@ -143,15 +149,15 @@ def get_all_scoring_rule_metrics(y_pred, y_std, y_true, resolution, scaled, verb
 
 def _print_adversarial_group_calibration(adv_group_metric_dic, print_group_num=3):
 
-    for adv_group_cali_type, adv_group_cali_dic in adv_group_metric_dic.items():
-        num_groups = adv_group_cali_dic["group_sizes"].shape[0]
+    for a_group_cali_type, a_group_cali_dic in adv_group_metric_dic.items():
+        num_groups = a_group_cali_dic["group_sizes"].shape[0]
         print_idxs = [int(x) for x in np.linspace(1, num_groups - 1, print_group_num)]
-        print("  {}".format(METRIC_NAMES[adv_group_cali_type]))
+        print("  {}".format(METRIC_NAMES[a_group_cali_type]))
         for idx in print_idxs:
             print(
                 "     Group Size: {:.2f} -- Calibration Error: {:.3f}".format(
-                    adv_group_cali_dic["group_sizes"][idx],
-                    adv_group_cali_dic["adv_group_cali_mean"][idx],
+                    a_group_cali_dic["group_sizes"][idx],
+                    a_group_cali_dic["adv_group_cali_mean"][idx],
                 )
             )
 
@@ -160,44 +166,45 @@ def get_all_metrics(
     y_pred, y_std, y_true, num_bins=100, resolution=99, scaled=True, verbose=True
 ):
 
-    """ Accuracy """
+    # Accuracy
     accuracy_metrics = get_all_accuracy_metrics(y_pred, y_true, verbose)
 
-    """ Calibration """
+    # Calibration
     calibration_metrics = get_all_average_calibration(
         y_pred, y_std, y_true, num_bins, verbose
     )
 
-    """ Adversarial Group Calibration """
+    # Adversarial Group Calibration
     adv_group_cali_metrics = get_all_adversarial_group_calibration(
         y_pred, y_std, y_true, num_bins, verbose
     )
 
-    """ Sharpness """
+    # Sharpness
     sharpness_metrics = get_all_sharpness_metrics(y_std, verbose)
 
-    """ Proper Scoring Rules """
+    # Proper Scoring Rules
     scoring_rule_metrics = get_all_scoring_rule_metrics(
         y_pred, y_std, y_true, resolution, scaled, verbose
     )
-    print("**Finished Calculating All Metrics**")
 
     # Print all outputs
-    print("\n")
-    print(" Accuracy Metrics ".center(60, "="))
-    for acc_metric, acc_val in accuracy_metrics.items():
-        print("  {:<13} {:.3f}".format(METRIC_NAMES[acc_metric], acc_val))
-    print(" Average Calibration Metrics ".center(60, "="))
-    for cali_metric, cali_val in calibration_metrics.items():
-        print("  {:<37} {:.3f}".format(METRIC_NAMES[cali_metric], cali_val))
-    print(" Adversarial Group Calibration Metrics ".center(60, "="))
-    _print_adversarial_group_calibration(adv_group_cali_metrics, print_group_num=3)
-    print(" Sharpness Metrics ".center(60, "="))
-    for sharp_metric, sharp_val in sharpness_metrics.items():
-        print("  {:}   {:.3f}".format(METRIC_NAMES[sharp_metric], sharp_val))
-    print(" Scoring Rule Metrics ".center(60, "="))
-    for sr_metric, sr_val in scoring_rule_metrics.items():
-        print("  {:<25} {:.3f}".format(METRIC_NAMES[sr_metric], sr_val))
+    if verbose:
+        print("**Finished Calculating All Metrics**")
+        print("\n")
+        print(" Accuracy Metrics ".center(60, "="))
+        for acc_metric, acc_val in accuracy_metrics.items():
+            print("  {:<13} {:.3f}".format(METRIC_NAMES[acc_metric], acc_val))
+        print(" Average Calibration Metrics ".center(60, "="))
+        for cali_metric, cali_val in calibration_metrics.items():
+            print("  {:<37} {:.3f}".format(METRIC_NAMES[cali_metric], cali_val))
+        print(" Adversarial Group Calibration Metrics ".center(60, "="))
+        _print_adversarial_group_calibration(adv_group_cali_metrics, print_group_num=3)
+        print(" Sharpness Metrics ".center(60, "="))
+        for sharp_metric, sharp_val in sharpness_metrics.items():
+            print("  {:}   {:.3f}".format(METRIC_NAMES[sharp_metric], sharp_val))
+        print(" Scoring Rule Metrics ".center(60, "="))
+        for sr_metric, sr_val in scoring_rule_metrics.items():
+            print("  {:<25} {:.3f}".format(METRIC_NAMES[sr_metric], sr_val))
 
     all_scores = {
         "accuracy": accuracy_metrics,
@@ -208,10 +215,3 @@ def get_all_metrics(
     }
 
     return all_scores
-
-
-if __name__ == "__main__":
-    y_pred = np.array([1, 2, 3, 4])
-    y_std = np.array([1, 2, 3, 4])
-    y_true = np.array([1.3, 2.3, 3.3, 4])
-    get_all_metrics(y_pred, y_std, y_true)

uncertainty_toolbox/metrics_accuracy.py

@@ -1,7 +1,6 @@
 """
 Metrics for assessing the quality of predictive uncertainty quantification.
 """
-
 import numpy as np
 from sklearn.metrics import (
     mean_absolute_error,
@@ -10,8 +9,6 @@
     median_absolute_error,
 )
 
-""" Error, Calibration, Sharpness Metrics """
-
 
 def prediction_error_metrics(y_pred, y_true):
     """