Add Lambert W x F distributions to XGBoostLSS

.gitignore

@@ -2,6 +2,8 @@
 .idea/
 dist/
 latex_distributions
+venv/
+xgboostlss.egg-info/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

docs/examples/Gaussian_Regression.ipynb

@@ -1167,7 +1167,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -1181,7 +1181,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.0"
+   "version": "3.8.10"
   }
  },
  "nbformat": 4,

pyproject.toml

@@ -10,7 +10,7 @@ license = { text = "Apache License 2.0" }
 requires-python = ">=3.10"
 dependencies = [
     "xgboost~=2.0.3",
-    "torch~=2.1.2",
+    "torch>=2.1.2",
     "pyro-ppl~=1.8.6",
     "optuna~=3.5.0",
     "properscoring~=0.1",
@@ -23,7 +23,8 @@ dependencies = [
     "seaborn~=0.13.1",
     "tqdm~=4.66.1",
     "matplotlib~=3.8.2",
-    "ipython~=8.20.0"
+    "ipython~=8.20.0",
+    "torchlambertw @ git+ssh://[email protected]/gmgeorg/torchlambertw.git#egg=torchlambertw-0.0.3"
 ]
 
 [project.optional-dependencies]
@@ -53,3 +54,7 @@ xgboostlss = ["datasets/*.csv"]
 [build-system]
 requires = ["setuptools>=61", "wheel"]
 build-backend = "setuptools.build_meta"
+
+dependencies = [
+    "torchlambertw @ git+ssh://[email protected]/gmgeorg/torchlambertw.git#egg=torchlambertw-0.0.3"
+]

tests/test_distribution_utils/test_dist_select.py

@@ -9,16 +9,25 @@
     LogNormal,
     Weibull,
     Gumbel,
-    Laplace)
+    Laplace,
+)
 from xgboostlss.distributions.Mixture import *
 from xgboostlss.distributions.SplineFlow import *
 from xgboostlss.distributions.MVN import *
 from xgboostlss.distributions.MVT import *
 from xgboostlss.distributions.MVN_LoRa import *
-from xgboostlss.distributions.distribution_utils import DistributionClass as univariate_dist_class
-from xgboostlss.distributions.multivariate_distribution_utils import Multivariate_DistributionClass as multivariate_dist_class
+from xgboostlss.distributions.distribution_utils import (
+    DistributionClass as univariate_dist_class,
+)
+from xgboostlss.distributions.multivariate_distribution_utils import (
+    Multivariate_DistributionClass as multivariate_dist_class,
+)
 from xgboostlss.distributions.flow_utils import NormalizingFlowClass as flow_dist_class
-from xgboostlss.distributions.mixture_distribution_utils import MixtureDistributionClass as mixture_dist_class
+from xgboostlss.distributions.mixture_distribution_utils import (
+    MixtureDistributionClass as mixture_dist_class,
+)
+
+import xgboostlss.distributions.distribution_utils as du
 
 
 class TestClass(BaseTestClass):
@@ -28,38 +37,58 @@ class TestClass(BaseTestClass):
     def test_univar_dist_select(self):
         # Create data for testing
         target = np.array([0.2, 0.4, 0.6, 0.8]).reshape(-1, 1)
-        candidate_distributions = [Beta, Gaussian, StudentT, Gamma, Cauchy, LogNormal, Weibull, Gumbel, Laplace]
+        candidate_distributions = [
+            Beta.Beta(),
+            Gaussian.Gaussian(),
+            StudentT.StudentT(),
+            Gamma.Gamma(),
+            Cauchy.Cauchy(),
+            LogNormal.LogNormal(),
+            Weibull.Weibull(),
+            Gumbel.Gumbel(),
+            Laplace.Laplace(),
+        ]
 
         # Call the function
-        dist_df = univariate_dist_class().dist_select(
+        dist_df = du.dist_select(
             target, candidate_distributions, plot=False, max_iter=2
         ).reset_index(drop=True)
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
         assert not dist_df.isna().any().any()
         assert isinstance(dist_df["distribution"].values[0], str)
-        assert np.issubdtype(dist_df["nll"].dtype, np.float64)
-        assert not np.isnan(dist_df["nll"].values).any()
-        assert not np.isinf(dist_df["nll"].values).any()
+        assert np.issubdtype(dist_df["loss"].dtype, np.float64)
+        assert not np.isnan(dist_df["loss"].values).any()
+        assert not np.isinf(dist_df["loss"].values).any()
 
     def test_univar_dist_select_plot(self):
         # Create data for testing
         target = np.array([0.2, 0.4, 0.6, 0.8]).reshape(-1, 1)
-        candidate_distributions = [Beta, Gaussian, StudentT, Gamma, Cauchy, LogNormal, Weibull, Gumbel, Laplace]
+        candidate_distributions = [
+            Beta.Beta(),
+            Gaussian.Gaussian(),
+            StudentT.StudentT(),
+            Gamma.Gamma(),
+            Cauchy.Cauchy(),
+            LogNormal.LogNormal(),
+            Weibull.Weibull(),
+            Gumbel.Gumbel(),
+            Laplace.Laplace(),
+        ]
 
         # Call the function
-        dist_df = univariate_dist_class().dist_select(
+        dist_df = du.dist_select(
             target, candidate_distributions, plot=True, max_iter=2
         ).reset_index(drop=True)
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
         assert not dist_df.isna().any().any()
         assert isinstance(dist_df["distribution"].values[0], str)
-        assert np.issubdtype(dist_df["nll"].dtype, np.float64)
-        assert not np.isnan(dist_df["nll"].values).any()
-        assert not np.isinf(dist_df["nll"].values).any()
+        assert np.issubdtype(dist_df["loss"].dtype, np.float64)
+        assert not np.isnan(dist_df["loss"].values).any()
+        assert not np.isinf(dist_df["loss"].values).any()
 
     ####################################################################################################################
     # Normalizing Flows
@@ -71,14 +100,23 @@ def test_flow_select(self):
         target_support = "real"
 
         candidate_flows = [
-            SplineFlow(target_support=target_support, count_bins=2, bound=bound, order="linear"),
-            SplineFlow(target_support=target_support, count_bins=2, bound=bound, order="quadratic")
+            SplineFlow(
+                target_support=target_support, count_bins=2, bound=bound, order="linear"
+            ),
+            SplineFlow(
+                target_support=target_support,
+                count_bins=2,
+                bound=bound,
+                order="quadratic",
+            ),
         ]
 
         # Call the function
-        dist_df = flow_dist_class().flow_select(
-            target, candidate_flows, plot=False, max_iter=2
-        ).reset_index(drop=True)
+        dist_df = (
+            flow_dist_class()
+            .flow_select(target, candidate_flows, plot=False, max_iter=2)
+            .reset_index(drop=True)
+        )
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
@@ -95,14 +133,23 @@ def test_flow_select_plot(self):
         target_support = "real"
 
         candidate_flows = [
-            SplineFlow(target_support=target_support, count_bins=2, bound=bound, order="linear"),
-            SplineFlow(target_support=target_support, count_bins=2, bound=bound, order="quadratic")
+            SplineFlow(
+                target_support=target_support, count_bins=2, bound=bound, order="linear"
+            ),
+            SplineFlow(
+                target_support=target_support,
+                count_bins=2,
+                bound=bound,
+                order="quadratic",
+            ),
         ]
 
         # Call the function
-        dist_df = flow_dist_class().flow_select(
-            target, candidate_flows, plot=True, max_iter=2
-        ).reset_index(drop=True)
+        dist_df = (
+            flow_dist_class()
+            .flow_select(target, candidate_flows, plot=True, max_iter=2)
+            .reset_index(drop=True)
+        )
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
@@ -127,13 +174,15 @@ def test_mixture_dist_select(self):
             Mixture(LogNormal.LogNormal()),
             Mixture(Weibull.Weibull()),
             Mixture(Gumbel.Gumbel()),
-            Mixture(Laplace.Laplace())
+            Mixture(Laplace.Laplace()),
         ]
 
         # Call the function
-        dist_df = mixture_dist_class().dist_select(
-            target, candidate_distributions, plot=False, max_iter=2
-        ).reset_index(drop=True)
+        dist_df = (
+            mixture_dist_class()
+            .dist_select(target, candidate_distributions, plot=False, max_iter=2)
+            .reset_index(drop=True)
+        )
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
@@ -155,13 +204,15 @@ def test_mixture_dist_select_plot(self):
             Mixture(LogNormal.LogNormal()),
             Mixture(Weibull.Weibull()),
             Mixture(Gumbel.Gumbel()),
-            Mixture(Laplace.Laplace())
+            Mixture(Laplace.Laplace()),
         ]
 
         # Call the function
-        dist_df = mixture_dist_class().dist_select(
-            target, candidate_distributions, plot=True, max_iter=2
-        ).reset_index(drop=True)
+        dist_df = (
+            mixture_dist_class()
+            .dist_select(target, candidate_distributions, plot=True, max_iter=2)
+            .reset_index(drop=True)
+        )
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
@@ -179,16 +230,16 @@ def test_multivar_dist_select(self):
         multivar_dist_class = MVN()
         target = np.arange(0.1, 0.9, 0.1)
         target = multivar_dist_class.target_append(
-            target,
-            multivar_dist_class.n_targets,
-            multivar_dist_class.n_dist_param
-        )[:, :multivar_dist_class.n_targets]
+            target, multivar_dist_class.n_targets, multivar_dist_class.n_dist_param
+        )[:, : multivar_dist_class.n_targets]
         candidate_distributions = [MVN(), MVT(), MVN_LoRa()]
 
         # Call the function
-        dist_df = multivariate_dist_class().dist_select(
-            target, candidate_distributions, plot=False, max_iter=2
-        ).reset_index(drop=True)
+        dist_df = (
+            multivariate_dist_class()
+            .dist_select(target, candidate_distributions, plot=False, max_iter=2)
+            .reset_index(drop=True)
+        )
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)
@@ -203,16 +254,16 @@ def test_multivar_dist_select_plot(self):
         multivar_dist_class = MVN()
         target = np.arange(0.1, 0.9, 0.1)
         target = multivar_dist_class.target_append(
-            target,
-            multivar_dist_class.n_targets,
-            multivar_dist_class.n_dist_param
-        )[:, :multivar_dist_class.n_targets]
+            target, multivar_dist_class.n_targets, multivar_dist_class.n_dist_param
+        )[:, : multivar_dist_class.n_targets]
         candidate_distributions = [MVN(), MVT(), MVN_LoRa()]
 
         # Call the function
-        dist_df = multivariate_dist_class().dist_select(
-            target, candidate_distributions, plot=True, ncol=1, max_iter=2
-        ).reset_index(drop=True)
+        dist_df = (
+            multivariate_dist_class()
+            .dist_select(target, candidate_distributions, plot=True, ncol=1, max_iter=2)
+            .reset_index(drop=True)
+        )
 
         # Assertions
         assert isinstance(dist_df, pd.DataFrame)

tests/test_distribution_utils/test_draw_samples.py

@@ -8,11 +8,13 @@ class TestClass(BaseTestClass):
     def test_draw_samples(self, dist_class):
         if dist_class.dist.univariate:
             # Create data for testing
-            predt_params = pd.DataFrame(np.array([0.5 for _ in range(dist_class.dist.n_dist_param)], dtype="float32")).T
-
+            predt_params_dict = {
+                k: [float(v(torch.tensor(0.0)).numpy())]
+                for k, v in dist_class.dist.param_dict.items()
+            }
+            predt_params = pd.DataFrame.from_dict(predt_params_dict).astype("float32")
             # Call the function
             dist_samples = dist_class.dist.draw_samples(predt_params)
-
             # Assertions
             if str(dist_class.dist).split(".")[2] != "Expectile":
                 assert isinstance(dist_samples, (pd.DataFrame, type(None)))
@@ -25,18 +27,26 @@ def test_draw_samples(self, dist_class):
             predt = np.array([0.5 for _ in range(dist_class.dist.n_dist_param)])
             predt = predt.reshape(-1, dist_class.dist.n_dist_param)
             predt = [
-                torch.tensor(predt[:, i].reshape(-1, 1), requires_grad=False) for i in
-                range(dist_class.dist.n_dist_param)
+                torch.tensor(predt[:, i].reshape(-1, 1), requires_grad=False)
+                for i in range(dist_class.dist.n_dist_param)
             ]
-            predt_transformed = dist_class.dist.param_transform(predt, dist_class.dist.param_dict,
-                                                                dist_class.dist.n_targets, rank=dist_class.dist.rank,
-                                                                n_obs=n_obs)
+            predt_transformed = dist_class.dist.param_transform(
+                predt,
+                dist_class.dist.param_dict,
+                dist_class.dist.n_targets,
+                rank=dist_class.dist.rank,
+                n_obs=n_obs,
+            )
 
             # Call the function
             if dist_class.dist.distribution.__name__ == "Dirichlet":
-                dist_kwargs = dict(zip(dist_class.dist.distribution_arg_names, [predt_transformed]))
+                dist_kwargs = dict(
+                    zip(dist_class.dist.distribution_arg_names, [predt_transformed])
+                )
             else:
-                dist_kwargs = dict(zip(dist_class.dist.distribution_arg_names, predt_transformed))
+                dist_kwargs = dict(
+                    zip(dist_class.dist.distribution_arg_names, predt_transformed)
+                )
             dist_pred = dist_class.dist.distribution(**dist_kwargs)
             dist_samples = dist_class.dist.draw_samples(dist_pred)
 
@@ -47,7 +57,11 @@ def test_draw_samples(self, dist_class):
 
     def test_draw_samples_mixture(self, mixture_class):
         # Create data for testing
-        predt_params = pd.DataFrame(np.array([0.5 for _ in range(mixture_class.dist.n_dist_param)], dtype="float32")).T
+        predt_params = pd.DataFrame(
+            np.array(
+                [0.5 for _ in range(mixture_class.dist.n_dist_param)], dtype="float32"
+            )
+        ).T
 
         # Call the function
         dist_samples = mixture_class.dist.draw_samples(predt_params)
@@ -56,5 +70,3 @@ def test_draw_samples_mixture(self, mixture_class):
         assert isinstance(dist_samples, (pd.DataFrame, type(None)))
         assert not dist_samples.isna().any().any()
         assert not np.isinf(dist_samples).any().any()
-
-