Merge branch 'automl_paper' of https://github.com/suzhoum/autogluon-bench into automl_paper

Author: suzhoum

sample_configs/cloud_configs/bench_all.py

@@ -3,13 +3,13 @@
 seeds = []
 for i in range(n_experiments):
     seeds.append(random.randint(0, 100))
-print(seeds)
 
 seeds = [22, 92, 54, 86, 41]
+
 config_paths = [
     "sample_configs/paper_text_tabular_local_configs.yaml",
-    # "sample_configs/paper_text_local_configs.yaml",
-    # "sample_configs/paper_image_local_configs.yaml",
+    "sample_configs/paper_text_local_configs.yaml",
+    "sample_configs/paper_image_local_configs.yaml",
     # "sample_configs/multimodal_cloud_text_configs.yaml",
     # "sample_configs/multimodal_cloud_text_fs_configs.yaml",
     # "sample_configs/multimodal_cloud_text_tabular_configs.yaml",
@@ -18,13 +18,14 @@
 ]
 frameworks = [
     # "AutoGluon_best_master",
+    # "ablation_base",
     # "ablation_greedy_soup",
     # "ablation_gradient_clip",
-    # "ablation_warmup_steps",
-    # "ablation_cosine_decay",
-    # "ablation_weight_decay",
+    "ablation_warmup_steps",
+    "ablation_cosine_decay",
+    "ablation_weight_decay",
     # "ablation_lr_decay",
-    "autokeras_master",
+    # "autokeras_master",
     # "torch_compile_best",
     # "AutoGluon_best_master",
     # "AutoGluon_high_master",
@@ -47,8 +48,8 @@
     5,
     10
 ]
-module = "autokeras"
-# module = "multimodal"
+# module = "autokeras"
+module = "multimodal"
 
 import yaml
 import os
@@ -58,6 +59,7 @@
 os.makedirs(config_root, exist_ok=True)
 
 for seed in seeds:
+    print("Seed: ", seed)
     for constraint in constraints:
         os.makedirs(f"{config_root}/{constraint}", exist_ok=True)
         for framework in frameworks:
@@ -75,11 +77,11 @@
                         configs["module"] = module
                         configs["seed"] = seed 
                         # configs["custom_dataloader"]["shot"] = shot
-                        # configs["benchmark_name"] = f"{configs['benchmark_name']}-{shot}"
+                        configs["benchmark_name"] = f"{configs['benchmark_name']}-{seed}"
                         new_config_path = os.path.join(config_dir, os.path.basename(config_path))
                         with open(new_config_path, "w") as new_f:
                             yaml.dump(configs, new_f)
-
+                        print("Running config: ", new_config_path)
                         command = ["agbench", "run", new_config_path]
                         subprocess.run(command)