Skip to content

Commit d89bf1b

Browse files
Ubuntuprateekdesai04
Ubuntu
authored and
prateekdesai04
committed
Squashed commits
1 parent 26c7e70 commit d89bf1b

23 files changed

Lines changed: 2570 additions & 6 deletions

‎examples/run_scripts_v5/download_tabforestpfn_weights.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
from autogluon.common.loaders import load_s3
2+
3+
4+
if __name__ == '__main__':
5+
output_prefix = "/home/ubuntu/workspace/tabpfn_weights/"
6+
s3_bucket = "autogluon-zeroshot"
7+
s3_prefix = "tabpfn/tabforestpfn/"
8+
9+
filenames = [
10+
"tabpfn.pt",
11+
"tabforest.pt",
12+
"tabforestpfn.pt",
13+
]
14+
15+
for filename in filenames:
16+
load_s3.download(s3_bucket, f"{s3_prefix}{filename}", f"{output_prefix}/{filename}")
17+
18+
s3_bucket = "mmts-tb"
19+
s3_prefix = "tabular/TabPFN_mix_7_models/"
20+
21+
filenames = [
22+
"TabPFN_mix_7_step_500000.pt",
23+
"TabPFN_mix_7_step_600000.pt",
24+
"TabPFN_mix_7_step_300000.pt",
25+
]
26+
27+
for filename in filenames:
28+
load_s3.download(s3_bucket, f"{s3_prefix}{filename}", f"{output_prefix}/{filename}")
29+
30+
s3_bucket = "autogluon-zeroshot"
31+
s3_prefix = "tabpfn/tabdpt/"
32+
output_prefix = "/home/ubuntu/workspace/tabdpt_weights/"
33+
34+
filenames = [
35+
"tabdpt_76M.ckpt",
36+
]
37+
38+
for filename in filenames:
39+
load_s3.download(s3_bucket, f"{s3_prefix}{filename}", f"{output_prefix}/{filename}")
40+
41+
s3_bucket = "mmts-tb"
42+
s3_prefix = "tabular/"
43+
44+
filenames = [
45+
"TabPFN_real_mix_7_models/model_step_500000.pt",
46+
]
47+
48+
for filename in filenames:
49+
load_s3.download(s3_bucket, f"{s3_prefix}{filename}", f"{output_prefix}/{filename}")

‎examples/run_scripts_v5/run_simple_benchmark_w_simulator_realmlp.py‎

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
from __future__ import annotations
2+
3+
import pandas as pd
4+
5+
from tabrepo import EvaluationRepository, EvaluationRepositoryCollection, Evaluator
6+
from tabrepo.benchmark.experiment import AGModelBagExperiment, ExperimentBatchRunner
7+
8+
from tabrepo.benchmark.models.ag import RealMLPModel
9+
10+
11+
# To re-use the pre-computed results if you have the file "tabrepo_artifacts_realmlp_20250201.zip":
12+
# cd {this_dir}
13+
# unzip tabrepo_artifacts_realmlp_20250201.zip
14+
# Note: This file is currently located in "s3://tabrepo/artifacts/methods/realmlp/tabrepo_artifacts_realmlp_20250201.zip"
15+
# Not publicly available
16+
# You can regenerate this artifact from scratch by running the code. On a 192 CPU core machine, this will take approximately 25 hours.
17+
# If the artifact is present, it will be used and the models will not be re-run.
18+
if __name__ == '__main__':
19+
# Load Context
20+
context_name = "D244_F3_C1530_200" # 200 smallest datasets. To run larger, set to "D244_F3_C1530_200"
21+
expname = "./initial_experiment_simple_simulator" # folder location of all experiment artifacts
22+
ignore_cache = False # set to True to overwrite existing caches and re-run experiments from scratch
23+
24+
# TODO: in future shouldn't require downloading all repo_og preds (100+ GB) before running experiments
25+
# Should only need preds for ensembling part, but not for comparisons
26+
repo_og: EvaluationRepository = EvaluationRepository.from_context(context_name, cache=True)
27+
28+
# df_out = get_feature_info(repo_og)
29+
#
30+
# a = df_out[("int", ("bool",))]
31+
# print(a)
32+
# b = a[a > 0]
33+
# datasets_with_bool = list(b.index)
34+
#
35+
# # Sample for a quick demo
36+
datasets = repo_og.datasets()
37+
# datasets = repo_og.datasets(problem_type="regression")
38+
# datasets_filter = repo_og.datasets(problem_type="binary") + repo_og.datasets(problem_type="multiclass")
39+
# datasets = [d for d in datasets if d in datasets_filter] + repo_og.datasets(problem_type="regression")
40+
# # datasets = datasets[:173]
41+
# datasets_og = datasets
42+
# datasets = [d for d in datasets_og if d in datasets_with_bool] + [d for d in datasets_og if d not in datasets_with_bool]
43+
44+
# datasets = repo_og.datasets(problem_type="regression")
45+
# datasets = datasets[:6] # FIXME: ImputeF crashes on GAMETES_Epistasis_2-Way_1000atts_0_4H_EDM-1_EDM-1_1 fold 0
46+
folds = [0, 1, 2]
47+
# datasets = ["Internet-Advertisements"]
48+
49+
# To run everything:
50+
# datasets = repo_og.datasets()
51+
# folds = repo_og.folds
52+
53+
# TODO: Why is RealMLP slow when running sequentially / not in a bag? Way slower than it should be. Torch threads?
54+
methods = [
55+
AGModelBagExperiment( # 2025/02/01 num_cpus=192, pytabkit==1.1.3
56+
name="RealMLP_c1_BAG_L1_v4_noes_r0",
57+
model_cls=RealMLPModel,
58+
model_hyperparameters={
59+
"random_state": 0,
60+
"use_early_stopping": False,
61+
"use_ls": None,
62+
"bool_to_cat": False,
63+
"impute_bool": True,
64+
},
65+
),
66+
AGModelBagExperiment( # 2025/02/01 num_cpus=192, pytabkit==1.1.3
67+
name="RealMLP_c2_BAG_L1_AutoLS",
68+
model_cls=RealMLPModel,
69+
model_hyperparameters={
70+
"random_state": 0,
71+
"use_early_stopping": False,
72+
"use_ls": "auto",
73+
"bool_to_cat": False,
74+
"impute_bool": True,
75+
},
76+
),
77+
AGModelBagExperiment( # 2025/02/01 num_cpus=192, pytabkit==1.1.3
78+
name="RealMLP_c2_BAG_L1_AutoLS_AUCStop",
79+
model_cls=RealMLPModel,
80+
model_hyperparameters={
81+
"random_state": 0,
82+
"use_early_stopping": False,
83+
"use_ls": "auto",
84+
"bool_to_cat": False,
85+
"impute_bool": True,
86+
# "use_roc_auc_to_stop": True,
87+
},
88+
),
89+
AGModelBagExperiment( # 2025/02/07 num_cpus=192, pytabkit==1.2.1
90+
name="RealMLP_c2_BAG_L1_AutoLS_AUCStop_boolcat_impF_naT",
91+
model_cls=RealMLPModel,
92+
model_hyperparameters={
93+
"random_state": 0,
94+
"use_early_stopping": False,
95+
"use_ls": "auto",
96+
# "use_roc_auc_to_stop": True,
97+
"bool_to_cat": True,
98+
"impute_bool": False,
99+
"name_categories": True,
100+
},
101+
),
102+
AGModelBagExperiment( # 2025/02/12 num_cpus=192, pytabkit==1.2.1
103+
name="RealMLP_c2_BAG_L1_TD",
104+
model_cls=RealMLPModel,
105+
model_hyperparameters={
106+
"random_state": 0,
107+
"use_early_stopping": False,
108+
"use_ls": "auto",
109+
# "use_roc_auc_to_stop": True,
110+
"bool_to_cat": True,
111+
"impute_bool": False,
112+
"name_categories": True,
113+
# "td_s_reg": False,
114+
},
115+
),
116+
# AGModelBagExperiment( # 2025/03/05 num_cpus=192, pytabkit==1.2.1
117+
# name="RealMLP_c1_BAG_L1",
118+
# model_cls=RealMLPModel,
119+
# model_hyperparameters={},
120+
# ),
121+
]
122+
123+
exp_batch_runner = ExperimentBatchRunner(
124+
expname=expname,
125+
task_metadata=repo_og.task_metadata,
126+
cache_path_format="task_first",
127+
)
128+
129+
# results_lst = exp_batch_runner.load_results(
130+
# methods=methods,
131+
# datasets=datasets,
132+
# folds=folds,
133+
# )
134+
135+
results_lst = exp_batch_runner.run(
136+
methods=methods,
137+
datasets=datasets,
138+
folds=folds,
139+
ignore_cache=ignore_cache,
140+
)
141+
142+
repo = exp_batch_runner.repo_from_results(results_lst=results_lst)
143+
144+
# TODO: repo.configs_type should not be None for custom methods
145+
repo.print_info()
146+
147+
save_path = "repo_new"
148+
repo.to_dir(path=save_path) # Load the repo later via `EvaluationRepository.from_dir(save_path)`
149+
150+
print(f"New Configs : {repo.configs()}")
151+
152+
shared_datasets = [d for d in repo.datasets(union=False) if d in repo_og.datasets()]
153+
154+
# repo_tabforestpfn = EvaluationRepository.from_dir(path="tabforestpfn_sim")
155+
# shared_datasets = [d for d in shared_datasets if d in repo_tabforestpfn.datasets(union=False)]
156+
# repo_combined = EvaluationRepositoryCollection(repos=[repo_og, repo, repo_tabforestpfn], config_fallback="ExtraTrees_c1_BAG_L1")
157+
repo_combined = EvaluationRepositoryCollection(repos=[repo_og, repo], config_fallback="ExtraTrees_c1_BAG_L1")
158+
repo_combined = repo_combined.subset(datasets=shared_datasets)
159+
repo_combined.set_config_fallback("ExtraTrees_c1_BAG_L1")
160+
evaluator = Evaluator(repo=repo_combined)
161+
162+
repo_combined.print_info()
163+
164+
comparison_configs_og = [
165+
"RandomForest_c1_BAG_L1",
166+
"ExtraTrees_c1_BAG_L1",
167+
"LightGBM_c1_BAG_L1",
168+
"XGBoost_c1_BAG_L1",
169+
"CatBoost_c1_BAG_L1",
170+
"NeuralNetTorch_c1_BAG_L1",
171+
"NeuralNetFastAI_c1_BAG_L1",
172+
]
173+
174+
comparison_configs = comparison_configs_og + [
175+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_121",
176+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_121_bool_to_cat",
177+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_121_bool_to_cat_false",
178+
179+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_121_bool_to_cat2",
180+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_121_bool_to_cat_false2",
181+
# "RealMLP_c2_BAG_L1_AutoLS_bool_to_cat",
182+
# "RealMLP_c2_BAG_L1_AutoLS_impute_true",
183+
# "RealMLP_c2_BAG_L1_AutoLS_bool_to_cat_impute_true",
184+
185+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_boolcat_imputeT", # FIXME: CRASHES?
186+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_boolcat_imputeF", # FIXME: CRASHES?
187+
188+
# "RealMLP_c1_BAG_L1_v4_noes_r0", # did 600 runs (200x3)
189+
# "RealMLP_c2_BAG_L1_AutoLS", # 200x3
190+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop", # 200x3
191+
"RealMLP_c2_BAG_L1_AutoLS_AUCStop_boolcat_impF_naT", # 200x3
192+
# "RealMLP_c2_BAG_L1_AutoLS_AUCStop_impF_naT", # 175x3
193+
"RealMLP_c2_BAG_L1_TD", # 200x3
194+
# "RealMLP_c1_BAG_L1",
195+
]
196+
197+
evaluator.compute_avg_config_prediction_delta(configs=comparison_configs + ["EBM_BAG_L1", "TabPFNv2_N4_BAG_L1", "TabPFN_Mix7_600000_N4_E30_FIX_BAG_L1"])
198+
199+
# comparison_configs += [
200+
# "TabForestPFN_N4_E10_BAG_L1",
201+
# "TabForestPFN_N4_E30_BAG_L1",
202+
# "TabForestPFN_N4_E50_BAG_L1",
203+
# "TabForestPFN_N1_E10_S4_BAG_L1",
204+
# "TabPFN_Mix7_500000_N4_E30_BAG_L1",
205+
# "TabPFN_Mix7_600000_N4_E30_BAG_L1",
206+
# "TabPFN_Mix7_300000_N4_E30_BAG_L1",
207+
# "TabPFN_Mix7_600000_N4_E30_S4_BAG_L1",
208+
# "TabPFNv2_N4_BAG_L1",
209+
# "EBM_BAG_L1",
210+
# "TabPFN_Mix7_600000_N1_E0_BAG_L1",
211+
# "TabPFN_Mix7_600000_N4_E0_BAG_L1",
212+
# "LightGBM_c1_BAG_L1_V2",
213+
# "TabDPT_N1_E0_BAG_L1",
214+
# "TabRMix7_500000_N1_E0_BAG_L1",
215+
# "TabRMix7_500000_N4_E30_BAG_L1",
216+
# "TabPFN_Mix7_600000_N4_E30_FIX_BAG_L1",
217+
# "TabPFN_Mix7_600000_N4_E50_FIX_BAG_L1",
218+
# "TabPFN_Mix7_600000_N4_E30_FIX_BAG_L1_COMPARISON",
219+
# ]
220+
221+
222+
223+
df_ensemble_results, df_ensemble_weights = repo_combined.evaluate_ensembles(configs=comparison_configs, ensemble_size=40)
224+
df_ensemble_results = df_ensemble_results.reset_index()
225+
df_ensemble_results["framework"] = "ensemble_with_RealMLP_c1"
226+
227+
df_ensemble_results_og, df_ensemble_weights_og = repo_combined.evaluate_ensembles(configs=comparison_configs_og, ensemble_size=40)
228+
df_ensemble_results_og = df_ensemble_results_og.reset_index()
229+
df_ensemble_results_og["framework"] = "ensemble_og"
230+
231+
# from script_utils import load_ag11_bq_baseline
232+
# df_processed_ag11_2024 = load_ag11_bq_baseline(datasets=repo_combined.datasets(), folds=repo_combined.folds, repo=repo_combined)
233+
234+
repo_og.set_config_fallback("ExtraTrees_c1_BAG_L1")
235+
df_zeroshot_portfolio_og = evaluator.zeroshot_portfolio(configs=repo_og.configs())
236+
df_zeroshot_portfolio_og["framework"] = "zeroshot_og"
237+
238+
df_zeroshot_portfolio_w_realmlp = evaluator.zeroshot_portfolio(configs=repo_combined.configs())
239+
df_zeroshot_portfolio_w_realmlp["framework"] = "zeroshot_w_realmlp"
240+
241+
df_zeroshot_portfolio_w_realmlp_single = evaluator.zeroshot_portfolio(configs=repo_og.configs() + ["RealMLP_c2_BAG_L1_TD"])
242+
df_zeroshot_portfolio_w_realmlp_single["framework"] = "zeroshot_w_realmlp_single"
243+
244+
df_zeroshot_portfolio_w_realmlp_2 = evaluator.zeroshot_portfolio(configs=repo_og.configs() + ["RealMLP_c2_BAG_L1_TD", "RealMLP_c2_BAG_L1_AutoLS_AUCStop_boolcat_impF_naT"])
245+
df_zeroshot_portfolio_w_realmlp_2["framework"] = "zeroshot_w_realmlp_2"
246+
247+
df_zeroshot_portfolio_w_realmlp_n5 = evaluator.zeroshot_portfolio(configs=repo_combined.configs(), n_portfolios=10)
248+
df_zeroshot_portfolio_w_realmlp_n5["framework"] = "zeroshot_w_realmlp_n10"
249+
250+
df_zeroshot_portfolio_n5 = evaluator.zeroshot_portfolio(configs=repo_og.configs(), n_portfolios=10)
251+
df_zeroshot_portfolio_n5["framework"] = "zeroshot_og_n10"
252+
253+
results_df = pd.concat([
254+
df_ensemble_results,
255+
df_ensemble_results_og,
256+
# df_processed_ag11_2024,
257+
df_zeroshot_portfolio_og,
258+
df_zeroshot_portfolio_w_realmlp,
259+
df_zeroshot_portfolio_w_realmlp_single,
260+
df_zeroshot_portfolio_w_realmlp_2,
261+
df_zeroshot_portfolio_w_realmlp_n5,
262+
df_zeroshot_portfolio_n5,
263+
], ignore_index=True)
264+
265+
baselines = [
266+
"AutoGluon_bq_4h8c_2023_11_14",
267+
# "AutoGluon_bq_4h8c_2024_10_25",
268+
]
269+
270+
p = evaluator.plot_ensemble_weights(df_ensemble_weights=df_ensemble_weights, figsize=(16, 60))
271+
p.savefig("ensemble_weights_w_RealMLP_c1")
272+
273+
metrics = evaluator.compare_metrics(
274+
results_df=results_df,
275+
datasets=datasets,
276+
folds=folds,
277+
baselines=baselines,
278+
configs=comparison_configs,
279+
)
280+
281+
metrics_tmp = metrics.reset_index(drop=False)
282+
283+
with pd.option_context("display.max_rows", None, "display.max_columns", None, "display.width", 1000):
284+
print(f"Config Metrics Example:\n{metrics.head(100)}")
285+
286+
evaluator_output = evaluator.plot_overall_rank_comparison(
287+
results_df=metrics,
288+
save_dir=expname,
289+
evaluator_kwargs={
290+
"treat_folds_as_datasets": True,
291+
"frameworks_compare_vs_all": [
292+
"RealMLP_c2_BAG_L1_TD",
293+
"zeroshot_w_realmlp",
294+
],
295+
},
296+
)

0 commit comments

Comments
 (0)