diff --git a/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py b/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py index 0b0cb3cde..e81a50ac6 100644 --- a/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py +++ b/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py @@ -103,7 +103,6 @@ def objective(trial, benchmark, valid_budgets, configspace): cfg.benchmark.algo, device=cfg.benchmark.device) sampler = TPESampler(seed=cfg.optimizer.seed) - study = optuna.create_study(direction='minimize', sampler=sampler) if cfg.optimizer.type == 'tpe_md': pruner = MedianPruner() sh_iters = precompute_sh_iters(cfg.optimizer.min_budget, @@ -124,7 +123,9 @@ def objective(trial, benchmark, valid_budgets, configspace): ] else: raise NotImplementedError - + study = optuna.create_study(direction='minimize', + sampler=sampler, + pruner=pruner) study.optimize(func=partial( objective, benchmark=benchmark, diff --git a/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml b/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml index c2514c62e..224875cdc 100644 --- a/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml +++ b/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml @@ -17,7 +17,7 @@ data: splits: [0.8,0.2,0.0] batch_size: 32 num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 0.5}] diff --git a/benchmark/FedHPOBench/scripts/cnn/femnist.yaml b/benchmark/FedHPOBench/scripts/cnn/femnist.yaml index c64356b44..58312f120 100644 --- a/benchmark/FedHPOBench/scripts/cnn/femnist.yaml +++ b/benchmark/FedHPOBench/scripts/cnn/femnist.yaml @@ -17,7 +17,7 @@ data: splits: [0.6,0.2,0.2] batch_size: 16 subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] num_workers: 0 model: type: convnet2 diff --git a/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml b/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml index d422df2bd..8dbe21f2b 100644 --- a/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml +++ b/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml @@ -28,7 +28,7 @@ data: splits: [0.6,0.2,0.2] batch_size: 16 subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] num_workers: 0 model: type: convnet2 diff --git a/federatedscope/autotune/algos.py b/federatedscope/autotune/algos.py index cf09c72ba..f29b4e607 100644 --- a/federatedscope/autotune/algos.py +++ b/federatedscope/autotune/algos.py @@ -13,8 +13,10 @@ from federatedscope.core.auxiliaries.worker_builder import get_client_cls, \ get_server_cls from federatedscope.core.auxiliaries.runner_builder import get_runner +from federatedscope.core.configs.yacs_config import CfgNode from federatedscope.autotune.utils import parse_search_space, \ - config2cmdargs, config2str, summarize_hpo_results, log2wandb + config2cmdargs, config2str, summarize_hpo_results, log2wandb, \ + flatten2nestdict logger = logging.getLogger(__name__) @@ -70,14 +72,17 @@ def get_scheduler(init_cfg, client_cfgs=None): client_cfgs: client-specific configuration """ - if init_cfg.hpo.scheduler in [ - 'sha', 'rs', 'bo_kde', 'bohb', 'hb', 'bo_gp', 'bo_rf' - ]: - scheduler = SuccessiveHalvingAlgo(init_cfg, client_cfgs) + # TODO: fix wrap_sha + support_optimizer = [ + 'sha', 'rs', 'bo_kde', 'hb', 'bohb', 'wrap_rs', 'wrap_bo_kde', + 'wrap_hb', 'wrap_bohb', 'bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf', + 'multi' + ] + assert init_cfg.hpo.scheduler in support_optimizer, \ + f'`init_cfg.hpo.scheduler` must be one of {support_optimizer}.' + scheduler = SuccessiveHalvingAlgo(init_cfg, client_cfgs) # elif init_cfg.hpo.scheduler == 'pbt': # scheduler = PBT(init_cfg) - elif init_cfg.hpo.scheduler.startswith('wrap', client_cfgs): - scheduler = SHAWrapFedex(init_cfg) return scheduler @@ -100,6 +105,16 @@ def __init__(self, cfg, client_cfgs=None): os.makedirs(self._cfg.hpo.working_folder, exist_ok=True) self._search_space = parse_search_space(self._cfg.hpo.ss) + # Convert to client_cfg + if self._cfg.hpo.personalized_ss: + # Do not support wrap_scheduler + client_num = self._cfg.federate.client_num + ss_client = CS.ConfigurationSpace() + for i in range(1, client_num + 1): + ss_client.add_configuration_space(f'client_{i}', + self._search_space, + delimiter='.') + self._search_space = ss_client self._init_configs = self._setup() logger.info(self._init_configs) @@ -159,7 +174,14 @@ def _evaluate(self, configs): thread_results[available_worker].clear() trial_cfg = self._cfg.clone() - trial_cfg.merge_from_list(config2cmdargs(config)) + if self._cfg.hpo.personalized_ss: + if isinstance(self._client_cfgs, CS.Configuration): + self._client_cfgs.merge_from_list( + config2cmdargs(config)) + else: + self._client_cfgs = CfgNode(flatten2nestdict(config)) + else: + trial_cfg.merge_from_list(config2cmdargs(config)) flags[available_worker].clear() trial = TrialExecutor(i, flags[available_worker], thread_results[available_worker], @@ -185,7 +207,14 @@ def _evaluate(self, configs): perfs = [None] * len(configs) for i, config in enumerate(configs): trial_cfg = self._cfg.clone() - trial_cfg.merge_from_list(config2cmdargs(config)) + if self._cfg.hpo.personalized_ss: + if isinstance(self._client_cfgs, CS.Configuration): + self._client_cfgs.merge_from_list( + config2cmdargs(config)) + else: + self._client_cfgs = CfgNode(flatten2nestdict(config)) + else: + trial_cfg.merge_from_list(config2cmdargs(config)) results = make_trial(trial_cfg, self._client_cfgs) key1, key2 = trial_cfg.hpo.metric.split('.') perfs[i] = results[key1][key2] @@ -327,85 +356,85 @@ def _generate_next_population(self, configs, perfs): return next_population -class SHAWrapFedex(SuccessiveHalvingAlgo): - """This SHA is customized as a wrapper for FedEx algorithm.""" - def _make_local_perturbation(self, config): - neighbor = dict() - for k in config: - if 'fedex' in k or 'fedopt' in k or k in [ - 'federate.save_to', 'federate.total_round_num', 'eval.freq' - ]: - # a workaround - continue - hyper = self._search_space.get(k) - if isinstance(hyper, CS.UniformFloatHyperparameter): - lb, ub = hyper.lower, hyper.upper - diameter = self._cfg.hpo.table.eps * (ub - lb) - new_val = (config[k] - - 0.5 * diameter) + np.random.uniform() * diameter - neighbor[k] = float(np.clip(new_val, lb, ub)) - elif isinstance(hyper, CS.UniformIntegerHyperparameter): - lb, ub = hyper.lower, hyper.upper - diameter = self._cfg.hpo.table.eps * (ub - lb) - new_val = round( - float((config[k] - 0.5 * diameter) + - np.random.uniform() * diameter)) - neighbor[k] = int(np.clip(new_val, lb, ub)) - elif isinstance(hyper, CS.CategoricalHyperparameter): - if len(hyper.choices) == 1: - neighbor[k] = config[k] - else: - threshold = self._cfg.hpo.table.eps * len( - hyper.choices) / (len(hyper.choices) - 1) - rn = np.random.uniform() - new_val = np.random.choice( - hyper.choices) if rn <= threshold else config[k] - if type(new_val) in [np.int32, np.int64]: - neighbor[k] = int(new_val) - elif type(new_val) in [np.float32, np.float64]: - neighbor[k] = float(new_val) - else: - neighbor[k] = str(new_val) - else: - raise TypeError("Value of {} has an invalid type {}".format( - k, type(config[k]))) - - return neighbor - - def _setup(self): - # self._cache_yaml() - init_configs = super(SHAWrapFedex, self)._setup() - new_init_configs = [] - for idx, trial_cfg in enumerate(init_configs): - arms = dict(("arm{}".format(1 + j), - self._make_local_perturbation(trial_cfg)) - for j in range(self._cfg.hpo.table.num - 1)) - arms['arm0'] = dict( - (k, v) for k, v in trial_cfg.items() if k in arms['arm1']) - with open( - os.path.join(self._cfg.hpo.working_folder, - f'{idx}_tmp_grid_search_space.yaml'), - 'w') as f: - yaml.dump(arms, f) - new_trial_cfg = dict() - for k in trial_cfg: - if k not in arms['arm0']: - new_trial_cfg[k] = trial_cfg[k] - new_trial_cfg['hpo.table.idx'] = idx - new_trial_cfg['hpo.fedex.ss'] = os.path.join( - self._cfg.hpo.working_folder, - f"{new_trial_cfg['hpo.table.idx']}_tmp_grid_search_space.yaml") - new_trial_cfg['federate.save_to'] = os.path.join( - self._cfg.hpo.working_folder, "idx_{}.pth".format(idx)) - new_init_configs.append(new_trial_cfg) - - self._search_space.add_hyperparameter( - CS.CategoricalHyperparameter("hpo.table.idx", - choices=list( - range(len(new_init_configs))))) - - return new_init_configs - +# class SHAWrapFedex(SuccessiveHalvingAlgo): +# """This SHA is customized as a wrapper for FedEx algorithm.""" +# def _make_local_perturbation(self, config): +# neighbor = dict() +# for k in config: +# if 'fedex' in k or 'fedopt' in k or k in [ +# 'federate.save_to', 'federate.total_round_num', +# 'eval.freq' +# ]: +# # a workaround +# continue +# hyper = self._search_space.get(k) +# if isinstance(hyper, CS.UniformFloatHyperparameter): +# lb, ub = hyper.lower, hyper.upper +# diameter = self._cfg.fedex.wrapper.eps * (ub - lb) +# new_val = (config[k] - +# 0.5 * diameter) + np.random.uniform() * diameter +# neighbor[k] = float(np.clip(new_val, lb, ub)) +# elif isinstance(hyper, CS.UniformIntegerHyperparameter): +# lb, ub = hyper.lower, hyper.upper +# diameter = self._cfg.fedex.wrapper.eps * (ub - lb) +# new_val = round( +# float((config[k] - 0.5 * diameter) + +# np.random.uniform() * diameter)) +# neighbor[k] = int(np.clip(new_val, lb, ub)) +# elif isinstance(hyper, CS.CategoricalHyperparameter): +# if len(hyper.choices) == 1: +# neighbor[k] = config[k] +# else: +# threshold = self._cfg.fedex.wrapper.eps * len( +# hyper.choices) / (len(hyper.choices) - 1) +# rn = np.random.uniform() +# new_val = np.random.choice( +# hyper.choices) if rn <= threshold else config[k] +# if type(new_val) in [np.int32, np.int64]: +# neighbor[k] = int(new_val) +# elif type(new_val) in [np.float32, np.float64]: +# neighbor[k] = float(new_val) +# else: +# neighbor[k] = str(new_val) +# else: +# raise TypeError("Value of {} has an invalid type {}".format( +# k, type(config[k]))) +# +# return neighbor +# +# def _setup(self): +# # self._cache_yaml() +# init_configs = super(SHAWrapFedex, self)._setup() +# new_init_configs = [] +# for idx, trial_cfg in enumerate(init_configs): +# arms = dict(("arm{}".format(1 + j), +# self._make_local_perturbation(trial_cfg)) +# for j in range(self._cfg.hpo.fedex.wrapper.arm - 1)) +# arms['arm0'] = dict( +# (k, v) for k, v in trial_cfg.items() if k in arms['arm1']) +# with open( +# os.path.join(self._cfg.hpo.working_folder, +# f'{idx}_tmp_grid_search_space.yaml'), +# 'w') as f: +# yaml.dump(arms, f) +# new_trial_cfg = dict() +# for k in trial_cfg: +# if k not in arms['arm0']: +# new_trial_cfg[k] = trial_cfg[k] +# new_trial_cfg['hpo.table.idx'] = idx +# new_trial_cfg['hpo.fedex.ss'] = os.path.join( +# self._cfg.hpo.working_folder, +# f"{new_trial_cfg['hpo.table.idx']}_tmp_grid_search_space.yaml") +# new_trial_cfg['federate.save_to'] = os.path.join( +# self._cfg.hpo.working_folder, "idx_{}.pth".format(idx)) +# new_init_configs.append(new_trial_cfg) +# +# self._search_space.add_hyperparameter( +# CS.CategoricalHyperparameter("hpo.table.idx", +# choices=list( +# range(len(new_init_configs))))) +# +# return new_init_configs # TODO: refactor PBT to enable async parallel # class PBT(IterativeScheduler): diff --git a/federatedscope/autotune/fedex/client.py b/federatedscope/autotune/fedex/client.py index 6e07ef61d..204afee78 100644 --- a/federatedscope/autotune/fedex/client.py +++ b/federatedscope/autotune/fedex/client.py @@ -1,6 +1,7 @@ import logging import json import copy +import numpy as np from federatedscope.core.message import Message from federatedscope.core.workers import Client @@ -56,6 +57,10 @@ def callback_funcs_for_model_para(self, message: Message): rnd=self.state, role='Client #{}'.format(self.ID), return_raw=True)) + # Inject + if self.ID in self._cfg.hpo.fedex.attack.id: + results['val_avg_loss_after'] += \ + self._cfg.hpo.fedex.attack.sigma * np.random.randn() results['arms'] = arms results['client_id'] = self.ID - 1 diff --git a/federatedscope/autotune/hpbandster.py b/federatedscope/autotune/hpbandster.py index a0bda9e8b..e44798ab4 100644 --- a/federatedscope/autotune/hpbandster.py +++ b/federatedscope/autotune/hpbandster.py @@ -14,12 +14,11 @@ logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) - -def clear_cache(working_folder): - # Clear cached ckpt - for name in os.listdir(working_folder): - if name.endswith('.pth'): - os.remove(osp(working_folder, name)) +# def clear_cache(working_folder): +# # Clear cached ckpt +# for name in os.listdir(working_folder): +# if name.endswith('.pth'): +# os.remove(osp(working_folder, name)) class MyRandomSearch(RandomSearch): @@ -33,11 +32,11 @@ def __init__(self, working_folder, **kwargs): self.working_folder = working_folder super(MyBOHB, self).__init__(**kwargs) - def get_next_iteration(self, iteration, iteration_kwargs={}): - if os.path.exists(self.working_folder): - clear_cache(self.working_folder) - return super(MyBOHB, self).get_next_iteration(iteration, - iteration_kwargs) + # def get_next_iteration(self, iteration, iteration_kwargs={}): + # if os.path.exists(self.working_folder): + # clear_cache(self.working_folder) + # return super(MyBOHB, self).get_next_iteration(iteration, + # iteration_kwargs) class MyHyperBand(HyperBand): @@ -45,11 +44,11 @@ def __init__(self, working_folder, **kwargs): self.working_folder = working_folder super(MyHyperBand, self).__init__(**kwargs) - def get_next_iteration(self, iteration, iteration_kwargs={}): - if os.path.exists(self.working_folder): - clear_cache(self.working_folder) - return super(MyHyperBand, - self).get_next_iteration(iteration, iteration_kwargs) + # def get_next_iteration(self, iteration, iteration_kwargs={}): + # if os.path.exists(self.working_folder): + # clear_cache(self.working_folder) + # return super(MyHyperBand, + # self).get_next_iteration(iteration, iteration_kwargs) class MyWorker(Worker): @@ -68,8 +67,9 @@ def __init__(self, self._init_configs = [] self._perfs = [] - def compute(self, config, budget, **kwargs): - results = eval_in_fs(self.cfg, config, int(budget), self.client_cfgs) + def compute(self, config, budget, config_id, **kwargs): + results = eval_in_fs(self.cfg, config, int(budget), config_id, + self._ss, self.client_cfgs) key1, key2 = self.cfg.hpo.metric.split('.') res = results[key1][key2] config = dict(config) @@ -83,7 +83,7 @@ def compute(self, config, budget, **kwargs): log2wandb(len(self._perfs) - 1, config, results, self.cfg) return {'loss': float(res), 'info': res} - def summarize(self): + def summarize(self, res=None): from federatedscope.autotune.utils import summarize_hpo_results results = summarize_hpo_results(self._init_configs, self._perfs, @@ -94,16 +94,13 @@ def summarize(self): "========================== HPO Final ==========================") logger.info("\n{}".format(results)) logger.info("====================================================") + logger.info(f'Winner config_id: {res.get_incumbent_id()}') return results def run_hpbandster(cfg, scheduler, client_cfgs=None): config_space = scheduler._search_space - if cfg.hpo.scheduler.startswith('wrap_'): - ss = CS.ConfigurationSpace() - ss.add_hyperparameter(config_space['hpo.table.idx']) - config_space = ss NS = hpns.NameServer(run_id=cfg.hpo.scheduler, host='127.0.0.1', port=0) ns_host, ns_port = NS.start() w = MyWorker(sleep_interval=0, @@ -143,6 +140,6 @@ def run_hpbandster(cfg, scheduler, client_cfgs=None): optimizer.shutdown(shutdown_workers=True) NS.shutdown() all_runs = res.get_all_runs() - w.summarize() + w.summarize(res) return [x.info for x in all_runs] diff --git a/federatedscope/autotune/run.py b/federatedscope/autotune/run.py index bdb2d27b6..c80911d32 100644 --- a/federatedscope/autotune/run.py +++ b/federatedscope/autotune/run.py @@ -8,7 +8,8 @@ def run_scheduler(scheduler, cfg, client_cfgs=None): cfg: The configurations of the FL course. client_cfgs: The clients' configurations. """ - if cfg.hpo.scheduler in ['sha', 'wrap_sha']: + # TODO: Fix 'wrap_sha' + if cfg.hpo.scheduler in ['sha']: _ = scheduler.optimize() elif cfg.hpo.scheduler in [ 'rs', 'bo_kde', 'hb', 'bohb', 'wrap_rs', 'wrap_bo_kde', 'wrap_hb', @@ -16,7 +17,9 @@ def run_scheduler(scheduler, cfg, client_cfgs=None): ]: from federatedscope.autotune.hpbandster import run_hpbandster run_hpbandster(cfg, scheduler, client_cfgs) - elif cfg.hpo.scheduler in ['bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf']: + elif cfg.hpo.scheduler in [ + 'bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf', 'multi' + ]: from federatedscope.autotune.smac import run_smac run_smac(cfg, scheduler, client_cfgs) else: diff --git a/federatedscope/autotune/smac.py b/federatedscope/autotune/smac.py index 067f53147..4facbb86f 100644 --- a/federatedscope/autotune/smac.py +++ b/federatedscope/autotune/smac.py @@ -1,16 +1,19 @@ import logging import numpy as np -import ConfigSpace as CS from federatedscope.autotune.utils import eval_in_fs, log2wandb from smac.facade.smac_bb_facade import SMAC4BB from smac.facade.smac_hpo_facade import SMAC4HPO from smac.scenario.scenario import Scenario +from smac.multi_objective.parego import ParEGO logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) +config_id = 0 + def run_smac(cfg, scheduler, client_cfgs=None): + config_space = scheduler._search_space init_configs = [] perfs = [] @@ -23,14 +26,29 @@ def optimization_function_wrapper(config): Returns: Best results of server of specific FS run. """ + global config_id + budget = cfg.hpo.sha.budgets[-1] - results = eval_in_fs(cfg, config, budget, client_cfgs) + results = eval_in_fs(cfg, config, budget, config_id, config_space, + client_cfgs) key1, key2 = cfg.hpo.metric.split('.') res = results[key1][key2] + if cfg.hpo.scheduler == 'multi': + new_res = {cfg.hpo.metric: res} + for key, w in zip(cfg.hpo.multi_obj.key, cfg.hpo.multi_obj.weight): + key1, key2 = key.split('.') + new_res[key] = w * results[key1][key2] + res = new_res config = dict(config) config['federate.total_round_num'] = budget init_configs.append(config) - perfs.append(res) + + if isinstance(res, dict): + perfs.append(sum(res.values())) + else: + perfs.append(res) + + config_id += 1 logger.info(f'Evaluate the {len(perfs)-1}-th config ' f'{config}, and get performance {res}') if cfg.wandb.use: @@ -48,14 +66,9 @@ def summarize(): "========================== HPO Final ==========================") logger.info("\n{}".format(results)) logger.info("====================================================") + logger.info(f'Winner config_id: {np.argmin(perfs)}') - return perfs - - config_space = scheduler._search_space - if cfg.hpo.scheduler.startswith('wrap_'): - ss = CS.ConfigurationSpace() - ss.add_hyperparameter(config_space['hpo.table.idx']) - config_space = ss + return results if cfg.hpo.sha.iter != 0: n_iterations = cfg.hpo.sha.iter @@ -64,13 +77,19 @@ def summarize(): np.log(cfg.hpo.sha.budgets[0] / cfg.hpo.sha.budgets[-1]) / np.log(cfg.hpo.sha.elim_rate)) + 1 + if cfg.hpo.scheduler == 'multi': + multi_obj_kwargs = { + "multi_objectives": [cfg.hpo.metric] + cfg.hpo.multi_obj.key, + } + scenario = Scenario({ "run_obj": "quality", "runcount-limit": n_iterations, "cs": config_space, "output_dir": cfg.hpo.working_folder, "deterministic": "true", - "limit_resources": False + "limit_resources": False, + **multi_obj_kwargs }) if cfg.hpo.scheduler.endswith('bo_gp'): @@ -80,6 +99,24 @@ def summarize(): elif cfg.hpo.scheduler.endswith('bo_rf'): smac = SMAC4HPO(scenario=scenario, tae_runner=optimization_function_wrapper) + elif cfg.hpo.scheduler == 'multi': + if cfg.hpo.multi_obj.algo == 'mean': + smac = SMAC4BB( + scenario=scenario, + tae_runner=optimization_function_wrapper, + ) + elif cfg.hpo.multi_obj.algo == 'parego': + smac = SMAC4HPO( + scenario=scenario, + tae_runner=optimization_function_wrapper, + multi_objective_algorithm=ParEGO, + multi_objective_kwargs={ + "rho": 0.05, + }, + ) + else: + raise ValueError(f'Unsupported `cfg.hpo.multi_obj.algo`' + f' {cfg.hpo.multi_obj.algo}') else: raise NotImplementedError try: diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 5e533a146..910d77efc 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -1,8 +1,13 @@ +import os.path + import yaml import logging import pandas as pd +import numpy as np import ConfigSpace as CS +from federatedscope.core.configs.yacs_config import CfgNode + logger = logging.getLogger(__name__) @@ -39,7 +44,7 @@ def parse_condition_param(condition, ss): f'should be in' \ f' {str_func_mapping.keys()}.' - if cond_type in ['and', 'in']: + if cond_type in ['and', 'in', 'or']: return str_func_mapping[cond_type]( parse_condition_param(condition['child'], ss), parse_condition_param(condition['parent'], ss), @@ -63,6 +68,7 @@ def parse_search_space(config_path): """ + # TODO: add seed for `ConfigurationSpace` ss = CS.ConfigurationSpace() conditions = [] @@ -236,13 +242,15 @@ def process(file_path): plt.close() -def eval_in_fs(cfg, config, budget, client_cfgs=None): +def eval_in_fs(cfg, config, budget, config_id, ss, client_cfgs=None): """ Args: cfg: fs cfg config: sampled trial CS.Configuration budget: budget round for this trial + config_id: Identifier to generate somadditional files + ss: search space of HPO client_cfgs: client-wise cfg Returns: @@ -258,19 +266,28 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): if isinstance(config, CS.Configuration): config = dict(config) - # Add FedEx related keys to config - if 'hpo.table.idx' in config.keys(): - idx = config['hpo.table.idx'] - config['hpo.fedex.ss'] = osp(cfg.hpo.working_folder, - f"{idx}_tmp_grid_search_space.yaml") + if 'wrap' in cfg.hpo.scheduler: + logger.info('FedEx is wrapped by scheduler.') + config['hpo.fedex.ss'] = osp( + cfg.hpo.working_folder, f"{config_id}_tmp_grid_search_space.yaml") + if not os.path.exists(config['hpo.fedex.ss']): + generate_arm(cfg, config, config_id, ss) + config['federate.save_to'] = osp(cfg.hpo.working_folder, - f"idx_{idx}.pth") + f"idx_{config_id}.pth") config['federate.restore_from'] = osp(cfg.hpo.working_folder, - f"idx_{idx}.pth") + f"idx_{config_id}.pth") # Global cfg trial_cfg = cfg.clone() # specify the configuration of interest - trial_cfg.merge_from_list(config2cmdargs(config)) + if cfg.hpo.personalized_ss: + if isinstance(client_cfgs, CS.Configuration): + client_cfgs.merge_from_list(config2cmdargs(config)) + else: + client_cfgs = CfgNode(flatten2nestdict(config)) + else: + trial_cfg.merge_from_list(config2cmdargs(config)) + # specify the budget trial_cfg.merge_from_list( ["federate.total_round_num", @@ -290,6 +307,73 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): return results +def generate_arm(cfg, config, config_id, ss): + def make_local_perturbation(config): + neighbor = dict() + for k in config: + if 'fedex' in k or 'fedopt' in k or k in [ + 'federate.save_to', 'federate.total_round_num', 'eval.freq' + ]: + # a workaround + continue + hyper = ss.get(k) + if isinstance(hyper, CS.UniformFloatHyperparameter): + lb, ub = hyper.lower, hyper.upper + diameter = cfg.hpo.fedex.wrapper.eps * (ub - lb) + new_val = (config[k] - + 0.5 * diameter) + np.random.uniform() * diameter + neighbor[k] = float(np.clip(new_val, lb, ub)) + elif isinstance(hyper, CS.UniformIntegerHyperparameter): + lb, ub = hyper.lower, hyper.upper + diameter = cfg.hpo.fedex.wrapper.eps * (ub - lb) + new_val = round( + float((config[k] - 0.5 * diameter) + + np.random.uniform() * diameter)) + neighbor[k] = int(np.clip(new_val, lb, ub)) + elif isinstance(hyper, CS.CategoricalHyperparameter): + if len(hyper.choices) == 1: + neighbor[k] = config[k] + else: + threshold = cfg.hpo.fedex.wrapper.eps * len( + hyper.choices) / (len(hyper.choices) - 1) + rn = np.random.uniform() + new_val = np.random.choice( + hyper.choices) if rn <= threshold else config[k] + if type(new_val) in [np.int32, np.int64]: + neighbor[k] = int(new_val) + elif type(new_val) in [np.float32, np.float64]: + neighbor[k] = float(new_val) + else: + neighbor[k] = str(new_val) + else: + raise TypeError("Value of {} has an invalid type {}".format( + k, type(config[k]))) + + return neighbor + + arms = dict(("arm{}".format(1 + j), make_local_perturbation(config)) + for j in range(cfg.hpo.fedex.wrapper.arm - 1)) + arms['arm0'] = dict((k, v) for k, v in config.items() if k in arms['arm1']) + with open( + os.path.join(cfg.hpo.working_folder, + f'{config_id}_tmp_grid_search_space.yaml'), 'w') as f: + yaml.dump(arms, f) + + +def flatten2nestdict(raw_dict, delimiter='.'): + # TODO: delete this for the function of `arm2dict` + def nested_set(dic, keys, value): + for key in keys[:-1]: + dic = dic.setdefault(key, {}) + dic[keys[-1]] = value + + new_dict = dict() + for key, value in raw_dict.items(): + keys = key.split(delimiter) + nested_set(new_dict, keys, value) + return new_dict + + def config_bool2int(config): # TODO: refactor bool/str to int import copy diff --git a/federatedscope/core/configs/README.md b/federatedscope/core/configs/README.md index ae1c85b22..1b4c09a7b 100644 --- a/federatedscope/core/configs/README.md +++ b/federatedscope/core/configs/README.md @@ -360,11 +360,10 @@ These arguments are exposed for customizing our provided auto-tuning components. #### Wrappers for FedEx -| Name | (Type) Default Value | Description | Note | -|:----:|:--------------------:|:-------------------------------------------|:-----| -| `hpo.table.eps` | (float) 0.1 | The probability to make local perturbation. | Larger values lead to drastically different arms of the bandit FedEx attempts to solve. | -| `hpo.table.num` | (int) 27 | The number of arms of the bandit FedEx attempts to solve. | - | -| `hpo.table.idx` | (int) 0 | The key (i.e., name) of the hyperparameter wrapper considers. | No need to change this argument. | +| Name | (Type) Default Value | Description | Note | +|:-----------------------:|:--------------------:|:-------------------------------------------|:-----| +| `hpo.fedex.wrapper.eps` | (float) 0.1 | The probability to make local perturbation. | Larger values lead to drastically different arms of the bandit FedEx attempts to solve. | +| `hpo.fedex.wrapper.arm` | (int) 27 | The number of arms of the bandit FedEx attempts to solve. | - | ### Attack diff --git a/federatedscope/core/configs/cfg_hpo.py b/federatedscope/core/configs/cfg_hpo.py index 72a403118..efbeb88b1 100644 --- a/federatedscope/core/configs/cfg_hpo.py +++ b/federatedscope/core/configs/cfg_hpo.py @@ -10,12 +10,21 @@ def extend_hpo_cfg(cfg): cfg.hpo = CN() cfg.hpo.working_folder = 'hpo' cfg.hpo.ss = '' + cfg.hpo.personalized_ss = False # If True, the ss will be discrete to + # hadamard product of client's ss cfg.hpo.num_workers = 0 cfg.hpo.init_cand_num = 16 cfg.hpo.larger_better = False cfg.hpo.scheduler = 'rs' cfg.hpo.metric = 'client_summarized_weighted_avg.val_loss' + # multi-objective-optimization + cfg.hpo.multi_obj = CN() + cfg.hpo.multi_obj.algo = '' # Choose from 'mean', 'parego' + cfg.hpo.multi_obj.key = [] + cfg.hpo.multi_obj.weight = [] # `cfg.hpo.metric` weight is 1.0 when + # multi-objective-optimization algo is `mean` + # SHA cfg.hpo.sha = CN() cfg.hpo.sha.elim_rate = 3 @@ -44,11 +53,14 @@ def extend_hpo_cfg(cfg): cfg.hpo.fedex.psn = False cfg.hpo.fedex.pi_lr = 0.01 - # Table - cfg.hpo.table = CN() - cfg.hpo.table.eps = 0.1 - cfg.hpo.table.num = 27 - cfg.hpo.table.idx = 0 + # FedEx wrap optimizer related args + cfg.hpo.fedex.wrapper = CN() + cfg.hpo.fedex.wrapper.eps = 0.1 + cfg.hpo.fedex.wrapper.arm = 27 + + cfg.hpo.fedex.attack = CN() + cfg.hpo.fedex.attack.id = [] # client IDs who inject noise into policy + cfg.hpo.fedex.attack.sigma = 1.0 # sigma of white noise def assert_hpo_cfg(cfg): diff --git a/federatedscope/hpo.py b/federatedscope/hpo.py index 789c897e7..ff1a2147f 100644 --- a/federatedscope/hpo.py +++ b/federatedscope/hpo.py @@ -10,6 +10,7 @@ from federatedscope.core.auxiliaries.utils import setup_seed from federatedscope.core.auxiliaries.logging import update_logger from federatedscope.core.cmd_args import parse_args, parse_client_cfg +from federatedscope.core.auxiliaries.data_builder import get_data from federatedscope.core.configs.config import global_cfg, CfgNode from federatedscope.autotune import get_scheduler, run_scheduler @@ -42,5 +43,10 @@ else: client_cfgs = None + # Update client num + data, modified_cfg = get_data(config=init_cfg.clone(), + client_cfgs=client_cfgs) + init_cfg.merge_from_other_cfg(modified_cfg) + scheduler = get_scheduler(init_cfg, client_cfgs) run_scheduler(scheduler, init_cfg, client_cfgs) diff --git a/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml b/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml index 09430d4d8..850676fb2 100644 --- a/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml +++ b/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml @@ -17,7 +17,7 @@ data: # form: dataloader splits: [0.6,0.2,0.2] subsample: 0.05 - # transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + # transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] transform: [ [ 'ToTensor' ] ] dataloader: batch_size: 32 diff --git a/scripts/example_configs/cora/hpo_ss_fedex.yaml b/scripts/example_configs/cora/hpo_ss_fedex.yaml deleted file mode 100644 index edece27da..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -hpo.fedex.eta0: - type: cate - choices: [-1.0, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] -hpo.fedex.gamma: - type: float - lower: 0.0 - upper: 1.0 - log: False -hpo.fedex.diff: - type: cate - choices: [True, False] \ No newline at end of file diff --git a/scripts/example_configs/cora/hpo_ss_fedex_arm.yaml b/scripts/example_configs/cora/hpo_ss_fedex_arm.yaml deleted file mode 100644 index 3b52d0c79..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex_arm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -train.optimizer.lr: - type: cate - choices: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] -model.dropout: - type: cate - choices: [0.0, 0.5] -train.local_update_steps: - type: cate - choices: [1, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file diff --git a/scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml b/scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml deleted file mode 100644 index d40abc429..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml +++ /dev/null @@ -1,3 +0,0 @@ -hpo.table.idx: - type: cate - choices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80] diff --git a/scripts/example_configs/cora/hpo_ss_fedex_grid.yaml b/scripts/example_configs/cora/hpo_ss_fedex_grid.yaml deleted file mode 100644 index 4f9699dcf..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex_grid.yaml +++ /dev/null @@ -1,4 +0,0 @@ -train.optimizer.lr: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: [0.0, 0.001, 0.01, 0.1] -model.dropout: [0.0, 0.5] -train.local_update_steps: [1, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file diff --git a/scripts/example_configs/cora/run.sh b/scripts/example_configs/cora/run.sh deleted file mode 100644 index 3d57fb137..000000000 --- a/scripts/example_configs/cora/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -# SHA -python hpo.py --cfg scripts/example_configs/cora/sha.yaml - -# SHA wrap FedEX (FedEX related param) -python hpo.py --cfg scripts/example_configs/cora/sha_wrap_fedex.yaml - -# SHA wrap FedEX (arm) -python hpo.py --cfg scripts/example_configs/cora/sha_wrap_fedex_arm.yaml \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_fedex.yaml b/scripts/example_configs/femnist/hpo_ss_fedex.yaml deleted file mode 100644 index edece27da..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -hpo.fedex.eta0: - type: cate - choices: [-1.0, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] -hpo.fedex.gamma: - type: float - lower: 0.0 - upper: 1.0 - log: False -hpo.fedex.diff: - type: cate - choices: [True, False] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml deleted file mode 100644 index ea438a1ad..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml +++ /dev/null @@ -1,15 +0,0 @@ -train.optimizer.lr: - type: cate - choices: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] -model.dropout: - type: cate - choices: [0.0, 0.5] -train.local_update_steps: - type: cate - choices: [1, 2, 3, 4] -dataloader.batch_size: - type: cate - choices: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml deleted file mode 100644 index d40abc429..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml +++ /dev/null @@ -1,3 +0,0 @@ -hpo.table.idx: - type: cate - choices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80] diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml deleted file mode 100644 index 774265c55..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml +++ /dev/null @@ -1,5 +0,0 @@ -train.optimizer.lr: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: [0.0, 0.001, 0.01, 0.1] -model.dropout: [0.0, 0.5] -train.local_update_steps: [1, 2, 3, 4] -dataloader.batch_size: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/run.sh b/scripts/example_configs/femnist/run.sh deleted file mode 100644 index 803e70cc9..000000000 --- a/scripts/example_configs/femnist/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -# SHA -python hpo.py --cfg scripts/example_configs/femnist/sha.yaml - -# SHA wrap FedEX (FedEX related param) -python hpo.py --cfg scripts/example_configs/femnist/sha_wrap_fedex.yaml - -# SHA wrap FedEX (arm) -python hpo.py --cfg scripts/example_configs/femnist/sha_wrap_fedex_arm.yaml \ No newline at end of file diff --git a/scripts/example_configs/femnist/sha_wrap_fedex.yaml b/scripts/example_configs/femnist/sha_wrap_fedex.yaml deleted file mode 100644 index 58309fe31..000000000 --- a/scripts/example_configs/femnist/sha_wrap_fedex.yaml +++ /dev/null @@ -1,52 +0,0 @@ -use_gpu: True -device: 3 -early_stop: - patience: 100 -seed: 12345 -federate: - mode: standalone - total_round_num: 500 - sample_client_rate: 1.0 - share_local_model: True - online_aggr: True - use_diff: True -data: - root: data/ - type: femnist - splits: [0.6,0.2,0.2] - subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] -dataloader: - batch_size: 16 -model: - type: convnet2 - hidden: 2048 - out_channels: 62 - dropout: 0.5 -train: - batch_or_epoch: epoch - local_update_steps: 1 - optimizer: - lr: 0.01 - weight_decay: 0.0 -grad: - grad_clip: 5.0 -criterion: - type: CrossEntropyLoss -trainer: - type: cvtrainer -eval: - freq: 1 - metrics: ['acc', 'correct', 'f1'] - split: ['test', 'val', 'train'] -hpo: - scheduler: sha - num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/femnist/hpo_ss_fedex.yaml' - sha: - budgets: [2, 4, 12, 36] - fedex: - use: True - ss: 'scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml' - metric: 'client_summarized_weighted_avg.val_avg_loss' diff --git a/scripts/hpo_exp_scripts/analysis.py b/scripts/hpo_exp_scripts/analysis.py new file mode 100644 index 000000000..3708c699c --- /dev/null +++ b/scripts/hpo_exp_scripts/analysis.py @@ -0,0 +1,221 @@ +import os +import yaml +import numpy as np +import matplotlib.pyplot as plt +from tqdm import tqdm + +FONTSIZE = 40 +MARKSIZE = 25 + +METHOD = { + 'rs': 'RS', + 'bo_gp': 'BO_GP', + 'bo_rf': 'BO_RF', + 'bo_kde': 'BO_KDE', + 'hb': 'HB', + 'bohb': 'BOHB', + 'rs_wrap': 'RS+FedEx', + 'bo_gp_wrap': 'BO_GP+FedEx', + 'bo_rf_wrap': 'BO_RF+FedEx', + 'bo_kde_wrap': 'BO_KDE+FedEx', + 'hb_wrap': 'HB+FedEx', + 'bohb_wrap': 'BOHB+FedEx', +} + +TRIAL2SEED = { + 't0': 12345, + 't1': 12346, + 't2': 12347, +} + +COLORS = [ + u'#1f77b4', u'#ff7f0e', u'#2ca02c', u'#d62728', u'#8c564b', u'#9467bd' +] + + +def parse_logs(root, + file_list, + eval_key='client_summarized_weighted_avg', + suf=''): + def process(file): + history = [] + with open(file, 'r') as F: + F = F.readlines() + for idx, line in tqdm(enumerate(F)): + if "'Round': 'Final'" in line: + last_line = F[idx - 2] + _, last_line = last_line.split('INFO: ') + last_line = eval(last_line) + config = {'federate.total_round_num': last_line['Round']} + try: + state, line = line.split('INFO: ') + results = eval(line) + performance = results['Results_raw'][eval_key][ + 'val_avg_loss'] + history.append((config, performance)) + except Exception as error: + continue + best_seen = np.inf + tol_budget = 0 + x, y = [], [] + + for config, performance in history: + tol_budget += config['federate.total_round_num'] + if best_seen > performance or config[ + 'federate.total_round_num'] > tmp_b: + best_seen = performance + x.append(tol_budget) + y.append(best_seen) + tmp_b = config['federate.total_round_num'] + return np.array(x) / tol_budget, np.array(y) + + # Draw + plt.figure(figsize=(10, 7.5)) + plt.xticks(fontsize=FONTSIZE) + plt.yticks(fontsize=FONTSIZE) + + plt.xlabel('Fraction of budget', size=FONTSIZE) + plt.ylabel('Loss', size=FONTSIZE) + + data_x = {} + data_y = {} + for file in file_list: + data_x[file] = [] + data_y[file] = [] + + for prefix in root: + for file in file_list: + if 'wrap' in file: + new_name = f'wrap_{file[:-5]}' + else: + new_name = file + + if 'hb' in file: + budget = [9, 81] + else: + budget = [50, 50] + x, y = process( + os.path.join( + prefix, f'{file}{suf}', + f'{new_name}_{budget}_server_global_eval.val_avg_loss', + 'exp_print.log')) + data_x[file].append(x) + data_y[file].append(y) + + for i, file in enumerate(file_list): + if 'wrap' in file: + linestyle = '--' + else: + linestyle = '-' + plt.plot(np.mean(data_x[file], axis=0), + np.mean(data_y[file], axis=0), + linewidth=1, + color=COLORS[i % len(COLORS)], + markersize=MARKSIZE, + linestyle=linestyle) + if file_list[0].endswith('_opt.log'): + suffix = 'opt' + else: + suffix = 'avg' + plt.xscale("log") + plt.xticks([0.01, 0.1, 1], ['1e-2', '1e-1', '1']) + plt.legend(list(METHOD.values()), + loc='upper right', + prop={'size': 22}, + bbox_to_anchor=(1.5, 1), + borderaxespad=0) + plt.savefig(f'exp2_{suffix}.pdf', bbox_inches='tight') + plt.show() + plt.close() + + +def args_run_from_scratch(root, + file_list, + fs_yaml='********', + suf='', + device=8): + def get_args(root, logs): + log_file = os.path.join(root, logs) + arg_key = [] + arg_val = [] + with open(log_file, 'r') as f: + flag = False + cnt = 0 + for line in f.readlines(): + if 'wrap' in log_file: + if 'Winner config_id:' in line: + idx = line.split('Winner config_id: ')[-1] + if idx.endswith(' ') or idx.endswith('\n'): + idx = idx[:-1] + break + else: + if '== HPO Final ==' in line: + flag = True + + if flag: + if line.startswith(' '): + args = [x for x in line.split(' ') if len(x)] + arg_key += args + if line.startswith('0 '): + args = [x for x in line.split(' ') if len(x)][1:] + arg_val += args + + if 'wrap' in log_file: + fedex_yaml = os.path.join(root, f'idx_{idx}_fedex.yaml') + arm_yaml = os.path.join(root, f'{idx}_tmp_grid_search_space.yaml') + + with open(fedex_yaml, 'r') as y: + fedex = dict(yaml.load(y, Loader=yaml.FullLoader)) + + with open(arm_yaml, 'r') as y: + arm = dict(yaml.load(y, Loader=yaml.FullLoader)) + + best_arm = np.argmax(fedex['z'][0]) + arg = '' + for key, val in arm[f'arm{best_arm}'].items(): + arg += f'{key} {val} ' + else: + arg_key = [x for x in arg_key if not x.endswith('\n')] + arg_val = [x for x in arg_val if not x.endswith('\n')] + arg = '' + for key, val in zip(arg_key[:-1], arg_val[:-1]): + arg += f'{key} {val} ' + return arg + + d = 0 + for prefix in root: + for file in file_list: + if 'wrap' in file: + new_name = f'wrap_{file[:-5]}' + else: + new_name = file + + if 'hb' in file: + budget = [9, 81] + else: + budget = [50, 50] + args = get_args( + os.path.join(prefix, f'{file}{suf}'), + os.path.join( + f'{new_name}_{budget}_server_global_eval.val_avg_loss', + 'exp_print.log')) + + seed = TRIAL2SEED[prefix] + print( + f'nohup python federatedscope/main.py --cfg {fs_yaml} {args} device {d % device} > {file}_{seed}{suf}.log &' + ) + d += 1 + + +if __name__ == '__main__': + parse_logs(list(TRIAL2SEED.keys()), + list(METHOD.keys()), + eval_key='server_global_eval', + suf='_pubmed_avg') + + args_run_from_scratch( + list(TRIAL2SEED.keys()), + list(METHOD.keys()), + fs_yaml= + 'scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml', + suf='_pubmed_avg') diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml new file mode 100644 index 000000000..f7e7beea8 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: bo_gp_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml new file mode 100644 index 000000000..38bce0697 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: bo_kde_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml new file mode 100644 index 000000000..675fd0e1a --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: bo_rf_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml new file mode 100644 index 000000000..2caffeb2c --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: bohb_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml new file mode 100644 index 000000000..07504656f --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: hb_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_wrap_femnist_avg_robust diff --git a/scripts/example_configs/femnist/sha.yaml b/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml similarity index 68% rename from scripts/example_configs/femnist/sha.yaml rename to scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml index 2343d43d6..1435cd8dc 100644 --- a/scripts/example_configs/femnist/sha.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml @@ -1,5 +1,5 @@ use_gpu: True -device: 3 +device: 0 early_stop: patience: 100 seed: 12345 @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: @@ -39,11 +39,3 @@ eval: freq: 1 metrics: ['acc', 'correct', 'f1'] split: ['test', 'val', 'train'] -hpo: - scheduler: sha - num_workers: 0 - init_cand_num: 27 - ss: 'scripts/example_configs/femnist/hpo_ss_sha.yaml' - sha: - budgets: [12, 13, 19] - metric: 'client_summarized_weighted_avg.val_avg_loss' diff --git a/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml new file mode 100644 index 000000000..e9c852614 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: rs_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_rs + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh new file mode 100644 index 000000000..7fd0554b0 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -0,0 +1,26 @@ +set -e + +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4]" 1.0 12345 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4]" 1.0 12346 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4]" 1.0 12347 8 +# +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]" 1.0 12345 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]" 1.0 12346 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]" 1.0 12347 8 +# +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]" 1.0 12345 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]" 1.0 12346 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]" 1.0 12347 8 + +attacker=$1 +sigma=$2 +seed=$3 +device=$4 + +methods=('rs_wrap' 'bo_gp_wrap' 'bo_kde_wrap' 'bo_rf_wrap' 'hb_wrap' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%${device})) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & +done diff --git a/scripts/example_configs/femnist/avg/ss.yaml b/scripts/hpo_exp_scripts/robustness/attack/ss.yaml similarity index 100% rename from scripts/example_configs/femnist/avg/ss.yaml rename to scripts/hpo_exp_scripts/robustness/attack/ss.yaml diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml new file mode 100644 index 000000000..50a2057b9 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml @@ -0,0 +1,55 @@ +use_gpu: True +device: 3 +outdir: bo_gp_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml new file mode 100644 index 000000000..f454522ce --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml @@ -0,0 +1,62 @@ +use_gpu: True +device: 3 +outdir: bo_gp_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml new file mode 100644 index 000000000..935e22a9e --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml @@ -0,0 +1,55 @@ +use_gpu: True +device: 3 +outdir: bo_kde_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml new file mode 100644 index 000000000..bc1afc4fb --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml @@ -0,0 +1,62 @@ +use_gpu: True +device: 3 +outdir: bo_kde_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml new file mode 100644 index 000000000..faac26886 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml @@ -0,0 +1,55 @@ +use_gpu: True +device: 3 +outdir: bo_rf_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml new file mode 100644 index 000000000..759082840 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml @@ -0,0 +1,62 @@ +use_gpu: True +device: 3 +outdir: bo_rf_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml new file mode 100644 index 000000000..bd5107ce5 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml @@ -0,0 +1,55 @@ +use_gpu: True +device: 3 +outdir: bohb_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml new file mode 100644 index 000000000..580c99f1d --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml @@ -0,0 +1,62 @@ +use_gpu: True +device: 3 +outdir: bohb_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml new file mode 100644 index 000000000..6ad5acf72 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml @@ -0,0 +1,55 @@ +use_gpu: True +device: 3 +outdir: hb_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml new file mode 100644 index 000000000..709776bbe --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml @@ -0,0 +1,62 @@ +use_gpu: True +device: 3 +outdir: hb_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml new file mode 100644 index 000000000..c9882a1bb --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml @@ -0,0 +1,45 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 0.05}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml new file mode 100644 index 000000000..1f70a6c05 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml @@ -0,0 +1,45 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 0.5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml new file mode 100644 index 000000000..ebd3225ec --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml @@ -0,0 +1,45 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml new file mode 100644 index 000000000..d765de21c --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml @@ -0,0 +1,56 @@ +use_gpu: True +device: 3 +outdir: rs_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: rs + num_workers: 0 + init_cand_num: 10 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml new file mode 100644 index 000000000..29342770e --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml @@ -0,0 +1,62 @@ +use_gpu: True +device: 3 +outdir: rs_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_rs + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh new file mode 100644 index 000000000..7f2928fb1 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh @@ -0,0 +1,26 @@ +set -e + +# How to use: +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12345 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12346 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12347 8 +# +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12345 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12346 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12347 8 +# +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12345 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12346 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12347 8 + +alpha=$1 +seed=$2 +device=$3 + +methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/cifar10/${methods[$m]}.yaml device $((m%${device})) seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & +done diff --git a/scripts/example_configs/femnist/hpo_ss_sha.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml similarity index 66% rename from scripts/example_configs/femnist/hpo_ss_sha.yaml rename to scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml index 4a65f2cef..e7d57b0ae 100644 --- a/scripts/example_configs/femnist/hpo_ss_sha.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml @@ -4,14 +4,16 @@ train.optimizer.lr: upper: 1.0 log: True train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] + type: float + lower: 0.0 + upper: 1.0 model.dropout: type: cate choices: [0.0, 0.5] train.local_update_steps: - type: cate - choices: [1, 2, 3, 4] + type: int + lower: 1 + upper: 4 dataloader.batch_size: type: cate - choices: [16, 32, 64] \ No newline at end of file + choices: [16, 32, 64] diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/learn_from_scratch/cifar.yaml.py b/scripts/hpo_exp_scripts/uniqueness/fairness/learn_from_scratch/cifar.yaml.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml b/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml new file mode 100644 index 000000000..445480bbb --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml @@ -0,0 +1,59 @@ +use_gpu: True +device: 3 +outdir: multi_mean_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: multi + num_workers: 0 + multi_obj: + algo: 'mean' + key: ['client_summarized_fairness.val_avg_loss_std'] + weight: [1.0] + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: multi_mean_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/run.sh b/scripts/hpo_exp_scripts/uniqueness/fairness/run.sh new file mode 100644 index 000000000..ac399c42d --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/run.sh @@ -0,0 +1,34 @@ +set -e + +# How to use: +#bash scripts/hpo_exp_scripts/uniqueness/fairness/run.sh 8 + +device=$1 + + +alphas=(0.05 0.5 5.0) +seeds=(12345 12346 12347) +weights=(0.1 1.0 10.0) +trial=0 + +for (( a=0; a<${#alphas[@]}; a++ )) +do + alpha=${alphas[$a]} + for (( s=0; s<${#seeds[@]}; s++ )) + do + seed=${seeds[$s]} + for (( w=0; w<${#weights[@]}; w++ )) + do + weight=${weights[$w]} + folder="${alpha}_${weight}_${seed}_multi_mean_cifar10_avg" + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml outdir $folder hpo.working_folder $folder device $((trial%${device})) hpo.multi_obj.weight "[${weight}]" seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & + trial=$((trial+1)) + sleep 0.5 + done + + weight=1.0 + folder="${alpha}_${weight}_${seed}_multi_parego_cifar10_avg" + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml outdir $folder hpo.working_folder $folder device $((trial%${device})) hpo.multi_obj.algo parego hpo.multi_obj.weight "[${weight}]" seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & + trial=$((trial+1)) + done +done \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml b/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml new file mode 100644 index 000000000..2e8bc1e3a --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml @@ -0,0 +1,58 @@ +use_gpu: True +device: 3 +outdir: test_multi_mean_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: multi + num_workers: 0 + multi_obj: + algo: 'mean' + key: ['client_summarized_fairness.val_avg_loss_std'] + weight: [1.0] + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 2, 2 ] + iter: 2 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: test_multi_mean_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml new file mode 100644 index 000000000..b87e8b5d8 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bo_gp_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml new file mode 100644 index 000000000..f4add5608 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 3 +outdir: bo_gp_wrap_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml new file mode 100644 index 000000000..2b29f75ae --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bo_kde_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml new file mode 100644 index 000000000..2edb6d618 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: bo_kde_wrap_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml new file mode 100644 index 000000000..60206cd5d --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bo_rf_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml new file mode 100644 index 000000000..66703064f --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml @@ -0,0 +1,61 @@ +use_gpu: True +device: 3 +outdir: bo_rf_wrap_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml new file mode 100644 index 000000000..394d9be57 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bohb_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml new file mode 100644 index 000000000..31354e834 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 3 +outdir: bohb_wrap_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml new file mode 100644 index 000000000..919f6426f --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: hb_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml new file mode 100644 index 000000000..b6a441699 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 3 +outdir: hb_wrap_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml new file mode 100644 index 000000000..03219db30 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml @@ -0,0 +1,40 @@ +use_gpu: True +device: 3 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml new file mode 100644 index 000000000..b52b27c5e --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 3 +outdir: rs_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: rs + num_workers: 0 + init_cand_num: 10 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml new file mode 100644 index 000000000..201ca2625 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 3 +outdir: rs_wrap_femnist_avg_per +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_rs + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh new file mode 100644 index 000000000..7815fc92e --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh @@ -0,0 +1,12 @@ +set -e + +seed=$1 +device=$2 + +methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/personalized/${methods[$m]}.yaml device $((m%${device})) seed ${seed} >/dev/null 2>&1 & +done \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml new file mode 100644 index 000000000..e7d57b0ae --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml @@ -0,0 +1,19 @@ +train.optimizer.lr: + type: float + lower: 0.01 + upper: 1.0 + log: True +train.optimizer.weight_decay: + type: float + lower: 0.0 + upper: 1.0 +model.dropout: + type: cate + choices: [0.0, 0.5] +train.local_update_steps: + type: int + lower: 1 + upper: 4 +dataloader.batch_size: + type: cate + choices: [16, 32, 64] diff --git a/scripts/example_configs/femnist/avg/bo_gp.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_gp.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bo_gp.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_gp.yaml index 35915c31f..af01e2cc5 100644 --- a/scripts/example_configs/femnist/avg/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_gp.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_gp_wrap.yaml similarity index 89% rename from scripts/example_configs/femnist/avg/bo_gp_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_gp_wrap.yaml index 815325682..9d79e5db9 100644 --- a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_gp_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/bo_kde.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_kde.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bo_kde.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_kde.yaml index e8096feeb..b91327f59 100644 --- a/scripts/example_configs/femnist/avg/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_kde.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_kde_wrap.yaml similarity index 93% rename from scripts/example_configs/femnist/avg/bo_kde_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_kde_wrap.yaml index 1ad4f0003..38d0d5099 100644 --- a/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_kde_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_rf.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_rf.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bo_rf.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_rf.yaml index 1e8bca28e..145ad9785 100644 --- a/scripts/example_configs/femnist/avg/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_rf.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_rf_wrap.yaml similarity index 89% rename from scripts/example_configs/femnist/avg/bo_rf_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_rf_wrap.yaml index f2977a2eb..31f05fdbd 100644 --- a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_rf_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_rf num_workers: 0 - init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/bohb.yaml b/scripts/hpo_exp_scripts/usability/femnist/bohb.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bohb.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bohb.yaml index b60970464..c53d94ff9 100644 --- a/scripts/example_configs/femnist/avg/bohb.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bohb.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bohb_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bohb_wrap.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bohb_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bohb_wrap.yaml index e3ca1f733..dbae06851 100644 --- a/scripts/example_configs/femnist/avg/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bohb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: @@ -49,8 +49,6 @@ hpo: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/hb.yaml b/scripts/hpo_exp_scripts/usability/femnist/hb.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/hb.yaml rename to scripts/hpo_exp_scripts/usability/femnist/hb.yaml index f48d9de93..6c6267cf9 100644 --- a/scripts/example_configs/femnist/avg/hb.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/hb.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/hb_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/hb_wrap.yaml similarity index 93% rename from scripts/example_configs/femnist/avg/hb_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/hb_wrap.yaml index 1cfdeeca6..4a9cd60ee 100644 --- a/scripts/example_configs/femnist/avg/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/hb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/learn_from_scratch.yaml b/scripts/hpo_exp_scripts/usability/femnist/learn_from_scratch.yaml similarity index 89% rename from scripts/example_configs/femnist/avg/learn_from_scratch.yaml rename to scripts/hpo_exp_scripts/usability/femnist/learn_from_scratch.yaml index ac484e7dc..f6b322da5 100644 --- a/scripts/example_configs/femnist/avg/learn_from_scratch.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/learn_from_scratch.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/rs.yaml b/scripts/hpo_exp_scripts/usability/femnist/rs.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/rs.yaml rename to scripts/hpo_exp_scripts/usability/femnist/rs.yaml index d6dd868a0..67616e04f 100644 --- a/scripts/example_configs/femnist/avg/rs.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/rs.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/rs_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/rs_wrap.yaml similarity index 90% rename from scripts/example_configs/femnist/avg/rs_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/rs_wrap.yaml index 9d4680524..70223b024 100644 --- a/scripts/example_configs/femnist/avg/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/rs_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: @@ -49,13 +49,13 @@ hpo: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff: False eta0: -1.0 gamma: 0.0 + wrapper: + arm: 27 metric: 'client_summarized_weighted_avg.val_avg_loss' working_folder: rs_wrap_femnist_avg diff --git a/scripts/example_configs/femnist/avg/rs_wrap_pfedex.sh b/scripts/hpo_exp_scripts/usability/femnist/rs_wrap_pfedex.sh similarity index 100% rename from scripts/example_configs/femnist/avg/rs_wrap_pfedex.sh rename to scripts/hpo_exp_scripts/usability/femnist/rs_wrap_pfedex.sh diff --git a/scripts/example_configs/femnist/avg/sha.yaml b/scripts/hpo_exp_scripts/usability/femnist/sha.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/sha.yaml rename to scripts/hpo_exp_scripts/usability/femnist/sha.yaml index 0c9350fe0..cf6d5a387 100644 --- a/scripts/example_configs/femnist/avg/sha.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/sha.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/sha_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/sha_wrap.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/sha_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/sha_wrap.yaml index cf476e100..879c99a03 100644 --- a/scripts/example_configs/femnist/avg/sha_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/sha_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/usability/femnist/ss.yaml b/scripts/hpo_exp_scripts/usability/femnist/ss.yaml new file mode 100644 index 000000000..e7d57b0ae --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/femnist/ss.yaml @@ -0,0 +1,19 @@ +train.optimizer.lr: + type: float + lower: 0.01 + upper: 1.0 + log: True +train.optimizer.weight_decay: + type: float + lower: 0.0 + upper: 1.0 +model.dropout: + type: cate + choices: [0.0, 0.5] +train.local_update_steps: + type: int + lower: 1 + upper: 4 +dataloader.batch_size: + type: cate + choices: [16, 32, 64] diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml new file mode 100644 index 000000000..6bcebf3b1 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml @@ -0,0 +1,50 @@ +use_gpu: True +device: 2 +outdir: bo_gp_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_gp_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml new file mode 100644 index 000000000..43d1e449e --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml @@ -0,0 +1,57 @@ +use_gpu: True +device: 2 +outdir: bo_gp_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_gp_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml new file mode 100644 index 000000000..7e185cd6f --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml @@ -0,0 +1,50 @@ +use_gpu: True +device: 2 +outdir: bo_kde_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_kde_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml new file mode 100644 index 000000000..b7e58f604 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml @@ -0,0 +1,57 @@ +use_gpu: True +device: 2 +outdir: bo_kde_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_kde_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml new file mode 100644 index 000000000..a8ff27a9e --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml @@ -0,0 +1,50 @@ +use_gpu: True +device: 2 +outdir: bo_rf_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_rf_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml new file mode 100644 index 000000000..fbdb20efe --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml @@ -0,0 +1,57 @@ +use_gpu: True +device: 2 +outdir: bo_rf_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_rf_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml new file mode 100644 index 000000000..d80b62efa --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml @@ -0,0 +1,50 @@ +use_gpu: True +device: 2 +outdir: bohb_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'server_global_eval.val_avg_loss' + working_folder: bohb_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml new file mode 100644 index 000000000..752f1f0ea --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml @@ -0,0 +1,57 @@ +use_gpu: True +device: 2 +outdir: bohb_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bohb_wrap_pubmed_avg diff --git a/scripts/example_configs/cora/sha.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml similarity index 65% rename from scripts/example_configs/cora/sha.yaml rename to scripts/hpo_exp_scripts/usability/pubmed/hb.yaml index 00444abb6..d0aace870 100644 --- a/scripts/example_configs/cora/sha.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml @@ -1,5 +1,6 @@ use_gpu: True -device: 3 +device: 2 +outdir: hb_pubmed_avg early_stop: patience: 100 seed: 12345 @@ -10,23 +11,22 @@ federate: total_round_num: 500 share_local_model: True online_aggr: True - use_diff: True data: root: data/ - type: cora + type: pubmed splitter: 'louvain' -dataloader: - batch_size: 1 model: type: gcn hidden: 64 dropout: 0.5 - out_channels: 7 + out_channels: 3 + task: node train: + batch_or_epoch: epoch local_update_steps: 1 optimizer: - lr: 0.25 - weight_decay: 0.0005 + lr: 0.01 + weight_decay: 0.0 criterion: type: CrossEntropyLoss trainer: @@ -35,11 +35,16 @@ eval: freq: 1 metrics: ['acc', 'correct', 'f1'] split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 hpo: - scheduler: sha + scheduler: hb num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/cora/hpo_ss_sha.yaml' + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: - budgets: [2, 4, 12, 36] + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 metric: 'server_global_eval.val_avg_loss' + working_folder: hb_pubmed_avg \ No newline at end of file diff --git a/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml similarity index 63% rename from scripts/example_configs/cora/sha_wrap_fedex_arm.yaml rename to scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml index 6566791d5..69b5e5683 100644 --- a/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml @@ -1,5 +1,6 @@ use_gpu: True -device: 3 +device: 2 +outdir: hb_wrap_pubmed_avg early_stop: patience: 100 seed: 12345 @@ -13,20 +14,20 @@ federate: use_diff: True data: root: data/ - type: cora + type: pubmed splitter: 'louvain' -dataloader: - batch_size: 1 model: type: gcn hidden: 64 dropout: 0.5 - out_channels: 7 + out_channels: 3 + task: node train: + batch_or_epoch: epoch local_update_steps: 1 optimizer: - lr: 0.25 - weight_decay: 0.0005 + lr: 0.01 + weight_decay: 0.0 criterion: type: CrossEntropyLoss trainer: @@ -35,20 +36,22 @@ eval: freq: 1 metrics: ['acc', 'correct', 'f1'] split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 hpo: - scheduler: wrap_sha + scheduler: wrap_hb num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml' - table: - ss: 'scripts/example_configs/cora/hpo_ss_fedex_arm.yaml' - num: 4 - cand: 81 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: - budgets: [2, 4, 12, 36] + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 fedex: + sched: 'aggressive' use: True diff: False - eta0: 0.050 - gamma: 0.495861 + eta0: -1.0 + gamma: 0.0 metric: 'server_global_eval.val_avg_loss' + working_folder: hb_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml new file mode 100644 index 000000000..d20ef1ccd --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml @@ -0,0 +1,40 @@ +use_gpu: True +device: 2 +outdir: rs_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 diff --git a/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh new file mode 100644 index 000000000..cf0f2064c --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh @@ -0,0 +1,43 @@ +set -e + +# Seed 12345 +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345 >rs_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.7170358969751284 train.optimizer.weight_decay 0.0 device 1 seed 12345 >rs_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 4 train.optimizer.lr 0.999311 train.optimizer.weight_decay 8.674074e-07 device 2 seed 12345 >bo_gp_12345.log & +# >rs_wrap_12345.log >bo_gp_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46701 train.optimizer.weight_decay 0.000853 device 0 seed 12345 >bo_kde_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46852259676975294 train.optimizer.weight_decay 0.263664919539593 device 1 seed 12345 >bo_kde_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.984828 train.optimizer.weight_decay 0.00233 device 2 seed 12345 >bo_rf_12345.log & +# >rs_wrap_12345.log >bo_rf_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345 >hb_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.453625286588245 train.optimizer.weight_decay 0.06289557213350952 device 1 seed 12345 >hb_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 2 seed 12345 >bohb_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.14495809658233344 train.optimizer.weight_decay 0.04477752881699114 device 3 seed 12345 >bohb_wrap_12345.log & + +# Seed 12346 +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346 >rs_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.11480094171726124 train.optimizer.weight_decay 0.15500565663984123 device 1 seed 12346 >rs_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.999636 train.optimizer.weight_decay 9.7e-05 device 2 seed 12346 >bo_gp_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.6231045010965764 train.optimizer.weight_decay 0.020651830583413487 device 3 seed 12346 >bo_gp_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.094777 train.optimizer.weight_decay 0.019946 device 0 seed 12346 >bo_kde_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.09470439245585746 train.optimizer.weight_decay 0.1134997725167636 device 1 seed 12346 >bo_kde_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.991872 train.optimizer.weight_decay 0.000695 device 2 seed 12346 >bo_rf_12346.log & +# >bo_gp_wrap_12346.log >bo_rf_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346 >hb_12346.log & +# >bo_kde_wrap_12346.log >hb_wrap_12346.log & +# >hb_12346.log >bohb_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.19783377983946915 train.optimizer.weight_decay 0.3087280858375391 device 3 seed 12346 >bohb_wrap_12346.log & + +# Seed 12347 +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347 >rs_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 3 train.optimizer.lr 0.043324742759307006 train.optimizer.weight_decay 0.0 device 1 seed 12347 >rs_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 8 train.optimizer.lr 0.999856 train.optimizer.weight_decay 1.2e-05 device 2 seed 12347 >bo_gp_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 2 train.optimizer.lr 0.05512741305783294 train.optimizer.weight_decay 0.32323708666212847 device 3 seed 12347 >bo_gp_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.020666 train.optimizer.weight_decay 0.010875 device 0 seed 12347 >bo_kde_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.029033027691434364 train.optimizer.weight_decay 0.7362670096896027 device 1 seed 12347 >bo_kde_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.998016 train.optimizer.weight_decay 0.001249 device 2 seed 12347 >bo_rf_12347.log & +# >bo_gp_wrap_12347.log >bo_rf_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347 >hb_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 5 train.optimizer.lr 0.07982026000525726 train.optimizer.weight_decay 0.07891495031558063 device 1 seed 12347 >hb_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.015962 train.optimizer.weight_decay 1.5e-05 device 2 seed 12347 >bohb_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 1 train.optimizer.lr 0.04927136943461437 train.optimizer.weight_decay 0.08747080914279645 device 3 seed 12347 >bohb_wrap_12347.log & diff --git a/scripts/example_configs/cora/sha_wrap_fedex.yaml b/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml similarity index 65% rename from scripts/example_configs/cora/sha_wrap_fedex.yaml rename to scripts/hpo_exp_scripts/usability/pubmed/rs.yaml index a5c729c14..b748f26ce 100644 --- a/scripts/example_configs/cora/sha_wrap_fedex.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml @@ -1,5 +1,6 @@ use_gpu: True -device: 3 +device: 2 +outdir: rs_pubmed_avg early_stop: patience: 100 seed: 12345 @@ -10,23 +11,22 @@ federate: total_round_num: 500 share_local_model: True online_aggr: True - use_diff: True data: root: data/ - type: cora + type: pubmed splitter: 'louvain' -dataloader: - batch_size: 1 model: type: gcn hidden: 64 dropout: 0.5 - out_channels: 7 + out_channels: 3 + task: node train: + batch_or_epoch: epoch local_update_steps: 1 optimizer: - lr: 0.25 - weight_decay: 0.0005 + lr: 0.01 + weight_decay: 0.0 criterion: type: CrossEntropyLoss trainer: @@ -35,14 +35,16 @@ eval: freq: 1 metrics: ['acc', 'correct', 'f1'] split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 hpo: - scheduler: sha + scheduler: rs num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/cora/hpo_ss_fedex.yaml' + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: - budgets: [2, 4, 12, 36] - fedex: - use: True - ss: 'scripts/example_configs/cora/hpo_ss_fedex_grid.yaml' + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 metric: 'server_global_eval.val_avg_loss' + working_folder: rs_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml new file mode 100644 index 000000000..d02dc104f --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml @@ -0,0 +1,57 @@ +use_gpu: True +device: 2 +outdir: rs_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_rs + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: rs_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/run.sh new file mode 100644 index 000000000..ca7aab7ba --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/run.sh @@ -0,0 +1,12 @@ +set -e + +seed=$1 +device=$2 + +methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%${device})) seed ${seed} >/dev/null 2>&1 +done diff --git a/scripts/example_configs/cora/hpo_ss_sha.yaml b/scripts/hpo_exp_scripts/usability/pubmed/ss.yaml similarity index 65% rename from scripts/example_configs/cora/hpo_ss_sha.yaml rename to scripts/hpo_exp_scripts/usability/pubmed/ss.yaml index 5928eb21c..af60a261a 100644 --- a/scripts/example_configs/cora/hpo_ss_sha.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/ss.yaml @@ -4,11 +4,13 @@ train.optimizer.lr: upper: 1.0 log: True train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] + type: float + lower: 0.0 + upper: 1.0 model.dropout: type: cate choices: [0.0, 0.5] train.local_update_steps: - type: cate - choices: [1, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file + type: int + lower: 1 + upper: 8 diff --git a/scripts/tianchi/submit.yaml b/scripts/tianchi/submit.yaml index 1ee4fbb61..9ecdbd23c 100644 --- a/scripts/tianchi/submit.yaml +++ b/scripts/tianchi/submit.yaml @@ -17,7 +17,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 10 model: diff --git a/setup.py b/setup.py index 892a94318..a8fffb21e 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ ] benchmark_hpo_requires = [ - 'configspace==0.5.0', 'hpbandster==0.7.4', 'smac==1.3.3', 'optuna==2.10.0' + 'configspace==0.5.0', 'hpbandster==0.7.4', 'smac==1.4.0', 'optuna==2.10.0' ] benchmark_htl_requires = ['learn2learn']